GDAL
cpl_aws.h
1 /**********************************************************************
2  * $Id$
3  *
4  * Name: cpl_aws.h
5  * Project: CPL - Common Portability Library
6  * Purpose: Amazon Web Services routines
7  * Author: Even Rouault <even.rouault at spatialys.com>
8  *
9  **********************************************************************
10  * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11  *
12  * Permission is hereby granted, free of charge, to any person obtaining a
13  * copy of this software and associated documentation files (the "Software"),
14  * to deal in the Software without restriction, including without limitation
15  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16  * and/or sell copies of the Software, and to permit persons to whom the
17  * Software is furnished to do so, subject to the following conditions:
18  *
19  * The above copyright notice and this permission notice shall be included
20  * in all copies or substantial portions of the Software.
21  *
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28  * DEALINGS IN THE SOFTWARE.
29  ****************************************************************************/
30 
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33 
34 #ifndef DOXYGEN_SKIP
35 
36 #ifdef HAVE_CURL
37 
38 #include <cstddef>
39 #include <mutex>
40 
41 #include "cpl_string.h"
42 
43 #include <curl/curl.h>
44 #include <map>
45 
46 CPLString CPLGetLowerCaseHexSHA256( const void *pabyData, size_t nBytes );
47 CPLString CPLGetLowerCaseHexSHA256( const CPLString& osStr );
48 
49 CPLString CPLGetAWS_SIGN4_Timestamp();
50 
51 CPLString CPLAWSURLEncode(const CPLString& osURL, bool bEncodeSlash = true);
52 
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist* psExistingHeaders,
54  const char* pszKey);
55 
57 CPLGetAWS_SIGN4_Signature( const CPLString& osSecretAccessKey,
58  const CPLString& osAccessToken,
59  const CPLString& osRegion,
60  const CPLString& osRequestPayer,
61  const CPLString& osService,
62  const CPLString& osVerb,
63  const struct curl_slist* psExistingHeaders,
64  const CPLString& osHost,
65  const CPLString& osCanonicalURI,
66  const CPLString& osCanonicalQueryString,
67  const CPLString& osXAMZContentSHA256,
68  bool bAddHeaderAMZContentSHA256,
69  const CPLString& osTimestamp,
70  CPLString& osSignedHeaders );
71 
72 CPLString CPLGetAWS_SIGN4_Authorization(const CPLString& osSecretAccessKey,
73  const CPLString& osAccessKeyId,
74  const CPLString& osAccessToken,
75  const CPLString& osRegion,
76  const CPLString& osRequestPayer,
77  const CPLString& osService,
78  const CPLString& osVerb,
79  const struct curl_slist* psExistingHeaders,
80  const CPLString& osHost,
81  const CPLString& osCanonicalURI,
82  const CPLString& osCanonicalQueryString,
83  const CPLString& osXAMZContentSHA256,
84  const CPLString& osTimestamp);
85 
86 class IVSIS3LikeHandleHelper
87 {
88  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
89 
90 protected:
91  std::map<CPLString, CPLString> m_oMapQueryParameters{};
92 
93  virtual void RebuildURL() = 0;
94  CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
95 
96 public:
97  IVSIS3LikeHandleHelper() = default;
98  virtual ~IVSIS3LikeHandleHelper() = default;
99 
100  void ResetQueryParameters();
101  void AddQueryParameter(const CPLString& osKey, const CPLString& osValue);
102 
103  virtual struct curl_slist* GetCurlHeaders(const CPLString& osVerb,
104  const struct curl_slist* psExistingHeaders,
105  const void *pabyDataContent = nullptr,
106  size_t nBytesContent = 0) const = 0;
107 
108  virtual bool AllowAutomaticRedirection() { return true; }
109  virtual bool CanRestartOnError(const char*, const char* /* pszHeaders*/,
110  bool /*bSetError*/, bool* /*pbUpdateMap*/ = nullptr) { return false;}
111 
112  virtual const CPLString& GetURL() const = 0;
113  CPLString GetURLNoKVP() const;
114 
115  virtual CPLString GetCopySourceHeader() const { return std::string(); }
116  virtual const char* GetMetadataDirectiveREPLACE() const { return ""; }
117 
118  static bool GetBucketAndObjectKey(const char* pszURI,
119  const char* pszFSPrefix,
120  bool bAllowNoObject,
121  CPLString &osBucketOut,
122  CPLString &osObjectKeyOut);
123 
124  static CPLString BuildCanonicalizedHeaders(
125  std::map<CPLString, CPLString>& oSortedMapHeaders,
126  const struct curl_slist* psExistingHeaders,
127  const char* pszHeaderPrefix);
128 
129  static CPLString GetRFC822DateTime();
130 };
131 
132 enum class AWSCredentialsSource
133 {
134  REGULAR, // credentials from env variables or ~/.aws/crediential
135  EC2, // credentials from EC2 private networking
136  ASSUMED_ROLE // credentials from an STS assumed role
137  // See https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
138  // and https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
139 };
140 
141 class VSIS3HandleHelper final: public IVSIS3LikeHandleHelper
142 {
143  CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
144 
145  CPLString m_osURL{};
146  mutable CPLString m_osSecretAccessKey{};
147  mutable CPLString m_osAccessKeyId{};
148  mutable CPLString m_osSessionToken{};
149  CPLString m_osEndpoint{};
150  CPLString m_osRegion{};
151  CPLString m_osRequestPayer{};
152  CPLString m_osBucket{};
153  CPLString m_osObjectKey{};
154  bool m_bUseHTTPS = false;
155  bool m_bUseVirtualHosting = false;
156  AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
157 
158  void RebuildURL() override;
159 
160  static bool GetConfigurationFromEC2(const std::string& osPathForOption,
161  CPLString& osSecretAccessKey,
162  CPLString& osAccessKeyId,
163  CPLString& osSessionToken);
164 
165  static bool GetConfigurationFromAWSConfigFiles(
166  const std::string& osPathForOption,
167  CPLString& osSecretAccessKey,
168  CPLString& osAccessKeyId,
169  CPLString& osSessionToken,
170  CPLString& osRegion,
171  CPLString& osCredentials,
172  CPLString& osRoleArn,
173  CPLString& osSourceProfile,
174  CPLString& osExternalId,
175  CPLString& osMFASerial,
176  CPLString& osRoleSessionName);
177 
178  static bool GetConfiguration(const std::string& osPathForOption,
179  CSLConstList papszOptions,
180  CPLString& osSecretAccessKey,
181  CPLString& osAccessKeyId,
182  CPLString& osSessionToken,
183  CPLString& osRegion,
184  AWSCredentialsSource& eCredentialsSource);
185  protected:
186 
187  public:
188  VSIS3HandleHelper(const CPLString& osSecretAccessKey,
189  const CPLString& osAccessKeyId,
190  const CPLString& osSessionToken,
191  const CPLString& osEndpoint,
192  const CPLString& osRegion,
193  const CPLString& osRequestPayer,
194  const CPLString& osBucket,
195  const CPLString& osObjectKey,
196  bool bUseHTTPS, bool bUseVirtualHosting,
197  AWSCredentialsSource eCredentialsSource);
198  ~VSIS3HandleHelper();
199 
200  static VSIS3HandleHelper* BuildFromURI(const char* pszURI,
201  const char* pszFSPrefix,
202  bool bAllowNoObject,
203  CSLConstList papszOptions = nullptr);
204  static CPLString BuildURL(const CPLString& osEndpoint,
205  const CPLString& osBucket,
206  const CPLString& osObjectKey,
207  bool bUseHTTPS, bool bUseVirtualHosting);
208 
209  struct curl_slist* GetCurlHeaders(
210  const CPLString& osVerb,
211  const struct curl_slist* psExistingHeaders,
212  const void *pabyDataContent = nullptr,
213  size_t nBytesContent = 0) const override;
214 
215  bool AllowAutomaticRedirection() override { return false; }
216  bool CanRestartOnError(const char*, const char* pszHeaders,
217  bool bSetError,
218  bool* pbUpdateMap = nullptr) override;
219 
220  const CPLString& GetURL() const override { return m_osURL; }
221  const CPLString& GetBucket() const { return m_osBucket; }
222  const CPLString& GetObjectKey() const { return m_osObjectKey; }
223  const CPLString& GetEndpoint()const { return m_osEndpoint; }
224  const CPLString& GetRegion() const { return m_osRegion; }
225  const CPLString& GetRequestPayer() const { return m_osRequestPayer; }
226  bool GetVirtualHosting() const { return m_bUseVirtualHosting; }
227  void SetEndpoint(const CPLString &osStr);
228  void SetRegion(const CPLString &osStr);
229  void SetRequestPayer(const CPLString &osStr);
230  void SetVirtualHosting(bool b);
231 
232  CPLString GetCopySourceHeader() const override { return "x-amz-copy-source"; }
233  const char* GetMetadataDirectiveREPLACE() const override { return "x-amz-metadata-directive: REPLACE"; }
234 
235  CPLString GetSignedURL(CSLConstList papszOptions);
236 
237  static void CleanMutex();
238  static void ClearCache();
239 };
240 
241 class VSIS3UpdateParams
242 {
243  public:
244  CPLString m_osRegion{};
245  CPLString m_osEndpoint{};
246  CPLString m_osRequestPayer{};
247  bool m_bUseVirtualHosting = false;
248 
249  VSIS3UpdateParams() = default;
250 
251  explicit VSIS3UpdateParams(const VSIS3HandleHelper* poHelper) :
252  m_osRegion(poHelper->GetRegion()),
253  m_osEndpoint(poHelper->GetEndpoint()),
254  m_osRequestPayer(poHelper->GetRequestPayer()),
255  m_bUseVirtualHosting(poHelper->GetVirtualHosting()) {}
256 
257  void UpdateHandlerHelper(VSIS3HandleHelper* poHelper) {
258  poHelper->SetRegion(m_osRegion);
259  poHelper->SetEndpoint(m_osEndpoint);
260  poHelper->SetRequestPayer(m_osRequestPayer);
261  poHelper->SetVirtualHosting(m_bUseVirtualHosting);
262  }
263 
264  static std::mutex gsMutex;
265  static std::map< CPLString, VSIS3UpdateParams > goMapBucketsToS3Params;
266  static void UpdateMapFromHandle( IVSIS3LikeHandleHelper* poHandleHelper );
267  static void UpdateHandleFromMap( IVSIS3LikeHandleHelper* poHandleHelper );
268  static void ClearCache();
269 };
270 
271 #endif /* HAVE_CURL */
272 
273 #endif /* #ifndef DOXYGEN_SKIP */
274 
275 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:319
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1053
cpl_string.h
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:927