GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48 
50 
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53 
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55 
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57  const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59  struct curl_slist* poSrcToDestroy );
60 
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62  const char *pszPath);
63 
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65  CSLConstList papszOptions,
66  const char *pszPath);
67 
68 namespace cpl {
69 
70 typedef enum
71 {
72  EXIST_UNKNOWN = -1,
73  EXIST_NO,
74  EXIST_YES,
75 } ExistStatus;
76 
77 class FileProp
78 {
79  public:
80  unsigned int nGenerationAuthParameters = 0;
81  ExistStatus eExists = EXIST_UNKNOWN;
82  vsi_l_offset fileSize = 0;
83  time_t mTime = 0;
84  time_t nExpireTimestampLocal = 0;
85  CPLString osRedirectURL{};
86  bool bHasComputedFileSize = false;
87  bool bIsDirectory = false;
88  int nMode = 0; // st_mode member of struct stat
89  bool bS3LikeRedirect = false;
90  CPLString ETag{};
91 };
92 
93 struct CachedDirList
94 {
95  bool bGotFileList = false;
96  unsigned int nGenerationAuthParameters = 0;
97  CPLStringList oFileList{}; /* only file name without path */
98 };
99 
100 struct WriteFuncStruct
101 {
102  char* pBuffer = nullptr;
103  size_t nSize = 0;
104  bool bIsHTTP = false;
105  bool bMultiRange = false;
106  vsi_l_offset nStartOffset = 0;
107  vsi_l_offset nEndOffset = 0;
108  int nHTTPCode = 0;
109  vsi_l_offset nContentLength = 0;
110  bool bFoundContentRange = false;
111  bool bError = false;
112  bool bInterruptDownload = false;
113  bool bDetectRangeDownloadingError = false;
114  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115 
116  VSILFILE *fp = nullptr;
117  VSICurlReadCbkFunc pfnReadCbk = nullptr;
118  void *pReadCbkUserData = nullptr;
119  bool bInterrupted = false;
120  bool bInterruptIfNonErrorPayload = false;
121 
122 #if !CURL_AT_LEAST_VERSION(7,54,0)
123  // Workaround to ignore extra HTTP response headers from
124  // proxies in older versions of curl.
125  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
126  bool bIsProxyConnectHeader = false;
127 #endif
128 };
129 
130 struct PutData
131 {
132  const GByte* pabyData = nullptr;
133  size_t nOff = 0;
134  size_t nTotalSize = 0;
135 
136  static size_t ReadCallBackBuffer( char *buffer, size_t size,
137  size_t nitems, void *instream )
138  {
139  PutData* poThis = static_cast<PutData *>(instream);
140  const size_t nSizeMax = size * nitems;
141  const size_t nSizeToWrite =
142  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
143  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
144  poThis->nOff += nSizeToWrite;
145  return nSizeToWrite;
146  }
147 };
148 
149 /************************************************************************/
150 /* VSICurlFilesystemHandler */
151 /************************************************************************/
152 
153 class VSICurlHandle;
154 
155 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
156 {
157  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
158 
159  struct FilenameOffsetPair
160  {
161  std::string filename_;
162  vsi_l_offset offset_;
163 
164  FilenameOffsetPair(const std::string& filename,
165  vsi_l_offset offset) :
166  filename_(filename), offset_(offset) {}
167 
168  bool operator==(const FilenameOffsetPair& other) const
169  {
170  return filename_ == other.filename_ &&
171  offset_ == other.offset_;
172  }
173  };
174  struct FilenameOffsetPairHasher
175  {
176  std::size_t operator()(const FilenameOffsetPair& k) const
177  {
178  return std::hash<std::string>()(k.filename_) ^
179  std::hash<vsi_l_offset>()(k.offset_);
180  }
181  };
182 
183  using RegionCacheType =
184  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
185  lru11::NullLock,
186  std::unordered_map<
187  FilenameOffsetPair,
188  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
189  std::shared_ptr<std::string>>>::iterator,
190  FilenameOffsetPairHasher>>;
191 
192  std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
193  RegionCacheType* GetRegionCache();
194 
195  // LRU cache that just keeps in memory if this file system handler is
196  // spposed to know the file properties of a file. The actual cache is a
197  // shared one among all network file systems.
198  // The aim of that design is that invalidating /vsis3/foo results in
199  // /vsis3_streaming/foo to be invalidated as well.
200  lru11::Cache<std::string, bool> oCacheFileProp;
201 
202  int nCachedFilesInDirList = 0;
203  lru11::Cache<std::string, CachedDirList> oCacheDirList;
204 
205  char** ParseHTMLFileList(const char* pszFilename,
206  int nMaxFiles,
207  char* pszData,
208  bool* pbGotFileList);
209 
210 protected:
211  CPLMutex *hMutex = nullptr;
212 
213  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
214  virtual char** GetFileList(const char *pszFilename,
215  int nMaxFiles,
216  bool* pbGotFileList);
217 
218  void RegisterEmptyDir( const CPLString& osDirname );
219 
220  bool AnalyseS3FileList( const CPLString& osBaseURL,
221  const char* pszXML,
222  CPLStringList& osFileList,
223  int nMaxFiles,
224  const std::set<std::string>& oSetIgnoredStorageClasses,
225  bool& bIsTruncated );
226 
227  void AnalyseSwiftFileList( const CPLString& osBaseURL,
228  const CPLString& osPrefix,
229  const char* pszJson,
230  CPLStringList& osFileList,
231  int nMaxFilesThisQuery,
232  int nMaxFiles,
233  bool& bIsTruncated,
234  CPLString& osNextMarker );
235 
236  static const char* GetOptionsStatic();
237 
238  static bool IsAllowedFilename( const char* pszFilename );
239 
240  VSICurlFilesystemHandlerBase();
241 
242 public:
243  ~VSICurlFilesystemHandlerBase() override;
244 
245  VSIVirtualHandle *Open( const char *pszFilename,
246  const char *pszAccess,
247  bool bSetError,
248  CSLConstList /* papszOptions */ ) override;
249 
250  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
251  int nFlags ) override;
252  int Unlink( const char *pszFilename ) override;
253  int Rename( const char *oldpath, const char *newpath ) override;
254  int Mkdir( const char *pszDirname, long nMode ) override;
255  int Rmdir( const char *pszDirname ) override;
256  char **ReadDir( const char *pszDirname ) override
257  { return ReadDirEx(pszDirname, 0); }
258  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
259  char **SiblingFiles( const char *pszFilename ) override;
260 
261  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
262  override { return true; }
263 
264  const char* GetActualURL(const char* pszFilename) override;
265 
266  const char* GetOptions() override;
267 
268  char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
269  CSLConstList papszOptions ) override;
270 
271  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
272  bool* pbGotFileList );
273  void InvalidateDirContent( const char *pszDirname );
274 
275  virtual const char* GetDebugKey() const = 0;
276 
277  virtual CPLString GetFSPrefix() const = 0;
278  virtual bool AllowCachedDataFor(const char* pszFilename);
279 
280  virtual bool IsLocal( const char* /* pszPath */ ) override { return false; }
281  virtual bool SupportsSequentialWrite( const char* /* pszPath */, bool /* bAllowLocalTempFile */ ) override { return false; }
282  virtual bool SupportsRandomWrite( const char* /* pszPath */, bool /* bAllowLocalTempFile */ ) override { return false; }
283 
284  std::shared_ptr<std::string> GetRegion( const char* pszURL,
285  vsi_l_offset nFileOffsetStart );
286 
287  void AddRegion( const char* pszURL,
288  vsi_l_offset nFileOffsetStart,
289  size_t nSize,
290  const char *pData );
291 
292  bool GetCachedFileProp( const char* pszURL,
293  FileProp& oFileProp );
294  void SetCachedFileProp( const char* pszURL,
295  FileProp& oFileProp );
296  void InvalidateCachedData( const char* pszURL );
297 
298  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
299 
300  virtual void ClearCache();
301  virtual void PartialClearCache(const char* pszFilename);
302 
303 
304  bool GetCachedDirList( const char* pszURL,
305  CachedDirList& oCachedDirList );
306  void SetCachedDirList( const char* pszURL,
307  CachedDirList& oCachedDirList );
308  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
309 
310  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
311 
312  std::string GetStreamingFilename(const std::string& osFilename) const override = 0;
313 
314  static std::set<std::string> GetS3IgnoredStorageClasses();
315 };
316 
317 
318 class VSICurlFilesystemHandler: public VSICurlFilesystemHandlerBase
319 {
320  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
321 
322 public:
323  VSICurlFilesystemHandler() = default;
324 
325  const char* GetDebugKey() const override { return "VSICURL"; }
326 
327  CPLString GetFSPrefix() const override { return "/vsicurl/"; }
328 
329  std::string GetStreamingFilename(const std::string& osFilename) const override;
330 };
331 
332 /************************************************************************/
333 /* VSICurlHandle */
334 /************************************************************************/
335 
336 class VSICurlHandle : public VSIVirtualHandle
337 {
338  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
339 
340  protected:
341  VSICurlFilesystemHandlerBase* poFS = nullptr;
342 
343  bool m_bCached = true;
344 
345  mutable FileProp oFileProp{};
346 
347  mutable std::mutex m_oMutex{};
348  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
349  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
350  mutable std::string m_osQueryString{}; // e.g. an Azure SAS
351 
352  char **m_papszHTTPOptions = nullptr;
353 
354  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
355  int nBlocksToDownload = 1;
356 
357  bool bStopOnInterruptUntilUninstall = false;
358  bool bInterrupted = false;
359  VSICurlReadCbkFunc pfnReadCbk = nullptr;
360  void *pReadCbkUserData = nullptr;
361 
362  int m_nMaxRetry = 0;
363  double m_dfRetryDelay = 0.0;
364 
365  CPLStringList m_aosHeaders{};
366 
367  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
368  const int nBlocks,
369  const char* pBuffer,
370  size_t nSize );
371 
372  private:
373 
374  vsi_l_offset curOffset = 0;
375 
376  bool bEOF = false;
377 
378  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
379 
380  bool m_bUseHead = false;
381  bool m_bUseRedirectURLIfNoQueryStringParams = false;
382 
383  // Specific to Planetary Computer signing: https://planetarycomputer.microsoft.com/docs/concepts/sas/
384  mutable bool m_bPlanetaryComputerURLSigning = false;
385  mutable std::string m_osPlanetaryComputerCollection{};
386  void ManagePlanetaryComputerSigning() const;
387 
388  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
389  const vsi_l_offset* panOffsets,
390  const size_t* panSizes );
391  CPLString GetRedirectURLIfValid(bool& bHasExpired) const;
392 
393  void UpdateRedirectInfo( CURL* hCurlHandle,
394  const WriteFuncStruct& sWriteFuncHeaderData );
395 
396  protected:
397  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
398  const struct curl_slist* /* psExistingHeaders */)
399  { return nullptr; }
400  virtual bool AllowAutomaticRedirection() { return true; }
401  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
402  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
403  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
404  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
405  void SetURL(const char* pszURL);
406  virtual bool Authenticate(const char* /* pszFilename */) { return false; }
407 
408  public:
409 
410  VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
411  const char* pszFilename,
412  const char* pszURLIn = nullptr );
413  ~VSICurlHandle() override;
414 
415  int Seek( vsi_l_offset nOffset, int nWhence ) override;
416  vsi_l_offset Tell() override;
417  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
418  int ReadMultiRange( int nRanges, void ** ppData,
419  const vsi_l_offset* panOffsets,
420  const size_t* panSizes ) override;
421  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
422  int Eof() override;
423  int Flush() override;
424  int Close() override;
425 
426  bool HasPRead() const override { return true; }
427  size_t PRead( void* pBuffer, size_t nSize, vsi_l_offset nOffset ) const override;
428 
429  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
430  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
431  virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
432  bool Exists( bool bSetError );
433  bool IsDirectory() const { return oFileProp.bIsDirectory; }
434  int GetMode() const { return oFileProp.nMode; }
435  time_t GetMTime() const { return oFileProp.mTime; }
436  const CPLStringList& GetHeaders() { return m_aosHeaders; }
437 
438  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
439  void* pfnUserData,
440  int bStopOnInterruptUntilUninstall );
441  int UninstallReadCbk();
442 
443  const char *GetURL() const { return m_pszURL; }
444 };
445 
446 /************************************************************************/
447 /* IVSIS3LikeFSHandler */
448 /************************************************************************/
449 
450 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandlerBase
451 {
452  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
453 
454  bool CopyFile(VSILFILE* fpIn,
455  vsi_l_offset nSourceSize,
456  const char* pszSource,
457  const char* pszTarget,
458  CSLConstList papszOptions,
459  GDALProgressFunc pProgressFunc,
460  void *pProgressData);
461  virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
462 
463  protected:
464  char** GetFileList( const char *pszFilename,
465  int nMaxFiles,
466  bool* pbGotFileList ) override;
467 
468  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
469  const char* pszURI, bool bAllowNoObject) = 0;
470 
471  virtual int CopyObject( const char *oldpath, const char *newpath,
472  CSLConstList papszMetadata );
473 
474  int RmdirRecursiveInternal( const char* pszDirname, int nBatchSize);
475 
476  virtual bool IsAllowedHeaderForObjectCreation( const char* /* pszHeaderName */ ) { return false; }
477 
478  IVSIS3LikeFSHandler() = default;
479 
480  public:
481  int Unlink( const char *pszFilename ) override;
482  int Mkdir( const char *pszDirname, long nMode ) override;
483  int Rmdir( const char *pszDirname ) override;
484  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
485  int nFlags ) override;
486  int Rename( const char *oldpath, const char *newpath ) override;
487 
488  virtual int DeleteObject( const char *pszFilename );
489 
490  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
491  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
492 
493  bool Sync( const char* pszSource, const char* pszTarget,
494  const char* const * papszOptions,
495  GDALProgressFunc pProgressFunc,
496  void *pProgressData,
497  char*** ppapszOutputs ) override;
498 
499  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
500  const char* const *papszOptions) override;
501 
502  // Multipart upload
503  virtual bool SupportsParallelMultipartUpload() const { return false; }
504 
505  virtual CPLString InitiateMultipartUpload(
506  const std::string& osFilename,
507  IVSIS3LikeHandleHelper *poS3HandleHelper,
508  int nMaxRetry,
509  double dfRetryDelay,
510  CSLConstList papszOptions);
511  virtual CPLString UploadPart(const CPLString& osFilename,
512  int nPartNumber,
513  const std::string& osUploadID,
514  vsi_l_offset nPosition,
515  const void* pabyBuffer,
516  size_t nBufferSize,
517  IVSIS3LikeHandleHelper *poS3HandleHelper,
518  int nMaxRetry,
519  double dfRetryDelay,
520  CSLConstList papszOptions);
521  virtual bool CompleteMultipart(const CPLString& osFilename,
522  const CPLString& osUploadID,
523  const std::vector<CPLString>& aosEtags,
524  vsi_l_offset nTotalSize,
525  IVSIS3LikeHandleHelper *poS3HandleHelper,
526  int nMaxRetry,
527  double dfRetryDelay);
528  virtual bool AbortMultipart(const CPLString& osFilename,
529  const CPLString& osUploadID,
530  IVSIS3LikeHandleHelper *poS3HandleHelper,
531  int nMaxRetry,
532  double dfRetryDelay);
533 
534  bool AbortPendingUploads(const char* pszFilename) override;
535 };
536 
537 /************************************************************************/
538 /* IVSIS3LikeHandle */
539 /************************************************************************/
540 
541 class IVSIS3LikeHandle: public VSICurlHandle
542 {
543  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
544 
545  protected:
546  bool UseLimitRangeGetInsteadOfHead() override { return true; }
547  bool IsDirectoryFromExists( const char* pszVerb,
548  int response_code ) override
549  {
550  // A bit dirty, but on S3, a GET on a existing directory returns a 416
551  return response_code == 416 && EQUAL(pszVerb, "GET") &&
552  CPLString(m_pszURL).back() == '/';
553  }
554  void ProcessGetFileSizeResult( const char* pszContent ) override
555  {
556  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
557  }
558 
559  public:
560  IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
561  const char* pszFilename,
562  const char* pszURLIn ) :
563  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
564  ~IVSIS3LikeHandle() override {}
565 };
566 
567 /************************************************************************/
568 /* VSIS3WriteHandle */
569 /************************************************************************/
570 
571 class VSIS3WriteHandle final : public VSIVirtualHandle
572 {
573  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
574 
575  IVSIS3LikeFSHandler *m_poFS = nullptr;
576  CPLString m_osFilename{};
577  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
578  bool m_bUseChunked = false;
579  CPLStringList m_aosOptions{};
580 
581  vsi_l_offset m_nCurOffset = 0;
582  int m_nBufferOff = 0;
583  int m_nBufferSize = 0;
584  bool m_bClosed = false;
585  GByte *m_pabyBuffer = nullptr;
586  CPLString m_osUploadID{};
587  int m_nPartNumber = 0;
588  std::vector<CPLString> m_aosEtags{};
589  bool m_bError = false;
590 
591  CURLM *m_hCurlMulti = nullptr;
592  CURL *m_hCurl = nullptr;
593  const void *m_pBuffer = nullptr;
594  CPLString m_osCurlErrBuf{};
595  size_t m_nChunkedBufferOff = 0;
596  size_t m_nChunkedBufferSize = 0;
597  size_t m_nWrittenInPUT = 0;
598 
599  int m_nMaxRetry = 0;
600  double m_dfRetryDelay = 0.0;
601  WriteFuncStruct m_sWriteFuncHeaderData{};
602 
603  bool UploadPart();
604  bool DoSinglePartPUT();
605 
606  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
607  size_t nitems, void *instream );
608  size_t WriteChunked( const void *pBuffer,
609  size_t nSize, size_t nMemb );
610  int FinishChunkedTransfer();
611 
612  void InvalidateParentDirectory();
613 
614  public:
615  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
616  const char* pszFilename,
617  IVSIS3LikeHandleHelper* poS3HandleHelper,
618  bool bUseChunked,
619  CSLConstList papszOptions );
620  ~VSIS3WriteHandle() override;
621 
622  int Seek( vsi_l_offset nOffset, int nWhence ) override;
623  vsi_l_offset Tell() override;
624  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
625  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
626  int Eof() override;
627  int Close() override;
628 
629  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
630 };
631 
632 /************************************************************************/
633 /* VSIAppendWriteHandle */
634 /************************************************************************/
635 
636 class VSIAppendWriteHandle : public VSIVirtualHandle
637 {
638  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
639 
640  protected:
641 
642  VSICurlFilesystemHandlerBase* m_poFS = nullptr;
643  CPLString m_osFSPrefix{};
644  CPLString m_osFilename{};
645 
646  vsi_l_offset m_nCurOffset = 0;
647  int m_nBufferOff = 0;
648  int m_nBufferSize = 0;
649  int m_nBufferOffReadCallback = 0;
650  bool m_bClosed = false;
651  GByte *m_pabyBuffer = nullptr;
652  bool m_bError = false;
653 
654  static size_t ReadCallBackBuffer( char *buffer, size_t size,
655  size_t nitems, void *instream );
656  virtual bool Send(bool bIsLastBlock) = 0;
657 
658  public:
659  VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
660  const char* pszFSPrefix,
661  const char* pszFilename,
662  int nChunkSize );
663  virtual ~VSIAppendWriteHandle();
664 
665  int Seek( vsi_l_offset nOffset, int nWhence ) override;
666  vsi_l_offset Tell() override;
667  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
668  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
669  int Eof() override;
670  int Close() override;
671 
672  bool IsOK() { return m_pabyBuffer != nullptr; }
673 };
674 
675 /************************************************************************/
676 /* VSIDIRWithMissingDirSynthesis */
677 /************************************************************************/
678 
679 struct VSIDIRWithMissingDirSynthesis: public VSIDIR
680 {
681  std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
682 
683 protected:
684  std::vector<std::string> m_aosSubpathsStack{};
685 
686  void SynthetizeMissingDirectories(const std::string& osCurSubdir,
687  bool bAddEntryForThisSubdir);
688 };
689 
690 /************************************************************************/
691 /* CurlRequestHelper */
692 /************************************************************************/
693 
694 struct CurlRequestHelper
695 {
696  WriteFuncStruct sWriteFuncData{};
697  WriteFuncStruct sWriteFuncHeaderData{};
698  char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
699 
700  CurlRequestHelper();
701  ~CurlRequestHelper();
702  long perform(CURL* hCurlHandle,
703  struct curl_slist* headers, // ownership transferred
704  VSICurlFilesystemHandlerBase *poFS,
705  IVSIS3LikeHandleHelper *poS3HandleHelper);
706 };
707 
708 /************************************************************************/
709 /* NetworkStatisticsLogger */
710 /************************************************************************/
711 
712 class NetworkStatisticsLogger
713 {
714  static int gnEnabled;
715  static NetworkStatisticsLogger gInstance;
716 
717  NetworkStatisticsLogger() = default;
718 
719  std::mutex m_mutex{};
720 
721  struct Counters
722  {
723  GIntBig nHEAD = 0;
724  GIntBig nGET = 0;
725  GIntBig nPUT = 0;
726  GIntBig nPOST = 0;
727  GIntBig nDELETE = 0;
728  GIntBig nGETDownloadedBytes = 0;
729  GIntBig nPUTUploadedBytes = 0;
730  GIntBig nPOSTDownloadedBytes = 0;
731  GIntBig nPOSTUploadedBytes = 0;
732  };
733 
734  enum class ContextPathType
735  {
736  FILESYSTEM,
737  FILE,
738  ACTION,
739  };
740 
741  struct ContextPathItem
742  {
743  ContextPathType eType;
744  CPLString osName;
745 
746  ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
747  eType(eTypeIn), osName(osNameIn) {}
748 
749  bool operator< (const ContextPathItem& other ) const
750  {
751  if( static_cast<int>(eType) < static_cast<int>(other.eType) )
752  return true;
753  if( static_cast<int>(eType) > static_cast<int>(other.eType) )
754  return false;
755  return osName < other.osName;
756  }
757  };
758 
759  struct Stats
760  {
761  Counters counters{};
762  std::map<ContextPathItem, Stats> children{};
763 
764  void AsJSON(CPLJSONObject& oJSON) const;
765  };
766 
767  // Workaround bug in Coverity Scan
768  // coverity[generated_default_constructor_used_in_field_initializer]
769  Stats m_stats{};
770  std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
771 
772  static void ReadEnabled();
773 
774  std::vector<Counters*> GetCountersForContext();
775 
776 public:
777 
778  static inline bool IsEnabled()
779  {
780  if( gnEnabled < 0)
781  {
782  ReadEnabled();
783  }
784  return gnEnabled == TRUE;
785  }
786 
787  static void EnterFileSystem(const char* pszName);
788 
789  static void LeaveFileSystem();
790 
791  static void EnterFile(const char* pszName);
792 
793  static void LeaveFile();
794 
795  static void EnterAction(const char* pszName);
796 
797  static void LeaveAction();
798 
799  static void LogHEAD();
800 
801  static void LogGET(size_t nDownloadedBytes);
802 
803  static void LogPUT(size_t nUploadedBytes);
804 
805  static void LogPOST(size_t nUploadedBytes,
806  size_t nDownloadedBytes);
807 
808  static void LogDELETE();
809 
810  static void Reset();
811 
812  static CPLString GetReportAsSerializedJSON();
813 };
814 
815 struct NetworkStatisticsFileSystem
816 {
817  inline explicit NetworkStatisticsFileSystem(const char* pszName) {
818  NetworkStatisticsLogger::EnterFileSystem(pszName);
819  }
820 
821  inline ~NetworkStatisticsFileSystem()
822  {
823  NetworkStatisticsLogger::LeaveFileSystem();
824  }
825 };
826 
827 struct NetworkStatisticsFile
828 {
829  inline explicit NetworkStatisticsFile(const char* pszName) {
830  NetworkStatisticsLogger::EnterFile(pszName);
831  }
832 
833  inline ~NetworkStatisticsFile()
834  {
835  NetworkStatisticsLogger::LeaveFile();
836  }
837 };
838 
839 struct NetworkStatisticsAction
840 {
841  inline explicit NetworkStatisticsAction(const char* pszName) {
842  NetworkStatisticsLogger::EnterAction(pszName);
843  }
844 
845  inline ~NetworkStatisticsAction()
846  {
847  NetworkStatisticsLogger::LeaveAction();
848  }
849 };
850 
851 
852 int VSICURLGetDownloadChunkSize();
853 
854 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
855  VSILFILE *fp,
856  VSICurlReadCbkFunc pfnReadCbk,
857  void *pReadCbkUserData );
858 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
859  size_t nmemb, void *req );
860 void MultiPerform(CURLM* hCurlMultiHandle,
861  CURL* hEasyHandle = nullptr);
862 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
863 
864 int VSICurlParseUnixPermissions(const char* pszPermissions);
865 
866 // Cache of file properties (size, etc.)
867 bool VSICURLGetCachedFileProp( const char* pszURL,
868  FileProp& oFileProp );
869 void VSICURLSetCachedFileProp( const char* pszURL,
870  FileProp& oFileProp );
871 void VSICURLInvalidateCachedFileProp( const char* pszURL );
872 void VSICURLInvalidateCachedFilePropPrefix( const char* pszURL );
873 void VSICURLDestroyCacheFileProp();
874 
875 } // namespace cpl
876 
878 
879 #endif // HAVE_CURL
880 
881 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:203
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:429
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:320
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:543
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1056
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:53
cpl_string.h
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:142
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:230
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:355
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:200
cpl_port.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:144
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:930
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:158