GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48 
50 
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53 
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55 
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57  const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59  struct curl_slist* poSrcToDestroy );
60 
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62  const char *pszPath);
63 
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65  CSLConstList papszOptions,
66  const char *pszPath);
67 
68 namespace cpl {
69 
70 typedef enum
71 {
72  EXIST_UNKNOWN = -1,
73  EXIST_NO,
74  EXIST_YES,
75 } ExistStatus;
76 
77 class FileProp
78 {
79  public:
80  unsigned int nGenerationAuthParameters = 0;
81  ExistStatus eExists = EXIST_UNKNOWN;
82  vsi_l_offset fileSize = 0;
83  time_t mTime = 0;
84  time_t nExpireTimestampLocal = 0;
85  CPLString osRedirectURL{};
86  bool bHasComputedFileSize = false;
87  bool bIsDirectory = false;
88  int nMode = 0; // st_mode member of struct stat
89  bool bS3LikeRedirect = false;
90  CPLString ETag{};
91 };
92 
93 struct CachedDirList
94 {
95  bool bGotFileList = false;
96  unsigned int nGenerationAuthParameters = 0;
97  CPLStringList oFileList{}; /* only file name without path */
98 };
99 
100 struct WriteFuncStruct
101 {
102  char* pBuffer = nullptr;
103  size_t nSize = 0;
104  bool bIsHTTP = false;
105  bool bMultiRange = false;
106  vsi_l_offset nStartOffset = 0;
107  vsi_l_offset nEndOffset = 0;
108  int nHTTPCode = 0;
109  vsi_l_offset nContentLength = 0;
110  bool bFoundContentRange = false;
111  bool bError = false;
112  bool bInterruptDownload = false;
113  bool bDetectRangeDownloadingError = false;
114  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115 
116  VSILFILE *fp = nullptr;
117  VSICurlReadCbkFunc pfnReadCbk = nullptr;
118  void *pReadCbkUserData = nullptr;
119  bool bInterrupted = false;
120 
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
122  // Workaround to ignore extra HTTP response headers from
123  // proxies in older versions of curl.
124  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
125  bool bIsProxyConnectHeader = false;
126 #endif
127 };
128 
129 struct PutData
130 {
131  const GByte* pabyData = nullptr;
132  size_t nOff = 0;
133  size_t nTotalSize = 0;
134 
135  static size_t ReadCallBackBuffer( char *buffer, size_t size,
136  size_t nitems, void *instream )
137  {
138  PutData* poThis = static_cast<PutData *>(instream);
139  const size_t nSizeMax = size * nitems;
140  const size_t nSizeToWrite =
141  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143  poThis->nOff += nSizeToWrite;
144  return nSizeToWrite;
145  }
146 };
147 
148 /************************************************************************/
149 /* VSICurlFilesystemHandler */
150 /************************************************************************/
151 
152 class VSICurlHandle;
153 
154 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
155 {
156  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
157 
158  struct FilenameOffsetPair
159  {
160  std::string filename_;
161  vsi_l_offset offset_;
162 
163  FilenameOffsetPair(const std::string& filename,
164  vsi_l_offset offset) :
165  filename_(filename), offset_(offset) {}
166 
167  bool operator==(const FilenameOffsetPair& other) const
168  {
169  return filename_ == other.filename_ &&
170  offset_ == other.offset_;
171  }
172  };
173  struct FilenameOffsetPairHasher
174  {
175  std::size_t operator()(const FilenameOffsetPair& k) const
176  {
177  return std::hash<std::string>()(k.filename_) ^
178  std::hash<vsi_l_offset>()(k.offset_);
179  }
180  };
181 
182  using RegionCacheType =
183  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
184  lru11::NullLock,
185  std::unordered_map<
186  FilenameOffsetPair,
187  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188  std::shared_ptr<std::string>>>::iterator,
189  FilenameOffsetPairHasher>>;
190 
191  std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
192  RegionCacheType* GetRegionCache();
193 
194  // LRU cache that just keeps in memory if this file system handler is
195  // spposed to know the file properties of a file. The actual cache is a
196  // shared one among all network file systems.
197  // The aim of that design is that invalidating /vsis3/foo results in
198  // /vsis3_streaming/foo to be invalidated as well.
199  lru11::Cache<std::string, bool> oCacheFileProp;
200 
201  int nCachedFilesInDirList = 0;
202  lru11::Cache<std::string, CachedDirList> oCacheDirList;
203 
204  char** ParseHTMLFileList(const char* pszFilename,
205  int nMaxFiles,
206  char* pszData,
207  bool* pbGotFileList);
208 
209 protected:
210  CPLMutex *hMutex = nullptr;
211 
212  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
213  virtual char** GetFileList(const char *pszFilename,
214  int nMaxFiles,
215  bool* pbGotFileList);
216 
217  void RegisterEmptyDir( const CPLString& osDirname );
218 
219  bool AnalyseS3FileList( const CPLString& osBaseURL,
220  const char* pszXML,
221  CPLStringList& osFileList,
222  int nMaxFiles,
223  const std::set<std::string>& oSetIgnoredStorageClasses,
224  bool& bIsTruncated );
225 
226  void AnalyseSwiftFileList( const CPLString& osBaseURL,
227  const CPLString& osPrefix,
228  const char* pszJson,
229  CPLStringList& osFileList,
230  int nMaxFilesThisQuery,
231  int nMaxFiles,
232  bool& bIsTruncated,
233  CPLString& osNextMarker );
234 
235  static const char* GetOptionsStatic();
236 
237  static bool IsAllowedFilename( const char* pszFilename );
238 
239  VSICurlFilesystemHandlerBase();
240 
241 public:
242  ~VSICurlFilesystemHandlerBase() override;
243 
244  VSIVirtualHandle *Open( const char *pszFilename,
245  const char *pszAccess,
246  bool bSetError,
247  CSLConstList /* papszOptions */ ) override;
248 
249  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
250  int nFlags ) override;
251  int Unlink( const char *pszFilename ) override;
252  int Rename( const char *oldpath, const char *newpath ) override;
253  int Mkdir( const char *pszDirname, long nMode ) override;
254  int Rmdir( const char *pszDirname ) override;
255  char **ReadDir( const char *pszDirname ) override
256  { return ReadDirEx(pszDirname, 0); }
257  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
258  char **SiblingFiles( const char *pszFilename ) override;
259 
260  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
261  override { return true; }
262 
263  const char* GetActualURL(const char* pszFilename) override;
264 
265  const char* GetOptions() override;
266 
267  char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
268  CSLConstList papszOptions ) override;
269 
270  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
271  bool* pbGotFileList );
272  void InvalidateDirContent( const char *pszDirname );
273 
274  virtual const char* GetDebugKey() const = 0;
275 
276  virtual CPLString GetFSPrefix() const = 0;
277  virtual bool AllowCachedDataFor(const char* pszFilename);
278 
279  std::shared_ptr<std::string> GetRegion( const char* pszURL,
280  vsi_l_offset nFileOffsetStart );
281 
282  void AddRegion( const char* pszURL,
283  vsi_l_offset nFileOffsetStart,
284  size_t nSize,
285  const char *pData );
286 
287  bool GetCachedFileProp( const char* pszURL,
288  FileProp& oFileProp );
289  void SetCachedFileProp( const char* pszURL,
290  FileProp& oFileProp );
291  void InvalidateCachedData( const char* pszURL );
292 
293  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
294 
295  virtual void ClearCache();
296  virtual void PartialClearCache(const char* pszFilename);
297 
298 
299  bool GetCachedDirList( const char* pszURL,
300  CachedDirList& oCachedDirList );
301  void SetCachedDirList( const char* pszURL,
302  CachedDirList& oCachedDirList );
303  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
304 
305  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
306 
307  std::string GetStreamingFilename(const std::string& osFilename) const override = 0;
308 
309  static std::set<std::string> GetS3IgnoredStorageClasses();
310 };
311 
312 
313 class VSICurlFilesystemHandler: public VSICurlFilesystemHandlerBase
314 {
315  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
316 
317 public:
318  VSICurlFilesystemHandler() = default;
319 
320  const char* GetDebugKey() const override { return "VSICURL"; }
321 
322  CPLString GetFSPrefix() const override { return "/vsicurl/"; }
323 
324  std::string GetStreamingFilename(const std::string& osFilename) const override;
325 };
326 
327 /************************************************************************/
328 /* VSICurlHandle */
329 /************************************************************************/
330 
331 class VSICurlHandle : public VSIVirtualHandle
332 {
333  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
334 
335  protected:
336  VSICurlFilesystemHandlerBase* poFS = nullptr;
337 
338  bool m_bCached = true;
339 
340  FileProp oFileProp{};
341 
342  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
343  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
344  std::string m_osQueryString{}; // e.g. an Azure SAS
345 
346  char **m_papszHTTPOptions = nullptr;
347 
348  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
349  int nBlocksToDownload = 1;
350 
351  bool bStopOnInterruptUntilUninstall = false;
352  bool bInterrupted = false;
353  VSICurlReadCbkFunc pfnReadCbk = nullptr;
354  void *pReadCbkUserData = nullptr;
355 
356  int m_nMaxRetry = 0;
357  double m_dfRetryDelay = 0.0;
358 
359  CPLStringList m_aosHeaders{};
360 
361  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
362  const int nBlocks,
363  const char* pBuffer,
364  size_t nSize );
365 
366  private:
367 
368  vsi_l_offset curOffset = 0;
369 
370  bool bEOF = false;
371 
372  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
373 
374  bool m_bUseHead = false;
375  bool m_bUseRedirectURLIfNoQueryStringParams = false;
376 
377  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
378  const vsi_l_offset* panOffsets,
379  const size_t* panSizes );
380  CPLString GetRedirectURLIfValid(bool& bHasExpired);
381 
382  protected:
383  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
384  const struct curl_slist* /* psExistingHeaders */)
385  { return nullptr; }
386  virtual bool AllowAutomaticRedirection() { return true; }
387  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
388  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
389  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
390  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
391  void SetURL(const char* pszURL);
392  virtual bool Authenticate(const char* /* pszFilename */) { return false; }
393 
394  public:
395 
396  VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
397  const char* pszFilename,
398  const char* pszURLIn = nullptr );
399  ~VSICurlHandle() override;
400 
401  int Seek( vsi_l_offset nOffset, int nWhence ) override;
402  vsi_l_offset Tell() override;
403  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
404  int ReadMultiRange( int nRanges, void ** ppData,
405  const vsi_l_offset* panOffsets,
406  const size_t* panSizes ) override;
407  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
408  int Eof() override;
409  int Flush() override;
410  int Close() override;
411 
412  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
413  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
414  virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
415  bool Exists( bool bSetError );
416  bool IsDirectory() const { return oFileProp.bIsDirectory; }
417  int GetMode() const { return oFileProp.nMode; }
418  time_t GetMTime() const { return oFileProp.mTime; }
419  const CPLStringList& GetHeaders() { return m_aosHeaders; }
420 
421  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
422  void* pfnUserData,
423  int bStopOnInterruptUntilUninstall );
424  int UninstallReadCbk();
425 
426  const char *GetURL() const { return m_pszURL; }
427 };
428 
429 /************************************************************************/
430 /* IVSIS3LikeFSHandler */
431 /************************************************************************/
432 
433 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandlerBase
434 {
435  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
436 
437  bool CopyFile(VSILFILE* fpIn,
438  vsi_l_offset nSourceSize,
439  const char* pszSource,
440  const char* pszTarget,
441  CSLConstList papszOptions,
442  GDALProgressFunc pProgressFunc,
443  void *pProgressData);
444  virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
445 
446  protected:
447  char** GetFileList( const char *pszFilename,
448  int nMaxFiles,
449  bool* pbGotFileList ) override;
450 
451  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
452  const char* pszURI, bool bAllowNoObject) = 0;
453 
454  virtual int CopyObject( const char *oldpath, const char *newpath,
455  CSLConstList papszMetadata );
456 
457  int RmdirRecursiveInternal( const char* pszDirname, int nBatchSize);
458 
459  virtual bool IsAllowedHeaderForObjectCreation( const char* /* pszHeaderName */ ) { return false; }
460 
461  IVSIS3LikeFSHandler() = default;
462 
463  public:
464  int Unlink( const char *pszFilename ) override;
465  int Mkdir( const char *pszDirname, long nMode ) override;
466  int Rmdir( const char *pszDirname ) override;
467  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
468  int nFlags ) override;
469  int Rename( const char *oldpath, const char *newpath ) override;
470 
471  virtual int DeleteObject( const char *pszFilename );
472 
473  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
474  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
475 
476  bool Sync( const char* pszSource, const char* pszTarget,
477  const char* const * papszOptions,
478  GDALProgressFunc pProgressFunc,
479  void *pProgressData,
480  char*** ppapszOutputs ) override;
481 
482  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
483  const char* const *papszOptions) override;
484 
485  // Multipart upload
486  virtual bool SupportsParallelMultipartUpload() const { return false; }
487 
488  virtual CPLString InitiateMultipartUpload(
489  const std::string& osFilename,
490  IVSIS3LikeHandleHelper *poS3HandleHelper,
491  int nMaxRetry,
492  double dfRetryDelay,
493  CSLConstList papszOptions);
494  virtual CPLString UploadPart(const CPLString& osFilename,
495  int nPartNumber,
496  const std::string& osUploadID,
497  vsi_l_offset nPosition,
498  const void* pabyBuffer,
499  size_t nBufferSize,
500  IVSIS3LikeHandleHelper *poS3HandleHelper,
501  int nMaxRetry,
502  double dfRetryDelay,
503  CSLConstList papszOptions);
504  virtual bool CompleteMultipart(const CPLString& osFilename,
505  const CPLString& osUploadID,
506  const std::vector<CPLString>& aosEtags,
507  vsi_l_offset nTotalSize,
508  IVSIS3LikeHandleHelper *poS3HandleHelper,
509  int nMaxRetry,
510  double dfRetryDelay);
511  virtual bool AbortMultipart(const CPLString& osFilename,
512  const CPLString& osUploadID,
513  IVSIS3LikeHandleHelper *poS3HandleHelper,
514  int nMaxRetry,
515  double dfRetryDelay);
516 
517  bool AbortPendingUploads(const char* pszFilename) override;
518 };
519 
520 /************************************************************************/
521 /* IVSIS3LikeHandle */
522 /************************************************************************/
523 
524 class IVSIS3LikeHandle: public VSICurlHandle
525 {
526  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
527 
528  protected:
529  bool UseLimitRangeGetInsteadOfHead() override { return true; }
530  bool IsDirectoryFromExists( const char* pszVerb,
531  int response_code ) override
532  {
533  // A bit dirty, but on S3, a GET on a existing directory returns a 416
534  return response_code == 416 && EQUAL(pszVerb, "GET") &&
535  CPLString(m_pszURL).back() == '/';
536  }
537  void ProcessGetFileSizeResult( const char* pszContent ) override
538  {
539  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
540  }
541 
542  public:
543  IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
544  const char* pszFilename,
545  const char* pszURLIn ) :
546  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
547  ~IVSIS3LikeHandle() override {}
548 };
549 
550 /************************************************************************/
551 /* VSIS3WriteHandle */
552 /************************************************************************/
553 
554 class VSIS3WriteHandle final : public VSIVirtualHandle
555 {
556  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
557 
558  IVSIS3LikeFSHandler *m_poFS = nullptr;
559  CPLString m_osFilename{};
560  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
561  bool m_bUseChunked = false;
562  CPLStringList m_aosOptions{};
563 
564  vsi_l_offset m_nCurOffset = 0;
565  int m_nBufferOff = 0;
566  int m_nBufferSize = 0;
567  bool m_bClosed = false;
568  GByte *m_pabyBuffer = nullptr;
569  CPLString m_osUploadID{};
570  int m_nPartNumber = 0;
571  std::vector<CPLString> m_aosEtags{};
572  bool m_bError = false;
573 
574  CURLM *m_hCurlMulti = nullptr;
575  CURL *m_hCurl = nullptr;
576  const void *m_pBuffer = nullptr;
577  CPLString m_osCurlErrBuf{};
578  size_t m_nChunkedBufferOff = 0;
579  size_t m_nChunkedBufferSize = 0;
580  size_t m_nWrittenInPUT = 0;
581 
582  int m_nMaxRetry = 0;
583  double m_dfRetryDelay = 0.0;
584  WriteFuncStruct m_sWriteFuncHeaderData{};
585 
586  bool UploadPart();
587  bool DoSinglePartPUT();
588 
589  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
590  size_t nitems, void *instream );
591  size_t WriteChunked( const void *pBuffer,
592  size_t nSize, size_t nMemb );
593  int FinishChunkedTransfer();
594 
595  void InvalidateParentDirectory();
596 
597  public:
598  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
599  const char* pszFilename,
600  IVSIS3LikeHandleHelper* poS3HandleHelper,
601  bool bUseChunked,
602  CSLConstList papszOptions );
603  ~VSIS3WriteHandle() override;
604 
605  int Seek( vsi_l_offset nOffset, int nWhence ) override;
606  vsi_l_offset Tell() override;
607  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
608  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
609  int Eof() override;
610  int Close() override;
611 
612  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
613 };
614 
615 /************************************************************************/
616 /* VSIAppendWriteHandle */
617 /************************************************************************/
618 
619 class VSIAppendWriteHandle : public VSIVirtualHandle
620 {
621  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
622 
623  protected:
624 
625  VSICurlFilesystemHandlerBase* m_poFS = nullptr;
626  CPLString m_osFSPrefix{};
627  CPLString m_osFilename{};
628 
629  vsi_l_offset m_nCurOffset = 0;
630  int m_nBufferOff = 0;
631  int m_nBufferSize = 0;
632  int m_nBufferOffReadCallback = 0;
633  bool m_bClosed = false;
634  GByte *m_pabyBuffer = nullptr;
635  bool m_bError = false;
636 
637  static size_t ReadCallBackBuffer( char *buffer, size_t size,
638  size_t nitems, void *instream );
639  virtual bool Send(bool bIsLastBlock) = 0;
640 
641  public:
642  VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
643  const char* pszFSPrefix,
644  const char* pszFilename,
645  int nChunkSize );
646  virtual ~VSIAppendWriteHandle();
647 
648  int Seek( vsi_l_offset nOffset, int nWhence ) override;
649  vsi_l_offset Tell() override;
650  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
651  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
652  int Eof() override;
653  int Close() override;
654 
655  bool IsOK() { return m_pabyBuffer != nullptr; }
656 };
657 
658 /************************************************************************/
659 /* VSIDIRWithMissingDirSynthesis */
660 /************************************************************************/
661 
662 struct VSIDIRWithMissingDirSynthesis: public VSIDIR
663 {
664  std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
665 
666 protected:
667  std::vector<std::string> m_aosSubpathsStack{};
668 
669  void SynthetizeMissingDirectories(const std::string& osCurSubdir,
670  bool bAddEntryForThisSubdir);
671 };
672 
673 /************************************************************************/
674 /* CurlRequestHelper */
675 /************************************************************************/
676 
677 struct CurlRequestHelper
678 {
679  WriteFuncStruct sWriteFuncData{};
680  WriteFuncStruct sWriteFuncHeaderData{};
681  char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
682 
683  CurlRequestHelper();
684  ~CurlRequestHelper();
685  long perform(CURL* hCurlHandle,
686  struct curl_slist* headers, // ownership transferred
687  VSICurlFilesystemHandlerBase *poFS,
688  IVSIS3LikeHandleHelper *poS3HandleHelper);
689 };
690 
691 /************************************************************************/
692 /* NetworkStatisticsLogger */
693 /************************************************************************/
694 
695 class NetworkStatisticsLogger
696 {
697  static int gnEnabled;
698  static NetworkStatisticsLogger gInstance;
699 
700  NetworkStatisticsLogger() = default;
701 
702  std::mutex m_mutex{};
703 
704  struct Counters
705  {
706  GIntBig nHEAD = 0;
707  GIntBig nGET = 0;
708  GIntBig nPUT = 0;
709  GIntBig nPOST = 0;
710  GIntBig nDELETE = 0;
711  GIntBig nGETDownloadedBytes = 0;
712  GIntBig nPUTUploadedBytes = 0;
713  GIntBig nPOSTDownloadedBytes = 0;
714  GIntBig nPOSTUploadedBytes = 0;
715  };
716 
717  enum class ContextPathType
718  {
719  FILESYSTEM,
720  FILE,
721  ACTION,
722  };
723 
724  struct ContextPathItem
725  {
726  ContextPathType eType;
727  CPLString osName;
728 
729  ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
730  eType(eTypeIn), osName(osNameIn) {}
731 
732  bool operator< (const ContextPathItem& other ) const
733  {
734  if( static_cast<int>(eType) < static_cast<int>(other.eType) )
735  return true;
736  if( static_cast<int>(eType) > static_cast<int>(other.eType) )
737  return false;
738  return osName < other.osName;
739  }
740  };
741 
742  struct Stats
743  {
744  Counters counters{};
745  std::map<ContextPathItem, Stats> children{};
746 
747  void AsJSON(CPLJSONObject& oJSON) const;
748  };
749 
750  // Workaround bug in Coverity Scan
751  // coverity[generated_default_constructor_used_in_field_initializer]
752  Stats m_stats{};
753  std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
754 
755  static void ReadEnabled();
756 
757  std::vector<Counters*> GetCountersForContext();
758 
759 public:
760 
761  static inline bool IsEnabled()
762  {
763  if( gnEnabled < 0)
764  {
765  ReadEnabled();
766  }
767  return gnEnabled == TRUE;
768  }
769 
770  static void EnterFileSystem(const char* pszName);
771 
772  static void LeaveFileSystem();
773 
774  static void EnterFile(const char* pszName);
775 
776  static void LeaveFile();
777 
778  static void EnterAction(const char* pszName);
779 
780  static void LeaveAction();
781 
782  static void LogHEAD();
783 
784  static void LogGET(size_t nDownloadedBytes);
785 
786  static void LogPUT(size_t nUploadedBytes);
787 
788  static void LogPOST(size_t nUploadedBytes,
789  size_t nDownloadedBytes);
790 
791  static void LogDELETE();
792 
793  static void Reset();
794 
795  static CPLString GetReportAsSerializedJSON();
796 };
797 
798 struct NetworkStatisticsFileSystem
799 {
800  inline explicit NetworkStatisticsFileSystem(const char* pszName) {
801  NetworkStatisticsLogger::EnterFileSystem(pszName);
802  }
803 
804  inline ~NetworkStatisticsFileSystem()
805  {
806  NetworkStatisticsLogger::LeaveFileSystem();
807  }
808 };
809 
810 struct NetworkStatisticsFile
811 {
812  inline explicit NetworkStatisticsFile(const char* pszName) {
813  NetworkStatisticsLogger::EnterFile(pszName);
814  }
815 
816  inline ~NetworkStatisticsFile()
817  {
818  NetworkStatisticsLogger::LeaveFile();
819  }
820 };
821 
822 struct NetworkStatisticsAction
823 {
824  inline explicit NetworkStatisticsAction(const char* pszName) {
825  NetworkStatisticsLogger::EnterAction(pszName);
826  }
827 
828  inline ~NetworkStatisticsAction()
829  {
830  NetworkStatisticsLogger::LeaveAction();
831  }
832 };
833 
834 
835 int VSICURLGetDownloadChunkSize();
836 
837 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
838  VSILFILE *fp,
839  VSICurlReadCbkFunc pfnReadCbk,
840  void *pReadCbkUserData );
841 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
842  size_t nmemb, void *req );
843 void MultiPerform(CURLM* hCurlMultiHandle,
844  CURL* hEasyHandle = nullptr);
845 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
846 
847 int VSICurlParseUnixPermissions(const char* pszPermissions);
848 
849 // Cache of file properties (size, etc.)
850 bool VSICURLGetCachedFileProp( const char* pszURL,
851  FileProp& oFileProp );
852 void VSICURLSetCachedFileProp( const char* pszURL,
853  FileProp& oFileProp );
854 void VSICURLInvalidateCachedFileProp( const char* pszURL );
855 void VSICURLInvalidateCachedFilePropPrefix( const char* pszURL );
856 void VSICURLDestroyCacheFileProp();
857 
858 } // namespace cpl
859 
861 
862 #endif // HAVE_CURL
863 
864 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:203
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:428
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:319
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:540
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1053
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:53
cpl_string.h
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:230
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:329
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:198
cpl_port.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:927
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156