GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <algorithm>
45 #include <condition_variable>
46 #include <set>
47 #include <map>
48 #include <memory>
49 #include <mutex>
50 #include <thread>
51 
52 // To avoid aliasing to CopyFile to CopyFileA on Windows
53 #ifdef CopyFile
54 #undef CopyFile
55 #endif
56 
58 
59 // Leave it for backward compatibility, but deprecate.
60 #define HAVE_CURLINFO_REDIRECT_URL
61 
62 void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
63 
64 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
65  const char *const *papszOptions);
66 struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
67  struct curl_slist *poSrcToDestroy);
68 
69 struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
70  const char *pszPath);
71 
72 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
73  struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
74 
75 namespace cpl
76 {
77 
78 typedef enum
79 {
80  EXIST_UNKNOWN = -1,
81  EXIST_NO,
82  EXIST_YES,
83 } ExistStatus;
84 
85 class FileProp
86 {
87  public:
88  unsigned int nGenerationAuthParameters = 0;
89  ExistStatus eExists = EXIST_UNKNOWN;
90  vsi_l_offset fileSize = 0;
91  time_t mTime = 0;
92  time_t nExpireTimestampLocal = 0;
93  CPLString osRedirectURL{};
94  bool bHasComputedFileSize = false;
95  bool bIsDirectory = false;
96  int nMode = 0; // st_mode member of struct stat
97  bool bS3LikeRedirect = false;
98  CPLString ETag{};
99 };
100 
101 struct CachedDirList
102 {
103  bool bGotFileList = false;
104  unsigned int nGenerationAuthParameters = 0;
105  CPLStringList oFileList{}; /* only file name without path */
106 };
107 
108 struct WriteFuncStruct
109 {
110  char *pBuffer = nullptr;
111  size_t nSize = 0;
112  bool bIsHTTP = false;
113  bool bMultiRange = false;
114  vsi_l_offset nStartOffset = 0;
115  vsi_l_offset nEndOffset = 0;
116  int nHTTPCode = 0;
117  vsi_l_offset nContentLength = 0;
118  bool bFoundContentRange = false;
119  bool bError = false;
120  bool bInterruptDownload = false;
121  bool bDetectRangeDownloadingError = false;
122  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
123 
124  VSILFILE *fp = nullptr;
125  VSICurlReadCbkFunc pfnReadCbk = nullptr;
126  void *pReadCbkUserData = nullptr;
127  bool bInterrupted = false;
128 
129 #if !CURL_AT_LEAST_VERSION(7, 54, 0)
130  // Workaround to ignore extra HTTP response headers from
131  // proxies in older versions of curl.
132  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
133  bool bIsProxyConnectHeader = false;
134 #endif
135 };
136 
137 struct PutData
138 {
139  const GByte *pabyData = nullptr;
140  size_t nOff = 0;
141  size_t nTotalSize = 0;
142 
143  static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
144  void *instream)
145  {
146  PutData *poThis = static_cast<PutData *>(instream);
147  const size_t nSizeMax = size * nitems;
148  const size_t nSizeToWrite =
149  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
150  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
151  poThis->nOff += nSizeToWrite;
152  return nSizeToWrite;
153  }
154 };
155 
156 /************************************************************************/
157 /* VSICurlFilesystemHandler */
158 /************************************************************************/
159 
160 class VSICurlHandle;
161 
162 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
163 {
164  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
165 
166  struct FilenameOffsetPair
167  {
168  std::string filename_;
169  vsi_l_offset offset_;
170 
171  FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
172  : filename_(filename), offset_(offset)
173  {
174  }
175 
176  bool operator==(const FilenameOffsetPair &other) const
177  {
178  return filename_ == other.filename_ && offset_ == other.offset_;
179  }
180  };
181  struct FilenameOffsetPairHasher
182  {
183  std::size_t operator()(const FilenameOffsetPair &k) const
184  {
185  return std::hash<std::string>()(k.filename_) ^
186  std::hash<vsi_l_offset>()(k.offset_);
187  }
188  };
189 
190  using RegionCacheType = lru11::Cache<
191  FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
192  std::unordered_map<
193  FilenameOffsetPair,
194  typename std::list<lru11::KeyValuePair<
195  FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
196  FilenameOffsetPairHasher>>;
197 
198  std::unique_ptr<RegionCacheType>
199  m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
200  // GetRegionCache();
201  RegionCacheType *GetRegionCache();
202 
203  // LRU cache that just keeps in memory if this file system handler is
204  // spposed to know the file properties of a file. The actual cache is a
205  // shared one among all network file systems.
206  // The aim of that design is that invalidating /vsis3/foo results in
207  // /vsis3_streaming/foo to be invalidated as well.
208  lru11::Cache<std::string, bool> oCacheFileProp;
209 
210  int nCachedFilesInDirList = 0;
211  lru11::Cache<std::string, CachedDirList> oCacheDirList;
212 
213  char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
214  char *pszData, bool *pbGotFileList);
215 
216  // Data structure and map to store regions that are in progress, to
217  // avoid simultaneous downloads of the same region in different threads
218  // Cf https://github.com/OSGeo/gdal/issues/8041
219  struct RegionInDownload
220  {
221  std::mutex oMutex{};
222  std::condition_variable oCond{};
223  bool bDownloadInProgress = false;
224  int nWaiters = 0;
225  std::string osData{};
226  };
227  std::mutex m_oMutex{};
228  std::map<std::string, std::unique_ptr<RegionInDownload>>
229  m_oMapRegionInDownload{};
230 
231  protected:
232  CPLMutex *hMutex = nullptr;
233 
234  virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
235  virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
236  bool *pbGotFileList);
237 
238  void RegisterEmptyDir(const CPLString &osDirname);
239 
240  bool
241  AnalyseS3FileList(const CPLString &osBaseURL, const char *pszXML,
242  CPLStringList &osFileList, int nMaxFiles,
243  const std::set<std::string> &oSetIgnoredStorageClasses,
244  bool &bIsTruncated);
245 
246  void AnalyseSwiftFileList(const CPLString &osBaseURL,
247  const CPLString &osPrefix, const char *pszJson,
248  CPLStringList &osFileList, int nMaxFilesThisQuery,
249  int nMaxFiles, bool &bIsTruncated,
250  CPLString &osNextMarker);
251 
252  static const char *GetOptionsStatic();
253 
254  VSICurlFilesystemHandlerBase();
255 
256  public:
257  ~VSICurlFilesystemHandlerBase() override;
258 
259  static bool IsAllowedFilename(const char *pszFilename);
260 
261  VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
262  bool bSetError,
263  CSLConstList /* papszOptions */) override;
264 
265  int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
266  int nFlags) override;
267  int Unlink(const char *pszFilename) override;
268  int Rename(const char *oldpath, const char *newpath) override;
269  int Mkdir(const char *pszDirname, long nMode) override;
270  int Rmdir(const char *pszDirname) override;
271  char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
272  char **SiblingFiles(const char *pszFilename) override;
273 
274  int HasOptimizedReadMultiRange(const char * /* pszPath */) override
275  {
276  return true;
277  }
278 
279  const char *GetActualURL(const char *pszFilename) override;
280 
281  const char *GetOptions() override;
282 
283  char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
284  CSLConstList papszOptions) override;
285 
286  char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
287  bool *pbGotFileList);
288  void InvalidateDirContent(const char *pszDirname);
289 
290  virtual const char *GetDebugKey() const = 0;
291 
292  virtual CPLString GetFSPrefix() const = 0;
293  virtual bool AllowCachedDataFor(const char *pszFilename);
294 
295  virtual bool IsLocal(const char * /* pszPath */) override
296  {
297  return false;
298  }
299  virtual bool
300  SupportsSequentialWrite(const char * /* pszPath */,
301  bool /* bAllowLocalTempFile */) override
302  {
303  return false;
304  }
305  virtual bool SupportsRandomWrite(const char * /* pszPath */,
306  bool /* bAllowLocalTempFile */) override
307  {
308  return false;
309  }
310 
311  std::shared_ptr<std::string> GetRegion(const char *pszURL,
312  vsi_l_offset nFileOffsetStart);
313 
314  void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
315  size_t nSize, const char *pData);
316 
317  std::string NotifyStartDownloadRegion(const std::string &osURL,
318  vsi_l_offset startOffset,
319  int nBlocks);
320  void NotifyStopDownloadRegion(const std::string &osURL,
321  vsi_l_offset startOffset, int nBlocks,
322  const std::string &osData);
323 
324  bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
325  void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
326  void InvalidateCachedData(const char *pszURL);
327 
328  CURLM *GetCurlMultiHandleFor(const CPLString &osURL);
329 
330  virtual void ClearCache();
331  virtual void PartialClearCache(const char *pszFilename);
332 
333  bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
334  void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
335  bool ExistsInCacheDirList(const CPLString &osDirname, bool *pbIsDir);
336 
337  virtual CPLString GetURLFromFilename(const CPLString &osFilename);
338 
339  std::string
340  GetStreamingFilename(const std::string &osFilename) const override = 0;
341 
342  static std::set<std::string> GetS3IgnoredStorageClasses();
343 };
344 
345 class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
346 {
347  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
348 
349  public:
350  VSICurlFilesystemHandler() = default;
351 
352  const char *GetDebugKey() const override
353  {
354  return "VSICURL";
355  }
356 
357  CPLString GetFSPrefix() const override
358  {
359  return "/vsicurl/";
360  }
361 
362  std::string
363  GetStreamingFilename(const std::string &osFilename) const override;
364 };
365 
366 /************************************************************************/
367 /* VSICurlHandle */
368 /************************************************************************/
369 
370 class VSICurlHandle : public VSIVirtualHandle
371 {
372  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
373 
374  protected:
375  VSICurlFilesystemHandlerBase *poFS = nullptr;
376 
377  bool m_bCached = true;
378 
379  mutable FileProp oFileProp{};
380 
381  mutable std::mutex m_oMutex{};
382  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
383  char *m_pszURL = nullptr; // e.g "http://example.com/foo"
384  mutable std::string m_osQueryString{}; // e.g. an Azure SAS
385 
386  char **m_papszHTTPOptions = nullptr;
387 
388  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
389  int nBlocksToDownload = 1;
390 
391  bool bStopOnInterruptUntilUninstall = false;
392  bool bInterrupted = false;
393  VSICurlReadCbkFunc pfnReadCbk = nullptr;
394  void *pReadCbkUserData = nullptr;
395 
396  int m_nMaxRetry = 0;
397  double m_dfRetryDelay = 0.0;
398 
399  CPLStringList m_aosHeaders{};
400 
401  void DownloadRegionPostProcess(const vsi_l_offset startOffset,
402  const int nBlocks, const char *pBuffer,
403  size_t nSize);
404 
405  private:
406  vsi_l_offset curOffset = 0;
407 
408  bool bEOF = false;
409 
410  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
411 
412  bool m_bUseHead = false;
413  bool m_bUseRedirectURLIfNoQueryStringParams = false;
414 
415  // Specific to Planetary Computer signing:
416  // https://planetarycomputer.microsoft.com/docs/concepts/sas/
417  mutable bool m_bPlanetaryComputerURLSigning = false;
418  mutable std::string m_osPlanetaryComputerCollection{};
419  void ManagePlanetaryComputerSigning() const;
420 
421  int ReadMultiRangeSingleGet(int nRanges, void **ppData,
422  const vsi_l_offset *panOffsets,
423  const size_t *panSizes);
424  CPLString GetRedirectURLIfValid(bool &bHasExpired) const;
425 
426  void UpdateRedirectInfo(CURL *hCurlHandle,
427  const WriteFuncStruct &sWriteFuncHeaderData);
428 
429  // Used by AdviseRead()
430  struct AdviseReadRange
431  {
432  bool bDone = false;
433  std::mutex oMutex{};
434  std::condition_variable oCV{};
435  vsi_l_offset nStartOffset = 0;
436  size_t nSize = 0;
437  std::vector<GByte> abyData{};
438  };
439  std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
440  std::thread m_oThreadAdviseRead{};
441 
442  protected:
443  virtual struct curl_slist *
444  GetCurlHeaders(const CPLString & /*osVerb*/,
445  const struct curl_slist * /* psExistingHeaders */)
446  {
447  return nullptr;
448  }
449  virtual bool AllowAutomaticRedirection()
450  {
451  return true;
452  }
453  virtual bool CanRestartOnError(const char *, const char *, bool)
454  {
455  return false;
456  }
457  virtual bool UseLimitRangeGetInsteadOfHead()
458  {
459  return false;
460  }
461  virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
462  int /*response_code*/)
463  {
464  return false;
465  }
466  virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
467  {
468  }
469  void SetURL(const char *pszURL);
470  virtual bool Authenticate(const char * /* pszFilename */)
471  {
472  return false;
473  }
474 
475  public:
476  VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
477  const char *pszURLIn = nullptr);
478  ~VSICurlHandle() override;
479 
480  int Seek(vsi_l_offset nOffset, int nWhence) override;
481  vsi_l_offset Tell() override;
482  size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
483  int ReadMultiRange(int nRanges, void **ppData,
484  const vsi_l_offset *panOffsets,
485  const size_t *panSizes) override;
486  size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
487  int Eof() override;
488  int Flush() override;
489  int Close() override;
490 
491  bool HasPRead() const override
492  {
493  return true;
494  }
495  size_t PRead(void *pBuffer, size_t nSize,
496  vsi_l_offset nOffset) const override;
497 
498  void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
499  const size_t *panSizes) override;
500 
501  bool IsKnownFileSize() const
502  {
503  return oFileProp.bHasComputedFileSize;
504  }
505  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
506  virtual vsi_l_offset GetFileSize(bool bSetError)
507  {
508  return GetFileSizeOrHeaders(bSetError, false);
509  }
510  bool Exists(bool bSetError);
511  bool IsDirectory() const
512  {
513  return oFileProp.bIsDirectory;
514  }
515  int GetMode() const
516  {
517  return oFileProp.nMode;
518  }
519  time_t GetMTime() const
520  {
521  return oFileProp.mTime;
522  }
523  const CPLStringList &GetHeaders()
524  {
525  return m_aosHeaders;
526  }
527 
528  int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
529  int bStopOnInterruptUntilUninstall);
530  int UninstallReadCbk();
531 
532  const char *GetURL() const
533  {
534  return m_pszURL;
535  }
536 };
537 
538 /************************************************************************/
539 /* VSICurlFilesystemHandlerBaseWritable */
540 /************************************************************************/
541 
542 class VSICurlFilesystemHandlerBaseWritable : public VSICurlFilesystemHandlerBase
543 {
544  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBaseWritable)
545 
546  protected:
547  VSICurlFilesystemHandlerBaseWritable() = default;
548 
549  virtual VSIVirtualHandleUniquePtr
550  CreateWriteHandle(const char *pszFilename, CSLConstList papszOptions) = 0;
551 
552  public:
553  VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
554  bool bSetError, CSLConstList papszOptions) override;
555 
556  bool SupportsSequentialWrite(const char * /* pszPath */,
557  bool /* bAllowLocalTempFile */) override
558  {
559  return true;
560  }
561  bool SupportsRandomWrite(const char * /* pszPath */,
562  bool /* bAllowLocalTempFile */) override;
563 };
564 
565 /************************************************************************/
566 /* IVSIS3LikeFSHandler */
567 /************************************************************************/
568 
569 class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBaseWritable
570 {
571  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
572 
573  virtual int MkdirInternal(const char *pszDirname, long nMode,
574  bool bDoStatCheck);
575 
576  protected:
577  char **GetFileList(const char *pszFilename, int nMaxFiles,
578  bool *pbGotFileList) override;
579 
580  virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
581  bool bAllowNoObject) = 0;
582 
583  virtual int CopyObject(const char *oldpath, const char *newpath,
584  CSLConstList papszMetadata);
585 
586  int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
587 
588  virtual bool
589  IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
590  {
591  return false;
592  }
593 
594  IVSIS3LikeFSHandler() = default;
595 
596  public:
597  int Unlink(const char *pszFilename) override;
598  int Mkdir(const char *pszDirname, long nMode) override;
599  int Rmdir(const char *pszDirname) override;
600  int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
601  int nFlags) override;
602  int Rename(const char *oldpath, const char *newpath) override;
603 
604  virtual int CopyFile(const char *pszSource, const char *pszTarget,
605  VSILFILE *fpSource, vsi_l_offset nSourceSize,
606  const char *const *papszOptions,
607  GDALProgressFunc pProgressFunc,
608  void *pProgressData) override;
609 
610  virtual int DeleteObject(const char *pszFilename);
611 
612  bool Sync(const char *pszSource, const char *pszTarget,
613  const char *const *papszOptions, GDALProgressFunc pProgressFunc,
614  void *pProgressData, char ***ppapszOutputs) override;
615 
616  VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
617  const char *const *papszOptions) override;
618 
619  // Multipart upload
620  virtual bool SupportsParallelMultipartUpload() const
621  {
622  return false;
623  }
624 
625  virtual CPLString InitiateMultipartUpload(
626  const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
627  int nMaxRetry, double dfRetryDelay, CSLConstList papszOptions);
628  virtual CPLString UploadPart(const CPLString &osFilename, int nPartNumber,
629  const std::string &osUploadID,
630  vsi_l_offset nPosition, const void *pabyBuffer,
631  size_t nBufferSize,
632  IVSIS3LikeHandleHelper *poS3HandleHelper,
633  int nMaxRetry, double dfRetryDelay,
634  CSLConstList papszOptions);
635  virtual bool CompleteMultipart(const CPLString &osFilename,
636  const CPLString &osUploadID,
637  const std::vector<CPLString> &aosEtags,
638  vsi_l_offset nTotalSize,
639  IVSIS3LikeHandleHelper *poS3HandleHelper,
640  int nMaxRetry, double dfRetryDelay);
641  virtual bool AbortMultipart(const CPLString &osFilename,
642  const CPLString &osUploadID,
643  IVSIS3LikeHandleHelper *poS3HandleHelper,
644  int nMaxRetry, double dfRetryDelay);
645 
646  bool AbortPendingUploads(const char *pszFilename) override;
647 };
648 
649 /************************************************************************/
650 /* IVSIS3LikeHandle */
651 /************************************************************************/
652 
653 class IVSIS3LikeHandle : public VSICurlHandle
654 {
655  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
656 
657  protected:
658  bool UseLimitRangeGetInsteadOfHead() override
659  {
660  return true;
661  }
662  bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
663  {
664  // A bit dirty, but on S3, a GET on a existing directory returns a 416
665  return response_code == 416 && EQUAL(pszVerb, "GET") &&
666  CPLString(m_pszURL).back() == '/';
667  }
668  void ProcessGetFileSizeResult(const char *pszContent) override
669  {
670  oFileProp.bIsDirectory =
671  strstr(pszContent, "ListBucketResult") != nullptr;
672  }
673 
674  public:
675  IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
676  const char *pszFilename, const char *pszURLIn)
677  : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
678  {
679  }
680  ~IVSIS3LikeHandle() override
681  {
682  }
683 };
684 
685 /************************************************************************/
686 /* VSIS3WriteHandle */
687 /************************************************************************/
688 
689 class VSIS3WriteHandle final : public VSIVirtualHandle
690 {
691  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
692 
693  IVSIS3LikeFSHandler *m_poFS = nullptr;
694  CPLString m_osFilename{};
695  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
696  bool m_bUseChunked = false;
697  CPLStringList m_aosOptions{};
698  CPLStringList m_aosHTTPOptions{};
699 
700  vsi_l_offset m_nCurOffset = 0;
701  int m_nBufferOff = 0;
702  int m_nBufferSize = 0;
703  bool m_bClosed = false;
704  GByte *m_pabyBuffer = nullptr;
705  CPLString m_osUploadID{};
706  int m_nPartNumber = 0;
707  std::vector<CPLString> m_aosEtags{};
708  bool m_bError = false;
709 
710  CURLM *m_hCurlMulti = nullptr;
711  CURL *m_hCurl = nullptr;
712  const void *m_pBuffer = nullptr;
713  CPLString m_osCurlErrBuf{};
714  size_t m_nChunkedBufferOff = 0;
715  size_t m_nChunkedBufferSize = 0;
716  size_t m_nWrittenInPUT = 0;
717 
718  int m_nMaxRetry = 0;
719  double m_dfRetryDelay = 0.0;
720  WriteFuncStruct m_sWriteFuncHeaderData{};
721 
722  bool UploadPart();
723  bool DoSinglePartPUT();
724 
725  static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
726  size_t nitems, void *instream);
727  size_t WriteChunked(const void *pBuffer, size_t nSize, size_t nMemb);
728  int FinishChunkedTransfer();
729 
730  void InvalidateParentDirectory();
731 
732  public:
733  VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
734  IVSIS3LikeHandleHelper *poS3HandleHelper, bool bUseChunked,
735  CSLConstList papszOptions);
736  ~VSIS3WriteHandle() override;
737 
738  int Seek(vsi_l_offset nOffset, int nWhence) override;
739  vsi_l_offset Tell() override;
740  size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
741  size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
742  int Eof() override;
743  int Close() override;
744 
745  bool IsOK()
746  {
747  return m_bUseChunked || m_pabyBuffer != nullptr;
748  }
749 };
750 
751 /************************************************************************/
752 /* VSIAppendWriteHandle */
753 /************************************************************************/
754 
755 class VSIAppendWriteHandle : public VSIVirtualHandle
756 {
757  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
758 
759  protected:
760  VSICurlFilesystemHandlerBase *m_poFS = nullptr;
761  CPLString m_osFSPrefix{};
762  CPLString m_osFilename{};
763 
764  vsi_l_offset m_nCurOffset = 0;
765  int m_nBufferOff = 0;
766  int m_nBufferSize = 0;
767  int m_nBufferOffReadCallback = 0;
768  bool m_bClosed = false;
769  GByte *m_pabyBuffer = nullptr;
770  bool m_bError = false;
771 
772  static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
773  void *instream);
774  virtual bool Send(bool bIsLastBlock) = 0;
775 
776  public:
777  VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
778  const char *pszFSPrefix, const char *pszFilename,
779  int nChunkSize);
780  virtual ~VSIAppendWriteHandle();
781 
782  int Seek(vsi_l_offset nOffset, int nWhence) override;
783  vsi_l_offset Tell() override;
784  size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
785  size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
786  int Eof() override;
787  int Close() override;
788 
789  bool IsOK()
790  {
791  return m_pabyBuffer != nullptr;
792  }
793 };
794 
795 /************************************************************************/
796 /* VSIDIRWithMissingDirSynthesis */
797 /************************************************************************/
798 
799 struct VSIDIRWithMissingDirSynthesis : public VSIDIR
800 {
801  std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
802 
803  protected:
804  std::vector<std::string> m_aosSubpathsStack{};
805 
806  void SynthetizeMissingDirectories(const std::string &osCurSubdir,
807  bool bAddEntryForThisSubdir);
808 };
809 
810 /************************************************************************/
811 /* CurlRequestHelper */
812 /************************************************************************/
813 
814 struct CurlRequestHelper
815 {
816  WriteFuncStruct sWriteFuncData{};
817  WriteFuncStruct sWriteFuncHeaderData{};
818  char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
819 
820  CurlRequestHelper();
821  ~CurlRequestHelper();
822  long perform(CURL *hCurlHandle,
823  struct curl_slist *headers, // ownership transferred
824  VSICurlFilesystemHandlerBase *poFS,
825  IVSIS3LikeHandleHelper *poS3HandleHelper);
826 };
827 
828 /************************************************************************/
829 /* NetworkStatisticsLogger */
830 /************************************************************************/
831 
832 class NetworkStatisticsLogger
833 {
834  static int gnEnabled;
835  static NetworkStatisticsLogger gInstance;
836 
837  NetworkStatisticsLogger() = default;
838 
839  std::mutex m_mutex{};
840 
841  struct Counters
842  {
843  GIntBig nHEAD = 0;
844  GIntBig nGET = 0;
845  GIntBig nPUT = 0;
846  GIntBig nPOST = 0;
847  GIntBig nDELETE = 0;
848  GIntBig nGETDownloadedBytes = 0;
849  GIntBig nPUTUploadedBytes = 0;
850  GIntBig nPOSTDownloadedBytes = 0;
851  GIntBig nPOSTUploadedBytes = 0;
852  };
853 
854  enum class ContextPathType
855  {
856  FILESYSTEM,
857  FILE,
858  ACTION,
859  };
860 
861  struct ContextPathItem
862  {
863  ContextPathType eType;
864  CPLString osName;
865 
866  ContextPathItem(ContextPathType eTypeIn, const CPLString &osNameIn)
867  : eType(eTypeIn), osName(osNameIn)
868  {
869  }
870 
871  bool operator<(const ContextPathItem &other) const
872  {
873  if (static_cast<int>(eType) < static_cast<int>(other.eType))
874  return true;
875  if (static_cast<int>(eType) > static_cast<int>(other.eType))
876  return false;
877  return osName < other.osName;
878  }
879  };
880 
881  struct Stats
882  {
883  Counters counters{};
884  std::map<ContextPathItem, Stats> children{};
885 
886  void AsJSON(CPLJSONObject &oJSON) const;
887  };
888 
889  // Workaround bug in Coverity Scan
890  // coverity[generated_default_constructor_used_in_field_initializer]
891  Stats m_stats{};
892  std::map<GIntBig, std::vector<ContextPathItem>>
893  m_mapThreadIdToContextPath{};
894 
895  static void ReadEnabled();
896 
897  std::vector<Counters *> GetCountersForContext();
898 
899  public:
900  static inline bool IsEnabled()
901  {
902  if (gnEnabled < 0)
903  {
904  ReadEnabled();
905  }
906  return gnEnabled == TRUE;
907  }
908 
909  static void EnterFileSystem(const char *pszName);
910 
911  static void LeaveFileSystem();
912 
913  static void EnterFile(const char *pszName);
914 
915  static void LeaveFile();
916 
917  static void EnterAction(const char *pszName);
918 
919  static void LeaveAction();
920 
921  static void LogHEAD();
922 
923  static void LogGET(size_t nDownloadedBytes);
924 
925  static void LogPUT(size_t nUploadedBytes);
926 
927  static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
928 
929  static void LogDELETE();
930 
931  static void Reset();
932 
933  static CPLString GetReportAsSerializedJSON();
934 };
935 
936 struct NetworkStatisticsFileSystem
937 {
938  inline explicit NetworkStatisticsFileSystem(const char *pszName)
939  {
940  NetworkStatisticsLogger::EnterFileSystem(pszName);
941  }
942 
943  inline ~NetworkStatisticsFileSystem()
944  {
945  NetworkStatisticsLogger::LeaveFileSystem();
946  }
947 };
948 
949 struct NetworkStatisticsFile
950 {
951  inline explicit NetworkStatisticsFile(const char *pszName)
952  {
953  NetworkStatisticsLogger::EnterFile(pszName);
954  }
955 
956  inline ~NetworkStatisticsFile()
957  {
958  NetworkStatisticsLogger::LeaveFile();
959  }
960 };
961 
962 struct NetworkStatisticsAction
963 {
964  inline explicit NetworkStatisticsAction(const char *pszName)
965  {
966  NetworkStatisticsLogger::EnterAction(pszName);
967  }
968 
969  inline ~NetworkStatisticsAction()
970  {
971  NetworkStatisticsLogger::LeaveAction();
972  }
973 };
974 
975 int VSICURLGetDownloadChunkSize();
976 
977 void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct, VSILFILE *fp,
978  VSICurlReadCbkFunc pfnReadCbk,
979  void *pReadCbkUserData);
980 size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
981  void *req);
982 void MultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr);
983 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
984 
985 int VSICurlParseUnixPermissions(const char *pszPermissions);
986 
987 // Cache of file properties (size, etc.)
988 bool VSICURLGetCachedFileProp(const char *pszURL, FileProp &oFileProp);
989 void VSICURLSetCachedFileProp(const char *pszURL, FileProp &oFileProp);
990 void VSICURLInvalidateCachedFileProp(const char *pszURL);
991 void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
992 void VSICURLDestroyCacheFileProp();
993 
994 } // namespace cpl
995 
997 
998 #endif // HAVE_CURL
999 
1000 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:196
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:437
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:562
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1178
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:62
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:226
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:384
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
cpl_port.h
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042