GDAL
cpl_vsil_curl_class.h
1/******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
14#define CPL_VSIL_CURL_CLASS_H_INCLUDED
15
16#ifdef HAVE_CURL
17
18#include "cpl_aws.h"
19#include "cpl_azure.h"
20#include "cpl_port.h"
21#include "cpl_json.h"
22#include "cpl_string.h"
23#include "cpl_vsil_curl_priv.h"
24#include "cpl_mem_cache.h"
25
26#include "cpl_curl_priv.h"
27
28#include <algorithm>
29#include <atomic>
30#include <condition_variable>
31#include <set>
32#include <map>
33#include <memory>
34#include <mutex>
35#include <thread>
36#include <utility>
37
38// To avoid aliasing to CopyFile to CopyFileA on Windows
39#ifdef CopyFile
40#undef CopyFile
41#endif
42
44
45// Leave it for backward compatibility, but deprecate.
46#define HAVE_CURLINFO_REDIRECT_URL
47
48void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
49
50struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
51 const char *const *papszOptions);
52struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
53 struct curl_slist *poSrcToDestroy);
54
55struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
56 const char *pszPath);
57
58struct curl_slist *VSICurlSetCreationHeadersFromOptions(
59 struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
60
61namespace cpl
62{
63
64typedef enum
65{
66 EXIST_UNKNOWN = -1,
67 EXIST_NO,
68 EXIST_YES,
69} ExistStatus;
70
71class FileProp
72{
73 public:
74 unsigned int nGenerationAuthParameters = 0;
75 ExistStatus eExists = EXIST_UNKNOWN;
76 int nHTTPCode = 0;
77 vsi_l_offset fileSize = 0;
78 time_t mTime = 0;
79 time_t nExpireTimestampLocal = 0;
80 std::string osRedirectURL{};
81 bool bHasComputedFileSize = false;
82 bool bIsDirectory = false;
83 int nMode = 0; // st_mode member of struct stat
84 bool bS3LikeRedirect = false;
85 std::string ETag{};
86};
87
88struct CachedDirList
89{
90 bool bGotFileList = false;
91 unsigned int nGenerationAuthParameters = 0;
92 CPLStringList oFileList{}; /* only file name without path */
93};
94
95struct WriteFuncStruct
96{
97 char *pBuffer = nullptr;
98 size_t nSize = 0;
99 bool bIsHTTP = false;
100 bool bMultiRange = false;
101 vsi_l_offset nStartOffset = 0;
102 vsi_l_offset nEndOffset = 0;
103 int nHTTPCode = 0; // potentially after redirect
104 int nFirstHTTPCode = 0; // the one of the redirect
105 vsi_l_offset nContentLength = 0;
106 bool bFoundContentRange = false;
107 bool bError = false;
108 bool bInterruptDownload = false;
109 bool bDetectRangeDownloadingError = false;
110 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
111
112 VSILFILE *fp = nullptr;
113 VSICurlReadCbkFunc pfnReadCbk = nullptr;
114 void *pReadCbkUserData = nullptr;
115 bool bInterrupted = false;
116};
117
118struct PutData
119{
120 const GByte *pabyData = nullptr;
121 size_t nOff = 0;
122 size_t nTotalSize = 0;
123
124 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
125 void *instream)
126 {
127 PutData *poThis = static_cast<PutData *>(instream);
128 const size_t nSizeMax = size * nitems;
129 const size_t nSizeToWrite =
130 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
131 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
132 poThis->nOff += nSizeToWrite;
133 return nSizeToWrite;
134 }
135};
136
137/************************************************************************/
138/* VSICurlFilesystemHandler */
139/************************************************************************/
140
141class VSICurlHandle;
142
143class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
144{
145 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
146
147 struct FilenameOffsetPair
148 {
149 std::string filename_;
150 vsi_l_offset offset_;
151
152 FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
153 : filename_(filename), offset_(offset)
154 {
155 }
156
157 bool operator==(const FilenameOffsetPair &other) const
158 {
159 return filename_ == other.filename_ && offset_ == other.offset_;
160 }
161 };
162
163 struct FilenameOffsetPairHasher
164 {
165 std::size_t operator()(const FilenameOffsetPair &k) const
166 {
167 return std::hash<std::string>()(k.filename_) ^
168 std::hash<vsi_l_offset>()(k.offset_);
169 }
170 };
171
172 using RegionCacheType = lru11::Cache<
173 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
174 std::unordered_map<
175 FilenameOffsetPair,
176 typename std::list<lru11::KeyValuePair<
177 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
178 FilenameOffsetPairHasher>>;
179
180 std::unique_ptr<RegionCacheType>
181 m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
182 // GetRegionCache();
183 RegionCacheType *GetRegionCache();
184
185 // LRU cache that just keeps in memory if this file system handler is
186 // spposed to know the file properties of a file. The actual cache is a
187 // shared one among all network file systems.
188 // The aim of that design is that invalidating /vsis3/foo results in
189 // /vsis3_streaming/foo to be invalidated as well.
190 lru11::Cache<std::string, bool> oCacheFileProp;
191
192 int nCachedFilesInDirList = 0;
193 lru11::Cache<std::string, CachedDirList> oCacheDirList;
194
195 char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
196 char *pszData, bool *pbGotFileList);
197
198 // Data structure and map to store regions that are in progress, to
199 // avoid simultaneous downloads of the same region in different threads
200 // Cf https://github.com/OSGeo/gdal/issues/8041
201 struct RegionInDownload
202 {
203 std::mutex oMutex{};
204 std::condition_variable oCond{};
205 bool bDownloadInProgress = false;
206 int nWaiters = 0;
207 std::string osData{};
208 };
209
210 std::mutex m_oMutex{};
211 std::map<std::string, std::unique_ptr<RegionInDownload>>
212 m_oMapRegionInDownload{};
213
214 protected:
215 CPLMutex *hMutex = nullptr;
216
217 virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
218 virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
219 bool *pbGotFileList);
220
221 void RegisterEmptyDir(const std::string &osDirname);
222
223 bool
224 AnalyseS3FileList(const std::string &osBaseURL, const char *pszXML,
225 CPLStringList &osFileList, int nMaxFiles,
226 const std::set<std::string> &oSetIgnoredStorageClasses,
227 bool &bIsTruncated);
228
229 void AnalyseSwiftFileList(const std::string &osBaseURL,
230 const std::string &osPrefix, const char *pszJson,
231 CPLStringList &osFileList, int nMaxFilesThisQuery,
232 int nMaxFiles, bool &bIsTruncated,
233 std::string &osNextMarker);
234
235 static const char *GetOptionsStatic();
236
237 VSICurlFilesystemHandlerBase();
238
239 public:
240 ~VSICurlFilesystemHandlerBase() override;
241
242 static bool IsAllowedFilename(const char *pszFilename);
243
244 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
245 bool bSetError,
246 CSLConstList /* papszOptions */) override;
247
248 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
249 int nFlags) override;
250 int Unlink(const char *pszFilename) override;
251 int Rename(const char *oldpath, const char *newpath) override;
252 int Mkdir(const char *pszDirname, long nMode) override;
253 int Rmdir(const char *pszDirname) override;
254 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
255 char **SiblingFiles(const char *pszFilename) override;
256
257 int HasOptimizedReadMultiRange(const char * /* pszPath */) override
258 {
259 return true;
260 }
261
262 const char *GetActualURL(const char *pszFilename) override;
263
264 const char *GetOptions() override;
265
266 char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
267 CSLConstList papszOptions) override;
268
269 char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
270 bool *pbGotFileList);
271 void InvalidateDirContent(const char *pszDirname);
272
273 virtual const char *GetDebugKey() const = 0;
274
275 virtual std::string GetFSPrefix() const = 0;
276 virtual bool AllowCachedDataFor(const char *pszFilename);
277
278 virtual bool IsLocal(const char * /* pszPath */) override
279 {
280 return false;
281 }
282
283 virtual bool
284 SupportsSequentialWrite(const char * /* pszPath */,
285 bool /* bAllowLocalTempFile */) override
286 {
287 return false;
288 }
289
290 virtual bool SupportsRandomWrite(const char * /* pszPath */,
291 bool /* bAllowLocalTempFile */) override
292 {
293 return false;
294 }
295
296 std::shared_ptr<std::string> GetRegion(const char *pszURL,
297 vsi_l_offset nFileOffsetStart);
298
299 void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
300 size_t nSize, const char *pData);
301
302 std::pair<bool, std::string>
303 NotifyStartDownloadRegion(const std::string &osURL,
304 vsi_l_offset startOffset, int nBlocks);
305 void NotifyStopDownloadRegion(const std::string &osURL,
306 vsi_l_offset startOffset, int nBlocks,
307 const std::string &osData);
308
309 bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
310 void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
311 void InvalidateCachedData(const char *pszURL);
312
313 CURLM *GetCurlMultiHandleFor(const std::string &osURL);
314
315 virtual void ClearCache();
316 virtual void PartialClearCache(const char *pszFilename);
317
318 bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
319 void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
320 bool ExistsInCacheDirList(const std::string &osDirname, bool *pbIsDir);
321
322 virtual std::string GetURLFromFilename(const std::string &osFilename) const;
323
324 std::string
325 GetStreamingFilename(const std::string &osFilename) const override = 0;
326
327 static std::set<std::string> GetS3IgnoredStorageClasses();
328};
329
330class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
331{
332 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
333
334 public:
335 VSICurlFilesystemHandler() = default;
336
337 const char *GetDebugKey() const override
338 {
339 return "VSICURL";
340 }
341
342 std::string GetFSPrefix() const override
343 {
344 return "/vsicurl/";
345 }
346
347 std::string
348 GetStreamingFilename(const std::string &osFilename) const override;
349};
350
351/************************************************************************/
352/* VSICurlHandle */
353/************************************************************************/
354
355class VSICurlHandle : public VSIVirtualHandle
356{
357 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
358
359 protected:
360 VSICurlFilesystemHandlerBase *poFS = nullptr;
361
362 bool m_bCached = true;
363
364 mutable FileProp oFileProp{};
365
366 mutable std::mutex m_oMutex{};
367 std::string m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
368 char *m_pszURL = nullptr; // e.g "http://example.com/foo"
369 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
370
371 CPLStringList m_aosHTTPOptions{};
372 CPLHTTPRetryParameters
373 m_oRetryParameters; // must be initialized in constructor
374
375 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
376 int nBlocksToDownload = 1;
377
378 bool bStopOnInterruptUntilUninstall = false;
379 bool bInterrupted = false;
380 VSICurlReadCbkFunc pfnReadCbk = nullptr;
381 void *pReadCbkUserData = nullptr;
382
383 CPLStringList m_aosHeaders{};
384
385 void DownloadRegionPostProcess(const vsi_l_offset startOffset,
386 const int nBlocks, const char *pBuffer,
387 size_t nSize);
388
389 private:
390 vsi_l_offset curOffset = 0;
391
392 bool bEOF = false;
393 bool bError = false;
394
395 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
396
397 bool m_bUseHead = false;
398 bool m_bUseRedirectURLIfNoQueryStringParams = false;
399
400 mutable std::atomic<bool> m_bInterrupt = false;
401
402 // Specific to Planetary Computer signing:
403 // https://planetarycomputer.microsoft.com/docs/concepts/sas/
404 mutable bool m_bPlanetaryComputerURLSigning = false;
405 mutable std::string m_osPlanetaryComputerCollection{};
406 void ManagePlanetaryComputerSigning() const;
407
408 int ReadMultiRangeSingleGet(int nRanges, void **ppData,
409 const vsi_l_offset *panOffsets,
410 const size_t *panSizes);
411 std::string GetRedirectURLIfValid(bool &bHasExpired,
412 CPLStringList &aosHTTPOptions) const;
413
414 void UpdateRedirectInfo(CURL *hCurlHandle,
415 const WriteFuncStruct &sWriteFuncHeaderData);
416
417 // Used by AdviseRead()
418 struct AdviseReadRange
419 {
420 bool bDone = false;
421 std::mutex oMutex{};
422 std::condition_variable oCV{};
423 vsi_l_offset nStartOffset = 0;
424 size_t nSize = 0;
425 std::vector<GByte> abyData{};
426 };
427
428 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
429 std::thread m_oThreadAdviseRead{};
430
431 protected:
432 virtual struct curl_slist *
433 GetCurlHeaders(const std::string & /*osVerb*/,
434 const struct curl_slist * /* psExistingHeaders */)
435 {
436 return nullptr;
437 }
438
439 virtual bool AllowAutomaticRedirection()
440 {
441 return true;
442 }
443
444 virtual bool CanRestartOnError(const char *, const char *, bool)
445 {
446 return false;
447 }
448
449 virtual bool UseLimitRangeGetInsteadOfHead()
450 {
451 return false;
452 }
453
454 virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
455 int /*response_code*/)
456 {
457 return false;
458 }
459
460 virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
461 {
462 }
463
464 void SetURL(const char *pszURL);
465
466 virtual bool Authenticate(const char * /* pszFilename */)
467 {
468 return false;
469 }
470
471 public:
472 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
473 const char *pszURLIn = nullptr);
474 ~VSICurlHandle() override;
475
476 int Seek(vsi_l_offset nOffset, int nWhence) override;
477 vsi_l_offset Tell() override;
478 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
479 int ReadMultiRange(int nRanges, void **ppData,
480 const vsi_l_offset *panOffsets,
481 const size_t *panSizes) override;
482 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
483 void ClearErr() override;
484 int Eof() override;
485 int Error() override;
486 int Flush() override;
487 int Close() override;
488
489 void Interrupt() override
490 {
491 m_bInterrupt = true;
492 }
493
494 bool HasPRead() const override
495 {
496 return true;
497 }
498
499 size_t PRead(void *pBuffer, size_t nSize,
500 vsi_l_offset nOffset) const override;
501
502 void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
503 const size_t *panSizes) override;
504
505 size_t GetAdviseReadTotalBytesLimit() const override;
506
507 bool IsKnownFileSize() const
508 {
509 return oFileProp.bHasComputedFileSize;
510 }
511
512 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
513
514 virtual vsi_l_offset GetFileSize(bool bSetError)
515 {
516 return GetFileSizeOrHeaders(bSetError, false);
517 }
518
519 bool Exists(bool bSetError);
520
521 bool IsDirectory() const
522 {
523 return oFileProp.bIsDirectory;
524 }
525
526 int GetMode() const
527 {
528 return oFileProp.nMode;
529 }
530
531 time_t GetMTime() const
532 {
533 return oFileProp.mTime;
534 }
535
536 const CPLStringList &GetHeaders()
537 {
538 return m_aosHeaders;
539 }
540
541 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
542 int bStopOnInterruptUntilUninstall);
543 int UninstallReadCbk();
544
545 const char *GetURL() const
546 {
547 return m_pszURL;
548 }
549};
550
551/************************************************************************/
552/* VSICurlFilesystemHandlerBaseWritable */
553/************************************************************************/
554
555class VSICurlFilesystemHandlerBaseWritable : public VSICurlFilesystemHandlerBase
556{
557 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBaseWritable)
558
559 protected:
560 VSICurlFilesystemHandlerBaseWritable() = default;
561
562 virtual VSIVirtualHandleUniquePtr
563 CreateWriteHandle(const char *pszFilename, CSLConstList papszOptions) = 0;
564
565 public:
566 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
567 bool bSetError, CSLConstList papszOptions) override;
568
569 bool SupportsSequentialWrite(const char * /* pszPath */,
570 bool /* bAllowLocalTempFile */) override
571 {
572 return true;
573 }
574
575 bool SupportsRandomWrite(const char * /* pszPath */,
576 bool /* bAllowLocalTempFile */) override;
577};
578
579/************************************************************************/
580/* IVSIS3LikeFSHandler */
581/************************************************************************/
582
583class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBaseWritable
584{
585 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
586
587 virtual int MkdirInternal(const char *pszDirname, long nMode,
588 bool bDoStatCheck);
589
590 protected:
591 char **GetFileList(const char *pszFilename, int nMaxFiles,
592 bool *pbGotFileList) override;
593
594 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
595 bool bAllowNoObject) = 0;
596
597 virtual int CopyObject(const char *oldpath, const char *newpath,
598 CSLConstList papszMetadata);
599
600 int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
601
602 virtual bool
603 IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
604 {
605 return false;
606 }
607
608 IVSIS3LikeFSHandler() = default;
609
610 public:
611 int Unlink(const char *pszFilename) override;
612 int Mkdir(const char *pszDirname, long nMode) override;
613 int Rmdir(const char *pszDirname) override;
614 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
615 int nFlags) override;
616 int Rename(const char *oldpath, const char *newpath) override;
617
618 virtual int CopyFile(const char *pszSource, const char *pszTarget,
619 VSILFILE *fpSource, vsi_l_offset nSourceSize,
620 const char *const *papszOptions,
621 GDALProgressFunc pProgressFunc,
622 void *pProgressData) override;
623
624 virtual int DeleteObject(const char *pszFilename);
625
626 virtual int *DeleteObjectBatch(CSLConstList papszFilesOrDirs);
627
628 bool Sync(const char *pszSource, const char *pszTarget,
629 const char *const *papszOptions, GDALProgressFunc pProgressFunc,
630 void *pProgressData, char ***ppapszOutputs) override;
631
632 VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
633 const char *const *papszOptions) override;
634};
635
636/************************************************************************/
637/* IVSIS3LikeFSHandlerWithMultipartUpload */
638/************************************************************************/
639
640class IVSIS3LikeFSHandlerWithMultipartUpload : public IVSIS3LikeFSHandler
641{
642 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandlerWithMultipartUpload)
643
644 protected:
645 IVSIS3LikeFSHandlerWithMultipartUpload() = default;
646
647 public:
648 virtual bool SupportsNonSequentialMultipartUpload() const
649 {
650 return true;
651 }
652
653 virtual bool SupportsParallelMultipartUpload() const
654 {
655 return true;
656 }
657
658 virtual bool SupportsMultipartAbort() const = 0;
659
660 size_t GetUploadChunkSizeInBytes(const char *pszFilename,
661 const char *pszSpecifiedValInBytes);
662
663 virtual int CopyFileRestartable(const char *pszSource,
664 const char *pszTarget,
665 const char *pszInputPayload,
666 char **ppszOutputPayload,
667 CSLConstList papszOptions,
668 GDALProgressFunc pProgressFunc,
669 void *pProgressData) override;
670
672 // Limit currently used by S3 and GS.
673 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
674 // and https://cloud.google.com/storage/quotas#requests
675 virtual int GetMaximumPartCount()
676 {
677 return 10000;
678 }
679
681 // Limit currently used by S3 and GS.
682 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
683 // and https://cloud.google.com/storage/quotas#requests
684 virtual int GetMinimumPartSizeInMiB()
685 {
686 return 5;
687 }
688
690 // Limit currently used by S3 and GS.
691 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
692 // and https://cloud.google.com/storage/quotas#requests
693 virtual int GetMaximumPartSizeInMiB()
694 {
695#if SIZEOF_VOIDP == 8
696 return 5 * 1024;
697#else
698 // Cannot be larger than 4, otherwise integer overflow would occur
699 // 1 GiB is the maximum reasonable value on a 32-bit machine
700 return 1 * 1024;
701#endif
702 }
703
705 virtual int GetDefaultPartSizeInMiB()
706 {
707 return 50;
708 }
709
710 virtual std::string
711 InitiateMultipartUpload(const std::string &osFilename,
712 IVSIS3LikeHandleHelper *poS3HandleHelper,
713 const CPLHTTPRetryParameters &oRetryParameters,
714 CSLConstList papszOptions);
715
716 virtual std::string
717 UploadPart(const std::string &osFilename, int nPartNumber,
718 const std::string &osUploadID, vsi_l_offset nPosition,
719 const void *pabyBuffer, size_t nBufferSize,
720 IVSIS3LikeHandleHelper *poS3HandleHelper,
721 const CPLHTTPRetryParameters &oRetryParameters,
722 CSLConstList papszOptions);
723
724 virtual bool CompleteMultipart(
725 const std::string &osFilename, const std::string &osUploadID,
726 const std::vector<std::string> &aosEtags, vsi_l_offset nTotalSize,
727 IVSIS3LikeHandleHelper *poS3HandleHelper,
728 const CPLHTTPRetryParameters &oRetryParameters);
729
730 virtual bool AbortMultipart(const std::string &osFilename,
731 const std::string &osUploadID,
732 IVSIS3LikeHandleHelper *poS3HandleHelper,
733 const CPLHTTPRetryParameters &oRetryParameters);
734
735 bool AbortPendingUploads(const char *pszFilename) override;
736
737 bool MultipartUploadGetCapabilities(int *pbNonSequentialUploadSupported,
738 int *pbParallelUploadSupported,
739 int *pbAbortSupported,
740 size_t *pnMinPartSize,
741 size_t *pnMaxPartSize,
742 int *pnMaxPartCount) override;
743
744 char *MultipartUploadStart(const char *pszFilename,
745 CSLConstList papszOptions) override;
746
747 char *MultipartUploadAddPart(const char *pszFilename,
748 const char *pszUploadId, int nPartNumber,
749 vsi_l_offset nFileOffset, const void *pData,
750 size_t nDataLength,
751 CSLConstList papszOptions) override;
752
753 bool MultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
754 size_t nPartIdsCount,
755 const char *const *apszPartIds,
756 vsi_l_offset nTotalSize,
757 CSLConstList papszOptions) override;
758
759 bool MultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
760 CSLConstList papszOptions) override;
761};
762
763/************************************************************************/
764/* IVSIS3LikeHandle */
765/************************************************************************/
766
767class IVSIS3LikeHandle : public VSICurlHandle
768{
769 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
770
771 protected:
772 bool UseLimitRangeGetInsteadOfHead() override
773 {
774 return true;
775 }
776
777 bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
778 {
779 // A bit dirty, but on S3, a GET on a existing directory returns a 416
780 return response_code == 416 && EQUAL(pszVerb, "GET") &&
781 std::string(m_pszURL).back() == '/';
782 }
783
784 void ProcessGetFileSizeResult(const char *pszContent) override
785 {
786 oFileProp.bIsDirectory =
787 strstr(pszContent, "ListBucketResult") != nullptr;
788 }
789
790 public:
791 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
792 const char *pszFilename, const char *pszURLIn)
793 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
794 {
795 }
796
797 ~IVSIS3LikeHandle() override
798 {
799 }
800};
801
802/************************************************************************/
803/* VSIMultipartWriteHandle */
804/************************************************************************/
805
806class VSIMultipartWriteHandle final : public VSIVirtualHandle
807{
808 CPL_DISALLOW_COPY_ASSIGN(VSIMultipartWriteHandle)
809
810 IVSIS3LikeFSHandlerWithMultipartUpload *m_poFS = nullptr;
811 std::string m_osFilename{};
812 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
813 CPLStringList m_aosOptions{};
814 CPLStringList m_aosHTTPOptions{};
815 CPLHTTPRetryParameters m_oRetryParameters;
816
817 vsi_l_offset m_nCurOffset = 0;
818 size_t m_nBufferOff = 0;
819 size_t m_nBufferSize = 0;
820 bool m_bClosed = false;
821 GByte *m_pabyBuffer = nullptr;
822 std::string m_osUploadID{};
823 int m_nPartNumber = 0;
824 std::vector<std::string> m_aosEtags{};
825 bool m_bError = false;
826
827 WriteFuncStruct m_sWriteFuncHeaderData{};
828
829 bool UploadPart();
830 bool DoSinglePartPUT();
831
832 void InvalidateParentDirectory();
833
834 public:
835 VSIMultipartWriteHandle(IVSIS3LikeFSHandlerWithMultipartUpload *poFS,
836 const char *pszFilename,
837 IVSIS3LikeHandleHelper *poS3HandleHelper,
838 CSLConstList papszOptions);
839 ~VSIMultipartWriteHandle() override;
840
841 int Seek(vsi_l_offset nOffset, int nWhence) override;
842 vsi_l_offset Tell() override;
843 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
844 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
845
846 void ClearErr() override
847 {
848 }
849
850 int Error() override
851 {
852 return FALSE;
853 }
854
855 int Eof() override
856 {
857 return FALSE;
858 }
859
860 int Close() override;
861
862 bool IsOK()
863 {
864 return m_pabyBuffer != nullptr;
865 }
866};
867
868/************************************************************************/
869/* VSIChunkedWriteHandle() */
870/************************************************************************/
871
875class VSIChunkedWriteHandle final : public VSIVirtualHandle
876{
877 CPL_DISALLOW_COPY_ASSIGN(VSIChunkedWriteHandle)
878
879 IVSIS3LikeFSHandler *m_poFS = nullptr;
880 std::string m_osFilename{};
881 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
882 CPLStringList m_aosOptions{};
883 CPLStringList m_aosHTTPOptions{};
884 CPLHTTPRetryParameters m_oRetryParameters;
885
886 vsi_l_offset m_nCurOffset = 0;
887 size_t m_nBufferOff = 0;
888 bool m_bError = false;
889 bool m_bClosed = false;
890
891 CURLM *m_hCurlMulti = nullptr;
892 CURL *m_hCurl = nullptr;
893 const void *m_pBuffer = nullptr;
894 std::string m_osCurlErrBuf{};
895 size_t m_nChunkedBufferOff = 0;
896 size_t m_nChunkedBufferSize = 0;
897 size_t m_nWrittenInPUT = 0;
898
899 WriteFuncStruct m_sWriteFuncHeaderData{};
900
901 static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
902 size_t nitems, void *instream);
903 int FinishChunkedTransfer();
904
905 bool DoEmptyPUT();
906
907 void InvalidateParentDirectory();
908
909 public:
910 VSIChunkedWriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
911 IVSIS3LikeHandleHelper *poS3HandleHelper,
912 CSLConstList papszOptions);
913 virtual ~VSIChunkedWriteHandle();
914
915 int Seek(vsi_l_offset nOffset, int nWhence) override;
916 vsi_l_offset Tell() override;
917 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
918 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
919
920 void ClearErr() override
921 {
922 }
923
924 int Error() override
925 {
926 return FALSE;
927 }
928
929 int Eof() override
930 {
931 return FALSE;
932 }
933
934 int Close() override;
935};
936
937/************************************************************************/
938/* VSIAppendWriteHandle */
939/************************************************************************/
940
941class VSIAppendWriteHandle CPL_NON_FINAL : public VSIVirtualHandle
942{
943 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
944
945 protected:
946 VSICurlFilesystemHandlerBase *m_poFS = nullptr;
947 std::string m_osFSPrefix{};
948 std::string m_osFilename{};
949 CPLHTTPRetryParameters m_oRetryParameters{};
950
951 vsi_l_offset m_nCurOffset = 0;
952 int m_nBufferOff = 0;
953 int m_nBufferSize = 0;
954 int m_nBufferOffReadCallback = 0;
955 bool m_bClosed = false;
956 GByte *m_pabyBuffer = nullptr;
957 bool m_bError = false;
958
959 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
960 void *instream);
961 virtual bool Send(bool bIsLastBlock) = 0;
962
963 public:
964 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
965 const char *pszFSPrefix, const char *pszFilename,
966 int nChunkSize);
967 virtual ~VSIAppendWriteHandle();
968
969 int Seek(vsi_l_offset nOffset, int nWhence) override;
970 vsi_l_offset Tell() override;
971 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
972 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
973
974 void ClearErr() override
975 {
976 }
977
978 int Error() override
979 {
980 return FALSE;
981 }
982
983 int Eof() override
984 {
985 return FALSE;
986 }
987
988 int Close() override;
989
990 bool IsOK()
991 {
992 return m_pabyBuffer != nullptr;
993 }
994};
995
996/************************************************************************/
997/* VSIDIRWithMissingDirSynthesis */
998/************************************************************************/
999
1000struct VSIDIRWithMissingDirSynthesis : public VSIDIR
1001{
1002 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
1003
1004 protected:
1005 std::vector<std::string> m_aosSubpathsStack{};
1006
1007 void SynthetizeMissingDirectories(const std::string &osCurSubdir,
1008 bool bAddEntryForThisSubdir);
1009};
1010
1011/************************************************************************/
1012/* CurlRequestHelper */
1013/************************************************************************/
1014
1015struct CurlRequestHelper
1016{
1017 WriteFuncStruct sWriteFuncData{};
1018 WriteFuncStruct sWriteFuncHeaderData{};
1019 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1020
1021 CurlRequestHelper();
1022 ~CurlRequestHelper();
1023 long perform(CURL *hCurlHandle,
1024 struct curl_slist *headers, // ownership transferred
1025 VSICurlFilesystemHandlerBase *poFS,
1026 IVSIS3LikeHandleHelper *poS3HandleHelper);
1027};
1028
1029/************************************************************************/
1030/* NetworkStatisticsLogger */
1031/************************************************************************/
1032
1033class NetworkStatisticsLogger
1034{
1035 static int gnEnabled;
1036 static NetworkStatisticsLogger gInstance;
1037
1038 NetworkStatisticsLogger() = default;
1039
1040 std::mutex m_mutex{};
1041
1042 struct Counters
1043 {
1044 GIntBig nHEAD = 0;
1045 GIntBig nGET = 0;
1046 GIntBig nPUT = 0;
1047 GIntBig nPOST = 0;
1048 GIntBig nDELETE = 0;
1049 GIntBig nGETDownloadedBytes = 0;
1050 GIntBig nPUTUploadedBytes = 0;
1051 GIntBig nPOSTDownloadedBytes = 0;
1052 GIntBig nPOSTUploadedBytes = 0;
1053 };
1054
1055 enum class ContextPathType
1056 {
1057 FILESYSTEM,
1058 FILE,
1059 ACTION,
1060 };
1061
1062 struct ContextPathItem
1063 {
1064 ContextPathType eType;
1065 std::string osName;
1066
1067 ContextPathItem(ContextPathType eTypeIn, const std::string &osNameIn)
1068 : eType(eTypeIn), osName(osNameIn)
1069 {
1070 }
1071
1072 bool operator<(const ContextPathItem &other) const
1073 {
1074 if (static_cast<int>(eType) < static_cast<int>(other.eType))
1075 return true;
1076 if (static_cast<int>(eType) > static_cast<int>(other.eType))
1077 return false;
1078 return osName < other.osName;
1079 }
1080 };
1081
1082 struct Stats
1083 {
1084 Counters counters{};
1085 std::map<ContextPathItem, Stats> children{};
1086
1087 void AsJSON(CPLJSONObject &oJSON) const;
1088 };
1089
1090 // Workaround bug in Coverity Scan
1091 // coverity[generated_default_constructor_used_in_field_initializer]
1092 Stats m_stats{};
1093 std::map<GIntBig, std::vector<ContextPathItem>>
1094 m_mapThreadIdToContextPath{};
1095
1096 static void ReadEnabled();
1097
1098 std::vector<Counters *> GetCountersForContext();
1099
1100 public:
1101 static inline bool IsEnabled()
1102 {
1103 if (gnEnabled < 0)
1104 {
1105 ReadEnabled();
1106 }
1107 return gnEnabled == TRUE;
1108 }
1109
1110 static void EnterFileSystem(const char *pszName);
1111
1112 static void LeaveFileSystem();
1113
1114 static void EnterFile(const char *pszName);
1115
1116 static void LeaveFile();
1117
1118 static void EnterAction(const char *pszName);
1119
1120 static void LeaveAction();
1121
1122 static void LogHEAD();
1123
1124 static void LogGET(size_t nDownloadedBytes);
1125
1126 static void LogPUT(size_t nUploadedBytes);
1127
1128 static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
1129
1130 static void LogDELETE();
1131
1132 static void Reset();
1133
1134 static std::string GetReportAsSerializedJSON();
1135};
1136
1137struct NetworkStatisticsFileSystem
1138{
1139 inline explicit NetworkStatisticsFileSystem(const char *pszName)
1140 {
1141 NetworkStatisticsLogger::EnterFileSystem(pszName);
1142 }
1143
1144 inline ~NetworkStatisticsFileSystem()
1145 {
1146 NetworkStatisticsLogger::LeaveFileSystem();
1147 }
1148};
1149
1150struct NetworkStatisticsFile
1151{
1152 inline explicit NetworkStatisticsFile(const char *pszName)
1153 {
1154 NetworkStatisticsLogger::EnterFile(pszName);
1155 }
1156
1157 inline ~NetworkStatisticsFile()
1158 {
1159 NetworkStatisticsLogger::LeaveFile();
1160 }
1161};
1162
1163struct NetworkStatisticsAction
1164{
1165 inline explicit NetworkStatisticsAction(const char *pszName)
1166 {
1167 NetworkStatisticsLogger::EnterAction(pszName);
1168 }
1169
1170 inline ~NetworkStatisticsAction()
1171 {
1172 NetworkStatisticsLogger::LeaveAction();
1173 }
1174};
1175
1176} // namespace cpl
1177
1178int VSICURLGetDownloadChunkSize();
1179
1180void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp,
1181 VSICurlReadCbkFunc pfnReadCbk,
1182 void *pReadCbkUserData);
1183size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
1184 void *req);
1185void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr,
1186 std::atomic<bool> *pbInterrupt = nullptr);
1187void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
1188
1189int VSICurlParseUnixPermissions(const char *pszPermissions);
1190
1191// Cache of file properties (size, etc.)
1192bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1193void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1194void VSICURLInvalidateCachedFileProp(const char *pszURL);
1195void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
1196void VSICURLDestroyCacheFileProp();
1197
1198void VSICURLMultiCleanup(CURLM *hCurlMultiHandle);
1199
1201
1202#endif // HAVE_CURL
1203
1204#endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:41
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:436
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define CPL_NON_FINAL
Mark that a class is explicitly recognized as non-final.
Definition: cpl_port.h:1023
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:535
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1030
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1179
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:169
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:199
Various convenience functions for working with strings and string lists.
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:189
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:132
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:388
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:130
Virtual file handle.
Definition: cpl_vsi_virtual.h:47