29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
45 #include <condition_variable>
60 #define HAVE_CURLINFO_REDIRECT_URL
62 void VSICurlStreamingClearCache(
void);
64 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle,
const char *pszURL,
65 const char *
const *papszOptions);
66 struct curl_slist *VSICurlMergeHeaders(
struct curl_slist *poDest,
67 struct curl_slist *poSrcToDestroy);
69 struct curl_slist *VSICurlSetContentTypeFromExt(
struct curl_slist *polist,
72 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
73 struct curl_slist *headers,
CSLConstList papszOptions,
const char *pszPath);
88 unsigned int nGenerationAuthParameters = 0;
89 ExistStatus eExists = EXIST_UNKNOWN;
92 time_t nExpireTimestampLocal = 0;
94 bool bHasComputedFileSize =
false;
95 bool bIsDirectory =
false;
97 bool bS3LikeRedirect =
false;
103 bool bGotFileList =
false;
104 unsigned int nGenerationAuthParameters = 0;
108 struct WriteFuncStruct
110 char *pBuffer =
nullptr;
112 bool bIsHTTP =
false;
113 bool bMultiRange =
false;
118 bool bFoundContentRange =
false;
120 bool bInterruptDownload =
false;
121 bool bDetectRangeDownloadingError =
false;
125 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
126 void *pReadCbkUserData =
nullptr;
127 bool bInterrupted =
false;
129 #if !CURL_AT_LEAST_VERSION(7, 54, 0)
133 bool bIsProxyConnectHeader =
false;
139 const GByte *pabyData =
nullptr;
141 size_t nTotalSize = 0;
143 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
146 PutData *poThis =
static_cast<PutData *
>(instream);
147 const size_t nSizeMax = size * nitems;
148 const size_t nSizeToWrite =
149 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
150 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
151 poThis->nOff += nSizeToWrite;
162 class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
166 struct FilenameOffsetPair
168 std::string filename_;
171 FilenameOffsetPair(
const std::string &filename,
vsi_l_offset offset)
172 : filename_(filename), offset_(offset)
176 bool operator==(
const FilenameOffsetPair &other)
const
178 return filename_ == other.filename_ && offset_ == other.offset_;
181 struct FilenameOffsetPairHasher
183 std::size_t operator()(
const FilenameOffsetPair &k)
const
185 return std::hash<std::string>()(k.filename_) ^
186 std::hash<vsi_l_offset>()(k.offset_);
190 using RegionCacheType = lru11::Cache<
191 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
194 typename std::list<lru11::KeyValuePair<
195 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
196 FilenameOffsetPairHasher>>;
198 std::unique_ptr<RegionCacheType>
199 m_poRegionCacheDoNotUseDirectly{};
201 RegionCacheType *GetRegionCache();
208 lru11::Cache<std::string, bool> oCacheFileProp;
210 int nCachedFilesInDirList = 0;
211 lru11::Cache<std::string, CachedDirList> oCacheDirList;
213 char **ParseHTMLFileList(
const char *pszFilename,
int nMaxFiles,
214 char *pszData,
bool *pbGotFileList);
217 CPLMutex *hMutex =
nullptr;
219 virtual VSICurlHandle *CreateFileHandle(
const char *pszFilename);
220 virtual char **GetFileList(
const char *pszFilename,
int nMaxFiles,
221 bool *pbGotFileList);
223 void RegisterEmptyDir(
const CPLString &osDirname);
226 AnalyseS3FileList(
const CPLString &osBaseURL,
const char *pszXML,
228 const std::set<std::string> &oSetIgnoredStorageClasses,
231 void AnalyseSwiftFileList(
const CPLString &osBaseURL,
232 const CPLString &osPrefix,
const char *pszJson,
234 int nMaxFiles,
bool &bIsTruncated,
237 static const char *GetOptionsStatic();
239 static bool IsAllowedFilename(
const char *pszFilename);
241 VSICurlFilesystemHandlerBase();
244 ~VSICurlFilesystemHandlerBase()
override;
250 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
251 int nFlags)
override;
252 int Unlink(
const char *pszFilename)
override;
253 int Rename(
const char *oldpath,
const char *newpath)
override;
254 int Mkdir(
const char *pszDirname,
long nMode)
override;
255 int Rmdir(
const char *pszDirname)
override;
256 char **ReadDir(
const char *pszDirname)
override
258 return ReadDirEx(pszDirname, 0);
260 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles)
override;
261 char **SiblingFiles(
const char *pszFilename)
override;
263 int HasOptimizedReadMultiRange(
const char * )
override
268 const char *GetActualURL(
const char *pszFilename)
override;
270 const char *GetOptions()
override;
272 char **GetFileMetadata(
const char *pszFilename,
const char *pszDomain,
275 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
276 bool *pbGotFileList);
277 void InvalidateDirContent(
const char *pszDirname);
279 virtual const char *GetDebugKey()
const = 0;
281 virtual CPLString GetFSPrefix()
const = 0;
282 virtual bool AllowCachedDataFor(
const char *pszFilename);
284 virtual bool IsLocal(
const char * )
override
289 SupportsSequentialWrite(
const char * ,
294 virtual bool SupportsRandomWrite(
const char * ,
300 std::shared_ptr<std::string> GetRegion(
const char *pszURL,
303 void AddRegion(
const char *pszURL,
vsi_l_offset nFileOffsetStart,
304 size_t nSize,
const char *pData);
306 bool GetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
307 void SetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
308 void InvalidateCachedData(
const char *pszURL);
310 CURLM *GetCurlMultiHandleFor(
const CPLString &osURL);
312 virtual void ClearCache();
313 virtual void PartialClearCache(
const char *pszFilename);
315 bool GetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
316 void SetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
317 bool ExistsInCacheDirList(
const CPLString &osDirname,
bool *pbIsDir);
322 GetStreamingFilename(
const std::string &osFilename)
const override = 0;
324 static std::set<std::string> GetS3IgnoredStorageClasses();
327 class VSICurlFilesystemHandler :
public VSICurlFilesystemHandlerBase
332 VSICurlFilesystemHandler() =
default;
334 const char *GetDebugKey()
const override
345 GetStreamingFilename(
const std::string &osFilename)
const override;
357 VSICurlFilesystemHandlerBase *poFS =
nullptr;
359 bool m_bCached =
true;
361 mutable FileProp oFileProp{};
363 mutable std::mutex m_oMutex{};
365 char *m_pszURL =
nullptr;
366 mutable std::string m_osQueryString{};
368 char **m_papszHTTPOptions =
nullptr;
371 int nBlocksToDownload = 1;
373 bool bStopOnInterruptUntilUninstall =
false;
374 bool bInterrupted =
false;
375 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
376 void *pReadCbkUserData =
nullptr;
379 double m_dfRetryDelay = 0.0;
383 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
384 const int nBlocks,
const char *pBuffer,
392 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
394 bool m_bUseHead =
false;
395 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
399 mutable bool m_bPlanetaryComputerURLSigning =
false;
400 mutable std::string m_osPlanetaryComputerCollection{};
401 void ManagePlanetaryComputerSigning()
const;
403 int ReadMultiRangeSingleGet(
int nRanges,
void **ppData,
405 const size_t *panSizes);
406 CPLString GetRedirectURLIfValid(
bool &bHasExpired)
const;
408 void UpdateRedirectInfo(CURL *hCurlHandle,
409 const WriteFuncStruct &sWriteFuncHeaderData);
412 struct AdviseReadRange
416 std::condition_variable oCV{};
419 std::vector<GByte> abyData{};
421 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
422 std::thread m_oThreadAdviseRead{};
425 virtual struct curl_slist *
427 const struct curl_slist * )
431 virtual bool AllowAutomaticRedirection()
435 virtual bool CanRestartOnError(
const char *,
const char *,
bool)
439 virtual bool UseLimitRangeGetInsteadOfHead()
443 virtual bool IsDirectoryFromExists(
const char * ,
448 virtual void ProcessGetFileSizeResult(
const char * )
451 void SetURL(
const char *pszURL);
452 virtual bool Authenticate(
const char * )
458 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS,
const char *pszFilename,
459 const char *pszURLIn =
nullptr);
460 ~VSICurlHandle()
override;
464 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
465 int ReadMultiRange(
int nRanges,
void **ppData,
467 const size_t *panSizes)
override;
468 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
470 int Flush()
override;
471 int Close()
override;
473 bool HasPRead()
const override
477 size_t PRead(
void *pBuffer,
size_t nSize,
480 void AdviseRead(
int nRanges,
const vsi_l_offset *panOffsets,
481 const size_t *panSizes)
override;
483 bool IsKnownFileSize()
const
485 return oFileProp.bHasComputedFileSize;
487 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
490 return GetFileSizeOrHeaders(bSetError,
false);
492 bool Exists(
bool bSetError);
493 bool IsDirectory()
const
495 return oFileProp.bIsDirectory;
499 return oFileProp.nMode;
501 time_t GetMTime()
const
503 return oFileProp.mTime;
510 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk,
void *pfnUserData,
511 int bStopOnInterruptUntilUninstall);
512 int UninstallReadCbk();
514 const char *GetURL()
const
524 class IVSIS3LikeFSHandler :
public VSICurlFilesystemHandlerBase
528 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
532 char **GetFileList(
const char *pszFilename,
int nMaxFiles,
533 bool *pbGotFileList)
override;
535 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(
const char *pszURI,
536 bool bAllowNoObject) = 0;
538 virtual int CopyObject(
const char *oldpath,
const char *newpath,
541 int RmdirRecursiveInternal(
const char *pszDirname,
int nBatchSize);
544 IsAllowedHeaderForObjectCreation(
const char * )
549 IVSIS3LikeFSHandler() =
default;
552 int Unlink(
const char *pszFilename)
override;
553 int Mkdir(
const char *pszDirname,
long nMode)
override;
554 int Rmdir(
const char *pszDirname)
override;
555 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
556 int nFlags)
override;
557 int Rename(
const char *oldpath,
const char *newpath)
override;
559 virtual int CopyFile(
const char *pszSource,
const char *pszTarget,
561 const char *
const *papszOptions,
562 GDALProgressFunc pProgressFunc,
563 void *pProgressData)
override;
565 virtual int DeleteObject(
const char *pszFilename);
567 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper *)
570 virtual void UpdateHandleFromMap(IVSIS3LikeHandleHelper *)
574 bool Sync(
const char *pszSource,
const char *pszTarget,
575 const char *
const *papszOptions, GDALProgressFunc pProgressFunc,
576 void *pProgressData,
char ***ppapszOutputs)
override;
578 VSIDIR *OpenDir(
const char *pszPath,
int nRecurseDepth,
579 const char *
const *papszOptions)
override;
582 virtual bool SupportsParallelMultipartUpload()
const
587 virtual CPLString InitiateMultipartUpload(
588 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
589 int nMaxRetry,
double dfRetryDelay,
CSLConstList papszOptions);
591 const std::string &osUploadID,
594 IVSIS3LikeHandleHelper *poS3HandleHelper,
595 int nMaxRetry,
double dfRetryDelay,
597 virtual bool CompleteMultipart(
const CPLString &osFilename,
599 const std::vector<CPLString> &aosEtags,
601 IVSIS3LikeHandleHelper *poS3HandleHelper,
602 int nMaxRetry,
double dfRetryDelay);
603 virtual bool AbortMultipart(
const CPLString &osFilename,
605 IVSIS3LikeHandleHelper *poS3HandleHelper,
606 int nMaxRetry,
double dfRetryDelay);
608 bool AbortPendingUploads(
const char *pszFilename)
override;
615 class IVSIS3LikeHandle :
public VSICurlHandle
620 bool UseLimitRangeGetInsteadOfHead()
override
624 bool IsDirectoryFromExists(
const char *pszVerb,
int response_code)
override
627 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
630 void ProcessGetFileSizeResult(
const char *pszContent)
override
632 oFileProp.bIsDirectory =
633 strstr(pszContent,
"ListBucketResult") !=
nullptr;
637 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
638 const char *pszFilename,
const char *pszURLIn)
639 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
642 ~IVSIS3LikeHandle()
override
655 IVSIS3LikeFSHandler *m_poFS =
nullptr;
657 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
658 bool m_bUseChunked =
false;
663 int m_nBufferOff = 0;
664 int m_nBufferSize = 0;
665 bool m_bClosed =
false;
666 GByte *m_pabyBuffer =
nullptr;
668 int m_nPartNumber = 0;
669 std::vector<CPLString> m_aosEtags{};
670 bool m_bError =
false;
672 CURLM *m_hCurlMulti =
nullptr;
673 CURL *m_hCurl =
nullptr;
674 const void *m_pBuffer =
nullptr;
676 size_t m_nChunkedBufferOff = 0;
677 size_t m_nChunkedBufferSize = 0;
678 size_t m_nWrittenInPUT = 0;
681 double m_dfRetryDelay = 0.0;
682 WriteFuncStruct m_sWriteFuncHeaderData{};
685 bool DoSinglePartPUT();
687 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
688 size_t nitems,
void *instream);
689 size_t WriteChunked(
const void *pBuffer,
size_t nSize,
size_t nMemb);
690 int FinishChunkedTransfer();
692 void InvalidateParentDirectory();
695 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS,
const char *pszFilename,
696 IVSIS3LikeHandleHelper *poS3HandleHelper,
bool bUseChunked,
698 ~VSIS3WriteHandle()
override;
702 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
703 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
705 int Close()
override;
709 return m_bUseChunked || m_pabyBuffer !=
nullptr;
722 VSICurlFilesystemHandlerBase *m_poFS =
nullptr;
727 int m_nBufferOff = 0;
728 int m_nBufferSize = 0;
729 int m_nBufferOffReadCallback = 0;
730 bool m_bClosed =
false;
731 GByte *m_pabyBuffer =
nullptr;
732 bool m_bError =
false;
734 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
736 virtual bool Send(
bool bIsLastBlock) = 0;
739 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
740 const char *pszFSPrefix,
const char *pszFilename,
742 virtual ~VSIAppendWriteHandle();
746 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
747 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
749 int Close()
override;
753 return m_pabyBuffer !=
nullptr;
761 struct VSIDIRWithMissingDirSynthesis :
public VSIDIR
763 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
766 std::vector<std::string> m_aosSubpathsStack{};
768 void SynthetizeMissingDirectories(
const std::string &osCurSubdir,
769 bool bAddEntryForThisSubdir);
776 struct CurlRequestHelper
778 WriteFuncStruct sWriteFuncData{};
779 WriteFuncStruct sWriteFuncHeaderData{};
780 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
783 ~CurlRequestHelper();
784 long perform(CURL *hCurlHandle,
785 struct curl_slist *headers,
786 VSICurlFilesystemHandlerBase *poFS,
787 IVSIS3LikeHandleHelper *poS3HandleHelper);
794 class NetworkStatisticsLogger
796 static int gnEnabled;
797 static NetworkStatisticsLogger gInstance;
799 NetworkStatisticsLogger() =
default;
801 std::mutex m_mutex{};
810 GIntBig nGETDownloadedBytes = 0;
812 GIntBig nPOSTDownloadedBytes = 0;
813 GIntBig nPOSTUploadedBytes = 0;
816 enum class ContextPathType
823 struct ContextPathItem
825 ContextPathType eType;
828 ContextPathItem(ContextPathType eTypeIn,
const CPLString &osNameIn)
829 : eType(eTypeIn), osName(osNameIn)
833 bool operator<(
const ContextPathItem &other)
const
835 if (
static_cast<int>(eType) <
static_cast<int>(other.eType))
837 if (
static_cast<int>(eType) >
static_cast<int>(other.eType))
839 return osName < other.osName;
846 std::map<ContextPathItem, Stats> children{};
854 std::map<GIntBig, std::vector<ContextPathItem>>
855 m_mapThreadIdToContextPath{};
857 static void ReadEnabled();
859 std::vector<Counters *> GetCountersForContext();
862 static inline bool IsEnabled()
868 return gnEnabled == TRUE;
871 static void EnterFileSystem(
const char *pszName);
873 static void LeaveFileSystem();
875 static void EnterFile(
const char *pszName);
877 static void LeaveFile();
879 static void EnterAction(
const char *pszName);
881 static void LeaveAction();
883 static void LogHEAD();
885 static void LogGET(
size_t nDownloadedBytes);
887 static void LogPUT(
size_t nUploadedBytes);
889 static void LogPOST(
size_t nUploadedBytes,
size_t nDownloadedBytes);
891 static void LogDELETE();
895 static CPLString GetReportAsSerializedJSON();
898 struct NetworkStatisticsFileSystem
900 inline explicit NetworkStatisticsFileSystem(
const char *pszName)
902 NetworkStatisticsLogger::EnterFileSystem(pszName);
905 inline ~NetworkStatisticsFileSystem()
907 NetworkStatisticsLogger::LeaveFileSystem();
911 struct NetworkStatisticsFile
913 inline explicit NetworkStatisticsFile(
const char *pszName)
915 NetworkStatisticsLogger::EnterFile(pszName);
918 inline ~NetworkStatisticsFile()
920 NetworkStatisticsLogger::LeaveFile();
924 struct NetworkStatisticsAction
926 inline explicit NetworkStatisticsAction(
const char *pszName)
928 NetworkStatisticsLogger::EnterAction(pszName);
931 inline ~NetworkStatisticsAction()
933 NetworkStatisticsLogger::LeaveAction();
937 int VSICURLGetDownloadChunkSize();
939 void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct,
VSILFILE *fp,
940 VSICurlReadCbkFunc pfnReadCbk,
941 void *pReadCbkUserData);
942 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
size_t nmemb,
944 void MultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle =
nullptr);
945 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
947 int VSICurlParseUnixPermissions(
const char *pszPermissions);
950 bool VSICURLGetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
951 void VSICURLSetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
952 void VSICURLInvalidateCachedFileProp(
const char *pszURL);
953 void VSICURLInvalidateCachedFilePropPrefix(
const char *pszURL);
954 void VSICURLDestroyCacheFileProp();
962 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED