29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
45 #include <condition_variable>
60 #define HAVE_CURLINFO_REDIRECT_URL
62 void VSICurlStreamingClearCache(
void);
64 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle,
const char *pszURL,
65 const char *
const *papszOptions);
66 struct curl_slist *VSICurlMergeHeaders(
struct curl_slist *poDest,
67 struct curl_slist *poSrcToDestroy);
69 struct curl_slist *VSICurlSetContentTypeFromExt(
struct curl_slist *polist,
72 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
73 struct curl_slist *headers,
CSLConstList papszOptions,
const char *pszPath);
88 unsigned int nGenerationAuthParameters = 0;
89 ExistStatus eExists = EXIST_UNKNOWN;
92 time_t nExpireTimestampLocal = 0;
94 bool bHasComputedFileSize =
false;
95 bool bIsDirectory =
false;
97 bool bS3LikeRedirect =
false;
103 bool bGotFileList =
false;
104 unsigned int nGenerationAuthParameters = 0;
108 struct WriteFuncStruct
110 char *pBuffer =
nullptr;
112 bool bIsHTTP =
false;
113 bool bMultiRange =
false;
118 bool bFoundContentRange =
false;
120 bool bInterruptDownload =
false;
121 bool bDetectRangeDownloadingError =
false;
125 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
126 void *pReadCbkUserData =
nullptr;
127 bool bInterrupted =
false;
129 #if !CURL_AT_LEAST_VERSION(7, 54, 0)
133 bool bIsProxyConnectHeader =
false;
139 const GByte *pabyData =
nullptr;
141 size_t nTotalSize = 0;
143 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
146 PutData *poThis =
static_cast<PutData *
>(instream);
147 const size_t nSizeMax = size * nitems;
148 const size_t nSizeToWrite =
149 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
150 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
151 poThis->nOff += nSizeToWrite;
162 class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
166 struct FilenameOffsetPair
168 std::string filename_;
171 FilenameOffsetPair(
const std::string &filename,
vsi_l_offset offset)
172 : filename_(filename), offset_(offset)
176 bool operator==(
const FilenameOffsetPair &other)
const
178 return filename_ == other.filename_ && offset_ == other.offset_;
181 struct FilenameOffsetPairHasher
183 std::size_t operator()(
const FilenameOffsetPair &k)
const
185 return std::hash<std::string>()(k.filename_) ^
186 std::hash<vsi_l_offset>()(k.offset_);
190 using RegionCacheType = lru11::Cache<
191 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
194 typename std::list<lru11::KeyValuePair<
195 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
196 FilenameOffsetPairHasher>>;
198 std::unique_ptr<RegionCacheType>
199 m_poRegionCacheDoNotUseDirectly{};
201 RegionCacheType *GetRegionCache();
208 lru11::Cache<std::string, bool> oCacheFileProp;
210 int nCachedFilesInDirList = 0;
211 lru11::Cache<std::string, CachedDirList> oCacheDirList;
213 char **ParseHTMLFileList(
const char *pszFilename,
int nMaxFiles,
214 char *pszData,
bool *pbGotFileList);
219 struct RegionInDownload
222 std::condition_variable oCond{};
223 bool bDownloadInProgress =
false;
225 std::string osData{};
227 std::mutex m_oMutex{};
228 std::map<std::string, std::unique_ptr<RegionInDownload>>
229 m_oMapRegionInDownload{};
232 CPLMutex *hMutex =
nullptr;
234 virtual VSICurlHandle *CreateFileHandle(
const char *pszFilename);
235 virtual char **GetFileList(
const char *pszFilename,
int nMaxFiles,
236 bool *pbGotFileList);
238 void RegisterEmptyDir(
const CPLString &osDirname);
241 AnalyseS3FileList(
const CPLString &osBaseURL,
const char *pszXML,
243 const std::set<std::string> &oSetIgnoredStorageClasses,
246 void AnalyseSwiftFileList(
const CPLString &osBaseURL,
247 const CPLString &osPrefix,
const char *pszJson,
249 int nMaxFiles,
bool &bIsTruncated,
252 static const char *GetOptionsStatic();
254 VSICurlFilesystemHandlerBase();
257 ~VSICurlFilesystemHandlerBase()
override;
259 static bool IsAllowedFilename(
const char *pszFilename);
265 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
266 int nFlags)
override;
267 int Unlink(
const char *pszFilename)
override;
268 int Rename(
const char *oldpath,
const char *newpath)
override;
269 int Mkdir(
const char *pszDirname,
long nMode)
override;
270 int Rmdir(
const char *pszDirname)
override;
271 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles)
override;
272 char **SiblingFiles(
const char *pszFilename)
override;
274 int HasOptimizedReadMultiRange(
const char * )
override
279 const char *GetActualURL(
const char *pszFilename)
override;
281 const char *GetOptions()
override;
283 char **GetFileMetadata(
const char *pszFilename,
const char *pszDomain,
286 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
287 bool *pbGotFileList);
288 void InvalidateDirContent(
const char *pszDirname);
290 virtual const char *GetDebugKey()
const = 0;
292 virtual CPLString GetFSPrefix()
const = 0;
293 virtual bool AllowCachedDataFor(
const char *pszFilename);
295 virtual bool IsLocal(
const char * )
override
300 SupportsSequentialWrite(
const char * ,
305 virtual bool SupportsRandomWrite(
const char * ,
311 std::shared_ptr<std::string> GetRegion(
const char *pszURL,
314 void AddRegion(
const char *pszURL,
vsi_l_offset nFileOffsetStart,
315 size_t nSize,
const char *pData);
317 std::string NotifyStartDownloadRegion(
const std::string &osURL,
320 void NotifyStopDownloadRegion(
const std::string &osURL,
322 const std::string &osData);
324 bool GetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
325 void SetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
326 void InvalidateCachedData(
const char *pszURL);
328 CURLM *GetCurlMultiHandleFor(
const CPLString &osURL);
330 virtual void ClearCache();
331 virtual void PartialClearCache(
const char *pszFilename);
333 bool GetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
334 void SetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
335 bool ExistsInCacheDirList(
const CPLString &osDirname,
bool *pbIsDir);
340 GetStreamingFilename(
const std::string &osFilename)
const override = 0;
342 static std::set<std::string> GetS3IgnoredStorageClasses();
345 class VSICurlFilesystemHandler :
public VSICurlFilesystemHandlerBase
350 VSICurlFilesystemHandler() =
default;
352 const char *GetDebugKey()
const override
363 GetStreamingFilename(
const std::string &osFilename)
const override;
375 VSICurlFilesystemHandlerBase *poFS =
nullptr;
377 bool m_bCached =
true;
379 mutable FileProp oFileProp{};
381 mutable std::mutex m_oMutex{};
383 char *m_pszURL =
nullptr;
384 mutable std::string m_osQueryString{};
386 char **m_papszHTTPOptions =
nullptr;
389 int nBlocksToDownload = 1;
391 bool bStopOnInterruptUntilUninstall =
false;
392 bool bInterrupted =
false;
393 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
394 void *pReadCbkUserData =
nullptr;
397 double m_dfRetryDelay = 0.0;
401 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
402 const int nBlocks,
const char *pBuffer,
410 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
412 bool m_bUseHead =
false;
413 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
417 mutable bool m_bPlanetaryComputerURLSigning =
false;
418 mutable std::string m_osPlanetaryComputerCollection{};
419 void ManagePlanetaryComputerSigning()
const;
421 int ReadMultiRangeSingleGet(
int nRanges,
void **ppData,
423 const size_t *panSizes);
424 CPLString GetRedirectURLIfValid(
bool &bHasExpired)
const;
426 void UpdateRedirectInfo(CURL *hCurlHandle,
427 const WriteFuncStruct &sWriteFuncHeaderData);
430 struct AdviseReadRange
434 std::condition_variable oCV{};
437 std::vector<GByte> abyData{};
439 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
440 std::thread m_oThreadAdviseRead{};
443 virtual struct curl_slist *
445 const struct curl_slist * )
449 virtual bool AllowAutomaticRedirection()
453 virtual bool CanRestartOnError(
const char *,
const char *,
bool)
457 virtual bool UseLimitRangeGetInsteadOfHead()
461 virtual bool IsDirectoryFromExists(
const char * ,
466 virtual void ProcessGetFileSizeResult(
const char * )
469 void SetURL(
const char *pszURL);
470 virtual bool Authenticate(
const char * )
476 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS,
const char *pszFilename,
477 const char *pszURLIn =
nullptr);
478 ~VSICurlHandle()
override;
482 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
483 int ReadMultiRange(
int nRanges,
void **ppData,
485 const size_t *panSizes)
override;
486 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
488 int Flush()
override;
489 int Close()
override;
491 bool HasPRead()
const override
495 size_t PRead(
void *pBuffer,
size_t nSize,
498 void AdviseRead(
int nRanges,
const vsi_l_offset *panOffsets,
499 const size_t *panSizes)
override;
501 bool IsKnownFileSize()
const
503 return oFileProp.bHasComputedFileSize;
505 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
508 return GetFileSizeOrHeaders(bSetError,
false);
510 bool Exists(
bool bSetError);
511 bool IsDirectory()
const
513 return oFileProp.bIsDirectory;
517 return oFileProp.nMode;
519 time_t GetMTime()
const
521 return oFileProp.mTime;
528 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk,
void *pfnUserData,
529 int bStopOnInterruptUntilUninstall);
530 int UninstallReadCbk();
532 const char *GetURL()
const
542 class VSICurlFilesystemHandlerBaseWritable :
public VSICurlFilesystemHandlerBase
547 VSICurlFilesystemHandlerBaseWritable() =
default;
549 virtual VSIVirtualHandleUniquePtr
550 CreateWriteHandle(
const char *pszFilename,
CSLConstList papszOptions) = 0;
556 bool SupportsSequentialWrite(
const char * ,
561 bool SupportsRandomWrite(
const char * ,
569 class IVSIS3LikeFSHandler :
public VSICurlFilesystemHandlerBaseWritable
573 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
577 char **GetFileList(
const char *pszFilename,
int nMaxFiles,
578 bool *pbGotFileList)
override;
580 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(
const char *pszURI,
581 bool bAllowNoObject) = 0;
583 virtual int CopyObject(
const char *oldpath,
const char *newpath,
586 int RmdirRecursiveInternal(
const char *pszDirname,
int nBatchSize);
589 IsAllowedHeaderForObjectCreation(
const char * )
594 IVSIS3LikeFSHandler() =
default;
597 int Unlink(
const char *pszFilename)
override;
598 int Mkdir(
const char *pszDirname,
long nMode)
override;
599 int Rmdir(
const char *pszDirname)
override;
600 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
601 int nFlags)
override;
602 int Rename(
const char *oldpath,
const char *newpath)
override;
604 virtual int CopyFile(
const char *pszSource,
const char *pszTarget,
606 const char *
const *papszOptions,
607 GDALProgressFunc pProgressFunc,
608 void *pProgressData)
override;
610 virtual int DeleteObject(
const char *pszFilename);
612 bool Sync(
const char *pszSource,
const char *pszTarget,
613 const char *
const *papszOptions, GDALProgressFunc pProgressFunc,
614 void *pProgressData,
char ***ppapszOutputs)
override;
616 VSIDIR *OpenDir(
const char *pszPath,
int nRecurseDepth,
617 const char *
const *papszOptions)
override;
620 virtual bool SupportsParallelMultipartUpload()
const
625 virtual CPLString InitiateMultipartUpload(
626 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
627 int nMaxRetry,
double dfRetryDelay,
CSLConstList papszOptions);
629 const std::string &osUploadID,
632 IVSIS3LikeHandleHelper *poS3HandleHelper,
633 int nMaxRetry,
double dfRetryDelay,
635 virtual bool CompleteMultipart(
const CPLString &osFilename,
637 const std::vector<CPLString> &aosEtags,
639 IVSIS3LikeHandleHelper *poS3HandleHelper,
640 int nMaxRetry,
double dfRetryDelay);
641 virtual bool AbortMultipart(
const CPLString &osFilename,
643 IVSIS3LikeHandleHelper *poS3HandleHelper,
644 int nMaxRetry,
double dfRetryDelay);
646 bool AbortPendingUploads(
const char *pszFilename)
override;
653 class IVSIS3LikeHandle :
public VSICurlHandle
658 bool UseLimitRangeGetInsteadOfHead()
override
662 bool IsDirectoryFromExists(
const char *pszVerb,
int response_code)
override
665 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
668 void ProcessGetFileSizeResult(
const char *pszContent)
override
670 oFileProp.bIsDirectory =
671 strstr(pszContent,
"ListBucketResult") !=
nullptr;
675 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
676 const char *pszFilename,
const char *pszURLIn)
677 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
680 ~IVSIS3LikeHandle()
override
693 IVSIS3LikeFSHandler *m_poFS =
nullptr;
695 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
696 bool m_bUseChunked =
false;
701 int m_nBufferOff = 0;
702 int m_nBufferSize = 0;
703 bool m_bClosed =
false;
704 GByte *m_pabyBuffer =
nullptr;
706 int m_nPartNumber = 0;
707 std::vector<CPLString> m_aosEtags{};
708 bool m_bError =
false;
710 CURLM *m_hCurlMulti =
nullptr;
711 CURL *m_hCurl =
nullptr;
712 const void *m_pBuffer =
nullptr;
714 size_t m_nChunkedBufferOff = 0;
715 size_t m_nChunkedBufferSize = 0;
716 size_t m_nWrittenInPUT = 0;
719 double m_dfRetryDelay = 0.0;
720 WriteFuncStruct m_sWriteFuncHeaderData{};
723 bool DoSinglePartPUT();
725 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
726 size_t nitems,
void *instream);
727 size_t WriteChunked(
const void *pBuffer,
size_t nSize,
size_t nMemb);
728 int FinishChunkedTransfer();
730 void InvalidateParentDirectory();
733 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS,
const char *pszFilename,
734 IVSIS3LikeHandleHelper *poS3HandleHelper,
bool bUseChunked,
736 ~VSIS3WriteHandle()
override;
740 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
741 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
743 int Close()
override;
747 return m_bUseChunked || m_pabyBuffer !=
nullptr;
760 VSICurlFilesystemHandlerBase *m_poFS =
nullptr;
765 int m_nBufferOff = 0;
766 int m_nBufferSize = 0;
767 int m_nBufferOffReadCallback = 0;
768 bool m_bClosed =
false;
769 GByte *m_pabyBuffer =
nullptr;
770 bool m_bError =
false;
772 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
774 virtual bool Send(
bool bIsLastBlock) = 0;
777 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
778 const char *pszFSPrefix,
const char *pszFilename,
780 virtual ~VSIAppendWriteHandle();
784 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
785 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
787 int Close()
override;
791 return m_pabyBuffer !=
nullptr;
799 struct VSIDIRWithMissingDirSynthesis :
public VSIDIR
801 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
804 std::vector<std::string> m_aosSubpathsStack{};
806 void SynthetizeMissingDirectories(
const std::string &osCurSubdir,
807 bool bAddEntryForThisSubdir);
814 struct CurlRequestHelper
816 WriteFuncStruct sWriteFuncData{};
817 WriteFuncStruct sWriteFuncHeaderData{};
818 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
821 ~CurlRequestHelper();
822 long perform(CURL *hCurlHandle,
823 struct curl_slist *headers,
824 VSICurlFilesystemHandlerBase *poFS,
825 IVSIS3LikeHandleHelper *poS3HandleHelper);
832 class NetworkStatisticsLogger
834 static int gnEnabled;
835 static NetworkStatisticsLogger gInstance;
837 NetworkStatisticsLogger() =
default;
839 std::mutex m_mutex{};
848 GIntBig nGETDownloadedBytes = 0;
850 GIntBig nPOSTDownloadedBytes = 0;
851 GIntBig nPOSTUploadedBytes = 0;
854 enum class ContextPathType
861 struct ContextPathItem
863 ContextPathType eType;
866 ContextPathItem(ContextPathType eTypeIn,
const CPLString &osNameIn)
867 : eType(eTypeIn), osName(osNameIn)
871 bool operator<(
const ContextPathItem &other)
const
873 if (
static_cast<int>(eType) <
static_cast<int>(other.eType))
875 if (
static_cast<int>(eType) >
static_cast<int>(other.eType))
877 return osName < other.osName;
884 std::map<ContextPathItem, Stats> children{};
892 std::map<GIntBig, std::vector<ContextPathItem>>
893 m_mapThreadIdToContextPath{};
895 static void ReadEnabled();
897 std::vector<Counters *> GetCountersForContext();
900 static inline bool IsEnabled()
906 return gnEnabled == TRUE;
909 static void EnterFileSystem(
const char *pszName);
911 static void LeaveFileSystem();
913 static void EnterFile(
const char *pszName);
915 static void LeaveFile();
917 static void EnterAction(
const char *pszName);
919 static void LeaveAction();
921 static void LogHEAD();
923 static void LogGET(
size_t nDownloadedBytes);
925 static void LogPUT(
size_t nUploadedBytes);
927 static void LogPOST(
size_t nUploadedBytes,
size_t nDownloadedBytes);
929 static void LogDELETE();
933 static CPLString GetReportAsSerializedJSON();
936 struct NetworkStatisticsFileSystem
938 inline explicit NetworkStatisticsFileSystem(
const char *pszName)
940 NetworkStatisticsLogger::EnterFileSystem(pszName);
943 inline ~NetworkStatisticsFileSystem()
945 NetworkStatisticsLogger::LeaveFileSystem();
949 struct NetworkStatisticsFile
951 inline explicit NetworkStatisticsFile(
const char *pszName)
953 NetworkStatisticsLogger::EnterFile(pszName);
956 inline ~NetworkStatisticsFile()
958 NetworkStatisticsLogger::LeaveFile();
962 struct NetworkStatisticsAction
964 inline explicit NetworkStatisticsAction(
const char *pszName)
966 NetworkStatisticsLogger::EnterAction(pszName);
969 inline ~NetworkStatisticsAction()
971 NetworkStatisticsLogger::LeaveAction();
975 int VSICURLGetDownloadChunkSize();
977 void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct,
VSILFILE *fp,
978 VSICurlReadCbkFunc pfnReadCbk,
979 void *pReadCbkUserData);
980 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
size_t nmemb,
982 void MultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle =
nullptr);
983 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
985 int VSICurlParseUnixPermissions(
const char *pszPermissions);
988 bool VSICURLGetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
989 void VSICURLSetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
990 void VSICURLInvalidateCachedFileProp(
const char *pszURL);
991 void VSICURLInvalidateCachedFilePropPrefix(
const char *pszURL);
992 void VSICURLDestroyCacheFileProp();
1000 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED