29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
52 #define HAVE_CURLINFO_REDIRECT_URL
54 void VSICurlStreamingClearCache(
void );
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
57 const char *
const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
61 struct curl_slist* VSICurlSetContentTypeFromExt(
struct curl_slist* polist,
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(
struct curl_slist* headers,
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
84 time_t nExpireTimestampLocal = 0;
86 bool bHasComputedFileSize =
false;
87 bool bIsDirectory =
false;
89 bool bS3LikeRedirect =
false;
95 bool bGotFileList =
false;
96 unsigned int nGenerationAuthParameters = 0;
100 struct WriteFuncStruct
102 char* pBuffer =
nullptr;
104 bool bIsHTTP =
false;
105 bool bMultiRange =
false;
110 bool bFoundContentRange =
false;
112 bool bInterruptDownload =
false;
113 bool bDetectRangeDownloadingError =
false;
117 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
118 void *pReadCbkUserData =
nullptr;
119 bool bInterrupted =
false;
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
125 bool bIsProxyConnectHeader =
false;
131 const GByte* pabyData =
nullptr;
133 size_t nTotalSize = 0;
135 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
136 size_t nitems,
void *instream )
138 PutData* poThis =
static_cast<PutData *
>(instream);
139 const size_t nSizeMax = size * nitems;
140 const size_t nSizeToWrite =
141 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143 poThis->nOff += nSizeToWrite;
154 class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
158 struct FilenameOffsetPair
160 std::string filename_;
163 FilenameOffsetPair(
const std::string& filename,
165 filename_(filename), offset_(offset) {}
167 bool operator==(
const FilenameOffsetPair& other)
const
169 return filename_ == other.filename_ &&
170 offset_ == other.offset_;
173 struct FilenameOffsetPairHasher
175 std::size_t operator()(
const FilenameOffsetPair& k)
const
177 return std::hash<std::string>()(k.filename_) ^
178 std::hash<vsi_l_offset>()(k.offset_);
182 using RegionCacheType =
183 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
187 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188 std::shared_ptr<std::string>>>::iterator,
189 FilenameOffsetPairHasher>>;
191 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{};
192 RegionCacheType* GetRegionCache();
199 lru11::Cache<std::string, bool> oCacheFileProp;
201 int nCachedFilesInDirList = 0;
202 lru11::Cache<std::string, CachedDirList> oCacheDirList;
204 char** ParseHTMLFileList(
const char* pszFilename,
207 bool* pbGotFileList);
210 CPLMutex *hMutex =
nullptr;
212 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
213 virtual char** GetFileList(
const char *pszFilename,
215 bool* pbGotFileList);
217 void RegisterEmptyDir(
const CPLString& osDirname );
219 bool AnalyseS3FileList(
const CPLString& osBaseURL,
223 const std::set<std::string>& oSetIgnoredStorageClasses,
224 bool& bIsTruncated );
226 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
230 int nMaxFilesThisQuery,
235 static const char* GetOptionsStatic();
237 static bool IsAllowedFilename(
const char* pszFilename );
239 VSICurlFilesystemHandlerBase();
242 ~VSICurlFilesystemHandlerBase()
override;
245 const char *pszAccess,
249 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
250 int nFlags )
override;
251 int Unlink(
const char *pszFilename )
override;
252 int Rename(
const char *oldpath,
const char *newpath )
override;
253 int Mkdir(
const char *pszDirname,
long nMode )
override;
254 int Rmdir(
const char *pszDirname )
override;
255 char **ReadDir(
const char *pszDirname )
override
256 {
return ReadDirEx(pszDirname, 0); }
257 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
258 char **SiblingFiles(
const char *pszFilename )
override;
260 int HasOptimizedReadMultiRange(
const char* )
261 override {
return true; }
263 const char* GetActualURL(
const char* pszFilename)
override;
265 const char* GetOptions()
override;
267 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
270 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
271 bool* pbGotFileList );
272 void InvalidateDirContent(
const char *pszDirname );
274 virtual const char* GetDebugKey()
const = 0;
276 virtual CPLString GetFSPrefix()
const = 0;
277 virtual bool AllowCachedDataFor(
const char* pszFilename);
279 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
282 void AddRegion(
const char* pszURL,
287 bool GetCachedFileProp(
const char* pszURL,
288 FileProp& oFileProp );
289 void SetCachedFileProp(
const char* pszURL,
290 FileProp& oFileProp );
291 void InvalidateCachedData(
const char* pszURL );
293 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
295 virtual void ClearCache();
296 virtual void PartialClearCache(
const char* pszFilename);
299 bool GetCachedDirList(
const char* pszURL,
300 CachedDirList& oCachedDirList );
301 void SetCachedDirList(
const char* pszURL,
302 CachedDirList& oCachedDirList );
303 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
307 std::string GetStreamingFilename(
const std::string& osFilename)
const override = 0;
309 static std::set<std::string> GetS3IgnoredStorageClasses();
313 class VSICurlFilesystemHandler:
public VSICurlFilesystemHandlerBase
318 VSICurlFilesystemHandler() =
default;
320 const char* GetDebugKey()
const override {
return "VSICURL"; }
322 CPLString GetFSPrefix()
const override {
return "/vsicurl/"; }
324 std::string GetStreamingFilename(
const std::string& osFilename)
const override;
336 VSICurlFilesystemHandlerBase* poFS =
nullptr;
338 bool m_bCached =
true;
340 FileProp oFileProp{};
343 char* m_pszURL =
nullptr;
344 std::string m_osQueryString{};
346 char **m_papszHTTPOptions =
nullptr;
349 int nBlocksToDownload = 1;
351 bool bStopOnInterruptUntilUninstall =
false;
352 bool bInterrupted =
false;
353 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
354 void *pReadCbkUserData =
nullptr;
357 double m_dfRetryDelay = 0.0;
361 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
372 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
374 bool m_bUseHead =
false;
375 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
377 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
379 const size_t* panSizes );
380 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
383 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
384 const struct curl_slist* )
386 virtual bool AllowAutomaticRedirection() {
return true; }
387 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
388 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
389 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
390 virtual void ProcessGetFileSizeResult(
const char* ) {}
391 void SetURL(
const char* pszURL);
392 virtual bool Authenticate(
const char* ) {
return false; }
396 VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
397 const char* pszFilename,
398 const char* pszURLIn =
nullptr );
399 ~VSICurlHandle()
override;
403 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
404 int ReadMultiRange(
int nRanges,
void ** ppData,
406 const size_t* panSizes )
override;
407 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
409 int Flush()
override;
410 int Close()
override;
412 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
413 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
414 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
415 bool Exists(
bool bSetError );
416 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
417 int GetMode()
const {
return oFileProp.nMode; }
418 time_t GetMTime()
const {
return oFileProp.mTime; }
421 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
423 int bStopOnInterruptUntilUninstall );
424 int UninstallReadCbk();
426 const char *GetURL()
const {
return m_pszURL; }
433 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandlerBase
439 const char* pszSource,
440 const char* pszTarget,
442 GDALProgressFunc pProgressFunc,
443 void *pProgressData);
444 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
bool bDoStatCheck );
447 char** GetFileList(
const char *pszFilename,
449 bool* pbGotFileList )
override;
451 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
452 const char* pszURI,
bool bAllowNoObject) = 0;
454 virtual int CopyObject(
const char *oldpath,
const char *newpath,
457 int RmdirRecursiveInternal(
const char* pszDirname,
int nBatchSize);
459 virtual bool IsAllowedHeaderForObjectCreation(
const char* ) {
return false; }
461 IVSIS3LikeFSHandler() =
default;
464 int Unlink(
const char *pszFilename )
override;
465 int Mkdir(
const char *pszDirname,
long nMode )
override;
466 int Rmdir(
const char *pszDirname )
override;
467 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
468 int nFlags )
override;
469 int Rename(
const char *oldpath,
const char *newpath )
override;
471 virtual int DeleteObject(
const char *pszFilename );
473 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
474 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
476 bool Sync(
const char* pszSource,
const char* pszTarget,
477 const char*
const * papszOptions,
478 GDALProgressFunc pProgressFunc,
480 char*** ppapszOutputs )
override;
482 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
483 const char*
const *papszOptions)
override;
486 virtual bool SupportsParallelMultipartUpload()
const {
return false; }
488 virtual CPLString InitiateMultipartUpload(
489 const std::string& osFilename,
490 IVSIS3LikeHandleHelper *poS3HandleHelper,
496 const std::string& osUploadID,
498 const void* pabyBuffer,
500 IVSIS3LikeHandleHelper *poS3HandleHelper,
504 virtual bool CompleteMultipart(
const CPLString& osFilename,
506 const std::vector<CPLString>& aosEtags,
508 IVSIS3LikeHandleHelper *poS3HandleHelper,
510 double dfRetryDelay);
511 virtual bool AbortMultipart(
const CPLString& osFilename,
513 IVSIS3LikeHandleHelper *poS3HandleHelper,
515 double dfRetryDelay);
517 bool AbortPendingUploads(
const char* pszFilename)
override;
524 class IVSIS3LikeHandle:
public VSICurlHandle
529 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
530 bool IsDirectoryFromExists(
const char* pszVerb,
531 int response_code )
override
534 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
537 void ProcessGetFileSizeResult(
const char* pszContent )
override
539 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
543 IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
544 const char* pszFilename,
545 const char* pszURLIn ) :
546 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
547 ~IVSIS3LikeHandle()
override {}
558 IVSIS3LikeFSHandler *m_poFS =
nullptr;
560 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
561 bool m_bUseChunked =
false;
565 int m_nBufferOff = 0;
566 int m_nBufferSize = 0;
567 bool m_bClosed =
false;
568 GByte *m_pabyBuffer =
nullptr;
570 int m_nPartNumber = 0;
571 std::vector<CPLString> m_aosEtags{};
572 bool m_bError =
false;
574 CURLM *m_hCurlMulti =
nullptr;
575 CURL *m_hCurl =
nullptr;
576 const void *m_pBuffer =
nullptr;
578 size_t m_nChunkedBufferOff = 0;
579 size_t m_nChunkedBufferSize = 0;
580 size_t m_nWrittenInPUT = 0;
583 double m_dfRetryDelay = 0.0;
584 WriteFuncStruct m_sWriteFuncHeaderData{};
587 bool DoSinglePartPUT();
589 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
590 size_t nitems,
void *instream );
591 size_t WriteChunked(
const void *pBuffer,
592 size_t nSize,
size_t nMemb );
593 int FinishChunkedTransfer();
595 void InvalidateParentDirectory();
598 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
599 const char* pszFilename,
600 IVSIS3LikeHandleHelper* poS3HandleHelper,
603 ~VSIS3WriteHandle()
override;
607 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
608 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
610 int Close()
override;
612 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
625 VSICurlFilesystemHandlerBase* m_poFS =
nullptr;
630 int m_nBufferOff = 0;
631 int m_nBufferSize = 0;
632 int m_nBufferOffReadCallback = 0;
633 bool m_bClosed =
false;
634 GByte *m_pabyBuffer =
nullptr;
635 bool m_bError =
false;
637 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
638 size_t nitems,
void *instream );
639 virtual bool Send(
bool bIsLastBlock) = 0;
642 VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
643 const char* pszFSPrefix,
644 const char* pszFilename,
646 virtual ~VSIAppendWriteHandle();
650 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
651 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
653 int Close()
override;
655 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
662 struct VSIDIRWithMissingDirSynthesis:
public VSIDIR
664 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
667 std::vector<std::string> m_aosSubpathsStack{};
669 void SynthetizeMissingDirectories(
const std::string& osCurSubdir,
670 bool bAddEntryForThisSubdir);
677 struct CurlRequestHelper
679 WriteFuncStruct sWriteFuncData{};
680 WriteFuncStruct sWriteFuncHeaderData{};
681 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
684 ~CurlRequestHelper();
685 long perform(CURL* hCurlHandle,
686 struct curl_slist* headers,
687 VSICurlFilesystemHandlerBase *poFS,
688 IVSIS3LikeHandleHelper *poS3HandleHelper);
695 class NetworkStatisticsLogger
697 static int gnEnabled;
698 static NetworkStatisticsLogger gInstance;
700 NetworkStatisticsLogger() =
default;
702 std::mutex m_mutex{};
711 GIntBig nGETDownloadedBytes = 0;
713 GIntBig nPOSTDownloadedBytes = 0;
714 GIntBig nPOSTUploadedBytes = 0;
717 enum class ContextPathType
724 struct ContextPathItem
726 ContextPathType eType;
729 ContextPathItem(ContextPathType eTypeIn,
const CPLString& osNameIn):
730 eType(eTypeIn), osName(osNameIn) {}
732 bool operator< (
const ContextPathItem& other )
const
734 if(
static_cast<int>(eType) <
static_cast<int>(other.eType) )
736 if(
static_cast<int>(eType) >
static_cast<int>(other.eType) )
738 return osName < other.osName;
745 std::map<ContextPathItem, Stats> children{};
753 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
755 static void ReadEnabled();
757 std::vector<Counters*> GetCountersForContext();
761 static inline bool IsEnabled()
767 return gnEnabled == TRUE;
770 static void EnterFileSystem(
const char* pszName);
772 static void LeaveFileSystem();
774 static void EnterFile(
const char* pszName);
776 static void LeaveFile();
778 static void EnterAction(
const char* pszName);
780 static void LeaveAction();
782 static void LogHEAD();
784 static void LogGET(
size_t nDownloadedBytes);
786 static void LogPUT(
size_t nUploadedBytes);
788 static void LogPOST(
size_t nUploadedBytes,
789 size_t nDownloadedBytes);
791 static void LogDELETE();
795 static CPLString GetReportAsSerializedJSON();
798 struct NetworkStatisticsFileSystem
800 inline explicit NetworkStatisticsFileSystem(
const char* pszName) {
801 NetworkStatisticsLogger::EnterFileSystem(pszName);
804 inline ~NetworkStatisticsFileSystem()
806 NetworkStatisticsLogger::LeaveFileSystem();
810 struct NetworkStatisticsFile
812 inline explicit NetworkStatisticsFile(
const char* pszName) {
813 NetworkStatisticsLogger::EnterFile(pszName);
816 inline ~NetworkStatisticsFile()
818 NetworkStatisticsLogger::LeaveFile();
822 struct NetworkStatisticsAction
824 inline explicit NetworkStatisticsAction(
const char* pszName) {
825 NetworkStatisticsLogger::EnterAction(pszName);
828 inline ~NetworkStatisticsAction()
830 NetworkStatisticsLogger::LeaveAction();
835 int VSICURLGetDownloadChunkSize();
837 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
839 VSICurlReadCbkFunc pfnReadCbk,
840 void *pReadCbkUserData );
841 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
842 size_t nmemb,
void *req );
843 void MultiPerform(CURLM* hCurlMultiHandle,
844 CURL* hEasyHandle =
nullptr);
845 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
847 int VSICurlParseUnixPermissions(
const char* pszPermissions);
850 bool VSICURLGetCachedFileProp(
const char* pszURL,
851 FileProp& oFileProp );
852 void VSICURLSetCachedFileProp(
const char* pszURL,
853 FileProp& oFileProp );
854 void VSICURLInvalidateCachedFileProp(
const char* pszURL );
855 void VSICURLInvalidateCachedFilePropPrefix(
const char* pszURL );
856 void VSICURLDestroyCacheFileProp();
864 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED