29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
52 #define HAVE_CURLINFO_REDIRECT_URL
54 void VSICurlStreamingClearCache(
void );
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
57 const char *
const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
61 struct curl_slist* VSICurlSetContentTypeFromExt(
struct curl_slist* polist,
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(
struct curl_slist* headers,
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
84 time_t nExpireTimestampLocal = 0;
86 bool bHasComputedFileSize =
false;
87 bool bIsDirectory =
false;
89 bool bS3LikeRedirect =
false;
95 bool bGotFileList =
false;
96 unsigned int nGenerationAuthParameters = 0;
100 struct WriteFuncStruct
102 char* pBuffer =
nullptr;
104 bool bIsHTTP =
false;
105 bool bMultiRange =
false;
110 bool bFoundContentRange =
false;
112 bool bInterruptDownload =
false;
113 bool bDetectRangeDownloadingError =
false;
117 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
118 void *pReadCbkUserData =
nullptr;
119 bool bInterrupted =
false;
120 bool bInterruptIfNonErrorPayload =
false;
122 #if !CURL_AT_LEAST_VERSION(7,54,0)
126 bool bIsProxyConnectHeader =
false;
132 const GByte* pabyData =
nullptr;
134 size_t nTotalSize = 0;
136 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
137 size_t nitems,
void *instream )
139 PutData* poThis =
static_cast<PutData *
>(instream);
140 const size_t nSizeMax = size * nitems;
141 const size_t nSizeToWrite =
142 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
143 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
144 poThis->nOff += nSizeToWrite;
155 class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
159 struct FilenameOffsetPair
161 std::string filename_;
164 FilenameOffsetPair(
const std::string& filename,
166 filename_(filename), offset_(offset) {}
168 bool operator==(
const FilenameOffsetPair& other)
const
170 return filename_ == other.filename_ &&
171 offset_ == other.offset_;
174 struct FilenameOffsetPairHasher
176 std::size_t operator()(
const FilenameOffsetPair& k)
const
178 return std::hash<std::string>()(k.filename_) ^
179 std::hash<vsi_l_offset>()(k.offset_);
183 using RegionCacheType =
184 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
188 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
189 std::shared_ptr<std::string>>>::iterator,
190 FilenameOffsetPairHasher>>;
192 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{};
193 RegionCacheType* GetRegionCache();
200 lru11::Cache<std::string, bool> oCacheFileProp;
202 int nCachedFilesInDirList = 0;
203 lru11::Cache<std::string, CachedDirList> oCacheDirList;
205 char** ParseHTMLFileList(
const char* pszFilename,
208 bool* pbGotFileList);
211 CPLMutex *hMutex =
nullptr;
213 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
214 virtual char** GetFileList(
const char *pszFilename,
216 bool* pbGotFileList);
218 void RegisterEmptyDir(
const CPLString& osDirname );
220 bool AnalyseS3FileList(
const CPLString& osBaseURL,
224 const std::set<std::string>& oSetIgnoredStorageClasses,
225 bool& bIsTruncated );
227 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
231 int nMaxFilesThisQuery,
236 static const char* GetOptionsStatic();
238 static bool IsAllowedFilename(
const char* pszFilename );
240 VSICurlFilesystemHandlerBase();
243 ~VSICurlFilesystemHandlerBase()
override;
246 const char *pszAccess,
250 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
251 int nFlags )
override;
252 int Unlink(
const char *pszFilename )
override;
253 int Rename(
const char *oldpath,
const char *newpath )
override;
254 int Mkdir(
const char *pszDirname,
long nMode )
override;
255 int Rmdir(
const char *pszDirname )
override;
256 char **ReadDir(
const char *pszDirname )
override
257 {
return ReadDirEx(pszDirname, 0); }
258 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
259 char **SiblingFiles(
const char *pszFilename )
override;
261 int HasOptimizedReadMultiRange(
const char* )
262 override {
return true; }
264 const char* GetActualURL(
const char* pszFilename)
override;
266 const char* GetOptions()
override;
268 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
271 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
272 bool* pbGotFileList );
273 void InvalidateDirContent(
const char *pszDirname );
275 virtual const char* GetDebugKey()
const = 0;
277 virtual CPLString GetFSPrefix()
const = 0;
278 virtual bool AllowCachedDataFor(
const char* pszFilename);
280 virtual bool IsLocal(
const char* )
override {
return false; }
281 virtual bool SupportsSequentialWrite(
const char* ,
bool )
override {
return false; }
282 virtual bool SupportsRandomWrite(
const char* ,
bool )
override {
return false; }
284 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
287 void AddRegion(
const char* pszURL,
292 bool GetCachedFileProp(
const char* pszURL,
293 FileProp& oFileProp );
294 void SetCachedFileProp(
const char* pszURL,
295 FileProp& oFileProp );
296 void InvalidateCachedData(
const char* pszURL );
298 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
300 virtual void ClearCache();
301 virtual void PartialClearCache(
const char* pszFilename);
304 bool GetCachedDirList(
const char* pszURL,
305 CachedDirList& oCachedDirList );
306 void SetCachedDirList(
const char* pszURL,
307 CachedDirList& oCachedDirList );
308 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
312 std::string GetStreamingFilename(
const std::string& osFilename)
const override = 0;
314 static std::set<std::string> GetS3IgnoredStorageClasses();
318 class VSICurlFilesystemHandler:
public VSICurlFilesystemHandlerBase
323 VSICurlFilesystemHandler() =
default;
325 const char* GetDebugKey()
const override {
return "VSICURL"; }
327 CPLString GetFSPrefix()
const override {
return "/vsicurl/"; }
329 std::string GetStreamingFilename(
const std::string& osFilename)
const override;
341 VSICurlFilesystemHandlerBase* poFS =
nullptr;
343 bool m_bCached =
true;
345 mutable FileProp oFileProp{};
347 mutable std::mutex m_oMutex{};
349 char* m_pszURL =
nullptr;
350 mutable std::string m_osQueryString{};
352 char **m_papszHTTPOptions =
nullptr;
355 int nBlocksToDownload = 1;
357 bool bStopOnInterruptUntilUninstall =
false;
358 bool bInterrupted =
false;
359 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
360 void *pReadCbkUserData =
nullptr;
363 double m_dfRetryDelay = 0.0;
367 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
378 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
380 bool m_bUseHead =
false;
381 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
384 mutable bool m_bPlanetaryComputerURLSigning =
false;
385 mutable std::string m_osPlanetaryComputerCollection{};
386 void ManagePlanetaryComputerSigning()
const;
388 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
390 const size_t* panSizes );
391 CPLString GetRedirectURLIfValid(
bool& bHasExpired)
const;
393 void UpdateRedirectInfo( CURL* hCurlHandle,
394 const WriteFuncStruct& sWriteFuncHeaderData );
397 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
398 const struct curl_slist* )
400 virtual bool AllowAutomaticRedirection() {
return true; }
401 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
402 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
403 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
404 virtual void ProcessGetFileSizeResult(
const char* ) {}
405 void SetURL(
const char* pszURL);
406 virtual bool Authenticate(
const char* ) {
return false; }
410 VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
411 const char* pszFilename,
412 const char* pszURLIn =
nullptr );
413 ~VSICurlHandle()
override;
417 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
418 int ReadMultiRange(
int nRanges,
void ** ppData,
420 const size_t* panSizes )
override;
421 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
423 int Flush()
override;
424 int Close()
override;
426 bool HasPRead()
const override {
return true; }
427 size_t PRead(
void* pBuffer,
size_t nSize,
vsi_l_offset nOffset )
const override;
429 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
430 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
431 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
432 bool Exists(
bool bSetError );
433 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
434 int GetMode()
const {
return oFileProp.nMode; }
435 time_t GetMTime()
const {
return oFileProp.mTime; }
438 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
440 int bStopOnInterruptUntilUninstall );
441 int UninstallReadCbk();
443 const char *GetURL()
const {
return m_pszURL; }
450 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandlerBase
456 const char* pszSource,
457 const char* pszTarget,
459 GDALProgressFunc pProgressFunc,
460 void *pProgressData);
461 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
bool bDoStatCheck );
464 char** GetFileList(
const char *pszFilename,
466 bool* pbGotFileList )
override;
468 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
469 const char* pszURI,
bool bAllowNoObject) = 0;
471 virtual int CopyObject(
const char *oldpath,
const char *newpath,
474 int RmdirRecursiveInternal(
const char* pszDirname,
int nBatchSize);
476 virtual bool IsAllowedHeaderForObjectCreation(
const char* ) {
return false; }
478 IVSIS3LikeFSHandler() =
default;
481 int Unlink(
const char *pszFilename )
override;
482 int Mkdir(
const char *pszDirname,
long nMode )
override;
483 int Rmdir(
const char *pszDirname )
override;
484 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
485 int nFlags )
override;
486 int Rename(
const char *oldpath,
const char *newpath )
override;
488 virtual int DeleteObject(
const char *pszFilename );
490 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
491 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
493 bool Sync(
const char* pszSource,
const char* pszTarget,
494 const char*
const * papszOptions,
495 GDALProgressFunc pProgressFunc,
497 char*** ppapszOutputs )
override;
499 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
500 const char*
const *papszOptions)
override;
503 virtual bool SupportsParallelMultipartUpload()
const {
return false; }
505 virtual CPLString InitiateMultipartUpload(
506 const std::string& osFilename,
507 IVSIS3LikeHandleHelper *poS3HandleHelper,
513 const std::string& osUploadID,
515 const void* pabyBuffer,
517 IVSIS3LikeHandleHelper *poS3HandleHelper,
521 virtual bool CompleteMultipart(
const CPLString& osFilename,
523 const std::vector<CPLString>& aosEtags,
525 IVSIS3LikeHandleHelper *poS3HandleHelper,
527 double dfRetryDelay);
528 virtual bool AbortMultipart(
const CPLString& osFilename,
530 IVSIS3LikeHandleHelper *poS3HandleHelper,
532 double dfRetryDelay);
534 bool AbortPendingUploads(
const char* pszFilename)
override;
541 class IVSIS3LikeHandle:
public VSICurlHandle
546 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
547 bool IsDirectoryFromExists(
const char* pszVerb,
548 int response_code )
override
551 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
554 void ProcessGetFileSizeResult(
const char* pszContent )
override
556 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
560 IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
561 const char* pszFilename,
562 const char* pszURLIn ) :
563 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
564 ~IVSIS3LikeHandle()
override {}
575 IVSIS3LikeFSHandler *m_poFS =
nullptr;
577 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
578 bool m_bUseChunked =
false;
582 int m_nBufferOff = 0;
583 int m_nBufferSize = 0;
584 bool m_bClosed =
false;
585 GByte *m_pabyBuffer =
nullptr;
587 int m_nPartNumber = 0;
588 std::vector<CPLString> m_aosEtags{};
589 bool m_bError =
false;
591 CURLM *m_hCurlMulti =
nullptr;
592 CURL *m_hCurl =
nullptr;
593 const void *m_pBuffer =
nullptr;
595 size_t m_nChunkedBufferOff = 0;
596 size_t m_nChunkedBufferSize = 0;
597 size_t m_nWrittenInPUT = 0;
600 double m_dfRetryDelay = 0.0;
601 WriteFuncStruct m_sWriteFuncHeaderData{};
604 bool DoSinglePartPUT();
606 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
607 size_t nitems,
void *instream );
608 size_t WriteChunked(
const void *pBuffer,
609 size_t nSize,
size_t nMemb );
610 int FinishChunkedTransfer();
612 void InvalidateParentDirectory();
615 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
616 const char* pszFilename,
617 IVSIS3LikeHandleHelper* poS3HandleHelper,
620 ~VSIS3WriteHandle()
override;
624 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
625 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
627 int Close()
override;
629 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
642 VSICurlFilesystemHandlerBase* m_poFS =
nullptr;
647 int m_nBufferOff = 0;
648 int m_nBufferSize = 0;
649 int m_nBufferOffReadCallback = 0;
650 bool m_bClosed =
false;
651 GByte *m_pabyBuffer =
nullptr;
652 bool m_bError =
false;
654 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
655 size_t nitems,
void *instream );
656 virtual bool Send(
bool bIsLastBlock) = 0;
659 VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
660 const char* pszFSPrefix,
661 const char* pszFilename,
663 virtual ~VSIAppendWriteHandle();
667 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
668 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
670 int Close()
override;
672 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
679 struct VSIDIRWithMissingDirSynthesis:
public VSIDIR
681 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
684 std::vector<std::string> m_aosSubpathsStack{};
686 void SynthetizeMissingDirectories(
const std::string& osCurSubdir,
687 bool bAddEntryForThisSubdir);
694 struct CurlRequestHelper
696 WriteFuncStruct sWriteFuncData{};
697 WriteFuncStruct sWriteFuncHeaderData{};
698 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
701 ~CurlRequestHelper();
702 long perform(CURL* hCurlHandle,
703 struct curl_slist* headers,
704 VSICurlFilesystemHandlerBase *poFS,
705 IVSIS3LikeHandleHelper *poS3HandleHelper);
712 class NetworkStatisticsLogger
714 static int gnEnabled;
715 static NetworkStatisticsLogger gInstance;
717 NetworkStatisticsLogger() =
default;
719 std::mutex m_mutex{};
728 GIntBig nGETDownloadedBytes = 0;
730 GIntBig nPOSTDownloadedBytes = 0;
731 GIntBig nPOSTUploadedBytes = 0;
734 enum class ContextPathType
741 struct ContextPathItem
743 ContextPathType eType;
746 ContextPathItem(ContextPathType eTypeIn,
const CPLString& osNameIn):
747 eType(eTypeIn), osName(osNameIn) {}
749 bool operator< (
const ContextPathItem& other )
const
751 if(
static_cast<int>(eType) <
static_cast<int>(other.eType) )
753 if(
static_cast<int>(eType) >
static_cast<int>(other.eType) )
755 return osName < other.osName;
762 std::map<ContextPathItem, Stats> children{};
770 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
772 static void ReadEnabled();
774 std::vector<Counters*> GetCountersForContext();
778 static inline bool IsEnabled()
784 return gnEnabled == TRUE;
787 static void EnterFileSystem(
const char* pszName);
789 static void LeaveFileSystem();
791 static void EnterFile(
const char* pszName);
793 static void LeaveFile();
795 static void EnterAction(
const char* pszName);
797 static void LeaveAction();
799 static void LogHEAD();
801 static void LogGET(
size_t nDownloadedBytes);
803 static void LogPUT(
size_t nUploadedBytes);
805 static void LogPOST(
size_t nUploadedBytes,
806 size_t nDownloadedBytes);
808 static void LogDELETE();
812 static CPLString GetReportAsSerializedJSON();
815 struct NetworkStatisticsFileSystem
817 inline explicit NetworkStatisticsFileSystem(
const char* pszName) {
818 NetworkStatisticsLogger::EnterFileSystem(pszName);
821 inline ~NetworkStatisticsFileSystem()
823 NetworkStatisticsLogger::LeaveFileSystem();
827 struct NetworkStatisticsFile
829 inline explicit NetworkStatisticsFile(
const char* pszName) {
830 NetworkStatisticsLogger::EnterFile(pszName);
833 inline ~NetworkStatisticsFile()
835 NetworkStatisticsLogger::LeaveFile();
839 struct NetworkStatisticsAction
841 inline explicit NetworkStatisticsAction(
const char* pszName) {
842 NetworkStatisticsLogger::EnterAction(pszName);
845 inline ~NetworkStatisticsAction()
847 NetworkStatisticsLogger::LeaveAction();
852 int VSICURLGetDownloadChunkSize();
854 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
856 VSICurlReadCbkFunc pfnReadCbk,
857 void *pReadCbkUserData );
858 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
859 size_t nmemb,
void *req );
860 void MultiPerform(CURLM* hCurlMultiHandle,
861 CURL* hEasyHandle =
nullptr);
862 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
864 int VSICurlParseUnixPermissions(
const char* pszPermissions);
867 bool VSICURLGetCachedFileProp(
const char* pszURL,
868 FileProp& oFileProp );
869 void VSICURLSetCachedFileProp(
const char* pszURL,
870 FileProp& oFileProp );
871 void VSICURLInvalidateCachedFileProp(
const char* pszURL );
872 void VSICURLInvalidateCachedFilePropPrefix(
const char* pszURL );
873 void VSICURLDestroyCacheFileProp();
881 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED