GDAL
cpl_vsi_virtual.h
1 /******************************************************************************
2  * $Id$
3  *
4  * Project: VSI Virtual File System
5  * Purpose: Declarations for classes related to the virtual filesystem.
6  * These would only be normally required by applications implementing
7  * their own virtual file system classes which should be rare.
8  * The class interface may be fragile through versions.
9  * Author: Frank Warmerdam, warmerdam@pobox.com
10  *
11  ******************************************************************************
12  * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
13  * Copyright (c) 2010-2014, Even Rouault <even dot rouault at spatialys.com>
14  *
15  * Permission is hereby granted, free of charge, to any person obtaining a
16  * copy of this software and associated documentation files (the "Software"),
17  * to deal in the Software without restriction, including without limitation
18  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
19  * and/or sell copies of the Software, and to permit persons to whom the
20  * Software is furnished to do so, subject to the following conditions:
21  *
22  * The above copyright notice and this permission notice shall be included
23  * in all copies or substantial portions of the Software.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
26  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
28  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
31  * DEALINGS IN THE SOFTWARE.
32  ****************************************************************************/
33 
34 #ifndef CPL_VSI_VIRTUAL_H_INCLUDED
35 #define CPL_VSI_VIRTUAL_H_INCLUDED
36 
37 #include "cpl_vsi.h"
38 #include "cpl_vsi_error.h"
39 #include "cpl_string.h"
40 #include "cpl_multiproc.h"
41 
42 #include <map>
43 #include <memory>
44 #include <vector>
45 #include <string>
46 
47 // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
48 #ifdef GetDiskFreeSpace
49 #undef GetDiskFreeSpace
50 #endif
51 
52 // To avoid aliasing to CopyFile to CopyFileA on Windows
53 #ifdef CopyFile
54 #undef CopyFile
55 #endif
56 
57 /************************************************************************/
58 /* VSIVirtualHandle */
59 /************************************************************************/
60 
62 struct CPL_DLL VSIVirtualHandle
63 {
64  public:
65  virtual int Seek(vsi_l_offset nOffset, int nWhence) = 0;
66  virtual vsi_l_offset Tell() = 0;
67  virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) = 0;
68  virtual int ReadMultiRange(int nRanges, void **ppData,
69  const vsi_l_offset *panOffsets,
70  const size_t *panSizes);
71 
84  virtual void AdviseRead(CPL_UNUSED int nRanges,
85  CPL_UNUSED const vsi_l_offset *panOffsets,
86  CPL_UNUSED const size_t *panSizes)
87  {
88  }
89 
90  virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount) = 0;
91  virtual int Eof() = 0;
92  virtual int Flush()
93  {
94  return 0;
95  }
96  virtual int Close() = 0;
97  // Base implementation that only supports file extension.
98  virtual int Truncate(vsi_l_offset nNewSize);
99  virtual void *GetNativeFileDescriptor()
100  {
101  return nullptr;
102  }
103  virtual VSIRangeStatus GetRangeStatus(CPL_UNUSED vsi_l_offset nOffset,
104  CPL_UNUSED vsi_l_offset nLength)
105  {
107  }
108  virtual bool HasPRead() const;
109  virtual size_t PRead(void *pBuffer, size_t nSize,
110  vsi_l_offset nOffset) const;
111 
112  // NOTE: when adding new methods, besides the "actual" implementations,
113  // also consider the VSICachedFile one.
114 
115  virtual ~VSIVirtualHandle()
116  {
117  }
118 };
119 
120 /************************************************************************/
121 /* VSIVirtualHandleCloser */
122 /************************************************************************/
123 
127 {
129  void operator()(VSIVirtualHandle *poHandle)
130  {
131  if (poHandle)
132  {
133  poHandle->Close();
134  delete poHandle;
135  }
136  }
137 };
138 
140 typedef std::unique_ptr<VSIVirtualHandle, VSIVirtualHandleCloser>
141  VSIVirtualHandleUniquePtr;
142 
143 /************************************************************************/
144 /* VSIFilesystemHandler */
145 /************************************************************************/
146 
147 #ifndef DOXYGEN_SKIP
148 class CPL_DLL VSIFilesystemHandler
149 {
150 
151  public:
152  virtual ~VSIFilesystemHandler()
153  {
154  }
155 
156  VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess);
157 
158  virtual VSIVirtualHandle *Open(const char *pszFilename,
159  const char *pszAccess, bool bSetError,
160  CSLConstList papszOptions) = 0;
161  virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
162  int nFlags) = 0;
163  virtual int Unlink(const char *pszFilename)
164  {
165  (void)pszFilename;
166  errno = ENOENT;
167  return -1;
168  }
169  virtual int *UnlinkBatch(CSLConstList papszFiles);
170  virtual int Mkdir(const char *pszDirname, long nMode)
171  {
172  (void)pszDirname;
173  (void)nMode;
174  errno = ENOENT;
175  return -1;
176  }
177  virtual int Rmdir(const char *pszDirname)
178  {
179  (void)pszDirname;
180  errno = ENOENT;
181  return -1;
182  }
183  virtual int RmdirRecursive(const char *pszDirname);
184  char **ReadDir(const char *pszDirname)
185  {
186  return ReadDirEx(pszDirname, 0);
187  }
188  virtual char **ReadDirEx(const char * /*pszDirname*/, int /* nMaxFiles */)
189  {
190  return nullptr;
191  }
192  virtual char **SiblingFiles(const char * /*pszFilename*/)
193  {
194  return nullptr;
195  }
196  virtual int Rename(const char *oldpath, const char *newpath)
197  {
198  (void)oldpath;
199  (void)newpath;
200  errno = ENOENT;
201  return -1;
202  }
203  virtual int IsCaseSensitive(const char *pszFilename)
204  {
205  (void)pszFilename;
206  return TRUE;
207  }
208  virtual GIntBig GetDiskFreeSpace(const char * /* pszDirname */)
209  {
210  return -1;
211  }
212  virtual int SupportsSparseFiles(const char * /* pszPath */)
213  {
214  return FALSE;
215  }
216  virtual int HasOptimizedReadMultiRange(const char * /* pszPath */)
217  {
218  return FALSE;
219  }
220  virtual const char *GetActualURL(const char * /*pszFilename*/)
221  {
222  return nullptr;
223  }
224  virtual const char *GetOptions()
225  {
226  return nullptr;
227  }
228  virtual char *GetSignedURL(const char * /*pszFilename*/,
229  CSLConstList /* papszOptions */)
230  {
231  return nullptr;
232  }
233  virtual bool Sync(const char *pszSource, const char *pszTarget,
234  const char *const *papszOptions,
235  GDALProgressFunc pProgressFunc, void *pProgressData,
236  char ***ppapszOutputs);
237 
238  virtual int CopyFile(const char *pszSource, const char *pszTarget,
239  VSILFILE *fpSource, vsi_l_offset nSourceSize,
240  const char *const *papszOptions,
241  GDALProgressFunc pProgressFunc, void *pProgressData);
242 
243  virtual VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
244  const char *const *papszOptions);
245 
246  virtual char **GetFileMetadata(const char *pszFilename,
247  const char *pszDomain,
248  CSLConstList papszOptions);
249 
250  virtual bool SetFileMetadata(const char *pszFilename,
251  CSLConstList papszMetadata,
252  const char *pszDomain,
253  CSLConstList papszOptions);
254 
255  virtual bool AbortPendingUploads(const char * /*pszFilename*/)
256  {
257  return true;
258  }
259 
260  virtual std::string
261  GetStreamingFilename(const std::string &osFilename) const
262  {
263  return osFilename;
264  }
265 
273  virtual std::string
274  GetCanonicalFilename(const std::string &osFilename) const
275  {
276  return osFilename;
277  }
278 
279  virtual bool IsLocal(const char * /* pszPath */)
280  {
281  return true;
282  }
283  virtual bool SupportsSequentialWrite(const char * /* pszPath */,
284  bool /* bAllowLocalTempFile */)
285  {
286  return true;
287  }
288  virtual bool SupportsRandomWrite(const char * /* pszPath */,
289  bool /* bAllowLocalTempFile */)
290  {
291  return true;
292  }
293  virtual bool SupportsRead(const char * /* pszPath */)
294  {
295  return true;
296  }
297 
298  virtual VSIFilesystemHandler *Duplicate(const char * /* pszPrefix */)
299  {
300  CPLError(CE_Failure, CPLE_NotSupported,
301  "Duplicate() not supported on this file system");
302  return nullptr;
303  }
304 };
305 #endif /* #ifndef DOXYGEN_SKIP */
306 
307 /************************************************************************/
308 /* VSIFileManager */
309 /************************************************************************/
310 
311 #ifndef DOXYGEN_SKIP
312 class CPL_DLL VSIFileManager
313 {
314  private:
315  VSIFilesystemHandler *poDefaultHandler = nullptr;
316  std::map<std::string, VSIFilesystemHandler *> oHandlers{};
317 
318  VSIFileManager();
319 
320  static VSIFileManager *Get();
321 
322  CPL_DISALLOW_COPY_ASSIGN(VSIFileManager)
323 
324  public:
325  ~VSIFileManager();
326 
327  static VSIFilesystemHandler *GetHandler(const char *);
328  static void InstallHandler(const std::string &osPrefix,
329  VSIFilesystemHandler *);
330  /* RemoveHandler is never defined. */
331  /* static void RemoveHandler( const std::string& osPrefix ); */
332 
333  static char **GetPrefixes();
334 };
335 #endif /* #ifndef DOXYGEN_SKIP */
336 
337 /************************************************************************/
338 /* ==================================================================== */
339 /* VSIArchiveFilesystemHandler */
340 /* ==================================================================== */
341 /************************************************************************/
342 
343 #ifndef DOXYGEN_SKIP
344 
345 class VSIArchiveEntryFileOffset
346 {
347  public:
348  virtual ~VSIArchiveEntryFileOffset();
349 };
350 
351 typedef struct
352 {
353  char *fileName;
354  vsi_l_offset uncompressed_size;
355  VSIArchiveEntryFileOffset *file_pos;
356  int bIsDir;
357  GIntBig nModifiedTime;
358 } VSIArchiveEntry;
359 
360 class VSIArchiveContent
361 {
362  public:
363  time_t mTime = 0;
364  vsi_l_offset nFileSize = 0;
365  int nEntries = 0;
366  VSIArchiveEntry *entries = nullptr;
367 
368  ~VSIArchiveContent();
369 };
370 
371 class VSIArchiveReader
372 {
373  public:
374  virtual ~VSIArchiveReader();
375 
376  virtual int GotoFirstFile() = 0;
377  virtual int GotoNextFile() = 0;
378  virtual VSIArchiveEntryFileOffset *GetFileOffset() = 0;
379  virtual GUIntBig GetFileSize() = 0;
380  virtual CPLString GetFileName() = 0;
381  virtual GIntBig GetModifiedTime() = 0;
382  virtual int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) = 0;
383 };
384 
385 class VSIArchiveFilesystemHandler : public VSIFilesystemHandler
386 {
387  CPL_DISALLOW_COPY_ASSIGN(VSIArchiveFilesystemHandler)
388 
389  protected:
390  CPLMutex *hMutex = nullptr;
391  /* We use a cache that contains the list of files contained in a VSIArchive
392  * file as */
393  /* unarchive.c is quite inefficient in listing them. This speeds up access
394  * to VSIArchive files */
395  /* containing ~1000 files like a CADRG product */
396  std::map<CPLString, VSIArchiveContent *> oFileList{};
397 
398  virtual const char *GetPrefix() = 0;
399  virtual std::vector<CPLString> GetExtensions() = 0;
400  virtual VSIArchiveReader *CreateReader(const char *pszArchiveFileName) = 0;
401 
402  public:
403  VSIArchiveFilesystemHandler();
404  virtual ~VSIArchiveFilesystemHandler();
405 
406  int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
407  int nFlags) override;
408  int Unlink(const char *pszFilename) override;
409  int Rename(const char *oldpath, const char *newpath) override;
410  int Mkdir(const char *pszDirname, long nMode) override;
411  int Rmdir(const char *pszDirname) override;
412  char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
413 
414  virtual const VSIArchiveContent *
415  GetContentOfArchive(const char *archiveFilename,
416  VSIArchiveReader *poReader = nullptr);
417  virtual char *SplitFilename(const char *pszFilename,
418  CPLString &osFileInArchive,
419  int bCheckMainFileExists);
420  virtual VSIArchiveReader *OpenArchiveFile(const char *archiveFilename,
421  const char *fileInArchiveName);
422  virtual int FindFileInArchive(const char *archiveFilename,
423  const char *fileInArchiveName,
424  const VSIArchiveEntry **archiveEntry);
425 
426  virtual bool IsLocal(const char *pszPath) override;
427  virtual bool
428  SupportsSequentialWrite(const char * /* pszPath */,
429  bool /* bAllowLocalTempFile */) override
430  {
431  return false;
432  }
433  virtual bool SupportsRandomWrite(const char * /* pszPath */,
434  bool /* bAllowLocalTempFile */) override
435  {
436  return false;
437  }
438 };
439 
440 /************************************************************************/
441 /* VSIDIR */
442 /************************************************************************/
443 
444 struct CPL_DLL VSIDIR
445 {
446  VSIDIR() = default;
447  virtual ~VSIDIR();
448 
449  virtual const VSIDIREntry *NextDirEntry() = 0;
450 
451  private:
452  VSIDIR(const VSIDIR &) = delete;
453  VSIDIR &operator=(const VSIDIR &) = delete;
454 };
455 
456 #endif /* #ifndef DOXYGEN_SKIP */
457 
458 VSIVirtualHandle CPL_DLL *
459 VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
461 VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
462  const GByte *pabyBeginningContent,
463  vsi_l_offset nCheatFileSize);
464 constexpr int VSI_CACHED_DEFAULT_CHUNK_SIZE = 32768;
465 VSIVirtualHandle CPL_DLL *
466 VSICreateCachedFile(VSIVirtualHandle *poBaseHandle,
467  size_t nChunkSize = VSI_CACHED_DEFAULT_CHUNK_SIZE,
468  size_t nCacheSize = 0);
469 
470 const int CPL_DEFLATE_TYPE_GZIP = 0;
471 const int CPL_DEFLATE_TYPE_ZLIB = 1;
472 const int CPL_DEFLATE_TYPE_RAW_DEFLATE = 2;
473 VSIVirtualHandle CPL_DLL *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
474  int nDeflateType,
475  int bAutoCloseBaseHandle);
476 
477 VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
478  int nDeflateType,
479  bool bAutoCloseBaseHandle, int nThreads,
480  size_t nChunkSize,
481  size_t nSOZIPIndexEltSize,
482  std::vector<uint8_t> *panSOZIPIndex);
483 
485 VSICreateUploadOnCloseFile(VSIVirtualHandleUniquePtr &&poWritableHandle,
486  VSIVirtualHandleUniquePtr &&poTmpFile,
487  const std::string &osTmpFilename);
488 
489 #endif /* ndef CPL_VSI_VIRTUAL_H_INCLUDED */
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:196
VSIVirtualHandleCloser::operator()
void operator()(VSIVirtualHandle *poHandle)
Operator () that closes and deletes the file handle.
Definition: cpl_vsi_virtual.h:129
cpl_vsi.h
VSIVirtualHandle::Close
virtual int Close()=0
Close file.
VSIRangeStatus
VSIRangeStatus
Range status.
Definition: cpl_vsi.h:181
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
VSI_RANGE_STATUS_UNKNOWN
@ VSI_RANGE_STATUS_UNKNOWN
Unknown.
Definition: cpl_vsi.h:183
VSIVirtualHandleCloser
Helper close to use with a std:unique_ptr<VSIVirtualHandle>, such as VSIVirtualHandleUniquePtr.
Definition: cpl_vsi_virtual.h:126
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1178
GUIntBig
unsigned long long GUIntBig
Large unsigned integer type (generally 64-bit unsigned integer type).
Definition: cpl_port.h:229
VSIDIREntry
Directory entry.
Definition: cpl_vsi.h:394
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:62
CPLError
void CPLError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt,...)
Report an error.
Definition: cpl_error.cpp:323
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
CPLE_NotSupported
#define CPLE_NotSupported
Not supported.
Definition: cpl_error.h:110
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:226
CPL_UNUSED
#define CPL_UNUSED
Qualifier for an argument that is unused.
Definition: cpl_port.h:985
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:384
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042