GDAL
cpl_vsi_virtual.h
1 /******************************************************************************
2  * $Id$
3  *
4  * Project: VSI Virtual File System
5  * Purpose: Declarations for classes related to the virtual filesystem.
6  * These would only be normally required by applications implementing
7  * their own virtual file system classes which should be rare.
8  * The class interface may be fragile through versions.
9  * Author: Frank Warmerdam, warmerdam@pobox.com
10  *
11  ******************************************************************************
12  * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
13  * Copyright (c) 2010-2014, Even Rouault <even dot rouault at spatialys.com>
14  *
15  * Permission is hereby granted, free of charge, to any person obtaining a
16  * copy of this software and associated documentation files (the "Software"),
17  * to deal in the Software without restriction, including without limitation
18  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
19  * and/or sell copies of the Software, and to permit persons to whom the
20  * Software is furnished to do so, subject to the following conditions:
21  *
22  * The above copyright notice and this permission notice shall be included
23  * in all copies or substantial portions of the Software.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
26  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
28  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
31  * DEALINGS IN THE SOFTWARE.
32  ****************************************************************************/
33 
34 #ifndef CPL_VSI_VIRTUAL_H_INCLUDED
35 #define CPL_VSI_VIRTUAL_H_INCLUDED
36 
37 #include "cpl_vsi.h"
38 #include "cpl_vsi_error.h"
39 #include "cpl_string.h"
40 #include "cpl_multiproc.h"
41 
42 #include <map>
43 #include <vector>
44 #include <string>
45 
46 // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
47 #ifdef GetDiskFreeSpace
48 #undef GetDiskFreeSpace
49 #endif
50 
51 // To avoid aliasing to CopyFile to CopyFileA on Windows
52 #ifdef CopyFile
53 #undef CopyFile
54 #endif
55 
56 /************************************************************************/
57 /* VSIVirtualHandle */
58 /************************************************************************/
59 
61 struct CPL_DLL VSIVirtualHandle
62 {
63  public:
64  virtual int Seek(vsi_l_offset nOffset, int nWhence) = 0;
65  virtual vsi_l_offset Tell() = 0;
66  virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) = 0;
67  virtual int ReadMultiRange(int nRanges, void **ppData,
68  const vsi_l_offset *panOffsets,
69  const size_t *panSizes);
70 
83  virtual void AdviseRead(CPL_UNUSED int nRanges,
84  CPL_UNUSED const vsi_l_offset *panOffsets,
85  CPL_UNUSED const size_t *panSizes)
86  {
87  }
88 
89  virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount) = 0;
90  virtual int Eof() = 0;
91  virtual int Flush()
92  {
93  return 0;
94  }
95  virtual int Close() = 0;
96  // Base implementation that only supports file extension.
97  virtual int Truncate(vsi_l_offset nNewSize);
98  virtual void *GetNativeFileDescriptor()
99  {
100  return nullptr;
101  }
102  virtual VSIRangeStatus GetRangeStatus(CPL_UNUSED vsi_l_offset nOffset,
103  CPL_UNUSED vsi_l_offset nLength)
104  {
106  }
107  virtual bool HasPRead() const;
108  virtual size_t PRead(void *pBuffer, size_t nSize,
109  vsi_l_offset nOffset) const;
110 
111  // NOTE: when adding new methods, besides the "actual" implementations,
112  // also consider the VSICachedFile one.
113 
114  virtual ~VSIVirtualHandle()
115  {
116  }
117 };
118 
119 /************************************************************************/
120 /* VSIVirtualHandleCloser */
121 /************************************************************************/
122 
126 {
128  void operator()(VSIVirtualHandle *poHandle)
129  {
130  if (poHandle)
131  {
132  poHandle->Close();
133  delete poHandle;
134  }
135  }
136 };
137 
139 typedef std::unique_ptr<VSIVirtualHandle, VSIVirtualHandleCloser>
140  VSIVirtualHandleUniquePtr;
141 
142 /************************************************************************/
143 /* VSIFilesystemHandler */
144 /************************************************************************/
145 
146 #ifndef DOXYGEN_SKIP
147 class CPL_DLL VSIFilesystemHandler
148 {
149 
150  public:
151  virtual ~VSIFilesystemHandler()
152  {
153  }
154 
155  VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess);
156 
157  virtual VSIVirtualHandle *Open(const char *pszFilename,
158  const char *pszAccess, bool bSetError,
159  CSLConstList papszOptions) = 0;
160  virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
161  int nFlags) = 0;
162  virtual int Unlink(const char *pszFilename)
163  {
164  (void)pszFilename;
165  errno = ENOENT;
166  return -1;
167  }
168  virtual int *UnlinkBatch(CSLConstList papszFiles);
169  virtual int Mkdir(const char *pszDirname, long nMode)
170  {
171  (void)pszDirname;
172  (void)nMode;
173  errno = ENOENT;
174  return -1;
175  }
176  virtual int Rmdir(const char *pszDirname)
177  {
178  (void)pszDirname;
179  errno = ENOENT;
180  return -1;
181  }
182  virtual int RmdirRecursive(const char *pszDirname);
183  virtual char **ReadDir(const char *pszDirname)
184  {
185  (void)pszDirname;
186  return nullptr;
187  }
188  virtual char **ReadDirEx(const char *pszDirname, int /* nMaxFiles */)
189  {
190  return ReadDir(pszDirname);
191  }
192  virtual char **SiblingFiles(const char * /*pszFilename*/)
193  {
194  return nullptr;
195  }
196  virtual int Rename(const char *oldpath, const char *newpath)
197  {
198  (void)oldpath;
199  (void)newpath;
200  errno = ENOENT;
201  return -1;
202  }
203  virtual int IsCaseSensitive(const char *pszFilename)
204  {
205  (void)pszFilename;
206  return TRUE;
207  }
208  virtual GIntBig GetDiskFreeSpace(const char * /* pszDirname */)
209  {
210  return -1;
211  }
212  virtual int SupportsSparseFiles(const char * /* pszPath */)
213  {
214  return FALSE;
215  }
216  virtual int HasOptimizedReadMultiRange(const char * /* pszPath */)
217  {
218  return FALSE;
219  }
220  virtual const char *GetActualURL(const char * /*pszFilename*/)
221  {
222  return nullptr;
223  }
224  virtual const char *GetOptions()
225  {
226  return nullptr;
227  }
228  virtual char *GetSignedURL(const char * /*pszFilename*/,
229  CSLConstList /* papszOptions */)
230  {
231  return nullptr;
232  }
233  virtual bool Sync(const char *pszSource, const char *pszTarget,
234  const char *const *papszOptions,
235  GDALProgressFunc pProgressFunc, void *pProgressData,
236  char ***ppapszOutputs);
237 
238  virtual int CopyFile(const char *pszSource, const char *pszTarget,
239  VSILFILE *fpSource, vsi_l_offset nSourceSize,
240  const char *const *papszOptions,
241  GDALProgressFunc pProgressFunc, void *pProgressData);
242 
243  virtual VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
244  const char *const *papszOptions);
245 
246  virtual char **GetFileMetadata(const char *pszFilename,
247  const char *pszDomain,
248  CSLConstList papszOptions);
249 
250  virtual bool SetFileMetadata(const char *pszFilename,
251  CSLConstList papszMetadata,
252  const char *pszDomain,
253  CSLConstList papszOptions);
254 
255  virtual bool AbortPendingUploads(const char * /*pszFilename*/)
256  {
257  return true;
258  }
259 
260  virtual std::string
261  GetStreamingFilename(const std::string &osFilename) const
262  {
263  return osFilename;
264  }
265 
266  virtual bool IsLocal(const char * /* pszPath */)
267  {
268  return true;
269  }
270  virtual bool SupportsSequentialWrite(const char * /* pszPath */,
271  bool /* bAllowLocalTempFile */)
272  {
273  return true;
274  }
275  virtual bool SupportsRandomWrite(const char * /* pszPath */,
276  bool /* bAllowLocalTempFile */)
277  {
278  return true;
279  }
280  virtual bool SupportsRead(const char * /* pszPath */)
281  {
282  return true;
283  }
284 
285  virtual VSIFilesystemHandler *Duplicate(const char * /* pszPrefix */)
286  {
287  CPLError(CE_Failure, CPLE_NotSupported,
288  "Duplicate() not supported on this file system");
289  return nullptr;
290  }
291 };
292 #endif /* #ifndef DOXYGEN_SKIP */
293 
294 /************************************************************************/
295 /* VSIFileManager */
296 /************************************************************************/
297 
298 #ifndef DOXYGEN_SKIP
299 class CPL_DLL VSIFileManager
300 {
301  private:
302  VSIFilesystemHandler *poDefaultHandler = nullptr;
303  std::map<std::string, VSIFilesystemHandler *> oHandlers{};
304 
305  VSIFileManager();
306 
307  static VSIFileManager *Get();
308 
309  CPL_DISALLOW_COPY_ASSIGN(VSIFileManager)
310 
311  public:
312  ~VSIFileManager();
313 
314  static VSIFilesystemHandler *GetHandler(const char *);
315  static void InstallHandler(const std::string &osPrefix,
316  VSIFilesystemHandler *);
317  /* RemoveHandler is never defined. */
318  /* static void RemoveHandler( const std::string& osPrefix ); */
319 
320  static char **GetPrefixes();
321 };
322 #endif /* #ifndef DOXYGEN_SKIP */
323 
324 /************************************************************************/
325 /* ==================================================================== */
326 /* VSIArchiveFilesystemHandler */
327 /* ==================================================================== */
328 /************************************************************************/
329 
330 #ifndef DOXYGEN_SKIP
331 
332 class VSIArchiveEntryFileOffset
333 {
334  public:
335  virtual ~VSIArchiveEntryFileOffset();
336 };
337 
338 typedef struct
339 {
340  char *fileName;
341  vsi_l_offset uncompressed_size;
342  VSIArchiveEntryFileOffset *file_pos;
343  int bIsDir;
344  GIntBig nModifiedTime;
345 } VSIArchiveEntry;
346 
347 class VSIArchiveContent
348 {
349  public:
350  time_t mTime = 0;
351  vsi_l_offset nFileSize = 0;
352  int nEntries = 0;
353  VSIArchiveEntry *entries = nullptr;
354 
355  ~VSIArchiveContent();
356 };
357 
358 class VSIArchiveReader
359 {
360  public:
361  virtual ~VSIArchiveReader();
362 
363  virtual int GotoFirstFile() = 0;
364  virtual int GotoNextFile() = 0;
365  virtual VSIArchiveEntryFileOffset *GetFileOffset() = 0;
366  virtual GUIntBig GetFileSize() = 0;
367  virtual CPLString GetFileName() = 0;
368  virtual GIntBig GetModifiedTime() = 0;
369  virtual int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) = 0;
370 };
371 
372 class VSIArchiveFilesystemHandler : public VSIFilesystemHandler
373 {
374  CPL_DISALLOW_COPY_ASSIGN(VSIArchiveFilesystemHandler)
375 
376  protected:
377  CPLMutex *hMutex = nullptr;
378  /* We use a cache that contains the list of files contained in a VSIArchive
379  * file as */
380  /* unarchive.c is quite inefficient in listing them. This speeds up access
381  * to VSIArchive files */
382  /* containing ~1000 files like a CADRG product */
383  std::map<CPLString, VSIArchiveContent *> oFileList{};
384 
385  virtual const char *GetPrefix() = 0;
386  virtual std::vector<CPLString> GetExtensions() = 0;
387  virtual VSIArchiveReader *CreateReader(const char *pszArchiveFileName) = 0;
388 
389  public:
390  VSIArchiveFilesystemHandler();
391  virtual ~VSIArchiveFilesystemHandler();
392 
393  int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
394  int nFlags) override;
395  int Unlink(const char *pszFilename) override;
396  int Rename(const char *oldpath, const char *newpath) override;
397  int Mkdir(const char *pszDirname, long nMode) override;
398  int Rmdir(const char *pszDirname) override;
399  char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
400 
401  virtual const VSIArchiveContent *
402  GetContentOfArchive(const char *archiveFilename,
403  VSIArchiveReader *poReader = nullptr);
404  virtual char *SplitFilename(const char *pszFilename,
405  CPLString &osFileInArchive,
406  int bCheckMainFileExists);
407  virtual VSIArchiveReader *OpenArchiveFile(const char *archiveFilename,
408  const char *fileInArchiveName);
409  virtual int FindFileInArchive(const char *archiveFilename,
410  const char *fileInArchiveName,
411  const VSIArchiveEntry **archiveEntry);
412 
413  virtual bool IsLocal(const char *pszPath) override;
414  virtual bool
415  SupportsSequentialWrite(const char * /* pszPath */,
416  bool /* bAllowLocalTempFile */) override
417  {
418  return false;
419  }
420  virtual bool SupportsRandomWrite(const char * /* pszPath */,
421  bool /* bAllowLocalTempFile */) override
422  {
423  return false;
424  }
425 };
426 
427 /************************************************************************/
428 /* VSIDIR */
429 /************************************************************************/
430 
431 struct CPL_DLL VSIDIR
432 {
433  VSIDIR() = default;
434  virtual ~VSIDIR();
435 
436  virtual const VSIDIREntry *NextDirEntry() = 0;
437 
438  private:
439  VSIDIR(const VSIDIR &) = delete;
440  VSIDIR &operator=(const VSIDIR &) = delete;
441 };
442 
443 #endif /* #ifndef DOXYGEN_SKIP */
444 
445 VSIVirtualHandle CPL_DLL *
446 VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
448 VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
449  const GByte *pabyBeginningContent,
450  vsi_l_offset nCheatFileSize);
451 VSIVirtualHandle CPL_DLL *VSICreateCachedFile(VSIVirtualHandle *poBaseHandle,
452  size_t nChunkSize = 32768,
453  size_t nCacheSize = 0);
454 
455 const int CPL_DEFLATE_TYPE_GZIP = 0;
456 const int CPL_DEFLATE_TYPE_ZLIB = 1;
457 const int CPL_DEFLATE_TYPE_RAW_DEFLATE = 2;
458 VSIVirtualHandle CPL_DLL *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
459  int nDeflateType,
460  int bAutoCloseBaseHandle);
461 
462 VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
463  int nDeflateType,
464  bool bAutoCloseBaseHandle, int nThreads,
465  size_t nChunkSize,
466  size_t nSOZIPIndexEltSize,
467  std::vector<uint8_t> *panSOZIPIndex);
468 
469 VSIVirtualHandle *VSICreateUploadOnCloseFile(VSIVirtualHandle *poBaseHandle);
470 
471 #endif /* ndef CPL_VSI_VIRTUAL_H_INCLUDED */
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:205
VSIVirtualHandleCloser::operator()
void operator()(VSIVirtualHandle *poHandle)
Operator () that closes and deletes the file handle.
Definition: cpl_vsi_virtual.h:128
cpl_vsi.h
VSIVirtualHandle::Close
virtual int Close()=0
Close file.
VSIRangeStatus
VSIRangeStatus
Range status.
Definition: cpl_vsi.h:181
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
VSI_RANGE_STATUS_UNKNOWN
@ VSI_RANGE_STATUS_UNKNOWN
Unknown.
Definition: cpl_vsi.h:183
VSIVirtualHandleCloser
Helper close to use with a std:unique_ptr<VSIVirtualHandle>, such as VSIVirtualHandleUniquePtr.
Definition: cpl_vsi_virtual.h:125
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1195
GUIntBig
unsigned long long GUIntBig
Large unsigned integer type (generally 64-bit unsigned integer type).
Definition: cpl_port.h:238
VSIDIREntry
Directory entry.
Definition: cpl_vsi.h:392
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:61
CPLError
void CPLError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt,...)
Report an error.
Definition: cpl_error.cpp:329
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
CPLE_NotSupported
#define CPLE_NotSupported
Not supported.
Definition: cpl_error.h:110
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:235
CPL_UNUSED
#define CPL_UNUSED
Qualifier for an argument that is unused.
Definition: cpl_port.h:994
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:382
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1051