GDAL

cpl_vsil_curl_class.h
1/******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
14#define CPL_VSIL_CURL_CLASS_H_INCLUDED
15
16#ifdef HAVE_CURL
17
18#include "cpl_aws.h"
19#include "cpl_azure.h"
20#include "cpl_port.h"
21#include "cpl_json.h"
22#include "cpl_http.h"
23#include "cpl_string.h"
24#include "cpl_vsil_curl_priv.h"
25#include "cpl_mem_cache.h"
26#include "cpl_multiproc.h"
27
28#include "cpl_curl_priv.h"
29
30#include <algorithm>
31#include <atomic>
32#include <condition_variable>
33#include <set>
34#include <map>
35#include <memory>
36#include <mutex>
37#include <thread>
38#include <utility>
39
40// To avoid aliasing to CopyFile to CopyFileA on Windows
41#ifdef CopyFile
42#undef CopyFile
43#endif
44
46
47// Leave it for backward compatibility, but deprecate.
48#define HAVE_CURLINFO_REDIRECT_URL
49
50 void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
51
52 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
53 const char *const *papszOptions);
54
55 struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
56 const char *pszPath);
57
58 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
59 struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
60
61 namespace cpl
62{
63
64 typedef enum
65{
66 EXIST_UNKNOWN = -1,
67 EXIST_NO,
68 EXIST_YES,
69} ExistStatus;
70
71 class FileProp
72{
73 public:
74 unsigned int nGenerationAuthParameters = 0;
75 ExistStatus eExists = EXIST_UNKNOWN;
76 int nHTTPCode = 0;
77 vsi_l_offset fileSize = 0;
78 time_t mTime = 0;
79 time_t nExpireTimestampLocal = 0;
80 std::string osRedirectURL{};
81 bool bHasComputedFileSize = false;
82 bool bIsDirectory = false;
83 bool bIsAzureFolder = false;
84 int nMode = 0; // st_mode member of struct stat
85 bool bS3LikeRedirect = false;
86 std::string ETag{};
87};
88
89 struct CachedDirList
90{
91 bool bGotFileList = false;
92 unsigned int nGenerationAuthParameters = 0;
93 CPLStringList oFileList{}; /* only file name without path */
94};
95
96 struct WriteFuncStruct
97{
98 char *pBuffer = nullptr;
99 size_t nSize = 0;
100 bool bIsHTTP = false;
101 bool bMultiRange = false;
102 vsi_l_offset nStartOffset = 0;
103 vsi_l_offset nEndOffset = 0;
104 int nHTTPCode = 0; // potentially after redirect
105 int nFirstHTTPCode = 0; // the one of the redirect
106 vsi_l_offset nContentLength = 0;
107 bool bFoundContentRange = false;
108 bool bError = false;
109 bool bInterruptDownload = false;
110 bool bDetectRangeDownloadingError = false;
111 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
112
113 VSILFILE *fp = nullptr;
114 VSICurlReadCbkFunc pfnReadCbk = nullptr;
115 void *pReadCbkUserData = nullptr;
116 bool bInterrupted = false;
117};
118
119 struct PutData
120{
121 const GByte *pabyData = nullptr;
122 size_t nOff = 0;
123 size_t nTotalSize = 0;
124
125 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
126 void *instream)
127 {
128 PutData *poThis = static_cast<PutData *>(instream);
129 const size_t nSizeMax = size * nitems;
130 const size_t nSizeToWrite =
131 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
132 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
133 poThis->nOff += nSizeToWrite;
134 return nSizeToWrite;
135 }
136};
137
138/************************************************************************/
139/* VSICurlFilesystemHandler */
140/************************************************************************/
141
142 class VSICurlHandle;
143
144 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
145{
146 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
147
148 struct FilenameOffsetPair
149 {
150 std::string filename_;
151 vsi_l_offset offset_;
152
153 FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
154 : filename_(filename), offset_(offset)
155 {
156 }
157
158 bool operator==(const FilenameOffsetPair &other) const
159 {
160 return filename_ == other.filename_ && offset_ == other.offset_;
161 }
162 };
163
164 struct FilenameOffsetPairHasher
165 {
166 std::size_t operator()(const FilenameOffsetPair &k) const
167 {
168 return std::hash<std::string>()(k.filename_) ^
169 std::hash<vsi_l_offset>()(k.offset_);
170 }
171 };
172
173 using RegionCacheType = lru11::Cache<
174 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
175 std::unordered_map<
176 FilenameOffsetPair,
177 typename std::list<lru11::KeyValuePair<
178 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
179 FilenameOffsetPairHasher>>;
180
181 std::unique_ptr<RegionCacheType>
182 m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
183 // GetRegionCache();
184 RegionCacheType *GetRegionCache();
185
186 // LRU cache that just keeps in memory if this file system handler is
187 // spposed to know the file properties of a file. The actual cache is a
188 // shared one among all network file systems.
189 // The aim of that design is that invalidating /vsis3/foo results in
190 // /vsis3_streaming/foo to be invalidated as well.
191 lru11::Cache<std::string, bool> oCacheFileProp;
192
193 int nCachedFilesInDirList = 0;
194 lru11::Cache<std::string, CachedDirList> oCacheDirList;
195
196 char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
197 char *pszData, bool *pbGotFileList);
198
199 // Data structure and map to store regions that are in progress, to
200 // avoid simultaneous downloads of the same region in different threads
201 // Cf https://github.com/OSGeo/gdal/issues/8041
202 struct RegionInDownload
203 {
204 std::mutex oMutex{};
205 std::condition_variable oCond{};
206 bool bDownloadInProgress = false;
207 int nWaiters = 0;
208 std::string osData{};
209 };
210
211 std::mutex m_oMutex{};
212 std::map<std::string, std::unique_ptr<RegionInDownload>>
213 m_oMapRegionInDownload{};
214
215 protected:
216 CPLMutex *hMutex = nullptr;
217
218 virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
219 virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
220 bool *pbGotFileList);
221
222 void RegisterEmptyDir(const std::string &osDirname);
223
224 bool
225 AnalyseS3FileList(const std::string &osBaseURL, const char *pszXML,
226 CPLStringList &osFileList, int nMaxFiles,
227 const std::set<std::string> &oSetIgnoredStorageClasses,
228 bool &bIsTruncated);
229
230 void AnalyseSwiftFileList(const std::string &osBaseURL,
231 const std::string &osPrefix, const char *pszJson,
232 CPLStringList &osFileList, int nMaxFilesThisQuery,
233 int nMaxFiles, bool &bIsTruncated,
234 std::string &osNextMarker);
235
236 static const char *GetOptionsStatic();
237
238 VSICurlFilesystemHandlerBase();
239
240 public:
241 ~VSICurlFilesystemHandlerBase() override;
242
243 static bool IsAllowedFilename(const char *pszFilename);
244
245 VSIVirtualHandleUniquePtr Open(const char *pszFilename,
246 const char *pszAccess, bool bSetError,
247 CSLConstList /* papszOptions */) override;
248
249 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
250 int nFlags) override;
251 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
252 char **SiblingFiles(const char *pszFilename) override;
253
254 int HasOptimizedReadMultiRange(const char * /* pszPath */) override
255 {
256 return true;
257 }
258
259 const char *GetActualURL(const char *pszFilename) override;
260
261 const char *GetOptions() override;
262
263 char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
264 CSLConstList papszOptions) override;
265
266 char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
267 bool *pbGotFileList);
268 void InvalidateDirContent(const std::string &osDirname);
269
270 virtual const char *GetDebugKey() const = 0;
271
272 virtual std::string GetFSPrefix() const = 0;
273 virtual bool AllowCachedDataFor(const char *pszFilename);
274
275 bool IsLocal(const char * /* pszPath */) const override
276 {
277 return false;
278 }
279
280 virtual bool
281 SupportsSequentialWrite(const char * /* pszPath */,
282 bool /* bAllowLocalTempFile */) override
283 {
284 return false;
285 }
286
287 virtual bool SupportsRandomWrite(const char * /* pszPath */,
288 bool /* bAllowLocalTempFile */) override
289 {
290 return false;
291 }
292
293 std::shared_ptr<std::string> GetRegion(const char *pszURL,
294 vsi_l_offset nFileOffsetStart);
295
296 void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
297 size_t nSize, const char *pData);
298
299 std::pair<bool, std::string>
300 NotifyStartDownloadRegion(const std::string &osURL,
301 vsi_l_offset startOffset, int nBlocks);
302 void NotifyStopDownloadRegion(const std::string &osURL,
303 vsi_l_offset startOffset, int nBlocks,
304 const std::string &osData);
305
306 bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
307 void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
308 void InvalidateCachedData(const char *pszURL);
309
310 CURLM *GetCurlMultiHandleFor(const std::string &osURL);
311
312 virtual void ClearCache();
313 virtual void PartialClearCache(const char *pszFilename);
314
315 bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
316 void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
317 bool ExistsInCacheDirList(const std::string &osDirname, bool *pbIsDir);
318
319 virtual std::string GetURLFromFilename(const std::string &osFilename) const;
320
321 std::string
322 GetStreamingFilename(const std::string &osFilename) const override = 0;
323
324 static std::set<std::string> GetS3IgnoredStorageClasses();
325};
326
327 class VSICurlFilesystemHandler final : public VSICurlFilesystemHandlerBase
328{
329 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
330
331 public:
332 VSICurlFilesystemHandler() = default;
333
334 const char *GetDebugKey() const override
335 {
336 return "VSICURL";
337 }
338
339 std::string GetFSPrefix() const override
340 {
341 return "/vsicurl/";
342 }
343
344 std::string
345 GetStreamingFilename(const std::string &osFilename) const override;
346};
347
348/************************************************************************/
349/* VSICurlHandle */
350/************************************************************************/
351
352 class VSICurlHandle /* non final*/ : public VSIVirtualHandle
353{
354 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
355
356 protected:
357 VSICurlFilesystemHandlerBase *poFS = nullptr;
358
359 bool m_bCached = true;
360
361 mutable FileProp oFileProp{};
362
363 mutable std::mutex m_oMutex{};
364 std::string m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
365 char *m_pszURL = nullptr; // e.g "http://example.com/foo"
366 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
367
368 CPLStringList m_aosHTTPOptions{};
369 CPLHTTPRetryParameters
370 m_oRetryParameters; // must be initialized in constructor
371
372 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
373 int nBlocksToDownload = 1;
374
375 bool bStopOnInterruptUntilUninstall = false;
376 bool bInterrupted = false;
377 VSICurlReadCbkFunc pfnReadCbk = nullptr;
378 void *pReadCbkUserData = nullptr;
379
380 CPLStringList m_aosHeaders{};
381
382 void DownloadRegionPostProcess(const vsi_l_offset startOffset,
383 const int nBlocks, const char *pBuffer,
384 size_t nSize);
385
386 private:
387 vsi_l_offset curOffset = 0;
388
389 bool bEOF = false;
390 bool bError = false;
391
392 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
393
394 bool m_bUseHead = false;
395 bool m_bUseRedirectURLIfNoQueryStringParams = false;
396
397 mutable std::atomic<bool> m_bInterrupt = false;
398
399 // Specific to Planetary Computer signing:
400 // https://planetarycomputer.microsoft.com/docs/concepts/sas/
401 mutable bool m_bPlanetaryComputerURLSigning = false;
402 mutable std::string m_osPlanetaryComputerCollection{};
403 void ManagePlanetaryComputerSigning() const;
404
405 void UpdateQueryString() const;
406
407 int ReadMultiRangeSingleGet(int nRanges, void **ppData,
408 const vsi_l_offset *panOffsets,
409 const size_t *panSizes);
410 std::string GetRedirectURLIfValid(bool &bHasExpired,
411 CPLStringList &aosHTTPOptions) const;
412
413 void UpdateRedirectInfo(CURL *hCurlHandle,
414 const WriteFuncStruct &sWriteFuncHeaderData);
415
416 // Used by AdviseRead()
417 struct AdviseReadRange
418 {
419 bool bDone = false;
420 bool bToRetry = true;
421 double dfSleepDelay = 0.0;
422 std::mutex oMutex{};
423 std::condition_variable oCV{};
424 vsi_l_offset nStartOffset = 0;
425 size_t nSize = 0;
426 std::vector<GByte> abyData{};
427 CPLHTTPRetryContext retryContext;
428
429 explicit AdviseReadRange(const CPLHTTPRetryParameters &oRetryParameters)
430 : retryContext(oRetryParameters)
431 {
432 }
433
434 AdviseReadRange(const AdviseReadRange &) = delete;
435 AdviseReadRange &operator=(const AdviseReadRange &) = delete;
436 AdviseReadRange(AdviseReadRange &&) = delete;
437 AdviseReadRange &operator=(AdviseReadRange &&) = delete;
438 };
439
440 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
441 std::thread m_oThreadAdviseRead{};
442 CURLM *m_hCurlMultiHandleForAdviseRead = nullptr;
443
444 protected:
445 virtual struct curl_slist *GetCurlHeaders(const std::string & /*osVerb*/,
446 struct curl_slist *psHeaders)
447 {
448 return psHeaders;
449 }
450
451 virtual bool AllowAutomaticRedirection()
452 {
453 return true;
454 }
455
456 virtual bool CanRestartOnError(const char *, const char *, bool)
457 {
458 return false;
459 }
460
461 virtual bool UseLimitRangeGetInsteadOfHead()
462 {
463 return false;
464 }
465
466 virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
467 int /*response_code*/)
468 {
469 return false;
470 }
471
472 virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
473 {
474 }
475
476 void SetURL(const char *pszURL);
477
478 virtual bool Authenticate(const char * /* pszFilename */)
479 {
480 return false;
481 }
482
483 public:
484 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
485 const char *pszURLIn = nullptr);
486 ~VSICurlHandle() override;
487
488 int Seek(vsi_l_offset nOffset, int nWhence) override;
489 vsi_l_offset Tell() override;
490 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
491 int ReadMultiRange(int nRanges, void **ppData,
492 const vsi_l_offset *panOffsets,
493 const size_t *panSizes) override;
494 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
495 void ClearErr() override;
496 int Eof() override;
497 int Error() override;
498 int Flush() override;
499 int Close() override;
500
501 void Interrupt() override
502 {
503 m_bInterrupt = true;
504 }
505
506 bool HasPRead() const override
507 {
508 return true;
509 }
510
511 size_t PRead(void *pBuffer, size_t nSize,
512 vsi_l_offset nOffset) const override;
513
514 void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
515 const size_t *panSizes) override;
516
517 size_t GetAdviseReadTotalBytesLimit() const override;
518
519 bool IsKnownFileSize() const
520 {
521 return oFileProp.bHasComputedFileSize;
522 }
523
524 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
525
526 virtual vsi_l_offset GetFileSize(bool bSetError)
527 {
528 return GetFileSizeOrHeaders(bSetError, false);
529 }
530
531 bool Exists(bool bSetError);
532
533 bool IsDirectory() const
534 {
535 return oFileProp.bIsDirectory;
536 }
537
538 int GetMode() const
539 {
540 return oFileProp.nMode;
541 }
542
543 time_t GetMTime() const
544 {
545 return oFileProp.mTime;
546 }
547
548 const CPLStringList &GetHeaders()
549 {
550 return m_aosHeaders;
551 }
552
553 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
554 int bStopOnInterruptUntilUninstall);
555 int UninstallReadCbk();
556
557 const char *GetURL() const
558 {
559 return m_pszURL;
560 }
561
563 void SetCache(bool bCache)
564 {
565 m_bCached = bCache;
566 }
567};
568
569/************************************************************************/
570/* VSICurlFilesystemHandlerBaseWritable */
571/************************************************************************/
572
573 class VSICurlFilesystemHandlerBaseWritable /* non final */
574 : public VSICurlFilesystemHandlerBase
575{
576 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBaseWritable)
577
578 protected:
579 VSICurlFilesystemHandlerBaseWritable() = default;
580
581 virtual VSIVirtualHandleUniquePtr
582 CreateWriteHandle(const char *pszFilename, CSLConstList papszOptions) = 0;
583
584 public:
585 VSIVirtualHandleUniquePtr Open(const char *pszFilename,
586 const char *pszAccess, bool bSetError,
587 CSLConstList /* papszOptions */) override;
588
589 bool SupportsSequentialWrite(const char * /* pszPath */,
590 bool /* bAllowLocalTempFile */) override
591 {
592 return true;
593 }
594
595 bool SupportsRandomWrite(const char * /* pszPath */,
596 bool /* bAllowLocalTempFile */) override;
597};
598
599/************************************************************************/
600/* IVSIS3LikeFSHandler */
601/************************************************************************/
602
603 class IVSIS3LikeFSHandler /* non final */
604 : public VSICurlFilesystemHandlerBaseWritable
605{
606 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
607
608 virtual int MkdirInternal(const char *pszDirname, long nMode,
609 bool bDoStatCheck);
610
611 protected:
612 char **GetFileList(const char *pszFilename, int nMaxFiles,
613 bool *pbGotFileList) override;
614
615 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
616 bool bAllowNoObject) = 0;
617
618 virtual int CopyObject(const char *oldpath, const char *newpath,
619 CSLConstList papszMetadata);
620
621 int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
622
623 virtual bool
624 IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
625 {
626 return false;
627 }
628
629 IVSIS3LikeFSHandler() = default;
630
631 public:
632 int Unlink(const char *pszFilename) override;
633 int Mkdir(const char *pszDirname, long nMode) override;
634 int Rmdir(const char *pszDirname) override;
635 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
636 int nFlags) override;
637 int Rename(const char *oldpath, const char *newpath, GDALProgressFunc,
638 void *) override;
639
640 virtual int CopyFile(const char *pszSource, const char *pszTarget,
641 VSILFILE *fpSource, vsi_l_offset nSourceSize,
642 const char *const *papszOptions,
643 GDALProgressFunc pProgressFunc,
644 void *pProgressData) override;
645
646 virtual int DeleteObject(const char *pszFilename);
647
648 virtual int *DeleteObjectBatch(CSLConstList papszFilesOrDirs);
649
650 bool Sync(const char *pszSource, const char *pszTarget,
651 const char *const *papszOptions, GDALProgressFunc pProgressFunc,
652 void *pProgressData, char ***ppapszOutputs) override;
653
654 VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
655 const char *const *papszOptions) override;
656};
657
658/************************************************************************/
659/* IVSIS3LikeFSHandlerWithMultipartUpload */
660/************************************************************************/
661
662 class IVSIS3LikeFSHandlerWithMultipartUpload /* non final */
663 : public IVSIS3LikeFSHandler
664{
665 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandlerWithMultipartUpload)
666
667 protected:
668 IVSIS3LikeFSHandlerWithMultipartUpload() = default;
669
670 public:
671 virtual bool SupportsNonSequentialMultipartUpload() const
672 {
673 return true;
674 }
675
676 virtual bool SupportsParallelMultipartUpload() const
677 {
678 return true;
679 }
680
681 virtual bool SupportsMultipartAbort() const = 0;
682
683 size_t GetUploadChunkSizeInBytes(const char *pszFilename,
684 const char *pszSpecifiedValInBytes);
685
686 virtual int CopyFileRestartable(const char *pszSource,
687 const char *pszTarget,
688 const char *pszInputPayload,
689 char **ppszOutputPayload,
690 CSLConstList papszOptions,
691 GDALProgressFunc pProgressFunc,
692 void *pProgressData) override;
693
695 // Limit currently used by S3 and GS.
696 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
697 // and https://cloud.google.com/storage/quotas#requests
698 virtual int GetMaximumPartCount()
699 {
700 return 10000;
701 }
702
704 // Limit currently used by S3 and GS.
705 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
706 // and https://cloud.google.com/storage/quotas#requests
707 virtual int GetMinimumPartSizeInMiB()
708 {
709 return 5;
710 }
711
713 // Limit currently used by S3 and GS.
714 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
715 // and https://cloud.google.com/storage/quotas#requests
716 virtual int GetMaximumPartSizeInMiB()
717 {
718#if SIZEOF_VOIDP == 8
719 return 5 * 1024;
720#else
721 // Cannot be larger than 4, otherwise integer overflow would occur
722 // 1 GiB is the maximum reasonable value on a 32-bit machine
723 return 1 * 1024;
724#endif
725 }
726
728 virtual int GetDefaultPartSizeInMiB()
729 {
730 return 50;
731 }
732
733 virtual std::string
734 InitiateMultipartUpload(const std::string &osFilename,
735 IVSIS3LikeHandleHelper *poS3HandleHelper,
736 const CPLHTTPRetryParameters &oRetryParameters,
737 CSLConstList papszOptions);
738
739 virtual std::string
740 UploadPart(const std::string &osFilename, int nPartNumber,
741 const std::string &osUploadID, vsi_l_offset nPosition,
742 const void *pabyBuffer, size_t nBufferSize,
743 IVSIS3LikeHandleHelper *poS3HandleHelper,
744 const CPLHTTPRetryParameters &oRetryParameters,
745 CSLConstList papszOptions);
746
747 virtual bool CompleteMultipart(
748 const std::string &osFilename, const std::string &osUploadID,
749 const std::vector<std::string> &aosEtags, vsi_l_offset nTotalSize,
750 IVSIS3LikeHandleHelper *poS3HandleHelper,
751 const CPLHTTPRetryParameters &oRetryParameters);
752
753 virtual bool AbortMultipart(const std::string &osFilename,
754 const std::string &osUploadID,
755 IVSIS3LikeHandleHelper *poS3HandleHelper,
756 const CPLHTTPRetryParameters &oRetryParameters);
757
758 bool AbortPendingUploads(const char *pszFilename) override;
759
760 bool MultipartUploadGetCapabilities(int *pbNonSequentialUploadSupported,
761 int *pbParallelUploadSupported,
762 int *pbAbortSupported,
763 size_t *pnMinPartSize,
764 size_t *pnMaxPartSize,
765 int *pnMaxPartCount) override;
766
767 char *MultipartUploadStart(const char *pszFilename,
768 CSLConstList papszOptions) override;
769
770 char *MultipartUploadAddPart(const char *pszFilename,
771 const char *pszUploadId, int nPartNumber,
772 vsi_l_offset nFileOffset, const void *pData,
773 size_t nDataLength,
774 CSLConstList papszOptions) override;
775
776 bool MultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
777 size_t nPartIdsCount,
778 const char *const *apszPartIds,
779 vsi_l_offset nTotalSize,
780 CSLConstList papszOptions) override;
781
782 bool MultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
783 CSLConstList papszOptions) override;
784};
785
786/************************************************************************/
787/* IVSIS3LikeHandle */
788/************************************************************************/
789
790 class IVSIS3LikeHandle /* non final */ : public VSICurlHandle
791{
792 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
793
794 protected:
795 bool UseLimitRangeGetInsteadOfHead() override
796 {
797 return true;
798 }
799
800 bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
801 {
802 // A bit dirty, but on S3, a GET on a existing directory returns a 416
803 return response_code == 416 && EQUAL(pszVerb, "GET") &&
804 std::string(m_pszURL).back() == '/';
805 }
806
807 void ProcessGetFileSizeResult(const char *pszContent) override
808 {
809 oFileProp.bIsDirectory =
810 strstr(pszContent, "ListBucketResult") != nullptr;
811 }
812
813 public:
814 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
815 const char *pszFilename, const char *pszURLIn)
816 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
817 {
818 }
819
820 ~IVSIS3LikeHandle() override;
821};
822
823/************************************************************************/
824/* VSIMultipartWriteHandle */
825/************************************************************************/
826
827 class VSIMultipartWriteHandle final : public VSIVirtualHandle
828{
829 CPL_DISALLOW_COPY_ASSIGN(VSIMultipartWriteHandle)
830
831 IVSIS3LikeFSHandlerWithMultipartUpload *m_poFS = nullptr;
832 std::string m_osFilename{};
833 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
834 CPLStringList m_aosOptions{};
835 CPLStringList m_aosHTTPOptions{};
836 CPLHTTPRetryParameters m_oRetryParameters;
837
838 vsi_l_offset m_nCurOffset = 0;
839 size_t m_nBufferOff = 0;
840 size_t m_nBufferSize = 0;
841 bool m_bClosed = false;
842 GByte *m_pabyBuffer = nullptr;
843 std::string m_osUploadID{};
844 int m_nPartNumber = 0;
845 std::vector<std::string> m_aosEtags{};
846 bool m_bError = false;
847
848 WriteFuncStruct m_sWriteFuncHeaderData{};
849
850 bool UploadPart();
851 bool DoSinglePartPUT();
852
853 void InvalidateParentDirectory();
854
855 public:
856 VSIMultipartWriteHandle(IVSIS3LikeFSHandlerWithMultipartUpload *poFS,
857 const char *pszFilename,
858 IVSIS3LikeHandleHelper *poS3HandleHelper,
859 CSLConstList papszOptions);
860 ~VSIMultipartWriteHandle() override;
861
862 int Seek(vsi_l_offset nOffset, int nWhence) override;
863 vsi_l_offset Tell() override;
864 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
865 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
866
867 void ClearErr() override
868 {
869 }
870
871 int Error() override
872 {
873 return FALSE;
874 }
875
876 int Eof() override
877 {
878 return FALSE;
879 }
880
881 int Close() override;
882
883 bool IsOK()
884 {
885 return m_pabyBuffer != nullptr;
886 }
887};
888
889/************************************************************************/
890/* VSIChunkedWriteHandle() */
891/************************************************************************/
892
896 class VSIChunkedWriteHandle final : public VSIVirtualHandle
897{
898 CPL_DISALLOW_COPY_ASSIGN(VSIChunkedWriteHandle)
899
900 IVSIS3LikeFSHandler *m_poFS = nullptr;
901 std::string m_osFilename{};
902 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
903 CPLStringList m_aosOptions{};
904 CPLStringList m_aosHTTPOptions{};
905 CPLHTTPRetryParameters m_oRetryParameters;
906
907 vsi_l_offset m_nCurOffset = 0;
908 size_t m_nBufferOff = 0;
909 bool m_bError = false;
910 bool m_bClosed = false;
911
912 CURLM *m_hCurlMulti = nullptr;
913 CURL *m_hCurl = nullptr;
914 const void *m_pBuffer = nullptr;
915 std::string m_osCurlErrBuf{};
916 size_t m_nChunkedBufferOff = 0;
917 size_t m_nChunkedBufferSize = 0;
918 size_t m_nWrittenInPUT = 0;
919
920 WriteFuncStruct m_sWriteFuncHeaderData{};
921
922 static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
923 size_t nitems, void *instream);
924 int FinishChunkedTransfer();
925
926 bool DoEmptyPUT();
927
928 void InvalidateParentDirectory();
929
930 public:
931 VSIChunkedWriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
932 IVSIS3LikeHandleHelper *poS3HandleHelper,
933 CSLConstList papszOptions);
934 ~VSIChunkedWriteHandle() override;
935
936 int Seek(vsi_l_offset nOffset, int nWhence) override;
937 vsi_l_offset Tell() override;
938 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
939 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
940
941 void ClearErr() override
942 {
943 }
944
945 int Error() override
946 {
947 return FALSE;
948 }
949
950 int Eof() override
951 {
952 return FALSE;
953 }
954
955 int Close() override;
956};
957
958/************************************************************************/
959/* VSIAppendWriteHandle */
960/************************************************************************/
961
962 class VSIAppendWriteHandle CPL_NON_FINAL : public VSIVirtualHandle
963{
964 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
965
966 protected:
967 VSICurlFilesystemHandlerBase *m_poFS = nullptr;
968 std::string m_osFSPrefix{};
969 std::string m_osFilename{};
970 CPLHTTPRetryParameters m_oRetryParameters{};
971
972 vsi_l_offset m_nCurOffset = 0;
973 int m_nBufferOff = 0;
974 int m_nBufferSize = 0;
975 int m_nBufferOffReadCallback = 0;
976 bool m_bClosed = false;
977 GByte *m_pabyBuffer = nullptr;
978 bool m_bError = false;
979
980 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
981 void *instream);
982 virtual bool Send(bool bIsLastBlock) = 0;
983
984 public:
985 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
986 const char *pszFSPrefix, const char *pszFilename,
987 int nChunkSize);
988 ~VSIAppendWriteHandle() override;
989
990 int Seek(vsi_l_offset nOffset, int nWhence) override;
991 vsi_l_offset Tell() override;
992 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
993 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
994
995 void ClearErr() override
996 {
997 }
998
999 int Error() override
1000 {
1001 return FALSE;
1002 }
1003
1004 int Eof() override
1005 {
1006 return FALSE;
1007 }
1008
1009 int Close() override;
1010
1011 bool IsOK()
1012 {
1013 return m_pabyBuffer != nullptr;
1014 }
1015};
1016
1017/************************************************************************/
1018/* VSIDIRWithMissingDirSynthesis */
1019/************************************************************************/
1020
1021 struct VSIDIRWithMissingDirSynthesis /* non final */ : public VSIDIR
1022{
1023 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
1024
1025 protected:
1026 ~VSIDIRWithMissingDirSynthesis() override;
1027
1028 std::vector<std::string> m_aosSubpathsStack{};
1029
1030 void SynthetizeMissingDirectories(const std::string &osCurSubdir,
1031 bool bAddEntryForThisSubdir);
1032};
1033
1034/************************************************************************/
1035/* VSIDIRS3Like */
1036/************************************************************************/
1037
1038 struct VSIDIRS3Like /* non final */ : public VSIDIRWithMissingDirSynthesis
1039{
1040 const std::string m_osDirName;
1041
1042 int nRecurseDepth = 0;
1043
1044 std::string osNextMarker{};
1045 int nPos = 0;
1046
1047 std::string osBucket{};
1048 std::string osObjectKey{};
1049 VSICurlFilesystemHandlerBase *poFS = nullptr;
1050 IVSIS3LikeFSHandler *poS3FS = nullptr;
1051 std::unique_ptr<IVSIS3LikeHandleHelper> poHandleHelper{};
1052 int nMaxFiles = 0;
1053 bool bCacheEntries = true;
1054 bool m_bSynthetizeMissingDirectories = false;
1055 std::string m_osFilterPrefix{};
1056
1057 // used when listing only the file system prefix
1058 std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> m_subdir{nullptr,
1059 VSICloseDir};
1060
1061 VSIDIRS3Like(const std::string &osDirName, IVSIS3LikeFSHandler *poFSIn)
1062 : m_osDirName(osDirName), poFS(poFSIn), poS3FS(poFSIn)
1063 {
1064 }
1065
1066 VSIDIRS3Like(const std::string &osDirName,
1067 VSICurlFilesystemHandlerBase *poFSIn)
1068 : m_osDirName(osDirName), poFS(poFSIn)
1069 {
1070 }
1071
1072 VSIDIRS3Like(const VSIDIRS3Like &) = delete;
1073 VSIDIRS3Like &operator=(const VSIDIRS3Like &) = delete;
1074
1075 const VSIDIREntry *NextDirEntry() override;
1076
1077 virtual bool IssueListDir() = 0;
1078 void clear();
1079};
1080
1081/************************************************************************/
1082/* CurlRequestHelper */
1083/************************************************************************/
1084
1085 struct CurlRequestHelper
1086{
1087 WriteFuncStruct sWriteFuncData{};
1088 WriteFuncStruct sWriteFuncHeaderData{};
1089 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1090
1091 CurlRequestHelper();
1092 ~CurlRequestHelper();
1093 long perform(CURL *hCurlHandle,
1094 struct curl_slist *headers, // ownership transferred
1095 VSICurlFilesystemHandlerBase *poFS,
1096 IVSIS3LikeHandleHelper *poS3HandleHelper);
1097};
1098
1099/************************************************************************/
1100/* NetworkStatisticsLogger */
1101/************************************************************************/
1102
1103 class NetworkStatisticsLogger
1104{
1105 static int gnEnabled;
1106 static NetworkStatisticsLogger gInstance;
1107
1108 NetworkStatisticsLogger() = default;
1109
1110 std::mutex m_mutex{};
1111
1112 struct Counters
1113 {
1114 GIntBig nHEAD = 0;
1115 GIntBig nGET = 0;
1116 GIntBig nPUT = 0;
1117 GIntBig nPOST = 0;
1118 GIntBig nDELETE = 0;
1119 GIntBig nGETDownloadedBytes = 0;
1120 GIntBig nPUTUploadedBytes = 0;
1121 GIntBig nPOSTDownloadedBytes = 0;
1122 GIntBig nPOSTUploadedBytes = 0;
1123 };
1124
1125 enum class ContextPathType
1126 {
1127 FILESYSTEM,
1128 FILE,
1129 ACTION,
1130 };
1131
1132 struct ContextPathItem
1133 {
1134 ContextPathType eType;
1135 std::string osName;
1136
1137 ContextPathItem(ContextPathType eTypeIn, const std::string &osNameIn)
1138 : eType(eTypeIn), osName(osNameIn)
1139 {
1140 }
1141
1142 bool operator<(const ContextPathItem &other) const
1143 {
1144 if (static_cast<int>(eType) < static_cast<int>(other.eType))
1145 return true;
1146 if (static_cast<int>(eType) > static_cast<int>(other.eType))
1147 return false;
1148 return osName < other.osName;
1149 }
1150 };
1151
1152 struct Stats
1153 {
1154 Counters counters{};
1155 std::map<ContextPathItem, Stats> children{};
1156
1157 void AsJSON(CPLJSONObject &oJSON) const;
1158 };
1159
1160 // Workaround bug in Coverity Scan
1161 // coverity[generated_default_constructor_used_in_field_initializer]
1162 Stats m_stats{};
1163 std::map<GIntBig, std::vector<ContextPathItem>>
1164 m_mapThreadIdToContextPath{};
1165
1166 static void ReadEnabled();
1167
1168 std::vector<Counters *> GetCountersForContext();
1169
1170 public:
1171 static inline bool IsEnabled()
1172 {
1173 if (gnEnabled < 0)
1174 {
1175 ReadEnabled();
1176 }
1177 return gnEnabled == TRUE;
1178 }
1179
1180 static void EnterFileSystem(const char *pszName);
1181
1182 static void LeaveFileSystem();
1183
1184 static void EnterFile(const char *pszName);
1185
1186 static void LeaveFile();
1187
1188 static void EnterAction(const char *pszName);
1189
1190 static void LeaveAction();
1191
1192 static void LogHEAD();
1193
1194 static void LogGET(size_t nDownloadedBytes);
1195
1196 static void LogPUT(size_t nUploadedBytes);
1197
1198 static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
1199
1200 static void LogDELETE();
1201
1202 static void Reset();
1203
1204 static std::string GetReportAsSerializedJSON();
1205};
1206
1207 struct NetworkStatisticsFileSystem
1208{
1209 inline explicit NetworkStatisticsFileSystem(const char *pszName)
1210 {
1211 NetworkStatisticsLogger::EnterFileSystem(pszName);
1212 }
1213
1214 inline ~NetworkStatisticsFileSystem()
1215 {
1216 NetworkStatisticsLogger::LeaveFileSystem();
1217 }
1218};
1219
1220 struct NetworkStatisticsFile
1221{
1222 inline explicit NetworkStatisticsFile(const char *pszName)
1223 {
1224 NetworkStatisticsLogger::EnterFile(pszName);
1225 }
1226
1227 inline ~NetworkStatisticsFile()
1228 {
1229 NetworkStatisticsLogger::LeaveFile();
1230 }
1231};
1232
1233 struct NetworkStatisticsAction
1234{
1235 inline explicit NetworkStatisticsAction(const char *pszName)
1236 {
1237 NetworkStatisticsLogger::EnterAction(pszName);
1238 }
1239
1240 inline ~NetworkStatisticsAction()
1241 {
1242 NetworkStatisticsLogger::LeaveAction();
1243 }
1244};
1245
1246} // namespace cpl
1247
1248 int VSICURLGetDownloadChunkSize();
1249
1250 void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp,
1251 VSICurlReadCbkFunc pfnReadCbk,
1252 void *pReadCbkUserData);
1253 size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
1254 void *req);
1255 void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr,
1256 std::atomic<bool> *pbInterrupt = nullptr);
1257 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
1258
1259 int VSICurlParseUnixPermissions(const char *pszPermissions);
1260
1261// Cache of file properties (size, etc.)
1262 bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1263 void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1264 void VSICURLInvalidateCachedFileProp(const char *pszURL);
1265 void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
1266 void VSICURLDestroyCacheFileProp();
1267
1268 void VSICURLMultiCleanup(CURLM *hCurlMultiHandle);
1269
1271
1272#endif // HAVE_CURL
1273
1274#endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition cpl_json.h:56
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition cpl_string.h:458
cpl_http.h
Interface for downloading HTTP, FTP documents.
cpl_json.h
Interface for read and write JSON documents.
cpl_port.h
Core portability definitions for CPL.
CPL_NON_FINAL
#define CPL_NON_FINAL
Mark that a class is explicitly recognized as non-final.
Definition cpl_port.h:929
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition cpl_port.h:541
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:936
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1087
GByte
unsigned char GByte
Unsigned byte type.
Definition cpl_port.h:175
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition cpl_port.h:205
cpl_string.h
Various convenience functions for working with strings and string lists.
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition cpl_vsi.h:195
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition cpl_vsi.h:138
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition cpl_vsi.h:396
VSICloseDir
void VSICloseDir(VSIDIR *dir)
Close a directory.
Definition cpl_vsil.cpp:599
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition cpl_vsi.h:136
VSIDIREntry
Directory entry.
Definition cpl_vsi.h:408
VSIVirtualHandle
Virtual file handle.
Definition cpl_vsi_virtual.h:48