GDAL

cpl_http.h
Go to the documentation of this file.
1/******************************************************************************
2 *
3 * Project: Common Portability Library
4 * Purpose: Function wrapper for libcurl HTTP access.
5 * Author: Frank Warmerdam, warmerdam@pobox.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2006, Frank Warmerdam
9 * Copyright (c) 2009, Even Rouault <even dot rouault at spatialys.com>
10 *
11 * SPDX-License-Identifier: MIT
12 ****************************************************************************/
13
14#ifndef CPL_HTTP_H_INCLUDED
15#define CPL_HTTP_H_INCLUDED
16
17#include "cpl_conv.h"
18#include "cpl_string.h"
19#include "cpl_progress.h"
20#include "cpl_vsi.h"
21
29#ifndef CPL_HTTP_MAX_RETRY
30#define CPL_HTTP_MAX_RETRY 0
31#endif
32
33#ifndef CPL_HTTP_RETRY_DELAY
34#define CPL_HTTP_RETRY_DELAY 30.0
35#endif
38 CPL_C_START
39
41 typedef struct
42{
char **papszHeaders;
44
GByte *pabyData; int nDataLen;
47} CPLMimePart;
48
50 typedef struct
51{
53 int nStatus;
54
56 char *pszContentType;
57
59 char *pszErrBuf;
60
62 int nDataLen;
64 int nDataAlloc;
65
67 GByte *pabyData;
68
70 char **papszHeaders;
71
73 int nMimePartCount;
74
76 CPLMimePart *pasMimePart;
77
78} CPLHTTPResult;
79
81 typedef size_t (*CPLHTTPFetchWriteFunc)(void *pBuffer, size_t nSize,
82 size_t nMemb, void *pWriteArg);
85 int CPL_DLL CPLHTTPEnabled(void);
86 CPLHTTPResult CPL_DLL *CPLHTTPFetch(const char *pszURL,
87 CSLConstList papszOptions);
88 CPLHTTPResult CPL_DLL *
89 CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
90 GDALProgressFunc pfnProgress, void *pProgressArg,
91 CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg);
92 CPLHTTPResult CPL_DLL **CPLHTTPMultiFetch(const char *const *papszURL,
93 int nURLCount, int nMaxSimultaneous,
94 CSLConstList papszOptions);
95
96 void CPL_DLL CPLHTTPCleanup(void);
97 void CPL_DLL CPLHTTPDestroyResult(CPLHTTPResult *psResult);
98 void CPL_DLL CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount);
99 int CPL_DLL CPLHTTPParseMultipartMime(CPLHTTPResult *psResult);
100
101 void CPL_DLL CPLHTTPSetDefaultUserAgent(const char *pszUserAgent);
102
103/* -------------------------------------------------------------------- */
104/* To install an alternate network layer to the default Curl one */
105/* -------------------------------------------------------------------- */
124 typedef CPLHTTPResult *(*CPLHTTPFetchCallbackFunc)(
125 const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress,
126 void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg,
127 void *pUserData);
128
129 void CPL_DLL CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc,
130 void *pUserData);
131
132 int CPL_DLL CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc,
133 void *pUserData);
134 int CPL_DLL CPLHTTPPopFetchCallback(void);
135
136/* -------------------------------------------------------------------- */
137/* The following is related to OAuth2 authorization around */
138/* google services like fusion tables, and potentially others */
139/* in the future. Code in cpl_google_oauth2.cpp. */
140/* */
141/* These services are built on CPL HTTP services. */
142/* -------------------------------------------------------------------- */
143
144 char CPL_DLL *GOA2GetAuthorizationURL(const char *pszScope);
145 char CPL_DLL *GOA2GetRefreshToken(const char *pszAuthToken,
146 const char *pszScope);
147 char CPL_DLL *GOA2GetAccessToken(const char *pszRefreshToken,
148 const char *pszScope);
149
150 char CPL_DLL **GOA2GetAccessTokenFromServiceAccount(
151 const char *pszPrivateKey, const char *pszClientEmail, const char *pszScope,
152 CSLConstList papszAdditionalClaims, CSLConstList papszOptions);
153
154 char CPL_DLL **GOA2GetAccessTokenFromCloudEngineVM(CSLConstList papszOptions);
155
156 CPL_C_END
157
158#if defined(__cplusplus) && !defined(CPL_SUPRESS_CPLUSPLUS)
160// Not sure if this belong here, used in cpl_http.cpp, cpl_vsil_curl.cpp and
161// frmts/wms/gdalhttp.cpp
162 void CPL_DLL *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
163 const char *const *papszOptions);
164 char **CPLHTTPGetOptionsFromEnv(const char *pszFilename);
165
167 struct CPLHTTPRetryParameters
168{
169 int nMaxRetry = CPL_HTTP_MAX_RETRY;
170 double dfInitialDelay = CPL_HTTP_RETRY_DELAY;
171 std::string osRetryCodes{};
172
173 CPLHTTPRetryParameters() = default;
174 explicit CPLHTTPRetryParameters(const CPLStringList &aosHTTPOptions);
175};
176
178 class CPLHTTPRetryContext
179{
180 public:
181 explicit CPLHTTPRetryContext(const CPLHTTPRetryParameters &oParams);
182
183 bool CanRetry(int response_code, const char *pszErrBuf,
184 const char *pszCurlError);
185 bool CanRetry();
186
188 double GetCurrentDelay() const;
189
191 void ResetCounter()
192 {
193 m_nRetryCount = 0;
194 }
195
196 private:
197 CPLHTTPRetryParameters m_oParameters{};
198 int m_nRetryCount = 0;
199 double m_dfCurDelay = 0.0;
200 double m_dfNextDelay = 0.0;
201};
202
203 void CPL_DLL *CPLHTTPIgnoreSigPipe();
204 void CPL_DLL CPLHTTPRestoreSigPipeHandler(void *old_handler);
205 bool CPLMultiPerformWait(void *hCurlMultiHandle, int &repeats);
208 bool CPL_DLL CPLIsMachinePotentiallyGCEInstance();
209 bool CPLIsMachineForSureGCEInstance();
210
217 class GOA2Manager
218{
219 public:
220 GOA2Manager();
221
223 typedef enum
224 {
225 NONE,
226 GCE,
227 ACCESS_TOKEN_FROM_REFRESH,
228 SERVICE_ACCOUNT
229 } AuthMethod;
230
231 bool SetAuthFromGCE(CSLConstList papszOptions);
232 bool SetAuthFromRefreshToken(const char *pszRefreshToken,
233 const char *pszClientId,
234 const char *pszClientSecret,
235 CSLConstList papszOptions);
236 bool SetAuthFromServiceAccount(const char *pszPrivateKey,
237 const char *pszClientEmail,
238 const char *pszScope,
239 CSLConstList papszAdditionalClaims,
240 CSLConstList papszOptions);
241
243 AuthMethod GetAuthMethod() const
244 {
245 return m_eMethod;
246 }
247
248 const char *GetBearer() const;
249
251 const CPLString &GetPrivateKey() const
252 {
253 return m_osPrivateKey;
254 }
255
257 const CPLString &GetClientEmail() const
258 {
259 return m_osClientEmail;
260 }
261
265 std::string GetKey() const
266 {
267 std::string osKey(std::to_string(static_cast<int>(m_eMethod))
268 .append(",client-id=")
269 .append(m_osClientId)
270 .append(",client-secret=")
271 .append(m_osClientSecret)
272 .append(",refresh-token=")
273 .append(m_osRefreshToken)
274 .append(",private-key=")
275 .append(m_osPrivateKey)
276 .append(",client-email=")
277 .append(m_osClientEmail)
278 .append(",scope=")
279 .append(m_osScope));
280 osKey.append(",additional-claims=");
281 for (const auto *pszOption : m_aosAdditionalClaims)
282 {
283 osKey.append(pszOption);
284 osKey.append("+");
285 }
286 osKey.append(",options=");
287 for (const auto *pszOption : m_aosOptions)
288 {
289 osKey.append(pszOption);
290 osKey.append("+");
291 }
292 return osKey;
293 }
294
295 private:
296 mutable CPLString m_osCurrentBearer{};
297 mutable time_t m_nExpirationTime = 0;
298
299 AuthMethod m_eMethod = NONE;
300
301 // for ACCESS_TOKEN_FROM_REFRESH
302 CPLString m_osClientId{};
303 CPLString m_osClientSecret{};
304 CPLString m_osRefreshToken{};
305
306 // for SERVICE_ACCOUNT
307 CPLString m_osPrivateKey{};
308 CPLString m_osClientEmail{};
309 CPLString m_osScope{};
310 CPLStringList m_aosAdditionalClaims{};
311
312 CPLStringList m_aosOptions{};
313};
314
315#endif // __cplusplus
316
317#endif /* ndef CPL_HTTP_H_INCLUDED */
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition cpl_string.h:458
CPLString
Convenient string class based on std::string.
Definition cpl_string.h:320
GOA2Manager
Manager of Google OAuth2 authentication.
Definition cpl_http.h:218
GOA2Manager::GetClientEmail
const CPLString & GetClientEmail() const
Returns client email for SERVICE_ACCOUNT method.
Definition cpl_http.h:257
GOA2Manager::GOA2Manager
GOA2Manager()
Constructor.
GOA2Manager::GetAuthMethod
AuthMethod GetAuthMethod() const
Returns the authentication method.
Definition cpl_http.h:243
GOA2Manager::SetAuthFromRefreshToken
bool SetAuthFromRefreshToken(const char *pszRefreshToken, const char *pszClientId, const char *pszClientSecret, CSLConstList papszOptions)
Specifies that the authentication will be done using the OAuth2 client id method.
Definition cpl_google_oauth2.cpp:546
GOA2Manager::GetKey
std::string GetKey() const
Returns a key that can be used to uniquely identify the instance parameters (excluding bearer)
Definition cpl_http.h:265
GOA2Manager::SetAuthFromServiceAccount
bool SetAuthFromServiceAccount(const char *pszPrivateKey, const char *pszClientEmail, const char *pszScope, CSLConstList papszAdditionalClaims, CSLConstList papszOptions)
Specifies that the authentication will be done using the OAuth2 service account method.
Definition cpl_google_oauth2.cpp:581
GOA2Manager::GetPrivateKey
const CPLString & GetPrivateKey() const
Returns private key for SERVICE_ACCOUNT method.
Definition cpl_http.h:251
GOA2Manager::GetBearer
const char * GetBearer() const
Return the access token.
Definition cpl_google_oauth2.cpp:624
GOA2Manager::SetAuthFromGCE
bool SetAuthFromGCE(CSLConstList papszOptions)
Specifies that the authentication will be done using the local credentials of the current Google Comp...
Definition cpl_google_oauth2.cpp:522
GOA2Manager::AuthMethod
AuthMethod
Authentication method.
Definition cpl_http.h:224
cpl_conv.h
Various convenience functions for CPL.
CPLHTTPPopFetchCallback
int CPLHTTPPopFetchCallback(void)
Uninstalls a callback set by CPLHTTPPushFetchCallback().
Definition cpl_http.cpp:1011
CPLHTTPMultiFetch
CPLHTTPResult ** CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount, int nMaxSimultaneous, CSLConstList papszOptions)
Fetch several documents at once.
Definition cpl_http.cpp:1775
GOA2GetRefreshToken
char * GOA2GetRefreshToken(const char *pszAuthToken, const char *pszScope)
Turn Auth Token into a Refresh Token.
Definition cpl_google_oauth2.cpp:107
CPLHTTPPushFetchCallback
int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
Installs an alternate callback to the default implementation of CPLHTTPFetchEx().
Definition cpl_http.cpp:991
CPLHTTPCleanup
void CPLHTTPCleanup(void)
Cleanup function to call at application termination.
Definition cpl_http.cpp:2812
GOA2GetAccessTokenFromServiceAccount
char ** GOA2GetAccessTokenFromServiceAccount(const char *pszPrivateKey, const char *pszClientEmail, const char *pszScope, CSLConstList papszAdditionalClaims, CSLConstList papszOptions)
Fetch access token using Service Account OAuth2.
Definition cpl_google_oauth2.cpp:408
CPLIsMachinePotentiallyGCEInstance
bool CPLIsMachinePotentiallyGCEInstance()
Returns whether the current machine is potentially a Google Compute Engine instance.
Definition cpl_google_cloud.cpp:136
CPLHTTPFetch
CPLHTTPResult * CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
Fetch a document from an url and return in a string.
Definition cpl_http.cpp:1250
GOA2GetAccessTokenFromCloudEngineVM
char ** GOA2GetAccessTokenFromCloudEngineVM(CSLConstList papszOptions)
Fetch access token using Cloud Engine internal REST API.
Definition cpl_google_oauth2.cpp:369
CPLIsMachineForSureGCEInstance
bool CPLIsMachineForSureGCEInstance()
Returns whether the current machine is surely a Google Compute Engine instance.
Definition cpl_google_cloud.cpp:90
CPLHTTPSetFetchCallback
void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
Installs an alternate callback to the default implementation of CPLHTTPFetchEx().
Definition cpl_http.cpp:968
CPLHTTPFetchCallbackFunc
CPLHTTPResult *(* CPLHTTPFetchCallbackFunc)(const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg, void *pUserData)
Callback function to process network requests.
Definition cpl_http.h:124
GOA2GetAuthorizationURL
char * GOA2GetAuthorizationURL(const char *pszScope)
Return authorization url for a given scope.
Definition cpl_google_oauth2.cpp:72
CPLHTTPFetchEx
CPLHTTPResult * CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
Fetch a document from an url and return in a string.
Definition cpl_http.cpp:1270
GOA2GetAccessToken
char * GOA2GetAccessToken(const char *pszRefreshToken, const char *pszScope)
Fetch access token using refresh token.
Definition cpl_google_oauth2.cpp:339
CPLHTTPEnabled
int CPLHTTPEnabled(void)
Return if CPLHTTP services can be useful.
Definition cpl_http.cpp:2795
CPLHTTPDestroyResult
void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
Clean the memory associated with the return value of CPLHTTPFetch()
Definition cpl_http.cpp:2868
CPLHTTPDestroyMultiResult
void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
Clean the memory associated with the return value of CPLHTTPMultiFetch()
Definition cpl_http.cpp:2029
CPLHTTPSetDefaultUserAgent
void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
Set the default user agent.
Definition cpl_http.cpp:489
CPLHTTPParseMultipartMime
int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
Parses a MIME multipart message.
Definition cpl_http.cpp:2901
CPL_C_END
#define CPL_C_END
Macro to end a block of C symbols.
Definition cpl_port.h:289
CPL_C_START
#define CPL_C_START
Macro to start a block of C symbols.
Definition cpl_port.h:285
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1087
GByte
unsigned char GByte
Unsigned byte type.
Definition cpl_port.h:175
cpl_string.h
Various convenience functions for working with strings and string lists.
cpl_vsi.h
Standard C Covers.
CPLHTTPResult
Definition cpl_http.h:51
CPLHTTPResult::pabyData
GByte * pabyData
Definition cpl_http.h:67
CPLHTTPResult::nStatus
int nStatus
Definition cpl_http.h:53
CPLHTTPResult::pasMimePart
CPLMimePart * pasMimePart
Definition cpl_http.h:76
CPLHTTPResult::pszContentType
char * pszContentType
Definition cpl_http.h:56
CPLHTTPResult::nDataAlloc
int nDataAlloc
Definition cpl_http.h:64
CPLHTTPResult::papszHeaders
char ** papszHeaders
Definition cpl_http.h:70
CPLHTTPResult::pszErrBuf
char * pszErrBuf
Definition cpl_http.h:59
CPLHTTPResult::nMimePartCount
int nMimePartCount
Definition cpl_http.h:73
CPLHTTPResult::nDataLen
int nDataLen
Definition cpl_http.h:62
CPLMimePart
Definition cpl_http.h:42