GDAL

gdalalg_abstract_pipeline.h
1/******************************************************************************
2 *
3 * Project: GDAL
4 * Purpose: gdal "raster/vector pipeline" subcommand
5 * Author: Even Rouault <even dot rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13#ifndef GDALALG_ABSTRACT_PIPELINE_INCLUDED
14#define GDALALG_ABSTRACT_PIPELINE_INCLUDED
15
17
18#include "cpl_json.h"
19
20#include "gdalalgorithm.h"
21#include "gdal_priv.h"
22
23#include <algorithm>
24
25// This is an easter egg to pay tribute to PROJ pipeline syntax
26// We accept "gdal vector +gdal=pipeline +step +gdal=read +input=in.tif +step +gdal=reproject +dst-crs=EPSG:32632 +step +gdal=write +output=out.tif +overwrite"
27// as an alternative to (recommended):
28// "gdal vector pipeline ! read in.tif ! reproject--dst-crs=EPSG:32632 ! write out.tif --overwrite"
29#ifndef GDAL_PIPELINE_PROJ_NOSTALGIA
30#define GDAL_PIPELINE_PROJ_NOSTALGIA
31#endif
32
33/************************************************************************/
34/* GDALPipelineStepRunContext */
35/************************************************************************/
36
37 class GDALPipelineStepAlgorithm;
38
39 class GDALPipelineStepRunContext
40{
41 public:
42 GDALPipelineStepRunContext() = default;
43
44 // Progress callback to use during execution of the step
45 GDALProgressFunc m_pfnProgress = nullptr;
46 void *m_pProgressData = nullptr;
47
48 // If there is a step in the pipeline immediately following step to which
49 // this instance of GDALRasterPipelineStepRunContext is passed, and that
50 // this next step is usable by the current step (as determined by
51 // CanHandleNextStep()), then this member will point to this next step.
52 GDALPipelineStepAlgorithm *m_poNextUsableStep = nullptr;
53
54 private:
55 CPL_DISALLOW_COPY_ASSIGN(GDALPipelineStepRunContext)
56};
57
58/************************************************************************/
59/* GDALPipelineStepAlgorithm */
60/************************************************************************/
61
62 class GDALPipelineStepAlgorithm /* non final */ : public GDALAlgorithm
63{
64 public:
65 std::vector<GDALArgDatasetValue> &GetInputDatasets()
66 {
67 return m_inputDataset;
68 }
69
70 const std::vector<GDALArgDatasetValue> &GetInputDatasets() const
71 {
72 return m_inputDataset;
73 }
74
75 GDALArgDatasetValue &GetOutputDataset()
76 {
77 return m_outputDataset;
78 }
79
80 const GDALArgDatasetValue &GetOutputDataset() const
81 {
82 return m_outputDataset;
83 }
84
85 const std::string &GetOutputString() const
86 {
87 return m_output;
88 }
89
90 const std::string &GetOutputLayerName() const
91 {
92 return m_outputLayerName;
93 }
94
95 const std::string &GetOutputFormat() const
96 {
97 return m_format;
98 }
99
100 const std::vector<std::string> &GetCreationOptions() const
101 {
102 return m_creationOptions;
103 }
104
105 const std::vector<std::string> &GetLayerCreationOptions() const
106 {
107 return m_layerCreationOptions;
108 }
109
110 bool GetOverwriteLayer() const
111 {
112 return m_overwriteLayer;
113 }
114
115 bool GetAppendLayer() const
116 {
117 return m_appendLayer;
118 }
119
120 virtual int GetInputType() const = 0;
121
122 virtual int GetOutputType() const = 0;
123
124 bool Finalize() override;
125
126 // Used by GDALDispatcherAlgorithm for vector info/convert
127 GDALDataset *GetInputDatasetRef()
128 {
129 return m_inputDataset.empty() ? nullptr
130 : m_inputDataset[0].GetDatasetRef();
131 }
132
133 // Used by GDALDispatcherAlgorithm for vector info/convert
134 void SetInputDataset(GDALDataset *poDS);
135
136 protected:
137 struct ConstructorOptions
138 {
139 bool standaloneStep = false;
140 bool addDefaultArguments = true;
141 bool autoOpenInputDatasets = true;
142 bool outputDatasetRequired = true;
143 bool addInputLayerNameArgument = true; // only for vector input
144 bool addUpdateArgument = true; // only for vector output
145 bool addAppendLayerArgument = true; // only for vector output
146 bool addOverwriteLayerArgument = true; // only for vector output
147 bool addUpsertArgument = true; // only for vector output
148 bool addSkipErrorsArgument = true; // only for vector output
149 bool addOutputLayerNameArgument = true; // only for vector output
150 int inputDatasetMaxCount = 1;
151 std::string inputDatasetHelpMsg{};
152 std::string inputDatasetAlias{};
153 std::string inputDatasetMetaVar = "INPUT";
154 std::string outputDatasetHelpMsg{};
155 std::string updateMutualExclusionGroup{};
156 std::string outputDatasetMutualExclusionGroup{};
157 std::string outputFormatCreateCapability = GDAL_DCAP_CREATECOPY;
158
159 inline ConstructorOptions &SetStandaloneStep(bool b)
160 {
161 standaloneStep = b;
162 return *this;
163 }
164
165 inline ConstructorOptions &SetAddDefaultArguments(bool b)
166 {
167 addDefaultArguments = b;
168 return *this;
169 }
170
171 inline ConstructorOptions &SetAddInputLayerNameArgument(bool b)
172 {
173 addInputLayerNameArgument = b;
174 return *this;
175 }
176
177 inline ConstructorOptions &SetInputDatasetMaxCount(int maxCount)
178 {
179 inputDatasetMaxCount = maxCount;
180 return *this;
181 }
182
183 inline ConstructorOptions &SetInputDatasetHelpMsg(const std::string &s)
184 {
185 inputDatasetHelpMsg = s;
186 return *this;
187 }
188
189 inline ConstructorOptions &SetInputDatasetAlias(const std::string &s)
190 {
191 inputDatasetAlias = s;
192 return *this;
193 }
194
195 inline ConstructorOptions &SetInputDatasetMetaVar(const std::string &s)
196 {
197 inputDatasetMetaVar = s;
198 return *this;
199 }
200
201 inline ConstructorOptions &SetOutputDatasetHelpMsg(const std::string &s)
202 {
203 outputDatasetHelpMsg = s;
204 return *this;
205 }
206
207 inline ConstructorOptions &SetAutoOpenInputDatasets(bool b)
208 {
209 autoOpenInputDatasets = b;
210 return *this;
211 }
212
213 inline ConstructorOptions &SetOutputDatasetRequired(bool b)
214 {
215 outputDatasetRequired = b;
216 return *this;
217 }
218
219 inline ConstructorOptions &
220 SetUpdateMutualExclusionGroup(const std::string &s)
221 {
222 updateMutualExclusionGroup = s;
223 return *this;
224 }
225
226 inline ConstructorOptions &
227 SetOutputDatasetMutualExclusionGroup(const std::string &s)
228 {
229 outputDatasetMutualExclusionGroup = s;
230 return *this;
231 }
232
233 inline ConstructorOptions &
234 SetOutputFormatCreateCapability(const std::string &capability)
235 {
236 outputFormatCreateCapability = capability;
237 return *this;
238 }
239
240 inline ConstructorOptions &SetAddAppendLayerArgument(bool b)
241 {
242 addAppendLayerArgument = b;
243 return *this;
244 }
245
246 inline ConstructorOptions &SetAddOverwriteLayerArgument(bool b)
247 {
248 addOverwriteLayerArgument = b;
249 return *this;
250 }
251
252 inline ConstructorOptions &SetAddUpdateArgument(bool b)
253 {
254 addUpdateArgument = b;
255 return *this;
256 }
257
258 inline ConstructorOptions &SetAddUpsertArgument(bool b)
259 {
260 addUpsertArgument = b;
261 return *this;
262 }
263
264 inline ConstructorOptions &SetAddSkipErrorsArgument(bool b)
265 {
266 addSkipErrorsArgument = b;
267 return *this;
268 }
269
270 inline ConstructorOptions &SetAddOutputLayerNameArgument(bool b)
271 {
272 addOutputLayerNameArgument = b;
273 return *this;
274 }
275 };
276
277 GDALPipelineStepAlgorithm(const std::string &name,
278 const std::string &description,
279 const std::string &helpURL,
280 const ConstructorOptions &);
281
282 friend class GDALPipelineAlgorithm;
283 friend class GDALRasterPipelineAlgorithm;
284 friend class GDALVectorPipelineAlgorithm;
285 friend class GDALAbstractPipelineAlgorithm;
286
287 virtual bool CanBeFirstStep() const
288 {
289 return false;
290 }
291
292 virtual bool CanBeMiddleStep() const
293 {
294 return !CanBeFirstStep() && !CanBeLastStep();
295 }
296
297 virtual bool CanBeLastStep() const
298 {
299 return false;
300 }
301
303 virtual bool GeneratesFilesFromUserInput() const
304 {
305 return false;
306 }
307
308 virtual bool IsNativelyStreamingCompatible() const
309 {
310 return true;
311 }
312
313 virtual bool SupportsInputMultiThreading() const
314 {
315 return false;
316 }
317
318 virtual bool CanHandleNextStep(GDALPipelineStepAlgorithm *) const
319 {
320 return false;
321 }
322
323 virtual bool OutputDatasetAllowedBeforeRunningStep() const
324 {
325 return false;
326 }
327
328 virtual CPLJSONObject Get_OGR_SCHEMA_OpenOption_Layer() const
329 {
330 CPLJSONObject obj;
331 obj.Deinit();
332 return obj;
333 }
334
335 virtual bool RunStep(GDALPipelineStepRunContext &ctxt) = 0;
336
337 bool m_standaloneStep = false;
338 const ConstructorOptions m_constructorOptions;
339 bool m_outputVRTCompatible = true;
340 std::string m_helpDocCategory{};
341
342 // Input arguments
343 std::vector<GDALArgDatasetValue> m_inputDataset{};
344 std::vector<std::string> m_openOptions{};
345 std::vector<std::string> m_inputFormats{};
346 std::vector<std::string> m_inputLayerNames{};
347
348 // Output arguments
349 bool m_stdout = false;
350 std::string m_output{};
351 GDALArgDatasetValue m_outputDataset{};
352 std::string m_format{};
353 std::vector<std::string> m_outputOpenOptions{};
354 std::vector<std::string> m_creationOptions{};
355 bool m_overwrite = false;
356 std::string m_outputLayerName{};
357 GDALInConstructionAlgorithmArg *m_outputFormatArg = nullptr;
358 bool m_appendRaster = false;
359
360 // Output arguments (vector specific)
361 std::vector<std::string> m_layerCreationOptions{};
362 bool m_update = false;
363 bool m_overwriteLayer = false;
364 bool m_appendLayer = false;
365 bool m_upsert = false;
366 bool m_skipErrors = false;
367
368 void AddRasterInputArgs(bool openForMixedRasterVector, bool hiddenForCLI);
369 void AddRasterOutputArgs(bool hiddenForCLI);
370 void AddRasterHiddenInputDatasetArg();
371
372 void AddVectorInputArgs(bool hiddenForCLI);
373 void AddVectorOutputArgs(bool hiddenForCLI,
374 bool shortNameOutputLayerAllowed);
375
376 private:
377 bool RunImpl(GDALProgressFunc pfnProgress, void *pProgressData) override;
378 GDALAlgorithm::ProcessGDALGOutputRet ProcessGDALGOutput() override;
379 bool CheckSafeForStreamOutput() override;
380
381 CPL_DISALLOW_COPY_ASSIGN(GDALPipelineStepAlgorithm)
382};
383
384/************************************************************************/
385/* GDALAbstractPipelineAlgorithm */
386/************************************************************************/
387
388 class GDALAbstractPipelineAlgorithm CPL_NON_FINAL
389 : public GDALPipelineStepAlgorithm
390{
391 public:
392 std::vector<std::string> GetAutoComplete(std::vector<std::string> &args,
393 bool lastWordIsComplete,
394 bool /* showAllOptions*/) override;
395
396 bool Finalize() override;
397
398 std::string GetUsageAsJSON() const override;
399
400 bool
401 ParseCommandLineArguments(const std::vector<std::string> &args) override;
402
403 bool HasSteps() const
404 {
405 return !m_steps.empty();
406 }
407
408 static constexpr const char *OPEN_NESTED_PIPELINE = "[";
409 static constexpr const char *CLOSE_NESTED_PIPELINE = "]";
410
411 static constexpr const char *RASTER_SUFFIX = "-raster";
412 static constexpr const char *VECTOR_SUFFIX = "-vector";
413
414 protected:
415 friend class GDALTeeStepAlgorithmAbstract;
416
417 GDALAbstractPipelineAlgorithm(
418 const std::string &name, const std::string &description,
419 const std::string &helpURL,
420 const GDALPipelineStepAlgorithm::ConstructorOptions &options)
421 : GDALPipelineStepAlgorithm(
422 name, description, helpURL,
423 ConstructorOptions(options).SetAutoOpenInputDatasets(false))
424 {
425 }
426
427 std::string m_pipeline{};
428
429 virtual GDALAlgorithmRegistry &GetStepRegistry() = 0;
430
431 virtual const GDALAlgorithmRegistry &GetStepRegistry() const = 0;
432
433 std::unique_ptr<GDALPipelineStepAlgorithm>
434 GetStepAlg(const std::string &name) const;
435
436 bool HasOutputString() const override;
437
438 static bool IsReadSpecificArgument(const char *pszArgName);
439 static bool IsWriteSpecificArgument(const char *pszArgName);
440
441 private:
442 friend class GDALPipelineAlgorithm;
443 friend class GDALRasterPipelineAlgorithm;
444 friend class GDALVectorPipelineAlgorithm;
445
446 std::vector<std::unique_ptr<GDALPipelineStepAlgorithm>> m_steps{};
447
448 std::unique_ptr<GDALPipelineStepAlgorithm> m_stepOnWhichHelpIsRequested{};
449
450 bool m_bInnerPipeline = false;
451 bool m_bExpectReadStep = true;
452
453 enum class StepConstraint
454 {
455 MUST_BE,
456 CAN_BE,
457 CAN_NOT_BE
458 };
459
460 StepConstraint m_eLastStepAsWrite = StepConstraint::CAN_BE;
461
462 std::vector<std::unique_ptr<GDALAbstractPipelineAlgorithm>>
463 m_apoNestedPipelines{};
464
465 // More would lead to unreadable pipelines
466 static constexpr int MAX_NESTING_LEVEL = 3;
467
468 bool
469 CheckFirstAndLastStep(const std::vector<GDALPipelineStepAlgorithm *> &steps,
470 bool forAutoComplete) const;
471
472 bool ParseCommandLineArguments(const std::vector<std::string> &args,
473 bool forAutoComplete);
474
475 bool RunStep(GDALPipelineStepRunContext &ctxt) override;
476
477 std::string
478 BuildNestedPipeline(GDALPipelineStepAlgorithm *curAlg,
479 std::vector<std::string> &nestedPipelineArgs,
480 bool forAutoComplete);
481
482 bool SaveGDALGFile(const std::string &outFilename,
483 std::string &outString) const;
484
485 virtual std::unique_ptr<GDALAbstractPipelineAlgorithm>
486 CreateNestedPipeline() const = 0;
487};
488
490
491#endif
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition cpl_json.h:56
CPLJSONObject::Deinit
void Deinit()
Decrement reference counter and make pointer NULL.
Definition cpl_json.cpp:1343
GDALAlgorithmRegistry
Registry of GDAL algorithms.
Definition gdalalgorithm_cpp.h:2132
GDALAlgorithm
GDAL algorithm.
Definition gdalalgorithm_cpp.h:2261
GDALAlgorithm::Finalize
virtual bool Finalize()
Complete any pending actions, and return the final status.
Definition gdalalgorithm.cpp:5790
GDALAlgorithm::ProcessGDALGOutputRet
ProcessGDALGOutputRet
Return value for ProcessGDALGOutput.
Definition gdalalgorithm_cpp.h:2925
GDALAlgorithm::CheckSafeForStreamOutput
virtual bool CheckSafeForStreamOutput()
Method executed by Run() when m_executionForStreamOutput is set to ensure the command is safe to exec...
Definition gdalalgorithm.cpp:5767
GDALAlgorithm::ProcessGDALGOutput
virtual ProcessGDALGOutputRet ProcessGDALGOutput()
Process output to a .gdalg file.
Definition gdalalgorithm.cpp:5203
GDALArgDatasetValue
Value for an argument that points to a GDALDataset.
Definition gdalalgorithm_cpp.h:149
GDALDataset
A set of associated raster bands, usually from one file.
Definition gdal_dataset.h:76
GDALInConstructionAlgorithmArg
Technical class used by GDALAlgorithm when constructing argument declarations.
Definition gdalalgorithm_cpp.h:1771
cpl_json.h
Interface for read and write JSON documents.
CPL_NON_FINAL
#define CPL_NON_FINAL
Mark that a class is explicitly recognized as non-final.
Definition cpl_port.h:929
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:936
GDAL_DCAP_CREATECOPY
#define GDAL_DCAP_CREATECOPY
Capability set by a driver that implements the CreateCopy() API.
Definition gdal.h:599
gdal_priv.h
This file is legacy since GDAL 3.12, but will be kept at least in the whole GDAL 3....