23#include "ogr_recordbatch.h"
25class CPL_DLL OGRArrowArrayHelper
27 OGRArrowArrayHelper(
const OGRArrowArrayHelper &) =
delete;
28 OGRArrowArrayHelper &operator=(
const OGRArrowArrayHelper &) =
delete;
31 bool m_bIncludeFID =
false;
32 int m_nMaxBatchSize = 0;
34 const int m_nFieldCount = 0;
35 const int m_nGeomFieldCount = 0;
36 std::vector<int> m_mapOGRFieldToArrowField{};
37 std::vector<int> m_mapOGRGeomFieldToArrowField{};
38 std::vector<bool> m_abNullableFields{};
39 std::vector<uint32_t> m_anArrowFieldMaxAlloc{};
40 std::vector<int> m_anTZFlags{};
41 int64_t *m_panFIDValues =
nullptr;
42 struct ArrowArray *m_out_array =
nullptr;
44 static uint32_t GetMemLimit();
47 GetMaxFeaturesInBatch(
const CPLStringList &aosArrowArrayStreamOptions);
51 struct ArrowArray *out_array);
53 bool SetNull(
int iArrowField,
int iFeat)
55 auto psArray = m_out_array->children[iArrowField];
56 ++psArray->null_count;
58 static_cast<uint8_t *
>(
const_cast<void *
>(psArray->buffers[0]));
59 if (psArray->buffers[0] ==
nullptr)
61 pabyNull =
static_cast<uint8_t *
>(
63 if (pabyNull ==
nullptr)
67 memset(pabyNull, 0xFF, (m_nMaxBatchSize + 7) / 8);
68 psArray->buffers[0] = pabyNull;
70 pabyNull[iFeat / 8] &=
static_cast<uint8_t
>(~(1 << (iFeat % 8)));
72 if (psArray->n_buffers == 3)
75 static_cast<int32_t *
>(
const_cast<void *
>(psArray->buffers[1]));
76 panOffsets[iFeat + 1] = panOffsets[iFeat];
81 inline static void SetBoolOn(
struct ArrowArray *psArray,
int iFeat)
83 static_cast<uint8_t *
>(
84 const_cast<void *
>(psArray->buffers[1]))[iFeat / 8] |=
85 static_cast<uint8_t
>(1 << (iFeat % 8));
88 inline static void SetInt8(
struct ArrowArray *psArray,
int iFeat,
91 static_cast<int8_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
95 inline static void SetUInt8(
struct ArrowArray *psArray,
int iFeat,
98 static_cast<uint8_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
102 inline static void SetInt16(
struct ArrowArray *psArray,
int iFeat,
105 static_cast<int16_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
109 inline static void SetUInt16(
struct ArrowArray *psArray,
int iFeat,
112 static_cast<uint16_t *
>(
113 const_cast<void *
>(psArray->buffers[1]))[iFeat] = nVal;
116 inline static void SetInt32(
struct ArrowArray *psArray,
int iFeat,
119 static_cast<int32_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
123 inline static void SetUInt32(
struct ArrowArray *psArray,
int iFeat,
126 static_cast<uint32_t *
>(
127 const_cast<void *
>(psArray->buffers[1]))[iFeat] = nVal;
130 inline static void SetInt64(
struct ArrowArray *psArray,
int iFeat,
133 static_cast<int64_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
137 inline static void SetUInt64(
struct ArrowArray *psArray,
int iFeat,
140 static_cast<uint64_t *
>(
141 const_cast<void *
>(psArray->buffers[1]))[iFeat] = nVal;
144 inline static void SetFloat(
struct ArrowArray *psArray,
int iFeat,
147 static_cast<float *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
151 inline static void SetDouble(
struct ArrowArray *psArray,
int iFeat,
154 static_cast<double *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
158 static void SetDate(
struct ArrowArray *psArray,
int iFeat,
159 struct tm &brokenDown,
const OGRField &ogrField)
161 brokenDown.tm_year = ogrField.Date.Year - 1900;
162 brokenDown.tm_mon = ogrField.Date.Month - 1;
163 brokenDown.tm_mday = ogrField.Date.Day;
164 brokenDown.tm_hour = 0;
165 brokenDown.tm_min = 0;
166 brokenDown.tm_sec = 0;
167 static_cast<int32_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
168 static_cast<int>(CPLYMDHMSToUnixTime(&brokenDown) / 86400);
171 static void SetDateTime(
struct ArrowArray *psArray,
int iFeat,
172 struct tm &brokenDown,
int nFieldTZFlag,
175 brokenDown.tm_year = ogrField.Date.Year - 1900;
176 brokenDown.tm_mon = ogrField.Date.Month - 1;
177 brokenDown.tm_mday = ogrField.Date.Day;
178 brokenDown.tm_hour = ogrField.Date.Hour;
179 brokenDown.tm_min = ogrField.Date.Minute;
180 brokenDown.tm_sec =
static_cast<int>(ogrField.Date.Second);
182 CPLYMDHMSToUnixTime(&brokenDown) * 1000 +
183 (
static_cast<int>(ogrField.Date.Second * 1000 + 0.5) % 1000);
188 const int TZOffset = (ogrField.Date.TZFlag -
OGR_TZFLAG_UTC) * 15;
189 const int TZOffsetMS = TZOffset * 60 * 1000;
192 static_cast<int64_t *
>(
const_cast<void *
>(psArray->buffers[1]))[iFeat] =
196 GByte *GetPtrForStringOrBinary(
int iArrowField,
int iFeat,
size_t nLen)
198 auto psArray = m_out_array->children[iArrowField];
200 static_cast<int32_t *
>(
const_cast<void *
>(psArray->buffers[1]));
201 const uint32_t nCurLength =
static_cast<uint32_t
>(panOffsets[iFeat]);
202 if (nLen > m_anArrowFieldMaxAlloc[iArrowField] - nCurLength)
205 static_cast<uint32_t
>(std::numeric_limits<int32_t>::max()) -
209 "Too large string or binary content");
212 uint32_t nNewSize = nCurLength +
static_cast<uint32_t
>(nLen);
213 if ((m_anArrowFieldMaxAlloc[iArrowField] >> 31) == 0)
215 const uint32_t nDoubleSize =
216 2U * m_anArrowFieldMaxAlloc[iArrowField];
217 if (nNewSize < nDoubleSize)
218 nNewSize = nDoubleSize;
221 if (newBuffer ==
nullptr)
223 m_anArrowFieldMaxAlloc[iArrowField] = nNewSize;
224 memcpy(newBuffer, psArray->buffers[2], nCurLength);
226 psArray->buffers[2] = newBuffer;
229 static_cast<GByte *
>(
const_cast<void *
>(psArray->buffers[2])) +
231 panOffsets[iFeat + 1] = panOffsets[iFeat] +
static_cast<int32_t
>(nLen);
235 static void SetEmptyStringOrBinary(
struct ArrowArray *psArray,
int iFeat)
238 static_cast<int32_t *
>(
const_cast<void *
>(psArray->buffers[1]));
239 panOffsets[iFeat + 1] = panOffsets[iFeat];
242 void Shrink(
int nFeatures)
244 if (nFeatures < m_nMaxBatchSize)
246 m_out_array->length = nFeatures;
247 for (
int i = 0; i < m_nChildren; i++)
249 m_out_array->children[i]->length = nFeatures;
256 if (m_out_array->release)
257 m_out_array->release(m_out_array);
258 memset(m_out_array, 0,
sizeof(*m_out_array));
261 static bool FillDict(
struct ArrowArray *psChild,
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:436
A set of associated raster bands, usually from one file.
Definition: gdal_priv.h:495
Definition of a coded / enumerated field domain.
Definition: ogr_feature.h:1742
Definition of a feature class or feature layer.
Definition: ogr_feature.h:501
#define CPLE_AppDefined
Application defined error.
Definition: cpl_error.h:84
void CPLError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt,...)
Report an error.
Definition: cpl_error.cpp:314
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:169
#define VSI_MALLOC_ALIGNED_AUTO_VERBOSE(size)
VSIMallocAlignedAutoVerbose() with FILE and LINE reporting.
Definition: cpl_vsi.h:305
void VSIFreeAligned(void *ptr)
Free a buffer allocated with VSIMallocAligned().
Definition: cpl_vsisimple.cpp:979
#define OGR_TZFLAG_UTC
Time zone flag indicating UTC.
Definition: ogr_core.h:894
#define OGR_TZFLAG_MIXED_TZ
Time zone flag only returned by OGRFieldDefn::GetTZFlag() to indicate that all values in the field ha...
Definition: ogr_core.h:886
Classes related to registration of format support, and opening datasets.
OGRFeature field attribute value union.
Definition: ogr_core.h:905