1 /**
2 * Copyright 2020-2022 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONSTANTS_H_
17 #define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONSTANTS_H_
18
19 #include <cstdint>
20 #include <limits>
21 #include <random>
22
23 #include "include/api/types.h"
24
25 namespace mindspore {
26 namespace dataset {
27 // Various type defines for convenience
28 using uchar = unsigned char;
29 using dsize_t = int64_t;
30
31 /// \brief The modulation in Flanger
32 enum class DATASET_API Modulation {
33 kSinusoidal = 0, ///< Use sinusoidal modulation.
34 kTriangular = 1 ///< Use triangular modulation.
35 };
36
37 /// \brief The interpolation in Flanger
38 enum class DATASET_API Interpolation {
39 kLinear = 0, ///< Use linear for delay-line interpolation.
40 kQuadratic = 1 ///< Use quadratic for delay-line interpolation.
41 };
42
43 /// \brief The dataset auto augment policy in AutoAugment
44 enum class DATASET_API AutoAugmentPolicy {
45 kImageNet = 0, ///< AutoAugment policy learned on the ImageNet dataset.
46 kCifar10 = 1, ///< AutoAugment policy learned on the Cifar10 dataset.
47 kSVHN = 2 ///< AutoAugment policy learned on the SVHN dataset.
48 };
49
50 /// \brief The color conversion code
51 enum class DATASET_API ConvertMode {
52 COLOR_BGR2BGRA = 0, ///< Add alpha channel to BGR image.
53 COLOR_RGB2RGBA = COLOR_BGR2BGRA, ///< Add alpha channel to RGB image.
54 COLOR_BGRA2BGR = 1, ///< Remove alpha channel to BGR image.
55 COLOR_RGBA2RGB = COLOR_BGRA2BGR, ///< Remove alpha channel to RGB image.
56 COLOR_BGR2RGBA = 2, ///< Convert BGR image to RGBA image.
57 COLOR_RGB2BGRA = COLOR_BGR2RGBA, ///< Convert RGB image to BGRA image.
58 COLOR_RGBA2BGR = 3, ///< Convert RGBA image to BGR image.
59 COLOR_BGRA2RGB = COLOR_RGBA2BGR, ///< Convert BGRA image to RGB image.
60 COLOR_BGR2RGB = 4, ///< Convert BGR image to RGB image.
61 COLOR_RGB2BGR = COLOR_BGR2RGB, ///< Convert RGB image to BGR image.
62 COLOR_BGRA2RGBA = 5, ///< Convert BGRA image to RGBA image.
63 COLOR_RGBA2BGRA = COLOR_BGRA2RGBA, ///< Convert RGBA image to BGRA image.
64 COLOR_BGR2GRAY = 6, ///< Convert BGR image to GRAY image.
65 COLOR_RGB2GRAY = 7, ///< Convert RGB image to GRAY image.
66 COLOR_GRAY2BGR = 8, ///< Convert GRAY image to BGR image.
67 COLOR_GRAY2RGB = COLOR_GRAY2BGR, ///< Convert GRAY image to RGB image.
68 COLOR_GRAY2BGRA = 9, ///< Convert GRAY image to BGRA image.
69 COLOR_GRAY2RGBA = COLOR_GRAY2BGRA, ///< Convert GRAY image to RGBA image.
70 COLOR_BGRA2GRAY = 10, ///< Convert BGRA image to GRAY image.
71 COLOR_RGBA2GRAY = 11 ///< Convert RGBA image to GRAY image.
72 };
73
74 /// \brief The mode for reading a image file.
75 enum class DATASET_API ImageReadMode {
76 kUNCHANGED = 0, ///< Remain the output in the original format.
77 kGRAYSCALE = 1, ///< Convert the output into one channel grayscale data.
78 kCOLOR = 2, ///< Convert the output into three channels RGB color data.
79 };
80
81 // \brief Possible density function in Dither.
82 enum DATASET_API DensityFunction {
83 kTPDF = 0, ///< Use triangular probability density function.
84 kRPDF = 1, ///< Use rectangular probability density function.
85 kGPDF = 2 ///< Use gaussian probability density function.
86 };
87
88 /// \brief Values of norm in CreateDct.
89 enum class DATASET_API NormMode {
90 kNone = 0, ///< None type norm.
91 kOrtho = 1 ///< Ortho type norm.
92 };
93
94 /// \brief Possible options for norm in MelscaleFbanks.
95 enum class DATASET_API NormType {
96 kNone = 0, ///< None type norm.
97 kSlaney = 1, ///< Slaney type norm.
98 };
99
100 /// \brief The mode for manual offload.
101 enum class DATASET_API ManualOffloadMode {
102 kUnspecified, ///< Not set, will use auto_offload setting instead.
103 kDisabled, ///< Do not perform offload.
104 kEnabled ///< Attempt to offload.
105 };
106
107 /// \brief Target devices to perform map operation.
108 enum class DATASET_API MapTargetDevice {
109 kCpu = 0, ///< CPU Device.
110 kGpu, ///< Gpu Device.
111 kAscend310, ///< Ascend310 Device.
112 kAscend910B, ///< Ascend910B Device.
113 kInvalid = 100
114 };
115
116 /// \brief Possible options for mel_type in MelscaleFbanks.
117 enum class DATASET_API MelType {
118 kHtk = 0, ///< Htk scale type.
119 kSlaney = 1, ///< Slaney scale type.
120 };
121
122 /// \brief The initial type of tensor implementation.
123 enum class DATASET_API TensorImpl {
124 kNone, ///< None type tensor.
125 kFlexible, ///< Flexible type tensor, can be converted to any type.
126 kCv, ///< CV type tensor.
127 kNP ///< Numpy type tensor.
128 };
129
130 /// \brief The mode for shuffling data.
131 enum class DATASET_API ShuffleMode {
132 kFalse = 0, ///< No shuffling is performed.
133 kFiles = 1, ///< Shuffle files only.
134 kGlobal = 2, ///< Shuffle both the files and samples.
135 kInfile = 3 ///< Shuffle data within each file.
136 };
137
138 /// \brief Possible scale for input audio.
139 enum class DATASET_API ScaleType {
140 kMagnitude = 0, ///< Audio scale is magnitude.
141 kPower = 1, ///< Audio scale is power.
142 };
143
144 /// \brief The scale for gain type.
145 enum class DATASET_API GainType {
146 kAmplitude = 0, ///< Audio gain type is amplitude.
147 kPower = 1, ///< Audio gain type is power.
148 kDb = 2, ///< Audio gain type is db.
149 };
150
151 /// \brief The method of padding.
152 enum class DATASET_API BorderType {
153 kConstant = 0, ///< Fill the border with constant values.
154 kEdge = 1, ///< Fill the border with the last value on the edge.
155 kReflect = 2, ///< Reflect the values on the edge omitting the last value of edge.
156 kSymmetric = 3 ///< Reflect the values on the edge repeating the last value of edge.
157 };
158
159 /// \brief Possible fix rotation angle for Rotate Op.
160 enum class DATASET_API FixRotationAngle {
161 k0Degree = 1, ///< Rotate 0 degree.
162 k0DegreeAndMirror = 2, ///< Rotate 0 degree and apply horizontal flip.
163 k180Degree = 3, ///< Rotate 180 degree.
164 k180DegreeAndMirror = 4, ///< Rotate 180 degree and apply horizontal flip.
165 k90DegreeAndMirror = 5, ///< Rotate 90 degree and apply horizontal flip.
166 k90Degree = 6, ///< Rotate 90 degree.
167 k270DegreeAndMirror = 7, ///< Rotate 270 degree and apply horizontal flip.
168 k270Degree = 8, ///< Rotate 270 degree.
169 };
170
171 /// \brief Possible types for windows function.
172 enum class DATASET_API WindowType {
173 kBartlett = 0, ///< Bartlett window function.
174 kBlackman = 1, ///< Blackman window function.
175 kHamming = 2, ///< Hamming window function.
176 kHann = 3, ///< Hann window function.
177 kKaiser = 4 ///< Kaiser window function.
178 };
179
180 /// \brief Possible options for Image format types in a batch.
181 enum class DATASET_API ImageBatchFormat {
182 kNHWC = 0, ///< Indicate the input batch is of NHWC format.
183 kNCHW = 1 ///< Indicate the input batch is of NCHW format.
184 };
185
186 /// \brief Possible options for Image format types.
187 enum class DATASET_API ImageFormat {
188 HWC = 0, ///< Indicate the input batch is of NHWC format
189 CHW = 1, ///< Indicate the input batch is of NHWC format
190 HW = 2 ///< Indicate the input batch is of NHWC format
191 };
192
193 /// \brief Possible options for interpolation method.
194 enum class DATASET_API InterpolationMode {
195 kLinear = 0, ///< Interpolation method is linear interpolation.
196 kNearestNeighbour = 1, ///< Interpolation method is nearest-neighbor interpolation.
197 kCubic = 2, ///< Interpolation method is bicubic interpolation.
198 kArea = 3, ///< Interpolation method is pixel area interpolation.
199 kCubicPil = 4 ///< Interpolation method is bicubic interpolation like implemented in pillow.
200 };
201
202 /// \brief Possible formats for Vdec output image.
203 enum class DATASET_API VdecOutputFormat {
204 kYuvSemiplanar420 = 1, ///< Output image with PIXEL_FORMAT_YUV_SEMIPLANAR_420.
205 kYvuSemiplanar420 = 2, ///< Output image with PIXEL_FORMAT_YVU_SEMIPLANAR_420.
206 };
207
208 /// \brief Possible formats for Vdec input video.
209 enum class DATASET_API VdecStreamFormat {
210 kH265MainLevel = 0, ///< Input video with H265_MAIN_LEVEL
211 kH264BaselineLevel, ///< Input video with H264_BASELINE_LEVEL
212 kH264MainLevel, ///< Input video with H264_MAIN_LEVEL
213 kH264HighLevel ///< Input video with H264_HIGH_LEVEL
214 };
215
216 /// \brief Possible tokenize modes for JiebaTokenizer.
217 enum class DATASET_API JiebaMode {
218 kMix = 0, ///< Tokenize with MPSegment algorithm.
219 kMp = 1, ///< Tokenize with Hiddel Markov Model Segment algorithm.
220 kHmm = 2 ///< Tokenize with a mix of MPSegment and HMMSegment algorithm.
221 };
222
223 /// \brief Possible options for SPieceTokenizerOutType.
224 enum class DATASET_API SPieceTokenizerOutType {
225 kString = 0, ///< Output of sentencepiece tokenizer is string type.
226 kInt = 1 ///< Output of sentencepiece tokenizer is int type.
227 };
228
229 /// \brief Possible options for SPieceTokenizerLoadType.
230 enum class DATASET_API SPieceTokenizerLoadType {
231 kFile = 0, ///< Load sentencepiece tokenizer from local sentencepiece vocab file.
232 kModel = 1 ///< Load sentencepiece tokenizer from sentencepiece vocab instance.
233 };
234
235 /// \brief Type options for SentencePiece Model.
236 enum class DATASET_API SentencePieceModel {
237 kUnigram = 0, ///< Based on Unigram model.
238 kBpe = 1, ///< Based on Byte Pair Encoding (BPE) model.
239 kChar = 2, ///< Based on Char model.
240 kWord = 3 ///< Based on Word model.
241 };
242
243 /// \brief Possible options to specify a specific normalize mode.
244 enum class DATASET_API NormalizeForm {
245 kNone = 0, ///< Keep the input string tensor unchanged.
246 kNfc, ///< Normalize with Normalization Form C.
247 kNfkc, ///< Normalize with Normalization Form KC.
248 kNfd, ///< Normalize with Normalization Form D.
249 kNfkd, ///< Normalize with Normalization Form KD.
250 };
251
252 /// \brief Possible options for Mask.
253 enum class DATASET_API RelationalOp {
254 kEqual = 0, ///< equal to `==`
255 kNotEqual, ///< equal to `!=`
256 kLess, ///< equal to `<`
257 kLessEqual, ///< equal to `<=`
258 kGreater, ///< equal to `>`
259 kGreaterEqual, ///< equal to `>=`
260 };
261
262 /// \brief Possible modes for slice patches.
263 enum class DATASET_API SliceMode {
264 kPad = 0, ///< Pad some pixels before slice to patches.
265 kDrop = 1, ///< Drop remainder pixels before slice to patches.
266 };
267
268 /// \brief Possible options for SamplingStrategy.
269 enum class DATASET_API SamplingStrategy {
270 kRandom = 0, ///< Random sampling with replacement.
271 kEdgeWeight = 1 ///< Sampling with edge weight as probability.
272 };
273
274 /// \brief Possible options for fade shape.
275 enum class DATASET_API FadeShape {
276 kLinear = 0, ///< Fade shape is linear mode.
277 kExponential = 1, ///< Fade shape is exponential mode.
278 kLogarithmic = 2, ///< Fade shape is logarithmic mode.
279 kQuarterSine = 3, ///< Fade shape is quarter_sine mode.
280 kHalfSine = 4, ///< Fade shape is half_sine mode.
281 };
282
283 /// \brief Sample method for audio resample.
284 enum class DATASET_API ResampleMethod {
285 kSincInterpolation = 0, ///< Resample audio by sinc interpolation method
286 kKaiserWindow = 1, ///< Resample audio by Kaiser window
287 };
288
289 /// \brief Possible configuration methods for processing error samples.
290 enum class DATASET_API ErrorSamplesMode {
291 kReturn = 0, ///< Erroneous sample results in error raised and returned
292 kReplace = 1, ///< Erroneous sample is replaced with an internally determined sample
293 kSkip = 2 ///< Erroneous sample is skipped
294 };
295
296 /// \brief Convenience function to check bitmask for a 32bit int
297 /// \param[in] bits a 32bit int to be tested
298 /// \param[in] bitMask a 32bit int representing bit mask
299 /// \return bool Result for the check
BitTest(uint32_t bits,uint32_t bitMask)300 inline bool DATASET_API BitTest(uint32_t bits, uint32_t bitMask) { return (bits & bitMask) == bitMask; }
301
302 /// \brief Convenience function to set bitmask for a 32bit int
303 /// \param[in] bits a 32bit int to deal with
304 /// \param[in] bitMask a 32bit int representing bit mask
BitSet(uint32_t * bits,uint32_t bitMask)305 inline void DATASET_API BitSet(uint32_t *bits, uint32_t bitMask) {
306 if (bits == nullptr) {
307 return;
308 }
309 *bits |= bitMask;
310 }
311
312 /// \brief Convenience function to clear bitmask from a 32bit int
313 /// \param[in] bits a 32bit int to deal with
314 /// \param[in] bitMask a 32bit int representing bit mask
BitClear(uint32_t * bits,uint32_t bitMask)315 inline void DATASET_API BitClear(uint32_t *bits, uint32_t bitMask) {
316 if (bits == nullptr) {
317 return;
318 }
319 *bits &= (~bitMask);
320 }
321
322 constexpr uint32_t kFrameWidthMax = 4096;
323 constexpr uint32_t kFrameHeightMax = 4096;
324 constexpr uint32_t kFrameWidthMin = 128;
325 constexpr uint32_t kFrameHeightMin = 128;
326
327 constexpr int64_t kDeMaxDim = std::numeric_limits<int64_t>::max();
328 constexpr int32_t kDeMaxRank = std::numeric_limits<int32_t>::max();
329 constexpr int64_t kDeMaxFreq = std::numeric_limits<int64_t>::max(); // 9223372036854775807 or 2^(64-1)
330 constexpr int64_t kDeMaxTopk = std::numeric_limits<int64_t>::max();
331
332 constexpr uint32_t kCfgRowsPerBuffer = 1;
333 constexpr uint32_t kCfgParallelWorkers = 8;
334 constexpr uint32_t kCfgWorkerConnectorSize = 16;
335 constexpr uint32_t kCfgOpConnectorSize = 16;
336 constexpr uint32_t kCfgSendingBatch = 0;
337 constexpr int32_t kCfgDefaultRankId = -1;
338 constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed;
339 constexpr uint32_t kCfgMonitorSamplingInterval = 1000; // timeout value for monitor sampling interval in
340 // milliseconds
341 constexpr uint32_t kCfgCallbackTimeout = 60; // timeout value for callback in seconds
342 constexpr uint32_t kCfgMultiprocessingTimeoutInterval = 300; // timeout value for multiprocessing interval in seconds
343 constexpr int32_t kCfgDefaultCachePort = 50052;
344 constexpr char kCfgDefaultCacheHost[] = "127.0.0.1";
345 constexpr int32_t kDftCachePrefetchSize = 20;
346 constexpr int32_t kDftNumConnections = 12;
347 constexpr bool kDftAutoNumWorkers = false;
348 constexpr char kDftMetaColumnPrefix[] = "_meta-";
349 constexpr int32_t kDecimal = 10; // used in strtol() to convert a string value according to decimal numeral system
350 constexpr int32_t kMinLegalPort = 1025;
351 constexpr int32_t kMaxLegalPort = 65535;
352
353 // Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h)
354 constexpr uint8_t kCVInvalidType = 255;
355
356 using connection_id_type = uint64_t;
357 using session_id_type = uint32_t;
358 using row_id_type = int64_t;
359
360 constexpr uint32_t kCfgAutoTuneInterval = 0; // default number of steps
361 } // namespace dataset
362 } // namespace mindspore
363 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_CONSTANTS_H_
364