1 /** 2 * Copyright 2020-2024 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_TENSOR_OP_H_ 17 #define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_TENSOR_OP_H_ 18 19 #include <memory> 20 #include <string> 21 #include <vector> 22 23 #include "nlohmann/json.hpp" 24 25 #include "minddata/dataset/core/device_resource.h" 26 #include "minddata/dataset/core/device_tensor.h" 27 #if !defined(BUILD_LITE) && defined(ENABLE_D) 28 #include "minddata/dataset/core/device_tensor_ascend910b.h" 29 #endif 30 #include "minddata/dataset/core/tensor.h" 31 #include "minddata/dataset/core/tensor_row.h" 32 #include "minddata/dataset/engine/perf/info_collector.h" 33 #include "minddata/dataset/util/random.h" 34 #include "minddata/dataset/util/status.h" 35 36 #define IO_CHECK(input, output) \ 37 do { \ 38 if (input == nullptr || output == nullptr) { \ 39 RETURN_STATUS_UNEXPECTED("input or output is null."); \ 40 } \ 41 } while (false) 42 43 #define IO_CHECK_VECTOR(input, output) \ 44 do { \ 45 if (output == nullptr) { \ 46 RETURN_STATUS_UNEXPECTED("output is null."); \ 47 } \ 48 for (auto &_i : input) { \ 49 if (_i == nullptr) { \ 50 RETURN_STATUS_UNEXPECTED("input is null."); \ 51 } \ 52 } \ 53 } while (false) 54 55 namespace mindspore { 56 namespace dataset { 57 // base class 58 constexpr char kTensorOp[] = "TensorOp"; 59 60 // image 61 constexpr char kAdjustBrightnessOp[] = "AdjustBrightnessOp"; 62 constexpr char kAdjustContrastOp[] = "AdjustContrastOp"; 63 constexpr char kAdjustGammaOp[] = "AdjustGammaOp"; 64 constexpr char kAdjustHueOp[] = "AdjustHueOp"; 65 constexpr char kAdjustSaturationOp[] = "AdjustSaturationOp"; 66 constexpr char kAffineOp[] = "AffineOp"; 67 constexpr char kAutoAugmentOp[] = "AutoAugmentOp"; 68 constexpr char kAutoContrastOp[] = "AutoContrastOp"; 69 constexpr char kBoundingBoxAugmentOp[] = "BoundingBoxAugmentOp"; 70 constexpr char kDecodeOp[] = "DecodeOp"; 71 constexpr char kDecodeVideoOp[] = "DecodeVideoOp"; 72 constexpr char kCenterCropOp[] = "CenterCropOp"; 73 constexpr char kConvertColorOp[] = "ConvertColorOp"; 74 constexpr char kCutMixBatchOp[] = "CutMixBatchOp"; 75 constexpr char kCutOutOp[] = "CutOutOp"; 76 constexpr char kCropOp[] = "CropOp"; 77 // Ascend310 DVPP just support C++ Interface API 78 constexpr char kDvppCropJpegOp[] = "DvppCropJpegOp"; 79 constexpr char kDvppDecodeResizeCropJpegOp[] = "DvppDecodeResizeCropJpegOp"; 80 constexpr char kDvppDecodeResizeJpegOp[] = "DvppDecodeResizeJpegOp"; 81 constexpr char kDvppDecodeJpegOp[] = "DvppDecodeJpegOp"; 82 constexpr char kDvppDecodePngOp[] = "DvppDecodePngOp"; 83 constexpr char kDvppNormalizeOp[] = "DvppNormalizeOp"; // used by Ascend310 and Ascend910B 84 constexpr char kDvppResizeJpegOp[] = "DvppResizeJpegOp"; 85 // Ascend910B DVPP used for Python Interface API 86 constexpr char kDvppAdjustBrightnessOp[] = "DvppAdjustBrightnessOp"; 87 constexpr char kDvppAdjustContrastOp[] = "DvppAdjustContrastOp"; 88 constexpr char kDvppAdjustHueOp[] = "DvppAdjustHueOp"; 89 constexpr char kDvppAdjustSaturationOp[] = "DvppAdjustSaturationOp"; 90 constexpr char kDvppAdjustSharpnessOp[] = "DvppAdjustSharpnessOp"; 91 constexpr char kDvppAffineOp[] = "DvppAffineOp"; 92 constexpr char kDvppAutoContrastOp[] = "DvppAutoContrastOp"; 93 constexpr char kDvppConvertColorOp[] = "DvppConvertColorOp"; 94 constexpr char kDvppCropOp[] = "DvppCropOp"; 95 constexpr char kDvppDecodeOp[] = "DvppDecodeOp"; 96 constexpr char kDvppEqualizeOp[] = "DvppEqualizeOp"; 97 constexpr char kDvppEraseOp[] = "DvppEraseOp"; 98 constexpr char kDvppGaussianBlurOp[] = "DvppGaussianBlurOp"; 99 constexpr char kDvppHorizontalFlipOp[] = "DvppHorizontalFlipOp"; 100 constexpr char kDvppInvertOp[] = "DvppInvertOp"; 101 constexpr char kDvppPadOp[] = "DvppPadOp"; 102 constexpr char kDvppPerspectiveOp[] = "DvppPerspectiveOp"; 103 constexpr char kDvppPosterizeOp[] = "DvppPosterizeOp"; 104 constexpr char kDvppResizeOp[] = "DvppResizeOp"; 105 constexpr char kDvppResizedCropOp[] = "DvppResizedCropOp"; 106 constexpr char kDvppRotateOp[] = "DvppRotateOp"; 107 constexpr char kDvppSolarizeOp[] = "DvppSolarizeOp"; 108 constexpr char kDvppVerticalFlipOp[] = "DvppVerticalFlipOp"; 109 constexpr char kEqualizeOp[] = "EqualizeOp"; 110 constexpr char kEraseOp[] = "EraseOp"; 111 constexpr char kGaussianBlurOp[] = "GaussianBlurOp"; 112 constexpr char kHorizontalFlipOp[] = "HorizontalFlipOp"; 113 constexpr char kHwcToChwOp[] = "HWC2CHWOp"; 114 constexpr char kInvertOp[] = "InvertOp"; 115 constexpr char kMixUpBatchOp[] = "MixUpBatchOp"; 116 constexpr char kNormalizeOp[] = "NormalizeOp"; 117 constexpr char kNormalizePadOp[] = "NormalizePadOp"; 118 constexpr char kPadOp[] = "PadOp"; 119 constexpr char kPadToSizeOp[] = "PadToSizeOp"; 120 constexpr char kPerspectiveOp[] = "PerspectiveOp"; 121 constexpr char kPosterizeOp[] = "PosterizeOp"; 122 constexpr char kRandAugmentOp[] = "RandAugmentOp"; 123 constexpr char kRandomAdjustSharpnessOp[] = "RandomAdjustSharpnessOp"; 124 constexpr char kRandomAffineOp[] = "RandomAffineOp"; 125 constexpr char kRandomAutoContrastOp[] = "RandomAutoContrastOp"; 126 constexpr char kRandomColorAdjustOp[] = "RandomColorAdjustOp"; 127 constexpr char kRandomColorOp[] = "RandomColorOp"; 128 constexpr char kRandomCropAndResizeOp[] = "RandomCropAndResizeOp"; 129 constexpr char kRandomCropAndResizeWithBBoxOp[] = "RandomCropAndResizeWithBBoxOp"; 130 constexpr char kRandomCropDecodeResizeOp[] = "RandomCropDecodeResizeOp"; 131 constexpr char kRandomCropOp[] = "RandomCropOp"; 132 constexpr char kRandomCropWithBBoxOp[] = "RandomCropWithBBoxOp"; 133 constexpr char kRandomEqualizeOp[] = "RandomEqualizeOp"; 134 constexpr char kRandomHorizontalFlipWithBBoxOp[] = "RandomHorizontalFlipWithBBoxOp"; 135 constexpr char kRandomHorizontalFlipOp[] = "RandomHorizontalFlipOp"; 136 constexpr char kRandomInvertOp[] = "RandomInvertOp"; 137 constexpr char kRandomLightingOp[] = "RandomLightingOp"; 138 constexpr char kRandomPosterizeOp[] = "RandomPosterizeOp"; 139 constexpr char kRandomResizeOp[] = "RandomResizeOp"; 140 constexpr char kRandomResizeWithBBoxOp[] = "RandomResizeWithBBoxOp"; 141 constexpr char kRandomRotationOp[] = "RandomRotationOp"; 142 constexpr char kRandomSolarizeOp[] = "RandomSolarizeOp"; 143 constexpr char kRandomSharpnessOp[] = "RandomSharpnessOp"; 144 constexpr char kRandomVerticalFlipOp[] = "RandomVerticalFlipOp"; 145 constexpr char kRandomVerticalFlipWithBBoxOp[] = "RandomVerticalFlipWithBBoxOp"; 146 constexpr char kRescaleOp[] = "RescaleOp"; 147 constexpr char kResizeBilinearOp[] = "ResizeBilinearOp"; 148 constexpr char kResizedCropOp[] = "ResizedCropOp"; 149 constexpr char kResizeOp[] = "ResizeOp"; 150 constexpr char kResizePreserveAROp[] = "ResizePreserveAROp"; 151 constexpr char kResizeWithBBoxOp[] = "ResizeWithBBoxOp"; 152 constexpr char kRgbaToBgrOp[] = "RgbaToBgrOp"; 153 constexpr char kRgbaToRgbOp[] = "RgbaToRgbOp"; 154 constexpr char kRgbToBgrOp[] = "RgbToBgrOp"; 155 constexpr char kRgbToGrayOp[] = "RgbToGrayOp"; 156 constexpr char kRotateOp[] = "RotateOp"; 157 constexpr char kSharpnessOp[] = "SharpnessOp"; 158 constexpr char kSlicePatchesOp[] = "SlicePatchesOp"; 159 constexpr char kSolarizeOp[] = "SolarizeOp"; 160 constexpr char kSwapRedBlueOp[] = "SwapRedBlueOp"; 161 constexpr char kToTensorOp[] = "ToTensorOp"; 162 constexpr char kUniformAugOp[] = "UniformAugmentOp"; 163 constexpr char kVerticalFlipOp[] = "VerticalFlipOp"; 164 165 // video 166 constexpr char kDvppDecodeVideoOp[] = "DvppDecodeVideoOp"; 167 168 // text 169 constexpr char kAddTokenOp[] = "AddTokenOp"; 170 constexpr char kBasicTokenizerOp[] = "BasicTokenizerOp"; 171 constexpr char kBertTokenizerOp[] = "BertTokenizerOp"; 172 constexpr char kCaseFoldOp[] = "CaseFoldOp"; 173 constexpr char kFilterWikipediaXMLOp[] = "FilterWikipediaXMLOp"; 174 constexpr char kJiebaTokenizerOp[] = "JiebaTokenizerOp"; 175 constexpr char kLookupOp[] = "LookupOp"; 176 constexpr char kNgramOp[] = "NgramOp"; 177 constexpr char kSlidingWindowOp[] = "SlidingWindowOp"; 178 constexpr char kNormalizeUTF8Op[] = "NormalizeUTF8Op"; 179 constexpr char kRegexReplaceOp[] = "RegexReplaceOp"; 180 constexpr char kRegexTokenizerOp[] = "RegexTokenizerOp"; 181 constexpr char kToNumberOp[] = "ToNumberOp"; 182 constexpr char kToVectorsOp[] = "ToVectorsOp"; 183 constexpr char kTruncateOp[] = "TruncateOp"; 184 constexpr char kTruncateSequencePairOp[] = "TruncateSequencePairOp"; 185 constexpr char kUnicodeCharTokenizerOp[] = "UnicodeCharTokenizerOp"; 186 constexpr char kUnicodeScriptTokenizerOp[] = "UnicodeScriptTokenizerOp"; 187 constexpr char kWhitespaceTokenizerOp[] = "WhitespaceTokenizerOp"; 188 constexpr char kWordpieceTokenizerOp[] = "WordpieceTokenizerOp"; 189 constexpr char kRandomChoiceOp[] = "RandomChoiceOp"; 190 constexpr char kRandomApplyOp[] = "RandomApplyOp"; 191 constexpr char kComposeOp[] = "ComposeOp"; 192 constexpr char kRandomSelectSubpolicyOp[] = "RandomSelectSubpolicyOp"; 193 constexpr char kSentencepieceTokenizerOp[] = "SentencepieceTokenizerOp"; 194 195 // audio 196 constexpr char kAllpassBiquadOp[] = "AllpassBiquadOp"; 197 constexpr char kAmplitudeToDBOp[] = "AmplitudeToDBOp"; 198 constexpr char kAngleOp[] = "AngleOp"; 199 constexpr char kBandBiquadOp[] = "BandBiquadOp"; 200 constexpr char kBandpassBiquadOp[] = "BandpassBiquadOp"; 201 constexpr char kBandrejectBiquadOp[] = "BandrejectBiquadOp"; 202 constexpr char kBassBiquadOp[] = "BassBiquadOp"; 203 constexpr char kBiquadOp[] = "BiquadOp"; 204 constexpr char kComplexNormOp[] = "ComplexNormOp"; 205 constexpr char kComputeDeltasOp[] = "ComputeDeltasOp"; 206 constexpr char kContrastOp[] = "ContrastOp"; 207 constexpr char kDBToAmplitudeOp[] = " DBToAmplitudeOp"; 208 constexpr char kDCShiftOp[] = "DCShiftOp"; 209 constexpr char kDeemphBiquadOp[] = "DeemphBiquadOp"; 210 constexpr char kDetectPitchFrequencyOp[] = "DetectPitchFrequencyOp"; 211 constexpr char kDitherOp[] = "DitherOp"; 212 constexpr char kEqualizerBiquadOp[] = "EqualizerBiquadOp"; 213 constexpr char kFadeOp[] = "FadeOp"; 214 constexpr char kFiltfiltOp[] = "FiltfiltOp"; 215 constexpr char kFlangerOp[] = "FlangerOp"; 216 constexpr char kFrequencyMaskingOp[] = "FrequencyMaskingOp"; 217 constexpr char kGainOp[] = "GainOp"; 218 constexpr char kGriffinLimOp[] = "GriffinLimOp"; 219 constexpr char kHighpassBiquadOp[] = "HighpassBiquadOp"; 220 constexpr char kInverseMelScaleOp[] = "InverseMelScaleOp"; 221 constexpr char kInverseSpectrogramOp[] = "InverseSpectrogramOp"; 222 constexpr char kLFCCOp[] = "LFCCOp"; 223 constexpr char kLFilterOp[] = "LFilterOp"; 224 constexpr char kLowpassBiquadOp[] = "LowpassBiquadOp"; 225 constexpr char kMagphaseOp[] = "MagphaseOp"; 226 constexpr char kMaskAlongAxisIIDOp[] = "MaskAlongAxisIIDOp"; 227 constexpr char kMaskAlongAxisOp[] = "MaskAlongAxisOp"; 228 constexpr char kMelScaleOp[] = "MelScaleOp"; 229 constexpr char kMelSpectrogramOp[] = "MelSpectrogramOp"; 230 constexpr char kMFCCOp[] = "MFCCOp"; 231 constexpr char kMuLawDecodingOp[] = "MuLawDecodingOp"; 232 constexpr char kMuLawEncodingOp[] = "MuLawEncodingOp"; 233 constexpr char kOverdriveOp[] = "OverdriveOp"; 234 constexpr char kPhaserOp[] = "PhaserOp"; 235 constexpr char kPhaseVocoderOp[] = "PhaseVocoderOp"; 236 constexpr char kPitchShiftOp[] = "PitchShiftOp"; 237 constexpr char kResampleOp[] = "ResampleOp"; 238 constexpr char kRiaaBiquadOp[] = "RiaaBiquadOp"; 239 constexpr char kSlidingWindowCmnOp[] = "SlidingWindowCmnOp"; 240 constexpr char kSpectralCentroidOp[] = "SpectralCentroidOp"; 241 constexpr char kSpectrogramOp[] = "SpectrogramOp"; 242 constexpr char kTimeMaskingOp[] = "TimeMaskingOp"; 243 constexpr char kTimeStretchOp[] = "TimeStretchOp"; 244 constexpr char kTrebleBiquadOp[] = "TrebleBiquadOp"; 245 constexpr char kVadOp[] = "VadOp"; 246 constexpr char kVolOp[] = "VolOp"; 247 248 // data 249 constexpr char kConcatenateOp[] = "ConcatenateOp"; 250 constexpr char kDuplicateOp[] = "DuplicateOp"; 251 constexpr char kFillOp[] = "FillOp"; 252 constexpr char kMaskOp[] = "MaskOp"; 253 constexpr char kOneHotOp[] = "OneHotOp"; 254 constexpr char kPadEndOp[] = "PadEndOp"; 255 constexpr char kParseExampleOp[] = "ParseExampleOp"; 256 constexpr char kSliceOp[] = "SliceOp"; 257 constexpr char kToFloat16Op[] = "ToFloat16Op"; 258 constexpr char kTypeCastOp[] = "TypeCastOp"; 259 constexpr char kUniqueOp[] = "UniqueOp"; 260 261 // other 262 constexpr char kCFuncOp[] = "CFuncOp"; 263 constexpr char kPyFuncOp[] = "PyFuncOp"; 264 constexpr char kPluginOp[] = "PluginOp"; 265 constexpr char kNoOp[] = "NoOp"; 266 267 // A class that does a computation on a Tensor 268 class TensorOp { 269 public: 270 TensorOp() = default; 271 272 virtual ~TensorOp() = default; 273 274 // A function that prints info about the tensor operation 275 // @param out Print(std::ostream & out)276 virtual void Print(std::ostream &out) const { out << Name() << std::endl; } 277 278 // Provide stream operator for displaying it 279 // @param output stream 280 // @param so the TensorOp object to be printed 281 // @return output stream 282 friend std::ostream &operator<<(std::ostream &out, const TensorOp &so) { 283 so.Print(out); 284 return out; 285 } 286 287 // Perform an operation on one Tensor and produce one Tensor. This is for 1-to-1 column MapOp 288 // @param input shares the ownership of the Tensor (increase the ref count). 289 // @param output the address to a shared_ptr where the result will be placed. 290 // @return Status 291 virtual Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output); 292 293 // Perform an operation on Tensors from multiple columns, and produce multiple Tensors. 294 // This is for m-to-n column MapOp. 295 // @param input is a vector of shared_ptr to Tensor (pass by const reference). 296 // @param output is the address to an empty vector of shared_ptr to Tensor. 297 // @return Status 298 virtual Status Compute(const TensorRow &input, TensorRow *output); 299 300 // Perform an operation on one DeviceTensor and produce one DeviceTensor. This is for 1-to-1 column MapOp 301 // @param input shares the ownership of the DeviceTensor (increase the ref count). 302 // @param output the address to a shared_ptr where the result will be placed. 303 // @return Status 304 virtual Status Compute(const std::shared_ptr<DeviceTensor> &input, std::shared_ptr<DeviceTensor> *output); 305 306 #if !defined(BUILD_LITE) && defined(ENABLE_D) 307 virtual Status Compute(const std::vector<std::shared_ptr<DeviceTensorAscend910B>> &input, 308 std::vector<std::shared_ptr<DeviceTensorAscend910B>> *output); 309 310 // Perform an operation on one DeviceTensorAscend910B and produce one DeviceTensorAscend910B. This is for 1-to-1 311 // column MapOp 312 // @param input shares the ownership of the DeviceTensorAscned910B (increase the ref count). 313 // @param output the address to a shared_ptr where the result will be placed. 314 // @return Status 315 virtual Status Compute(const std::shared_ptr<DeviceTensorAscend910B> &input, 316 std::shared_ptr<DeviceTensorAscend910B> *output); 317 #endif 318 319 // Returns true oif the TensorOp takes one input and returns one output. 320 // @return true/false OneToOne()321 bool OneToOne() { return NumInput() == 1 && NumOutput() == 1; } 322 323 // Returns true oif the TensorOp produces deterministic result. 324 // @return true/false Deterministic()325 bool Deterministic() const { return is_deterministic_; } 326 327 // Function to determine the number of inputs the TensorOp can take. 0: means undefined. 328 // @return uint32_t NumInput()329 virtual uint32_t NumInput() { return 1; } 330 331 // Function to determine the number of output the TensorOp generates. 0: means undefined. 332 // @return uint32_t NumOutput()333 virtual uint32_t NumOutput() { return 1; } 334 335 // Function to determine the shapes of the output tensor given the input tensors' shapes. 336 // If a subclass did not override this function, it means that the shape does not change. 337 // @param inputs in: vector of the shapes of the input tensors. 338 // @param outputs out: vector of the shapes of the output tensors to be filled. 339 // @return Status 340 virtual Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs); 341 342 // Function to determine the types of the output tensor given the input tensor's types. 343 // If a subclass did not override this function, it means that the type does not change. 344 // @param inputs in: vector of the types of the input tensors. 345 // @param outputs out: vector of the types of the output tensors to be filled. 346 // @return Status 347 virtual Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs); 348 349 virtual std::string Name() const = 0; 350 to_json(nlohmann::json * out_json)351 virtual Status to_json(nlohmann::json *out_json) { return Status::OK(); } 352 353 virtual Status SetAscendResource(const std::shared_ptr<DeviceResource> &resource); 354 IsDvppOp()355 virtual bool IsDvppOp() { return false; } 356 IsHWC()357 virtual bool IsHWC() { return true; } // the input of the op is HWC in default 358 359 // Currently, it's used by PyFuncOp which can release global executor when map with thread/process mode ReleaseResource()360 virtual Status ReleaseResource() { return Status::OK(); } 361 SetSeed(uint32_t seed)362 virtual void SetSeed(uint32_t seed) {} 363 364 protected: 365 bool is_deterministic_{true}; 366 }; 367 368 class RandomTensorOp : public TensorOp { 369 public: 370 RandomTensorOp(); 371 372 ~RandomTensorOp() override = default; 373 374 protected: 375 void SetSeed(uint32_t seed) override; 376 377 std::mt19937 random_generator_; 378 }; 379 } // namespace dataset 380 } // namespace mindspore 381 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_TENSOR_OP_H_ 382