1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "minddata/dataset/engine/ir/datasetops/source/samplers/subset_sampler_ir.h"
18 #include "minddata/dataset/engine/datasetops/source/sampler/subset_sampler.h"
19 #include "minddata/dataset/core/config_manager.h"
20
21 #ifndef ENABLE_ANDROID
22 #include "minddata/dataset/util/random.h"
23 #include "minddata/mindrecord/include/shard_distributed_sample.h"
24 #include "minddata/mindrecord/include/shard_operator.h"
25 #include "minddata/mindrecord/include/shard_pk_sample.h"
26 #include "minddata/mindrecord/include/shard_sample.h"
27 #include "minddata/mindrecord/include/shard_sequential_sample.h"
28 #include "minddata/mindrecord/include/shard_shuffle.h"
29 #endif
30
31 namespace mindspore {
32 namespace dataset {
33 // Constructor
SubsetSamplerObj(std::vector<int64_t> indices,int64_t num_samples)34 SubsetSamplerObj::SubsetSamplerObj(std::vector<int64_t> indices, int64_t num_samples)
35 : indices_(std::move(indices)), num_samples_(num_samples) {}
36
37 // Destructor
38 SubsetSamplerObj::~SubsetSamplerObj() = default;
39
ValidateParams()40 Status SubsetSamplerObj::ValidateParams() {
41 if (num_samples_ < 0) {
42 RETURN_STATUS_UNEXPECTED("SubsetRandomSampler: num_samples must be greater than or equal to 0, but got: " +
43 std::to_string(num_samples_));
44 }
45
46 return Status::OK();
47 }
48
SamplerBuild(std::shared_ptr<SamplerRT> * sampler)49 Status SubsetSamplerObj::SamplerBuild(std::shared_ptr<SamplerRT> *sampler) {
50 // runtime sampler object
51 *sampler = std::make_shared<dataset::SubsetSamplerRT>(indices_, num_samples_);
52 Status s = BuildChildren(sampler);
53 sampler = s.IsOk() ? sampler : nullptr;
54 return s;
55 }
56
57 #ifndef ENABLE_ANDROID
BuildForMindDataset()58 std::shared_ptr<mindrecord::ShardOperator> SubsetSamplerObj::BuildForMindDataset() {
59 // runtime mindrecord sampler object
60 auto mind_sampler = std::make_shared<mindrecord::ShardSample>(indices_);
61
62 return mind_sampler;
63 }
64 #endif
to_json(nlohmann::json * const out_json)65 Status SubsetSamplerObj::to_json(nlohmann::json *const out_json) {
66 nlohmann::json args;
67 RETURN_IF_NOT_OK(SamplerObj::to_json(&args));
68 args["sampler_name"] = "SubsetSampler";
69 args["indices"] = indices_;
70 args["num_samples"] = num_samples_;
71 *out_json = args;
72 return Status::OK();
73 }
74
75 #ifndef ENABLE_ANDROID
from_json(nlohmann::json json_obj,int64_t num_samples,std::shared_ptr<SamplerObj> * sampler)76 Status SubsetSamplerObj::from_json(nlohmann::json json_obj, int64_t num_samples, std::shared_ptr<SamplerObj> *sampler) {
77 RETURN_IF_NOT_OK(ValidateParamInJson(json_obj, "indices", "SubsetSampler"));
78 std::vector<int64_t> indices = json_obj["indices"];
79 *sampler = std::make_shared<SubsetSamplerObj>(indices, num_samples);
80 // Run common code in super class to add children samplers
81 RETURN_IF_NOT_OK(SamplerObj::from_json(json_obj, sampler));
82 return Status::OK();
83 }
84 #endif
85
SamplerCopy()86 std::shared_ptr<SamplerObj> SubsetSamplerObj::SamplerCopy() {
87 auto sampler = std::make_shared<SubsetSamplerObj>(indices_, num_samples_);
88 for (const auto &child : children_) {
89 Status rc = sampler->AddChildSampler(child);
90 if (rc.IsError()) {
91 MS_LOG(ERROR) << "[Internal ERROR] Error in copying the sampler. Message: " << rc;
92 }
93 }
94 return sampler;
95 }
96 } // namespace dataset
97 } // namespace mindspore
98