| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/api/python/bindings/mindrecord/include/ |
| D | bindings.cc | 34 … (void)py::class_<mindrecord::ShardOperator, std::shared_ptr<mindrecord::ShardOperator>>( in __anone6b51b890102()
37 [](std::shared_ptr<mindrecord::ShardOperator> self, in __anone6b51b890102()
38 … std::shared_ptr<mindrecord::ShardOperator> child) { self->SetChildOp(child); }); in __anone6b51b890102()
42 (void)py::class_<mindrecord::ShardDistributedSample, mindrecord::ShardSample, in __anone6b51b890302()
43 std::shared_ptr<mindrecord::ShardDistributedSample>>(*m, in __anone6b51b890302()
50 …(void)py::class_<mindrecord::ShardPkSample, mindrecord::ShardOperator, std::shared_ptr<mindrecord:… in __anone6b51b890402()
54 …return std::make_shared<mindrecord::ShardPkSample>(kColumn, kVal, std::numeric_limits<int64_t>::ma… in __anone6b51b890402()
57 return std::make_shared<mindrecord::ShardPkSample>(kColumn, kVal, num_samples); in __anone6b51b890402()
64 …(void)py::class_<mindrecord::ShardSample, mindrecord::ShardOperator, std::shared_ptr<mindrecord::S… in __anone6b51b890602()
71 (void)py::class_<mindrecord::ShardSequentialSample, mindrecord::ShardSample, in __anone6b51b890702()
[all …]
|
| /third_party/mindspore/tests/ut/cpp/mindrecord/ |
| D | ut_shard_writer_test.cc | 39 namespace mindrecord { namespace
108 …mindrecord::GetAbsoluteFiles(path_dir, image_filenames); // get all files whose path within path_… in TEST_F()
117 mindrecord::ShardHeader header_data; in TEST_F()
126 …std::shared_ptr<mindrecord::Schema> image_schema = mindrecord::Schema::Build("picture", image_sche… in TEST_F()
142 std::shared_ptr<mindrecord::Statistics> static1 = in TEST_F()
143 mindrecord::Statistics::Build(static1_json["description"], static1_json["statistics"]); in TEST_F()
144 std::shared_ptr<mindrecord::Statistics> static2 = in TEST_F()
145 mindrecord::Statistics::Build(static2_json["description"], static2_json["statistics"]); in TEST_F()
193 mindrecord::ShardWriter fw; in TEST_F()
201 fw.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in TEST_F()
[all …]
|
| D | ut_common.cc | 24 namespace mindrecord { namespace
157 if (-1 == mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)) { in ShardWriterImageNet()
161 mindrecord::Img2DataUint8(filenames, bin_data); in ShardWriterImageNet()
169 …std::shared_ptr<mindrecord::Schema> anno_schema = mindrecord::Schema::Build("annotation", anno_sch… in ShardWriterImageNet()
212 fw_init.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in ShardWriterImageNet()
220 mindrecord::ShardWriter fw; in ShardWriterImageNet()
225 mindrecord::ShardIndexGenerator sg{filename}; in ShardWriterImageNet()
236 if (-1 == mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)) { in ShardWriterImageNetOneSample()
240 mindrecord::Img2DataUint8(filenames, bin_data); in ShardWriterImageNetOneSample()
243 mindrecord::ShardHeader header_data; in ShardWriterImageNetOneSample()
[all …]
|
| D | ut_shard_header_test.cc | 47 namespace mindrecord { namespace
69 mindrecord::ShardHeader header_data; in TEST_F()
123 mindrecord::ShardHeader header_data_new; in TEST_F()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/ |
| D | mindrecord_op.cc | 35 using mindrecord::kInt64Len;
36 using mindrecord::MSRStatus;
37 using mindrecord::Schema;
38 using mindrecord::ShardOperator;
39 using mindrecord::ShardReader;
45 … const mindrecord::json &sample_json, const std::map<std::string, std::string> &sample_bytes, in MindRecordOp()
74 …std::vector<mindrecord::ColumnDataType> col_data_types = shard_reader_->GetShardColumn()->GeColumn… in Init()
84 std::string type_str = mindrecord::ColumnDataTypeNameNormalized[col_data_types[i]]; in Init()
87 if (col_data_types[i] == mindrecord::ColumnBytes) { // rank = 1 in Init()
89 } else if (col_data_types[i] == mindrecord::ColumnString) { // rank = 0 in Init()
[all …]
|
| D | mindrecord_op.h | 47 using mindrecord::ShardOperator;
48 using mindrecord::ShardReader;
49 using ShardTuple = std::vector<std::tuple<std::vector<uint8_t>, mindrecord::json>>; /// Row of dat…
66 … const mindrecord::json &sample_json, const std::map<std::string, std::string> &sample_bytes_,
128 const mindrecord::json &columns_json, const mindrecord::TaskType task_type);
145 mindrecord::json sample_json_;
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/gnn/ |
| D | graph_feature_parser.cc | 28 using mindrecord::MSRStatus;
39 mindrecord::ColumnDataType col_type = mindrecord::ColumnNoDataType; in LoadFeatureTensor()
47 … std::move(DataType(mindrecord::ColumnDataTypeNameNormalized[col_type])), in LoadFeatureTensor()
59 mindrecord::ColumnDataType col_type = mindrecord::ColumnNoDataType; in LoadFeatureToSharedMemory()
84 mindrecord::ColumnDataType col_type = mindrecord::ColumnNoDataType; in LoadFeatureIndex()
95 if (col_type == mindrecord::ColumnInt32) { in LoadFeatureIndex()
97 } else if (col_type == mindrecord::ColumnInt64) { in LoadFeatureIndex()
|
| D | graph_loader.h | 43 using mindrecord::ShardReader;
86 …Status LoadNode(const std::vector<uint8_t> &blob, const mindrecord::json &jsn, std::shared_ptr<Nod…
95 …Status LoadEdge(const std::vector<uint8_t> &blob, const mindrecord::json &jsn, std::shared_ptr<Edg…
|
| D | graph_loader.cc | 28 using ShardTuple = std::vector<std::tuple<std::vector<uint8_t>, mindspore::mindrecord::json>>;
33 using mindrecord::MSRStatus;
102 mindrecord::json schema = graph_impl_->data_schema_["schema"]; in InitAndLoad()
136 Status GraphLoader::LoadNode(const std::vector<uint8_t> &col_blob, const mindrecord::json &col_jsn, in LoadNode()
185 Status GraphLoader::LoadEdge(const std::vector<uint8_t> &col_blob, const mindrecord::json &col_jsn, in LoadEdge()
247 mindrecord::json col_jsn = std::get<1>(tupled_row); in WorkerEntry()
|
| /third_party/mindspore/tests/perf_test/mindrecord/imagenet/ |
| D | perf_read_imagenet.py | 20 from mindspore.mindrecord import FileReader
30 def use_filereader(mindrecord): argument
33 reader = FileReader(file_name=mindrecord,
44 def use_minddataset(mindrecord): argument
47 data_set = ds.MindDataset(dataset_file=mindrecord,
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/ |
| D | pk_sampler_ir.cc | 85 std::shared_ptr<mindrecord::ShardOperator> PKSamplerObj::BuildForMindDataset() { in BuildForMindDataset()
87 std::shared_ptr<mindrecord::ShardOperator> mind_sampler; in BuildForMindDataset()
89 …mind_sampler = std::make_shared<mindrecord::ShardPkSample>("label", num_val_, std::numeric_limits<… in BuildForMindDataset()
92 mind_sampler = std::make_shared<mindrecord::ShardPkSample>("label", num_val_, num_samples_); in BuildForMindDataset()
|
| D | prebuilt_sampler_ir.h | 42 explicit PreBuiltSamplerObj(std::shared_ptr<mindrecord::ShardOperator> sampler);
50 std::shared_ptr<mindrecord::ShardOperator> BuildForMindDataset() override;
62 std::shared_ptr<mindrecord::ShardOperator> sp_minddataset_;
|
| D | mindrecord_sampler_ir.h | 58 Status GetShardReader(std::unique_ptr<mindrecord::ShardReader> *shard_reader);
61 std::unique_ptr<mindrecord::ShardReader> shard_reader_;
|
| D | mindrecord_sampler_ir.cc | 34 shard_reader_ = std::make_unique<mindrecord::ShardReader>(); in SamplerBuild()
47 Status MindRecordSamplerObj::GetShardReader(std::unique_ptr<mindrecord::ShardReader> *shard_reader)… in GetShardReader()
|
| D | prebuilt_sampler_ir.cc | 40 PreBuiltSamplerObj::PreBuiltSamplerObj(std::shared_ptr<mindrecord::ShardOperator> sampler) in PreBuiltSamplerObj()
56 std::shared_ptr<mindrecord::ShardOperator> PreBuiltSamplerObj::BuildForMindDataset() { return sp_mi… in BuildForMindDataset()
|
| D | subset_random_sampler_ir.cc | 49 std::shared_ptr<mindrecord::ShardOperator> SubsetRandomSamplerObj::BuildForMindDataset() { in BuildForMindDataset()
51 auto mind_sampler = std::make_shared<mindrecord::ShardSample>(indices_, GetSeed()); in BuildForMindDataset()
|
| D | sequential_sampler_ir.cc | 85 std::shared_ptr<mindrecord::ShardOperator> SequentialSamplerObj::BuildForMindDataset() { in BuildForMindDataset()
87 …auto mind_sampler = std::make_shared<mindrecord::ShardSequentialSample>(num_samples_, start_index_… in BuildForMindDataset()
|
| D | subset_sampler_ir.cc | 58 std::shared_ptr<mindrecord::ShardOperator> SubsetSamplerObj::BuildForMindDataset() { in BuildForMindDataset()
60 auto mind_sampler = std::make_shared<mindrecord::ShardSample>(indices_); in BuildForMindDataset()
|
| D | random_sampler_ir.cc | 82 std::shared_ptr<mindrecord::ShardOperator> RandomSamplerObj::BuildForMindDataset() { in BuildForMindDataset()
85 …std::make_shared<mindrecord::ShardShuffle>(GetSeed(), num_samples_, replacement_, reshuffle_each_e… in BuildForMindDataset()
|
| /third_party/mindspore/mindspore/mindrecord/ |
| D | __init__.inter | 16 Introduction to mindrecord:
20 mindrecord data through FileReader(FileWriter). Users could also
21 convert other format dataset to mindrecord data through
|
| /third_party/mindspore/tests/ut/python/mindrecord/ |
| D | test_cifar100_to_mindrecord.py | 20 from mindspore.mindrecord import Cifar100ToMR
21 from mindspore.mindrecord import FileReader
22 from mindspore.mindrecord import SUCCESS
|
| D | test_cifar10_to_mindrecord.py | 20 from mindspore.mindrecord import Cifar10ToMR
21 from mindspore.mindrecord import FileReader
22 from mindspore.mindrecord import SUCCESS
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/ |
| D | mind_record_sampler.h | 33 MindRecordSamplerRT(mindrecord::ShardReader *shard_reader,
59 mindrecord::ShardReader *shard_reader_; // back pointer to the shard reader
|
| /third_party/mindspore/tests/ut/cpp/dataset/ |
| D | mind_record_op_test.cc | 150 std::vector<std::shared_ptr<mindspore::mindrecord::ShardOperator>> operators; in TEST_F()
151 operators.push_back(std::make_shared<mindspore::mindrecord::ShardSample>(4)); in TEST_F()
214 std::vector<std::shared_ptr<mindspore::mindrecord::ShardOperator>> operators; in TEST_F()
215 operators.push_back(std::make_shared<mindspore::mindrecord::ShardShuffle>(1)); in TEST_F()
279 std::vector<std::shared_ptr<mindspore::mindrecord::ShardOperator>> operators; in TEST_F()
283 operators.push_back(std::make_shared<mindspore::mindrecord::ShardCategory>(categories)); in TEST_F()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/ |
| D | minddata_node.cc | 136 … std::vector<std::shared_ptr<mindrecord::ShardOperator>> *operators_, in BuildMindDatasetSamplerChain()
138 std::shared_ptr<mindrecord::ShardOperator> op = sampler->BuildForMindDataset(); in BuildMindDatasetSamplerChain()
146 std::stack<std::shared_ptr<mindrecord::ShardOperator>> stack_ops; in BuildMindDatasetSamplerChain()
153 auto distributed_sampler_op = std::dynamic_pointer_cast<mindrecord::ShardDistributedSample>(op); in BuildMindDatasetSamplerChain()
|