| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/meta/ |
| D | shard_header.cc | 37 ShardHeader::ShardHeader() : shard_count_(0), header_size_(0), page_size_(0), compression_size_(0) { in ShardHeader() function in mindspore::mindrecord::ShardHeader
41 Status ShardHeader::InitializeHeader(const std::vector<json> &headers, bool load_dataset) { in InitializeHeader()
62 Status ShardHeader::CheckFileStatus(const std::string &path) { in CheckFileStatus()
83 Status ShardHeader::ValidateHeader(const std::string &path, std::shared_ptr<json> *header_ptr) { in ValidateHeader()
123 Status ShardHeader::BuildSingleHeader(const std::string &file_path, std::shared_ptr<json> *header_p… in BuildSingleHeader()
141 Status ShardHeader::BuildDataset(const std::vector<std::string> &file_paths, bool load_dataset) { in BuildDataset()
159 …std::thread(&ShardHeader::GetHeadersOneTask, this, start_num, end_num, std::ref(headers), file_pat… in BuildDataset()
174 void ShardHeader::GetHeadersOneTask(int start, int end, std::vector<json> &headers, in GetHeadersOneTask()
198 Status ShardHeader::InitByFiles(const std::vector<std::string> &file_paths) { in InitByFiles()
214 Status ShardHeader::ParseIndexFields(const json &index_fields) { in ParseIndexFields()
[all …]
|
| D | shard_column.cc | 25 ShardColumn::ShardColumn(const std::shared_ptr<ShardHeader> &shard_header, bool compress_integer) { in ShardColumn()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/common/ |
| D | shard_pybind.cc | 79 (void)py::class_<ShardHeader, std::shared_ptr<ShardHeader>>(*m, "ShardHeader", py::module_local()) in BindShardHeader()
81 .def("add_schema", &ShardHeader::AddSchema) in BindShardHeader()
82 .def("add_statistics", &ShardHeader::AddStatistic) in BindShardHeader()
84 [](ShardHeader &s, const std::vector<std::string> &fields) { in BindShardHeader()
88 .def("get_meta", &ShardHeader::GetSchemas) in BindShardHeader()
89 .def("get_statistics", &ShardHeader::GetStatistics) in BindShardHeader()
90 .def("get_fields", &ShardHeader::GetFields) in BindShardHeader()
92 [](ShardHeader &s, int64_t schema_id) { in BindShardHeader()
97 .def("get_statistic_by_id", [](ShardHeader &s, int64_t statistic_id) { in BindShardHeader()
128 [](ShardWriter &s, std::shared_ptr<ShardHeader> header_data) { in BindShardWriter()
|
| /third_party/mindspore/tests/ut/cpp/mindrecord/ |
| D | ut_shard_writer_test.cc | 117 mindrecord::ShardHeader header_data; in TEST_F()
201 fw.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in TEST_F()
249 mindrecord::ShardHeader header_data; in TEST_F()
338 fw.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in TEST_F()
388 mindrecord::ShardHeader header_data; in TEST_F()
473 fw.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in TEST_F()
522 mindrecord::ShardHeader header_data; in TEST_F()
574 fw.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in TEST_F()
623 mindrecord::ShardHeader header_data; in TEST_F()
675 fw.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in TEST_F()
[all …]
|
| D | ut_common.cc | 164 ShardHeader header_data; in ShardWriterImageNet()
212 fw_init.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in ShardWriterImageNet()
243 mindrecord::ShardHeader header_data; in ShardWriterImageNetOneSample()
291 fw_init.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in ShardWriterImageNetOneSample()
331 mindrecord::ShardHeader header_data; in ShardWriterImageNetOpenForAppend()
379 fw_init.SetShardHeader(std::make_shared<mindrecord::ShardHeader>(header_data)); in ShardWriterImageNetOpenForAppend()
|
| D | ut_shard_header_test.cc | 69 mindrecord::ShardHeader header_data; in TEST_F()
123 mindrecord::ShardHeader header_data_new; in TEST_F()
|
| D | ut_shard.cc | 108 mindrecord::ShardHeader header_data; in TEST_F()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/include/ |
| D | shard_header.h | 34 class __attribute__((visibility("default"))) ShardHeader {
36 ShardHeader();
38 ~ShardHeader() = default;
138 static Status Initialize(const std::shared_ptr<ShardHeader> *header_ptr, const json &schema,
|
| D | shard_writer.h | 84 Status SetShardHeader(std::shared_ptr<ShardHeader> header_data);
243 std::shared_ptr<ShardHeader> shard_header_; // shard header
|
| D | shard_reader.h | 106 std::shared_ptr<ShardHeader> GetShardHeader() const;
299 std::shared_ptr<ShardHeader> shard_header_; // shard header
|
| D | shard_index_generator.h | 112 ShardHeader shard_header_;
|
| D | shard_column.h | 62 …explicit ShardColumn(const std::shared_ptr<ShardHeader> &shard_header, bool compress_integer = tru…
|
| /third_party/mindspore/tests/ut/python/mindrecord/ |
| D | skip_test_issue.py | 23 from mindspore.mindrecord import ShardHeader, SUCCESS, FAILED, ParamValueError
112 header = ShardHeader()
125 header = ShardHeader()
137 header = ShardHeader()
148 header = ShardHeader()
196 header = ShardHeader()
207 header = ShardHeader()
403 header = ShardHeader()
415 header = ShardHeader()
|
| D | skip_test_mindrecord_shard.py | 21 from mindspore.mindrecord import ShardHeader, SUCCESS
44 header = ShardHeader()
117 header = ShardHeader()
192 header = ShardHeader()
264 header = ShardHeader()
|
| /third_party/mindspore/mindspore/mindrecord/ |
| D | filereader.py | 19 from .shardheader import ShardHeader
58 self._header = ShardHeader(self._reader.get_header())
|
| D | shardheader.py | 25 class ShardHeader: class
35 self._header = ms.ShardHeader()
|
| D | shardsegment.py | 21 from .shardheader import ShardHeader
65 self._header = ShardHeader(self._segment.get_header())
|
| D | filewriter.py | 25 from .shardheader import ShardHeader
97 self._header = ShardHeader()
141 header = ShardHeader(reader.get_header())
|
| D | __init__.inter | 39 from .shardheader import ShardHeader
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/io/ |
| D | shard_index_generator.cc | 40 RETURN_IF_NOT_OK(ShardHeader::BuildSingleHeader(file_path_, &header_ptr)); in Build()
43 ShardHeader header = ShardHeader(); in Build()
|
| D | shard_writer.cc | 152 RETURN_IF_NOT_OK(ShardHeader::BuildSingleHeader(path, &header_ptr)); in OpenForAppend()
155 ShardHeader header = ShardHeader(); in OpenForAppend()
157 shard_header_ = std::make_shared<ShardHeader>(header); in OpenForAppend()
181 Status ShardWriter::SetShardHeader(std::shared_ptr<ShardHeader> header_data) { in SetShardHeader()
|
| D | shard_reader.cc | 61 RETURN_IF_NOT_OK(ShardHeader::BuildSingleHeader(file_path, &header_ptr)); in GetMeta()
94 ShardHeader sh = ShardHeader(); in Init()
96 shard_header_ = std::make_shared<ShardHeader>(sh); in Init()
285 std::shared_ptr<ShardHeader> ShardReader::GetShardHeader() const { return shard_header_; } in GetShardHeader()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/consumers/ |
| D | tree_consumer.cc | 256 auto mr_header = std::make_shared<mindrecord::ShardHeader>(); in Save()
288 …mindrecord::ShardHeader::Initialize(&mr_header, mr_json, index_fields, blob_fields, mr_schema_id)); in Save()
|