| /third_party/mindspore/mindspore/mindrecord/ |
| D | shardreader.py | 52 load_dataset = False
54 load_dataset = True
56 ret = self._reader.open(file_name, load_dataset, num_consumer, columns, operator)
|
| D | shardsegment.py | 57 load_dataset = False
59 load_dataset = True
61 ret = self._segment.open(file_name, load_dataset, num_consumer, self._columns, operator)
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/ |
| D | mindrecord_op.h | 63 …RecordOp(int32_t num_mind_record_workers, std::vector<std::string> dataset_file, bool load_dataset,
103 static Status CountTotalRows(const std::vector<std::string> dataset_path, bool load_dataset,
112 bool load_dataset() const { return load_dataset_; } in load_dataset() function
|
| D | mindrecord_op.cc | 42 …RecordOp(int32_t num_mind_record_workers, std::vector<std::string> dataset_file, bool load_dataset, in MindRecordOp() argument
50 load_dataset_(load_dataset), in MindRecordOp()
318 Status MindRecordOp::CountTotalRows(const std::vector<std::string> dataset_path, bool load_dataset, in CountTotalRows() argument
321 RETURN_IF_NOT_OK(shard_reader->CountTotalRows(dataset_path, load_dataset, op, count, num_padded)); in CountTotalRows()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/include/ |
| D | shard_header.h | 40 Status BuildDataset(const std::vector<std::string> &file_paths, bool load_dataset = true);
143 Status InitializeHeader(const std::vector<json> &headers, bool load_dataset);
164 Status ParsePage(const json &page, int shard_index, bool load_dataset);
|
| D | shard_reader.h | 82 Status Open(const std::vector<std::string> &file_paths, bool load_dataset, int n_consumer = 4,
122 Status CountTotalRows(const std::vector<std::string> &file_paths, bool load_dataset,
225 Status Init(const std::vector<std::string> &file_paths, bool load_dataset);
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/meta/ |
| D | shard_header.cc | 41 Status ShardHeader::InitializeHeader(const std::vector<json> &headers, bool load_dataset) { in InitializeHeader() argument
56 RETURN_IF_NOT_OK(ParsePage(header["page"], shard_index, load_dataset)); in InitializeHeader()
141 Status ShardHeader::BuildDataset(const std::vector<std::string> &file_paths, bool load_dataset) { in BuildDataset() argument
170 RETURN_IF_NOT_OK(InitializeHeader(headers, load_dataset)); in BuildDataset()
226 Status ShardHeader::ParsePage(const json &pages, int shard_index, bool load_dataset) { in ParsePage() argument
251 if (load_dataset == true) { in ParsePage()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/common/ |
| D | shard_pybind.cc | 158 …[](ShardReader &s, const std::vector<std::string> &file_paths, bool load_dataset, const int &n_con… in BindShardReader()
161 … THROW_IF_ERROR(s.Open(file_paths, load_dataset, n_consumer, selected_columns, operators)); in BindShardReader()
206 …[](ShardSegment &s, const std::vector<std::string> &file_paths, bool load_dataset, const int &n_co… in BindShardSegment()
209 … THROW_IF_ERROR(s.Open(file_paths, load_dataset, n_consumer, selected_columns, operators)); in BindShardSegment()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/io/ |
| D | shard_reader.cc | 70 Status ShardReader::Init(const std::vector<std::string> &file_paths, bool load_dataset) { in Init() argument
75 if (file_paths.size() == 1 && load_dataset == true) { in Init()
79 } else if (file_paths.size() >= 1 && load_dataset == false) { in Init()
83 … "] or 'load_dataset' [" + std::to_string(load_dataset) + "]is invalid."); in Init()
95 RETURN_IF_NOT_OK(sh.BuildDataset(file_paths_, load_dataset)); in Init()
924 Status ShardReader::CountTotalRows(const std::vector<std::string> &file_paths, bool load_dataset, in CountTotalRows() argument
926 RETURN_IF_NOT_OK(Init(file_paths, load_dataset)); in CountTotalRows()
982 Status ShardReader::Open(const std::vector<std::string> &file_paths, bool load_dataset, int n_consu… in Open() argument
988 RETURN_IF_NOT_OK(Init(file_paths, load_dataset)); in Open()
|
| /third_party/mindspore/tests/st/fl/albert/src/ |
| D | dataset.py | 78 def load_dataset(data_path, max_seq_length, tokenizer, batch_size, label_list=None, do_shuffle=True, function
|
| /third_party/mindspore/mindspore/dataset/engine/ |
| D | datasets.py | 3612 self.load_dataset = False
3614 self.load_dataset = True
|