Home
last modified time | relevance | path

Searched refs:shard_count_ (Results 1 – 6 of 6) sorted by relevance

/third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/meta/
Dshard_header.cc37 ShardHeader::ShardHeader() : shard_count_(0), header_size_(0), page_size_(0), compression_size_(0) { in ShardHeader()
42 shard_count_ = headers.size(); in InitializeHeader()
206 shard_count_ = file_paths.size(); in InitByFiles()
207 CHECK_FAIL_RETURN_UNEXPECTED(shard_count_ != 0 && (shard_count_ <= kMaxShardCount), in InitByFiles()
208 … "Invalid input, The number of MindRecord files " + std::to_string(shard_count_) + in InitByFiles()
210 pages_.resize(shard_count_); in InitByFiles()
228 …CHECK_FAIL_RETURN_UNEXPECTED(shard_count_ <= kMaxFileCount, "Invalid input, The number of MindReco… in ParsePage()
229 … std::to_string(shard_count_) + "is not int range (0, " + in ParsePage()
232 pages_.resize(shard_count_); in ParsePage()
301 if (shard_count_ > static_cast<int>(pages.size())) { in SerializeHeader()
[all …]
/third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/io/
Dshard_reader.cc46 shard_count_(0), in ShardReader()
458 std::vector<std::thread> threads = std::vector<std::thread>(shard_count_); in GetAllClasses()
459 for (int x = 0; x < shard_count_; x++) { in GetAllClasses()
463 for (int x = 0; x < shard_count_; x++) { in GetAllClasses()
498 shard_count_, std::vector<std::vector<uint64_t>>{}); in ReadAllRowGroup()
499 …auto col_val_ptr = std::make_shared<std::vector<std::vector<json>>>(shard_count_, std::vector<json… in ReadAllRowGroup()
515 std::vector<std::thread> thread_read_db = std::vector<std::thread>(shard_count_); in ReadAllRowGroup()
516 for (int x = 0; x < shard_count_; x++) { in ReadAllRowGroup()
520 for (int x = 0; x < shard_count_; x++) { in ReadAllRowGroup()
533 shard_count_, std::vector<std::vector<uint64_t>>{}); in ReadRowGroupByShardIDAndSampleID()
[all …]
Dshard_writer.cc33 …: shard_count_(1), header_size_(kDefaultHeaderSize), page_size_(kDefaultPageSize), row_count_(0), … in ShardWriter()
135 shard_count_ = paths.size(); in Open()
347 int batch_size = rawdata_iter->second.size() / shard_count_; in CheckData()
348 int thread_num = shard_count_; in CheckData()
658 int thread_num = static_cast<int>(shard_count_); in ParallelWriteData()
663 int left_thread = shard_count_; in ParallelWriteData()
992 int row_in_shard = row_count_ / shard_count_; in BreakIntoShards()
993 int remains = row_count_ % shard_count_; in BreakIntoShards()
995 std::vector<int> v_list(shard_count_); in BreakIntoShards()
1002 if (shard_count_ <= kMaxShardCount) { in BreakIntoShards()
[all …]
/third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/include/
Dshard_header.h116 int GetShardCount() const { return shard_count_; } in GetShardCount()
186 uint32_t shard_count_; variable
Dshard_writer.h232 int shard_count_; // number of files variable
Dshard_reader.h298 int shard_count_; // number of shards variable