Home
last modified time | relevance | path

Searched refs:worker_id (Results 1 – 25 of 77) sorted by relevance

1234

/third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/
Dconnector.h97 …virtual Status Pop(int32_t worker_id, // The worker-id of the caller. See the requirement at the … in Pop() argument
100 MS_ASSERT(worker_id < num_consumers_); in Pop()
102 … RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); in Pop()
118 Status Push(int32_t worker_id, const T &el) noexcept { in Push() argument
119 MS_ASSERT(worker_id < static_cast<int32_t>(queues_.size())); in Push()
120 MS_ASSERT(queues_[worker_id] != nullptr); in Push()
121 return (queues_[worker_id]->Add(el)); in Push()
131 virtual Status Push(int32_t worker_id, T &&el) noexcept { in Push() argument
132 MS_ASSERT(worker_id < static_cast<int32_t>(queues_.size())); in Push()
133 MS_ASSERT(queues_[worker_id] != nullptr); in Push()
[all …]
Ddb_connector.h48 Status Add(TensorRow &&el, int32_t worker_id = 0) noexcept {
49 return (Connector<TensorRow>::Push(worker_id, std::move(el)));
52 Status SendEOE(int32_t worker_id = 0) noexcept {
54 return Add(std::move(eoe), worker_id);
57 Status SendEOF(int32_t worker_id = 0) noexcept {
59 return Add(std::move(eof), worker_id);
69 Status PopWithRetry(int32_t worker_id, TensorRow *result, bool retry_if_eoe = false) noexcept {
75 …RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return (expect_consumer_ == worker_id) || end…
Djagged_connector.h45 Status Pop(int32_t worker_id, TensorRow *result) noexcept override { in Pop() argument
48 MS_ASSERT(worker_id < num_consumers_); in Pop()
50 … RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; })); in Pop()
Dgpu_item_connector.h48 Status Pop(int32_t worker_id, std::vector<device::DataItemGpu> *result) noexcept override { in Pop() argument
51 MS_ASSERT(worker_id < num_consumers_); in Pop()
53 … RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; })); in Pop()
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/
Drandom_data_op.cc171 Status RandomDataOp::EpochSync(int32_t worker_id, bool *quitting) { in EpochSync() argument
172 MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " syncing at end of epoch"; in EpochSync()
187 …MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " is the last one to sync. eoe sent as work… in EpochSync()
198 …MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " has no more epochs. sending eof as worke… in EpochSync()
205 MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " entering sync wait."; in EpochSync()
217 MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " last guy out clearing wait post."; in EpochSync()
224 MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " epoch sync complete."; in EpochSync()
229 Status RandomDataOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
230 MS_LOG(INFO) << "RandomDataOp worker " << worker_id << " entry"; in WorkerEntry()
241 if (worker_rows_packed_[worker_id] < worker_max_rows_[worker_id]) { in WorkerEntry()
[all …]
Dmappable_leaf_op.cc91 Status MappableLeafOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
94 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); in WorkerEntry()
103 RETURN_IF_NOT_OK(out_connector_->SendEOE(worker_id)); in WorkerEntry()
105 RETURN_IF_NOT_OK(out_connector_->SendEOF(worker_id)); in WorkerEntry()
112 RETURN_IF_NOT_OK(out_connector_->Add(std::move(trow), worker_id)); in WorkerEntry()
114 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); in WorkerEntry()
Dmindrecord_op.cc146 Status MindRecordOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
149 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); in WorkerEntry()
157 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); in WorkerEntry()
161 RETURN_IF_NOT_OK(out_connector_->SendEOE(worker_id)); in WorkerEntry()
162 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); in WorkerEntry()
166 RETURN_IF_NOT_OK(out_connector_->SendEOF(worker_id)); in WorkerEntry()
167 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); in WorkerEntry()
188 …MS_LOG(DEBUG) << "MindRecord operator consumed row " << row_id << " by worker " << worker_id << ".… in WorkerEntry()
190 RETURN_IF_NOT_OK(GetRowFromReader(&fetched_row, row_id, worker_id)); in WorkerEntry()
191 RETURN_IF_NOT_OK(out_connector_->Add(std::move(fetched_row), worker_id)); in WorkerEntry()
[all …]
Dnonmappable_leaf_op.cc131 Status NonMappableLeafOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
136 RETURN_IF_NOT_OK(PopIoBlockQueue(worker_id, &io_block)); in WorkerEntry()
145 RETURN_IF_NOT_OK(LoadFile(filename, start_offset, end_offset, worker_id)); in WorkerEntry()
146 … MS_LOG(DEBUG) << Name() << " operator worker " << worker_id << " loaded file " << filename << "."; in WorkerEntry()
150 RETURN_IF_NOT_OK(jagged_rows_connector_->Add(worker_id, std::move(eoe))); in WorkerEntry()
153 RETURN_IF_NOT_OK(PopIoBlockQueue(worker_id, &io_block)); in WorkerEntry()
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/datasetops/
Drepeat_op.cc60 Status RepeatOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
65 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, true)); in GetNextRow()
68 RETURN_IF_NOT_OK(EoeReceived(worker_id)); in GetNextRow()
72 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, true)); in GetNextRow()
76 RETURN_IF_NOT_OK(EofReceived(worker_id)); in GetNextRow()
82 Status RepeatOp::EoeReceived(int32_t worker_id) { in EoeReceived() argument
113 Status RepeatOp::EofReceived(int32_t worker_id) { in EofReceived() argument
Dcache_op.cc63 Status CacheOp::CacheAllRows(int32_t worker_id) { in CacheAllRows() argument
68 if (worker_id == 0) { in CacheAllRows()
71 MS_LOG(INFO) << "CacheOp first epoch SAVE mode started. Worker: " << worker_id; in CacheAllRows()
74 auto child_iterator = std::make_unique<ChildIterator>(this, worker_id, 0); in CacheAllRows()
163 Status CacheOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
165 RETURN_IF_NOT_OK(CacheAllRows(worker_id)); in WorkerEntry()
166 RETURN_IF_NOT_OK(FetchFromCache(worker_id)); in WorkerEntry()
180 Status CacheOp::EoeReceived(int32_t worker_id) { in EoeReceived() argument
185 Status CacheOp::EofReceived(int32_t worker_id) { in EofReceived() argument
Dzip_op.cc32 Status ZipOp::getNextZippedRow(TensorRow *const new_zip_row, int32_t *skip_child, int32_t worker_id, in getNextZippedRow() argument
38 RETURN_IF_NOT_OK(child_[i]->GetNextRow(&new_row, worker_id, retry_if_eoe)); in getNextZippedRow()
52 Status ZipOp::drainPipeline(int32_t skip_child, int32_t worker_id, bool retry_if_eoe) { in drainPipeline() argument
58 RETURN_IF_NOT_OK(child_[con]->GetNextRow(&row, worker_id, retry_if_eoe)); in drainPipeline()
122 Status ZipOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
124 RETURN_IF_NOT_OK(getNextZippedRow(row, &skip_child, worker_id, retry_if_eoe)); in GetNextRow()
128 RETURN_IF_NOT_OK(drainPipeline(skip_child, worker_id, retry_if_eoe)); in GetNextRow()
Dcache_op.h53 Status EoeReceived(int32_t worker_id) override;
58 Status EofReceived(int32_t worker_id) override;
67 Status WorkerEntry(int32_t worker_id) override;
88 Status CacheAllRows(int32_t worker_id);
Depoch_ctrl_op.cc43 Status EpochCtrlOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
49 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, false)); in GetNextRow()
55 RETURN_IF_NOT_OK(EoeReceived(worker_id)); in GetNextRow()
61 Status EpochCtrlOp::EoeReceived(int32_t worker_id) { in EoeReceived() argument
Dcache_base_op.cc75 auto send_to_que = [](QueueList<std::unique_ptr<IOBlock>> &qList, int32_t worker_id, in FetchSamplesToWorkers()
78 RETURN_IF_NOT_OK(qList[worker_id]->Add(std::move(blk))); in FetchSamplesToWorkers()
165 Status CacheBase::FetchFromCache(int32_t worker_id) { in FetchFromCache() argument
168 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&blk)); in FetchFromCache()
176 RETURN_IF_NOT_OK(out_connector_->SendEOF(worker_id)); in FetchFromCache()
178 RETURN_IF_NOT_OK(out_connector_->SendEOE(worker_id)); in FetchFromCache()
198 RETURN_IF_NOT_OK(out_connector_->Add(std::move(row), worker_id)); in FetchFromCache()
272 Status CacheBase::Prefetcher(int32_t worker_id) { in Prefetcher() argument
282 RETURN_IF_NOT_OK(prefetch_queues_[worker_id]->PopFront(&blk)); in Prefetcher()
318 RETURN_IF_NOT_OK(keys_miss_->Push(worker_id, cache_miss)); in Prefetcher()
Dcache_merge_op.cc72 Status CacheMergeOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
75 auto child_iterator = std::make_unique<ChildIterator>(this, worker_id, kCacheHitChildIdx); in WorkerEntry()
79 RETURN_IF_NOT_OK(EoeReceived(worker_id)); in WorkerEntry()
87 RETURN_IF_NOT_OK(out_connector_->Add(std::move(new_row), worker_id)); in WorkerEntry()
91 RETURN_IF_NOT_OK(EofReceived(worker_id)); in WorkerEntry()
219 Status CacheMergeOp::EoeReceived(int32_t worker_id) { in EoeReceived() argument
222 return out_connector_->SendEOE(worker_id); in EoeReceived()
226 Status CacheMergeOp::EofReceived(int32_t worker_id) { in EofReceived() argument
229 return out_connector_->SendEOF(worker_id); in EofReceived()
Dfilter_op.cc119 Status FilterOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
122 RETURN_IF_NOT_OK(worker_queues_[worker_id]->PopFront(&new_row)); in WorkerEntry()
127 …RETURN_IF_NOT_OK(filter_queues_[worker_id]->EmplaceBack(std::make_pair(new_row, filterCtrl::kFilte… in WorkerEntry()
129 …RETURN_IF_NOT_OK(filter_queues_[worker_id]->EmplaceBack(std::make_pair(new_row, filterCtrl::kFilte… in WorkerEntry()
138 …filter_queues_[worker_id]->EmplaceBack(std::make_pair(std::move(new_row), filterCtrl::kFilterFull)… in WorkerEntry()
141 …filter_queues_[worker_id]->EmplaceBack(std::make_pair(std::move(new_row), filterCtrl::kFilterEmpty… in WorkerEntry()
143 RETURN_IF_NOT_OK(worker_queues_[worker_id]->PopFront(&new_row)); in WorkerEntry()
Dzip_op.h67 Status GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) override;
76 Status drainPipeline(int32_t skip_child, int32_t worker_id, bool retry_if_eoe);
89 …Status getNextZippedRow(TensorRow *const new_zip_row, int32_t *skip_child, int32_t worker_id, bool…
Dskip_op.cc49 Status SkipOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
52 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
60 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
Dproject_op.h62 Status GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) override;
75 Status EoeReceived(int32_t worker_id) override;
80 Status EofReceived(int32_t worker_id) override;
Dproject_op.cc51 Status ProjectOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
52 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
100 Status ProjectOp::EoeReceived(int32_t worker_id) { in EoeReceived() argument
105 Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); } in EofReceived() argument
Dconcat_op.cc150 Status ConcatOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
157 RETURN_IF_NOT_OK(child_[cur_child_]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
163 RETURN_IF_NOT_OK(GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
182 RETURN_IF_NOT_OK(GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
188 RETURN_IF_NOT_OK(child_[i]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
Dtake_op.cc44 Status TakeOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { in GetNextRow() argument
47 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
58 RETURN_IF_NOT_OK(child_[0]->GetNextRow(row, worker_id, retry_if_eoe)); in GetNextRow()
Drepeat_op.h70 Status GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) override;
74 Status EoeReceived(int32_t worker_id) override;
78 Status EofReceived(int32_t worker_id) override;
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/cache/perf/
Dcache_pipeline_run.cc226 int32_t worker_id = 0; in RunFirstEpoch() local
230 RETURN_IF_NOT_OK(io_block_queues_[worker_id++ % num_workers]->Add(std::move(blk))); in RunFirstEpoch()
235 RETURN_IF_NOT_OK(io_block_queues_[worker_id++ % num_workers]->Add(std::move(blk))); in RunFirstEpoch()
260 Status CachePipelineRun::WriterWorkerEntry(int32_t worker_id) { in WriterWorkerEntry() argument
275 RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&blk)); in WriterWorkerEntry()
302 … MS_LOG(WARNING) << "Pipeline number " << (my_pipeline_ + 1) << " worker id " << worker_id << ": " in WriterWorkerEntry()
326 proto.set_worker(worker_id); in WriterWorkerEntry()
371 int32_t worker_id = 0; in RunReadEpoch() local
375 RETURN_IF_NOT_OK(io_block_queues_[worker_id++ % num_workers]->Add(std::move(blk))); in RunReadEpoch()
380 RETURN_IF_NOT_OK(io_block_queues_[worker_id++ % num_workers]->Add(std::move(blk))); in RunReadEpoch()
[all …]
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/
Dmap_op.cc77 Status MapOp::FetchNextWork(uint32_t worker_id, TensorRow *row, std::vector<std::shared_ptr<MapJob>… in FetchNextWork() argument
80 RETURN_IF_NOT_OK(local_queues_[worker_id]->PopFront(&worker_job)); in FetchNextWork()
200 Status MapOp::WorkerEntry(int32_t worker_id) { in WorkerEntry() argument
207 RETURN_IF_NOT_OK(FetchNextWork(worker_id, &in_row, &job_list)); in WorkerEntry()
224 RETURN_IF_NOT_OK(out_connector_->SendEOE(worker_id)); in WorkerEntry()
227 RETURN_IF_NOT_OK(out_connector_->SendEOF(worker_id)); in WorkerEntry()
231 RETURN_IF_NOT_OK(FetchNextWork(worker_id, &in_row, &job_list)); in WorkerEntry()
239 RETURN_IF_NOT_OK(out_connector_->Add(std::move(out_row), static_cast<int>(worker_id))); in WorkerEntry()
241 RETURN_IF_NOT_OK(FetchNextWork(worker_id, &in_row, &job_list)); in WorkerEntry()

1234