| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/meta/ |
| D | shard_schema.cc | 32 std::vector<std::string> blob_fields = PopulateBlobFields(schema); in Build() local
35 object_schema.blob_fields_ = std::move(blob_fields); in Build()
58 std::vector<std::string> blob_fields; in PopulateBlobFields() local
62 blob_fields.emplace_back(it.key()); in PopulateBlobFields()
65 return blob_fields; in PopulateBlobFields()
|
| D | shard_header.cc | 281 std::vector<std::string> blob_fields = schema["blob_fields"].get<std::vector<std::string>>(); in ParseSchema() local
632 … const std::vector<std::string> &index_fields, std::vector<std::string> &blob_fields, in Initialize() argument
647 blob_fields = build_schema_ptr->GetBlobFields(); in Initialize()
|
| D | shard_column.cc | 35 auto blob_fields = schema_json["blob_fields"]; in Init() local
63 for (const auto &field : blob_fields) { in Init()
|
| /third_party/mindspore/mindspore/mindrecord/ |
| D | shardutils.py | 141 def populate_data(raw, blob, columns, blob_fields, schema): argument
160 if not blob_fields:
166 if column in blob_fields:
169 loaded_columns = blob_fields
|
| D | shardsegment.py | 108 return [populate_data(raw, blob, self._columns, self._header.blob_fields,
127 return [populate_data(raw, blob, self._columns, self._header.blob_fields,
|
| D | shardwriter.py | 169 row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields})
174 … for field in self._header.schema.keys() - self._header.blob_fields if field in item}
|
| D | filereader.py | 74 … yield populate_data(raw, blob, self._columns, self._header.blob_fields, self._header.schema)
|
| D | shardheader.py | 134 def blob_fields(self): member in ShardHeader
|
| D | filewriter.py | 206 if field in self._header.blob_fields:
|
| /third_party/mindspore/tests/ut/python/mindrecord/ |
| D | skip_test_mindrecord_shard.py | 149 _, blob_fields = dataset.get_blob_fields()
153 raw[blob_fields[0]] = bytes(blob)
335 _, blob_fields = dataset.get_blob_fields()
339 raw[blob_fields[0]] = bytes(blob)
|
| D | skip_test_issue.py | 179 _, blob_fields = reader.get_blob_fields()
183 raw[blob_fields[0]] = bytes(blob)
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/io/ |
| D | shard_segment.cc | 312 std::vector<std::string> blob_fields; in GetBlobFields() local
316 blob_fields.assign(fields.begin(), fields.end()); in GetBlobFields()
318 return std::make_pair(kCV, blob_fields); in GetBlobFields()
|
| D | shard_writer.cc | 549 Status ShardWriter::MergeBlobData(const std::vector<string> &blob_fields, in MergeBlobData() argument
552 if (blob_fields.empty()) { in MergeBlobData()
555 if (blob_fields.size() == 1) { in MergeBlobData()
556 auto &blob = row_bin_data.at(blob_fields[0]); in MergeBlobData()
562 for (auto &field : blob_fields) { in MergeBlobData()
565 output_size += blob_fields.size() * sizeof(uint64_t); in MergeBlobData()
569 for (auto &field : blob_fields) { in MergeBlobData()
|
| D | shard_reader.cc | 685 std::vector<std::string> blob_fields; in GetBlobFields() local
689 blob_fields.assign(fields.begin(), fields.end()); in GetBlobFields()
692 return std::make_pair(kCV, blob_fields); in GetBlobFields()
1385 auto blob_fields = GetBlobFields().second; in UnCompressBlob() local
1387 …if (std::find(blob_fields.begin(), blob_fields.end(), loaded_columns[i_col]) == blob_fields.end())… in UnCompressBlob()
|
| /third_party/mindspore/mindspore/ccsrc/minddata/mindrecord/include/ |
| D | shard_header.h | 139 … const std::vector<std::string> &index_fields, std::vector<std::string> &blob_fields,
|
| D | shard_writer.h | 103 Status MergeBlobData(const std::vector<string> &blob_fields,
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/engine/consumers/ |
| D | tree_consumer.cc | 258 std::vector<std::string> blob_fields; in Save() local
288 …mindrecord::ShardHeader::Initialize(&mr_header, mr_json, index_fields, blob_fields, mr_schema_id)); in Save()
296 RETURN_IF_NOT_OK(mr_writer->MergeBlobData(blob_fields, row_bin_data, &output_bin_data)); in Save()
|