• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 // Note: ported from Chromium commit head: c3bd64c
5 
6 #include <algorithm>
7 #include <limits>
8 
9 #include "base/bind.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback_helpers.h"
12 #include "base/macros.h"
13 #include "base/numerics/safe_conversions.h"
14 #include "base/optional.h"
15 #include "base/stl_util.h"
16 #include "h264_decoder.h"
17 
18 namespace media {
19 
20 H264Decoder::H264Accelerator::H264Accelerator() = default;
21 
22 H264Decoder::H264Accelerator::~H264Accelerator() = default;
23 
H264Decoder(H264Accelerator * accelerator)24 H264Decoder::H264Decoder(H264Accelerator* accelerator)
25     : state_(kNeedStreamMetadata),
26       max_frame_num_(0),
27       max_pic_num_(0),
28       max_long_term_frame_idx_(0),
29       max_num_reorder_frames_(0),
30       accelerator_(accelerator) {
31   DCHECK(accelerator_);
32   Reset();
33 }
34 
35 H264Decoder::~H264Decoder() = default;
36 
Reset()37 void H264Decoder::Reset() {
38   curr_pic_ = nullptr;
39   curr_nalu_ = nullptr;
40   curr_slice_hdr_ = nullptr;
41   curr_sps_id_ = -1;
42   curr_pps_id_ = -1;
43 
44   prev_frame_num_ = -1;
45   prev_ref_frame_num_ = -1;
46   prev_frame_num_offset_ = -1;
47   prev_has_memmgmnt5_ = false;
48 
49   prev_ref_has_memmgmnt5_ = false;
50   prev_ref_top_field_order_cnt_ = -1;
51   prev_ref_pic_order_cnt_msb_ = -1;
52   prev_ref_pic_order_cnt_lsb_ = -1;
53   prev_ref_field_ = H264Picture::FIELD_NONE;
54 
55   ref_pic_list_p0_.clear();
56   ref_pic_list_b0_.clear();
57   ref_pic_list_b1_.clear();
58   dpb_.Clear();
59   parser_.Reset();
60   accelerator_->Reset();
61   last_output_poc_ = std::numeric_limits<int>::min();
62 
63   // If we are in kDecoding, we can resume without processing an SPS.
64   if (state_ == kDecoding)
65     state_ = kAfterReset;
66 }
67 
PrepareRefPicLists(const H264SliceHeader * slice_hdr)68 void H264Decoder::PrepareRefPicLists(const H264SliceHeader* slice_hdr) {
69   ConstructReferencePicListsP(slice_hdr);
70   ConstructReferencePicListsB(slice_hdr);
71 }
72 
ModifyReferencePicLists(const H264SliceHeader * slice_hdr,H264Picture::Vector * ref_pic_list0,H264Picture::Vector * ref_pic_list1)73 bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr,
74                                           H264Picture::Vector* ref_pic_list0,
75                                           H264Picture::Vector* ref_pic_list1) {
76   ref_pic_list0->clear();
77   ref_pic_list1->clear();
78 
79   // Fill reference picture lists for B and S/SP slices.
80   if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) {
81     *ref_pic_list0 = ref_pic_list_p0_;
82     return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0);
83   } else if (slice_hdr->IsBSlice()) {
84     *ref_pic_list0 = ref_pic_list_b0_;
85     *ref_pic_list1 = ref_pic_list_b1_;
86     return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) &&
87            ModifyReferencePicList(slice_hdr, 1, ref_pic_list1);
88   }
89 
90   return true;
91 }
92 
DecodePicture()93 bool H264Decoder::DecodePicture() {
94   DCHECK(curr_pic_.get());
95 
96   DVLOG(4) << "Decoding POC " << curr_pic_->pic_order_cnt;
97   return accelerator_->SubmitDecode(curr_pic_);
98 }
99 
InitNonexistingPicture(scoped_refptr<H264Picture> pic,int frame_num)100 bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic,
101                                          int frame_num) {
102   pic->nonexisting = true;
103   pic->nal_ref_idc = 1;
104   pic->frame_num = pic->pic_num = frame_num;
105   pic->adaptive_ref_pic_marking_mode_flag = false;
106   pic->ref = true;
107   pic->long_term_reference_flag = false;
108   pic->field = H264Picture::FIELD_NONE;
109 
110   return CalculatePicOrderCounts(pic);
111 }
112 
InitCurrPicture(const H264SliceHeader * slice_hdr)113 bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) {
114   DCHECK(curr_pic_.get());
115 
116   curr_pic_->idr = slice_hdr->idr_pic_flag;
117   if (curr_pic_->idr)
118     curr_pic_->idr_pic_id = slice_hdr->idr_pic_id;
119 
120   if (slice_hdr->field_pic_flag) {
121     curr_pic_->field = slice_hdr->bottom_field_flag ? H264Picture::FIELD_BOTTOM
122                                                     : H264Picture::FIELD_TOP;
123   } else {
124     curr_pic_->field = H264Picture::FIELD_NONE;
125   }
126 
127   if (curr_pic_->field != H264Picture::FIELD_NONE) {
128     DVLOG(1) << "Interlaced video not supported.";
129     return false;
130   }
131 
132   curr_pic_->nal_ref_idc = slice_hdr->nal_ref_idc;
133   curr_pic_->ref = slice_hdr->nal_ref_idc != 0;
134   // This assumes non-interlaced stream.
135   curr_pic_->frame_num = curr_pic_->pic_num = slice_hdr->frame_num;
136 
137   DCHECK_NE(curr_sps_id_, -1);
138   const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
139   if (!sps)
140     return false;
141 
142   curr_pic_->pic_order_cnt_type = sps->pic_order_cnt_type;
143   switch (curr_pic_->pic_order_cnt_type) {
144     case 0:
145       curr_pic_->pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb;
146       curr_pic_->delta_pic_order_cnt_bottom =
147           slice_hdr->delta_pic_order_cnt_bottom;
148       break;
149 
150     case 1:
151       curr_pic_->delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt0;
152       curr_pic_->delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt1;
153       break;
154 
155     case 2:
156       break;
157 
158     default:
159       NOTREACHED();
160       return false;
161   }
162 
163   if (!CalculatePicOrderCounts(curr_pic_))
164     return false;
165 
166   curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag;
167   curr_pic_->adaptive_ref_pic_marking_mode_flag =
168       slice_hdr->adaptive_ref_pic_marking_mode_flag;
169 
170   // If the slice header indicates we will have to perform reference marking
171   // process after this picture is decoded, store required data for that
172   // purpose.
173   if (slice_hdr->adaptive_ref_pic_marking_mode_flag) {
174     static_assert(sizeof(curr_pic_->ref_pic_marking) ==
175                       sizeof(slice_hdr->ref_pic_marking),
176                   "Array sizes of ref pic marking do not match.");
177     memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking,
178            sizeof(curr_pic_->ref_pic_marking));
179   }
180 
181   curr_pic_->visible_rect = visible_rect_;
182 
183   return true;
184 }
185 
CalculatePicOrderCounts(scoped_refptr<H264Picture> pic)186 bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) {
187   const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
188   if (!sps)
189     return false;
190 
191   switch (pic->pic_order_cnt_type) {
192     case 0: {
193       // See spec 8.2.1.1.
194       int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb;
195 
196       if (pic->idr) {
197         prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0;
198       } else {
199         if (prev_ref_has_memmgmnt5_) {
200           if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) {
201             prev_pic_order_cnt_msb = 0;
202             prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_;
203           } else {
204             prev_pic_order_cnt_msb = 0;
205             prev_pic_order_cnt_lsb = 0;
206           }
207         } else {
208           prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_;
209           prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_;
210         }
211       }
212 
213       int max_pic_order_cnt_lsb =
214           1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
215       DCHECK_NE(max_pic_order_cnt_lsb, 0);
216       if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) &&
217           (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >=
218            max_pic_order_cnt_lsb / 2)) {
219         pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb;
220       } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) &&
221                  (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb >
222                   max_pic_order_cnt_lsb / 2)) {
223         pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb;
224       } else {
225         pic->pic_order_cnt_msb = prev_pic_order_cnt_msb;
226       }
227 
228       if (pic->field != H264Picture::FIELD_BOTTOM) {
229         pic->top_field_order_cnt =
230             pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
231       }
232 
233       if (pic->field != H264Picture::FIELD_TOP) {
234         if (pic->field == H264Picture::FIELD_NONE) {
235           pic->bottom_field_order_cnt =
236               pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom;
237         } else {
238           pic->bottom_field_order_cnt =
239               pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
240         }
241       }
242       break;
243     }
244 
245     case 1: {
246       // See spec 8.2.1.2.
247       if (prev_has_memmgmnt5_)
248         prev_frame_num_offset_ = 0;
249 
250       if (pic->idr)
251         pic->frame_num_offset = 0;
252       else if (prev_frame_num_ > pic->frame_num)
253         pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
254       else
255         pic->frame_num_offset = prev_frame_num_offset_;
256 
257       int abs_frame_num = 0;
258       if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
259         abs_frame_num = pic->frame_num_offset + pic->frame_num;
260       else
261         abs_frame_num = 0;
262 
263       if (pic->nal_ref_idc == 0 && abs_frame_num > 0)
264         --abs_frame_num;
265 
266       int expected_pic_order_cnt = 0;
267       if (abs_frame_num > 0) {
268         if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) {
269           DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle "
270                    << "in stream";
271           return false;
272         }
273 
274         int pic_order_cnt_cycle_cnt =
275             (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle;
276         int frame_num_in_pic_order_cnt_cycle =
277             (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle;
278 
279         expected_pic_order_cnt = pic_order_cnt_cycle_cnt *
280                                  sps->expected_delta_per_pic_order_cnt_cycle;
281         // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser
282         for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i)
283           expected_pic_order_cnt += sps->offset_for_ref_frame[i];
284       }
285 
286       if (!pic->nal_ref_idc)
287         expected_pic_order_cnt += sps->offset_for_non_ref_pic;
288 
289       if (pic->field == H264Picture::FIELD_NONE) {
290         pic->top_field_order_cnt =
291             expected_pic_order_cnt + pic->delta_pic_order_cnt0;
292         pic->bottom_field_order_cnt = pic->top_field_order_cnt +
293                                       sps->offset_for_top_to_bottom_field +
294                                       pic->delta_pic_order_cnt1;
295       } else if (pic->field != H264Picture::FIELD_BOTTOM) {
296         pic->top_field_order_cnt =
297             expected_pic_order_cnt + pic->delta_pic_order_cnt0;
298       } else {
299         pic->bottom_field_order_cnt = expected_pic_order_cnt +
300                                       sps->offset_for_top_to_bottom_field +
301                                       pic->delta_pic_order_cnt0;
302       }
303       break;
304     }
305 
306     case 2: {
307       // See spec 8.2.1.3.
308       if (prev_has_memmgmnt5_)
309         prev_frame_num_offset_ = 0;
310 
311       if (pic->idr)
312         pic->frame_num_offset = 0;
313       else if (prev_frame_num_ > pic->frame_num)
314         pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
315       else
316         pic->frame_num_offset = prev_frame_num_offset_;
317 
318       int temp_pic_order_cnt;
319       if (pic->idr) {
320         temp_pic_order_cnt = 0;
321       } else if (!pic->nal_ref_idc) {
322         temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1;
323       } else {
324         temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num);
325       }
326 
327       if (pic->field == H264Picture::FIELD_NONE) {
328         pic->top_field_order_cnt = temp_pic_order_cnt;
329         pic->bottom_field_order_cnt = temp_pic_order_cnt;
330       } else if (pic->field == H264Picture::FIELD_BOTTOM) {
331         pic->bottom_field_order_cnt = temp_pic_order_cnt;
332       } else {
333         pic->top_field_order_cnt = temp_pic_order_cnt;
334       }
335       break;
336     }
337 
338     default:
339       DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type;
340       return false;
341   }
342 
343   switch (pic->field) {
344     case H264Picture::FIELD_NONE:
345       pic->pic_order_cnt =
346           std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt);
347       break;
348     case H264Picture::FIELD_TOP:
349       pic->pic_order_cnt = pic->top_field_order_cnt;
350       break;
351     case H264Picture::FIELD_BOTTOM:
352       pic->pic_order_cnt = pic->bottom_field_order_cnt;
353       break;
354   }
355 
356   return true;
357 }
358 
UpdatePicNums(int frame_num)359 void H264Decoder::UpdatePicNums(int frame_num) {
360   for (auto& pic : dpb_) {
361     if (!pic->ref)
362       continue;
363 
364     // 8.2.4.1. Assumes non-interlaced stream.
365     DCHECK_EQ(pic->field, H264Picture::FIELD_NONE);
366     if (pic->long_term) {
367       pic->long_term_pic_num = pic->long_term_frame_idx;
368     } else {
369       if (pic->frame_num > frame_num)
370         pic->frame_num_wrap = pic->frame_num - max_frame_num_;
371       else
372         pic->frame_num_wrap = pic->frame_num;
373 
374       pic->pic_num = pic->frame_num_wrap;
375     }
376   }
377 }
378 
379 struct PicNumDescCompare {
operator ()media::PicNumDescCompare380   bool operator()(const scoped_refptr<H264Picture>& a,
381                   const scoped_refptr<H264Picture>& b) const {
382     return a->pic_num > b->pic_num;
383   }
384 };
385 
386 struct LongTermPicNumAscCompare {
operator ()media::LongTermPicNumAscCompare387   bool operator()(const scoped_refptr<H264Picture>& a,
388                   const scoped_refptr<H264Picture>& b) const {
389     return a->long_term_pic_num < b->long_term_pic_num;
390   }
391 };
392 
ConstructReferencePicListsP(const H264SliceHeader * slice_hdr)393 void H264Decoder::ConstructReferencePicListsP(
394     const H264SliceHeader* slice_hdr) {
395   // RefPicList0 (8.2.4.2.1) [[1] [2]], where:
396   // [1] shortterm ref pics sorted by descending pic_num,
397   // [2] longterm ref pics by ascending long_term_pic_num.
398   ref_pic_list_p0_.clear();
399 
400   // First get the short ref pics...
401   dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_);
402   size_t num_short_refs = ref_pic_list_p0_.size();
403 
404   // and sort them to get [1].
405   std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(),
406             PicNumDescCompare());
407 
408   // Now get long term pics and sort them by long_term_pic_num to get [2].
409   dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_);
410   std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(),
411             LongTermPicNumAscCompare());
412 }
413 
414 struct POCAscCompare {
operator ()media::POCAscCompare415   bool operator()(const scoped_refptr<H264Picture>& a,
416                   const scoped_refptr<H264Picture>& b) const {
417     return a->pic_order_cnt < b->pic_order_cnt;
418   }
419 };
420 
421 struct POCDescCompare {
operator ()media::POCDescCompare422   bool operator()(const scoped_refptr<H264Picture>& a,
423                   const scoped_refptr<H264Picture>& b) const {
424     return a->pic_order_cnt > b->pic_order_cnt;
425   }
426 };
427 
ConstructReferencePicListsB(const H264SliceHeader * slice_hdr)428 void H264Decoder::ConstructReferencePicListsB(
429     const H264SliceHeader* slice_hdr) {
430   // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where:
431   // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC,
432   // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC,
433   // [3] longterm ref pics by ascending long_term_pic_num.
434   ref_pic_list_b0_.clear();
435   ref_pic_list_b1_.clear();
436   dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_);
437   size_t num_short_refs = ref_pic_list_b0_.size();
438 
439   // First sort ascending, this will put [1] in right place and finish [2].
440   std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare());
441 
442   // Find first with POC > curr_pic's POC to get first element in [2]...
443   H264Picture::Vector::iterator iter;
444   iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
445                           curr_pic_.get(), POCAscCompare());
446 
447   // and sort [1] descending, thus finishing sequence [1] [2].
448   std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare());
449 
450   // Now add [3] and sort by ascending long_term_pic_num.
451   dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_);
452   std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(),
453             LongTermPicNumAscCompare());
454 
455   // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where:
456   // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC,
457   // [2] shortterm ref pics with POC < curr_pic's POC by descending POC,
458   // [3] longterm ref pics by ascending long_term_pic_num.
459 
460   dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_);
461   num_short_refs = ref_pic_list_b1_.size();
462 
463   // First sort by descending POC.
464   std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare());
465 
466   // Find first with POC < curr_pic's POC to get first element in [2]...
467   iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(),
468                           curr_pic_.get(), POCDescCompare());
469 
470   // and sort [1] ascending.
471   std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare());
472 
473   // Now add [3] and sort by ascending long_term_pic_num
474   dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_);
475   std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(),
476             LongTermPicNumAscCompare());
477 
478   // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3)
479   if (ref_pic_list_b1_.size() > 1 &&
480       std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
481                  ref_pic_list_b1_.begin()))
482     std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]);
483 }
484 
485 // See 8.2.4
PicNumF(const scoped_refptr<H264Picture> & pic)486 int H264Decoder::PicNumF(const scoped_refptr<H264Picture>& pic) {
487   if (!pic)
488     return -1;
489 
490   if (!pic->long_term)
491     return pic->pic_num;
492   else
493     return max_pic_num_;
494 }
495 
496 // See 8.2.4
LongTermPicNumF(const scoped_refptr<H264Picture> & pic)497 int H264Decoder::LongTermPicNumF(const scoped_refptr<H264Picture>& pic) {
498   if (pic->ref && pic->long_term)
499     return pic->long_term_pic_num;
500   else
501     return 2 * (max_long_term_frame_idx_ + 1);
502 }
503 
504 // Shift elements on the |v| starting from |from| to |to|, inclusive,
505 // one position to the right and insert pic at |from|.
ShiftRightAndInsert(H264Picture::Vector * v,int from,int to,const scoped_refptr<H264Picture> & pic)506 static void ShiftRightAndInsert(H264Picture::Vector* v,
507                                 int from,
508                                 int to,
509                                 const scoped_refptr<H264Picture>& pic) {
510   // Security checks, do not disable in Debug mode.
511   CHECK(from <= to);
512   CHECK(to <= std::numeric_limits<int>::max() - 2);
513   // Additional checks. Debug mode ok.
514   DCHECK(v);
515   DCHECK(pic);
516   DCHECK((to + 1 == static_cast<int>(v->size())) ||
517          (to + 2 == static_cast<int>(v->size())));
518 
519   v->resize(to + 2);
520 
521   for (int i = to + 1; i > from; --i)
522     (*v)[i] = (*v)[i - 1];
523 
524   (*v)[from] = pic;
525 }
526 
ModifyReferencePicList(const H264SliceHeader * slice_hdr,int list,H264Picture::Vector * ref_pic_listx)527 bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr,
528                                          int list,
529                                          H264Picture::Vector* ref_pic_listx) {
530   bool ref_pic_list_modification_flag_lX;
531   int num_ref_idx_lX_active_minus1;
532   const H264ModificationOfPicNum* list_mod;
533 
534   // This can process either ref_pic_list0 or ref_pic_list1, depending on
535   // the list argument. Set up pointers to proper list to be processed here.
536   if (list == 0) {
537     ref_pic_list_modification_flag_lX =
538         slice_hdr->ref_pic_list_modification_flag_l0;
539     num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1;
540     list_mod = slice_hdr->ref_list_l0_modifications;
541   } else {
542     ref_pic_list_modification_flag_lX =
543         slice_hdr->ref_pic_list_modification_flag_l1;
544     num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1;
545     list_mod = slice_hdr->ref_list_l1_modifications;
546   }
547 
548   // Resize the list to the size requested in the slice header.
549   // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to
550   // indicate there should be more ref pics on list than we constructed.
551   // Those superfluous ones should be treated as non-reference and will be
552   // initialized to nullptr, which must be handled by clients.
553   DCHECK_GE(num_ref_idx_lX_active_minus1, 0);
554   ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
555 
556   if (!ref_pic_list_modification_flag_lX)
557     return true;
558 
559   // Spec 8.2.4.3:
560   // Reorder pictures on the list in a way specified in the stream.
561   int pic_num_lx_pred = curr_pic_->pic_num;
562   int ref_idx_lx = 0;
563   int pic_num_lx_no_wrap;
564   int pic_num_lx;
565   bool done = false;
566   scoped_refptr<H264Picture> pic;
567   for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) {
568     switch (list_mod->modification_of_pic_nums_idc) {
569       case 0:
570       case 1:
571         // Modify short reference picture position.
572         if (list_mod->modification_of_pic_nums_idc == 0) {
573           // Subtract given value from predicted PicNum.
574           pic_num_lx_no_wrap =
575               pic_num_lx_pred -
576               (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
577           // Wrap around max_pic_num_ if it becomes < 0 as result
578           // of subtraction.
579           if (pic_num_lx_no_wrap < 0)
580             pic_num_lx_no_wrap += max_pic_num_;
581         } else {
582           // Add given value to predicted PicNum.
583           pic_num_lx_no_wrap =
584               pic_num_lx_pred +
585               (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
586           // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result
587           // of the addition.
588           if (pic_num_lx_no_wrap >= max_pic_num_)
589             pic_num_lx_no_wrap -= max_pic_num_;
590         }
591 
592         // For use in next iteration.
593         pic_num_lx_pred = pic_num_lx_no_wrap;
594 
595         if (pic_num_lx_no_wrap > curr_pic_->pic_num)
596           pic_num_lx = pic_num_lx_no_wrap - max_pic_num_;
597         else
598           pic_num_lx = pic_num_lx_no_wrap;
599 
600         DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
601                   H264SliceHeader::kRefListModSize);
602         pic = dpb_.GetShortRefPicByPicNum(pic_num_lx);
603         if (!pic) {
604           DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx;
605           return false;
606         }
607         ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
608                             num_ref_idx_lX_active_minus1, pic);
609         ref_idx_lx++;
610 
611         for (int src = ref_idx_lx, dst = ref_idx_lx;
612              src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
613           if (PicNumF((*ref_pic_listx)[src]) != pic_num_lx)
614             (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
615         }
616         break;
617 
618       case 2:
619         // Modify long term reference picture position.
620         DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
621                   H264SliceHeader::kRefListModSize);
622         pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num);
623         if (!pic) {
624           DVLOG(1) << "Malformed stream, no pic num "
625                    << list_mod->long_term_pic_num;
626           return false;
627         }
628         ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
629                             num_ref_idx_lX_active_minus1, pic);
630         ref_idx_lx++;
631 
632         for (int src = ref_idx_lx, dst = ref_idx_lx;
633              src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
634           if (LongTermPicNumF((*ref_pic_listx)[src]) !=
635               static_cast<int>(list_mod->long_term_pic_num))
636             (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
637         }
638         break;
639 
640       case 3:
641         // End of modification list.
642         done = true;
643         break;
644 
645       default:
646         // May be recoverable.
647         DVLOG(1) << "Invalid modification_of_pic_nums_idc="
648                  << list_mod->modification_of_pic_nums_idc
649                  << " in position " << i;
650         break;
651     }
652 
653     ++list_mod;
654   }
655 
656   // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is
657   // temporarily made one element longer than the required final list.
658   // Resize the list back to its required size.
659   ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
660 
661   return true;
662 }
663 
OutputPic(scoped_refptr<H264Picture> pic)664 void H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) {
665   DCHECK(!pic->outputted);
666   pic->outputted = true;
667 
668   if (pic->nonexisting) {
669     DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num;
670     return;
671   }
672 
673   DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_)
674       << "Outputting out of order, likely a broken stream: "
675       << last_output_poc_ << " -> " << pic->pic_order_cnt;
676   last_output_poc_ = pic->pic_order_cnt;
677 
678   DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt;
679   accelerator_->OutputPicture(pic);
680 }
681 
ClearDPB()682 void H264Decoder::ClearDPB() {
683   // Clear DPB contents, marking the pictures as unused first.
684   dpb_.Clear();
685   last_output_poc_ = std::numeric_limits<int>::min();
686 }
687 
OutputAllRemainingPics()688 bool H264Decoder::OutputAllRemainingPics() {
689   // Output all pictures that are waiting to be outputted.
690   FinishPrevFrameIfPresent();
691   H264Picture::Vector to_output;
692   dpb_.GetNotOutputtedPicsAppending(&to_output);
693   // Sort them by ascending POC to output in order.
694   std::sort(to_output.begin(), to_output.end(), POCAscCompare());
695 
696   for (auto& pic : to_output)
697     OutputPic(pic);
698 
699   return true;
700 }
701 
Flush()702 bool H264Decoder::Flush() {
703   DVLOG(2) << "Decoder flush";
704 
705   if (!OutputAllRemainingPics())
706     return false;
707 
708   ClearDPB();
709   DVLOG(2) << "Decoder flush finished";
710   return true;
711 }
712 
StartNewFrame(const H264SliceHeader * slice_hdr)713 bool H264Decoder::StartNewFrame(const H264SliceHeader* slice_hdr) {
714   // TODO posciak: add handling of max_num_ref_frames per spec.
715   CHECK(curr_pic_.get());
716   DCHECK(slice_hdr);
717 
718   curr_pps_id_ = slice_hdr->pic_parameter_set_id;
719   const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
720   if (!pps)
721     return false;
722 
723   curr_sps_id_ = pps->seq_parameter_set_id;
724   const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
725   if (!sps)
726     return false;
727 
728   max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4);
729   int frame_num = slice_hdr->frame_num;
730   if (slice_hdr->idr_pic_flag)
731     prev_ref_frame_num_ = 0;
732 
733   // 7.4.3
734   if (frame_num != prev_ref_frame_num_ &&
735       frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) {
736     if (!HandleFrameNumGap(frame_num))
737       return false;
738   }
739 
740   if (!InitCurrPicture(slice_hdr))
741     return false;
742 
743   UpdatePicNums(frame_num);
744   PrepareRefPicLists(slice_hdr);
745 
746   if (!accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_,
747                                          ref_pic_list_b0_, ref_pic_list_b1_,
748                                          curr_pic_.get()))
749     return false;
750 
751   return true;
752 }
753 
HandleMemoryManagementOps(scoped_refptr<H264Picture> pic)754 bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) {
755   // 8.2.5.4
756   for (size_t i = 0; i < arraysize(pic->ref_pic_marking); ++i) {
757     // Code below does not support interlaced stream (per-field pictures).
758     H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i];
759     scoped_refptr<H264Picture> to_mark;
760     int pic_num_x;
761 
762     switch (ref_pic_marking->memory_mgmnt_control_operation) {
763       case 0:
764         // Normal end of operations' specification.
765         return true;
766 
767       case 1:
768         // Mark a short term reference picture as unused so it can be removed
769         // if outputted.
770         pic_num_x =
771             pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
772         to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
773         if (to_mark) {
774           to_mark->ref = false;
775         } else {
776           DVLOG(1) << "Invalid short ref pic num to unmark";
777           return false;
778         }
779         break;
780 
781       case 2:
782         // Mark a long term reference picture as unused so it can be removed
783         // if outputted.
784         to_mark = dpb_.GetLongRefPicByLongTermPicNum(
785             ref_pic_marking->long_term_pic_num);
786         if (to_mark) {
787           to_mark->ref = false;
788         } else {
789           DVLOG(1) << "Invalid long term ref pic num to unmark";
790           return false;
791         }
792         break;
793 
794       case 3:
795         // Mark a short term reference picture as long term reference.
796         pic_num_x =
797             pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
798         to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
799         if (to_mark) {
800           DCHECK(to_mark->ref && !to_mark->long_term);
801           to_mark->long_term = true;
802           to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
803         } else {
804           DVLOG(1) << "Invalid short term ref pic num to mark as long ref";
805           return false;
806         }
807         break;
808 
809       case 4: {
810         // Unmark all reference pictures with long_term_frame_idx over new max.
811         max_long_term_frame_idx_ =
812             ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
813         H264Picture::Vector long_terms;
814         dpb_.GetLongTermRefPicsAppending(&long_terms);
815         for (size_t i = 0; i < long_terms.size(); ++i) {
816           scoped_refptr<H264Picture>& long_term_pic = long_terms[i];
817           DCHECK(long_term_pic->ref && long_term_pic->long_term);
818           // Ok to cast, max_long_term_frame_idx is much smaller than 16bit.
819           if (long_term_pic->long_term_frame_idx >
820               static_cast<int>(max_long_term_frame_idx_))
821             long_term_pic->ref = false;
822         }
823         break;
824       }
825 
826       case 5:
827         // Unmark all reference pictures.
828         dpb_.MarkAllUnusedForRef();
829         max_long_term_frame_idx_ = -1;
830         pic->mem_mgmt_5 = true;
831         break;
832 
833       case 6: {
834         // Replace long term reference pictures with current picture.
835         // First unmark if any existing with this long_term_frame_idx...
836         H264Picture::Vector long_terms;
837         dpb_.GetLongTermRefPicsAppending(&long_terms);
838         for (size_t i = 0; i < long_terms.size(); ++i) {
839           scoped_refptr<H264Picture>& long_term_pic = long_terms[i];
840           DCHECK(long_term_pic->ref && long_term_pic->long_term);
841           // Ok to cast, long_term_frame_idx is much smaller than 16bit.
842           if (long_term_pic->long_term_frame_idx ==
843               static_cast<int>(ref_pic_marking->long_term_frame_idx))
844             long_term_pic->ref = false;
845         }
846 
847         // and mark the current one instead.
848         pic->ref = true;
849         pic->long_term = true;
850         pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
851         break;
852       }
853 
854       default:
855         // Would indicate a bug in parser.
856         NOTREACHED();
857     }
858   }
859 
860   return true;
861 }
862 
863 // This method ensures that DPB does not overflow, either by removing
864 // reference pictures as specified in the stream, or using a sliding window
865 // procedure to remove the oldest one.
866 // It also performs marking and unmarking pictures as reference.
867 // See spac 8.2.5.1.
ReferencePictureMarking(scoped_refptr<H264Picture> pic)868 bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) {
869   // If the current picture is an IDR, all reference pictures are unmarked.
870   if (pic->idr) {
871     dpb_.MarkAllUnusedForRef();
872 
873     if (pic->long_term_reference_flag) {
874       pic->long_term = true;
875       pic->long_term_frame_idx = 0;
876       max_long_term_frame_idx_ = 0;
877     } else {
878       pic->long_term = false;
879       max_long_term_frame_idx_ = -1;
880     }
881 
882     return true;
883   }
884 
885   // Not an IDR. If the stream contains instructions on how to discard pictures
886   // from DPB and how to mark/unmark existing reference pictures, do so.
887   // Otherwise, fall back to default sliding window process.
888   if (pic->adaptive_ref_pic_marking_mode_flag) {
889     DCHECK(!pic->nonexisting);
890     return HandleMemoryManagementOps(pic);
891   } else {
892     return SlidingWindowPictureMarking();
893   }
894 }
895 
SlidingWindowPictureMarking()896 bool H264Decoder::SlidingWindowPictureMarking() {
897   const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
898   if (!sps)
899     return false;
900 
901   // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture.
902   int num_ref_pics = dpb_.CountRefPics();
903   DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1));
904   if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) {
905     // Max number of reference pics reached, need to remove one of the short
906     // term ones. Find smallest frame_num_wrap short reference picture and mark
907     // it as unused.
908     scoped_refptr<H264Picture> to_unmark =
909         dpb_.GetLowestFrameNumWrapShortRefPic();
910     if (!to_unmark) {
911       DVLOG(1) << "Couldn't find a short ref picture to unmark";
912       return false;
913     }
914 
915     to_unmark->ref = false;
916   }
917 
918   return true;
919 }
920 
FinishPicture(scoped_refptr<H264Picture> pic)921 bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) {
922   // Finish processing the picture.
923   // Start by storing previous picture data for later use.
924   if (pic->ref) {
925     ReferencePictureMarking(pic);
926     prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5;
927     prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt;
928     prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb;
929     prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb;
930     prev_ref_field_ = pic->field;
931     prev_ref_frame_num_ = pic->frame_num;
932   }
933   prev_frame_num_ = pic->frame_num;
934   prev_has_memmgmnt5_ = pic->mem_mgmt_5;
935   prev_frame_num_offset_ = pic->frame_num_offset;
936 
937   // Remove unused (for reference or later output) pictures from DPB, marking
938   // them as such.
939   dpb_.DeleteUnused();
940 
941   DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num
942            << ", entries in DPB: " << dpb_.size();
943 
944   // The ownership of pic will either be transferred to DPB - if the picture is
945   // still needed (for output and/or reference) - or we will release it
946   // immediately if we manage to output it here and won't have to store it for
947   // future reference.
948 
949   // Get all pictures that haven't been outputted yet.
950   H264Picture::Vector not_outputted;
951   dpb_.GetNotOutputtedPicsAppending(&not_outputted);
952   // Include the one we've just decoded.
953   not_outputted.push_back(pic);
954 
955   // Sort in output order.
956   std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare());
957 
958   // Try to output as many pictures as we can. A picture can be output,
959   // if the number of decoded and not yet outputted pictures that would remain
960   // in DPB afterwards would at least be equal to max_num_reorder_frames.
961   // If the outputted picture is not a reference picture, it doesn't have
962   // to remain in the DPB and can be removed.
963   H264Picture::Vector::iterator output_candidate = not_outputted.begin();
964   size_t num_remaining = not_outputted.size();
965   while (num_remaining > max_num_reorder_frames_ ||
966          // If the condition below is used, this is an invalid stream. We should
967          // not be forced to output beyond max_num_reorder_frames in order to
968          // make room in DPB to store the current picture (if we need to do so).
969          // However, if this happens, ignore max_num_reorder_frames and try
970          // to output more. This may cause out-of-order output, but is not
971          // fatal, and better than failing instead.
972          ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) {
973     DVLOG_IF(1, num_remaining <= max_num_reorder_frames_)
974         << "Invalid stream: max_num_reorder_frames not preserved";
975 
976     OutputPic(*output_candidate);
977 
978     if (!(*output_candidate)->ref) {
979       // Current picture hasn't been inserted into DPB yet, so don't remove it
980       // if we managed to output it immediately.
981       int outputted_poc = (*output_candidate)->pic_order_cnt;
982       if (outputted_poc != pic->pic_order_cnt)
983         dpb_.DeleteByPOC(outputted_poc);
984     }
985 
986     ++output_candidate;
987     --num_remaining;
988   }
989 
990   // If we haven't managed to output the picture that we just decoded, or if
991   // it's a reference picture, we have to store it in DPB.
992   if (!pic->outputted || pic->ref) {
993     if (dpb_.IsFull()) {
994       // If we haven't managed to output anything to free up space in DPB
995       // to store this picture, it's an error in the stream.
996       DVLOG(1) << "Could not free up space in DPB!";
997       return false;
998     }
999 
1000     dpb_.StorePic(pic);
1001   }
1002 
1003   return true;
1004 }
1005 
LevelToMaxDpbMbs(int level)1006 static int LevelToMaxDpbMbs(int level) {
1007   // See table A-1 in spec.
1008   switch (level) {
1009     case 10:
1010       return 396;
1011     case 11:
1012       return 900;
1013     case 12:  //  fallthrough
1014     case 13:  //  fallthrough
1015     case 20:
1016       return 2376;
1017     case 21:
1018       return 4752;
1019     case 22:  //  fallthrough
1020     case 30:
1021       return 8100;
1022     case 31:
1023       return 18000;
1024     case 32:
1025       return 20480;
1026     case 40:  //  fallthrough
1027     case 41:
1028       return 32768;
1029     case 42:
1030       return 34816;
1031     case 50:
1032       return 110400;
1033     case 51:  //  fallthrough
1034     case 52:
1035       return 184320;
1036     default:
1037       DVLOG(1) << "Invalid codec level (" << level << ")";
1038       return 0;
1039   }
1040 }
1041 
UpdateMaxNumReorderFrames(const H264SPS * sps)1042 bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) {
1043   if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) {
1044     max_num_reorder_frames_ =
1045         base::checked_cast<size_t>(sps->max_num_reorder_frames);
1046     if (max_num_reorder_frames_ > dpb_.max_num_pics()) {
1047       DVLOG(1)
1048           << "max_num_reorder_frames present, but larger than MaxDpbFrames ("
1049           << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")";
1050       max_num_reorder_frames_ = 0;
1051       return false;
1052     }
1053     return true;
1054   }
1055 
1056   // max_num_reorder_frames not present, infer from profile/constraints
1057   // (see VUI semantics in spec).
1058   if (sps->constraint_set3_flag) {
1059     switch (sps->profile_idc) {
1060       case 44:
1061       case 86:
1062       case 100:
1063       case 110:
1064       case 122:
1065       case 244:
1066         max_num_reorder_frames_ = 0;
1067         break;
1068       default:
1069         max_num_reorder_frames_ = dpb_.max_num_pics();
1070         break;
1071     }
1072   } else {
1073     max_num_reorder_frames_ = dpb_.max_num_pics();
1074   }
1075 
1076   return true;
1077 }
1078 
ProcessSPS(int sps_id,bool * need_new_buffers)1079 bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) {
1080   DVLOG(4) << "Processing SPS id:" << sps_id;
1081 
1082   const H264SPS* sps = parser_.GetSPS(sps_id);
1083   if (!sps)
1084     return false;
1085 
1086   *need_new_buffers = false;
1087 
1088   if (sps->frame_mbs_only_flag == 0) {
1089     DVLOG(1) << "frame_mbs_only_flag != 1 not supported";
1090     return false;
1091   }
1092 
1093   Size new_pic_size = sps->GetCodedSize().value_or(Size());
1094   if (new_pic_size.IsEmpty()) {
1095     DVLOG(1) << "Invalid picture size";
1096     return false;
1097   }
1098 
1099   int width_mb = new_pic_size.width() / 16;
1100   int height_mb = new_pic_size.height() / 16;
1101 
1102   // Verify that the values are not too large before multiplying.
1103   if (std::numeric_limits<int>::max() / width_mb < height_mb) {
1104     DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString();
1105     return false;
1106   }
1107 
1108   int level = sps->level_idc;
1109   int max_dpb_mbs = LevelToMaxDpbMbs(level);
1110   if (max_dpb_mbs == 0)
1111     return false;
1112 
1113   // MaxDpbFrames from level limits per spec.
1114   size_t max_dpb_frames = std::min(max_dpb_mbs / (width_mb * height_mb),
1115                                    static_cast<int>(H264DPB::kDPBMaxSize));
1116   DVLOG(1) << "MaxDpbFrames: " << max_dpb_frames
1117            << ", max_num_ref_frames: " << sps->max_num_ref_frames
1118            << ", max_dec_frame_buffering: " << sps->max_dec_frame_buffering;
1119 
1120   // Set DPB size to at least the level limit, or what the stream requires.
1121   size_t max_dpb_size =
1122       std::max(static_cast<int>(max_dpb_frames),
1123                std::max(sps->max_num_ref_frames, sps->max_dec_frame_buffering));
1124   // Some non-conforming streams specify more frames are needed than the current
1125   // level limit. Allow this, but only up to the maximum number of reference
1126   // frames allowed per spec.
1127   DVLOG_IF(1, max_dpb_size > max_dpb_frames)
1128       << "Invalid stream, DPB size > MaxDpbFrames";
1129   if (max_dpb_size == 0 || max_dpb_size > H264DPB::kDPBMaxSize) {
1130     DVLOG(1) << "Invalid DPB size: " << max_dpb_size;
1131     return false;
1132   }
1133 
1134   if ((pic_size_ != new_pic_size) || (dpb_.max_num_pics() != max_dpb_size)) {
1135     if (!Flush())
1136       return false;
1137     DVLOG(1) << "Codec level: " << level << ", DPB size: " << max_dpb_size
1138              << ", Picture size: " << new_pic_size.ToString();
1139     *need_new_buffers = true;
1140     pic_size_ = new_pic_size;
1141     dpb_.set_max_num_pics(max_dpb_size);
1142   }
1143 
1144   Rect new_visible_rect = sps->GetVisibleRect().value_or(Rect());
1145   if (visible_rect_ != new_visible_rect) {
1146     DVLOG(2) << "New visible rect: " << new_visible_rect.ToString();
1147     visible_rect_ = new_visible_rect;
1148   }
1149 
1150   if (!UpdateMaxNumReorderFrames(sps))
1151     return false;
1152   DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_;
1153 
1154   return true;
1155 }
1156 
FinishPrevFrameIfPresent()1157 bool H264Decoder::FinishPrevFrameIfPresent() {
1158   // If we already have a frame waiting to be decoded, decode it and finish.
1159   if (curr_pic_) {
1160     if (!DecodePicture())
1161       return false;
1162 
1163     scoped_refptr<H264Picture> pic = curr_pic_;
1164     curr_pic_ = nullptr;
1165     return FinishPicture(pic);
1166   }
1167 
1168   return true;
1169 }
1170 
HandleFrameNumGap(int frame_num)1171 bool H264Decoder::HandleFrameNumGap(int frame_num) {
1172   const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
1173   if (!sps)
1174     return false;
1175 
1176   if (!sps->gaps_in_frame_num_value_allowed_flag) {
1177     DVLOG(1) << "Invalid frame_num: " << frame_num;
1178     return false;
1179   }
1180 
1181   DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->"
1182            << frame_num;
1183 
1184   // 7.4.3/7-23
1185   int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_;
1186   while (unused_short_term_frame_num != frame_num) {
1187     scoped_refptr<H264Picture> pic = new H264Picture();
1188     if (!InitNonexistingPicture(pic, unused_short_term_frame_num))
1189       return false;
1190 
1191     UpdatePicNums(unused_short_term_frame_num);
1192 
1193     if (!FinishPicture(pic))
1194       return false;
1195 
1196     unused_short_term_frame_num++;
1197     unused_short_term_frame_num %= max_frame_num_;
1198   }
1199 
1200   return true;
1201 }
1202 
IsNewPrimaryCodedPicture(const H264SliceHeader * slice_hdr) const1203 bool H264Decoder::IsNewPrimaryCodedPicture(
1204     const H264SliceHeader* slice_hdr) const {
1205   if (!curr_pic_)
1206     return true;
1207 
1208   // 7.4.1.2.4, assumes non-interlaced.
1209   if (slice_hdr->frame_num != curr_pic_->frame_num ||
1210       slice_hdr->pic_parameter_set_id != curr_pps_id_ ||
1211       slice_hdr->nal_ref_idc != curr_pic_->nal_ref_idc ||
1212       slice_hdr->idr_pic_flag != curr_pic_->idr ||
1213       (slice_hdr->idr_pic_flag &&
1214        (slice_hdr->idr_pic_id != curr_pic_->idr_pic_id ||
1215         // If we have two consecutive IDR slices, and the second one has
1216         // first_mb_in_slice == 0, treat it as a new picture.
1217         // Per spec, idr_pic_id should not be equal in this case (and we should
1218         // have hit the condition above instead, see spec 7.4.3 on idr_pic_id),
1219         // but some encoders neglect changing idr_pic_id for two consecutive
1220         // IDRs. Work around this by checking if the next slice contains the
1221         // zeroth macroblock, i.e. data that belongs to the next picture.
1222         slice_hdr->first_mb_in_slice == 0)))
1223     return true;
1224 
1225   const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
1226   if (!sps)
1227     return false;
1228 
1229   if (sps->pic_order_cnt_type == curr_pic_->pic_order_cnt_type) {
1230     if (curr_pic_->pic_order_cnt_type == 0) {
1231       if (slice_hdr->pic_order_cnt_lsb != curr_pic_->pic_order_cnt_lsb ||
1232           slice_hdr->delta_pic_order_cnt_bottom !=
1233               curr_pic_->delta_pic_order_cnt_bottom)
1234         return true;
1235     } else if (curr_pic_->pic_order_cnt_type == 1) {
1236       if (slice_hdr->delta_pic_order_cnt0 != curr_pic_->delta_pic_order_cnt0 ||
1237           slice_hdr->delta_pic_order_cnt1 != curr_pic_->delta_pic_order_cnt1)
1238         return true;
1239     }
1240   }
1241 
1242   return false;
1243 }
1244 
PreprocessCurrentSlice()1245 bool H264Decoder::PreprocessCurrentSlice() {
1246   const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
1247   DCHECK(slice_hdr);
1248 
1249   if (IsNewPrimaryCodedPicture(slice_hdr)) {
1250     // New picture, so first finish the previous one before processing it.
1251     if (!FinishPrevFrameIfPresent())
1252       return false;
1253 
1254     DCHECK(!curr_pic_);
1255 
1256     if (slice_hdr->first_mb_in_slice != 0) {
1257       DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: "
1258                << slice_hdr->first_mb_in_slice;
1259       return false;
1260     }
1261 
1262     // If the new picture is an IDR, flush DPB.
1263     if (slice_hdr->idr_pic_flag) {
1264       // Output all remaining pictures, unless we are explicitly instructed
1265       // not to do so.
1266       if (!slice_hdr->no_output_of_prior_pics_flag) {
1267         if (!Flush())
1268           return false;
1269       }
1270       dpb_.Clear();
1271       last_output_poc_ = std::numeric_limits<int>::min();
1272     }
1273   }
1274 
1275   return true;
1276 }
1277 
ProcessCurrentSlice()1278 bool H264Decoder::ProcessCurrentSlice() {
1279   DCHECK(curr_pic_);
1280 
1281   const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
1282   DCHECK(slice_hdr);
1283 
1284   if (slice_hdr->field_pic_flag == 0)
1285     max_pic_num_ = max_frame_num_;
1286   else
1287     max_pic_num_ = 2 * max_frame_num_;
1288 
1289   H264Picture::Vector ref_pic_list0, ref_pic_list1;
1290   if (!ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1))
1291     return false;
1292 
1293   const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
1294   if (!pps)
1295     return false;
1296 
1297   if (!accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1,
1298                                  curr_pic_.get(), slice_hdr->nalu_data,
1299                                  slice_hdr->nalu_size))
1300     return false;
1301 
1302   return true;
1303 }
1304 
1305 #define SET_ERROR_AND_RETURN()         \
1306   do {                                 \
1307     DVLOG(1) << "Error during decode"; \
1308     state_ = kError;                   \
1309     return H264Decoder::kDecodeError;  \
1310   } while (0)
1311 
SetStream(const uint8_t * ptr,size_t size)1312 void H264Decoder::SetStream(const uint8_t* ptr, size_t size) {
1313   DCHECK(ptr);
1314   DCHECK(size);
1315 
1316   DVLOG(4) << "New input stream at: " << (void*)ptr << " size: " << size;
1317   parser_.SetStream(ptr, size);
1318 }
1319 
Decode()1320 H264Decoder::DecodeResult H264Decoder::Decode() {
1321   if (state_ == kError) {
1322     DVLOG(1) << "Decoder in error state";
1323     return kDecodeError;
1324   }
1325 
1326   while (1) {
1327     H264Parser::Result par_res;
1328 
1329     if (!curr_nalu_) {
1330       curr_nalu_.reset(new H264NALU());
1331       par_res = parser_.AdvanceToNextNALU(curr_nalu_.get());
1332       if (par_res == H264Parser::kEOStream)
1333         return kRanOutOfStreamData;
1334       else if (par_res != H264Parser::kOk)
1335         SET_ERROR_AND_RETURN();
1336 
1337       DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type);
1338     }
1339 
1340     switch (curr_nalu_->nal_unit_type) {
1341       case H264NALU::kNonIDRSlice:
1342         // We can't resume from a non-IDR slice.
1343         if (state_ != kDecoding)
1344           break;
1345 
1346         // else fallthrough
1347       case H264NALU::kIDRSlice: {
1348         // TODO(posciak): the IDR may require an SPS that we don't have
1349         // available. For now we'd fail if that happens, but ideally we'd like
1350         // to keep going until the next SPS in the stream.
1351         if (state_ == kNeedStreamMetadata) {
1352           // We need an SPS, skip this IDR and keep looking.
1353           break;
1354         }
1355 
1356         // If after reset, we should be able to recover from an IDR.
1357         state_ = kDecoding;
1358 
1359         if (!curr_slice_hdr_) {
1360           curr_slice_hdr_.reset(new H264SliceHeader());
1361           par_res =
1362               parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get());
1363           if (par_res != H264Parser::kOk)
1364             SET_ERROR_AND_RETURN();
1365 
1366           if (!PreprocessCurrentSlice())
1367             SET_ERROR_AND_RETURN();
1368         }
1369 
1370         if (!curr_pic_) {
1371           // New picture/finished previous one, try to start a new one
1372           // or tell the client we need more surfaces.
1373           curr_pic_ = accelerator_->CreateH264Picture();
1374           if (!curr_pic_)
1375             return kRanOutOfSurfaces;
1376 
1377           if (!StartNewFrame(curr_slice_hdr_.get()))
1378             SET_ERROR_AND_RETURN();
1379         }
1380 
1381         if (!ProcessCurrentSlice())
1382           SET_ERROR_AND_RETURN();
1383 
1384         curr_slice_hdr_.reset();
1385         break;
1386       }
1387 
1388       case H264NALU::kSPS: {
1389         int sps_id;
1390 
1391         if (!FinishPrevFrameIfPresent())
1392           SET_ERROR_AND_RETURN();
1393 
1394         par_res = parser_.ParseSPS(&sps_id);
1395         if (par_res != H264Parser::kOk)
1396           SET_ERROR_AND_RETURN();
1397 
1398         bool need_new_buffers = false;
1399         if (!ProcessSPS(sps_id, &need_new_buffers))
1400           SET_ERROR_AND_RETURN();
1401 
1402         if (state_ == kNeedStreamMetadata)
1403           state_ = kAfterReset;
1404 
1405         if (need_new_buffers) {
1406           curr_pic_ = nullptr;
1407           curr_nalu_ = nullptr;
1408           ref_pic_list_p0_.clear();
1409           ref_pic_list_b0_.clear();
1410           ref_pic_list_b1_.clear();
1411 
1412           return kAllocateNewSurfaces;
1413         }
1414         break;
1415       }
1416 
1417       case H264NALU::kPPS: {
1418         int pps_id;
1419 
1420         if (!FinishPrevFrameIfPresent())
1421           SET_ERROR_AND_RETURN();
1422 
1423         par_res = parser_.ParsePPS(&pps_id);
1424         if (par_res != H264Parser::kOk)
1425           SET_ERROR_AND_RETURN();
1426 
1427         break;
1428       }
1429 
1430       case H264NALU::kAUD:
1431       case H264NALU::kEOSeq:
1432       case H264NALU::kEOStream:
1433         if (state_ != kDecoding)
1434           break;
1435 
1436         if (!FinishPrevFrameIfPresent())
1437           SET_ERROR_AND_RETURN();
1438 
1439         break;
1440 
1441       default:
1442         DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type;
1443         break;
1444     }
1445 
1446     DVLOG(4) << "NALU done";
1447     curr_nalu_.reset();
1448   }
1449 }
1450 
GetPicSize() const1451 Size H264Decoder::GetPicSize() const {
1452   return pic_size_;
1453 }
1454 
GetRequiredNumOfPictures() const1455 size_t H264Decoder::GetRequiredNumOfPictures() const {
1456   return dpb_.max_num_pics() + kPicsInPipeline;
1457 }
1458 
1459 }  // namespace media
1460