1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // Note: ported from Chromium commit head: 77be7ae 5 6 #ifndef H264_DECODER_H_ 7 #define H264_DECODER_H_ 8 9 #include <stddef.h> 10 #include <stdint.h> 11 12 #include <memory> 13 #include <vector> 14 15 #include "base/macros.h" 16 #include "base/memory/ref_counted.h" 17 #include "accelerated_video_decoder.h" 18 #include "h264_dpb.h" 19 #include "h264_parser.h" 20 #include "rect.h" 21 #include "size.h" 22 23 namespace media { 24 25 // Clients of this class are expected to pass H264 Annex-B byte stream 26 // and are expected to provide an implementation of H264Accelerator for 27 // offloading final steps of the decoding process. 28 // 29 // This class must be created, called and destroyed on a single thread, and 30 // does nothing internally on any other thread. 31 class H264Decoder : public AcceleratedVideoDecoder { 32 public: 33 class H264Accelerator { 34 public: 35 H264Accelerator(); 36 virtual ~H264Accelerator(); 37 38 // Create a new H264Picture that the decoder client can use for decoding 39 // and pass back to this accelerator for decoding or reference. 40 // When the picture is no longer needed by decoder, it will just drop 41 // its reference to it, and it may do so at any time. 42 // Note that this may return nullptr if accelerator is not able to provide 43 // any new pictures at given time. The decoder is expected to handle 44 // this situation as normal and return from Decode() with kRanOutOfSurfaces. 45 virtual scoped_refptr<H264Picture> CreateH264Picture() = 0; 46 47 // Submit metadata for the current frame, providing the current |sps| and 48 // |pps| for it, |dpb| has to contain all the pictures in DPB for current 49 // frame, and |ref_pic_p0/b0/b1| as specified in the H264 spec. Note that 50 // depending on the frame type, either p0, or b0 and b1 are used. |pic| 51 // contains information about the picture for the current frame. 52 // Note that this does not run decode in the accelerator and the decoder 53 // is expected to follow this call with one or more SubmitSlice() calls 54 // before calling SubmitDecode(). 55 // Return true if successful. 56 virtual bool SubmitFrameMetadata(const H264SPS* sps, 57 const H264PPS* pps, 58 const H264DPB& dpb, 59 const H264Picture::Vector& ref_pic_listp0, 60 const H264Picture::Vector& ref_pic_listb0, 61 const H264Picture::Vector& ref_pic_listb1, 62 const scoped_refptr<H264Picture>& pic) = 0; 63 64 // Submit one slice for the current frame, passing the current |pps| and 65 // |pic| (same as in SubmitFrameMetadata()), the parsed header for the 66 // current slice in |slice_hdr|, and the reordered |ref_pic_listX|, 67 // as per H264 spec. 68 // |data| pointing to the full slice (including the unparsed header| of 69 // |size| in bytes. 70 // This must be called one or more times per frame, before SubmitDecode(). 71 // Note that |data| does not have to remain valid after this call returns. 72 // Return true if successful. 73 virtual bool SubmitSlice(const H264PPS* pps, 74 const H264SliceHeader* slice_hdr, 75 const H264Picture::Vector& ref_pic_list0, 76 const H264Picture::Vector& ref_pic_list1, 77 const scoped_refptr<H264Picture>& pic, 78 const uint8_t* data, 79 size_t size) = 0; 80 81 // Execute the decode in hardware for |pic|, using all the slices and 82 // metadata submitted via SubmitFrameMetadata() and SubmitSlice() since 83 // the previous call to SubmitDecode(). 84 // Return true if successful. 85 virtual bool SubmitDecode(const scoped_refptr<H264Picture>& pic) = 0; 86 87 // Schedule output (display) of |pic|. Note that returning from this 88 // method does not mean that |pic| has already been outputted (displayed), 89 // but guarantees that all pictures will be outputted in the same order 90 // as this method was called for them. Decoder may drop its reference 91 // to |pic| after calling this method. 92 // Return true if successful. 93 virtual bool OutputPicture(const scoped_refptr<H264Picture>& pic) = 0; 94 95 // Reset any current state that may be cached in the accelerator, dropping 96 // any cached parameters/slices that have not been committed yet. 97 virtual void Reset() = 0; 98 99 private: 100 DISALLOW_COPY_AND_ASSIGN(H264Accelerator); 101 }; 102 103 H264Decoder(H264Accelerator* accelerator); 104 ~H264Decoder() override; 105 106 // AcceleratedVideoDecoder implementation. 107 bool Flush() override WARN_UNUSED_RESULT; 108 void Reset() override; 109 void SetStream(const uint8_t* ptr, size_t size) override; 110 DecodeResult Decode() override WARN_UNUSED_RESULT; 111 Size GetPicSize() const override; 112 size_t GetRequiredNumOfPictures() const override; 113 114 private: 115 // We need to keep at most kDPBMaxSize pictures in DPB for 116 // reference/to display later and an additional one for the one currently 117 // being decoded. We also ask for some additional ones since VDA needs 118 // to accumulate a few ready-to-output pictures before it actually starts 119 // displaying and giving them back. +2 instead of +1 because of subjective 120 // smoothness improvement during testing. 121 enum { 122 // TODO(johnylin): see if we could get rid of kMaxVideoFrames. 123 kMaxVideoFrames = 4, 124 kPicsInPipeline = kMaxVideoFrames + 2, 125 kMaxNumReqPictures = H264DPB::kDPBMaxSize + kPicsInPipeline, 126 }; 127 128 // Internal state of the decoder. 129 enum State { 130 kNeedStreamMetadata, // After initialization, need an SPS. 131 kDecoding, // Ready to decode from any point. 132 kAfterReset, // After Reset(), need a resume point. 133 kError, // Error in decode, can't continue. 134 }; 135 136 // Process H264 stream structures. 137 bool ProcessSPS(int sps_id, bool* need_new_buffers); 138 // Process current slice header to discover if we need to start a new picture, 139 // finishing up the current one. 140 bool PreprocessCurrentSlice(); 141 // Process current slice as a slice of the current picture. 142 bool ProcessCurrentSlice(); 143 144 // Return true if we need to start a new picture. 145 bool IsNewPrimaryCodedPicture(const H264SliceHeader* slice_hdr) const; 146 147 // Initialize the current picture according to data in |slice_hdr|. 148 bool InitCurrPicture(const H264SliceHeader* slice_hdr); 149 150 // Initialize |pic| as a "non-existing" picture (see spec) with |frame_num|, 151 // to be used for frame gap concealment. 152 bool InitNonexistingPicture(scoped_refptr<H264Picture> pic, int frame_num); 153 154 // Calculate picture order counts for |pic| on initialization 155 // of a new frame (see spec). 156 bool CalculatePicOrderCounts(scoped_refptr<H264Picture> pic); 157 158 // Update PicNum values in pictures stored in DPB on creation of 159 // a picture with |frame_num|. 160 void UpdatePicNums(int frame_num); 161 162 bool UpdateMaxNumReorderFrames(const H264SPS* sps); 163 164 // Prepare reference picture lists for the current frame. 165 void PrepareRefPicLists(const H264SliceHeader* slice_hdr); 166 // Prepare reference picture lists for the given slice. 167 bool ModifyReferencePicLists(const H264SliceHeader* slice_hdr, 168 H264Picture::Vector* ref_pic_list0, 169 H264Picture::Vector* ref_pic_list1); 170 171 // Construct initial reference picture lists for use in decoding of 172 // P and B pictures (see 8.2.4 in spec). 173 void ConstructReferencePicListsP(const H264SliceHeader* slice_hdr); 174 void ConstructReferencePicListsB(const H264SliceHeader* slice_hdr); 175 176 // Helper functions for reference list construction, per spec. 177 int PicNumF(const scoped_refptr<H264Picture>& pic); 178 int LongTermPicNumF(const scoped_refptr<H264Picture>& pic); 179 180 // Perform the reference picture lists' modification (reordering), as 181 // specified in spec (8.2.4). 182 // 183 // |list| indicates list number and should be either 0 or 1. 184 bool ModifyReferencePicList(const H264SliceHeader* slice_hdr, 185 int list, 186 H264Picture::Vector* ref_pic_listx); 187 188 // Perform reference picture memory management operations (marking/unmarking 189 // of reference pictures, long term picture management, discarding, etc.). 190 // See 8.2.5 in spec. 191 bool HandleMemoryManagementOps(scoped_refptr<H264Picture> pic); 192 bool ReferencePictureMarking(scoped_refptr<H264Picture> pic); 193 bool SlidingWindowPictureMarking(); 194 195 // Handle a gap in frame_num in the stream up to |frame_num|, by creating 196 // "non-existing" pictures (see spec). 197 bool HandleFrameNumGap(int frame_num); 198 199 // Start processing a new frame. 200 bool StartNewFrame(const H264SliceHeader* slice_hdr); 201 202 // All data for a frame received, process it and decode. 203 bool FinishPrevFrameIfPresent(); 204 205 // Called after we are done processing |pic|. Performs all operations to be 206 // done after decoding, including DPB management, reference picture marking 207 // and memory management operations. 208 // This will also output pictures if any have become ready to be outputted 209 // after processing |pic|. 210 bool FinishPicture(scoped_refptr<H264Picture> pic); 211 212 // Clear DPB contents and remove all surfaces in DPB from *in_use_ list. 213 // Cleared pictures will be made available for decode, unless they are 214 // at client waiting to be displayed. 215 void ClearDPB(); 216 217 // Commits all pending data for HW decoder and starts HW decoder. 218 bool DecodePicture(); 219 220 // Notifies client that a picture is ready for output. 221 void OutputPic(scoped_refptr<H264Picture> pic); 222 223 // Output all pictures in DPB that have not been outputted yet. 224 bool OutputAllRemainingPics(); 225 226 // Decoder state. 227 State state_; 228 229 // Parser in use. 230 H264Parser parser_; 231 232 // DPB in use. 233 H264DPB dpb_; 234 235 // Picture currently being processed/decoded. 236 scoped_refptr<H264Picture> curr_pic_; 237 238 // Reference picture lists, constructed for each frame. 239 H264Picture::Vector ref_pic_list_p0_; 240 H264Picture::Vector ref_pic_list_b0_; 241 H264Picture::Vector ref_pic_list_b1_; 242 243 // Global state values, needed in decoding. See spec. 244 int max_frame_num_; 245 int max_pic_num_; 246 int max_long_term_frame_idx_; 247 size_t max_num_reorder_frames_; 248 249 int prev_frame_num_; 250 int prev_ref_frame_num_; 251 int prev_frame_num_offset_; 252 bool prev_has_memmgmnt5_; 253 254 // Values related to previously decoded reference picture. 255 bool prev_ref_has_memmgmnt5_; 256 int prev_ref_top_field_order_cnt_; 257 int prev_ref_pic_order_cnt_msb_; 258 int prev_ref_pic_order_cnt_lsb_; 259 H264Picture::Field prev_ref_field_; 260 261 // Currently active SPS and PPS. 262 int curr_sps_id_; 263 int curr_pps_id_; 264 265 // Current NALU and slice header being processed. 266 std::unique_ptr<H264NALU> curr_nalu_; 267 std::unique_ptr<H264SliceHeader> curr_slice_hdr_; 268 269 // Output picture size. 270 Size pic_size_; 271 // Output visible cropping rect. 272 Rect visible_rect_; 273 274 // PicOrderCount of the previously outputted frame. 275 int last_output_poc_; 276 277 H264Accelerator* accelerator_; 278 279 DISALLOW_COPY_AND_ASSIGN(H264Decoder); 280 }; 281 282 } // namespace media 283 284 #endif // H264_DECODER_H_ 285