Source: lib/transmuxer/h264.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.transmuxer.H264');
  9. goog.require('shaka.util.ExpGolomb');
  10. goog.require('shaka.util.Uint8ArrayUtils');
  13. /**
  14.  * H.264 utils
  15.  */
  16. shaka.transmuxer.H264 = class {
  17.   /**
  18.    * Read a sequence parameter set and return some interesting video
  19.    * properties. A sequence parameter set is the H264 metadata that
  20.    * describes the properties of upcoming video frames.
  21.    *
  22.    * @param {!Array<shaka.extern.VideoNalu>} nalus
  23.    * @return {?{
  24.    *   height: number,
  25.    *   width: number,
  26.    *   videoConfig: !Uint8Array,
  27.    *   hSpacing: number,
  28.    *   vSpacing: number,
  29.    * }}
  30.    */
  31.   static parseInfo(nalus) {
  32.     const H264 = shaka.transmuxer.H264;
  33.     if (!nalus.length) {
  34.       return null;
  35.     }
  36.     const spsNalu = nalus.find((nalu) => {
  37.       return nalu.type == H264.NALU_TYPE_SPS_;
  38.     });
  39.     const ppsNalu = nalus.find((nalu) => {
  40.       return nalu.type == H264.NALU_TYPE_PPS_;
  41.     });
  42.     if (!spsNalu || !ppsNalu) {
  43.       return null;
  44.     }
  46.     const expGolombDecoder = new shaka.util.ExpGolomb(spsNalu.data);
  47.     // profile_idc
  48.     const profileIdc = expGolombDecoder.readUnsignedByte();
  49.     // constraint_set[0-5]_flag
  50.     expGolombDecoder.readUnsignedByte();
  51.     // level_idc u(8)
  52.     expGolombDecoder.readUnsignedByte();
  53.     // seq_parameter_set_id
  54.     expGolombDecoder.skipExpGolomb();
  56.     // some profiles have more optional data we don't need
  57.     if (H264.PROFILES_WITH_OPTIONAL_SPS_DATA_.includes(profileIdc)) {
  58.       const chromaFormatIdc = expGolombDecoder.readUnsignedExpGolomb();
  59.       if (chromaFormatIdc === 3) {
  60.         // separate_colour_plane_flag
  61.         expGolombDecoder.skipBits(1);
  62.       }
  63.       // bit_depth_luma_minus8
  64.       expGolombDecoder.skipExpGolomb();
  65.       // bit_depth_chroma_minus8
  66.       expGolombDecoder.skipExpGolomb();
  67.       // qpprime_y_zero_transform_bypass_flag
  68.       expGolombDecoder.skipBits(1);
  69.       // seq_scaling_matrix_present_flag
  70.       if (expGolombDecoder.readBoolean()) {
  71.         const scalingListCount = (chromaFormatIdc !== 3) ? 8 : 12;
  72.         for (let i = 0; i < scalingListCount; i++) {
  73.           // seq_scaling_list_present_flag[ i ]
  74.           if (expGolombDecoder.readBoolean()) {
  75.             if (i < 6) {
  76.               expGolombDecoder.skipScalingList(16);
  77.             } else {
  78.               expGolombDecoder.skipScalingList(64);
  79.             }
  80.           }
  81.         }
  82.       }
  83.     }
  85.     // log2_max_frame_num_minus4
  86.     expGolombDecoder.skipExpGolomb();
  87.     const picOrderCntType = expGolombDecoder.readUnsignedExpGolomb();
  89.     if (picOrderCntType === 0) {
  90.       // log2_max_pic_order_cnt_lsb_minus4
  91.       expGolombDecoder.readUnsignedExpGolomb();
  92.     } else if (picOrderCntType === 1) {
  93.       // delta_pic_order_always_zero_flag
  94.       expGolombDecoder.skipBits(1);
  95.       // offset_for_non_ref_pic
  96.       expGolombDecoder.skipExpGolomb();
  97.       // offset_for_top_to_bottom_field
  98.       expGolombDecoder.skipExpGolomb();
  99.       const numRefFramesInPicOrderCntCycle =
  100.           expGolombDecoder.readUnsignedExpGolomb();
  101.       for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
  102.         // offset_for_ref_frame[ i ]
  103.         expGolombDecoder.skipExpGolomb();
  104.       }
  105.     }
  107.     // max_num_ref_frames
  108.     expGolombDecoder.skipExpGolomb();
  109.     // gaps_in_frame_num_value_allowed_flag
  110.     expGolombDecoder.skipBits(1);
  112.     const picWidthInMbsMinus1 =
  113.         expGolombDecoder.readUnsignedExpGolomb();
  114.     const picHeightInMapUnitsMinus1 =
  115.         expGolombDecoder.readUnsignedExpGolomb();
  117.     const frameMbsOnlyFlag = expGolombDecoder.readBits(1);
  118.     if (frameMbsOnlyFlag === 0) {
  119.       // mb_adaptive_frame_field_flag
  120.       expGolombDecoder.skipBits(1);
  121.     }
  122.     // direct_8x8_inference_flag
  123.     expGolombDecoder.skipBits(1);
  125.     let frameCropLeftOffset = 0;
  126.     let frameCropRightOffset = 0;
  127.     let frameCropTopOffset = 0;
  128.     let frameCropBottomOffset = 0;
  130.     // frame_cropping_flag
  131.     if (expGolombDecoder.readBoolean()) {
  132.       frameCropLeftOffset = expGolombDecoder.readUnsignedExpGolomb();
  133.       frameCropRightOffset = expGolombDecoder.readUnsignedExpGolomb();
  134.       frameCropTopOffset = expGolombDecoder.readUnsignedExpGolomb();
  135.       frameCropBottomOffset = expGolombDecoder.readUnsignedExpGolomb();
  136.     }
  138.     let hSpacing = 1;
  139.     let vSpacing = 1;
  141.     // vui_parameters_present_flag
  142.     if (expGolombDecoder.readBoolean()) {
  143.       // aspect_ratio_info_present_flag
  144.       if (expGolombDecoder.readBoolean()) {
  145.         const aspectRatioIdc = expGolombDecoder.readUnsignedByte();
  146.         const hSpacingTable = [
  147.           1, 12, 10, 16, 40, 24, 20, 32, 80, 18, 15, 64, 160, 4, 3, 2,
  148.         ];
  149.         const vSpacingTable = [
  150.           1, 11, 11, 11, 33, 11, 11, 11, 33, 11, 11, 33, 99, 3, 2, 1,
  151.         ];
  152.         if (aspectRatioIdc > 0 && aspectRatioIdc <= 16) {
  153.           hSpacing = hSpacingTable[aspectRatioIdc - 1];
  154.           vSpacing = vSpacingTable[aspectRatioIdc - 1];
  155.         } else if (aspectRatioIdc === 255) {
  156.           hSpacing = expGolombDecoder.readBits(16);
  157.           vSpacing = expGolombDecoder.readBits(16);
  158.         }
  159.       }
  160.     }
  162.     const height = ((2 - frameMbsOnlyFlag) *
  163.         (picHeightInMapUnitsMinus1 + 1) * 16) - (frameCropTopOffset * 2) -
  164.         (frameCropBottomOffset * 2);
  165.     const width = ((picWidthInMbsMinus1 + 1) * 16) -
  166.         frameCropLeftOffset * 2 - frameCropRightOffset * 2;
  168.     // assemble the SPSs
  169.     let sps = [];
  170.     const spsData = spsNalu.fullData;
  171.     sps.push((spsData.byteLength >>> 8) & 0xff);
  172.     sps.push(spsData.byteLength & 0xff);
  173.     sps = sps.concat(...spsData);
  175.     // assemble the PPSs
  176.     let pps = [];
  177.     const ppsData = ppsNalu.fullData;
  178.     pps.push((ppsData.byteLength >>> 8) & 0xff);
  179.     pps.push(ppsData.byteLength & 0xff);
  180.     pps = pps.concat(...ppsData);
  182.     const videoConfig = new Uint8Array(
  183.         [
  184.           0x01, // version
  185.           sps[3], // profile
  186.           sps[4], // profile compat
  187.           sps[5], // level
  188.           0xfc | 3, // lengthSizeMinusOne, hard-coded to 4 bytes
  189.           0xe0 | 1, // 3bit reserved (111) + numOfSequenceParameterSets
  190.         ].concat(sps).concat([
  191.           1, // numOfPictureParameterSets
  192.         ]).concat(pps));
  194.     return {
  195.       height,
  196.       width,
  197.       videoConfig,
  198.       hSpacing,
  199.       vSpacing,
  200.     };
  201.   }
  203.   /**
  204.    * @param {!Array<shaka.extern.MPEG_PES>} videoData
  205.    * @return {!Array<shaka.extern.VideoSample>}
  206.    */
  207.   static getVideoSamples(videoData) {
  208.     const H264 = shaka.transmuxer.H264;
  210.     /** @type {!Array<shaka.extern.VideoSample>} */
  211.     const videoSamples = [];
  212.     /** @type {?shaka.extern.VideoSample} */
  213.     let lastVideoSample = null;
  214.     /** @type {boolean} */
  215.     let audFound = false;
  217.     const addLastVideoSample = () => {
  218.       if (!lastVideoSample) {
  219.         return;
  220.       }
  221.       if (!lastVideoSample.nalus.length || !lastVideoSample.frame) {
  222.         return;
  223.       }
  224.       const nalusData = [];
  225.       for (const nalu of lastVideoSample.nalus) {
  226.         const size = nalu.fullData.byteLength;
  227.         const naluLength = new Uint8Array(4);
  228.         naluLength[0] = (size >> 24) & 0xff;
  229.         naluLength[1] = (size >> 16) & 0xff;
  230.         naluLength[2] = (size >> 8) & 0xff;
  231.         naluLength[3] = size & 0xff;
  232.         nalusData.push(naluLength);
  233.         nalusData.push(nalu.fullData);
  234.       }
  235.       lastVideoSample.data = shaka.util.Uint8ArrayUtils.concat(...nalusData);
  236.       videoSamples.push(lastVideoSample);
  237.     };
  239.     const createLastVideoSample = (pes) => {
  240.       lastVideoSample = {
  241.         data: new Uint8Array([]),
  242.         frame: false,
  243.         isKeyframe: false,
  244.         pts: pes.pts,
  245.         dts: pes.dts,
  246.         nalus: [],
  247.       };
  248.     };
  250.     for (let i = 0; i < videoData.length; i++) {
  251.       const pes = videoData[i];
  252.       const nalus = pes.nalus;
  253.       let spsFound = false;
  255.       // If new NAL units found and last sample still there, let's push ...
  256.       // This helps parsing streams with missing AUD
  257.       // (only do this if AUD never found)
  258.       if (lastVideoSample && nalus.length && !audFound) {
  259.         addLastVideoSample();
  260.         createLastVideoSample(pes);
  261.       }
  263.       for (const nalu of pes.nalus) {
  264.         let push = false;
  265.         switch (nalu.type) {
  266.           case H264.NALU_TYPE_NDR_: {
  267.             let isKeyframe = false;
  268.             push = true;
  269.             const data = nalu.data;
  270.             // Only check slice type to detect KF in case SPS found in same
  271.             // packet (any keyframe is preceded by SPS ...)
  272.             if (spsFound && data.length > 4) {
  273.               // retrieve slice type by parsing beginning of NAL unit (follow
  274.               // H264 spec,slice_header definition) to detect keyframe embedded
  275.               // in NDR
  276.               const sliceType = new shaka.util.ExpGolomb(data).readSliceType();
  277.               // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
  278.               // SI slice : A slice that is coded using intra prediction only
  279.               // and using quantisation of the prediction samples.
  280.               // An SI slice can be coded such that its decoded samples can be
  281.               // constructed identically to an SP slice.
  282.               // I slice: A slice that is not an SI slice that is decoded using
  283.               // intra prediction only.
  284.               if (sliceType === 2 || sliceType === 4 ||
  285.                   sliceType === 7 || sliceType === 9) {
  286.                 isKeyframe = true;
  287.               }
  288.             }
  289.             if (isKeyframe) {
  290.               // If we have non-keyframe data already, that cannot belong to
  291.               // the same frame as a keyframe, so force a push
  292.               if (lastVideoSample &&
  293.                   lastVideoSample.frame && !lastVideoSample.isKeyframe) {
  294.                 addLastVideoSample();
  295.                 lastVideoSample = null;
  296.               }
  297.             }
  298.             if (!lastVideoSample) {
  299.               createLastVideoSample(pes);
  300.             }
  301.             lastVideoSample.frame = true;
  302.             lastVideoSample.isKeyframe = isKeyframe;
  303.             break;
  304.           }
  305.           case H264.NALU_TYPE_IDR_: {
  306.             push = true;
  307.             // Handle PES not starting with AUD
  308.             // If we have frame data already, that cannot belong to the same
  309.             // frame, so force a push
  310.             if (lastVideoSample &&
  311.                 lastVideoSample.frame && !lastVideoSample.isKeyframe) {
  312.               addLastVideoSample();
  313.               lastVideoSample = null;
  314.             }
  315.             if (!lastVideoSample) {
  316.               createLastVideoSample(pes);
  317.             }
  318.             lastVideoSample.frame = true;
  319.             lastVideoSample.isKeyframe = true;
  320.             break;
  321.           }
  322.           case H264.NALU_TYPE_SEI_:
  323.             push = true;
  324.             break;
  325.           case H264.NALU_TYPE_SPS_:
  326.             push = true;
  327.             spsFound = true;
  328.             break;
  329.           case H264.NALU_TYPE_PPS_:
  330.             push = true;
  331.             break;
  332.           case H264.NALU_TYPE_AUD_:
  333.             push = true;
  334.             audFound = true;
  335.             if (lastVideoSample && lastVideoSample.frame) {
  336.               addLastVideoSample();
  337.               lastVideoSample = null;
  338.             }
  339.             if (!lastVideoSample) {
  340.               createLastVideoSample(pes);
  341.             }
  342.             break;
  343.           case H264.NALU_TYPE_FILLER_DATA_:
  344.             push = true;
  345.             break;
  346.           default:
  347.             push = false;
  348.             break;
  349.         }
  350.         if (lastVideoSample && push) {
  351.           lastVideoSample.nalus.push(nalu);
  352.         }
  353.       }
  354.     }
  356.     // If last PES packet, push samples
  357.     addLastVideoSample();
  359.     return videoSamples;
  360.   }
  361. };
  364. /**
  365.  * NALU type for NDR for H.264.
  366.  * @const {number}
  367.  * @private
  368.  */
  369. shaka.transmuxer.H264.NALU_TYPE_NDR_ = 0x01;
  372. /**
  373.  * NALU type for Instantaneous Decoder Refresh (IDR) for H.264.
  374.  * @const {number}
  375.  * @private
  376.  */
  377. shaka.transmuxer.H264.NALU_TYPE_IDR_ = 0x05;
  380. /**
  381.  * NALU type for Supplemental Enhancement Information (SEI) for H.264.
  382.  * @const {number}
  383.  * @private
  384.  */
  385. shaka.transmuxer.H264.NALU_TYPE_SEI_ = 0x06;
  388. /**
  389.  * NALU type for Sequence Parameter Set (SPS) for H.264.
  390.  * @const {number}
  391.  * @private
  392.  */
  393. shaka.transmuxer.H264.NALU_TYPE_SPS_ = 0x07;
  396. /**
  397.  * NALU type for Picture Parameter Set (PPS) for H.264.
  398.  * @const {number}
  399.  * @private
  400.  */
  401. shaka.transmuxer.H264.NALU_TYPE_PPS_ = 0x08;
  404. /**
  405.  * NALU type for Access Unit Delimiter (AUD) for H.264.
  406.  * @const {number}
  407.  * @private
  408.  */
  409. shaka.transmuxer.H264.NALU_TYPE_AUD_ = 0x09;
  412. /**
  413.  * NALU type for Filler Data for H.264.
  414.  * @const {number}
  415.  * @private
  416.  */
  417. shaka.transmuxer.H264.NALU_TYPE_FILLER_DATA_ = 0x0c;
  420. /**
  421.  * Values of profile_idc that indicate additional fields are included in the
  422.  * SPS.
  423.  * see Recommendation ITU-T H.264 (4/2013)
  424.  * 7.3.2.1.1 Sequence parameter set data syntax
  425.  *
  426.  * @const {!Array<number>}
  427.  * @private
  428.  */
  429. shaka.transmuxer.H264.PROFILES_WITH_OPTIONAL_SPS_DATA_ =
  430.     [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134];