~ft/h264bsd

73e0ce9f876fc6ea042072addeb21cd9cbac443b — Sigrid Haflínudóttir 2 months ago 7d0cb59 master
report more errors; give up on non-baseline profile immediately
M src/h264bsd_byte_stream.c => src/h264bsd_byte_stream.c +8 -2
@@ 117,6 117,7 @@ u32 h264bsdExtractNalUnit(u8 *pByteStream, u32 len, strmData_t *pStrmData,
            {
                /* no start code prefix found -> error */
                *readBytes = len;
                werrstr("no start code prefix");
                return(HANTRO_NOK);
            }



@@ 187,6 188,7 @@ u32 h264bsdExtractNalUnit(u8 *pByteStream, u32 len, strmData_t *pStrmData,

    if (invalidStream)
    {
        werrstr("invalid stream");
        return(HANTRO_NOK);
    }



@@ 204,8 206,10 @@ u32 h264bsdExtractNalUnit(u8 *pByteStream, u32 len, strmData_t *pStrmData,
                 * following bytes: 0x00, 0x01, 0x02, 0x03. This implies that
                 * emulation prevention 0x03 byte shall not be the last byte
                 * of the stream. */
                if ( (i == 0) || (*(readPtr+1) > 0x03) )
                if ( (i == 0) || (*(readPtr+1) > 0x03) ) {
                    werrstr("emulation byte last in the stream");
                    return(HANTRO_NOK);
                }

                /* do not write emulation prevention byte */
                readPtr++;


@@ 215,8 219,10 @@ u32 h264bsdExtractNalUnit(u8 *pByteStream, u32 len, strmData_t *pStrmData,
            {
                /* NAL unit shall not contain byte sequences 0x000000,
                 * 0x000001 or 0x000002 */
                if ( (zeroCount == 2) && (*readPtr <= 0x02) )
                if ( (zeroCount == 2) && (*readPtr <= 0x02) ) {
                    werrstr("start sequence in NAL unit");
                    return(HANTRO_NOK);
                }

                if (*readPtr == 0)
                    zeroCount++;

M src/h264bsd_cavlc.c => src/h264bsd_cavlc.c +9 -3
@@ 784,8 784,10 @@ u32 h264bsdDecodeResidualBlockCavlc(
    BUFFER_FLUSH(bufferValue, bufferBits, LENGTH_TC(tmp));

    totalCoeff = TOTAL_COEFF(tmp);
    if (totalCoeff > maxNumCoeff)
    if (totalCoeff > maxNumCoeff) {
        werrstr("invalid total coeff: %d", totalCoeff);
        return(HANTRO_NOK);
    }
    trailingOnes = TRAILING_ONES(tmp);

    if (totalCoeff != 0)


@@ 813,8 815,10 @@ u32 h264bsdDecodeResidualBlockCavlc(
        {
            BUFFER_SHOW(bufferValue, bufferBits, bit, 16);
            levelPrefix = DecodeLevelPrefix(bit);
            if (levelPrefix == VLC_NOT_FOUND)
            if (levelPrefix == VLC_NOT_FOUND) {
                werrstr("level prefix unknown");
                return(HANTRO_NOK);
            }
            BUFFER_FLUSH(bufferValue, bufferBits, levelPrefix+1);

            if (levelPrefix < 14)


@@ 865,8 869,10 @@ u32 h264bsdDecodeResidualBlockCavlc(
            BUFFER_SHOW(bufferValue, bufferBits, bit,9);
            zerosLeft = DecodeTotalZeros(bit, totalCoeff,
                                        (u32)(maxNumCoeff == 4));
            if (!zerosLeft)
            if (!zerosLeft) {
                werrstr("no zeros left");
                return(HANTRO_NOK);
            }
            BUFFER_FLUSH(bufferValue, bufferBits, LENGTH(zerosLeft));
            zerosLeft = INFO(zerosLeft);
        }

M src/h264bsd_decoder.c => src/h264bsd_decoder.c +3 -1
@@ 103,8 103,10 @@ u32 h264bsdInit(storage_t *pStorage, u32 noOutputReordering)
    size = (sizeof(macroblockLayer_t) + 63) & ~0x3F;

    pStorage->mbLayer = (macroblockLayer_t*)malloc(size);
    if (!pStorage->mbLayer)
    if (!pStorage->mbLayer) {
        werrstr("no memory");
        return HANTRO_NOK;
    }

    if (noOutputReordering)
        pStorage->noReordering = HANTRO_TRUE;

M src/h264bsd_dpb.c => src/h264bsd_dpb.c +6 -2
@@ 285,8 285,10 @@ u32 h264bsdReorderRefPicList(
        }
        /* find corresponding picture from dpb */
        index = FindDpbPic(dpb, picNum, isShortTerm);
        if (index < 0 || !IS_EXISTING(dpb->buffer[index]))
        if (index < 0 || !IS_EXISTING(dpb->buffer[index])) {
            werrstr("dpb: picture not found");
            return(HANTRO_NOK);
        }

        /* shift pictures */
        for (j = numRefIdxActive; j > refIdx; j--)


@@ 652,7 654,7 @@ u32 h264bsdMarkDecRefPic(

    if (image->data != dpb->currentOut->data)
    {
        EPRINT("TRYING TO MARK NON-ALLOCATED IMAGE");
        werrstr("trying to mark non-allocated image");
        return(HANTRO_NOK);
    }



@@ 760,6 762,7 @@ u32 h264bsdMarkDecRefPic(
                        break;

                    default: /* invalid memory management control operation */
                        werrstr("invalid mem management control op %d", mark->operation[i].memoryManagementControlOperation);
                        status = HANTRO_NOK;
                        break;
                }


@@ 792,6 795,7 @@ u32 h264bsdMarkDecRefPic(
            /* no room */
            else
            {
                werrstr("no room for pic in dpb");
                status = HANTRO_NOK;
            }
        }

M src/h264bsd_inter_prediction.c => src/h264bsd_inter_prediction.c +36 -12
@@ 535,13 535,17 @@ static u32 MvPrediction16x16(mbStorage_t *pMb, mbPred_t *mbPred, dpbStorage_t *d
        mv.ver += mvPred.ver;

        /* horizontal motion vector range [-2048, 2047.75] */
        if ((u32)(i32)(mv.hor+8192) >= (16384))
        if ((u32)(i32)(mv.hor+8192) >= (16384)) {
            werrstr("horizontal motion vector out of range");
            return(HANTRO_NOK);
        }

        /* vertical motion vector range [-512, 511.75]
         * (smaller for low levels) */
        if ((u32)(i32)(mv.ver+2048) >= (4096))
        if ((u32)(i32)(mv.ver+2048) >= (4096)) {
            werrstr("vertical motion vector out of range");
            return(HANTRO_NOK);
        }
    }

    tmp = h264bsdGetRefPicData(dpb, refIndex);


@@ 623,12 627,16 @@ static u32 MvPrediction16x8(mbStorage_t *pMb, mbPred_t *mbPred, dpbStorage_t *dp
    mv.ver += mvPred.ver;

    /* horizontal motion vector range [-2048, 2047.75] */
    if ((u32)(i32)(mv.hor+8192) >= (16384))
    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
        werrstr("horizontal motion vector out of range");
        return(HANTRO_NOK);
    }

    /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
    if ((u32)(i32)(mv.ver+2048) >= (4096))
    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
        werrstr("vertical motion vector out of range");
        return(HANTRO_NOK);
    }

    tmp = h264bsdGetRefPicData(dpb, refIndex);
    if (tmp == NULL)


@@ 669,12 677,16 @@ static u32 MvPrediction16x8(mbStorage_t *pMb, mbPred_t *mbPred, dpbStorage_t *dp
    mv.ver += mvPred.ver;

    /* horizontal motion vector range [-2048, 2047.75] */
    if ((u32)(i32)(mv.hor+8192) >= (16384))
    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
        werrstr("horizontal motion vector out of range");
        return(HANTRO_NOK);
    }

    /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
    if ((u32)(i32)(mv.ver+2048) >= (4096))
    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
        werrstr("vertical motion vector out of range");
        return(HANTRO_NOK);
    }

    tmp = h264bsdGetRefPicData(dpb, refIndex);
    if (tmp == NULL)


@@ 742,12 754,16 @@ static u32 MvPrediction8x16(mbStorage_t *pMb, mbPred_t *mbPred, dpbStorage_t *dp
    mv.ver += mvPred.ver;

    /* horizontal motion vector range [-2048, 2047.75] */
    if ((u32)(i32)(mv.hor+8192) >= (16384))
    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
        werrstr("horizontal motion vector out of range");
        return(HANTRO_NOK);
    }

    /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
    if ((u32)(i32)(mv.ver+2048) >= (4096))
    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
        werrstr("vertical motion vector out of range");
        return(HANTRO_NOK);
    }

    tmp = h264bsdGetRefPicData(dpb, refIndex);
    if (tmp == NULL)


@@ 791,12 807,16 @@ static u32 MvPrediction8x16(mbStorage_t *pMb, mbPred_t *mbPred, dpbStorage_t *dp
    mv.ver += mvPred.ver;

    /* horizontal motion vector range [-2048, 2047.75] */
    if ((u32)(i32)(mv.hor+8192) >= (16384))
    if ((u32)(i32)(mv.hor+8192) >= (16384)) {
        werrstr("horizontal motion vector out of range");
        return(HANTRO_NOK);
    }

    /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
    if ((u32)(i32)(mv.ver+2048) >= (4096))
    if ((u32)(i32)(mv.ver+2048) >= (4096)) {
        werrstr("vertical motion vector out of range");
        return(HANTRO_NOK);
    }

    tmp = h264bsdGetRefPicData(dpb, refIndex);
    if (tmp == NULL)


@@ 909,12 929,16 @@ static u32 MvPrediction(mbStorage_t *pMb, subMbPred_t *subMbPred, u32 mbPartIdx,
    mv.ver += mvPred.ver;

    /* horizontal motion vector range [-2048, 2047.75] */
    if (((u32)(i32)(mv.hor+8192) >= (16384)))
    if (((u32)(i32)(mv.hor+8192) >= (16384))) {
        werrstr("horizontal motion vector out of range");
        return(HANTRO_NOK);
    }

    /* vertical motion vector range [-512, 511.75] (smaller for low levels) */
    if (((u32)(i32)(mv.ver+2048) >= (4096)))
    if (((u32)(i32)(mv.ver+2048) >= (4096))) {
        werrstr("vertical motion vector out of range");
        return(HANTRO_NOK);
    }

    switch (subMbPartMode)
    {

M src/h264bsd_nal_unit.c => src/h264bsd_nal_unit.c +3 -1
@@ 83,8 83,10 @@ u32 h264bsdDecodeNalUnit(strmData_t *pStrmData, nalUnit_t *pNalUnit)
    tmp = h264bsdGetBits(pStrmData, 1);
    /* Assuming that NAL unit starts from byte boundary �> don't have to check
     * following 7 bits for END_OF_STREAM */
    if (tmp == END_OF_STREAM)
    if (tmp == END_OF_STREAM) {
        werrstr("end of stream");
        return(HANTRO_NOK);
    }

    tmp = h264bsdGetBits(pStrmData, 2);
    pNalUnit->nalRefIdc = tmp;

M src/h264bsd_pic_param_set.c => src/h264bsd_pic_param_set.c +12 -12
@@ 109,7 109,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
        return(tmp);
    if (pPicParamSet->picParameterSetId >= MAX_NUM_PIC_PARAM_SETS)
    {
        EPRINT("pic_parameter_set_id");
        werrstr("pic_parameter_set_id out of range: %d", pPicParamSet->picParameterSetId);
        return(HANTRO_NOK);
    }



@@ 119,7 119,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
        return(tmp);
    if (pPicParamSet->seqParameterSetId >= MAX_NUM_SEQ_PARAM_SETS)
    {
        EPRINT("seq_param_set_id");
        werrstr("seq_param_set_id out of range: %d", pPicParamSet->seqParameterSetId);
        return(HANTRO_NOK);
    }



@@ 127,7 127,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
    tmp = h264bsdGetBits(pStrmData, 1);
    if (tmp)
    {
        EPRINT("entropy_coding_mode_flag");
        werrstr("unexpected entropy_coding_mode_flag");
        return(HANTRO_NOK);
    }



@@ 143,7 143,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
    pPicParamSet->numSliceGroups = value + 1;
    if (pPicParamSet->numSliceGroups > MAX_NUM_SLICE_GROUPS)
    {
        EPRINT("num_slice_groups_minus1");
        werrstr("num_slice_groups_minus1 out of range: %d", pPicParamSet->numSliceGroups);
        return(HANTRO_NOK);
    }



@@ 156,7 156,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
            return(tmp);
        if (pPicParamSet->sliceGroupMapType > 6)
        {
            EPRINT("slice_group_map_type");
            werrstr("slice_group_map_type out of range: %d", pPicParamSet->sliceGroupMapType);
            return(HANTRO_NOK);
        }



@@ 234,7 234,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
                if ( pPicParamSet->sliceGroupId[i] >=
                     pPicParamSet->numSliceGroups )
                {
                    EPRINT("slice_group_id");
                    werrstr("slice_group_id out of range");
                    return(HANTRO_NOK);
                }
            }


@@ 258,7 258,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
        return(tmp);
    if (value > 31)
    {
        EPRINT("num_ref_idx_l1_active_minus1");
        werrstr("num_ref_idx_l1_active_minus1 out of range: %d", value);
        return(HANTRO_NOK);
    }



@@ 266,7 266,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
    tmp = h264bsdGetBits(pStrmData, 1);
    if (tmp)
    {
        EPRINT("weighted_pred_flag");
        werrstr("unexpected weighted_pred_flag");
        return(HANTRO_NOK);
    }



@@ 274,7 274,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
    tmp = h264bsdGetBits(pStrmData, 2);
    if (tmp > 2)
    {
        EPRINT("weighted_bipred_idc");
        werrstr("weighted_bipred_idc out of range: %d", tmp);
        return(HANTRO_NOK);
    }



@@ 284,7 284,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
        return(tmp);
    if ((itmp < -26) || (itmp > 25))
    {
        EPRINT("pic_init_qp_minus26");
        werrstr("pic_init_qp_minus26 out of range: %d", itmp);
        return(HANTRO_NOK);
    }
    pPicParamSet->picInitQp = (u32)(itmp + 26);


@@ 295,7 295,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
        return(tmp);
    if ((itmp < -26) || (itmp > 25))
    {
        EPRINT("pic_init_qs_minus26");
        werrstr("pic_init_qs_minus26 out of range: %d", itmp);
        return(HANTRO_NOK);
    }



@@ 304,7 304,7 @@ u32 h264bsdDecodePicParamSet(strmData_t *pStrmData, picParamSet_t *pPicParamSet)
        return(tmp);
    if ((itmp < -12) || (itmp > 12))
    {
        EPRINT("chroma_qp_index_offset");
        werrstr("chroma_qp_index_offset out of range: %d", itmp);
        return(HANTRO_NOK);
    }
    pPicParamSet->chromaQpIndexOffset = itmp;

M src/h264bsd_sei.c => src/h264bsd_sei.c +3 -1
@@ 418,8 418,10 @@ static u32 DecodeBufferingPeriod(
      &pBufferingPeriod->seqParameterSetId);
    if (tmp != HANTRO_OK)
        return(tmp);
    if (pBufferingPeriod->seqParameterSetId > 31)
    if (pBufferingPeriod->seqParameterSetId > 31) {
        werrstr("seqParameterSetId out of range: %d", pBufferingPeriod->seqParameterSetId);
        return(HANTRO_NOK);
    }

    if (nalHrdBpPresentFlag)
    {

M src/h264bsd_seq_param_set.c => src/h264bsd_seq_param_set.c +14 -12
@@ 99,9 99,9 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
    tmp = h264bsdGetBits(pStrmData, 8);
    if (tmp == END_OF_STREAM)
        return(HANTRO_NOK);
    if (tmp != 66)
    {
        DEBUG(("NOT BASELINE PROFILE %d\n", tmp));
    if (tmp != 66) {
        werrstr("not baseline profile: %d", tmp);
        return(HANTRO_NOK);
    }
    pSeqParamSet->profileIdc = tmp;



@@ 131,7 131,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
        return(tmp);
    if (pSeqParamSet->seqParameterSetId >= MAX_NUM_SEQ_PARAM_SETS)
    {
        EPRINT("seq_param_set_id");
        werrstr("seq_param_set_id out of range: %d", pSeqParamSet->seqParameterSetId);
        return(HANTRO_NOK);
    }



@@ 141,7 141,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
        return(tmp);
    if (value > 12)
    {
        EPRINT("log2_max_frame_num_minus4");
        werrstr("log2_max_frame_num_minus4 out of range: %d", value);
        return(HANTRO_NOK);
    }
    /* maxFrameNum = 2^(log2_max_frame_num_minus4 + 4) */


@@ 153,7 153,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
        return(tmp);
    if (value > 2)
    {
        EPRINT("pic_order_cnt_type");
        werrstr("pic_order_cnt_type out of range: %d", value);
        return(HANTRO_NOK);
    }
    pSeqParamSet->picOrderCntType = value;


@@ 166,7 166,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
            return(tmp);
        if (value > 12)
        {
            EPRINT("log2_max_pic_order_cnt_lsb_minus4");
            werrstr("log2_max_pic_order_cnt_lsb_minus4 out of range: %d", value);
            return(HANTRO_NOK);
        }
        /* maxPicOrderCntLsb = 2^(log2_max_pic_order_cnt_lsb_minus4 + 4) */


@@ 196,7 196,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
            return(tmp);
        if (pSeqParamSet->numRefFramesInPicOrderCntCycle > 255)
        {
            EPRINT("num_ref_frames_in_pic_order_cnt_cycle");
            werrstr("num_ref_frames_in_pic_order_cnt_cycle out of range: %d", pSeqParamSet->numRefFramesInPicOrderCntCycle);
            return(HANTRO_NOK);
        }



@@ 228,7 228,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
        return(tmp);
    if (pSeqParamSet->numRefFrames > MAX_NUM_REF_PICS)
    {
        EPRINT("num_ref_frames");
        werrstr("num_ref_frames out of range: %d", pSeqParamSet->numRefFrames);
        return(HANTRO_NOK);
    }



@@ 254,7 254,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
        return(HANTRO_NOK);
    if (!tmp)
    {
        EPRINT("frame_mbs_only_flag");
        werrstr("frame_mbs_only_flag not set");
        return(HANTRO_NOK);
    }



@@ 296,7 296,7 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
               ( 8 * (i32)pSeqParamSet->picHeightInMbs -
                 ((i32)pSeqParamSet->frameCropBottomOffset + 1) ) ) )
        {
            EPRINT("frame_cropping");
            werrstr("frame_cropping invalid");
            return(HANTRO_NOK);
        }
    }


@@ 323,8 323,10 @@ u32 h264bsdDecodeSeqParamSet(strmData_t *pStrmData, seqParamSet_t *pSeqParamSet)
    if (pSeqParamSet->vuiParametersPresentFlag)
    {
        ALLOCATE(pSeqParamSet->vuiParameters, 1, vuiParameters_t);
        if (pSeqParamSet->vuiParameters == NULL)
        if (pSeqParamSet->vuiParameters == NULL) {
            werrstr("no memory");
            return(MEMORY_ALLOCATION_ERROR);
        }
        tmp = h264bsdDecodeVuiParameters(pStrmData,
            pSeqParamSet->vuiParameters);
        if (tmp != HANTRO_OK)

M src/h264bsd_vlc.c => src/h264bsd_vlc.c +7 -1
@@ 175,10 175,12 @@ u32 h264bsdDecodeExpGolombUnsigned(strmData_t *pStrmData, u32 *codeNum)
                else if (bits == 1)
                {
                    *codeNum = BIG_CODE_NUM;
                    werrstr("big_code_num");
                    return(HANTRO_NOK);
                }
            }
            /* if more zeros than 32, it is an error */
            werrstr("zeros > 32");
            return(HANTRO_NOK);
        }
        else


@@ 267,6 269,8 @@ u32 h264bsdDecodeExpGolombSigned(strmData_t *pStrmData, i32 *value)
        return(HANTRO_OK);
    }

    werrstr("h264bsdDecodeExpGolombSigned failed");

    return(HANTRO_NOK);

}


@@ 315,8 319,10 @@ u32 h264bsdDecodeExpGolombMapped(strmData_t *pStrmData, u32 *value,
    else
    {
        /* range of valid codeNums [0,47] */
        if (codeNum > 47)
        if (codeNum > 47) {
            werrstr("invalid codeNum %d", codeNum);
            return (HANTRO_NOK);
        }
        if (isIntra)
            *value = codedBlockPatternIntra4x4[codeNum];
        else