上个月看完了讲H.264视频编解码的经典书:“The H.264 Advanced Video Compression Standard”,看完后感觉也不是那么难。记得大约十年前第一次看到预测(prediction), 运动补偿(motion compensation)之类的术语和示意图,感觉是非常晦涩难懂的知识。而现在补了基础信号理论的知识,也扫除了很多畏难心理。
后来了解到比较常用的开源H.264编码器是X264,而FFmpeg则自带一个广泛使用的开源H.264解码器。于是,我就想对照着开源代码,再过一遍理论知识。那就选FFmpeg吧~毕竟名气太大。
首先,得用FFmpeg整一个简单的播放器跑起来才能调试。但是,网络上虽然有大量FFmpeg编写简单播放器的代码,绝大部分代码却是无法用最新的FFmpeg通过编译的。毕竟FFmpeg在持续开发中,连上层API也会有变化。我下载了目前(2022/Feb/13)最新的代码,版本号是5.0。旧FFmpeg播放器代码跑不起来。
好在我在youtube.com上找到一位FFmpeg核心维护者Matt Szatmary做的presentation,An Introduction to Building tools with FFmpeg libraries and APIs – Matt Szatmary | August 2019{.wp-editor-md-post-content-link},结合它的介绍,把简单的播放功能跑起来了。这里有个小遗憾没能找到Matt Szatmary的源代码,虽然他在视频中说是开源的。本来还想学习一下大佬的写法。
代码贴在这里。用这个小DEMO可以用调试器跟到FFmpeg的代码里看H264解码器的具体实现了。
#include <stdio.h> #define __STDC_CONSTANT_MACROS // Linux... #ifdef __cplusplus extern "C" { #endif #include <libavcodec/avcodec.h> #include <libavformat/avformat.h> #include <libswscale/swscale.h> #include <libavutil/imgutils.h> #include <SDL2/SDL.h> #ifdef __cplusplus }; #endif int sfp_refresh_thread(void *opaque); // Refresh Event #define SFM_REFRESH_EVENT (SDL_USEREVENT + 1) #define SFM_BREAK_EVENT (SDL_USEREVENT + 2) int thread_exit = 0; int thread_pause = 0; int sfp_refresh_thread(void *opaque) { SDL_Event event; thread_exit = 0; thread_pause = 0; while (!thread_exit) { if (!thread_pause) { SDL_Event event; event.type = SFM_REFRESH_EVENT; SDL_PushEvent(&event); } SDL_Delay(40); } thread_exit = 0; thread_pause = 0; // Break event.type = SFM_BREAK_EVENT; SDL_PushEvent(&event); return 0; } int main(int argc, char *argv[]) { AVFormatContext *pFormatCtx; int i, videoindex; AVCodecParameters *pCodecPar; AVCodecContext *pCodecCtx; const AVCodec *pCodec; AVFrame *pFrame, *pFrameYUV; unsigned char *out_buffer; AVPacket *packet; int ret, got_picture; //------------SDL---------------- int screen_w, screen_h; SDL_Window *screen; SDL_Renderer *sdlRenderer; SDL_Texture *sdlTexture; SDL_Rect sdlRect; SDL_Thread *video_tid; SDL_Event event; struct SwsContext *img_convert_ctx; char filepath[] = "video"; // avformat_network_init(); // pFormatCtx = avformat_alloc_context(); pFormatCtx = NULL; // MYNOTE: if pFormatCtx is not allocated explicitly using avformat_alloc_context() // pFormatCtx will be allocated implicitly in avformat_open_input. if (avformat_open_input(&pFormatCtx, filepath, NULL, NULL) != 0) { printf("Couldn't open input stream.\n"); return -1; } if (avformat_find_stream_info(pFormatCtx, NULL) < 0) { printf("Couldn't find stream information.\n"); return -1; } videoindex = -1; for (i = 0; i < pFormatCtx->nb_streams; i++) if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { videoindex = i; pCodecPar = pFormatCtx->streams[i]->codecpar; break; } if (videoindex == -1) { printf("Didn't find a video stream.\n"); return -1; } pCodec = avcodec_find_decoder(pCodecPar->codec_id); if (pCodec == NULL) { printf("Codec not found.\n"); return -1; } pCodecCtx = avcodec_alloc_context3(pCodec); if (!pCodecCtx) { fprintf(stderr, "Could not allocate video codec context\n"); exit(1); } // 这一步不能少,否则会崩溃。把CodecPar的参数再复制到codecCtx中 ret = avcodec_parameters_to_context(pCodecCtx, pCodecPar); if (ret < 0) { printf("avcodec_parameters_to_context failed..."); return -1; } // pCodecCtx = pFormatCtx->streams[videoindex]->codecpar; if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) { printf("Could not open codec.\n"); return -1; } pFrame = av_frame_alloc(); pFrameYUV = av_frame_alloc(); size_t sz = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height, 1); out_buffer = (unsigned char *)av_malloc(sz); av_image_fill_arrays(pFrameYUV->data, pFrameYUV->linesize, out_buffer, AV_PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height, 1); // Output Info----------------------------- printf("---------------- File Information ---------------\n"); av_dump_format(pFormatCtx, 0, filepath, 0); printf("-------------------------------------------------\n"); img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL); if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) { printf("Could not initialize SDL - %s\n", SDL_GetError()); return -1; } // SDL 2.0 Support for multiple windows screen_w = pCodecCtx->width; screen_h = pCodecCtx->height; screen = SDL_CreateWindow("Simplest ffmpeg player's Window", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, screen_w, screen_h, SDL_WINDOW_OPENGL); if (!screen) { printf("SDL: could not create window - exiting:%s\n", SDL_GetError()); return -1; } sdlRenderer = SDL_CreateRenderer(screen, -1, 0); // IYUV: Y + U + V (3 planes) // YV12: Y + V + U (3 planes) sdlTexture = SDL_CreateTexture(sdlRenderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING, pCodecCtx->width, pCodecCtx->height); sdlRect.x = 0; sdlRect.y = 0; sdlRect.w = screen_w; sdlRect.h = screen_h; packet = (AVPacket *)av_malloc(sizeof(AVPacket)); video_tid = SDL_CreateThread(sfp_refresh_thread, NULL, NULL); //------------SDL End------------ // Event Loop for (;;) { char buf[1024]; int ret; // Wait SDL_WaitEvent(&event); if (event.type == SFM_REFRESH_EVENT) { while (1) { if (av_read_frame(pFormatCtx, packet) < 0) thread_exit = 1; if (packet->stream_index == videoindex) break; } // ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, packet); // ret = my_decode(); ret = avcodec_send_packet(pCodecCtx, packet); if (ret < 0) { fprintf(stderr, "Error sending a packet for decoding\n"); exit(1); } while (ret >= 0) { ret = avcodec_receive_frame(pCodecCtx, pFrame); if (ret == AVERROR(EAGAIN)) { fprintf(stderr, "EAGAIN occurred...\n"); break; } if (ret == AVERROR_EOF) { printf("byebye...\n"); exit(0); } if (ret < 0) { fprintf(stderr, "Error during decoding\n"); exit(1); } printf("Got frame %3d\n", pCodecCtx->frame_number); fflush(stdout); /* the picture is allocated by the decoder. no need to free it */ // snprintf(buf, sizeof(buf), "%s-%d", "player.c", dec_ctx->frame_number); // pgm_save(frame->data[0], frame->linesize[0], frame->width, frame->height, buf); sws_scale(img_convert_ctx, (const unsigned char *const *)pFrame->data, pFrame->linesize, 0, pCodecCtx->height, pFrameYUV->data, pFrameYUV->linesize); // SDL--------------------------- SDL_UpdateTexture(sdlTexture, NULL, pFrameYUV->data[0], pFrameYUV->linesize[0]); SDL_RenderClear(sdlRenderer); SDL_RenderCopy(sdlRenderer, sdlTexture, NULL, NULL); SDL_RenderPresent(sdlRenderer); // SDL End----------------------- } // av_free_packet(packet); av_packet_unref(packet); } else if (event.type == SDL_KEYDOWN) { // Pause if (event.key.keysym.sym == SDLK_SPACE) thread_pause = !thread_pause; } else if (event.type == SDL_QUIT) { thread_exit = 1; } else if (event.type == SFM_BREAK_EVENT) { break; } } sws_freeContext(img_convert_ctx); SDL_Quit(); av_frame_free(&pFrameYUV); av_frame_free(&pFrame); avcodec_free_context(&pCodecCtx); avcodec_close(pCodecCtx); avformat_close_input(&pFormatCtx); return 0; }