上个月看完了讲H.264视频编解码的经典书：“The H.264 Advanced Video Compression Standard”，看完后感觉也不是那么难。记得大约十年前第一次看到预测(prediction), 运动补偿(motion compensation)之类的术语和示意图，感觉是非常晦涩难懂的知识。而现在补了基础信号理论的知识，也扫除了很多畏难心理。
后来了解到比较常用的开源H.264编码器是X264，而FFmpeg则自带一个广泛使用的开源H.264解码器。于是，我就想对照着开源代码，再过一遍理论知识。那就选FFmpeg吧～毕竟名气太大。
首先，得用FFmpeg整一个简单的播放器跑起来才能调试。但是，网络上虽然有大量FFmpeg编写简单播放器的代码，绝大部分代码却是无法用最新的FFmpeg通过编译的。毕竟FFmpeg在持续开发中，连上层API也会有变化。我下载了目前(2022/Feb/13)最新的代码，版本号是5.0。旧FFmpeg播放器代码跑不起来。
好在我在youtube.com上找到一位FFmpeg核心维护者Matt Szatmary做的presentation，An Introduction to Building tools with FFmpeg libraries and APIs – Matt Szatmary | August 2019，结合它的介绍，把简单的播放功能跑起来了。这里有个小遗憾没能找到Matt Szatmary的源代码，虽然他在视频中说是开源的。本来还想学习一下大佬的写法。
代码贴在这里。用这个小DEMO可以用调试器跟到FFmpeg的代码里看H264解码器的具体实现了。
#include <stdio.h>
#define __STDC_CONSTANT_MACROS
// Linux...
#ifdef __cplusplus
extern "C"
{
#endif
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
#include <SDL2/SDL.h>
#ifdef __cplusplus
};
#endif
int sfp_refresh_thread(void *opaque);
// Refresh Event
#define SFM_REFRESH_EVENT (SDL_USEREVENT + 1)
#define SFM_BREAK_EVENT (SDL_USEREVENT + 2)
int thread_exit = 0;
int thread_pause = 0;
int sfp_refresh_thread(void *opaque)
{
SDL_Event event;
thread_exit = 0;
thread_pause = 0;
while (!thread_exit)
{
if (!thread_pause)
{
SDL_Event event;
event.type = SFM_REFRESH_EVENT;
SDL_PushEvent(&event);
}
SDL_Delay(40);
}
thread_exit = 0;
thread_pause = 0;
// Break
event.type = SFM_BREAK_EVENT;
SDL_PushEvent(&event);
return 0;
}
int main(int argc, char *argv[])
{
AVFormatContext *pFormatCtx;
int i, videoindex;
AVCodecParameters *pCodecPar;
AVCodecContext *pCodecCtx;
const AVCodec *pCodec;
AVFrame *pFrame, *pFrameYUV;
unsigned char *out_buffer;
AVPacket *packet;
int ret, got_picture;
//------------SDL----------------
int screen_w, screen_h;
SDL_Window *screen;
SDL_Renderer *sdlRenderer;
SDL_Texture *sdlTexture;
SDL_Rect sdlRect;
SDL_Thread *video_tid;
SDL_Event event;
struct SwsContext *img_convert_ctx;
char filepath[] = "video";
// avformat_network_init();
// pFormatCtx = avformat_alloc_context();
pFormatCtx = NULL; // MYNOTE: if pFormatCtx is not allocated explicitly using avformat_alloc_context()
// pFormatCtx will be allocated implicitly in avformat_open_input.
if (avformat_open_input(&pFormatCtx, filepath, NULL, NULL) != 0)
{
printf("Couldn't open input stream.\n");
return -1;
}
if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
{
printf("Couldn't find stream information.\n");
return -1;
}
videoindex = -1;
for (i = 0; i < pFormatCtx->nb_streams; i++)
if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
{
videoindex = i;
pCodecPar = pFormatCtx->streams[i]->codecpar;
break;
}
if (videoindex == -1)
{
printf("Didn't find a video stream.\n");
return -1;
}
pCodec = avcodec_find_decoder(pCodecPar->codec_id);
if (pCodec == NULL)
{
printf("Codec not found.\n");
return -1;
}
pCodecCtx = avcodec_alloc_context3(pCodec);
if (!pCodecCtx)
{
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
// 这一步不能少，否则会崩溃。把CodecPar的参数再复制到codecCtx中
ret = avcodec_parameters_to_context(pCodecCtx, pCodecPar);
if (ret < 0)
{
printf("avcodec_parameters_to_context failed...");
return -1;
}
// pCodecCtx = pFormatCtx->streams[videoindex]->codecpar;
if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
{
printf("Could not open codec.\n");
return -1;
}
pFrame = av_frame_alloc();
pFrameYUV = av_frame_alloc();
size_t sz = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height, 1);
out_buffer = (unsigned char *)av_malloc(sz);
av_image_fill_arrays(pFrameYUV->data, pFrameYUV->linesize, out_buffer,
AV_PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height, 1);
// Output Info-----------------------------
printf("---------------- File Information ---------------\n");
av_dump_format(pFormatCtx, 0, filepath, 0);
printf("-------------------------------------------------\n");
img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt,
pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_YUV420P,
SWS_BICUBIC, NULL, NULL, NULL);
if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO |
SDL_INIT_TIMER))
{
printf("Could not initialize SDL - %s\n", SDL_GetError());
return -1;
}
// SDL 2.0 Support for multiple windows
screen_w = pCodecCtx->width;
screen_h = pCodecCtx->height;
screen = SDL_CreateWindow("Simplest ffmpeg player's Window",
SDL_WINDOWPOS_UNDEFINED,
SDL_WINDOWPOS_UNDEFINED,
screen_w, screen_h, SDL_WINDOW_OPENGL);
if (!screen)
{
printf("SDL: could not create window - exiting:%s\n", SDL_GetError());
return -1;
}
sdlRenderer = SDL_CreateRenderer(screen, -1, 0);
// IYUV: Y + U + V (3 planes)
// YV12: Y + V + U (3 planes)
sdlTexture = SDL_CreateTexture(sdlRenderer,
SDL_PIXELFORMAT_IYUV,
SDL_TEXTUREACCESS_STREAMING, pCodecCtx->width, pCodecCtx->height);
sdlRect.x = 0;
sdlRect.y = 0;
sdlRect.w = screen_w;
sdlRect.h = screen_h;
packet = (AVPacket *)av_malloc(sizeof(AVPacket));
video_tid = SDL_CreateThread(sfp_refresh_thread, NULL, NULL);
//------------SDL End------------
// Event Loop
for (;;)
{
char buf[1024];
int ret;
// Wait
SDL_WaitEvent(&event);
if (event.type == SFM_REFRESH_EVENT)
{
while (1)
{
if (av_read_frame(pFormatCtx, packet) < 0)
thread_exit = 1;
if (packet->stream_index == videoindex)
break;
}
// ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, packet);
// ret = my_decode();
ret = avcodec_send_packet(pCodecCtx, packet);
if (ret < 0)
{
fprintf(stderr, "Error sending a packet for decoding\n");
exit(1);
}
while (ret >= 0)
{
ret = avcodec_receive_frame(pCodecCtx, pFrame);
if (ret == AVERROR(EAGAIN))
{
fprintf(stderr, "EAGAIN occurred...\n");
break;
}
if (ret == AVERROR_EOF)
{
printf("byebye...\n");
exit(0);
}
if (ret < 0)
{
fprintf(stderr, "Error during decoding\n");
exit(1);
}
printf("Got frame %3d\n", pCodecCtx->frame_number);
fflush(stdout);
/* the picture is allocated by the decoder. no need to
free it */
// snprintf(buf, sizeof(buf), "%s-%d", "player.c", dec_ctx->frame_number);
// pgm_save(frame->data[0], frame->linesize[0], frame->width, frame->height, buf);
sws_scale(img_convert_ctx, (const unsigned char *const *)pFrame->data,
pFrame->linesize, 0, pCodecCtx->height, pFrameYUV->data, pFrameYUV->linesize);
// SDL---------------------------
SDL_UpdateTexture(sdlTexture, NULL, pFrameYUV->data[0],
pFrameYUV->linesize[0]);
SDL_RenderClear(sdlRenderer);
SDL_RenderCopy(sdlRenderer, sdlTexture, NULL, NULL);
SDL_RenderPresent(sdlRenderer);
// SDL End-----------------------
}
// av_free_packet(packet);
av_packet_unref(packet);
}
else if (event.type == SDL_KEYDOWN)
{
// Pause
if (event.key.keysym.sym == SDLK_SPACE)
thread_pause = !thread_pause;
}
else if (event.type == SDL_QUIT)
{
thread_exit = 1;
}
else if (event.type == SFM_BREAK_EVENT)
{
break;
}
}
sws_freeContext(img_convert_ctx);
SDL_Quit();
av_frame_free(&pFrameYUV);
av_frame_free(&pFrame);
avcodec_free_context(&pCodecCtx);
avcodec_close(pCodecCtx);
avformat_close_input(&pFormatCtx);
return 0;
}