src/feature/ffmpeg/ffmpeg-encoder.c (view raw)
1/* Copyright (c) 2013-2015 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include "ffmpeg-encoder.h"
7
8#include <mgba/core/core.h>
9#include <mgba/internal/gba/video.h>
10
11#include <libavcodec/version.h>
12#include <libavcodec/avcodec.h>
13
14#include <libavutil/version.h>
15#if LIBAVUTIL_VERSION_MAJOR >= 53
16#include <libavutil/buffer.h>
17#endif
18#include <libavutil/imgutils.h>
19#include <libavutil/mathematics.h>
20#include <libavutil/opt.h>
21
22#include <libavresample/avresample.h>
23#include <libswscale/swscale.h>
24
25static void _ffmpegPostVideoFrame(struct mAVStream*, const color_t* pixels, size_t stride);
26static void _ffmpegPostAudioFrame(struct mAVStream*, int16_t left, int16_t right);
27static void _ffmpegSetVideoDimensions(struct mAVStream*, unsigned width, unsigned height);
28
29enum {
30 PREFERRED_SAMPLE_RATE = 0x8000
31};
32
33void FFmpegEncoderInit(struct FFmpegEncoder* encoder) {
34 av_register_all();
35
36 encoder->d.videoDimensionsChanged = _ffmpegSetVideoDimensions;
37 encoder->d.postVideoFrame = _ffmpegPostVideoFrame;
38 encoder->d.postAudioFrame = _ffmpegPostAudioFrame;
39 encoder->d.postAudioBuffer = 0;
40
41 encoder->audioCodec = 0;
42 encoder->videoCodec = 0;
43 encoder->containerFormat = 0;
44 FFmpegEncoderSetAudio(encoder, "flac", 0);
45 FFmpegEncoderSetVideo(encoder, "png", 0);
46 FFmpegEncoderSetContainer(encoder, "matroska");
47 FFmpegEncoderSetDimensions(encoder, VIDEO_HORIZONTAL_PIXELS, VIDEO_VERTICAL_PIXELS);
48 encoder->iwidth = VIDEO_HORIZONTAL_PIXELS;
49 encoder->iheight = VIDEO_VERTICAL_PIXELS;
50 encoder->resampleContext = 0;
51 encoder->absf = 0;
52 encoder->context = 0;
53 encoder->scaleContext = NULL;
54}
55
56bool FFmpegEncoderSetAudio(struct FFmpegEncoder* encoder, const char* acodec, unsigned abr) {
57 static const struct {
58 int format;
59 int priority;
60 } priorities[] = {
61 { AV_SAMPLE_FMT_S16, 0 },
62 { AV_SAMPLE_FMT_S16P, 1 },
63 { AV_SAMPLE_FMT_S32, 2 },
64 { AV_SAMPLE_FMT_S32P, 2 },
65 { AV_SAMPLE_FMT_FLT, 3 },
66 { AV_SAMPLE_FMT_FLTP, 3 },
67 { AV_SAMPLE_FMT_DBL, 4 },
68 { AV_SAMPLE_FMT_DBLP, 4 }
69 };
70
71 if (!acodec) {
72 encoder->audioCodec = 0;
73 return true;
74 }
75
76 AVCodec* codec = avcodec_find_encoder_by_name(acodec);
77 if (!codec) {
78 return false;
79 }
80
81 if (!codec->sample_fmts) {
82 return false;
83 }
84 size_t i;
85 size_t j;
86 int priority = INT_MAX;
87 encoder->sampleFormat = AV_SAMPLE_FMT_NONE;
88 for (i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; ++i) {
89 for (j = 0; j < sizeof(priorities) / sizeof(*priorities); ++j) {
90 if (codec->sample_fmts[i] == priorities[j].format && priority > priorities[j].priority) {
91 priority = priorities[j].priority;
92 encoder->sampleFormat = codec->sample_fmts[i];
93 }
94 }
95 }
96 if (encoder->sampleFormat == AV_SAMPLE_FMT_NONE) {
97 return false;
98 }
99 encoder->sampleRate = PREFERRED_SAMPLE_RATE;
100 if (codec->supported_samplerates) {
101 for (i = 0; codec->supported_samplerates[i]; ++i) {
102 if (codec->supported_samplerates[i] < PREFERRED_SAMPLE_RATE) {
103 continue;
104 }
105 if (encoder->sampleRate == PREFERRED_SAMPLE_RATE || encoder->sampleRate > codec->supported_samplerates[i]) {
106 encoder->sampleRate = codec->supported_samplerates[i];
107 }
108 }
109 } else if (codec->id == AV_CODEC_ID_AAC) {
110 // HACK: AAC doesn't support 32768Hz (it rounds to 32000), but libfaac doesn't tell us that
111 encoder->sampleRate = 44100;
112 }
113 encoder->audioCodec = acodec;
114 encoder->audioBitrate = abr;
115 return true;
116}
117
118bool FFmpegEncoderSetVideo(struct FFmpegEncoder* encoder, const char* vcodec, unsigned vbr) {
119 static const struct {
120 enum AVPixelFormat format;
121 int priority;
122 } priorities[] = {
123 { AV_PIX_FMT_RGB555, 0 },
124 { AV_PIX_FMT_BGR555, 0 },
125 { AV_PIX_FMT_RGB565, 1 },
126 { AV_PIX_FMT_BGR565, 1 },
127 { AV_PIX_FMT_RGB24, 2 },
128 { AV_PIX_FMT_BGR24, 2 },
129#ifndef USE_LIBAV
130 { AV_PIX_FMT_BGR0, 3 },
131 { AV_PIX_FMT_RGB0, 3 },
132 { AV_PIX_FMT_0BGR, 3 },
133 { AV_PIX_FMT_0RGB, 3 },
134#endif
135 { AV_PIX_FMT_YUV422P, 4 },
136 { AV_PIX_FMT_YUV444P, 5 },
137 { AV_PIX_FMT_YUV420P, 6 }
138 };
139 AVCodec* codec = avcodec_find_encoder_by_name(vcodec);
140 if (!codec) {
141 return false;
142 }
143
144 size_t i;
145 size_t j;
146 int priority = INT_MAX;
147 encoder->pixFormat = AV_PIX_FMT_NONE;
148 for (i = 0; codec->pix_fmts[i] != AV_PIX_FMT_NONE; ++i) {
149 for (j = 0; j < sizeof(priorities) / sizeof(*priorities); ++j) {
150 if (codec->pix_fmts[i] == priorities[j].format && priority > priorities[j].priority) {
151 priority = priorities[j].priority;
152 encoder->pixFormat = codec->pix_fmts[i];
153 }
154 }
155 }
156 if (encoder->pixFormat == AV_PIX_FMT_NONE) {
157 return false;
158 }
159 encoder->videoCodec = vcodec;
160 encoder->videoBitrate = vbr;
161 return true;
162}
163
164bool FFmpegEncoderSetContainer(struct FFmpegEncoder* encoder, const char* container) {
165 AVOutputFormat* oformat = av_guess_format(container, 0, 0);
166 if (!oformat) {
167 return false;
168 }
169 encoder->containerFormat = container;
170 return true;
171}
172
173void FFmpegEncoderSetDimensions(struct FFmpegEncoder* encoder, int width, int height) {
174 encoder->width = width > 0 ? width : VIDEO_HORIZONTAL_PIXELS;
175 encoder->height = height > 0 ? height : VIDEO_VERTICAL_PIXELS;
176}
177
178bool FFmpegEncoderVerifyContainer(struct FFmpegEncoder* encoder) {
179 AVOutputFormat* oformat = av_guess_format(encoder->containerFormat, 0, 0);
180 AVCodec* acodec = avcodec_find_encoder_by_name(encoder->audioCodec);
181 AVCodec* vcodec = avcodec_find_encoder_by_name(encoder->videoCodec);
182 if ((encoder->audioCodec && !acodec) || !vcodec || !oformat) {
183 return false;
184 }
185 if (encoder->audioCodec && !avformat_query_codec(oformat, acodec->id, FF_COMPLIANCE_EXPERIMENTAL)) {
186 return false;
187 }
188 if (!avformat_query_codec(oformat, vcodec->id, FF_COMPLIANCE_EXPERIMENTAL)) {
189 return false;
190 }
191 return true;
192}
193
194bool FFmpegEncoderOpen(struct FFmpegEncoder* encoder, const char* outfile) {
195 AVCodec* acodec = avcodec_find_encoder_by_name(encoder->audioCodec);
196 AVCodec* vcodec = avcodec_find_encoder_by_name(encoder->videoCodec);
197 if ((encoder->audioCodec && !acodec) || !vcodec || !FFmpegEncoderVerifyContainer(encoder)) {
198 return false;
199 }
200
201 encoder->currentAudioSample = 0;
202 encoder->currentAudioFrame = 0;
203 encoder->currentVideoFrame = 0;
204 encoder->nextAudioPts = 0;
205
206 AVOutputFormat* oformat = av_guess_format(encoder->containerFormat, 0, 0);
207#ifndef USE_LIBAV
208 avformat_alloc_output_context2(&encoder->context, oformat, 0, outfile);
209#else
210 encoder->context = avformat_alloc_context();
211 strncpy(encoder->context->filename, outfile, sizeof(encoder->context->filename) - 1);
212 encoder->context->filename[sizeof(encoder->context->filename) - 1] = '\0';
213 encoder->context->oformat = oformat;
214#endif
215
216 if (acodec) {
217#ifdef FFMPEG_USE_CODECPAR
218 encoder->audioStream = avformat_new_stream(encoder->context, NULL);
219 encoder->audio = avcodec_alloc_context3(acodec);
220#else
221 encoder->audioStream = avformat_new_stream(encoder->context, acodec);
222 encoder->audio = encoder->audioStream->codec;
223#endif
224 encoder->audio->bit_rate = encoder->audioBitrate;
225 encoder->audio->channels = 2;
226 encoder->audio->channel_layout = AV_CH_LAYOUT_STEREO;
227 encoder->audio->sample_rate = encoder->sampleRate;
228 encoder->audio->sample_fmt = encoder->sampleFormat;
229 AVDictionary* opts = 0;
230 av_dict_set(&opts, "strict", "-2", 0);
231 if (encoder->context->oformat->flags & AVFMT_GLOBALHEADER) {
232 encoder->audio->flags |= CODEC_FLAG_GLOBAL_HEADER;
233 }
234 avcodec_open2(encoder->audio, acodec, &opts);
235 av_dict_free(&opts);
236#if LIBAVCODEC_VERSION_MAJOR >= 55
237 encoder->audioFrame = av_frame_alloc();
238#else
239 encoder->audioFrame = avcodec_alloc_frame();
240#endif
241 if (!encoder->audio->frame_size) {
242 encoder->audio->frame_size = 1;
243 }
244 encoder->audioFrame->nb_samples = encoder->audio->frame_size;
245 encoder->audioFrame->format = encoder->audio->sample_fmt;
246 encoder->audioFrame->pts = 0;
247 encoder->resampleContext = avresample_alloc_context();
248 av_opt_set_int(encoder->resampleContext, "in_channel_layout", AV_CH_LAYOUT_STEREO, 0);
249 av_opt_set_int(encoder->resampleContext, "out_channel_layout", AV_CH_LAYOUT_STEREO, 0);
250 av_opt_set_int(encoder->resampleContext, "in_sample_rate", PREFERRED_SAMPLE_RATE, 0);
251 av_opt_set_int(encoder->resampleContext, "out_sample_rate", encoder->sampleRate, 0);
252 av_opt_set_int(encoder->resampleContext, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
253 av_opt_set_int(encoder->resampleContext, "out_sample_fmt", encoder->sampleFormat, 0);
254 avresample_open(encoder->resampleContext);
255 encoder->audioBufferSize = (encoder->audioFrame->nb_samples * PREFERRED_SAMPLE_RATE / encoder->sampleRate) * 4;
256 encoder->audioBuffer = av_malloc(encoder->audioBufferSize);
257 encoder->postaudioBufferSize = av_samples_get_buffer_size(0, encoder->audio->channels, encoder->audio->frame_size, encoder->audio->sample_fmt, 0);
258 encoder->postaudioBuffer = av_malloc(encoder->postaudioBufferSize);
259 avcodec_fill_audio_frame(encoder->audioFrame, encoder->audio->channels, encoder->audio->sample_fmt, (const uint8_t*) encoder->postaudioBuffer, encoder->postaudioBufferSize, 0);
260
261 if (encoder->audio->codec->id == AV_CODEC_ID_AAC &&
262 (strcasecmp(encoder->containerFormat, "mp4") ||
263 strcasecmp(encoder->containerFormat, "m4v") ||
264 strcasecmp(encoder->containerFormat, "mov"))) {
265 // MP4 container doesn't support the raw ADTS AAC format that the encoder spits out
266#ifdef FFMPEG_USE_NEW_BSF
267 av_bsf_alloc(av_bsf_get_by_name("aac_adtstoasc"), &encoder->absf);
268 avcodec_parameters_from_context(encoder->absf->par_in, encoder->audio);
269 av_bsf_init(encoder->absf);
270#else
271 encoder->absf = av_bitstream_filter_init("aac_adtstoasc");
272#endif
273 }
274#ifdef FFMPEG_USE_CODECPAR
275 avcodec_parameters_from_context(encoder->audioStream->codecpar, encoder->audio);
276#endif
277 }
278
279#ifdef FFMPEG_USE_CODECPAR
280 encoder->videoStream = avformat_new_stream(encoder->context, NULL);
281 encoder->video = avcodec_alloc_context3(vcodec);
282#else
283 encoder->videoStream = avformat_new_stream(encoder->context, vcodec);
284 encoder->video = encoder->videoStream->codec;
285#endif
286 encoder->video->bit_rate = encoder->videoBitrate;
287 encoder->video->width = encoder->width;
288 encoder->video->height = encoder->height;
289 encoder->video->time_base = (AVRational) { VIDEO_TOTAL_LENGTH, GBA_ARM7TDMI_FREQUENCY };
290 encoder->video->pix_fmt = encoder->pixFormat;
291 encoder->video->gop_size = 60;
292 encoder->video->max_b_frames = 3;
293 if (encoder->context->oformat->flags & AVFMT_GLOBALHEADER) {
294 encoder->video->flags |= CODEC_FLAG_GLOBAL_HEADER;
295 }
296 if (strcmp(vcodec->name, "libx264") == 0) {
297 // Try to adaptively figure out when you can use a slower encoder
298 if (encoder->width * encoder->height > 1000000) {
299 av_opt_set(encoder->video->priv_data, "preset", "superfast", 0);
300 } else if (encoder->width * encoder->height > 500000) {
301 av_opt_set(encoder->video->priv_data, "preset", "veryfast", 0);
302 } else {
303 av_opt_set(encoder->video->priv_data, "preset", "faster", 0);
304 }
305 av_opt_set(encoder->video->priv_data, "tune", "zerolatency", 0);
306 }
307
308 if (encoder->video->codec->id == AV_CODEC_ID_H264 &&
309 (strcasecmp(encoder->containerFormat, "mp4") ||
310 strcasecmp(encoder->containerFormat, "m4v") ||
311 strcasecmp(encoder->containerFormat, "mov"))) {
312 // QuickTime and a few other things require YUV420
313 encoder->video->pix_fmt = AV_PIX_FMT_YUV420P;
314 }
315 avcodec_open2(encoder->video, vcodec, 0);
316#if LIBAVCODEC_VERSION_MAJOR >= 55
317 encoder->videoFrame = av_frame_alloc();
318#else
319 encoder->videoFrame = avcodec_alloc_frame();
320#endif
321 encoder->videoFrame->format = encoder->video->pix_fmt;
322 encoder->videoFrame->width = encoder->video->width;
323 encoder->videoFrame->height = encoder->video->height;
324 encoder->videoFrame->pts = 0;
325 _ffmpegSetVideoDimensions(&encoder->d, encoder->iwidth, encoder->iheight);
326 av_image_alloc(encoder->videoFrame->data, encoder->videoFrame->linesize, encoder->video->width, encoder->video->height, encoder->video->pix_fmt, 32);
327#ifdef FFMPEG_USE_CODECPAR
328 avcodec_parameters_from_context(encoder->videoStream->codecpar, encoder->video);
329#endif
330
331 if (avio_open(&encoder->context->pb, outfile, AVIO_FLAG_WRITE) < 0) {
332 return false;
333 }
334 return avformat_write_header(encoder->context, 0) >= 0;
335}
336
337void FFmpegEncoderClose(struct FFmpegEncoder* encoder) {
338 if (!encoder->context) {
339 return;
340 }
341 av_write_trailer(encoder->context);
342 avio_close(encoder->context->pb);
343
344 if (encoder->audioCodec) {
345 av_free(encoder->postaudioBuffer);
346 if (encoder->audioBuffer) {
347 av_free(encoder->audioBuffer);
348 }
349#if LIBAVCODEC_VERSION_MAJOR >= 55
350 av_frame_free(&encoder->audioFrame);
351#else
352 avcodec_free_frame(&encoder->audioFrame);
353#endif
354 avcodec_close(encoder->audio);
355
356 if (encoder->resampleContext) {
357 avresample_close(encoder->resampleContext);
358 }
359
360 if (encoder->absf) {
361#ifdef FFMPEG_USE_NEW_BSF
362 av_bsf_free(&encoder->absf);
363#else
364 av_bitstream_filter_close(encoder->absf);
365 encoder->absf = 0;
366#endif
367 }
368 }
369
370#if LIBAVCODEC_VERSION_MAJOR >= 55
371 av_frame_free(&encoder->videoFrame);
372#else
373 avcodec_free_frame(&encoder->videoFrame);
374#endif
375 avcodec_close(encoder->video);
376
377 sws_freeContext(encoder->scaleContext);
378 encoder->scaleContext = NULL;
379
380 avformat_free_context(encoder->context);
381 encoder->context = 0;
382}
383
384bool FFmpegEncoderIsOpen(struct FFmpegEncoder* encoder) {
385 return !!encoder->context;
386}
387
388void _ffmpegPostAudioFrame(struct mAVStream* stream, int16_t left, int16_t right) {
389 struct FFmpegEncoder* encoder = (struct FFmpegEncoder*) stream;
390 if (!encoder->context || !encoder->audioCodec) {
391 return;
392 }
393
394 if (encoder->absf && !left) {
395 // XXX: AVBSF doesn't like silence. Figure out why.
396 left = 1;
397 }
398
399 encoder->audioBuffer[encoder->currentAudioSample * 2] = left;
400 encoder->audioBuffer[encoder->currentAudioSample * 2 + 1] = right;
401
402 ++encoder->currentAudioSample;
403
404 if (encoder->currentAudioSample * 4 < encoder->audioBufferSize) {
405 return;
406 }
407
408 int channelSize = 2 * av_get_bytes_per_sample(encoder->audio->sample_fmt);
409 avresample_convert(encoder->resampleContext, 0, 0, 0,
410 (uint8_t**) &encoder->audioBuffer, 0, encoder->audioBufferSize / 4);
411
412 encoder->currentAudioSample = 0;
413 if (avresample_available(encoder->resampleContext) < encoder->audioFrame->nb_samples) {
414 return;
415 }
416#if LIBAVCODEC_VERSION_MAJOR >= 55
417 av_frame_make_writable(encoder->audioFrame);
418#endif
419 int samples = avresample_read(encoder->resampleContext, encoder->audioFrame->data, encoder->postaudioBufferSize / channelSize);
420
421 encoder->audioFrame->pts = av_rescale_q(encoder->currentAudioFrame, encoder->audio->time_base, encoder->audioStream->time_base);
422 encoder->currentAudioFrame += samples;
423
424 AVPacket packet;
425 av_init_packet(&packet);
426 packet.data = 0;
427 packet.size = 0;
428 packet.pts = encoder->audioFrame->pts;
429
430 int gotData;
431#ifdef FFMPEG_USE_PACKETS
432 avcodec_send_frame(encoder->audio, encoder->audioFrame);
433 gotData = avcodec_receive_packet(encoder->audio, &packet);
434 gotData = (gotData == 0) && packet.size;
435#else
436 avcodec_encode_audio2(encoder->audio, &packet, encoder->audioFrame, &gotData);
437#endif
438 if (gotData) {
439 if (encoder->absf) {
440 AVPacket tempPacket;
441
442#ifdef FFMPEG_USE_NEW_BSF
443 int success = av_bsf_send_packet(encoder->absf, &packet);
444 if (success >= 0) {
445 success = av_bsf_receive_packet(encoder->absf, &tempPacket);
446 }
447#else
448 int success = av_bitstream_filter_filter(encoder->absf, encoder->audio, 0,
449 &tempPacket.data, &tempPacket.size,
450 packet.data, packet.size, 0);
451#endif
452
453 if (success >= 0) {
454#if LIBAVUTIL_VERSION_MAJOR >= 53
455 tempPacket.buf = av_buffer_create(tempPacket.data, tempPacket.size, av_buffer_default_free, 0, 0);
456#endif
457
458#ifdef FFMPEG_USE_PACKET_UNREF
459 av_packet_move_ref(&packet, &tempPacket);
460#else
461 av_free_packet(&packet);
462 packet = tempPacket;
463#endif
464
465 packet.stream_index = encoder->audioStream->index;
466 av_interleaved_write_frame(encoder->context, &packet);
467 }
468 } else {
469 packet.stream_index = encoder->audioStream->index;
470 av_interleaved_write_frame(encoder->context, &packet);
471 }
472 }
473#ifdef FFMPEG_USE_PACKET_UNREF
474 av_packet_unref(&packet);
475#else
476 av_free_packet(&packet);
477#endif
478}
479
480void _ffmpegPostVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
481 struct FFmpegEncoder* encoder = (struct FFmpegEncoder*) stream;
482 if (!encoder->context) {
483 return;
484 }
485 stride *= BYTES_PER_PIXEL;
486
487 AVPacket packet;
488
489 av_init_packet(&packet);
490 packet.data = 0;
491 packet.size = 0;
492#if LIBAVCODEC_VERSION_MAJOR >= 55
493 av_frame_make_writable(encoder->videoFrame);
494#endif
495 encoder->videoFrame->pts = av_rescale_q(encoder->currentVideoFrame, encoder->video->time_base, encoder->videoStream->time_base);
496 packet.pts = encoder->videoFrame->pts;
497 ++encoder->currentVideoFrame;
498
499 sws_scale(encoder->scaleContext, (const uint8_t* const*) &pixels, (const int*) &stride, 0, encoder->iheight, encoder->videoFrame->data, encoder->videoFrame->linesize);
500
501 int gotData;
502#ifdef FFMPEG_USE_PACKETS
503 avcodec_send_frame(encoder->video, encoder->videoFrame);
504 gotData = avcodec_receive_packet(encoder->video, &packet) == 0;
505#else
506 avcodec_encode_video2(encoder->video, &packet, encoder->videoFrame, &gotData);
507#endif
508 if (gotData) {
509#ifndef FFMPEG_USE_PACKET_UNREF
510 if (encoder->video->coded_frame->key_frame) {
511 packet.flags |= AV_PKT_FLAG_KEY;
512 }
513#endif
514 packet.stream_index = encoder->videoStream->index;
515 av_interleaved_write_frame(encoder->context, &packet);
516 }
517#ifdef FFMPEG_USE_PACKET_UNREF
518 av_packet_unref(&packet);
519#else
520 av_free_packet(&packet);
521#endif
522}
523
524static void _ffmpegSetVideoDimensions(struct mAVStream* stream, unsigned width, unsigned height) {
525 struct FFmpegEncoder* encoder = (struct FFmpegEncoder*) stream;
526 encoder->iwidth = width;
527 encoder->iheight = height;
528 if (encoder->scaleContext) {
529 sws_freeContext(encoder->scaleContext);
530 }
531 encoder->scaleContext = sws_getContext(encoder->iwidth, encoder->iheight,
532#ifdef COLOR_16_BIT
533#ifdef COLOR_5_6_5
534 AV_PIX_FMT_RGB565,
535#else
536 AV_PIX_FMT_BGR555,
537#endif
538#else
539#ifndef USE_LIBAV
540 AV_PIX_FMT_0BGR32,
541#else
542 AV_PIX_FMT_BGR32,
543#endif
544#endif
545 encoder->videoFrame->width, encoder->videoFrame->height, encoder->video->pix_fmt,
546 SWS_POINT, 0, 0, 0);
547}