Android：使用MediaCodec编码音频和视频
我正在尝试使用MediaCodec和MediaMuxer对摄像机中的视频和麦克风中的音频进行编码。我使用OpenGL在录制时将文本覆盖在图像上 我以这些课程为例：Android：使用MediaCodec编码音频和视频,android,audio,video,android-mediacodec,mediamuxer,Android,Audio,Video,Android Mediacodec,Mediamuxer,我正在尝试使用MediaCodec和MediaMuxer对摄像机中的视频和麦克风中的音频进行编码。我使用OpenGL在录制时将文本覆盖在图像上 我以这些课程为例： 我编写了一个执行编码的主类。它产生2个用于录制音频和视频的线程。它不工作（生成的文件无效），但如果我对其中一个线程（音频或视频）进行注释，它工作正常。另外，我需要将轨道计数设置为1。这是主类的代码： import android.graphics.SurfaceTexture; import android.media.Aud
import android.graphics.SurfaceTexture;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.media.MediaRecorder;
import com.google.common.base.Throwables;
import java.io.IOException;
import java.nio.ByteBuffer;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Class for recording a reply including a text message.
*/
public class ReplyRecorder {
// Encoding state
private boolean encoding;
long startWhen;
// Muxer
private static final int TRACK_COUNT = 2;
private Muxer mMuxer;
// Video
private static final String VIDEO_MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
private static final int FRAME_RATE = 15; // 30fps
private static final int IFRAME_INTERVAL = 10; // 5 seconds between I-frames
private static final int BIT_RATE = 2000000;
private Encoder mVideoEncoder;
private CodecInputSurface mInputSurface;
private SurfaceTextureManager mStManager;
// Audio
private static final String AUDIO_MIME_TYPE = "audio/mp4a-latm";
private static final int SAMPLE_RATE = 44100;
private static final int SAMPLES_PER_FRAME = 1024;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
private static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private Encoder mAudioEncoder;
private AudioRecord audioRecord;
public void start(final CameraManager cameraManager, final String messageText, final String filePath) {
checkNotNull(cameraManager);
checkNotNull(messageText);
checkNotNull(filePath);
try {
// Create a MediaMuxer. We can't add the video track and start() the muxer here,
// because our MediaFormat doesn't have the Magic Goodies. These can only be
// obtained from the encoder after it has started processing data.
mMuxer = new Muxer(new MediaMuxer(filePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4), TRACK_COUNT);
startWhen = System.nanoTime();
encoding = true;
new Thread(new Runnable() {
@Override
public void run() {
initVideoComponents(cameraManager, messageText);
encodeVideo(cameraManager);
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
initAudioComponents();
encodeAudio();
}
}).start();
} catch (IOException e) {
release();
throw Throwables.propagate(e);
}
}
private void initVideoComponents(CameraManager cameraManager,
String messageText) {
try {
MediaFormat format = MediaFormat.createVideoFormat(VIDEO_MIME_TYPE, cameraManager.getEncWidth(), cameraManager.getEncHeight());
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
// Create a MediaCodec encoder, and configure it with our format. Get a Surface
// we can use for input and wrap it with a class that handles the EGL work.
//
// If you want to have two EGL contexts -- one for display, one for recording --
// you will likely want to defer instantiation of CodecInputSurface until after the
// "display" EGL context is created, then modify the eglCreateContext call to
// take eglGetCurrentContext() as the share_context argument.
mVideoEncoder = new Encoder(VIDEO_MIME_TYPE, format, mMuxer);
mInputSurface = new CodecInputSurface(mVideoEncoder.getEncoder().createInputSurface());
mVideoEncoder.getEncoder().start();
mInputSurface.makeCurrent();
mStManager = new SurfaceTextureManager(messageText, cameraManager.getEncWidth(), cameraManager.getEncHeight());
} catch (RuntimeException e) {
releaseVideo();
throw e;
}
}
private void encodeVideo(CameraManager cameraManager) {
try {
SurfaceTexture st = mStManager.getSurfaceTexture();
cameraManager.record(st);
while (encoding) {
// Feed any pending encoder output into the muxer.
mVideoEncoder.drain(false);
// Acquire a new frame of input, and render it to the Surface. If we had a
// GLSurfaceView we could switch EGL contexts and call drawImage() a second
// time to render it on screen. The texture can be shared between contexts by
// passing the GLSurfaceView's EGLContext as eglCreateContext()'s share_context
// argument.
mStManager.awaitNewImage();
mStManager.drawImage();
// Set the presentation time stamp from the SurfaceTexture's time stamp. This
// will be used by MediaMuxer to set the PTS in the video.
mInputSurface.setPresentationTime(st.getTimestamp() - startWhen);
// Submit it to the encoder. The eglSwapBuffers call will block if the input
// is full, which would be bad if it stayed full until we dequeued an output
// buffer (which we can't do, since we're stuck here). So long as we fully drain
// the encoder before supplying additional input, the system guarantees that we
// can supply another frame without blocking.
mInputSurface.swapBuffers();
}
// send end-of-stream to encoder, and drain remaining output
mVideoEncoder.drain(true);
} finally {
releaseVideo();
}
}
private void initAudioComponents() {
try {
int min_buffer_size = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT);
int buffer_size = SAMPLES_PER_FRAME * 10;
if (buffer_size < min_buffer_size)
buffer_size = ((min_buffer_size / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
audioRecord = new AudioRecord(
MediaRecorder.AudioSource.MIC, // source
SAMPLE_RATE, // sample rate, hz
CHANNEL_CONFIG, // channels
AUDIO_FORMAT, // audio format
buffer_size); // buffer size (bytes)
/////////////////
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, AUDIO_MIME_TYPE);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 44100);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
format.setInteger(MediaFormat.KEY_BIT_RATE, 128000);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16384);
mAudioEncoder = new Encoder(AUDIO_MIME_TYPE, format, mMuxer);
mAudioEncoder.getEncoder().start();
} catch (RuntimeException e) {
releaseAudio();
throw e;
}
}
private void encodeAudio() {
try {
audioRecord.startRecording();
while (encoding) {
mAudioEncoder.drain(false);
sendAudioToEncoder(false);
}
//TODO: Sending "false" because calling signalEndOfInputStream fails on this encoder
mAudioEncoder.drain(false);
} finally {
releaseAudio();
}
}
public void sendAudioToEncoder(boolean endOfStream) {
// send current frame data to encoder
ByteBuffer[] inputBuffers = mAudioEncoder.getEncoder().getInputBuffers();
int inputBufferIndex = mAudioEncoder.getEncoder().dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
long presentationTimeNs = System.nanoTime();
int inputLength = audioRecord.read(inputBuffer, SAMPLES_PER_FRAME);
presentationTimeNs -= (inputLength / SAMPLE_RATE) / 1000000000;
long presentationTimeUs = (presentationTimeNs - startWhen) / 1000;
if (endOfStream) {
mAudioEncoder.getEncoder().queueInputBuffer(inputBufferIndex, 0, inputLength, presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mAudioEncoder.getEncoder().queueInputBuffer(inputBufferIndex, 0, inputLength, presentationTimeUs, 0);
}
}
}
public void stop() {
encoding = false;
}
/**
* Releases encoder resources.
*/
public void release() {
releaseVideo();
releaseAudio();
}
private void releaseVideo() {
if (mVideoEncoder != null) {
mVideoEncoder.release();
mVideoEncoder = null;
}
if (mInputSurface != null) {
mInputSurface.release();
mInputSurface = null;
}
if (mStManager != null) {
mStManager.release();
mStManager = null;
}
releaseMuxer();
}
private void releaseAudio() {
if (audioRecord != null) {
audioRecord.stop();
audioRecord = null;
}
if (mAudioEncoder != null) {
mAudioEncoder.release();
mAudioEncoder = null;
}
releaseMuxer();
}
private void releaseMuxer() {
if (mMuxer != null && mVideoEncoder == null && mAudioEncoder == null) {
mMuxer.release();
mMuxer = null;
}
}
public boolean isRecording() {
return mMuxer != null;
}
}
import android.media.MediaCodec;
import android.media.MediaFormat;
import com.google.common.base.Throwables;
import java.io.IOException;
import java.nio.ByteBuffer;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
/**
* Class responsible for encoding.
*/
public class Encoder {
private final MediaCodec encoder;
private final Muxer muxer;
private final MediaCodec.BufferInfo bufferInfo;
private int trackIndex;
public Encoder(String mimeType, MediaFormat format, Muxer muxer) {
checkNotNull(mimeType);
checkNotNull(format);
checkNotNull(muxer);
try {
encoder = MediaCodec.createEncoderByType(mimeType);
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
this.muxer = muxer;
bufferInfo = new MediaCodec.BufferInfo();
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
public MediaCodec getEncoder() {
return encoder;
}
/**
* Extracts all pending data from the encoder and forwards it to the muxer.
* <p/>
* If endOfStream is not set, this returns when there is no more data to drain. If it
* is set, we send EOS to the encoder, and then iterate until we see EOS on the output.
* Calling this with endOfStream set should be done once, right before stopping the muxer.
* <p/>
* We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream).
*/
public void drain(boolean endOfStream) {
final int TIMEOUT_USEC = 10000;
if (endOfStream) {
encoder.signalEndOfInputStream();
}
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (!endOfStream) {
break; // out of while
}
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// now that we have the Magic Goodies, start the muxer
trackIndex = muxer.addTrack(encoder.getOutputFormat());
} else if (encoderStatus < 0) {
// let's ignore it
} else {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
checkState(encodedData != null, "encoderOutputBuffer %s was null", encoderStatus);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
break; // out of while
}
}
}
}
public void release() {
if (encoder != null) {
try {
encoder.stop();
} catch (Exception e) {
}
encoder.release();
}
}
}
导入android.media.MediaCodec；
导入android.media.MediaFormat；
导入com.google.common.base.Throwables；
导入java.io.IOException；
导入java.nio.ByteBuffer；
导入静态com.google.common.base.premissions.checkNotNull；
导入静态com.google.common.base.premissions.checkState；
/**
*类负责编码。
*/
公共类编码器{
专用最终媒体编解码器编码器；
专用最终多路复用器多路复用器；
私有最终MediaCodec.BufferInfo BufferInfo；
私有int跟踪索引；
公共编码器（字符串mimeType、MediaFormat格式、Muxer-Muxer）{
checkNotNull（mimeType）；
checkNotNull（格式）；
checkNotNull（muxer）；
试一试{
编码器=MediaCodec.createEncoderByType（mimeType）；
编码器.configure（格式、空、空、MediaCodec.configure\u标志\u编码）；
this.muxer=muxer；
bufferInfo=新的MediaCodec.bufferInfo（）；
}捕获（IOE异常）{
抛掷物。传播（e）；
}
}
公共媒体编解码器getEncoder（）{
返回编码器；
}
/**
*从编码器中提取所有挂起的数据并将其转发给muxer。
*
*如果未设置endOfStream，则在没有更多数据要排出时返回
*设置后，我们将EOS发送到编码器，然后迭代，直到在输出上看到EOS。
*使用endOfStream set调用此函数应该在停止muxer之前完成一次。
*
*我们只是使用muxer来获取一个.mp4文件（而不是原始的H.264流）。
*/
公共空排水管（布尔endOfStream）{
最终整数超时时间=10000；
if（内流）{
encoder.signalEndOfInputStream（）；
}
ByteBuffer[]encoderOutputBuffers=encoder.getOutputBuffers（）；
while（true）{
int encoderStatus=encoder.dequeueOutputBuffer（bufferInfo，TIMEOUT\u USEC）；
如果（encoderStatus==MediaCodec.INFO\u请稍后再试）{
//还没有可用的输出
如果（！endOfStream）{
中断；//暂停
}
}else if（encoderStatus==MediaCodec.INFO\u输出\u缓冲区\u更改）{
//编码器不需要
encoderOutputBuffers=encoder.getOutputBuffers（）；
}else if（encoderStatus==MediaCodec.INFO\u输出\u格式\u更改）{
//现在我们有了神奇的糖果，启动muxer
trackIndex=muxer.addTrack（encoder.getOutputFormat（））；
}否则如果（编码器状态<0）{
//让我们忽略它
}否则{
ByteBuffer encodedData=encoderOutputBuffers[encoderStatus]；
检查状态（encodedData！=null，“encoderOutputBuffer%s为null”，encoderStatus）；
if（（bufferInfo.flags和MediaCodec.BUFFER\u FLAG\u CODEC\u CONFIG）！=0）{
//当我们得到时，编解码器配置数据被取出并馈送到muxer
//信息输出格式已更改状态。忽略它。
bufferInfo.size=0；
}
如果（bufferInfo.size！=0）{
//调整ByteBuffer值以匹配BufferInfo（不需要？）
编码数据位置（缓冲信息偏移量）；
encodedData.limit（bufferInfo.offset+bufferInfo.size）；
写入示例数据（trackIndex、encodedData、bufferInfo）；
}
编码器.releaseOutputBuffer（编码器状态，false）；
if（（bufferInfo.flags和MediaCodec.BUFFER\u FLAG\u END\u流）！=0）{
中断；//暂停
}
}
}
}
公开无效释放（）{
如果（编码器！=null）{
试一试{
编码器。停止（）；
}捕获（例外e）{
}
编码器。释放（）；
}
}
}
知道为什么并发运行会失败吗？好的，我终于实现了原始海报的最终目标。问题正如我所预料的那样，它必须处理为音频曲目生成的时间戳，而这些时间戳与视频曲目提供给我们的不完全匹配 我的解决方案是将
视频编码器缓冲信息音频编码器AudioEncoderBufferInfomvideocoder.onFrameAvailableAudioEncoderimport android.media.MediaCodec;
import android.media.MediaFormat;
import com.google.common.base.Throwables;
import java.io.IOException;
import java.nio.ByteBuffer;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
/**
* Class responsible for encoding.
*/
public class Encoder {
private final MediaCodec encoder;
private final Muxer muxer;
private final MediaCodec.BufferInfo bufferInfo;
private int trackIndex;
public Encoder(String mimeType, MediaFormat format, Muxer muxer) {
checkNotNull(mimeType);
checkNotNull(format);
checkNotNull(muxer);
try {
encoder = MediaCodec.createEncoderByType(mimeType);
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
this.muxer = muxer;
bufferInfo = new MediaCodec.BufferInfo();
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
public MediaCodec getEncoder() {
return encoder;
}
/**
* Extracts all pending data from the encoder and forwards it to the muxer.
* <p/>
* If endOfStream is not set, this returns when there is no more data to drain. If it
* is set, we send EOS to the encoder, and then iterate until we see EOS on the output.
* Calling this with endOfStream set should be done once, right before stopping the muxer.
* <p/>
* We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream).
*/
public void drain(boolean endOfStream) {
final int TIMEOUT_USEC = 10000;
if (endOfStream) {
encoder.signalEndOfInputStream();
}
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (!endOfStream) {
break; // out of while
}
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// now that we have the Magic Goodies, start the muxer
trackIndex = muxer.addTrack(encoder.getOutputFormat());
} else if (encoderStatus < 0) {
// let's ignore it
} else {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
checkState(encodedData != null, "encoderOutputBuffer %s was null", encoderStatus);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
break; // out of while
}
}
}
}
public void release() {
if (encoder != null) {
try {
encoder.stop();
} catch (Exception e) {
}
encoder.release();
}
}
}