Android中音视频合成方案分析

字数统计: 1,074字   |   阅读时长: 6分
Android

Android下音视频合成，在当前调研方案中主要有三大类方法：MediaMux硬解码，mp4parser，FFmepg。三种方法均可实现，但是也有不同的局限和问题，先将实现和问题记录于此，便于之后的总结学习。

方法一（Fail）

利用MediaMux实现音视频的合成。
效果：可以实现音视频的合并，利用Android原生的VideoView和SurfaceView播放正常，大部分的播放器也播放正常，但是，但是，在上传Youtube就会出现问题：音频不连续，分析主要是上传Youtube时会被再次的压缩，可能在压缩的过程中出现音频的帧率出现问题。
分析：在MediaCodec.BufferInfo的处理中，时间戳presentationTimeUs出现问题，导致Youtube的压缩造成音频的紊乱。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
public static void muxVideoAndAudio(String videoPath, String audioPath, String muxPath) {
       try {
           MediaExtractor videoExtractor = new MediaExtractor();
           videoExtractor.setDataSource(videoPath);
           MediaFormat videoFormat = null;
           int videoTrackIndex = -1;
           int videoTrackCount = videoExtractor.getTrackCount();
           for (int i = 0; i < videoTrackCount; i++) {
               videoFormat = videoExtractor.getTrackFormat(i);
               String mimeType = videoFormat.getString(MediaFormat.KEY_MIME);
               if (mimeType.startsWith("video/")) {
                   videoTrackIndex = i;
                   break;
               }
           }
           MediaExtractor audioExtractor = new MediaExtractor();
           audioExtractor.setDataSource(audioPath);
           MediaFormat audioFormat = null;
           int audioTrackIndex = -1;
           int audioTrackCount = audioExtractor.getTrackCount();
           for (int i = 0; i < audioTrackCount; i++) {
               audioFormat = audioExtractor.getTrackFormat(i);
               String mimeType = audioFormat.getString(MediaFormat.KEY_MIME);
               if (mimeType.startsWith("audio/")) {
                   audioTrackIndex = i;
                   break;
               }
           }
           videoExtractor.selectTrack(videoTrackIndex);
           audioExtractor.selectTrack(audioTrackIndex);
           MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
           MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
           MediaMuxer mediaMuxer = new MediaMuxer(muxPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
           int writeVideoTrackIndex = mediaMuxer.addTrack(videoFormat);
           int writeAudioTrackIndex = mediaMuxer.addTrack(audioFormat);
           mediaMuxer.start();
           ByteBuffer byteBuffer = ByteBuffer.allocate(500 * 1024);
           long sampleTime = 0;
           {
               videoExtractor.readSampleData(byteBuffer, 0);
               if (videoExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {
                   videoExtractor.advance();
               }
               videoExtractor.readSampleData(byteBuffer, 0);
               long secondTime = videoExtractor.getSampleTime();
               videoExtractor.advance();
               long thirdTime = videoExtractor.getSampleTime();
               sampleTime = Math.abs(thirdTime - secondTime);
           }
           videoExtractor.unselectTrack(videoTrackIndex);
           videoExtractor.selectTrack(videoTrackIndex);
           while (true) {
               int readVideoSampleSize = videoExtractor.readSampleData(byteBuffer, 0);
               if (readVideoSampleSize < 0) {
                   break;
               }
               videoBufferInfo.size = readVideoSampleSize;
               videoBufferInfo.presentationTimeUs += sampleTime;
               videoBufferInfo.offset = 0;
               //noinspection WrongConstant
               videoBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;//videoExtractor.getSampleFlags()
               mediaMuxer.writeSampleData(writeVideoTrackIndex, byteBuffer, videoBufferInfo);
               videoExtractor.advance();
           }
           while (true) {
               int readAudioSampleSize = audioExtractor.readSampleData(byteBuffer, 0);
               if (readAudioSampleSize < 0) {
                   break;
               }
               audioBufferInfo.size = readAudioSampleSize;
               audioBufferInfo.presentationTimeUs += sampleTime;
               audioBufferInfo.offset = 0;
               //noinspection WrongConstant
               audioBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;// videoExtractor.getSampleFlags()
               mediaMuxer.writeSampleData(writeAudioTrackIndex, byteBuffer, audioBufferInfo);
               audioExtractor.advance();
           }
           mediaMuxer.stop();
           mediaMuxer.release();
           videoExtractor.release();
           audioExtractor.release();
       } catch (IOException e) {
           e.printStackTrace();
       }
   }

方法二（Success）

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
public static void muxVideoAudio(String videoFilePath, String audioFilePath, String outputFile) {
        try {
            MediaExtractor videoExtractor = new MediaExtractor();
            videoExtractor.setDataSource(videoFilePath);
            MediaExtractor audioExtractor = new MediaExtractor();
            audioExtractor.setDataSource(audioFilePath);
            MediaMuxer muxer = new MediaMuxer(outputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
            videoExtractor.selectTrack(0);
            MediaFormat videoFormat = videoExtractor.getTrackFormat(0);
            int videoTrack = muxer.addTrack(videoFormat);
            audioExtractor.selectTrack(0);
            MediaFormat audioFormat = audioExtractor.getTrackFormat(0);
            int audioTrack = muxer.addTrack(audioFormat);
            LogUtil.d(TAG, "Video Format " + videoFormat.toString());
            LogUtil.d(TAG, "Audio Format " + audioFormat.toString());
            boolean sawEOS = false;
            int frameCount = 0;
            int offset = 100;
            int sampleSize = 256 * 1024;
            ByteBuffer videoBuf = ByteBuffer.allocate(sampleSize);
            ByteBuffer audioBuf = ByteBuffer.allocate(sampleSize);
            MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
            MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
            videoExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
            audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
            muxer.start();
            while (!sawEOS) {
                videoBufferInfo.offset = offset;
                videoBufferInfo.size = videoExtractor.readSampleData(videoBuf, offset);
                if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) {
                    sawEOS = true;
                    videoBufferInfo.size = 0;
                } else {
                    videoBufferInfo.presentationTimeUs = videoExtractor.getSampleTime();
                    //noinspection WrongConstant
                    videoBufferInfo.flags = videoExtractor.getSampleFlags();
                    muxer.writeSampleData(videoTrack, videoBuf, videoBufferInfo);
                    videoExtractor.advance();
                    frameCount++;
                }
            }
            
            boolean sawEOS2 = false;
            int frameCount2 = 0;
            while (!sawEOS2) {
                frameCount2++;
                audioBufferInfo.offset = offset;
                audioBufferInfo.size = audioExtractor.readSampleData(audioBuf, offset);
                if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) {
                    sawEOS2 = true;
                    audioBufferInfo.size = 0;
                } else {
                    audioBufferInfo.presentationTimeUs = audioExtractor.getSampleTime();
                    //noinspection WrongConstant
                    audioBufferInfo.flags = audioExtractor.getSampleFlags();
                    muxer.writeSampleData(audioTrack, audioBuf, audioBufferInfo);
                    audioExtractor.advance();
                }
            }
            muxer.stop();
            muxer.release();
            LogUtil.d(TAG,"Output: "+outputFile);
        } catch (IOException e) {
            LogUtil.d(TAG, "Mixer Error 1 " + e.getMessage());
        } catch (Exception e) {
            LogUtil.d(TAG, "Mixer Error 2 " + e.getMessage());
        }
    }

方法三

利用mp4parser实现

compile “com.googlecode.mp4parser:isoparser:1.1.21”

问题：上传Youtube压缩后，视频数据丢失严重，大部分就只剩下一秒钟的时长，相当于把视频变成图片了，囧

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 public boolean mux(String videoFile, String audioFile, final String outputFile) {
        if (isStopMux) {
            return false;
        }
        Movie video;
        try {
            video = MovieCreator.build(videoFile);
        } catch (RuntimeException e) {
            e.printStackTrace();
            return false;
        } catch (IOException e) {
            e.printStackTrace();
            return false;
        }
        Movie audio;
        try {
            audio = MovieCreator.build(audioFile);
        } catch (IOException e) {
            e.printStackTrace();
            return false;
        } catch (NullPointerException e) {
            e.printStackTrace();
            return false;
        }
        Track audioTrack = audio.getTracks().get(0);
        video.addTrack(audioTrack);
        Container out = new DefaultMp4Builder().build(video);
        FileOutputStream fos;
        try {
            fos = new FileOutputStream(outputFile);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
            return false;
        }
        BufferedWritableFileByteChannel byteBufferByteChannel = new
                BufferedWritableFileByteChannel(fos);
        try {
            out.writeContainer(byteBufferByteChannel);
            byteBufferByteChannel.close();
            fos.close();
            if (isStopMux) {
                return false;
            }
            runOnUiThread(new Runnable() {
                @Override
                public void run() {
                    mCustomeProgressDialog.setProgress(100);
                    goShareActivity(outputFile);
//                    FileUtils.insertMediaDB(AddAudiosActivity.this,outputFile);//
                }
            });
        } catch (IOException e) {
            e.printStackTrace();
            if (mCustomeProgressDialog.isShowing()) {
                mCustomeProgressDialog.dismiss();
            }
            ToastUtil.showShort(getString(R.string.process_failed));
            return false;
        }
        return true;
    }
    private static class BufferedWritableFileByteChannel implements WritableByteChannel {
        private static final int BUFFER_CAPACITY = 2000000;
        private boolean isOpen = true;
        private final OutputStream outputStream;
        private final ByteBuffer byteBuffer;
        private final byte[] rawBuffer = new byte[BUFFER_CAPACITY];
        private BufferedWritableFileByteChannel(OutputStream outputStream) {
            this.outputStream = outputStream;
            this.byteBuffer = ByteBuffer.wrap(rawBuffer);
        }
        @Override
        public int write(ByteBuffer inputBuffer) throws IOException {
            int inputBytes = inputBuffer.remaining();
            if (inputBytes > byteBuffer.remaining()) {
                dumpToFile();
                byteBuffer.clear();
                if (inputBytes > byteBuffer.remaining()) {
                    throw new BufferOverflowException();
                }
            }
            byteBuffer.put(inputBuffer);
            return inputBytes;
        }
        @Override
        public boolean isOpen() {
            return isOpen;
        }
        @Override
        public void close() throws IOException {
            dumpToFile();
            isOpen = false;
        }
        private void dumpToFile() {
            try {
                outputStream.write(rawBuffer, 0, byteBuffer.position());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }

方法四

利用FFmpeg大法

Merge Video /Audio and retain both audios

可以实现，兼容性强，但由于是软解码，合并速度很慢，忍受不了，而相应的FFmpeg优化还不太了解，囧…….

以上，谢谢阅读。

本文标题:Android中音视频合成方案分析

文章作者:AnswerZhao

发布时间:2017年03月11日 - 20时30分

最后更新:2017年12月02日 - 15时41分

原始链接:http://answerzhao.github.io/2017/03/11/Android中音视频合成方案分析/

许可协议: "署名-非商用-相同方式共享 3.0" 转载请保留原文链接及作者。

文章目录
  1. 1. 方法一（Fail）
  2. 2. 方法二（Success）
  3. 3. 方法三
  4. 4. 方法四
,