android上的openGL ES 2.0，YUV到RGB，并使用ffMpeg进行渲染

我的渲染器在视频显示后1~2帧死亡

致命错误11:blabla…（恰好发生在GLD元素（Y部分））

我认为问题在于“glPixelStorei”或“GL_RGB”，“GL_亮度”，但。。我不明白

我的渲染方式：

解码从网络获取的数据，（SDK获取->NDK解码），排队

退出另一个线程的队列（当然是同步的）准备好安装OpenGL ES 2.0。（SDK）

当调用onDrawFrame、onSurfaceCreated和onSurfaceChanged方法时，它会收缩到NDK。（NDK中的我的渲染器源将附加到下面。）

渲染

正如您所知，片段着色器用于转换。 我的数据是YUV 420p（pix_fmt_YUV420p）（每像素12位）

这是我的全部资料来源

我以前对OpenGL ES一无所知，这是第一次。

请让我知道我在做什么来提高绩效。

我在“glTexImage2D”、“glTexSubImage2D”、“glRenderbufferStorage”中使用的参数是什么？？？？ 亮度？格尔巴？德国劳埃德大学？（现在正在使用GL_亮度）

---

您不能调用glTexSubImage2D（）并期望它能跟上视频帧速率。这是一个非常常见的错误。glTexSubImage2D（）和glTexImage2D（）对于视频来说太慢

您应该改用EGL图像扩展。这里有一个Android的例子：

---

您不能调用glTexSubImage2D（）并期望它跟上视频帧速率。这是一个非常常见的错误。glTexSubImage2D（）和glTexImage2D（）对于视频来说太慢

您应该改用EGL图像扩展。这里有一个Android的例子：

---

非常感谢您！！我刚刚解决了我的问题，并担心帧速率！！！！！！谢谢无需打开GraphicBuffer。只需使用“new”构造它并锁定它，如本文清单5所示。GraphicBuffer的唯一用途是为您提供指向EGL图像中纹理的虚拟内存指针，以便您可以将视频帧数据复制到纹理中。最好让您的视频DSP/编解码器编写，但您可以使用memcpy（）。首先，感谢您的回复。我是说……”“开放”是。。。当我使用GraphicBuffer（值声明、分配等）时，android不知道GraphicBuffer。在执行ndk构建时，她说“不命名类型”。我试图修复Android.mk，#include，通过谷歌搜索“使用私有api”，但没有解决。特别是#include没有这样的文件或目录。我添加了我的Android.mk-lui。我的目标=android-10（我尝试全部改变10~17个）哦，我明白了。。我想我的本地库无法连接libui.so。因为该库位于frameworks/native（框架部分）中，所以。。。我应该找到如何使用框架库。是吗？NDK中有一些例子。看：非常感谢！！我刚刚解决了我的问题，并担心帧速率！！！！！！谢谢无需打开GraphicBuffer。只需使用“new”构造它并锁定它，如本文清单5所示。GraphicBuffer的唯一用途是为您提供指向EGL图像中纹理的虚拟内存指针，以便您可以将视频帧数据复制到纹理中。最好让您的视频DSP/编解码器编写，但您可以使用memcpy（）。首先，感谢您的回复。我是说……”“开放”是。。。当我使用GraphicBuffer（值声明、分配等）时，android不知道GraphicBuffer。在执行ndk构建时，她说“不命名类型”。我试图修复Android.mk，#include，通过谷歌搜索“使用私有api”，但没有解决。特别是#include没有这样的文件或目录。我添加了我的Android.mk-lui。我的目标=android-10（我尝试全部改变10~17个）哦，我明白了。。我想我的本地库无法连接libui.so。因为该库位于frameworks/native（框架部分）中，所以。。。我应该找到如何使用框架库。是吗？NDK中有一些例子。见：

void Renderer::set_draw_frame(JNIEnv* jenv, jbyteArray yData, jbyteArray uData, jbyteArray vData)
{
    for (int i = 0; i < 3; i++) {
        if (yuv_data_[i] != NULL) {
            free(yuv_data_[i]);
        }
    }

  int YSIZE = -1;
  int USIZE = -1;
  int VSIZE = -1;

  if (yData != NULL) {
        YSIZE = (int)jenv->GetArrayLength(yData);
    LOG_DEBUG("YSIZE : %d", YSIZE);
        yuv_data_[0] = (unsigned char*)malloc(sizeof(unsigned char) * YSIZE);
    memset(yuv_data_[0], 0, YSIZE);
        jenv->GetByteArrayRegion(yData, 0, YSIZE, (jbyte*)yuv_data_[0]);
    yuv_data_[0] = reinterpret_cast<unsigned char*>(yuv_data_[0]);
    } else {
        YSIZE = (int)jenv->GetArrayLength(yData);
        yuv_data_[0] = (unsigned char*)malloc(sizeof(unsigned char) * YSIZE);
    memset(yuv_data_[0], 1, YSIZE);
  }

    if (uData != NULL) {
        USIZE = (int)jenv->GetArrayLength(uData);
    LOG_DEBUG("USIZE : %d", USIZE);
        yuv_data_[1] = (unsigned char*)malloc(sizeof(unsigned char) * USIZE);
    memset(yuv_data_[1], 0, USIZE);
        jenv->GetByteArrayRegion(uData, 0, USIZE, (jbyte*)yuv_data_[1]);
    yuv_data_[1] = reinterpret_cast<unsigned char*>(yuv_data_[1]);
    } else {
        USIZE = YSIZE/4;
        yuv_data_[1] = (unsigned char*)malloc(sizeof(unsigned char) * USIZE);
    memset(yuv_data_[1], 1, USIZE);
  }

    if (vData != NULL) {
        VSIZE = (int)jenv->GetArrayLength(vData);
    LOG_DEBUG("VSIZE : %d", VSIZE);
        yuv_data_[2] = (unsigned char*)malloc(sizeof(unsigned char) * VSIZE);
    memset(yuv_data_[2], 0, VSIZE);
        jenv->GetByteArrayRegion(vData, 0, VSIZE, (jbyte*)yuv_data_[2]);
    yuv_data_[2] = reinterpret_cast<unsigned char*>(yuv_data_[2]);
    } else {
        VSIZE = YSIZE/4;
        yuv_data_[2] = (unsigned char*)malloc(sizeof(unsigned char) * VSIZE);
    memset(yuv_data_[2], 1, VSIZE);
  }

    glClearColor(1.0F, 1.0F, 1.0F, 1.0F);
    check_gl_error("glClearColor");
    glClear(GL_COLOR_BUFFER_BIT);
    check_gl_error("glClear");
}

void Renderer::draw_frame()
{
  // Binding created FBO
  glBindFramebuffer(GL_FRAMEBUFFER, frame_buffer_object_);
  check_gl_error("glBindFramebuffer");
    // Add program to OpenGL environment
    glUseProgram(program_object_);
    check_gl_error("glUseProgram");

  for (int i = 0; i < 3; i++) {
    LOG_DEBUG("Success");
      //Bind texture
      glActiveTexture(GL_TEXTURE0 + i);
      check_gl_error("glActiveTexture");
      glBindTexture(GL_TEXTURE_2D, yuv_texture_id_[i]);
      check_gl_error("glBindTexture");
      glUniform1i(yuv_texture_object_[i], i);
      check_gl_error("glBindTexture");
    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, stream_yuv_width_[i], stream_yuv_height_[i], GL_RGBA, GL_UNSIGNED_BYTE, yuv_data_[i]);
      check_gl_error("glTexSubImage2D");
  }

  LOG_DEBUG("Success");
    // Load vertex information
    glVertexAttribPointer(position_object_, 2, GL_FLOAT, GL_FALSE, kStride, kVertexInformation);
    check_gl_error("glVertexAttribPointer");
    // Load texture information
    glVertexAttribPointer(texture_position_object_, 2, GL_SHORT, GL_FALSE, kStride, kTextureCoordinateInformation);
    check_gl_error("glVertexAttribPointer");

LOG_DEBUG("9");
    glEnableVertexAttribArray(position_object_);
    check_gl_error("glEnableVertexAttribArray");
    glEnableVertexAttribArray(texture_position_object_);
    check_gl_error("glEnableVertexAttribArray");

  // Back to window buffer
  glBindFramebuffer(GL_FRAMEBUFFER, 0);
  check_gl_error("glBindFramebuffer");
  LOG_DEBUG("Success");
    // Draw the Square
    glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_SHORT, kIndicesInformation);
    check_gl_error("glDrawElements");
}

void Renderer::setup_render_to_texture()
{
    glGenFramebuffers(1, &frame_buffer_object_);
    check_gl_error("glGenFramebuffers");
    glBindFramebuffer(GL_FRAMEBUFFER, frame_buffer_object_);
    check_gl_error("glBindFramebuffer");
    glGenRenderbuffers(1, &render_buffer_object_);
    check_gl_error("glGenRenderbuffers");
    glBindRenderbuffer(GL_RENDERBUFFER, render_buffer_object_);
    check_gl_error("glBindRenderbuffer");
    glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA4, stream_yuv_width_[0], stream_yuv_height_[0]);
    check_gl_error("glRenderbufferStorage");
    glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, render_buffer_object_);
    check_gl_error("glFramebufferRenderbuffer");
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id_[0], 0);
    check_gl_error("glFramebufferTexture2D");  
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id_[1], 0);
    check_gl_error("glFramebufferTexture2D");  
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id_[2], 0);
    check_gl_error("glFramebufferTexture2D");  

  glBindFramebuffer(GL_FRAMEBUFFER, 0);
    check_gl_error("glBindFramebuffer");

    GLint status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
    if (status != GL_FRAMEBUFFER_COMPLETE) {
        print_log("renderer.cpp", "setup_graphics", "FBO setting fault.", LOGERROR);
        LOG_ERROR("%d\n", status);
        return;
    }
}

void Renderer::setup_yuv_texture() 
{
    // Use tightly packed data
    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
    check_gl_error("glPixelStorei");

  for (int i = 0; i < 3; i++) {
    if (yuv_texture_id_[i]) {
      glDeleteTextures(1, &yuv_texture_id_[i]);
      check_gl_error("glDeleteTextures");
    }
      glActiveTexture(GL_TEXTURE0+i);
      check_gl_error("glActiveTexture"); 
      // Generate texture object
      glGenTextures(1, &yuv_texture_id_[i]);
      check_gl_error("glGenTextures");
      glBindTexture(GL_TEXTURE_2D, yuv_texture_id_[i]);
      check_gl_error("glBindTexture");
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
      check_gl_error("glTexParameteri");
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
      check_gl_error("glTexParameteri");
      glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
      check_gl_error("glTexParameterf");
      glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
      check_gl_error("glTexParameterf"); 
    glEnable(GL_TEXTURE_2D);
    check_gl_error("glEnable");
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, maximum_yuv_width_[i], maximum_yuv_height_[i], 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
      check_gl_error("glTexImage2D");
  }
}

void Renderer::setup_graphics()
{
    print_gl_string("Version", GL_VERSION);
    print_gl_string("Vendor", GL_VENDOR);
    print_gl_string("Renderer", GL_RENDERER);
    print_gl_string("Extensions", GL_EXTENSIONS);

    program_object_ = create_program(kVertexShader, kFragmentShader);
    if (!program_object_) {
        print_log("renderer.cpp", "setup_graphics", "Could not create program.", LOGERROR);
        return;
    }

    position_object_ = glGetAttribLocation(program_object_, "vPosition");
    check_gl_error("glGetAttribLocation");
    texture_position_object_ = glGetAttribLocation(program_object_, "vTexCoord");
    check_gl_error("glGetAttribLocation");

    yuv_texture_object_[0] = glGetUniformLocation(program_object_, "yTexture");
    check_gl_error("glGetUniformLocation");
  yuv_texture_object_[1] = glGetUniformLocation(program_object_, "uTexture");
    check_gl_error("glGetUniformLocation");
    yuv_texture_object_[2] = glGetUniformLocation(program_object_, "vTexture");
    check_gl_error("glGetUniformLocation");

  setup_yuv_texture();
    setup_render_to_texture();

  glViewport(0, 0, stream_yuv_width_[0], stream_yuv_height_[0]);//736, 480);//1920, 1080);//maximum_yuv_width_[0], maximum_yuv_height_[0]);
  check_gl_error("glViewport");
}

GLuint Renderer::create_program(const char* vertex_source, const char* fragment_source)
{
    GLuint vertexShader = load_shader(GL_VERTEX_SHADER, vertex_source);
    if (!vertexShader) {
        return 0;
    }

    GLuint pixelShader = load_shader(GL_FRAGMENT_SHADER, fragment_source);
    if (!pixelShader) {
        return 0;
    }

    GLuint program = glCreateProgram();
    if (program) {
        glAttachShader(program, vertexShader);
        check_gl_error("glAttachShader");
        glAttachShader(program, pixelShader);
        check_gl_error("glAttachShader");
        glLinkProgram(program);
        /* Get a Status */
        GLint linkStatus = GL_FALSE;
        glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
        if (linkStatus != GL_TRUE) {
            GLint bufLength = 0;
            glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength);
            if (bufLength) {
                char* buf = (char*) malloc(bufLength);
                if (buf) {
                    glGetProgramInfoLog(program, bufLength, NULL, buf);
                    print_log("renderer.cpp", "create_program", "Could not link program.", LOGERROR);
                    LOG_ERROR("%s\n", buf);
                    free(buf);
                }
            }
            glDeleteProgram(program);
            program = 0;
        }
    }
    return program;
}

GLuint Renderer::load_shader(GLenum shaderType, const char* pSource)
{
    GLuint shader = glCreateShader(shaderType);
        if (shader) {
            glShaderSource(shader, 1, &pSource, NULL);
            glCompileShader(shader);
            /* Get a Status */
            GLint compiled = 0;
            glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
            if (!compiled) {
                GLint infoLen = 0;
                glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);
                if (infoLen) {
                    char* buf = (char*) malloc(infoLen);
                    if (buf) {
                        glGetShaderInfoLog(shader, infoLen, NULL, buf);
                        print_log("renderer.cpp", "load_shader", "Could not link program.", LOGERROR);
                                  LOG_ERROR("%d :: %s\n", shaderType, buf);
                        free(buf);
                    }
                    glDeleteShader(shader);
                    shader = 0;
                }
            }
        }
    return shader;
}


void Renderer::onDrawFrame(JNIEnv* jenv, jbyteArray yData, jbyteArray uData, jbyteArray vData)
{
    set_draw_frame(jenv, yData, uData, vData);
    draw_frame();
    return;
}

void Renderer::setSize(int stream_width, int stream_height) {
  stream_yuv_width_[0] = stream_width;
  stream_yuv_width_[1] = stream_width/2;
  stream_yuv_width_[2] = stream_width/2;
  stream_yuv_height_[0] = stream_height;
  stream_yuv_height_[1] = stream_height/2;
  stream_yuv_height_[2] = stream_height/2;
}

void Renderer::onSurfaceChanged(int width, int height)
{
  mobile_yuv_width_[0] = width;
  mobile_yuv_width_[1] = width/2;
  mobile_yuv_width_[2] = width/2; 
  mobile_yuv_height_[0] = height;
  mobile_yuv_height_[1] = height/2;
  mobile_yuv_height_[2] = height/2;

  maximum_yuv_width_[0] = 1920;
  maximum_yuv_width_[1] = 1920/2;
  maximum_yuv_width_[2] = 1920/2;
  maximum_yuv_height_[0] = 1080;
  maximum_yuv_height_[1] = 1080/2;
  maximum_yuv_height_[2] = 1080/2;

  // If stream size not setting, default size D1
  //if (stream_yuv_width_[0] == 0) {
    stream_yuv_width_[0] = 736;
    stream_yuv_width_[1] = 736/2;
    stream_yuv_width_[2] = 736/2;
    stream_yuv_height_[0] = 480;
    stream_yuv_height_[1] = 480/2;
    stream_yuv_height_[2] = 480/2;
  //}

    setup_graphics();
    return;
}

static const char kVertexShader[] =
    "attribute vec4 vPosition;      \n"
      "attribute vec2 vTexCoord;        \n"
      "varying vec2 v_vTexCoord;        \n"
    "void main() {                        \n"
        "gl_Position = vPosition;       \n"
        "v_vTexCoord = vTexCoord;       \n"
    "}                                          \n";

static const char kFragmentShader[] =
        "precision mediump float;               \n"
        "varying vec2 v_vTexCoord;          \n"
        "uniform sampler2D yTexture;        \n"
        "uniform sampler2D uTexture;        \n"
        "uniform sampler2D vTexture;        \n"
        "void main() {                      \n"
            "float y=texture2D(yTexture, v_vTexCoord).r;\n"
            "float u=texture2D(uTexture, v_vTexCoord).r - 0.5;\n"
            "float v=texture2D(vTexture, v_vTexCoord).r - 0.5;\n"
            "float r=y + 1.13983 * v;\n"
            "float g=y - 0.39465 * u - 0.58060 * v;\n"
            "float b=y + 2.03211 * u;\n"
            "gl_FragColor = vec4(r, g, b, 1.0);\n"
        "}\n";

static const GLfloat kVertexInformation[] =
{
         -1.0f, 1.0f,           // TexCoord 0 top left
         -1.0f,-1.0f,           // TexCoord 1 bottom left
          1.0f,-1.0f,           // TexCoord 2 bottom right
          1.0f, 1.0f            // TexCoord 3 top right
};
static const GLshort kTextureCoordinateInformation[] =
{
          0, 0,         // TexCoord 0 top left
          0, 1,         // TexCoord 1 bottom left
          1, 1,         // TexCoord 2 bottom right
          1, 0          // TexCoord 3 top right
};
static const GLuint kStride = 0;//COORDS_PER_VERTEX * 4;
static const GLshort kIndicesInformation[] =
{
    0, 1, 2, 
    0, 2, 3
};