Android native OpenGL ES 3.0 PBO Rendering leerer Bildschirm

Question

Ich entwickle Kamera Vorschau App mit Android.
Mein App Flow ist wie folgt.

1) JAVA: die Kameravorschau Puffer Schnappen und es JNI
2) CPP passieren: Stellen Textur von Pufferkameravorschau mit OpenGL ES 3.0
3) CPP: Textur mit OpenGL ES Render 3.0

Also, meine App funktioniert gut mit normalen Textur Update/Rendering Problemumgehung. Es ist in Ordnung.

Das Problem ist die Leistung. Ich muss jedes Bild auf den GPU-Speicher mit glTexSubImage2D(...) aktualisieren und es ist etwas langsam.
Also ich versuche, PBO zu verwenden, aber gerenderten Bildschirm ist immer leer.

Ich weiß nicht, was ich vermisse, kann jemand helfen?

Hier ist mein Code:

#include "native-lib.h" 
#include "textureloader.h" 
#include "vecmath.h" 
#include "glutil.h" 
#include "logger.h" 

GLuint program; 
GLuint p_mvp, p_vertices, p_uvs, p_tex; 
GLuint tex; 
GLuint pboIds[2]; 
GLfloat vertices[] = {0,0,0,0,0,0,0,0}; 
GLfloat uvs[] = {0,1,1,1,1,0,0,0}; 
GLushort indices[] = {0,1,2,0,2,3}; 
int dataSize; 
float zoom; 
mat4 mvp; 

bool usePBO = true; 
bool doublePBO = false; 

cv::Mat mat; 

std::string vs = 
     "#version 300 es\n" 
       "in vec4 a_vertices;\n" 
       "in vec2 a_uvs;\n" 
       "uniform mat4 u_mvp;\n" 
       "out vec2 v_texCoord;\n" 
       "void main(){\n" 
       " gl_Position = u_mvp * a_vertices;\n" 
       " v_texCoord = a_uvs;\n" 
       "}\n"; 
std::string fs = 
     "#version 300 es\n" 
       "uniform sampler2D u_tex;\n" 
       "in vec2 v_texCoord;\n" 
       "out vec4 fragColor;\n" 
       "void main(){\n" 
       " fragColor = texture(u_tex, v_texCoord);\n" 
       " fragColor.a = 1.0f;\n" 
       "}\n"; 


JNIEXPORT jint JNICALL 
Java_com_quram_vmtest3_Native_glInit(JNIEnv *env, jclass) 
{ 
    // create program 
    program = loadProgram(vs, fs, false); 

    // create texture basic data 
    p_vertices = glGetAttribLocation(program, "a_vertices"); 
    p_uvs = glGetAttribLocation(program, "a_uvs"); 
    p_mvp = glGetUniformLocation(program, "u_mvp"); 
    p_tex = glGetUniformLocation(program, "u_tex"); 

    // create PBO 
    if(usePBO) { 
     glGenBuffers(2, pboIds); 
    } 

    return 1; 
} 

JNIEXPORT jint JNICALL 
Java_com_quram_vmtest3_Native_surfaceChanged(JNIEnv *env, jclass, 
                jint w, jint h) 
{ 
    glViewport(0, 0, w, h); 

    zoom = (w < h ? w : h)/2; 
    vec4 target = vec4(w/2, h/2, 0, 0); 
    vec4 eye = target + vec4(0, 0, zoom, 1); 
    mat4 view(mat4::lookAt(eye, target, vec4(0, 1, 0, 0))); 
    mat4 proj(mat4::perspective(90, (float)w/(float)h, 1, 1000)); 
    mvp = proj * view; 

    vertices[5] = vertices[7] = h; 
    vertices[0] = vertices[6] = w; 

    glGenTextures(1, texId); 
    glBindTexture(GL_TEXTURE_2D, *texId); 
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL); 

    if(usePBO) { 
     dataSize = w * h * 3; 

     glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[0]); 
     glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW); 
     glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[1]); 
     glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, NULL, GL_STREAM_DRAW); 
     glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); 
    } 

    return 1; 
} 


JNIEXPORT jint JNICALL 
Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray, 
             jint w, jint h) 
{ 
    jbyte *buf = env->GetByteArrayElements(bytearray, 0); 

    if(mat.empty()) 
     mat = cv::Mat(h+h/2, w, CV_8UC1, buf); 
    else 
     mat.data = reinterpret_cast<uchar*>(buf); 
    cv::cvtColor(mat, mat, CV_YUV2RGB_NV21); 
    rotateMat(mat, mat, -90); 

    if(usePBO) { 
     static int index = 0; 
     if(doublePBO) { 
      int nextIndex = 0; 

      index = (index + 1) % 2; 
      nextIndex = (index + 1) % 2; 

      // bind the texture and PBO 
      glBindTexture(GL_TEXTURE_2D, tex); 
      glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]); 
      glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 

      // copy pixels from PBO to texture object 
      // Use offset instead of ponter. 
      glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0); 
      glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[nextIndex]); 
      glBufferData(GL_PIXEL_UNPACK_BUFFER, dataSize, 0, GL_STREAM_DRAW); 
      GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT); 
      if (ptr) { 
       memcpy(ptr, mat.ptr(), dataSize); 
       glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer 
      } 
      glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); 
      glBindTexture(GL_TEXTURE_2D, 0); 
     } 
     else { 
      glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pboIds[index]); 
      glBindTexture(GL_TEXTURE_2D, tex); 
      GLubyte *ptr = (GLubyte *) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, dataSize, GL_MAP_WRITE_BIT); 
      if (ptr) { 
       // memcpy(ptr, mat.ptr(), dataSize); 
       memset(ptr, 255, dataSize); 
       glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0); 
       glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER); // release pointer to mapping buffer 
      } 
      glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); 
      glBindTexture(GL_TEXTURE_2D, 0); 
     } 
    } 
    else { 
     loadTextureWithMat(mat, &tex, GL_RGB, GL_RGB, GL_UNSIGNED_BYTE); 
    } 

    // enable depth test 
    glEnable(GL_DEPTH_TEST); 
    glDepthFunc(GL_LEQUAL); 

    // enable alpha blending option 
    glEnable(GL_BLEND); 
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); 

    glClearColor(1.f, 0.f, 0.f, 0.f); 
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); 

    glUseProgram(program); 

    glUniformMatrix4fv(p_mvp, 1, GL_FALSE, &mvp.x.x); 
    glEnableVertexAttribArray(p_vertices); 
    glVertexAttribPointer(p_vertices, 2, GL_FLOAT, GL_FALSE, 0, vertices); 
    glEnableVertexAttribArray(p_uvs); 
    glVertexAttribPointer(p_uvs, 2, GL_FLOAT, GL_FALSE, 0, uvs); 
    glUniform1i(p_tex, 0); 

    glActiveTexture(GL_TEXTURE0); 
    glBindTexture(GL_TEXTURE_2D, tex); 

    glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); 

    glBindTexture(GL_TEXTURE_2D, 0); 
    glUseProgram(0); 

    // disable alpha blending option 
    glDisable(GL_BLEND); 

    // disable depth test 
    glDisable(GL_DEPTH_TEST); 

    env->ReleaseByteArrayElements(bytearray, buf, JNI_ABORT); 
    mat.release(); 

    return 1; 
} 

void rotateMat(cv::Mat const &src, cv::Mat &dst, int angle) 
{ 
    if(angle == 270 || angle == -90){ 
     // Rotate clockwise 270 degrees 
     cv::transpose(src, dst); 
     cv::flip(dst, dst, 0); 
    }else if(angle == 180 || angle == -180){ 
     // Rotate clockwise 180 degrees 
     cv::flip(src, dst, -1); 
    }else if(angle == 90 || angle == -270){ 
     // Rotate clockwise 90 degrees 
     cv::transpose(src, dst); 
     cv::flip(dst, dst, 1); 
    }else if(angle == 360 || angle == 0 || angle == -360){ 
     if(src.data != dst.data){ 
      src.copyTo(dst); 
     } 
    } 
} 

Wenn ich nicht alles PBO verwenden funktioniert gut (außer fps) so gehe ich davon aus Codes innerhalb usePBO und doublePBO Flaggen Inspektion wäre genug.

Answer 1

Nach einigen Schwierigkeiten fand ich endlich meinen kleinen (aber kritischen) Fehler.

Ich habe meinen cv :: Mat gedreht und so wurde die Breite und Höhe vertauscht.

Java_com_quram_vmtest3_Native_process(JNIEnv *env, jclass, jbyteArray bytearray, 
             jint w, jint h) 
{ 
    mat = cv::Mat(h+h/2, w, CV_8UC1, buf); // created w*(h*1.5) size mat 
    cv::cvtColor(mat, mat, CV_YUV2RGB_NV21); // This makes mat height from h*1.5 to h 
    rotateMat(mat, mat, -90); // Now our mat is h*w 
} 

Ich verwendete Breite, Höhe mit Funktionsparameterwert (die nicht ausgelagert wird).

glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGB, GL_UNSIGNED_BYTE, 0); 

änderte ich den Code oben mit diesem:

glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mat.cols, mat.rows, GL_RGB, GL_UNSIGNED_BYTE, 0); 

Ja, alles funktioniert gut :)

HINWEIS:
Btw die Leistung fast genauso aussieht wie vorher.
Vielleicht die Verwendung von PBO garantiert nicht immer Leistungssteigerung ...