Overlay 3d model on camera frame background opengl

I am working on an augmented reality project. The frame is taken as input and converted into an OpenGL texture. That OpenGL texture is then set as background of the OpenGL scene. I would like to project a cube on the scene. I done that by using the homography matrix on opencv and then convert the homography matrix into model view matrix (projection matrix for opengl). The cube is successfully projected but the whole scene is also overlayed by another black background (I need only the cube to be projected while maintaining my camera frame as background). I am not sure whats wrong with the texture, please help to take a look on it, thanks in advance!

Code for projection and scene:

import numpy as np
import cv2
import math
from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *
from PIL import Image
from main import run_ar
from spin_cube import Cube

cap = cv2.VideoCapture("src/video.mov")
query = cv2.imread("src/cards.jpg")
calibration_matrix = np.array([[800, 0, 320], [0, 800, 240], [0, 0, 1]])

def init_gl():
    global texture_background, texture_object
    glClearColor(0.0, 0.0, 0.0, 0.0)
    glClearDepth(1.0)
    glDepthFunc(GL_LESS)
    glEnable(GL_DEPTH_TEST)
    glShadeModel(GL_SMOOTH)
    glMatrixMode(GL_MODELVIEW)
    glEnable(GL_DEPTH_TEST)
    glEnable(GL_LIGHTING)
    glEnable(GL_LIGHT0)
    texture_background = glGenTextures(1)
    texture_object = glGenTextures(1)


def main():
    glutInit()
    glutInitWindowSize(640, 480)
    glutInitWindowPosition(625, 100)
    glutInitDisplayMode(GLUT_RGB | GLUT_DEPTH | GLUT_DOUBLE)
    window_id = glutCreateWindow("OpenGL")
    init_gl()
    glutDisplayFunc(drawGLScene)
    glutIdleFunc(drawGLScene)
    glutReshapeFunc(resize)
    glutMainLoop()

def init_object_texture(image_filepath):
        tex = cv2.imread(image_filepath)
        tex = cv2.flip(tex, 0)
        tex = Image.fromarray(tex)
        ix = tex.size[0]
        iy = tex.size[1]
        tex = tex.tobytes("raw","BGRX", 0, -1)

        glActiveTexture(GL_TEXTURE1)
        glBindTexture(GL_TEXTURE_2D, texture_background)
        glEnable(GL_TEXTURE_2D)

        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
        glTexImage2D(GL_TEXTURE_2D, 0,GL_RGBA, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, tex)
        return None

def resize(w,h):
        ratio = 1.0* w / h
        glMatrixMode(GL_PROJECTION)
        gluPerspective(45, ratio, 0.1, 100.0)

def draw_background(img):
 #CONVERTING CAMERA FRAME INTO OPENGL USABLE FORM
        tex_back = cv2.flip(img, 0)
        tex_back = Image.fromarray(tex_back)
        ix = tex_back.size[0]
        iy = tex_back.size[1]
        tex_back = tex_back.tobytes("raw","BGRX", 0, -1)

        #DEFINING TEXTURE TYPE FOR BACKGROUND
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, texture_background)
        glEnable(GL_TEXTURE_2D)

        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)

        glTexImage2D(GL_TEXTURE_2D, 0,GL_RGBA, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, tex_back)


        glTranslatef(0.0,0.0,-4.5)
        glBegin(GL_QUADS)

        glTexCoord2fv([0.0, 1.0]);
        glVertex3fv([-4.0, -3.0, 0.0])
        glTexCoord2fv([1.0, 1.0]);
        glVertex3fv([ 4.0, -3.0, 0.0])
        glTexCoord2fv([1.0, 0.0]);
        glVertex3fv( [4.0,  3.0, 0.0])
        glTexCoord2fv([0.0, 0.0]);
        glVertex3fv([-4.0,  3.0, 0.0])
        glEnd( )

        return None

def decompose_homo(homography_matrix):
    num, rvec, tvec, _ = cv2.decomposeHomographyMat(homography_matrix, calibration_matrix)
    rmtx = np.array(rvec)
    tmtx = np.array(tvec)

    view_matrix = np.array([[rmtx[0][0][0],rmtx[0][0][1],rmtx[0][0][2],  tmtx[0][0]],
                            [rmtx[0][1][0],rmtx[0][1][1],rmtx[0][1][2],  tmtx[0][1]],
                            [rmtx[0][2][0],rmtx[0][2][1],rmtx[0][2][2],  tmtx[0][2]],
                            [0.0       ,0.0       ,0.0       ,1.0    ]])

    view_matrix = np.transpose(view_matrix)
    return view_matrix

def drawGLScene():
    ret, frame = cap.read()
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    draw_background(frame)
    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()
    homography_matrix = run_ar(query, frame)
    init_object_texture("tex_src/wood.jpeg")
    view_matrix = decompose_homo(homography_matrix)
    cube_model = Cube(view_matrix)
    cube_model.update()
    cube_model.draw()
    cv2.imshow('frame',frame)
    cv2.waitKey(1)
    glutSwapBuffers()

main()

Code for cube model:

import pygame as pg
from OpenGL.GL import *
from OpenGL.GL.shaders import compileProgram,compileShader
import numpy as np
import pyrr
import cv2
from PIL import Image

class Cube:
    def __init__(self, view_matrix):
        # initialise opengl
        self.shader = self.createShader("shaders/vertex.txt", "shaders/fragment.txt")
        glUseProgram(self.shader)
        glUniform1i(glGetUniformLocation(self.shader, "imageTexture"), 0)
        glEnable(GL_DEPTH_TEST)

        self.view_matrix = view_matrix

        # x, y, z, s, t
        self.vertices = (
                -0.2, -0.2, -0.2, 0, 0,
                 0.2, -0.2, -0.2, 1, 0,
                 0.2,  0.2, -0.2, 1, 1,

                 0.2,  0.2, -0.2, 1, 1,
                -0.2,  0.2, -0.2, 0, 1,
                -0.2, -0.2, -0.2, 0, 0,

                -0.2, -0.2,  0.2, 0, 0,
                 0.2, -0.2,  0.2, 1, 0,
                 0.2,  0.2,  0.2, 1, 1,

                 0.2,  0.2,  0.2, 1, 1,
                -0.2,  0.2,  0.2, 0, 1,
                -0.2, -0.2,  0.2, 0, 0,

                -0.2,  0.2,  0.2, 1, 0,
                -0.2,  0.2, -0.2, 1, 1,
                -0.2, -0.2, -0.2, 0, 1,

                -0.2, -0.2, -0.2, 0, 1,
                -0.2, -0.2,  0.2, 0, 0,
                -0.2,  0.2,  0.2, 1, 0,

                 0.2,  0.2,  0.2, 1, 0,
                 0.2,  0.2, -0.2, 1, 1,
                 0.2, -0.2, -0.2, 0, 1,

                 0.2, -0.2, -0.2, 0, 1,
                 0.2, -0.2,  0.2, 0, 0,
                 0.2,  0.2,  0.2, 1, 0,

                -0.2, -0.2, -0.2, 0, 1,
                 0.2, -0.2, -0.2, 1, 1,
                 0.2, -0.2,  0.2, 1, 0,

                 0.2, -0.2,  0.2, 1, 0,
                -0.2, -0.2,  0.2, 0, 0,
                -0.2, -0.2, -0.2, 0, 1,

                -0.2,  0.2, -0.2, 0, 1,
                 0.2,  0.2, -0.2, 1, 1,
                 0.2,  0.2,  0.2, 1, 0,

                 0.2,  0.2,  0.2, 1, 0,
                -0.2,  0.2,  0.2, 0, 0,
                -0.2,  0.2, -0.2, 0, 1
            )
        self.vertex_count = len(self.vertices)//5
        self.vertices = np.array(self.vertices, dtype=np.float32)

        self.vao = glGenVertexArrays(1)
        glBindVertexArray(self.vao)
        self.vbo = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, self.vbo)
        glBufferData(GL_ARRAY_BUFFER, self.vertices.nbytes, self.vertices, GL_STATIC_DRAW)

        glEnableVertexAttribArray(0)
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 20, ctypes.c_void_p(0))

        glEnableVertexAttribArray(1)
        glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 20, ctypes.c_void_p(12))

    def createShader(self, vertexFilepath, fragmentFilepath):

        with open(vertexFilepath,'r') as f:
            vertex_src = f.readlines()

        with open(fragmentFilepath,'r') as f:
            fragment_src = f.readlines()

        shader = compileProgram(compileShader(vertex_src, GL_VERTEX_SHADER),
                                compileShader(fragment_src, GL_FRAGMENT_SHADER))

        return shader

    def update(self):
        glUniformMatrix4fv(glGetUniformLocation(self.shader,"model"),1,GL_FALSE,self.view_matrix)

    def draw(self):
        glUseProgram(self.shader)
        glBindVertexArray(self.vao)
        glDrawArrays(GL_TRIANGLES, 0, self.vertex_count)

The result I got (The window named OpenGL should had the window named frame as background scene while only cube is being projected on the scene but turns out the background on OpenGL is overlayed with black colour which is not what I was looking for): enter image description here

How many English words
do you know?
Test your English vocabulary size, and measure
how many words do you know
Online Test
Powered by Examplum