Python实现静态和动态增强现实

一、环境

pycharm+OpenGL+opengame

二、静态及动态增强现实实现

（一）静态（立方体和茶壶）

　　　　1.立方体的实现

　　　　　将静态立方体放到键盘上

　　　　　效果图：

　　　　　　　　　　　　（a）

　　　　　　　　　　　　（b）

　　　　　　　　　　　　　(c)

　　　　　　　　　　　　图一

　　　　　代码：

  1 from pylab import *
  2 from PIL import Image
  3 # If you have PCV installed, these imports should work
  4 from PCV.geometry import homography, camera
  5 from PCV.localdescriptors import sift
  6 
  7 """
  8 This is the augmented reality and pose estimation cube example from Section 4.3.
  9 """
 10 
 11 def draw_teapot(size):
 12     glEnable(GL_LIGHTING)
 13     glEnable(GL_LIGHT0)
 14 def cube_points(c, wid):
 15     """ Creates a list of points for plotting
 16         a cube with plot. (the first 5 points are
 17         the bottom square, some sides repeated). """
 18     p = []
 19     # bottom
 20     p.append([c[0]-wid, c[1]-wid, c[2]-wid])
 21     p.append([c[0]-wid, c[1]+wid, c[2]-wid])
 22     p.append([c[0]+wid, c[1]+wid, c[2]-wid])
 23     p.append([c[0]+wid, c[1]-wid, c[2]-wid])
 24     p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot
 25     
 26     # top
 27     p.append([c[0]-wid, c[1]-wid, c[2]+wid])
 28     p.append([c[0]-wid, c[1]+wid, c[2]+wid])
 29     p.append([c[0]+wid, c[1]+wid, c[2]+wid])
 30     p.append([c[0]+wid, c[1]-wid, c[2]+wid])
 31     p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot
 32     
 33     # vertical sides
 34     p.append([c[0]-wid, c[1]-wid, c[2]+wid])
 35     p.append([c[0]-wid, c[1]+wid, c[2]+wid])
 36     p.append([c[0]-wid, c[1]+wid, c[2]-wid])
 37     p.append([c[0]+wid, c[1]+wid, c[2]-wid])
 38     p.append([c[0]+wid, c[1]+wid, c[2]+wid])
 39     p.append([c[0]+wid, c[1]-wid, c[2]+wid])
 40     p.append([c[0]+wid, c[1]-wid, c[2]-wid])
 41     
 42     return array(p).T
 43 
 44 
 45 def my_calibration(sz):
 46     """
 47     Calibration function for the camera (iPhone4) used in this example.
 48     """
 49     row, col = sz
 50     fx = 2555*col/2592
 51     fy = 2586*row/1936
 52     K = diag([fx, fy, 1])
 53     K[0, 2] = 0.5*col
 54     K[1, 2] = 0.5*row
 55     return K
 56 
 57 
 58 
 59 # compute features
 60 sift.process_image('1.png', 'im0.sift')
 61 l0, d0 = sift.read_features_from_file('im0.sift')
 62 
 63 sift.process_image('1.png', 'im1.sift')
 64 l1, d1 = sift.read_features_from_file('im1.sift')
 65 
 66 
 67 # match features and estimate homography
 68 matches = sift.match_twosided(d0, d1)
 69 ndx = matches.nonzero()[0]
 70 fp = homography.make_homog(l0[ndx, :2].T)
 71 ndx2 = [int(matches[i]) for i in ndx]
 72 tp = homography.make_homog(l1[ndx2, :2].T)
 73 
 74 model = homography.RansacModel()
 75 H, inliers = homography.H_from_ransac(fp, tp, model)
 76 
 77 # camera calibration
 78 K = my_calibration((747, 1000))
 79 
 80 # 3D points at plane z=0 with sides of length 0.2
 81 box = cube_points([0, 0, 0.1], 0.1)
 82 
 83 # project bottom square in first image
 84 cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]])))))
 85 # first points are the bottom square
 86 box_cam1 = cam1.project(homography.make_homog(box[:, :5]))
 87 
 88 
 89 # use H to transfer points to the second image
 90 box_trans = homography.normalize(dot(H,box_cam1))
 91 
 92 # compute second camera matrix from cam1 and H
 93 cam2 = camera.Camera(dot(H, cam1.P))
 94 A = dot(linalg.inv(K), cam2.P[:, :3])
 95 A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
 96 cam2.P[:, :3] = dot(K, A)
 97 
 98 # project with the second camera
 99 box_cam2 = cam2.project(homography.make_homog(box))
100 
101 
102 
103 # plotting
104 im0 = array(Image.open('1.png'))
105 im1 = array(Image.open('2.png'))
106 
107 figure()
108 imshow(im0)
109 plot(box_cam1[0, :], box_cam1[1, :], linewidth=3)
110 title('2D projection of bottom square')
111 axis('off')
112 
113 figure()
114 imshow(im1)
115 plot(box_trans[0, :], box_trans[1, :], linewidth=3)
116 title('2D projection transfered with H')
117 axis('off')
118 
119 figure()
120 imshow(im1)
121 plot(box_cam2[0, :], box_cam2[1, :], linewidth=3)
122 title('3D points projected in second image')
123 axis('off')
124 
125 show()

立方体

　　　　2.茶壶的实现

　　　　　将茶壶放到书籍上

　　　　　效果图：

　　　　　　　　　　　　　　　　　　　　　　　　图二

　　　　　　代码：

import math
import pickle
from pylab import *
from OpenGL.GL import * 
from OpenGL.GLU import * 
from OpenGL.GLUT import * 
import pygame, pygame.image 
from pygame.locals import *
from PCV.geometry import homography, camera
from PCV.localdescriptors import sift

def cube_points(c, wid):
    """ Creates a list of points for plotting
        a cube with plot. (the first 5 points are
        the bottom square, some sides repeated). """
    p = []
    # bottom
    p.append([c[0]-wid, c[1]-wid, c[2]-wid])
    p.append([c[0]-wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]-wid, c[2]-wid])
    p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot
    
    # top
    p.append([c[0]-wid, c[1]-wid, c[2]+wid])
    p.append([c[0]-wid, c[1]+wid, c[2]+wid])
    p.append([c[0]+wid, c[1]+wid, c[2]+wid])
    p.append([c[0]+wid, c[1]-wid, c[2]+wid])
    p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot
    
    # vertical sides
    p.append([c[0]-wid, c[1]-wid, c[2]+wid])
    p.append([c[0]-wid, c[1]+wid, c[2]+wid])
    p.append([c[0]-wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]+wid, c[2]+wid])
    p.append([c[0]+wid, c[1]-wid, c[2]+wid])
    p.append([c[0]+wid, c[1]-wid, c[2]-wid])
    
    return array(p).T
    
def my_calibration(sz):
    row, col = sz
    fx = 2555*col/2592
    fy = 2586*row/1936
    K = diag([fx, fy, 1])
    K[0, 2] = 0.5*col
    K[1, 2] = 0.5*row
    return K

def set_projection_from_camera(K): 
   glMatrixMode(GL_PROJECTION) 
   glLoadIdentity()
   fx = K[0,0] 
   fy = K[1,1] 
   fovy = 2*math.atan(0.5*height/fy)*180/math.pi 
   aspect = (width*fy)/(height*fx)
   near = 0.1 
   far = 100.0
   gluPerspective(fovy,aspect,near,far) 
   glViewport(0,0,width,height)

def set_modelview_from_camera(Rt): 
   glMatrixMode(GL_MODELVIEW) 
   glLoadIdentity()
   Rx = np.array([[1,0,0],[0,0,-1],[0,1,0]])
   R = Rt[:,:3] 
   U,S,V = np.linalg.svd(R) 
   R = np.dot(U,V) 
   R[0,:] = -R[0,:]
   t = Rt[:,3]
   M = np.eye(4) 
   M[:3,:3] = np.dot(R,Rx) 
   M[:3,3] = t
   M = M.T
   m = M.flatten()
   glLoadMatrixf(m)

def draw_background(imname):
   bg_image = pygame.image.load(imname).convert() 
   bg_data = pygame.image.tostring(bg_image,"RGBX",1)
   glMatrixMode(GL_MODELVIEW) 
   glLoadIdentity()

   glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
   glEnable(GL_TEXTURE_2D) 
   glBindTexture(GL_TEXTURE_2D,glGenTextures(1)) 
   glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA,width,height,0,GL_RGBA,GL_UNSIGNED_BYTE,bg_data) 
   glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_NEAREST) 
   glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_NEAREST)
   glBegin(GL_QUADS) 
   glTexCoord2f(0.0,0.0); glVertex3f(-1.0,-1.0,-1.0) 
   glTexCoord2f(1.0,0.0); glVertex3f( 1.0,-1.0,-1.0) 
   glTexCoord2f(1.0,1.0); glVertex3f( 1.0, 1.0,-1.0) 
   glTexCoord2f(0.0,1.0); glVertex3f(-1.0, 1.0,-1.0) 
   glEnd()
   glDeleteTextures(1)


def draw_teapot(size):
   glEnable(GL_LIGHTING) 
   glEnable(GL_LIGHT0) 
   glEnable(GL_DEPTH_TEST) 
   glClear(GL_DEPTH_BUFFER_BIT)
   glMaterialfv(GL_FRONT,GL_AMBIENT,[0,0,0,0]) 
   glMaterialfv(GL_FRONT,GL_DIFFUSE,[0.5,0.0,0.0,0.0]) 
   glMaterialfv(GL_FRONT,GL_SPECULAR,[0.7,0.6,0.6,0.0]) 
   glMaterialf(GL_FRONT,GL_SHININESS,0.25*128.0) 
   glutSolidTeapot(size)

width,height = 1000,747
def setup():
   pygame.init() 
   pygame.display.set_mode((width,height),OPENGL | DOUBLEBUF) 
   pygame.display.set_caption("OpenGL AR demo")    

# compute features
sift.process_image('book_frontal.JPG', 'im0.sift')
l0, d0 = sift.read_features_from_file('im0.sift')

sift.process_image('book_perspective.JPG', 'im1.sift')
l1, d1 = sift.read_features_from_file('im1.sift')

# match features and estimate homography
matches = sift.match_twosided(d0, d1)
ndx = matches.nonzero()[0]
fp = homography.make_homog(l0[ndx, :2].T)
ndx2 = [int(matches[i]) for i in ndx]
tp = homography.make_homog(l1[ndx2, :2].T)

model = homography.RansacModel()
H, inliers = homography.H_from_ransac(fp, tp, model)

K = my_calibration((747, 1000))
cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]])))))
box = cube_points([0, 0, 0.1], 0.1)
box_cam1 = cam1.project(homography.make_homog(box[:, :5]))
box_trans = homography.normalize(dot(H,box_cam1))
cam2 = camera.Camera(dot(H, cam1.P))
A = dot(linalg.inv(K), cam2.P[:, :3])
A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = dot(K, A)

Rt=dot(linalg.inv(K),cam2.P)
 
setup() 
draw_background("book_perspective.bmp") 
set_projection_from_camera(K) 
set_modelview_from_camera(Rt)
draw_teapot(0.05)

pygame.display.flip()
while True: 
   for event in pygame.event.get():
      if event.type==pygame.QUIT:
         sys.exit()
import math
import pickle
from pylab import *
from OpenGL.GL import * 
from OpenGL.GLU import * 
from OpenGL.GLUT import * 
import pygame, pygame.image 
from pygame.locals import *
from PCV.geometry import homography, camera
from PCV.localdescriptors import sift

def cube_points(c, wid):
    """ Creates a list of points for plotting
        a cube with plot. (the first 5 points are
        the bottom square, some sides repeated). """
    p = []
    # bottom
    p.append([c[0]-wid, c[1]-wid, c[2]-wid])
    p.append([c[0]-wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]-wid, c[2]-wid])
    p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot
    
    # top
    p.append([c[0]-wid, c[1]-wid, c[2]+wid])
    p.append([c[0]-wid, c[1]+wid, c[2]+wid])
    p.append([c[0]+wid, c[1]+wid, c[2]+wid])
    p.append([c[0]+wid, c[1]-wid, c[2]+wid])
    p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot
    
    # vertical sides
    p.append([c[0]-wid, c[1]-wid, c[2]+wid])
    p.append([c[0]-wid, c[1]+wid, c[2]+wid])
    p.append([c[0]-wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]+wid, c[2]-wid])
    p.append([c[0]+wid, c[1]+wid, c[2]+wid])
    p.append([c[0]+wid, c[1]-wid, c[2]+wid])
    p.append([c[0]+wid, c[1]-wid, c[2]-wid])
    
    return array(p).T
    
def my_calibration(sz):
    row, col = sz
    fx = 2555*col/2592
    fy = 2586*row/1936
    K = diag([fx, fy, 1])
    K[0, 2] = 0.5*col
    K[1, 2] = 0.5*row
    return K

def set_projection_from_camera(K): 
   glMatrixMode(GL_PROJECTION) 
   glLoadIdentity()
   fx = K[0,0] 
   fy = K[1,1] 
   fovy = 2*math.atan(0.5*height/fy)*180/math.pi 
   aspect = (width*fy)/(height*fx)
   near = 0.1 
   far = 100.0
   gluPerspective(fovy,aspect,near,far) 
   glViewport(0,0,width,height)

def set_modelview_from_camera(Rt): 
   glMatrixMode(GL_MODELVIEW) 
   glLoadIdentity()
   Rx = np.array([[1,0,0],[0,0,-1],[0,1,0]])
   R = Rt[:,:3] 
   U,S,V = np.linalg.svd(R) 
   R = np.dot(U,V) 
   R[0,:] = -R[0,:]
   t = Rt[:,3]
   M = np.eye(4) 
   M[:3,:3] = np.dot(R,Rx) 
   M[:3,3] = t
   M = M.T
   m = M.flatten()
   glLoadMatrixf(m)

def draw_background(imname):
   bg_image = pygame.image.load(imname).convert() 
   bg_data = pygame.image.tostring(bg_image,"RGBX",1)
   glMatrixMode(GL_MODELVIEW) 
   glLoadIdentity()

   glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
   glEnable(GL_TEXTURE_2D) 
   glBindTexture(GL_TEXTURE_2D,glGenTextures(1)) 
   glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA,width,height,0,GL_RGBA,GL_UNSIGNED_BYTE,bg_data) 
   glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_NEAREST) 
   glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_NEAREST)
   glBegin(GL_QUADS) 
   glTexCoord2f(0.0,0.0); glVertex3f(-1.0,-1.0,-1.0) 
   glTexCoord2f(1.0,0.0); glVertex3f( 1.0,-1.0,-1.0) 
   glTexCoord2f(1.0,1.0); glVertex3f( 1.0, 1.0,-1.0) 
   glTexCoord2f(0.0,1.0); glVertex3f(-1.0, 1.0,-1.0) 
   glEnd()
   glDeleteTextures(1)


def draw_teapot(size):
   glEnable(GL_LIGHTING) 
   glEnable(GL_LIGHT0) 
   glEnable(GL_DEPTH_TEST) 
   glClear(GL_DEPTH_BUFFER_BIT)
   glMaterialfv(GL_FRONT,GL_AMBIENT,[0,0,0,0]) 
   glMaterialfv(GL_FRONT,GL_DIFFUSE,[0.5,0.0,0.0,0.0]) 
   glMaterialfv(GL_FRONT,GL_SPECULAR,[0.7,0.6,0.6,0.0]) 
   glMaterialf(GL_FRONT,GL_SHININESS,0.25*128.0) 
   glutSolidTeapot(size)

width,height = 1000,747
def setup():
   pygame.init() 
   pygame.display.set_mode((width,height),OPENGL | DOUBLEBUF) 
   pygame.display.set_caption("OpenGL AR demo")    

# compute features
sift.process_image('book_frontal.JPG', 'im0.sift')
l0, d0 = sift.read_features_from_file('im0.sift')

sift.process_image('book_perspective.JPG', 'im1.sift')
l1, d1 = sift.read_features_from_file('im1.sift')

# match features and estimate homography
matches = sift.match_twosided(d0, d1)
ndx = matches.nonzero()[0]
fp = homography.make_homog(l0[ndx, :2].T)
ndx2 = [int(matches[i]) for i in ndx]
tp = homography.make_homog(l1[ndx2, :2].T)

model = homography.RansacModel()
H, inliers = homography.H_from_ransac(fp, tp, model)

K = my_calibration((747, 1000))
cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]])))))
box = cube_points([0, 0, 0.1], 0.1)
box_cam1 = cam1.project(homography.make_homog(box[:, :5]))
box_trans = homography.normalize(dot(H,box_cam1))
cam2 = camera.Camera(dot(H, cam1.P))
A = dot(linalg.inv(K), cam2.P[:, :3])
A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = dot(K, A)

Rt=dot(linalg.inv(K),cam2.P)
 
setup() 
draw_background("book_perspective.bmp") 
set_projection_from_camera(K) 
set_modelview_from_camera(Rt)
draw_teapot(0.05)

pygame.display.flip()
while True: 
   for event in pygame.event.get():
      if event.type==pygame.QUIT:
         sys.exit()

茶壶

（二）动态

　　下图使用手机拍摄的gif，用的model是fox。

　　　　　　　　　　　　　　　　　　　　　　　图三

代码链接：https://download.csdn.net/download/weixin_43842653/11094470

三、实验中遇到的问题

1.下载OpenGL

链接：https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyopengl

ctrl+F搜索PyOpenGL，选择与自己python版本相应的版本下载。

2.实现动态增强现实时出现下图问题：

说明你的图书封面拍摄有问题，可以换成横着拍（或竖着排）如：

右图就无法实现将fox放到书上的效果，换成横着拍的左图即可。