GPU编程：OpenGL-图像处理算法

使用OpenGL处理鱼眼图像信号

本篇章是OpenGL编程的一个经典应用，可以广泛使用在带有GPU的各种设备上。通过阅读本篇章，可以完全了解如何使用标定好的相机内参数据对鱼眼图像进行去畸变处理。

OpenGL

OpenGL是一套通用GPU编程接口，是操纵GPU进行图像处理的一套方法，抑或是渲染管线。

关于OpenGL的基础环境、基础原理，可以参考：主页 - LearnOpenGL CN (learnopengl-cn.github.io)

本篇章就不再赘述现代渲染管线的一些基础知识。

​

超大FOV鱼眼相机的成像模型

参考OpenCV的官方文档，适合鱼眼的成像数学模型是K-B通用相机模型：Kanala-Brandt畸变模型。它不同于针孔相机的5参数径向切向畸变建模，而是采用了麦克劳林多项式对畸变的光路进行建模。参考论文：《A generic camera model and calibration method for conventional, wide-angle, and fish-eye lenses》在这里我把原文贴上，有需要可以直接拿取：《A generic camera model and calibration method for conventional, wide-angle, and fish-eye lenses》

这个成像模型可以对大于90度的相机进行适合的数学建模，OpenCV的fisheye模块也采用此篇论文的设计。

image-20240630195558301

参考OpenCV doc并结合论文，可以直接了解这个成像模型，后续也将在OpenGL里实现这个模型。

GLSL实现K-B鱼眼相机模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
//顶点着色器
#version 330 core
out vec4 FragColor;

in vec3 ourColor;
in vec2 TexCoord;

// texture sampler
uniform sampler2D texture1;

void main()
{
	FragColor = texture(texture1, TexCoord);
}


//段着色器
#version 330 core

uniform sampler2D inputTexture;
uniform float windowWidth;
uniform float windowHeight;
uniform mat3 cameraMatrix;
uniform vec4 distortVector;
uniform float textureWidth;
uniform float textureHeight;

in vec2 TexCoord;
out vec4 fragColor;

void undistortFisheye()
{
    float i = TexCoord.x * windowWidth;
    float j = TexCoord.y * windowHeight;
    float xc = (i - cameraMatrix[0][2]) / cameraMatrix[0][0];
    float yc = (j - cameraMatrix[1][2]) / cameraMatrix[1][1];
    float r = sqrt(xc * xc + yc * yc);
    float degree = atan(r);
    float rd = degree + distortVector[0] * pow(degree, 3) + distortVector[1] * pow(degree, 5) + distortVector[2] * pow(degree, 7) + distortVector[3] * pow(degree, 9);
    float scale = rd / r;
    float x = xc * scale;
    float y = yc * scale;
    float u = cameraMatrix[0][0] * (x + 0.01 * y) + cameraMatrix[0][2];
    float v = cameraMatrix[1][1] * y + cameraMatrix[1][2];
    vec2 coords = vec2(u / textureWidth, v / textureHeight);
    if (coords.x >= 0.0 && coords.x <= 1.0 && coords.y >= 0.0 && coords.y <= 1.0) {
        fragColor = texture(inputTexture, coords);
    } 
    else 
    {
        fragColor = vec4(0.0, 0.0, 0.0, 0.0);
    }
}

void main() {
    undistortFisheye();
}

上面是段着色器，虽然非常不推荐在着色器里使用if语句，但是没有找到更好的办法去搞归一化。暂且先这样，后续如果想到法子会进行优化。

传参说明：

uniform mat3 cameraMatrix;代表相机的内参矩阵，它是一个3x3的矩阵
uniform vec4 distortVector;代表KB相机模型代表的畸变系数D，它是一个有四个元素的向量

这两个参数都是作为uniform类型的变量从外部传进去的。当然，为了获得这两个参数需要对相机进行标定，标定的方法见我另一篇博客。

如何传入相机内参和畸变系数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
const unsigned int SCR_WIDTH = 800;
const unsigned int SCR_HEIGHT = 600;

int main()
{
	// glfw: initialize and configure
	// ------------------------------
	glfwInit();
	glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
	glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
	glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);

#ifdef __APPLE__
	glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);
#endif

	// glfw window creation
	// --------------------
	GLFWwindow *window = glfwCreateWindow(SCR_WIDTH, SCR_HEIGHT, "Undistort Fisheye", NULL, NULL);
	if (window == NULL)
	{
		std::cout << "Failed to create GLFW window" << std::endl;
		glfwTerminate();
		return -1;
	}
	glfwMakeContextCurrent(window);
	glfwSetFramebufferSizeCallback(window, framebuffer_size_callback);

	// glad: load all OpenGL function pointers
	if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress))
	{
		std::cout << "Failed to initialize GLAD" << std::endl;
		return -1;
	}

	// build and compile our shader program
	Shader ourShader("../data/shader_file/fisheye.vs", "../data/shader_file/undistort_fisheye.fs");

	// set up vertex data (and buffer(s)) and configure vertex attributes
    float vertices[] = {
        // positions          // colors               // texture coords
        0.9f,  0.9f,  0.0f,   1.0f, 0.0f, 0.0f,        1.0f, 1.0f,     // top right
        0.9f, -0.9f, 0.0f,    0.0f, 1.0f, 0.0f,        1.0f, 0.75f,    // bottom right
       -0.9f, -0.9f, 0.0f,    0.0f, 0.0f, 1.0f,        0.0f, 0.75f,    // bottom left
       -0.9f,  0.9f,  0.0f,   1.0f, 1.0f, 0.0f,        0.0f, 1.0f      // top left
    };
	float vertices_undistort[] = {
        // positions          // colors               // texture coords
        0.9f,  0.9f,  0.0f,   1.0f, 0.0f, 0.0f,        1.0f, 1.0f,    // top right
        0.9f, -0.9f, 0.0f,    0.0f, 1.0f, 0.0f,        1.0f, 0.0f,    // bottom right
       -0.9f, -0.9f, 0.0f,    0.0f, 0.0f, 1.0f,        0.0f, 0.0f,    // bottom left
       -0.9f,  0.9f,  0.0f,   1.0f, 1.0f, 0.0f,        0.0f, 1.0f     // top left
    };
	unsigned int indices[] = {
		0, 1, 3, // first triangle
		1, 2, 3	 // second triangle
	};
	unsigned int VBO, VAO, EBO;
	glGenVertexArrays(1, &VAO);
	glGenBuffers(1, &VBO);
	glGenBuffers(1, &EBO);
	glBindVertexArray(VAO);
	glBindBuffer(GL_ARRAY_BUFFER, VBO);
	glBufferData(GL_ARRAY_BUFFER, sizeof(vertices_undistort), vertices_undistort, GL_STATIC_DRAW);
	glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
	glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW);
	// position attribute
	glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(float), (void *)0);
	glEnableVertexAttribArray(0);
	// color attribute
	glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(float), (void *)(3 * sizeof(float)));
	glEnableVertexAttribArray(1);
	// texture coord attribute
	glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(float), (void *)(6 * sizeof(float)));
	glEnableVertexAttribArray(2);
	// load and create a texture 这个是第一个纹理，原图
	unsigned int texture1, texture2, texture3;
	glGenTextures(1, &texture1);
	glBindTexture(GL_TEXTURE_2D, texture1); // all upcoming GL_TEXTURE_2D operations now have effect on this texture object
	// set the texture wrapping parameters
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); // set texture wrapping to GL_REPEAT (default wrapping method)
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
	// set texture filtering parameters
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

	// load image, create texture and generate mipmaps
		int width, height, nrChannels;
		cv::Mat image = import_image("../data/raw/2023_04_22_14_52_31.bmp");
		cv::Mat single_image;
		single_image = image(cv::Rect(0, 0, 1280, 960));
		cv::flip(single_image, single_image, 0);

		width = single_image.cols;
		height = single_image.rows;
		nrChannels = single_image.channels();
		std::string params_file = "../data/params/2023_04_22_14_52_31_camera_params.txt";
		std::vector<cv::Mat> k_and_d_params;
		get_camera_para_and_distortion(params_file, k_and_d_params);
		get_new_camera_matrix(k_and_d_params[0],1,1,0,0);		
		unsigned char *data = single_image.data;

		if (data)
		{
			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_BGR, GL_UNSIGNED_BYTE, data);
			glGenerateMipmap(GL_TEXTURE_2D);
		}
		else
		{
			std::cout << "Failed to load texture" << std::endl;
		}


// tell opengl for each sampler to which texture unit it belongs to (only has to be done once)
    // -------------------------------------------------------------------------------------------
    ourShader.use(); // don't forget to activate/use the shader before setting uniforms!
    // either set it manually like so:
    glUniform1i(glGetUniformLocation(ourShader.ID, "texture1"), 0);

	// render loop
	while (!glfwWindowShouldClose(window))
	{
		processInput(window);
		glClearColor(0.2f, 0.3f, 0.3f, 1.0f);
		glClear(GL_COLOR_BUFFER_BIT);


		// bind textures on corresponding texture units
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, texture1);
        glActiveTexture(GL_TEXTURE1);
        glBindTexture(GL_TEXTURE_2D, texture2);
		
ourShader.use();
// 在设置着色器程序之前，获取uniform变量的位置
GLuint inputTextureLoc = glGetUniformLocation(ourShader.ID, "inputTexture");
GLuint windowWidthLoc = glGetUniformLocation(ourShader.ID, "windowWidth");
GLuint windowHeightLoc = glGetUniformLocation(ourShader.ID, "windowHeight");
GLuint cameraMatrixLoc = glGetUniformLocation(ourShader.ID, "cameraMatrix");
GLuint distortVectorLoc = glGetUniformLocation(ourShader.ID, "distortVector");
GLuint textureWidthLoc = glGetUniformLocation(ourShader.ID, "textureWidth");
GLuint textureHeightLoc = glGetUniformLocation(ourShader.ID, "textureHeight");

// 设置外部矩形大小参数
glUniform2f(glGetUniformLocation(ourShader.ID, "externalRectangleSize"), rectangleWidth, rectangleHeight);

// 设置矩形位置和透明度参数
glUniform2f(glGetUniformLocation(ourShader.ID, "rectanglePosition"), rectangleX, rectangleY);
glUniform2f(glGetUniformLocation(ourShader.ID, "rectangleSize"), rectangleWidth, rectangleHeight);
glUniform1f(glGetUniformLocation(ourShader.ID, "rectangleAlpha"), rectangleAlpha);

// 设置纹理单元
GLuint textureUnit = 0; // 假设纹理单元为0
glUniform1i(inputTextureLoc, textureUnit);

float windowWidth = 1280.0f; // 窗口宽度
float windowHeight = 960.0f; // 窗口高度

glm::mat3 cameraMatrix(
    k_and_d_params[0].at<double>(0,0), k_and_d_params[0].at<double>(0,1),k_and_d_params[0].at<double>(0,2),
    k_and_d_params[0].at<double>(1,0), k_and_d_params[0].at<double>(1,1), k_and_d_params[0].at<double>(1,2),
    k_and_d_params[0].at<double>(2,0), k_and_d_params[0].at<double>(2,1), k_and_d_params[0].at<double>(2,2)
);


glm::vec4 distortVector;
distortVector.x = k_and_d_params[1].at<double>(0,0);
distortVector.y = k_and_d_params[1].at<double>(0,1);
distortVector.z = k_and_d_params[1].at<double>(0,2);
distortVector.w = k_and_d_params[1].at<double>(0,3);

float textureWidth = 1280.0f; // 纹理宽度
float textureHeight = 960.0f; // 纹理高度

// 将uniform变量的值传递给着色器
glUniform1f(windowWidthLoc, windowWidth);
glUniform1f(windowHeightLoc, windowHeight);
glUniformMatrix3fv(cameraMatrixLoc, 1, GL_FALSE, glm::value_ptr(cameraMatrix));
glUniform4fv(distortVectorLoc, 1, glm::value_ptr(distortVector));
glUniform1f(textureWidthLoc, textureWidth);
glUniform1f(textureHeightLoc, textureHeight);

		// render container
		ourShader.use();
		glBindVertexArray(VAO);
		glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);

		// glfw: swap buffers and poll IO events (keys pressed/released, mouse moved etc.)
		// -------------------------------------------------------------------------------
		glfwSwapBuffers(window);
		glfwPollEvents();
	}

	// optional: de-allocate all resources once they've outlived their purpose:
	// ------------------------------------------------------------------------
	glDeleteVertexArrays(1, &VAO);
	glDeleteBuffers(1, &VBO);
	glDeleteBuffers(1, &EBO);

	// glfw: terminate, clearing all previously allocated GLFW resources.
	// ------------------------------------------------------------------
	glfwTerminate();
	return 0;
}

通过传入uniform类型的参数，将相机的内参和畸变系数送进shader，在GPU中统一进行处理，能够极大加速去畸变的速度，相较于opencv的方法，它提速了十倍以上。当然，这个例程是一个简单的程序，仅作学习层面的参考，实际的工程化方面还有非常多的考量。

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

继续努力<br>只做了一点微小的工作<br>谢谢大家<br>欢迎您给我发邮件沟通交流<br>ctysimple@qq.com<br>微信：CHENTY4869