|
多视图几何三维重建的基本原理：
从两个或者多个视点观察同一景物，已获得在多个不同的视角下对景物的多张感知图像，运用三角测量的基本原理计算图像像素间位置偏差，获得景物的三维深度信息，这一个过程与人类观察外面的世界的过程是一样的。
SfM：
SfM的全称为Structure from Motion，即通过相机的移动来确定目标的空间和几何关系，是三维重建的一种常见方法。它只需要普通的RGB摄像头，因此成本更低廉，且受环境约束较小，在室内和室外均能使用。但是，SfM背后需要复杂的理论和算法做支持，在精度和速度上都还有待提高，所以目前成熟的商业应用并不多。
本文将实现一个简易的增量式SfM系统。
书籍：计算机视觉中的多视图几何：原书第2版/（澳）理查德\cdot哈特利 北京：机械工业出版社，2019.8
代码参考：OpenCV实现SfM（二）：多目三维重建_arag2009的学习专栏-CSDN博客
数据集来源：MVS Data Set – 2014 | DTU Robot Image Data Sets
对极几何&基础矩阵
特征点提取和匹配
从上面的分析可知，要求取两个相机的相对关系，需要两幅图像中的对应点，这就变成的特征点的提取和匹配问题。
对于图像差别较大的情况，推荐使用SIFT特征，因为SIFT对旋转、尺度、透视都有较好的鲁棒性。(如果差别不大，可以考虑其他更快速的特征，比如SURF、ORB等。 )，然后采用knn最近邻算法进行匹配，距离计算采用欧氏距离。
由于OpenCV将SIFT包含在了扩展部分中，所以官网上下载的版本是没有SIFT的，
为此需要到GitHub - opencv/opencv_contrib: Repository for OpenCV's extra modules下载扩展包opencv_contrib，并按照里面的说明重新编译OpenCV。
编译opencv的时候要通过cmake -DOPENCV_EXTRA_MODULES_PATH将opencv_contrib的module编译进来，一定要保持两者的版本一致
opencv链接：GitHub - opencv/opencv: Open Source Computer Vision Library
重点说明的是，匹配结果往往有很多误匹配，为了排除这些错误，这里使用了Ratio Test方法，即使用KNN算法寻找与该特征最匹配的2个特征，若第一个特征的匹配距离与第二个特征的匹配距离之比小于某一阈值，就接受该匹配，否则视为误匹配。
void match_features(
cv::Mat & query,
cv::Mat & train,
std::vector& matches,
std::vector& knn_matches
)
{
cv::BFMatcher matcher(cv::NORM_L2);
matcher.knnMatch(query, train, knn_matches, 2);
//获取满足Ratio Test的最小匹配的距离
float min_dist = FLT_MAX;
for (int r = 0; r < knn_matches.size(); ++r)
{
//Ratio Test
if (knn_matches[r][0].distance > 0.6*knn_matches[r][1].distance)
continue;
float dist = knn_matches[r][0].distance;
if (dist < min_dist) min_dist = dist;
}
matches.clear();
for (size_t r = 0; r < knn_matches.size(); ++r)
{
//排除不满足Ratio Test的点和匹配距离过大的点
if (
knn_matches[r][0].distance > 0.6*knn_matches[r][1].distance ||
knn_matches[r][0].distance > 5 * cv::max(min_dist, 10.0f)
)
continue;
//保存匹配点
matches.push_back(knn_matches[r][0]);
}
}
之后可以参考《计算机视觉中的多视图几何》第11章 基本矩阵估计 算法 11.4
std::vector knn_matches;
match_features(this->leftimage->descriptor, this->rightimage->descriptor, m_matchs,knn_matches);//上文的匹配函数
auto& kp1 = this->leftimage->keypoints;
auto& kp2 = this->rightimage->keypoints;
cv::Mat m_Fundamental;
ransace_refine_matchs(kp1, kp2, m_matchs,m_Fundamental);//这里采用基于RANSAC的基础矩阵估计进行匹配点对的refine
int last_num = 0;
int next_num = 0;
double T = 10;
cv::Mat x1, x2;
cv::Point2f p1, p2;
double d;
int index_kp2;
do {
last_num = m_matchs.size();
std::vector label1(kp1.size(), -1);
std::vector label2(kp2.size(), -1);
for (int i = 0; i < m_matchs.size(); i++) {
label1[m_matchs[i].queryIdx] = 1;
label2[m_matchs[i].trainIdx] = 1;
}
for (int i = 0; i < knn_matches.size(); i++) {
if (label1[i] < 0) {
index_kp2 = knn_matches[i][0].trainIdx;
if (label2[index_kp2] < 0) {
p1 = kp1[knn_matches[i][0].queryIdx].pt;
p2 = kp2[index_kp2].pt;
x1 = (cv::Mat_(3, 1) SetBackground(0, 0, 0);//设置背景颜色
renderWindow->Render();
renderWindow->SetSize(800, 800);//设置窗口大小
renderWindowInteractor->Start();
vtkSmartPointer writer =
vtkSmartPointer::New();
writer->SetFileName("polyData.vtp");
writer->SetInputData(polyData);
writer->Write();
}
MultiViewStereoReconstructor.h
#pragma once
#ifndef MULTIVIEWSTEREORECONSTRUCTOR
#define MULTIVIEWSTEREORECONSTRUCTOR
#include
#include
#include
#include
class MyKeyPoint :public cv::KeyPoint {
public:
MyKeyPoint() {};
MyKeyPoint(cv::Point2f _pt, std::vector _color = std::vector(),
float _size = 1, float _angle = -1, float _response = 0, int _octave = 0, int _class_id = -1)
: cv::KeyPoint(_pt,_size,_angle,_response,_octave,_class_id),color(_color){};
MyKeyPoint(cv::KeyPoint _keypoint) :cv::KeyPoint(_keypoint) {};
MyKeyPoint(cv::KeyPoint _keypoint,cv::Vec3b _color) :cv::KeyPoint(_keypoint) {
setColor(_color);
};
MyKeyPoint(cv::KeyPoint _keypoint,std::vector _color) :cv::KeyPoint(_keypoint),color(_color) {};
void setColor(cv::Vec3b _color);
~MyKeyPoint() {};
bool operator==(MyKeyPoint &that);
public:
std::vector color;
};
class MyPoint3f :public cv::Point3f {
public:
MyPoint3f() {};
MyPoint3f(cv::Point3f _p) :cv::Point3f(_p) {};
MyPoint3f(cv::Point3f _p,cv::Vec3b _color) :cv::Point3f(_p) {
setColor(_color);
};
MyPoint3f(cv::Point3f _p,std::vector _color) :cv::Point3f(_p),color(_color) {};
void setColor(cv::Vec3b _color);
public:
std::vector color;
};
class UnitView {
public:
UnitView(cv::Mat& _image) {
image = _image.clone();
};
~UnitView() {};
bool extract_feature(cv::Ptr _executor);
public:
cv::Mat image;
cv::Mat descriptor;
std::vector keypoints;
cv::Mat rotation;//旋转矩阵
cv::Mat motion;
};
class PointCloudModel {
public:
PointCloudModel() {};
~PointCloudModel() {};
public:
std::vector correspond_struct_idx;
std::vector structure;
cv::Mat K;
cv::Mat distCoeffs;
};
class MyMatch {
public:
MyMatch(int _type=1): match_type(_type){};
~MyMatch() {};
bool SetUp();
bool Build();
bool init_structure();
void match_features(cv::Mat& query, cv::Mat& train,
std::vector& matches,
std::vector& knn_matches);
void ransace_refine_matchs(std::vector& kp1, std::vector & kp2,
std::vector& matchs,cv::Mat& m_Fundamental);
bool find_transform(cv::Mat& K, std::vector& p1, std::vector& p2, cv::Mat& R, cv::Mat& T, cv::Mat& mask);
void maskout_points(std::vector& p1, cv::Mat& mask);
void reconstruct(
cv::Mat& K,
cv::Mat& R1, cv::Mat& T1,
cv::Mat& R2, cv::Mat& T2,
std::vector& p1, std::vector& p2,
std::vector& structure);
void get_objpoints_and_imgpoints(
std::vector& matches,
std::vector& last_struct_indices,
std::vector& global_structure,
std::vector& key_points,
std::vector& object_points,
std::vector& image_points);
void fusion_structure(
std::vector& matches,
std::vector& last_struct_indices,
std::vector& next_struct_indices,
std::vector& global_structure,
std::vector& local_structure);
cv::Mat get_disparity(cv::Mat& img1,cv::Mat& img2);
public:
std::shared_ptr leftimage;
std::shared_ptr rightimage;
std::shared_ptr global_model;
int match_type = 1;//0表示这个匹配是第一张图像和第二张图像的匹配，否则为1; 默认为1;
private:
std::vector m_structure;
std::vector m_matchs;
std::vector m1;
std::vector m2;
};
class MultiViewStereoReconstructor {
public:
MultiViewStereoReconstructor() {};
~MultiViewStereoReconstructor() {};
void SetK(cv::Mat& _K) { K = _K.clone(); };
void SetDistCoeffs(cv::Mat _distCoeffs) { distCoeffs = _distCoeffs.clone(); };
void SetData(std::vector& _images);
bool Execute();
void Visualize();
public:
std::shared_ptr global_model;
std::vector images;
std::vector matchs;
cv::Mat K;
cv::Mat distCoeffs;
};
#endif // !MULTIVIEWSTEREORECONSTRUCTOR
MultiViewStereoReconstructor.cpp
#pragma once
#include "MultiViewStereoReconstructor.h"
#include
#include "GlobalVTKUtils.h"
#include
#include
void MyKeyPoint::setColor(cv::Vec3b _color)
{
this->color.resize(3);
for (int i = 0; i < 3; i++)this->color[i] = _color[i];
}
bool MyKeyPoint::operator==(MyKeyPoint & that)
{
return this->pt == that.pt;
}
void MyPoint3f::setColor(cv::Vec3b _color)
{
this->color.resize(3);
for (int i = 0; i < 3; i++)this->color[i] = _color[i];
}
bool UnitView::extract_feature(cv::Ptr _executor)
{
std::vector _keypoints;
_executor->detectAndCompute(this->image, cv::noArray(), _keypoints, this->descriptor);
if (_keypoints.size() keypoints.push_back(MyKeyPoint(key, this->image.at(key.pt.y, key.pt.x)));
}
}
bool MyMatch::SetUp()
{
//std::cout leftimage->descriptor, this->rightimage->descriptor, m_matchs,knn_matches);
auto& kp1 = this->leftimage->keypoints;
auto& kp2 = this->rightimage->keypoints;
cv::Mat m_Fundamental;
ransace_refine_matchs(kp1, kp2, m_matchs,m_Fundamental);
#if 1
int last_num = 0;
int next_num = 0;
double T = 10;
cv::Mat x1, x2;
cv::Point2f p1, p2;
double d;
int index_kp2;
do {
last_num = m_matchs.size();
std::vector label1(kp1.size(), -1);
std::vector label2(kp2.size(), -1);
for (int i = 0; i < m_matchs.size(); i++) {
label1[m_matchs[i].queryIdx] = 1;
label2[m_matchs[i].trainIdx] = 1;
}
for (int i = 0; i < knn_matches.size(); i++) {
if (label1[i] < 0) {
index_kp2 = knn_matches[i][0].trainIdx;
if (label2[index_kp2] < 0) {
p1 = kp1[knn_matches[i][0].queryIdx].pt;
p2 = kp2[index_kp2].pt;
x1 = (cv::Mat_(3, 1) extract_feature(_executor);
}
matchs.resize(images.size() - 1);
}
bool MultiViewStereoReconstructor::Execute() {
if (K.empty()) {
std::cout K = K.clone();
global_model->distCoeffs = distCoeffs.clone();
//#pragma omp parallel for num_threads(2*omp_get_num_procs()-1)
for (int i = 0; i < images.size() - 1; i++) {
MyMatch &match = matchs[i];
if (i == 0) {
match.match_type = 0;
}
match.leftimage = images[i];
match.rightimage = images[i + 1];
match.global_model = global_model;
//std::cout |