ACloudViewer  3.9.4
A Modern Library for 3D Data Processing
texturing.cc
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - CloudViewer: www.cloudViewer.org -
3 // ----------------------------------------------------------------------------
4 // Copyright (c) 2018-2024 www.cloudViewer.org
5 // SPDX-License-Identifier: MIT
6 // ----------------------------------------------------------------------------
7 
8 #include "mvs/texturing.h"
9 
10 #include <QImage>
11 #include <QImageReader>
12 #include <QPainter>
13 #include <algorithm>
14 #include <cmath>
15 #include <fstream>
16 #include <iomanip>
17 #include <limits>
18 #include <list>
19 #include <map>
20 #include <numeric>
21 #include <set>
22 #include <unordered_map>
23 #include <unordered_set>
24 
25 #include "base/reconstruction.h"
26 #include "mvs/workspace.h"
27 #include "util/logging.h"
28 #include "util/misc.h"
29 
30 // CV_CORE_LIB
31 #include <CVLog.h>
32 #include <CVTools.h>
33 #include <FileSystem.h>
34 
35 // CV_DB_LIB
37 
38 // CV_IO_LIB
39 #include <AutoIO.h>
40 #include <ImageIO.h>
41 #include <ObjFilter.h>
42 #include <ecvHObjectCaster.h>
43 #include <ecvMaterial.h>
44 #include <ecvMaterialSet.h>
45 #include <ecvMesh.h>
46 #include <ecvPointCloud.h>
47 
48 namespace colmap {
49 namespace mvs {
50 
51 using colmap::image_t;
52 
53 // Implementations of internal structures
54 TextureView::TextureView(std::size_t id,
55  const std::string& image_file,
56  const Eigen::Matrix3f& projection,
57  const Eigen::Matrix4f& world_to_cam,
58  const Eigen::Vector3f& pos,
59  const Eigen::Vector3f& viewdir,
60  int width,
61  int height)
62  : id(id),
63  image_file(image_file),
64  width(width),
65  height(height),
66  projection(projection),
67  world_to_cam(world_to_cam),
68  pos(pos),
69  viewdir(viewdir) {}
70 
72  const Eigen::Vector3f& vertex) const {
73  // Project vertex from world coordinates to image coordinates
74  // P = K * [R|t] * [X; Y; Z; 1]
75  // where [R|t] is world_to_cam (3x4), K is projection (3x3)
76  Eigen::Vector4f vertex_homogeneous(vertex.x(), vertex.y(), vertex.z(),
77  1.0f);
78 
79  // First transform to camera coordinates: [R|t] * [X; Y; Z; 1]
80  Eigen::Vector3f cam_coords =
81  world_to_cam.block<3, 4>(0, 0) * vertex_homogeneous;
82 
83  // Check if point is behind camera
84  if (cam_coords.z() <= 0) {
85  return Eigen::Vector2f(-1, -1);
86  }
87 
88  // Project to image plane: K * [x_cam; y_cam; z_cam]
89  Eigen::Vector3f proj = projection * cam_coords;
90 
91  // Normalize by depth to get pixel coordinates
92  Eigen::Vector2f pixel(proj.x() / proj.z(), proj.y() / proj.z());
93 
94  // Following mvs-texturing: pixel coordinates are centered (pixel[0] - 0.5,
95  // pixel[1] - 0.5) This means pixel (0,0) corresponds to center of top-left
96  // pixel
97  return Eigen::Vector2f(pixel.x() - 0.5f, pixel.y() - 0.5f);
98 }
99 
100 bool TextureView::ValidPixel(const Eigen::Vector2f& pixel) const {
101  return pixel.x() >= 0 && pixel.x() < width - 1 && pixel.y() >= 0 &&
102  pixel.y() < height - 1;
103 }
104 
105 bool TextureView::Inside(const Eigen::Vector3f& v1,
106  const Eigen::Vector3f& v2,
107  const Eigen::Vector3f& v3) const {
108  Eigen::Vector2f p1 = GetPixelCoords(v1);
109  Eigen::Vector2f p2 = GetPixelCoords(v2);
110  Eigen::Vector2f p3 = GetPixelCoords(v3);
111  return ValidPixel(p1) && ValidPixel(p2) && ValidPixel(p3);
112 }
113 
115  const std::vector<size_t>& faces,
116  const std::vector<Eigen::Vector2f>& texcoords,
117  std::shared_ptr<QImage> image,
118  int min_x,
119  int min_y,
120  int max_x,
121  int max_y)
122  : label(label),
123  faces(faces),
124  texcoords(texcoords),
125  image(image),
126  min_x(min_x),
127  min_y(min_y),
128  max_x(max_x),
129  max_y(max_y) {}
130 
132  const colmap::Reconstruction& reconstruction,
133  colmap::mvs::Workspace* workspace,
134  const std::string& image_path)
135  : options_(options),
136  reconstruction_(reconstruction),
137  workspace_(workspace),
138  image_path_(image_path) {}
139 
140 int MvsTexturing::GetWorkspaceImageIdx(const colmap::image_t image_id) const {
141  if (!workspace_) {
142  return -1;
143  }
144  const colmap::Image& image = reconstruction_.Image(image_id);
145  const std::string& image_name = image.Name();
146 
147  const auto& model = workspace_->GetModel();
148 
149  // GetImageIdx uses CHECK_GT and .at() which throws exception if not found
150  // Use try-catch to safely handle the case when image doesn't exist
151  try {
152  int workspace_image_idx = model.GetImageIdx(image_name);
153 
154  // Validate the index is within valid range
155  if (workspace_image_idx < 0 ||
156  static_cast<size_t>(workspace_image_idx) >= model.images.size()) {
157  return -1;
158  }
159 
160  return workspace_image_idx;
161  } catch (const std::exception&) {
162  // Image not found in workspace model or index out of range
163  return -1;
164  }
165 }
166 
167 bool MvsTexturing::IsPointVisible(const Eigen::Vector3d& point3d,
168  const colmap::Image& image,
169  const colmap::Camera& camera,
170  int workspace_image_idx) const {
171  if (!options_.use_depth_normal_maps || !workspace_ ||
172  workspace_image_idx < 0) {
173  return true;
174  }
175 
176  // Validate index is within model range before accessing cache
177  const auto& model = workspace_->GetModel();
178  if (static_cast<size_t>(workspace_image_idx) >= model.images.size()) {
179  return true; // Invalid index, assume visible
180  }
181 
182  if (!workspace_->HasDepthMap(workspace_image_idx)) {
183  return true;
184  }
185 
186  try {
187  const auto& depth_map = workspace_->GetDepthMap(workspace_image_idx);
188 
189  const Eigen::Matrix3x4d proj_matrix = image.ProjectionMatrix();
190  const Eigen::Vector3d proj = proj_matrix * point3d.homogeneous();
191 
192  if (proj(2) <= 0) {
193  return false;
194  }
195 
196  const double u = proj(0) / proj(2);
197  const double v = proj(1) / proj(2);
198 
199  if (u < 0 || u >= camera.Width() || v < 0 || v >= camera.Height()) {
200  return false;
201  }
202 
203  const size_t row = static_cast<size_t>(std::round(v));
204  const size_t col = static_cast<size_t>(std::round(u));
205 
206  if (row >= depth_map.GetHeight() || col >= depth_map.GetWidth()) {
207  return false;
208  }
209 
210  const float depth_map_value = depth_map.Get(row, col);
211  const float point_depth = static_cast<float>(proj(2));
212 
213  if (depth_map_value <= 0) {
214  return false;
215  }
216 
217  // Use very lenient depth check: allow larger depth error
218  // mvs-texturing doesn't use depth maps for visibility by default
219  // We use it as an optional filter, so be very lenient
220  const float depth_error =
221  std::abs(point_depth - depth_map_value) / depth_map_value;
222  // Increase tolerance: use 10x the configured error threshold (was 5x,
223  // but still too strict) Also add absolute tolerance: allow up to 0.1 *
224  // depth_map_value absolute error
225  const float relative_tolerance = options_.max_depth_error * 10.0f;
226  const float absolute_tolerance = 0.1f; // 10% absolute tolerance
227  return depth_error <= relative_tolerance ||
228  std::abs(point_depth - depth_map_value) <=
229  depth_map_value * absolute_tolerance;
230  } catch (const std::exception&) {
231  // Error accessing depth map, assume visible
232  return true;
233  }
234 }
235 
237  ccMesh& mesh,
238  const ::cloudViewer::camera::PinholeCameraTrajectory& camera_trajectory,
239  const std::string& output_path) {
240  if (options_.verbose) {
241  std::cout << StringPrintf("Starting mvs-texturing based texture mapping...\n");
242  }
243 
244  // Verify mesh has associated cloud
245  ccGenericPointCloud* generic_cloud = mesh.getAssociatedCloud();
246  if (!generic_cloud) {
247  std::cerr << "ERROR: " << StringPrintf(
248  "Mesh has no associated cloud! "
249  "This may be due to mesh merge failure. "
250  "Please ensure the mesh file has valid vertices.\n");
251  return false;
252  }
253 
254  ccPointCloud* cloud = ccHObjectCaster::ToPointCloud(generic_cloud);
255  if (!cloud || cloud->size() == 0) {
256  std::cerr << "ERROR: " << StringPrintf("Mesh has invalid or empty associated cloud!\n");
257  return false;
258  }
259 
260  if (mesh.size() == 0) {
261  std::cerr << "ERROR: " << StringPrintf("Mesh has no triangles!\n");
262  return false;
263  }
264 
265  if (options_.verbose) {
266  std::cout << StringPrintf("Using mesh: %u vertices, %u triangles\n", cloud->size(),
267  mesh.size());
268  }
269 
270  // Ensure normals are available (following mvs-texturing: prepare_mesh ->
271  // mesh->ensure_normals(true, true)) This should be done right after loading
272  // mesh, before processing
273  if (!cloud->hasNormals()) {
274  if (options_.verbose) {
275  std::cout << StringPrintf(
276  "Mesh has no normals, computing per-vertex normals...\n");
277  }
278  {
279  mesh.ComputeVertexNormals();
280  // Refresh cloud pointer after computing normals
281  generic_cloud = mesh.getAssociatedCloud();
282  cloud = ccHObjectCaster::ToPointCloud(generic_cloud);
283  if (!cloud) {
284  std::cerr << "ERROR: " << StringPrintf(
285  "Failed to get associated cloud after computing "
286  "normals!\n");
287  return false;
288  }
289  if (options_.verbose) {
290  std::cout << StringPrintf("Computed per-vertex normals for %u vertices",
291  cloud->size()) << "\n";
292  }
293  }
294  }
295 
296  // Create texture views
297  CreateTextureViews(camera_trajectory);
298 
299  if (texture_views_.empty()) {
300  std::cerr << "ERROR: " << StringPrintf("No valid texture views created!\n");
301  return false;
302  }
303 
304  if (options_.verbose) {
305  std::cout << StringPrintf("Created %zu texture views\n", texture_views_.size());
306  }
307 
308  // Build adjacency graph for mesh faces
309  BuildAdjacencyGraph(mesh);
310 
311  // Calculate data costs
312  CalculateDataCosts(mesh);
313 
314  // Select views using graph cut algorithm
315  SelectViews();
316 
317  // Generate texture patches
318  GenerateTexturePatches(mesh);
319 
320  // Seam leveling
321  SeamLeveling(mesh);
322 
323  // Generate texture atlases with packing
324  GenerateTextureAtlases();
325 
326  // Build and save OBJ model
327  if (!SaveOBJModel(output_path, mesh)) {
328  std::cerr << "ERROR: " << StringPrintf("Failed to save OBJ model to %s\n", output_path.c_str());
329  return false;
330  }
331 
332  return true;
333 }
334 
335 void MvsTexturing::CreateTextureViews(
336  const ::cloudViewer::camera::PinholeCameraTrajectory& camera_trajectory) {
337  texture_views_.clear();
338  view_to_image_id_.clear();
339  texture_views_.reserve(camera_trajectory.parameters_.size());
340  view_to_image_id_.reserve(camera_trajectory.parameters_.size());
341 
342  // Try to find image_id by matching texture_file_ with image names
343  for (size_t i = 0; i < camera_trajectory.parameters_.size(); ++i) {
344  const auto& params = camera_trajectory.parameters_[i];
345 
346  // Find corresponding image_id by matching texture_file_ with image
347  // names
349  std::string texture_file_base =
350  colmap::GetPathBaseName(params.texture_file_);
351  for (const colmap::image_t& img_id : reconstruction_.RegImageIds()) {
352  const colmap::Image& img = reconstruction_.Image(img_id);
353  if (colmap::GetPathBaseName(img.Name()) == texture_file_base) {
354  image_id = img_id;
355  break;
356  }
357  }
358 
359  view_to_image_id_.push_back(image_id);
360 
361  // Get projection matrix directly from COLMAP Image if available
362  // This ensures we use the same coordinate system as COLMAP
363  Eigen::Matrix3f projection = Eigen::Matrix3f::Identity();
364  Eigen::Matrix4f world_to_cam = Eigen::Matrix4f::Identity();
365  Eigen::Vector3f pos;
366  Eigen::Vector3f viewdir;
367 
368  if (image_id != colmap::kInvalidImageId) {
369  // Use COLMAP Image's projection matrix directly
370  // Image::ProjectionMatrix() returns [R|t] (world_to_cam), NOT
371  // K*[R|t] This ensures we use the exact same coordinate system as
372  // COLMAP
373  const colmap::Image& img = reconstruction_.Image(image_id);
374  const Eigen::Matrix3x4d Rt_matrix = img.ProjectionMatrix();
375 
376  // Build intrinsic matrix K
377  projection(0, 0) = params.intrinsic_.GetFocalLength().first;
378  projection(1, 1) = params.intrinsic_.GetFocalLength().second;
379  projection(0, 2) = params.intrinsic_.GetPrincipalPoint().first;
380  projection(1, 2) = params.intrinsic_.GetPrincipalPoint().second;
381 
382  // Image::ProjectionMatrix() already returns [R|t] (world_to_cam)
383  // No need to extract it - use directly!
384  world_to_cam.block<3, 3>(0, 0) =
385  Rt_matrix.leftCols<3>().cast<float>();
386  world_to_cam.block<3, 1>(0, 3) =
387  Rt_matrix.rightCols<1>().cast<float>();
388  world_to_cam(3, 0) = 0.0f;
389  world_to_cam(3, 1) = 0.0f;
390  world_to_cam(3, 2) = 0.0f;
391  world_to_cam(3, 3) = 1.0f;
392 
393  // Camera position: C = -R^T * t (in world coordinates)
394  Eigen::Matrix3f R = world_to_cam.block<3, 3>(0, 0);
395  Eigen::Vector3f t = world_to_cam.block<3, 1>(0, 3);
396  pos = -R.transpose() * t;
397 
398  // Viewing direction: camera -Z axis in world coordinates = -R^T *
399  // [0, 0, 1]^T
400  viewdir = -R.transpose() * Eigen::Vector3f(0, 0, 1);
401  viewdir.normalize();
402  } else {
403  // Fallback: use params directly
404  // NOTE: params.extrinsic_ is cam_to_world (from
405  // InverseProjectionMatrix) So we need to invert it to get
406  // world_to_cam
407  projection(0, 0) = params.intrinsic_.GetFocalLength().first;
408  projection(1, 1) = params.intrinsic_.GetFocalLength().second;
409  projection(0, 2) = params.intrinsic_.GetPrincipalPoint().first;
410  projection(1, 2) = params.intrinsic_.GetPrincipalPoint().second;
411 
412  // params.extrinsic_ is cam_to_world, so invert to get world_to_cam
413  Eigen::Matrix4f cam_to_world = params.extrinsic_.cast<float>();
414  world_to_cam = cam_to_world.inverse();
415 
416  pos = Eigen::Vector3f(cam_to_world(0, 3), cam_to_world(1, 3),
417  cam_to_world(2, 3));
418  viewdir = -cam_to_world.block<3, 1>(0, 2).normalized();
419  }
420 
421  // Image file path
422  std::string image_file = params.texture_file_;
423 
424  // Try multiple path resolution strategies
425  if (!boost::filesystem::path(image_file).is_absolute()) {
426  // First try: join with image_path_
427  std::string candidate = colmap::JoinPaths(image_path_, image_file);
428  if (ExistsFile(candidate)) {
429  image_file = candidate;
430  } else {
431  // Second try: check if texture_file_ is relative to workspace
432  // The texture_file_ might already be correct relative path
433  // Try joining with workspace path if available
434  if (workspace_) {
435  std::string workspace_path =
436  workspace_->GetOptions().workspace_path;
437  candidate = colmap::JoinPaths(
438  workspace_path, "images",
439  colmap::GetPathBaseName(image_file));
440  if (ExistsFile(candidate)) {
441  image_file = candidate;
442  } else {
443  // Fallback: use original path (might be resolved later)
444  image_file = colmap::JoinPaths(image_path_, image_file);
445  }
446  } else {
447  image_file = candidate;
448  }
449  }
450  }
451 
452  // Verify file exists, log warning if not
453  if (!ExistsFile(image_file)) {
454  std::cerr << "WARNING: " << StringPrintf(
455  "Image file does not exist: %s (will try to load anyway)",
456  image_file.c_str()) << "\n";
457  }
458 
459  auto view = std::make_unique<TextureView>(
460  i, image_file, projection, world_to_cam, pos, viewdir,
461  params.intrinsic_.width_, params.intrinsic_.height_);
462 
463  texture_views_.push_back(std::move(view));
464  }
465 }
466 
467 void MvsTexturing::CalculateDataCosts(const ccMesh& mesh) {
468  if (options_.verbose) {
469  std::cout << StringPrintf("Calculating data costs for %zu faces...\n", mesh.size());
470  }
471 
472  ccPointCloud* cloud =
474  if (!cloud) {
475  std::cerr << "ERROR: " << StringPrintf("Mesh has no associated cloud!\n");
476  return;
477  }
478 
479  // Get global shift/scale for coordinate conversion
480  // CloudViewer uses local coordinates (shifted/scaled), but camera
481  // transforms are in global coordinates. We need to convert mesh vertices to
482  // global coordinates before projection.
483  bool mesh_is_shifted = cloud->isShifted();
484  CCVector3d mesh_global_shift = cloud->getGlobalShift();
485  double mesh_global_scale = cloud->getGlobalScale();
486 
487  if (options_.verbose && mesh_is_shifted) {
488  std::cout << StringPrintf("Mesh has global shift: (%.6f, %.6f, %.6f), scale: %.6f",
489  mesh_global_shift.x, mesh_global_shift.y,
490  mesh_global_shift.z, mesh_global_scale) << "\n";
491  }
492 
493  const unsigned num_faces = mesh.size();
494  face_projection_infos_.clear();
495  face_projection_infos_.resize(num_faces);
496 
497  // Statistics for debugging
498  size_t total_faces_processed = 0;
499  size_t faces_inside_view = 0;
500  size_t faces_backface_culled = 0;
501  size_t faces_viewing_angle_too_steep = 0;
502  size_t faces_behind_camera = 0;
503  size_t faces_depth_check_passed = 0;
504  size_t faces_depth_check_failed = 0;
505  size_t faces_zero_area = 0;
506  size_t faces_quality_ok = 0;
507 
508  // For each view, test visibility and quality for each face
509  for (size_t view_id = 0; view_id < texture_views_.size(); ++view_id) {
510  const auto& view = texture_views_[view_id];
511  colmap::image_t image_id = view_to_image_id_[view_id];
512 
513  if (image_id == colmap::kInvalidImageId) {
514  if (options_.verbose) {
515  std::cerr << "WARNING: " << StringPrintf("View %zu has invalid image_id, skipping",
516  view_id) << "\n";
517  }
518  continue;
519  }
520 
521  const colmap::Image& image = reconstruction_.Image(image_id);
522  const colmap::Camera& camera = reconstruction_.Camera(image.CameraId());
523  int workspace_image_idx = GetWorkspaceImageIdx(image_id);
524 
525  // Process each face
526  for (unsigned face_id = 0; face_id < num_faces; ++face_id) {
527  total_faces_processed++;
528 
529  Eigen::Vector3i tri_idx;
530  mesh.getTriangleVertIndexes(face_id, tri_idx);
531 
532  if (tri_idx(0) >= static_cast<int>(cloud->size()) ||
533  tri_idx(1) >= static_cast<int>(cloud->size()) ||
534  tri_idx(2) >= static_cast<int>(cloud->size())) {
535  continue;
536  }
537 
538  const CCVector3* v1_ptr = cloud->getPoint(tri_idx(0));
539  const CCVector3* v2_ptr = cloud->getPoint(tri_idx(1));
540  const CCVector3* v3_ptr = cloud->getPoint(tri_idx(2));
541 
542  // Convert from local coordinates to global coordinates if mesh is
543  // shifted Camera transforms are in global coordinate system
544  Eigen::Vector3f v1, v2, v3;
545  if (mesh_is_shifted) {
546  // Convert local to global: Pglobal = Plocal / scale - shift
547  CCVector3d v1_global = cloud->toGlobal3d(*v1_ptr);
548  CCVector3d v2_global = cloud->toGlobal3d(*v2_ptr);
549  CCVector3d v3_global = cloud->toGlobal3d(*v3_ptr);
550  v1 = Eigen::Vector3f(static_cast<float>(v1_global.x),
551  static_cast<float>(v1_global.y),
552  static_cast<float>(v1_global.z));
553  v2 = Eigen::Vector3f(static_cast<float>(v2_global.x),
554  static_cast<float>(v2_global.y),
555  static_cast<float>(v2_global.z));
556  v3 = Eigen::Vector3f(static_cast<float>(v3_global.x),
557  static_cast<float>(v3_global.y),
558  static_cast<float>(v3_global.z));
559  } else {
560  v1 = Eigen::Vector3f(v1_ptr->x, v1_ptr->y, v1_ptr->z);
561  v2 = Eigen::Vector3f(v2_ptr->x, v2_ptr->y, v2_ptr->z);
562  v3 = Eigen::Vector3f(v3_ptr->x, v3_ptr->y, v3_ptr->z);
563  }
564 
565  // Check if face projects into view
566  if (!view->Inside(v1, v2, v3)) {
567  continue;
568  }
569  faces_inside_view++;
570 
571  // Compute face center and normal (following mvs-texturing approach)
572  Eigen::Vector3f face_center = (v1 + v2 + v3) / 3.0f;
573  Eigen::Vector3f face_normal_vec = (v2 - v1).cross(v3 - v1);
574  float face_normal_norm = face_normal_vec.norm();
575  if (face_normal_norm < 1e-6f) {
576  continue; // Degenerate face (zero area)
577  }
578  Eigen::Vector3f face_normal =
579  face_normal_vec / face_normal_norm; // Normalize
580 
581  // Convert to double for visibility/quality testing
582  Eigen::Vector3d face_center_d = face_center.cast<double>();
583  Eigen::Vector3d face_normal_d = face_normal.cast<double>();
584 
585  // Following mvs-texturing: view filtering based on viewing angle
586  const Eigen::Vector3d camera_pos = image.ProjectionCenter();
587  Eigen::Vector3d view_to_face_vec =
588  (face_center_d - camera_pos).normalized();
589  Eigen::Vector3d face_to_view_vec =
590  (camera_pos - face_center_d).normalized();
591 
592  // Backface culling: viewing_angle < 0 means face is facing away
593  float viewing_angle =
594  static_cast<float>(face_to_view_vec.dot(face_normal_d));
595  if (viewing_angle < 0.0f) {
596  faces_backface_culled++;
597  continue; // Backface culling
598  }
599 
600  // Viewing angle limit: reject faces viewed from too steep angle
601  // (mvs-texturing uses 75 degrees)
602  const float max_viewing_angle_rad =
603  options_.max_viewing_angle_deg * M_PI / 180.0f;
604  if (std::acos(viewing_angle) > max_viewing_angle_rad) {
605  faces_viewing_angle_too_steep++;
606  continue; // Viewing angle too steep
607  }
608 
609  // Frustum culling: check if face is in viewing direction
610  Eigen::Vector3d viewing_direction =
611  image.RotationMatrix()
612  .row(2)
613  .transpose(); // Camera -Z axis
614  if (viewing_direction.dot(view_to_face_vec) < 0.0) {
615  faces_behind_camera++;
616  continue; // Face is behind camera
617  }
618 
619  // Test visibility using depth map (if enabled) - geometric
620  // visibility test Note: mvs-texturing's geometric_visibility_test
621  // is optional and uses BVH ray casting Depth map check is very
622  // lenient - we use it as a soft filter, not a hard requirement Most
623  // faces should pass this check to allow proper texturing If depth
624  // map check fails, we still allow the face (depth maps are
625  // optional)
626  bool visible = true;
627  if (options_.use_depth_normal_maps && workspace_image_idx >= 0) {
628  // Try depth map check, but don't reject if it fails
629  // Depth maps may have noise, alignment issues, or missing data
630  // We use it as a quality hint, not a strict filter
631  bool depth_visible = IsPointVisible(
632  face_center_d, image, camera, workspace_image_idx);
633  // Use depth check result as a hint - don't reject faces based
634  // on it This ensures we get enough faces for texturing (The
635  // lenient tolerance in IsPointVisible should allow most faces
636  // through anyway)
637  visible = depth_visible; // Use result, but lenient tolerance
638  // should allow most through
639  if (visible) {
640  faces_depth_check_passed++;
641  } else {
642  faces_depth_check_failed++;
643  }
644  }
645  // IMPORTANT: Don't skip faces even if depth check failed!
646  // Depth maps are optional and may have errors. We need to ensure
647  // every face has at least some candidate views for texturing.
648  // Continue processing even if depth check failed - this ensures
649  // faces have candidate views
650 
651  // Compute quality following mvs-texturing: use projection area or GMI
652  Eigen::Vector2f p1 = view->GetPixelCoords(v1);
653  Eigen::Vector2f p2 = view->GetPixelCoords(v2);
654  Eigen::Vector2f p3 = view->GetPixelCoords(v3);
655 
656  // Calculate projected triangle area
657  float area = 0.5f * std::abs((p2(0) - p1(0)) * (p3(1) - p1(1)) -
658  (p3(0) - p1(0)) * (p2(1) - p1(1)));
659 
660  // Use a very small threshold instead of epsilon to allow more faces
661  // Some faces may have very small projected area but still valid
662  const float min_area_threshold =
663  1e-6f; // Very small but larger than epsilon
664  if (area < min_area_threshold) {
665  faces_zero_area++;
666  continue; // Zero or near-zero area projection
667  }
668 
669  float quality;
670  if (options_.use_gradient_magnitude) {
671  // Use GMI (Gradient Magnitude Image) for quality
672  quality = CalculateGMIQuality(view_id, p1, p2, p3);
673  // Fallback to area if GMI fails
674  if (quality <= 0.0f) {
675  quality = area;
676  }
677  } else {
678  // Use area as quality (DATA_TERM_AREA)
679  quality = area;
680  }
681 
682  // If depth map check failed, reduce quality slightly to prefer
683  // views that pass depth check, but still allow the face to be
684  // textured IMPORTANT: Don't skip faces even if depth check failed!
685  if (!visible && options_.use_depth_normal_maps) {
686  quality *= 0.5f; // Reduce quality by 50% if depth check failed
687  }
688 
689  faces_quality_ok++;
690 
691  // Get mean color from image (for outlier detection)
692  // TODO: Sample pixels in triangle to get mean color (for outlier
693  // detection)
694  Eigen::Vector3f mean_color(0.5f, 0.5f,
695  0.5f); // Default gray for now
696 
697  FaceProjectionInfo info;
698  info.view_id = view_id;
699  info.quality = quality;
700  info.mean_color = mean_color;
701 
702  face_projection_infos_[face_id].push_back(info);
703  }
704 
705  if (options_.verbose && (view_id + 1) % 10 == 0) {
706  std::cout << StringPrintf("Processed %zu/%zu views...\n", view_id + 1,
707  texture_views_.size());
708  }
709  }
710 
711  if (options_.verbose) {
712  std::cout << StringPrintf(
713  "\nData cost calculation stats:\n"
714  " Total face-view pairs processed: %zu\n"
715  " Inside view: %zu\n"
716  " Backface culled: %zu\n"
717  " Viewing angle too steep: %zu\n"
718  " Behind camera: %zu\n"
719  " Depth check passed: %zu\n"
720  " Depth check not passed: %zu\n"
721  " Zero area projection: %zu\n"
722  " Quality OK (added to data costs): %zu\n",
723  total_faces_processed, faces_inside_view, faces_backface_culled,
724  faces_viewing_angle_too_steep, faces_behind_camera,
725  faces_depth_check_passed, faces_depth_check_failed,
726  faces_zero_area, faces_quality_ok);
727  }
728 
729  // Postprocess face infos following mvs-texturing approach
730  // 1. Sort infos by view_id (for consistency)
731  for (size_t face_id = 0; face_id < face_projection_infos_.size();
732  ++face_id) {
733  auto& infos = face_projection_infos_[face_id];
734  std::sort(infos.begin(), infos.end(),
735  [](const FaceProjectionInfo& a, const FaceProjectionInfo& b) {
736  return a.view_id < b.view_id;
737  });
738  }
739 
740  // 2. Find global max quality and calculate percentile (99.5% like
741  // mvs-texturing)
742  float max_quality = 0.0f;
743  std::vector<float> all_qualities;
744  for (const auto& infos : face_projection_infos_) {
745  for (const auto& info : infos) {
746  max_quality = std::max(max_quality, info.quality);
747  all_qualities.push_back(info.quality);
748  }
749  }
750 
751  // Calculate 99.5th percentile
752  float percentile = max_quality;
753  if (!all_qualities.empty()) {
754  std::sort(all_qualities.begin(), all_qualities.end());
755  size_t percentile_idx = static_cast<size_t>(
756  std::round(0.995f * (all_qualities.size() - 1)));
757  percentile_idx = std::min(percentile_idx, all_qualities.size() - 1);
758  percentile = all_qualities[percentile_idx];
759  }
760 
761  if (options_.verbose) {
762  std::cout << StringPrintf("Maximum quality: %.6f, 99.5%% percentile: %.6f",
763  max_quality, percentile) << "\n";
764  }
765 
766  // 3. Convert face_projection_infos_ to data_costs_ format
767  // Following mvs-texturing: clamp to percentile and normalize
768  data_costs_.clear();
769  data_costs_.resize(face_projection_infos_.size());
770  for (size_t face_id = 0; face_id < face_projection_infos_.size();
771  ++face_id) {
772  const auto& infos = face_projection_infos_[face_id];
773  data_costs_[face_id].reserve(infos.size());
774 
775  for (const auto& info : infos) {
776  // Clamp to percentile and normalize (following mvs-texturing)
777  float normalized_quality =
778  percentile > 0 ? std::min(1.0f, info.quality / percentile)
779  : 0.0f;
780  // Convert quality to cost: cost = 1.0 - normalized_quality
781  float cost = 1.0f - normalized_quality;
782  // Note: mvs-texturing uses view_id + 1 (label 0 is undefined)
783  // We'll keep view_id as-is for now, but SelectViews should handle
784  // label 0
785  data_costs_[face_id].emplace_back(info.view_id, cost);
786  }
787  }
788 
789  if (options_.verbose) {
790  size_t total_projections = 0;
791  for (const auto& infos : face_projection_infos_) {
792  total_projections += infos.size();
793  }
794  std::cout << StringPrintf("Calculated data costs: %zu total face-view projections",
795  total_projections) << "\n";
796  }
797 }
798 
799 void MvsTexturing::BuildAdjacencyGraph(const ccMesh& mesh) {
800  if (options_.verbose) {
801  std::cout << StringPrintf("Building adjacency graph for mesh faces...\n");
802  }
803 
804  const unsigned num_faces = mesh.size();
805  face_adjacency_.clear();
806  face_adjacency_.resize(num_faces);
807 
808  ccPointCloud* cloud =
810  if (!cloud) {
811  std::cerr << "ERROR: " << StringPrintf("Mesh has no associated cloud!\n");
812  return;
813  }
814 
815  // Build edge-to-faces map
816  std::map<std::pair<unsigned, unsigned>, std::vector<unsigned>>
817  edge_to_faces;
818  for (unsigned face_id = 0; face_id < num_faces; ++face_id) {
819  Eigen::Vector3i tri_idx;
820  mesh.getTriangleVertIndexes(face_id, tri_idx);
821 
822  unsigned v1 = tri_idx(0), v2 = tri_idx(1), v3 = tri_idx(2);
823 
824  // Add edges (always store with smaller index first)
825  auto add_edge = [&edge_to_faces](unsigned a, unsigned b,
826  unsigned face) {
827  if (a > b) std::swap(a, b);
828  edge_to_faces[{a, b}].push_back(face);
829  };
830  add_edge(v1, v2, face_id);
831  add_edge(v2, v3, face_id);
832  add_edge(v3, v1, face_id);
833  }
834 
835  // Build adjacency list
836  for (const auto& [edge, faces] : edge_to_faces) {
837  for (size_t i = 0; i < faces.size(); ++i) {
838  for (size_t j = i + 1; j < faces.size(); ++j) {
839  unsigned face1 = faces[i];
840  unsigned face2 = faces[j];
841 
842  // Add bidirectional adjacency
843  if (std::find(face_adjacency_[face1].begin(),
844  face_adjacency_[face1].end(),
845  face2) == face_adjacency_[face1].end()) {
846  face_adjacency_[face1].push_back(face2);
847  }
848  if (std::find(face_adjacency_[face2].begin(),
849  face_adjacency_[face2].end(),
850  face1) == face_adjacency_[face2].end()) {
851  face_adjacency_[face2].push_back(face1);
852  }
853  }
854  }
855  }
856 
857  if (options_.verbose) {
858  size_t total_edges = 0;
859  for (const auto& adj : face_adjacency_) {
860  total_edges += adj.size();
861  }
862  std::cout << StringPrintf("Built adjacency graph: %zu faces, %zu edges\n", num_faces,
863  total_edges / 2);
864  }
865 }
866 
867 float MvsTexturing::ComputePairwiseCost(size_t face1,
868  size_t face2,
869  size_t view1,
870  size_t view2) const {
871  // Potts model: cost is 0 if same view, 1 if different view
872  if (view1 == view2) {
873  return 0.0f;
874  }
875  return 1.0f;
876 }
877 
878 void MvsTexturing::SelectViews() {
879  if (options_.verbose) {
880  std::cout << StringPrintf("Selecting views using graph cut algorithm...\n");
881  }
882 
883  face_labels_.clear();
884  face_labels_.resize(face_projection_infos_.size(), SIZE_MAX);
885 
886  // Initialize with greedy solution
887  for (size_t face_id = 0; face_id < face_projection_infos_.size();
888  ++face_id) {
889  const auto& infos = face_projection_infos_[face_id];
890  if (infos.empty()) {
891  continue;
892  }
893 
894  // Find view with minimum cost (highest quality)
895  float min_cost = std::numeric_limits<float>::max();
896  size_t best_view_id = SIZE_MAX;
897  for (const auto& [view_id, cost] : data_costs_[face_id]) {
898  if (cost < min_cost) {
899  min_cost = cost;
900  best_view_id = view_id;
901  }
902  }
903 
904  if (best_view_id != SIZE_MAX) {
905  face_labels_[face_id] = best_view_id;
906  }
907  }
908 
909  // Iterative refinement using alpha-expansion style algorithm
910  const int max_iterations = 10;
911  const float lambda = 0.5f; // Smoothness weight
912 
913  for (int iter = 0; iter < max_iterations; ++iter) {
914  bool changed = false;
915 
916  // Try to improve each face's label
917  for (size_t face_id = 0; face_id < face_labels_.size(); ++face_id) {
918  if (face_labels_[face_id] == SIZE_MAX) continue;
919  if (data_costs_[face_id].empty()) continue;
920 
921  float current_cost = std::numeric_limits<float>::max();
922  // Find current data cost
923  for (const auto& [view_id, cost] : data_costs_[face_id]) {
924  if (view_id == face_labels_[face_id]) {
925  current_cost = cost;
926  break;
927  }
928  }
929 
930  // Add pairwise costs from neighbors
931  float current_pairwise = 0.0f;
932  for (size_t adj_face : face_adjacency_[face_id]) {
933  if (adj_face < face_labels_.size() &&
934  face_labels_[adj_face] != SIZE_MAX) {
935  current_pairwise += ComputePairwiseCost(
936  face_id, adj_face, face_labels_[face_id],
937  face_labels_[adj_face]);
938  }
939  }
940  float current_total = current_cost + lambda * current_pairwise;
941 
942  // Try all alternative labels
943  float best_total = current_total;
944  size_t best_label = face_labels_[face_id];
945 
946  for (const auto& [view_id, cost] : data_costs_[face_id]) {
947  if (view_id == face_labels_[face_id]) continue;
948 
949  float pairwise = 0.0f;
950  for (size_t adj_face : face_adjacency_[face_id]) {
951  if (adj_face < face_labels_.size() &&
952  face_labels_[adj_face] != SIZE_MAX) {
953  pairwise +=
954  ComputePairwiseCost(face_id, adj_face, view_id,
955  face_labels_[adj_face]);
956  }
957  }
958 
959  float total = cost + lambda * pairwise;
960  if (total < best_total) {
961  best_total = total;
962  best_label = view_id;
963  changed = true;
964  }
965  }
966 
967  face_labels_[face_id] = best_label;
968  }
969 
970  if (!changed) break;
971 
972  if (options_.verbose && iter % 2 == 0) {
973  std::cout << StringPrintf("Graph cut iteration %d...\n", iter + 1);
974  }
975  }
976 
977  // Count faces with valid labels
978  size_t labeled_faces = 0;
979  size_t unlabeled_faces = 0;
980  for (size_t label : face_labels_) {
981  if (label != SIZE_MAX) {
982  labeled_faces++;
983  } else {
984  unlabeled_faces++;
985  }
986  }
987 
988  if (options_.verbose) {
989  std::cout << StringPrintf("Selected views: %zu/%zu faces have valid labels (%.1f%%)\n",
990  labeled_faces, face_labels_.size(),
991  100.0f * labeled_faces / face_labels_.size());
992  if (unlabeled_faces > 0) {
993  std::cerr << "WARNING: " << StringPrintf(
994  "%zu faces have no valid labels (will be handled as unseen "
995  "faces)\n",
996  unlabeled_faces);
997  }
998  }
999 }
1000 
1001 void MvsTexturing::GetSubgraphs(
1002  size_t label, std::vector<std::vector<size_t>>* subgraphs) const {
1003  // Following mvs-texturing's UniGraph::get_subgraphs implementation
1004  std::vector<bool> used(face_labels_.size(), false);
1005 
1006  for (size_t i = 0; i < face_labels_.size(); ++i) {
1007  if (face_labels_[i] == label && !used[i]) {
1008  subgraphs->push_back(std::vector<size_t>());
1009 
1010  std::list<size_t> queue;
1011  queue.push_back(i);
1012  used[i] = true;
1013 
1014  while (!queue.empty()) {
1015  size_t node = queue.front();
1016  queue.pop_front();
1017 
1018  subgraphs->back().push_back(node);
1019 
1020  // Add all unused neighbours with the same label to the queue
1021  for (size_t adj_face : face_adjacency_[node]) {
1022  if (adj_face < face_labels_.size() &&
1023  face_labels_[adj_face] == label && !used[adj_face]) {
1024  queue.push_back(adj_face);
1025  used[adj_face] = true;
1026  }
1027  }
1028  }
1029  }
1030  }
1031 }
1032 
1033 void MvsTexturing::MergeVertexProjectionInfos() {
1034  // Following mvs-texturing's merge_vertex_projection_infos implementation
1035  // Merge vertex infos within the same texture patch
1036  for (size_t i = 0; i < vertex_projection_infos_.size(); ++i) {
1037  auto& infos = vertex_projection_infos_[i];
1038 
1039  std::map<size_t, VertexProjectionInfo> info_map;
1040 
1041  for (const auto& info : infos) {
1042  size_t texture_patch_id = info.texture_patch_id;
1043  auto it = info_map.find(texture_patch_id);
1044  if (it == info_map.end()) {
1045  info_map[texture_patch_id] = info;
1046  } else {
1047  // Merge faces
1048  it->second.faces.insert(it->second.faces.end(),
1049  info.faces.begin(), info.faces.end());
1050  }
1051  }
1052 
1053  infos.clear();
1054  infos.reserve(info_map.size());
1055  for (const auto& [patch_id, info] : info_map) {
1056  infos.push_back(info);
1057  }
1058  }
1059 }
1060 
1061 // Helper structure for texture patch candidate (following mvs-texturing)
1063  int min_x, min_y, max_x, max_y; // Bounding box
1064  std::unique_ptr<TexturePatch> texture_patch;
1065 
1066  bool IsInside(const TexturePatchCandidate& other) const {
1067  return min_x >= other.min_x && max_x <= other.max_x &&
1068  min_y >= other.min_y && max_y <= other.max_y;
1069  }
1070 };
1071 
1072 // Helper function to generate a texture patch candidate (following
1073 // mvs-texturing's generate_candidate)
1075  TextureView* texture_view,
1076  const std::vector<size_t>& faces,
1077  const ccMesh& mesh,
1078  ccPointCloud* cloud,
1079  bool mesh_is_shifted = false) {
1080  // Load image if needed
1081  if (!texture_view->image_data) {
1082  std::string image_path = texture_view->image_file;
1083  if (!ExistsFile(image_path)) {
1084  // Try to find image in common locations
1085  std::vector<std::string> candidates;
1086  // Add path resolution strategies here if needed
1087  for (const auto& candidate : candidates) {
1088  if (ExistsFile(candidate)) {
1089  image_path = candidate;
1090  break;
1091  }
1092  }
1093  }
1094 
1095  QImageReader reader(QString::fromStdString(image_path));
1096  QImage img = reader.read();
1097  if (!img.isNull()) {
1098  texture_view->image_data = std::make_shared<QImage>(
1099  img.convertToFormat(QImage::Format_RGB888));
1100  }
1101  }
1102 
1103  if (!texture_view->image_data) {
1104  std::cerr << "ERROR: " << StringPrintf("Failed to load image for texture view\n");
1105  TexturePatchCandidate empty;
1106  empty.min_x = empty.min_y = empty.max_x = empty.max_y = 0;
1107  return empty;
1108  }
1109 
1110  // Calculate bounding box
1111  int min_x = texture_view->width, min_y = texture_view->height;
1112  int max_x = 0, max_y = 0;
1113  std::vector<Eigen::Vector2f> texcoords;
1114 
1115  for (size_t face_id : faces) {
1116  Eigen::Vector3i tri_idx;
1117  mesh.getTriangleVertIndexes(face_id, tri_idx);
1118 
1119  const CCVector3* v1_ptr = cloud->getPoint(tri_idx(0));
1120  const CCVector3* v2_ptr = cloud->getPoint(tri_idx(1));
1121  const CCVector3* v3_ptr = cloud->getPoint(tri_idx(2));
1122 
1123  // Convert from local coordinates to global coordinates if mesh is
1124  // shifted Camera transforms are in global coordinate system
1125  Eigen::Vector3f v1, v2, v3;
1126  if (mesh_is_shifted) {
1127  // Convert local to global: Pglobal = Plocal / scale - shift
1128  CCVector3d v1_global = cloud->toGlobal3d(*v1_ptr);
1129  CCVector3d v2_global = cloud->toGlobal3d(*v2_ptr);
1130  CCVector3d v3_global = cloud->toGlobal3d(*v3_ptr);
1131  v1 = Eigen::Vector3f(static_cast<float>(v1_global.x),
1132  static_cast<float>(v1_global.y),
1133  static_cast<float>(v1_global.z));
1134  v2 = Eigen::Vector3f(static_cast<float>(v2_global.x),
1135  static_cast<float>(v2_global.y),
1136  static_cast<float>(v2_global.z));
1137  v3 = Eigen::Vector3f(static_cast<float>(v3_global.x),
1138  static_cast<float>(v3_global.y),
1139  static_cast<float>(v3_global.z));
1140  } else {
1141  v1 = Eigen::Vector3f(v1_ptr->x, v1_ptr->y, v1_ptr->z);
1142  v2 = Eigen::Vector3f(v2_ptr->x, v2_ptr->y, v2_ptr->z);
1143  v3 = Eigen::Vector3f(v3_ptr->x, v3_ptr->y, v3_ptr->z);
1144  }
1145 
1146  Eigen::Vector2f p1 = texture_view->GetPixelCoords(v1);
1147  Eigen::Vector2f p2 = texture_view->GetPixelCoords(v2);
1148  Eigen::Vector2f p3 = texture_view->GetPixelCoords(v3);
1149 
1150  texcoords.push_back(p1);
1151  texcoords.push_back(p2);
1152  texcoords.push_back(p3);
1153 
1154  min_x = std::min({min_x, static_cast<int>(std::floor(p1.x())),
1155  static_cast<int>(std::floor(p2.x())),
1156  static_cast<int>(std::floor(p3.x()))});
1157  min_y = std::min({min_y, static_cast<int>(std::floor(p1.y())),
1158  static_cast<int>(std::floor(p2.y())),
1159  static_cast<int>(std::floor(p3.y()))});
1160  max_x = std::max({max_x, static_cast<int>(std::ceil(p1.x())),
1161  static_cast<int>(std::ceil(p2.x())),
1162  static_cast<int>(std::ceil(p3.x()))});
1163  max_y = std::max({max_y, static_cast<int>(std::ceil(p1.y())),
1164  static_cast<int>(std::ceil(p2.y())),
1165  static_cast<int>(std::ceil(p3.y()))});
1166  }
1167 
1168  // Validate bounding box
1169  if (min_x < 0 || min_y < 0 || max_x >= texture_view->width ||
1170  max_y >= texture_view->height) {
1171  std::cerr << "WARNING: " << StringPrintf("Bounding box out of bounds, clamping\n");
1172  min_x = std::max(0, min_x);
1173  min_y = std::max(0, min_y);
1174  max_x = std::min(texture_view->width - 1, max_x);
1175  max_y = std::min(texture_view->height - 1, max_y);
1176  }
1177 
1178  int width = max_x - min_x + 1;
1179  int height = max_y - min_y + 1;
1180 
1181  if (width <= 0 || height <= 0) {
1182  TexturePatchCandidate empty;
1183  empty.min_x = empty.min_y = empty.max_x = empty.max_y = 0;
1184  return empty;
1185  }
1186 
1187  // Add border
1188  const int texture_patch_border = 1;
1189  int patch_width = width + 2 * texture_patch_border;
1190  int patch_height = height + 2 * texture_patch_border;
1191  int patch_min_x = min_x - texture_patch_border;
1192  int patch_min_y = min_y - texture_patch_border;
1193 
1194  // Adjust texcoords relative to patch bounding box
1195  Eigen::Vector2f offset(patch_min_x, patch_min_y);
1196  for (auto& tc : texcoords) {
1197  tc = tc - offset;
1198  }
1199 
1200  // Extract image patch
1201  QImage patch(patch_width, patch_height, QImage::Format_RGB888);
1202  patch.fill(QColor(255, 0, 255)); // Magenta fill
1203 
1204  int src_x_start = std::max(0, patch_min_x);
1205  int src_y_start = std::max(0, patch_min_y);
1206  int src_x_end = std::min(texture_view->width, patch_min_x + patch_width);
1207  int src_y_end = std::min(texture_view->height, patch_min_y + patch_height);
1208 
1209  int dst_x_start = src_x_start - patch_min_x;
1210  int dst_y_start = src_y_start - patch_min_y;
1211  int copy_width = src_x_end - src_x_start;
1212  int copy_height = src_y_end - src_y_start;
1213 
1214  if (copy_width > 0 && copy_height > 0) {
1215  QImage src_region = texture_view->image_data->copy(
1216  src_x_start, src_y_start, copy_width, copy_height);
1217  QPainter painter(&patch);
1218  painter.drawImage(dst_x_start, dst_y_start, src_region);
1219  painter.end();
1220  }
1221 
1222  auto patch_ptr = std::make_shared<QImage>(patch);
1223  auto texture_patch = std::make_unique<TexturePatch>(
1224  label, faces, texcoords, patch_ptr, patch_min_x, patch_min_y,
1225  patch_min_x + patch_width - 1, patch_min_y + patch_height - 1);
1226 
1227  TexturePatchCandidate candidate;
1228  candidate.min_x = patch_min_x;
1229  candidate.min_y = patch_min_y;
1230  candidate.max_x = patch_min_x + patch_width - 1;
1231  candidate.max_y = patch_min_y + patch_height - 1;
1232  candidate.texture_patch = std::move(texture_patch);
1233 
1234  return candidate;
1235 }
1236 
1237 void MvsTexturing::GenerateTexturePatches(const ccMesh& mesh) {
1238  // Following mvs-texturing's generate_texture_patches implementation exactly
1239  if (options_.verbose) {
1240  std::cout << StringPrintf("Generating texture patches...\n");
1241  }
1242 
1243  texture_patches_.clear();
1244  ccPointCloud* cloud =
1246  if (!cloud) {
1247  std::cerr << "ERROR: " << StringPrintf("Mesh has no associated cloud!\n");
1248  return;
1249  }
1250 
1251  // Initialize vertex_projection_infos_ (following mvs-texturing)
1252  vertex_projection_infos_.clear();
1253  vertex_projection_infos_.resize(cloud->size());
1254 
1255  // Get global shift/scale for coordinate conversion
1256  bool mesh_is_shifted = cloud->isShifted();
1257 
1258  size_t num_patches = 0;
1259 
1260  // Process each texture view (following mvs-texturing: for each view_id)
1261  // Note: In our implementation, face_labels_ stores view_id (0-based)
1262  // In mvs-texturing, graph labels are view_id + 1 (1-based)
1263  for (size_t i = 0; i < texture_views_.size(); ++i) {
1264  size_t view_id = i;
1265  int label = static_cast<int>(
1266  view_id +
1267  1); // mvs-texturing uses label = view_id + 1 for TexturePatch
1268 
1269  // Get subgraphs for this view_id (connected components with same
1270  // view_id) Note: GetSubgraphs expects the label value stored in
1271  // face_labels_ (which is view_id)
1272  std::vector<std::vector<size_t>> subgraphs;
1273  GetSubgraphs(view_id, &subgraphs);
1274 
1275  if (subgraphs.empty()) continue;
1276 
1277  TextureView* texture_view = texture_views_[view_id].get();
1278 
1279  // Load image if not already loaded
1280  if (!texture_view->image_data) {
1281  std::string image_path = texture_view->image_file;
1282 
1283  // Try multiple path resolution strategies
1284  if (!ExistsFile(image_path)) {
1285  std::vector<std::string> candidates;
1286 
1287  // 1. Try workspace images directory
1288  if (workspace_) {
1289  std::string workspace_path =
1290  workspace_->GetOptions().workspace_path;
1291  candidates.push_back(colmap::JoinPaths(
1292  workspace_path, "images",
1293  colmap::GetPathBaseName(image_path)));
1294  }
1295 
1296  // 2. Try image_path_/images
1297  candidates.push_back(
1298  colmap::JoinPaths(image_path_, "images",
1299  colmap::GetPathBaseName(image_path)));
1300 
1301  // 3. Try image_path_ directly
1302  candidates.push_back(colmap::JoinPaths(
1303  image_path_, colmap::GetPathBaseName(image_path)));
1304 
1305  for (const auto& candidate : candidates) {
1306  if (ExistsFile(candidate)) {
1307  image_path = candidate;
1308  break;
1309  }
1310  }
1311  }
1312 
1313  QImageReader reader(QString::fromStdString(image_path));
1314  QImage img = reader.read();
1315  if (!img.isNull()) {
1316  texture_view->image_data = std::make_shared<QImage>(
1317  img.convertToFormat(QImage::Format_RGB888));
1318  } else {
1319  std::cerr << "WARNING: " << StringPrintf("Failed to load image: %s\n", image_path.c_str());
1320  continue;
1321  }
1322  }
1323 
1324  // Generate candidates for each subgraph
1325  std::list<TexturePatchCandidate> candidates;
1326  for (size_t j = 0; j < subgraphs.size(); ++j) {
1327  TexturePatchCandidate candidate =
1328  GenerateCandidate(label, texture_view, subgraphs[j], mesh,
1329  cloud, mesh_is_shifted);
1330  if (candidate.texture_patch) {
1331  candidates.push_back(std::move(candidate));
1332  }
1333  }
1334 
1335  // Merge candidates which contain the same image content (following
1336  // mvs-texturing) If one candidate's bounding box is inside another,
1337  // merge them mvs-texturing: if (it != sit &&
1338  // bounding_box.is_inside(&it->bounding_box)) This means: if sit's bbox
1339  // is inside it's bbox, merge sit into it
1340  for (auto it = candidates.begin(); it != candidates.end(); ++it) {
1341  for (auto sit = candidates.begin(); sit != candidates.end();) {
1342  if (it != sit) {
1343  // Check if sit's bounding box is inside it's bounding box
1344  bool sit_inside_it = (sit->min_x >= it->min_x &&
1345  sit->max_x <= it->max_x &&
1346  sit->min_y >= it->min_y &&
1347  sit->max_y <= it->max_y);
1348 
1349  if (sit_inside_it) {
1350  // Merge sit into it: add sit's faces and adjust
1351  // texcoords
1352  auto& it_faces = it->texture_patch->faces;
1353  auto& sit_faces = sit->texture_patch->faces;
1354  it_faces.insert(it_faces.end(), sit_faces.begin(),
1355  sit_faces.end());
1356 
1357  auto& it_texcoords = it->texture_patch->texcoords;
1358  auto& sit_texcoords = sit->texture_patch->texcoords;
1359  Eigen::Vector2f offset(sit->min_x - it->min_x,
1360  sit->min_y - it->min_y);
1361  for (const auto& tc : sit_texcoords) {
1362  it_texcoords.push_back(tc + offset);
1363  }
1364 
1365  sit = candidates.erase(sit);
1366  } else {
1367  ++sit;
1368  }
1369  } else {
1370  ++sit;
1371  }
1372  }
1373  }
1374 
1375  // Add candidates to texture_patches_ and update
1376  // vertex_projection_infos_
1377  for (auto it = candidates.begin(); it != candidates.end(); ++it) {
1378  size_t texture_patch_id;
1379 
1380  // Add to texture_patches_
1381  texture_patches_.push_back(std::move(it->texture_patch));
1382  texture_patch_id = num_patches++;
1383 
1384  // Update vertex_projection_infos_ (following mvs-texturing)
1385  const auto& patch = texture_patches_.back();
1386  const auto& faces = patch->faces;
1387  const auto& texcoords = patch->texcoords;
1388 
1389  for (size_t face_idx = 0; face_idx < faces.size(); ++face_idx) {
1390  size_t face_id = faces[face_idx];
1391  Eigen::Vector3i tri_idx;
1392  mesh.getTriangleVertIndexes(face_id, tri_idx);
1393 
1394  for (size_t j = 0; j < 3; ++j) {
1395  size_t vertex_id = tri_idx(j);
1396  if (vertex_id >= vertex_projection_infos_.size()) continue;
1397 
1398  Eigen::Vector2f projection = texcoords[face_idx * 3 + j];
1399 
1400  VertexProjectionInfo info;
1401  info.texture_patch_id = texture_patch_id;
1402  info.projection = projection;
1403  info.faces = {face_id};
1404 
1405  vertex_projection_infos_[vertex_id].push_back(info);
1406  }
1407  }
1408  }
1409  }
1410 
1411  // Merge vertex projection infos (following mvs-texturing)
1412  MergeVertexProjectionInfos();
1413 
1414  // Handle unseen faces (label = 0 in mvs-texturing, SIZE_MAX in our
1415  // implementation) Following mvs-texturing: get_subgraphs(0, &subgraphs) for
1416  // unseen faces
1417  {
1418  std::vector<size_t> unseen_faces;
1419  std::vector<std::vector<size_t>> subgraphs;
1420 
1421  // Get subgraphs with label = SIZE_MAX (unseen faces in our
1422  // implementation) In mvs-texturing, label = 0 means unseen We use
1423  // SIZE_MAX to represent unseen faces
1424  for (size_t face_id = 0; face_id < face_labels_.size(); ++face_id) {
1425  if (face_labels_[face_id] == SIZE_MAX) {
1426  unseen_faces.push_back(face_id);
1427  }
1428  }
1429 
1430  // Group unseen faces into connected components
1431  if (!unseen_faces.empty()) {
1432  std::vector<bool> used(face_labels_.size(), false);
1433  for (size_t face_id : unseen_faces) {
1434  if (used[face_id]) continue;
1435 
1436  subgraphs.push_back(std::vector<size_t>());
1437  std::list<size_t> queue;
1438  queue.push_back(face_id);
1439  used[face_id] = true;
1440 
1441  while (!queue.empty()) {
1442  size_t node = queue.front();
1443  queue.pop_front();
1444  subgraphs.back().push_back(node);
1445 
1446  for (size_t adj_face : face_adjacency_[node]) {
1447  if (adj_face < face_labels_.size() &&
1448  face_labels_[adj_face] == SIZE_MAX &&
1449  !used[adj_face]) {
1450  queue.push_back(adj_face);
1451  used[adj_face] = true;
1452  }
1453  }
1454  }
1455  }
1456  }
1457 
1458  // Process unseen faces (following mvs-texturing: fill_hole or
1459  // keep_unseen_faces) For now, we create a simple texture patch for
1460  // unseen faces
1461  // TODO: Implement proper hole filling if needed
1462  if (!unseen_faces.empty() && options_.verbose) {
1463  std::cout << StringPrintf("Found %zu unseen faces in %zu subgraphs\n",
1464  unseen_faces.size(), subgraphs.size());
1465 
1466  // Create a simple texture patch for unseen faces (following
1467  // mvs-texturing) Use a small default texture
1468  QImage default_image(3, 3, QImage::Format_RGB888);
1469  default_image.fill(QColor(128, 128, 128)); // Gray color
1470 
1471  std::vector<Eigen::Vector2f> texcoords;
1472  for (size_t i = 0; i < unseen_faces.size(); ++i) {
1473  // Use fixed texture coordinates (following mvs-texturing:
1474  // {{2.0f, 1.0f}, {1.0f, 1.0f}, {1.0f, 2.0f}})
1475  texcoords.push_back(Eigen::Vector2f(2.0f, 1.0f));
1476  texcoords.push_back(Eigen::Vector2f(1.0f, 1.0f));
1477  texcoords.push_back(Eigen::Vector2f(1.0f, 2.0f));
1478  }
1479 
1480  auto patch_ptr = std::make_shared<QImage>(default_image);
1481  auto texture_patch = std::make_unique<TexturePatch>(
1482  0, unseen_faces, texcoords, patch_ptr, 0, 0, 2, 2);
1483 
1484  size_t texture_patch_id = texture_patches_.size();
1485  texture_patches_.push_back(std::move(texture_patch));
1486 
1487  // Update vertex_projection_infos_ for unseen faces
1488  for (size_t i = 0; i < unseen_faces.size(); ++i) {
1489  size_t face_id = unseen_faces[i];
1490  Eigen::Vector3i tri_idx;
1491  mesh.getTriangleVertIndexes(face_id, tri_idx);
1492 
1493  for (size_t j = 0; j < 3; ++j) {
1494  size_t vertex_id = tri_idx(j);
1495  if (vertex_id >= vertex_projection_infos_.size()) continue;
1496 
1497  Eigen::Vector2f projection = texcoords[i * 3 + j];
1498 
1499  VertexProjectionInfo info;
1500  info.texture_patch_id = texture_patch_id;
1501  info.projection = projection;
1502  info.faces = {face_id};
1503 
1504  vertex_projection_infos_[vertex_id].push_back(info);
1505  }
1506  }
1507  }
1508  }
1509 
1510  // Merge vertex projection infos again (following mvs-texturing)
1511  MergeVertexProjectionInfos();
1512 
1513  if (options_.verbose) {
1514  std::cout << StringPrintf("Generated %zu texture patches\n", texture_patches_.size());
1515  }
1516 }
1517 
1518 void MvsTexturing::SeamLeveling(const ccMesh& mesh) {
1519  // Following mvs-texturing's seam leveling flow:
1520  // - If global_seam_leveling: call global_seam_leveling
1521  // - Else: call adjust_colors with zero adjust_values to calculate validity
1522  // masks
1523  // - If local_seam_leveling: call local_seam_leveling
1524  // Note: Our TexturePatch doesn't have adjust_colors/validity_mask, so we
1525  // use simplified approach
1526 
1527  if (options_.verbose) {
1528  std::cout << StringPrintf("Performing seam leveling...\n");
1529  }
1530 
1531  if (texture_patches_.empty()) {
1532  return;
1533  }
1534 
1535  // Find seam edges: edges between faces with different labels
1536  // Following mvs-texturing: find_seam_edges(graph, mesh, &seam_edges)
1537  seam_edges_.clear();
1538  std::set<std::pair<size_t, size_t>> processed_vertex_edges;
1539 
1540  for (size_t face_id = 0; face_id < face_labels_.size(); ++face_id) {
1541  if (face_labels_[face_id] == SIZE_MAX) continue;
1542 
1543  for (size_t adj_face : face_adjacency_[face_id]) {
1544  if (adj_face >= face_labels_.size() ||
1545  face_labels_[adj_face] == SIZE_MAX)
1546  continue;
1547  if (face_labels_[face_id] == face_labels_[adj_face]) continue;
1548 
1549  // Find shared edge vertices (following mvs-texturing:
1550  // find_seam_edges)
1551  Eigen::Vector3i tri1_idx, tri2_idx;
1552  mesh.getTriangleVertIndexes(face_id, tri1_idx);
1553  mesh.getTriangleVertIndexes(adj_face, tri2_idx);
1554 
1555  // Find shared vertices
1556  std::vector<size_t> shared_vertices;
1557  for (int i = 0; i < 3; ++i) {
1558  for (int j = 0; j < 3; ++j) {
1559  if (tri1_idx(i) == tri2_idx(j)) {
1560  shared_vertices.push_back(tri1_idx(i));
1561  }
1562  }
1563  }
1564 
1565  if (shared_vertices.size() == 2) {
1566  SeamEdge seam;
1567  seam.face1 = face_id;
1568  seam.face2 = adj_face;
1569  seam.v1 = std::min(shared_vertices[0], shared_vertices[1]);
1570  seam.v2 = std::max(shared_vertices[0], shared_vertices[1]);
1571 
1572  auto edge_key = std::make_pair(seam.v1, seam.v2);
1573  if (processed_vertex_edges.find(edge_key) ==
1574  processed_vertex_edges.end()) {
1575  processed_vertex_edges.insert(edge_key);
1576  seam_edges_.push_back(seam);
1577  }
1578  }
1579  }
1580  }
1581 
1582  // Following mvs-texturing: if global_seam_leveling is false, calculate
1583  // validity masks by calling adjust_colors with zero adjust_values Since our
1584  // TexturePatch doesn't have adjust_colors, we skip this step In a full
1585  // implementation, we would:
1586  // 1. Call adjust_colors with zero adjust_values to calculate validity
1587  // masks
1588  // 2. Optionally call global_seam_leveling if enabled
1589  // 3. Optionally call local_seam_leveling if enabled
1590 
1591  if (options_.verbose) {
1592  std::cout << StringPrintf("Seam leveling completed: %zu seam edges found\n",
1593  seam_edges_.size());
1594  }
1595 }
1596 
1597 void MvsTexturing::GenerateTextureAtlases() {
1598  if (options_.verbose) {
1599  std::cout << StringPrintf("Generating texture atlases with bin packing...\n");
1600  }
1601 
1602  if (texture_patches_.empty()) {
1603  return;
1604  }
1605 
1606  // Constants for texture atlas sizing
1607  const unsigned int MAX_TEXTURE_SIZE = 8192;
1608  const unsigned int PREF_TEXTURE_SIZE = 4096;
1609  // const unsigned int MIN_TEXTURE_SIZE = 256; // Reserved for future use
1610  const unsigned int PADDING = 2;
1611 
1612  // Sort patches by size (largest first) for better packing
1613  std::vector<size_t> patch_indices(texture_patches_.size());
1614  std::iota(patch_indices.begin(), patch_indices.end(), 0);
1615  std::sort(patch_indices.begin(), patch_indices.end(),
1616  [this](size_t a, size_t b) {
1617  int size_a = texture_patches_[a]->image->width() *
1618  texture_patches_[a]->image->height();
1619  int size_b = texture_patches_[b]->image->width() *
1620  texture_patches_[b]->image->height();
1621  return size_a > size_b;
1622  });
1623 
1624  // Simple bin packing: try to pack patches into atlases
1625  struct AtlasPatch {
1626  size_t patch_idx;
1627  unsigned int x, y;
1628  };
1629 
1630  struct Atlas {
1631  unsigned int width;
1632  unsigned int height;
1633  std::vector<AtlasPatch> patches;
1634  };
1635 
1636  std::vector<Atlas> atlases;
1637  unsigned int current_atlas_size = PREF_TEXTURE_SIZE;
1638 
1639  for (size_t patch_idx : patch_indices) {
1640  const auto& patch = texture_patches_[patch_idx];
1641  unsigned int patch_width = patch->image->width() + 2 * PADDING;
1642  unsigned int patch_height = patch->image->height() + 2 * PADDING;
1643 
1644  // Find or create atlas that can fit this patch
1645  bool placed = false;
1646  for (auto& atlas : atlases) {
1647  // Simple bottom-left bin packing
1648  // Try to place at different positions
1649  for (unsigned int y = 0; y + patch_height <= atlas.height;
1650  y += 64) {
1651  for (unsigned int x = 0; x + patch_width <= atlas.width;
1652  x += 64) {
1653  // Check if this position doesn't overlap with existing
1654  // patches
1655  bool overlaps = false;
1656  for (const auto& ap : atlas.patches) {
1657  const auto& existing_patch =
1658  texture_patches_[ap.patch_idx];
1659  unsigned int ew =
1660  existing_patch->image->width() + 2 * PADDING;
1661  unsigned int eh =
1662  existing_patch->image->height() + 2 * PADDING;
1663 
1664  if (!(x + patch_width <= ap.x || ap.x + ew <= x ||
1665  y + patch_height <= ap.y || ap.y + eh <= y)) {
1666  overlaps = true;
1667  break;
1668  }
1669  }
1670 
1671  if (!overlaps) {
1672  AtlasPatch ap;
1673  ap.patch_idx = patch_idx;
1674  ap.x = x;
1675  ap.y = y;
1676  atlas.patches.push_back(ap);
1677  placed = true;
1678  break;
1679  }
1680  }
1681  if (placed) break;
1682  }
1683  if (placed) break;
1684  }
1685 
1686  // Create new atlas if patch doesn't fit
1687  if (!placed) {
1688  // Determine atlas size
1689  unsigned int atlas_size = current_atlas_size;
1690  if (patch_width > atlas_size || patch_height > atlas_size) {
1691  atlas_size = std::max(patch_width, patch_height);
1692  // Round up to power of 2
1693  unsigned int next_power = 1;
1694  while (next_power < atlas_size &&
1695  next_power < MAX_TEXTURE_SIZE) {
1696  next_power <<= 1;
1697  }
1698  atlas_size = std::min(next_power, MAX_TEXTURE_SIZE);
1699  }
1700 
1701  Atlas new_atlas;
1702  new_atlas.width = atlas_size;
1703  new_atlas.height = atlas_size;
1704  AtlasPatch ap;
1705  ap.patch_idx = patch_idx;
1706  ap.x = PADDING;
1707  ap.y = PADDING;
1708  new_atlas.patches.push_back(ap);
1709  atlases.push_back(new_atlas);
1710  }
1711  }
1712 
1713  // Merge patches into atlas images and update texture coordinates
1714  texture_atlases_.clear();
1715  texture_atlases_.reserve(atlases.size());
1716 
1717  for (size_t atlas_idx = 0; atlas_idx < atlases.size(); ++atlas_idx) {
1718  const auto& atlas_info = atlases[atlas_idx];
1719 
1720  TextureAtlas atlas;
1721  atlas.width = atlas_info.width;
1722  atlas.height = atlas_info.height;
1723  atlas.image = std::make_shared<QImage>(
1724  atlas_info.width, atlas_info.height, QImage::Format_RGB888);
1725  atlas.image->fill(Qt::black);
1726 
1727  // size_t texcoord_id_offset = 0; // Currently unused
1728 
1729  // Merge all patches into this atlas
1730  for (const auto& ap : atlas_info.patches) {
1731  const auto& patch = texture_patches_[ap.patch_idx];
1732 
1733  // Copy patch image into atlas at the specified position
1734  QPainter painter(atlas.image.get());
1735  painter.drawImage(ap.x + PADDING, ap.y + PADDING, *patch->image);
1736  painter.end();
1737 
1738  // Update texture coordinates: convert from patch-local to
1739  // atlas-normalized
1740  for (size_t i = 0; i < patch->faces.size(); ++i) {
1741  size_t face_id = patch->faces[i];
1742  atlas.face_ids.push_back(face_id);
1743 
1744  // Get texture coordinates for this face (3 vertices)
1745  size_t face_texcoord_start = atlas.texcoords.size();
1746  for (int j = 0; j < 3; ++j) {
1747  size_t texcoord_idx = i * 3 + j;
1748  Eigen::Vector2f local_tc = patch->texcoords[texcoord_idx];
1749 
1750  // Convert to atlas coordinates: add offset and normalize
1751  float atlas_u =
1752  (local_tc.x() + ap.x + PADDING) / atlas.width;
1753  float atlas_v =
1754  (local_tc.y() + ap.y + PADDING) / atlas.height;
1755 
1756  atlas.texcoords.emplace_back(atlas_u, atlas_v);
1757  }
1758 
1759  // Store texture coordinate indices for this face (relative to
1760  // atlas start)
1761  atlas.texcoord_ids.push_back(face_texcoord_start);
1762  atlas.texcoord_ids.push_back(face_texcoord_start + 1);
1763  atlas.texcoord_ids.push_back(face_texcoord_start + 2);
1764  }
1765  }
1766 
1767  texture_atlases_.push_back(atlas);
1768  }
1769 
1770  if (options_.verbose) {
1771  std::cout << StringPrintf("Created %zu texture atlases from %zu patches",
1772  texture_atlases_.size(), texture_patches_.size()) << "\n";
1773  for (size_t i = 0; i < texture_atlases_.size(); ++i) {
1774  std::cout << StringPrintf(" Atlas %zu: %u x %u, %zu faces", i + 1,
1775  texture_atlases_[i].width, texture_atlases_[i].height,
1776  texture_atlases_[i].face_ids.size()) << "\n";
1777  }
1778  }
1779 }
1780 
1781 // ============================================================================
1782 // GMI (Gradient Magnitude Image) Implementation
1783 // Reference: OpenMVS (https://github.com/cdcseacave/openMVS)
1784 // ============================================================================
1785 
1786 // Compute gradient magnitude image using Sobel operator
1787 QImage MvsTexturing::ComputeGradientMagnitudeImage(const QImage& image) {
1788  // Convert to grayscale if needed
1789  QImage gray;
1790  if (image.format() == QImage::Format_Grayscale8) {
1791  gray = image;
1792  } else {
1793  gray = image.convertToFormat(QImage::Format_Grayscale8);
1794  }
1795 
1796  const int width = gray.width();
1797  const int height = gray.height();
1798 
1799  // Create output GMI image
1800  QImage gmi(width, height, QImage::Format_Grayscale8);
1801  gmi.fill(0);
1802 
1803  // Compute gradients using Sobel operator
1804  // Sobel kernels:
1805  // Gx = [-1 0 1; -2 0 2; -1 0 1]
1806  // Gy = [-1 -2 -1; 0 0 0; 1 2 1]
1807 
1808  std::vector<float> magnitudes;
1809  magnitudes.reserve((width - 2) * (height - 2));
1810 
1811  for (int y = 1; y < height - 1; y++) {
1812  const uchar* row_prev = gray.constScanLine(y - 1);
1813  const uchar* row_curr = gray.constScanLine(y);
1814  const uchar* row_next = gray.constScanLine(y + 1);
1815 
1816  for (int x = 1; x < width - 1; x++) {
1817  // Sobel Gx (horizontal gradient)
1818  int gx = -static_cast<int>(row_prev[x - 1]) + static_cast<int>(row_prev[x + 1])
1819  - 2 * static_cast<int>(row_curr[x - 1]) + 2 * static_cast<int>(row_curr[x + 1])
1820  - static_cast<int>(row_next[x - 1]) + static_cast<int>(row_next[x + 1]);
1821 
1822  // Sobel Gy (vertical gradient)
1823  int gy = -static_cast<int>(row_prev[x - 1]) - 2 * static_cast<int>(row_prev[x]) - static_cast<int>(row_prev[x + 1])
1824  + static_cast<int>(row_next[x - 1]) + 2 * static_cast<int>(row_next[x]) + static_cast<int>(row_next[x + 1]);
1825 
1826  // Gradient magnitude
1827  float mag = std::sqrt(static_cast<float>(gx * gx + gy * gy));
1828  magnitudes.push_back(mag);
1829  }
1830  }
1831 
1832  // Find max magnitude for normalization
1833  float max_mag = 0.0f;
1834  for (float mag : magnitudes) {
1835  max_mag = std::max(max_mag, mag);
1836  }
1837 
1838  // Normalize and store
1839  size_t idx = 0;
1840  for (int y = 1; y < height - 1; y++) {
1841  uchar* row = gmi.scanLine(y);
1842  for (int x = 1; x < width - 1; x++) {
1843  float normalized = (max_mag > 0.0f) ? (magnitudes[idx] / max_mag) : 0.0f;
1844  row[x] = static_cast<uchar>(normalized * 255.0f);
1845  idx++;
1846  }
1847  }
1848 
1849  return gmi;
1850 }
1851 
1852 // Check if point (px, py) is inside triangle (p1, p2, p3) using barycentric coordinates
1853 bool MvsTexturing::IsPointInTriangle(float px, float py,
1854  const Eigen::Vector2f& p1,
1855  const Eigen::Vector2f& p2,
1856  const Eigen::Vector2f& p3) const {
1857  // Use barycentric coordinates method
1858  float denom = (p2.y() - p3.y()) * (p1.x() - p3.x()) +
1859  (p3.x() - p2.x()) * (p1.y() - p3.y());
1860 
1861  if (std::abs(denom) < 1e-10f) {
1862  return false; // Degenerate triangle
1863  }
1864 
1865  float a = ((p2.y() - p3.y()) * (px - p3.x()) +
1866  (p3.x() - p2.x()) * (py - p3.y())) / denom;
1867  float b = ((p3.y() - p1.y()) * (px - p3.x()) +
1868  (p1.x() - p3.x()) * (py - p3.y())) / denom;
1869  float c = 1.0f - a - b;
1870 
1871  return a >= 0.0f && b >= 0.0f && c >= 0.0f;
1872 }
1873 
1874 // Calculate GMI quality for a projected triangle
1875 // Reference: OpenMVS texture quality computation based on gradient magnitude
1876 float MvsTexturing::CalculateGMIQuality(size_t view_id,
1877  const Eigen::Vector2f& p1,
1878  const Eigen::Vector2f& p2,
1879  const Eigen::Vector2f& p3) {
1880  // Ensure GMI is computed for this view
1881  if (view_id >= gradient_magnitude_images_.size() ||
1882  gradient_magnitude_images_[view_id].isNull()) {
1883  // Compute GMI on-demand if not yet done
1884  if (view_id < texture_views_.size()) {
1885  const auto& view = texture_views_[view_id];
1886  if (view && view->image_data && !view->image_data->isNull()) {
1887  if (view_id >= gradient_magnitude_images_.size()) {
1888  gradient_magnitude_images_.resize(texture_views_.size());
1889  }
1890  gradient_magnitude_images_[view_id] =
1891  ComputeGradientMagnitudeImage(*view->image_data);
1892  } else {
1893  return 0.0f; // Image not loaded
1894  }
1895  } else {
1896  return 0.0f;
1897  }
1898  }
1899 
1900  const QImage& gmi = gradient_magnitude_images_[view_id];
1901  if (gmi.isNull()) {
1902  return 0.0f;
1903  }
1904 
1905  // Calculate bounding box of triangle
1906  float min_x = std::min({p1.x(), p2.x(), p3.x()});
1907  float max_x = std::max({p1.x(), p2.x(), p3.x()});
1908  float min_y = std::min({p1.y(), p2.y(), p3.y()});
1909  float max_y = std::max({p1.y(), p2.y(), p3.y()});
1910 
1911  // Clamp to image bounds
1912  min_x = std::max(0.0f, min_x);
1913  max_x = std::min(static_cast<float>(gmi.width() - 1), max_x);
1914  min_y = std::max(0.0f, min_y);
1915  max_y = std::min(static_cast<float>(gmi.height() - 1), max_y);
1916 
1917  // Sample GMI values in the triangle region
1918  std::vector<float> samples;
1919  samples.reserve(static_cast<size_t>((max_x - min_x + 1) * (max_y - min_y + 1)));
1920 
1921  for (int y = static_cast<int>(std::floor(min_y)); y <= static_cast<int>(std::ceil(max_y)); y++) {
1922  if (y < 0 || y >= gmi.height()) continue;
1923  const uchar* row = gmi.constScanLine(y);
1924 
1925  for (int x = static_cast<int>(std::floor(min_x)); x <= static_cast<int>(std::ceil(max_x)); x++) {
1926  if (x < 0 || x >= gmi.width()) continue;
1927 
1928  // Check if point is inside triangle
1929  if (IsPointInTriangle(static_cast<float>(x), static_cast<float>(y), p1, p2, p3)) {
1930  float gmi_value = static_cast<float>(row[x]) / 255.0f;
1931  samples.push_back(gmi_value);
1932  }
1933  }
1934  }
1935 
1936  if (samples.empty()) {
1937  return 0.0f;
1938  }
1939 
1940  // Use median for robustness against outliers (like OpenMVS)
1941  std::sort(samples.begin(), samples.end());
1942  float median_gmi = samples[samples.size() / 2];
1943 
1944  // Calculate projected triangle area
1945  float area = 0.5f * std::abs((p2.x() - p1.x()) * (p3.y() - p1.y()) -
1946  (p3.x() - p1.x()) * (p2.y() - p1.y()));
1947 
1948  // Combine GMI with area: quality = GMI_median * sqrt(area)
1949  // Following OpenMVS approach: higher gradient = better texture quality
1950  // Scale by sqrt(area) to balance resolution vs coverage
1951  return median_gmi * std::sqrt(std::max(area, 1.0f));
1952 }
1953 
1954 bool MvsTexturing::SaveOBJModel(const std::string& output_path,
1955  const ccMesh& mesh) {
1956  if (options_.verbose) {
1957  std::cout << StringPrintf("Exporting OBJ model with ObjFilter...\n");
1958  }
1959 
1960  if (texture_atlases_.empty()) {
1961  std::cerr << "WARNING: No texture atlases to save!\n";
1962  return false;
1963  }
1964 
1966  if (!cloud) {
1967  std::cerr << "ERROR: Mesh has no associated cloud!\n";
1968  return false;
1969  }
1970 
1971  // Parse output path
1972  std::string root, ext;
1973  colmap::SplitFileExtension(output_path, &root, &ext);
1974  std::string prefix = (ext == ".obj" || ext == ".OBJ") ? root : output_path;
1975  std::string output_dir = colmap::GetParentDir(prefix);
1976  std::string base_name = colmap::GetPathBaseName(prefix);
1977 
1978  // Helper to format material names (material0000, material0001, ...)
1979  auto format_material_name = [](size_t n) -> std::string {
1980  std::string name = "material";
1981  std::string num_str = std::to_string(n);
1982  while (num_str.length() < 4) {
1983  num_str = "0" + num_str;
1984  }
1985  return name + num_str;
1986  };
1987 
1988  // Step 1: Create ccMesh directly from original data
1989  // Clone the point cloud (preserve vertices and normals)
1990  ccPointCloud* export_cloud = cloud->cloneThis();
1991  if (!export_cloud) {
1992  std::cerr << "ERROR: Failed to clone point cloud\n";
1993  return false;
1994  }
1995  export_cloud->setEnabled(false);
1996 
1997  // Create mesh
1998  ccMesh* export_mesh = new ccMesh(export_cloud);
1999  export_mesh->setName("textured-mesh");
2000  export_mesh->addChild(export_cloud);
2001 
2002  // Count total faces
2003  size_t total_faces = 0;
2004  for (const auto& atlas : texture_atlases_) {
2005  total_faces += atlas.face_ids.size();
2006  }
2007 
2008  if (!export_mesh->reserve(total_faces)) {
2009  std::cerr << "ERROR: Failed to reserve memory for export mesh\n";
2010  delete export_mesh;
2011  return false;
2012  }
2013 
2014  // Step 2: Create materials and save texture images
2015  ccMaterialSet* materials = new ccMaterialSet("materials");
2016 
2017  for (size_t atlas_idx = 0; atlas_idx < texture_atlases_.size(); ++atlas_idx) {
2018  const auto& atlas = texture_atlases_[atlas_idx];
2019  std::string material_name = format_material_name(atlas_idx);
2020 
2021  // Build texture path (ObjFilter's saveAsMTL will save it automatically)
2022  std::string texture_filename = base_name + "_" + material_name + "_map_Kd.png";
2023  std::string texture_path = colmap::JoinPaths(output_dir, texture_filename);
2024 
2025  // Create material
2026  ccMaterial::Shared mat(new ccMaterial(QString::fromStdString(material_name)));
2027  mat->setAmbient(ecvColor::Rgbaf(1.0f, 1.0f, 1.0f, 1.0f));
2028  mat->setDiffuse(ecvColor::Rgbaf(1.0f, 1.0f, 1.0f, 1.0f));
2029  mat->setSpecular(ecvColor::Rgbaf(0.0f, 0.0f, 0.0f, 1.0f));
2030  mat->setShininess(1.0f);
2031 
2032  // Register texture into ccMaterial's static database
2033  // ObjFilter will automatically save it when calling saveAsMTL()
2034  mat->setTexture(*atlas.image, QString::fromStdString(texture_path), false);
2035 
2036  materials->addMaterial(mat);
2037  }
2038 
2039  export_mesh->setMaterialSet(materials);
2040 
2041  // Step 3: Build texture coordinates table (all atlases combined)
2043 
2044  size_t total_texcoords = 0;
2045  for (const auto& atlas : texture_atlases_) {
2046  total_texcoords += atlas.texcoords.size();
2047  }
2048  texCoords->reserve(total_texcoords);
2049 
2050  for (const auto& atlas : texture_atlases_) {
2051  for (const auto& tc : atlas.texcoords) {
2052  // CRITICAL FIX: Flip Y axis to match ObjModel::SaveToFiles behavior
2053  // ObjModel writes: vt.x(), (1.0 - vt.y()) - flips Y when writing
2054  // ObjFilter writes: tc.tx, tc.ty - NO flip when writing
2055  // Therefore, we must flip Y here before storing
2056  texCoords->addElement(TexCoords2D(tc.x(), 1.0f - tc.y()));
2057  }
2058  }
2059 
2060  export_mesh->setTexCoordinatesTable(texCoords);
2061 
2062  // Step 4: Reserve per-triangle data
2063  if (!export_mesh->reservePerTriangleMtlIndexes() ||
2064  !export_mesh->reservePerTriangleTexCoordIndexes()) {
2065  std::cerr << "ERROR: Failed to reserve per-triangle data\n";
2066  delete export_mesh;
2067  return false;
2068  }
2069 
2070  // Step 5: Add faces by atlas order (naturally groups by material!)
2071  // This is the key: processing atlases in order means faces are naturally grouped
2072  size_t texcoord_offset = 0;
2073 
2074  for (size_t atlas_idx = 0; atlas_idx < texture_atlases_.size(); ++atlas_idx) {
2075  const auto& atlas = texture_atlases_[atlas_idx];
2076  int material_idx = static_cast<int>(atlas_idx);
2077 
2078  const auto& atlas_faces = atlas.face_ids;
2079  const auto& atlas_texcoord_ids = atlas.texcoord_ids;
2080 
2081  for (size_t i = 0; i < atlas_faces.size(); ++i) {
2082  size_t mesh_face_idx = atlas_faces[i];
2083 
2084  // Get vertex indices from original mesh
2085  const cloudViewer::VerticesIndexes* tri = mesh.getTriangleVertIndexes(mesh_face_idx);
2086 
2087  // Add triangle
2088  export_mesh->addTriangle(tri->i1, tri->i2, tri->i3);
2089 
2090  // Add material index (all faces in this loop have same material!)
2091  export_mesh->addTriangleMtlIndex(material_idx);
2092 
2093  // Add texture coordinate indices (offset by accumulated texcoords)
2094  int tc_idx0 = static_cast<int>(texcoord_offset + atlas_texcoord_ids[i * 3 + 0]);
2095  int tc_idx1 = static_cast<int>(texcoord_offset + atlas_texcoord_ids[i * 3 + 1]);
2096  int tc_idx2 = static_cast<int>(texcoord_offset + atlas_texcoord_ids[i * 3 + 2]);
2097 
2098  export_mesh->addTriangleTexCoordIndexes(tc_idx0, tc_idx1, tc_idx2);
2099  }
2100 
2101  texcoord_offset += atlas.texcoords.size();
2102  }
2103 
2104  // Step 6: Use ObjFilter to save
2105  if (options_.verbose) {
2106  std::cout << StringPrintf(" Saving with ObjFilter (%zu materials, %zu usemtl switches)...\n",
2107  texture_atlases_.size(), texture_atlases_.size());
2108  }
2109 
2110  ObjFilter obj_filter;
2112  params.alwaysDisplaySaveDialog = false;
2113  params.parentWidget = nullptr;
2114 
2115  QString qpath = QString::fromStdString(output_path);
2116  CC_FILE_ERROR result = obj_filter.saveToFile(export_mesh, qpath, params);
2117 
2118  // Clean up
2119  if (export_mesh) {
2120  delete export_mesh;
2121  export_mesh = nullptr;
2122  }
2123 
2124  if (result == CC_FERR_NO_ERROR) {
2125  if (options_.verbose) {
2126  std::cout << StringPrintf(
2127  "Saved OBJ model to %s\n"
2128  " Vertices: %u, Faces: %zu, Materials: %zu, usemtl switches: %zu\n",
2129  output_path.c_str(), cloud->size(), total_faces,
2130  texture_atlases_.size(), texture_atlases_.size());
2131  }
2132  return true;
2133  } else {
2134  std::cerr << "ERROR: ObjFilter failed to save, error code: " << result << std::endl;
2135  return false;
2136  }
2137 }
2138 
2139 } // namespace mvs
2140 } // namespace colmap
constexpr double M_PI
Pi.
Definition: CVConst.h:19
std::shared_ptr< core::Tensor > image
int width
std::string name
int height
int offset
CC_FILE_ERROR
Typical I/O filter errors.
Definition: FileIOFilter.h:20
@ CC_FERR_NO_ERROR
Definition: FileIOFilter.h:21
core::Tensor result
Definition: VtkUtils.cpp:76
Wavefront meshes file I/O filter.
Definition: ObjFilter.h:13
virtual CC_FILE_ERROR saveToFile(ccHObject *entity, const QString &filename, const SaveParameters &parameters) override
Saves an entity (or a group of) to a file.
Definition: ObjFilter.cpp:59
Array of 2D texture coordinates.
Type y
Definition: CVGeom.h:137
Type x
Definition: CVGeom.h:137
Type z
Definition: CVGeom.h:137
void addElement(const Type &value)
Definition: ecvArray.h:105
A 3D cloud interface with associated features (color, normals, octree, etc.)
static ccPointCloud * ToPointCloud(ccHObject *obj, bool *isLockedVertices=nullptr)
Converts current object to 'equivalent' ccPointCloud.
virtual bool addChild(ccHObject *child, int dependencyFlags=DP_PARENT_OF_OTHER, int insertIndex=-1)
Adds a child.
Definition: ecvHObject.cpp:534
Mesh (triangle) material.
int addMaterial(ccMaterial::CShared mat, bool allowDuplicateNames=false)
Adds a material.
Mesh (triangle) material.
Definition: ecvMaterial.h:28
QSharedPointer< ccMaterial > Shared
Shared type.
Definition: ecvMaterial.h:33
Triangular mesh.
Definition: ecvMesh.h:35
bool reservePerTriangleMtlIndexes()
Reserves memory to store per-triangle material index.
Definition: ecvMesh.cpp:3734
virtual unsigned size() const override
Returns the number of triangles.
Definition: ecvMesh.cpp:2143
void addTriangleMtlIndex(int mtlIndex)
Adds triangle material index for next triangle.
Definition: ecvMesh.cpp:3751
void setMaterialSet(ccMaterialSet *materialSet, bool autoReleaseOldMaterialSet=true)
Sets associated material set (may be shared)
Definition: ecvMesh.cpp:492
bool reserve(std::size_t n)
Reserves the memory to store the vertex indexes (3 per triangle)
Definition: ecvMesh.cpp:2475
ccGenericPointCloud * getAssociatedCloud() const override
Returns the vertices cloud.
Definition: ecvMesh.h:143
void addTriangleTexCoordIndexes(int i1, int i2, int i3)
Adds a triplet of tex coords indexes for next triangle.
Definition: ecvMesh.cpp:3678
void addTriangle(unsigned i1, unsigned i2, unsigned i3)
Adds a triangle to the mesh.
Definition: ecvMesh.cpp:2428
void setTexCoordinatesTable(TextureCoordsContainer *texCoordsTable, bool autoReleaseOldTable=true)
Sets per-triangle texture coordinates array (may be shared)
Definition: ecvMesh.cpp:3598
bool reservePerTriangleTexCoordIndexes()
Reserves memory to store per-triangle triplets of tex coords indexes.
Definition: ecvMesh.cpp:3659
ccMesh & ComputeVertexNormals(bool normalized=true)
Function to compute vertex normals, usually called before rendering.
cloudViewer::VerticesIndexes * getTriangleVertIndexes(unsigned triangleIndex) override
Returns the indexes of the vertices of a given triangle.
Definition: ecvMesh.cpp:2599
virtual void setName(const QString &name)
Sets object name.
Definition: ecvObject.h:75
virtual void setEnabled(bool state)
Sets the "enabled" property.
Definition: ecvObject.h:102
A 3D cloud and its associated features (color, normals, scalar fields, etc.)
bool hasNormals() const override
Returns whether normals are enabled or not.
ccPointCloud * cloneThis(ccPointCloud *destCloud=nullptr, bool ignoreChildren=false)
Clones this entity.
CCVector3d toGlobal3d(const Vector3Tpl< T > &Plocal) const
Returns the point back-projected into the original coordinates system.
bool isShifted() const
Returns whether the cloud is shifted or not.
virtual const CCVector3d & getGlobalShift() const
Returns the shift applied to original coordinates.
virtual double getGlobalScale() const
Returns the scale applied to original coordinates.
unsigned size() const override
Definition: PointCloudTpl.h:38
const CCVector3 * getPoint(unsigned index) const override
size_t Width() const
Definition: camera.h:160
size_t Height() const
Definition: camera.h:162
Eigen::Matrix3x4d ProjectionMatrix() const
Definition: image.cc:140
const std::string & Name() const
Definition: image.h:242
const class Image & Image(const image_t image_id) const
const class Camera & Camera(const camera_t camera_id) const
const std::vector< image_t > & RegImageIds() const
MvsTexturing(const Options &options, const Reconstruction &reconstruction, Workspace *workspace, const std::string &image_path)
Definition: texturing.cc:131
bool TextureMesh(ccMesh &mesh, const ::cloudViewer::camera::PinholeCameraTrajectory &camera_trajectory, const std::string &output_path)
Definition: texturing.cc:236
const Options & GetOptions() const
Definition: workspace.h:48
virtual const DepthMap & GetDepthMap(const int image_idx)
Definition: workspace.cc:123
const Model & GetModel() const
Definition: workspace.h:50
bool HasDepthMap(const int image_idx) const
Definition: workspace.cc:147
RGBA color structure.
std::vector< unsigned int > face
a[190]
normal_z y
normal_z x
Matrix< double, 3, 4 > Matrix3x4d
Definition: types.h:39
QTextStream & endl(QTextStream &stream)
Definition: QtCompat.h:718
CLOUDVIEWER_HOST_DEVICE Pair< First, Second > make_pair(const First &_first, const Second &_second)
Definition: SlabTraits.h:49
static const std::string path
Definition: PointCloud.cpp:59
MiniVec< float, N > floor(const MiniVec< float, N > &a)
Definition: MiniVec.h:75
MiniVec< float, N > ceil(const MiniVec< float, N > &a)
Definition: MiniVec.h:89
static TexturePatchCandidate GenerateCandidate(int label, TextureView *texture_view, const std::vector< size_t > &faces, const ccMesh &mesh, ccPointCloud *cloud, bool mesh_is_shifted=false)
Definition: texturing.cc:1074
std::string GetPathBaseName(const std::string &path)
Definition: misc.cc:118
void SplitFileExtension(const std::string &path, std::string *root, std::string *ext)
Definition: misc.cc:64
bool ExistsFile(const std::string &path)
Definition: misc.cc:100
std::string JoinPaths(T const &... paths)
Definition: misc.h:128
uint32_t image_t
Definition: types.h:61
std::string StringPrintf(const char *format,...)
Definition: string.cc:131
const image_t kInvalidImageId
Definition: types.h:76
std::string GetParentDir(const std::string &path)
Definition: misc.cc:128
constexpr Rgb black(0, 0, 0)
Eigen::Matrix< Index, 3, 1 > Vector3i
Definition: knncpp.h:30
Definition: Common.h:18
std::string to_string(const T &n)
Definition: Common.h:20
void swap(cloudViewer::core::SmallVectorImpl< T > &LHS, cloudViewer::core::SmallVectorImpl< T > &RHS)
Implement std::swap in terms of SmallVector swap.
Definition: SmallVector.h:1370
unsigned char uchar
Definition: shpopen.c:319
Generic saving parameters.
Definition: FileIOFilter.h:84
QWidget * parentWidget
Parent widget (if any)
Definition: FileIOFilter.h:93
2D texture coordinates
Triangle described by the indexes of its 3 vertices.
int GetImageIdx(const std::string &name) const
Definition: model.cc:127
std::unique_ptr< TexturePatch > texture_patch
Definition: texturing.cc:1064
bool IsInside(const TexturePatchCandidate &other) const
Definition: texturing.cc:1066
TexturePatch(int label, const std::vector< size_t > &faces, const std::vector< Eigen::Vector2f > &texcoords, std::shared_ptr< QImage > image, int min_x, int min_y, int max_x, int max_y)
Definition: texturing.cc:114
Eigen::Matrix4f world_to_cam
Definition: texturing.h:43
bool ValidPixel(const Eigen::Vector2f &pixel) const
Definition: texturing.cc:100
bool Inside(const Eigen::Vector3f &v1, const Eigen::Vector3f &v2, const Eigen::Vector3f &v3) const
Definition: texturing.cc:105
TextureView(std::size_t id, const std::string &image_file, const Eigen::Matrix3f &projection, const Eigen::Matrix4f &world_to_cam, const Eigen::Vector3f &pos, const Eigen::Vector3f &viewdir, int width, int height)
Definition: texturing.cc:54
Eigen::Vector2f GetPixelCoords(const Eigen::Vector3f &vertex) const
Definition: texturing.cc:71
Eigen::Matrix3f projection
Definition: texturing.h:42
std::string image_file
Definition: texturing.h:39
std::shared_ptr< QImage > image_data
Definition: texturing.h:48