ACloudViewer  3.9.4
A Modern Library for 3D Data Processing
ContinuousConvTranspose.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - CloudViewer: www.cloudViewer.org -
3 // ----------------------------------------------------------------------------
4 // Copyright (c) 2018-2024 www.cloudViewer.org
5 // SPDX-License-Identifier: MIT
6 // ----------------------------------------------------------------------------
7 
8 #pragma once
9 
10 #include <tbb/parallel_for.h>
11 
12 #include "ml/impl/continuous_conv/CoordinateTransformation.h"
13 
14 namespace cloudViewer {
15 namespace ml {
16 namespace impl {
17 
20 template <class TFeat,
21  class TOut,
22  class TReal,
23  class TIndex,
24  InterpolationMode INTERPOLATION,
25  CoordinateMapping MAPPING,
26  bool ALIGN_CORNERS,
27  bool INDIVIDUAL_EXTENT,
28  bool ISOTROPIC_EXTENT,
29  bool NORMALIZE>
30 void _CConvTransposeComputeFeaturesCPU(
31  TOut* out_features,
32  const std::vector<int>& filter_dims,
33  const TFeat* filter,
34  size_t num_out,
35  const TReal* out_positions,
36  const TFeat* out_importance,
37  size_t num_inp,
38  const TReal* inp_positions,
39  const TFeat* inp_features,
40  const TFeat* inp_neighbors_importance_sum,
41  const int64_t* inp_neighbors_row_splits,
42  size_t neighbors_index_size,
43  const TIndex* neighbors_index,
44  const TFeat* neighbors_importance,
45  const int64_t* neighbors_row_splits,
46  const TReal* extents,
47  const TReal* offsets) {
48  const bool NEIGHBORS_IMPORTANCE = inp_neighbors_importance_sum;
49  // const int VECSIZE = 32;
50 #define VECSIZE 32
51  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
52  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
53  InterpolationVec_t interpolation;
54 
55  const int in_channels = filter_dims[filter_dims.size() - 2];
56  const int out_channels = filter_dims[filter_dims.size() - 1];
57 
58  int spatial_filter_size = 1;
59  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
60  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
61  filter_dims[0]);
62 
63  memset(out_features, 0, sizeof(TOut) * num_out * out_channels);
64 
65  typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
66  typedef Eigen::Array<TReal, VECSIZE, 3> Matrix3C;
67 
68  tbb::parallel_for(
69  tbb::blocked_range<size_t>(0, num_out, 32),
70  [&](const tbb::blocked_range<size_t>& r) {
71  int range_length = r.end() - r.begin();
72 
73  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
74  in_channels * spatial_filter_size, range_length);
75  B.setZero();
76 
77  Matrix infeat(VECSIZE, in_channels);
78 
79  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
80  offsets[2]);
81 
82  Matrix3C inv_extents;
83  if (INDIVIDUAL_EXTENT == false) {
84  if (ISOTROPIC_EXTENT) {
85  inv_extents = 1 / extents[0];
86  } else {
87  inv_extents.col(0) = 1 / extents[0];
88  inv_extents.col(1) = 1 / extents[1];
89  inv_extents.col(2) = 1 / extents[2];
90  }
91  }
92 
93  for (size_t out_idx = r.begin(); out_idx != r.end();
94  ++out_idx) {
95  const int out_col = out_idx - r.begin();
96  const size_t neighbor_start = neighbors_row_splits[out_idx];
97  const size_t neighbor_end =
98  (out_idx + 1 < num_out
99  ? neighbors_row_splits[out_idx + 1]
100  : neighbors_index_size);
101 
102  typename InterpolationVec_t::Weight_t interp_weights;
103  typename InterpolationVec_t::Idx_t interp_indices;
104 
105  int vec_valid_count = 0;
106  Vec_t x, y, z;
107 
108  // set to zero to avoid problems with vectors with less than
109  // VECSIZE valid entries
110  x.setZero();
111  y.setZero();
112  z.setZero();
113  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
114  const size_t inp_idx = neighbors_index[n];
115 
116  const int i = vec_valid_count;
117  x(i) = out_positions[out_idx * 3 + 0] -
118  inp_positions[inp_idx * 3 + 0];
119  y(i) = out_positions[out_idx * 3 + 1] -
120  inp_positions[inp_idx * 3 + 1];
121  z(i) = out_positions[out_idx * 3 + 2] -
122  inp_positions[inp_idx * 3 + 2];
123 
124  if (INDIVIDUAL_EXTENT) {
125  if (ISOTROPIC_EXTENT) {
126  inv_extents.row(i) = 1 / extents[inp_idx];
127  } else {
128  inv_extents(i, 0) =
129  1 / extents[3 * inp_idx + 0];
130  inv_extents(i, 1) =
131  1 / extents[3 * inp_idx + 1];
132  inv_extents(i, 2) =
133  1 / extents[3 * inp_idx + 2];
134  }
135  }
136 
137  TFeat n_importance = NEIGHBORS_IMPORTANCE
138  ? neighbors_importance[n]
139  : TFeat(1);
140  for (int ic = 0; ic < in_channels; ++ic)
141  infeat(i, ic) =
142  inp_features[inp_idx * in_channels + ic] *
143  n_importance;
144 
145  if (NORMALIZE) {
146  TFeat normalizer(1);
147  if (NEIGHBORS_IMPORTANCE) {
148  if (inp_neighbors_importance_sum[inp_idx] !=
149  TFeat(0))
150  normalizer /= inp_neighbors_importance_sum
151  [inp_idx];
152  } else {
153  size_t num_inp_neighbors;
154  const size_t inp_neighbor_start =
155  inp_neighbors_row_splits[inp_idx];
156  const size_t inp_neighbor_end =
157  inp_neighbors_row_splits[inp_idx + 1];
158  num_inp_neighbors =
159  inp_neighbor_end - inp_neighbor_start;
160  if (num_inp_neighbors > 0)
161  normalizer /= TFeat(num_inp_neighbors);
162  }
163  for (int ic = 0; ic < in_channels; ++ic)
164  infeat(i, ic) *= normalizer;
165  }
166 
167  ++vec_valid_count;
168  if (vec_valid_count == VECSIZE ||
169  n + 1 == neighbor_end) {
170  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING,
171  TReal, VECSIZE>(
172  x, y, z, filter_size_xyz, inv_extents,
173  offsets_);
174  interpolation.Interpolate(
175  interp_weights, interp_indices, x, y, z,
176  filter_size_xyz, in_channels);
177  for (int k = 0; k < vec_valid_count; ++k) {
178  for (int j = 0; j < InterpolationVec_t::Size();
179  ++j) {
180  for (int ic = 0; ic < in_channels; ++ic)
181  B(interp_indices(j, k) + ic, out_col) +=
182  TFeat(interp_weights(j, k)) *
183  infeat(k, ic);
184  }
185  }
186  vec_valid_count = 0;
187  }
188  }
189 
190  } // out_idx
191 
192  Eigen::Map<const Eigen::Matrix<TFeat, Eigen::Dynamic,
193  Eigen::Dynamic>>
194  A(filter, out_channels,
195  spatial_filter_size * in_channels);
196  Eigen::Map<Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic>>
197  C(out_features + (r.begin() * out_channels),
198  out_channels, range_length);
199 
200  C = (A * B).template cast<TOut>();
201  if (out_importance) {
202  for (int i = 0; i < range_length; ++i)
203  C.col(i) *= TOut(out_importance[r.begin() + i]);
204  }
205  });
206 
207 #undef VECSIZE
208 }
209 
291 template <class TFeat, class TOut, class TReal, class TIndex>
292 void CConvTransposeComputeFeaturesCPU(TOut* out_features,
293  const std::vector<int>& filter_dims,
294  const TFeat* filter,
295  size_t num_out,
296  const TReal* out_positions,
297  const TFeat* out_importance,
298  size_t num_inp,
299  const TReal* inp_positions,
300  const TFeat* inp_features,
301  const TFeat* inp_neighbors_importance_sum,
302  const int64_t* inp_neighbors_row_splits,
303  size_t neighbors_index_size,
304  const TIndex* neighbors_index,
305  const TFeat* neighbors_importance,
306  const int64_t* neighbors_row_splits,
307  const TReal* extents,
308  const TReal* offsets,
309  InterpolationMode interpolation,
310  CoordinateMapping coordinate_mapping,
311  bool align_corners,
312  bool individual_extent,
313  bool isotropic_extent,
314  bool normalize) {
315 #define FN_PARAMETERS \
316  out_features, filter_dims, filter, num_out, out_positions, out_importance, \
317  num_inp, inp_positions, inp_features, \
318  inp_neighbors_importance_sum, inp_neighbors_row_splits, \
319  neighbors_index_size, neighbors_index, neighbors_importance, \
320  neighbors_row_splits, extents, offsets
321 
322 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
323  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE) \
324  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
325  ALIGN_CORNERS == align_corners && \
326  INDIVIDUAL_EXTENT == individual_extent && \
327  ISOTROPIC_EXTENT == isotropic_extent && NORMALIZE == normalize) \
328  _CConvTransposeComputeFeaturesCPU<TFeat, TOut, TReal, TIndex, \
329  INTERPOLATION, MAPPING, \
330  ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
331  ISOTROPIC_EXTENT, NORMALIZE>( \
332  FN_PARAMETERS);
333 
334 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
335  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
336  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
337  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
338  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
339  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
340  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
341  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
342  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
343  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
344  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
345  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
346  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
347  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
348  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
349  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
350  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
351 
352 #define CALL_TEMPLATE3(INTERPOLATION) \
353  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
354  CALL_TEMPLATE2(INTERPOLATION, \
355  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
356  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
357 
358 #define CALL_TEMPLATE4 \
359  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
360  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
361  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
362 
363  CALL_TEMPLATE4
364 
365 #undef CALL_TEMPLATE
366 #undef CALL_TEMPLATE2
367 #undef CALL_TEMPLATE3
368 #undef CALL_TEMPLATE4
369 
370 #undef FN_PARAMETERS
371 }
372 
373 } // namespace impl
374 } // namespace ml
375 } // namespace cloudViewer
CALL_TEMPLATE4
#define CALL_TEMPLATE4
VECSIZE
#define VECSIZE
normalize
__host__ __device__ float2 normalize(float2 v)
Definition: cutil_math.h:1179
cloudViewer::ml::impl::_CConvTransposeComputeFeaturesCPU
void _CConvTransposeComputeFeaturesCPU(TOut *out_features, const std::vector< int > &filter_dims, const TFeat *filter, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets)
Definition: ContinuousConvTranspose.h:30
cloudViewer::ml::impl::CConvTransposeComputeFeaturesCPU
void CConvTransposeComputeFeaturesCPU(TOut *out_features, const std::vector< int > &filter_dims, const TFeat *filter, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvTranspose.h:292
cloudViewer::ml::impl::ComputeFilterCoordinates
void ComputeFilterCoordinates(Eigen::Array< T, VECSIZE, 1 > &x, Eigen::Array< T, VECSIZE, 1 > &y, Eigen::Array< T, VECSIZE, 1 > &z, const Eigen::Array< int, 3, 1 > &filter_size, const Eigen::Array< T, VECSIZE, 3 > &inv_extents, const Eigen::Array< T, 3, 1 > &offset)
Definition: CoordinateTransformation.h:107
cloudViewer
Generic file read and write utility for python interface.
Definition: AutoSegmentationTools.h:16
cloudViewer::ml::impl::InterpolationVec
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:185