Skip to content

Commit b05321a

Browse files
authored
Merge pull request #268 from zhujun98/implement_parallel_azimuthal_integration
Implement parallel azimuthal integration
2 parents 3e24752 + 5eb2168 commit b05321a

File tree

8 files changed

+233
-32
lines changed

8 files changed

+233
-32
lines changed

benchmarks/azimuthal_integration.ipynb

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,22 @@
4444
"\n",
4545
"# img, mask = load_image(\"jf_ring.npy\")\n",
4646
"# cy, cx = -33, 1112\n",
47+
"# pixel1, pixel2 = 75e-6, 75e-6 # pixel size (y, x)\n",
4748
"\n",
4849
"# img, mask = load_image(\"jf_ring_6modules.npy\")\n",
49-
"# cy, cx = 537, 1132 \n",
50+
"# cy, cx = 537, 1132\n",
51+
"# pixel1, pixel2 = 75e-6, 75e-6 # pixel size (y, x)\n",
5052
"\n",
5153
"img, mask = load_image(\"lpd.npy\")\n",
52-
"cy, cx = 606, 554"
54+
"cy, cx = 606, 554\n",
55+
"pixel1, pixel2 = 200e-6, 200e-6 # pixel size (y, x)"
56+
]
57+
},
58+
{
59+
"cell_type": "markdown",
60+
"metadata": {},
61+
"source": [
62+
"#### Integrate a single image"
5363
]
5464
},
5565
{
@@ -60,7 +70,6 @@
6070
"source": [
6171
"dist = 1 # sample distance\n",
6272
"npt = 1024 # number of integration points\n",
63-
"pixel1, pixel2 = 0.75e-6, 0.75e-6 # pixel size (y, x)\n",
6473
"poni1, poni2 = cy * pixel1, cx * pixel2 # integration center (y, x)"
6574
]
6675
},
@@ -126,14 +135,54 @@
126135
"ax.legend(fontsize=16)"
127136
]
128137
},
138+
{
139+
"cell_type": "markdown",
140+
"metadata": {},
141+
"source": [
142+
"#### Integrate an array of images"
143+
]
144+
},
129145
{
130146
"cell_type": "code",
131147
"execution_count": null,
132148
"metadata": {},
133149
"outputs": [],
134150
"source": [
135-
"# %%timeit\n",
151+
"import multiprocessing as mp\n",
152+
"\n",
153+
"print(mp.cpu_count())"
154+
]
155+
},
156+
{
157+
"cell_type": "code",
158+
"execution_count": null,
159+
"metadata": {},
160+
"outputs": [],
161+
"source": [
162+
"img_array = np.tile(img, (40, 1, 1))\n",
163+
"print(img_array.shape)\n",
164+
"\n",
165+
"q_a, I_a = integrator.integrate1d(img_array, npt=npt)\n",
166+
"np.testing.assert_array_equal(q_a, q)\n",
167+
"np.testing.assert_array_equal(I_a[0], I)\n",
168+
"np.testing.assert_array_equal(I_a[39], I)\n",
136169
"\n",
170+
"%timeit integrator.integrate1d(img_array, npt=npt)"
171+
]
172+
},
173+
{
174+
"cell_type": "markdown",
175+
"metadata": {},
176+
"source": [
177+
"#### Concentric ring finding"
178+
]
179+
},
180+
{
181+
"cell_type": "code",
182+
"execution_count": null,
183+
"metadata": {},
184+
"outputs": [],
185+
"source": [
137186
"min_count = 500\n",
138187
"prominence = 100\n",
139188
"distance = 10\n",
@@ -142,6 +191,15 @@
142191
"cx, cy = finder.search(img, cx, cy, min_count=min_count)"
143192
]
144193
},
194+
{
195+
"cell_type": "code",
196+
"execution_count": null,
197+
"metadata": {},
198+
"outputs": [],
199+
"source": [
200+
"%timeit finder.search(img, cx, cy, min_count=min_count)"
201+
]
202+
},
145203
{
146204
"cell_type": "code",
147205
"execution_count": null,
@@ -172,9 +230,9 @@
172230
],
173231
"metadata": {
174232
"kernelspec": {
175-
"display_name": "foam-gcc6",
233+
"display_name": "Python [conda env:foam-test]",
176234
"language": "python",
177-
"name": "foam-gcc6"
235+
"name": "conda-env-foam-test-py"
178236
},
179237
"language_info": {
180238
"codemirror_mode": {
@@ -186,7 +244,7 @@
186244
"name": "python",
187245
"nbconvert_exporter": "python",
188246
"pygments_lexer": "ipython3",
189-
"version": "3.7.6"
247+
"version": "3.7.7"
190248
}
191249
},
192250
"nbformat": 4,

docs/installation.rst

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,19 +41,29 @@ Install **EXtra-foam**
4141
$ git submodule update --init
4242
4343
$ cd EXtra-foam
44+
$ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
45+
$ pip install .
4446
45-
# optional
46-
$ export FOAM_USE_TBB=0 # turn off intel TBB in extra-foam
47-
$ export XTENSOR_USE_TBB=0 # turn off intel TBB in xtensor
48-
$ export FOAM_USE_XSIMD=0 # turn off XSIMD in extra-foam
49-
$ export XTENSOR_USE_XSIMD=0 # turn off XSIMD in xtensor
5047
51-
# Note: This step is also required if one wants to change the above
52-
# environmental parameters.
53-
$ python setup.py clean # alternatively "rm -r build"
48+
Install your own **EXtra-foam** kernel on the Maxwell cluster for offline analysis
49+
----------------------------------------------------------------------------------
5450

55-
$ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
56-
$ pip install .
51+
For now, there is no documentation for the Python bindings of the C++ implementations in
52+
**EXtra-foam**. If you are interested in using those super fast C++ implementation to
53+
accelerate your analysis, please feel free to dig into the code and ask questions.
54+
55+
.. code-block:: bash
56+
57+
# ssh to the Maxwell cluster and then
58+
$ module load anaconda3
59+
$ conda create --name extra-foam-offline -y
60+
$ conda activate extra-foam-offline
61+
62+
# follow the previous steps to install EXtra-foam
63+
64+
$ conda install ipykernel nb_conda_kernels -y
65+
66+
# Now you should be able to load the newly created kernel on max-jhub.
5767
5868
5969
Install C++ API of **EXtra-foam** only

extra_foam/algorithms/tests/test_azimuthal_integ.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,20 @@ def testIntegrate1D(self, dtype):
3737
assert q0 == q1
3838
assert s0 == s1
3939

40-
q10, s10 = integrator.integrate1d(img, npt=10, min_count=img.size)
41-
assert not np.any(s10)
40+
# test threshold
41+
q10_0, s10_0 = integrator.integrate1d(img, npt=10, min_count=img.size)
42+
assert not np.any(s10_0)
43+
44+
# test integrate an array of images
45+
q10, s10 = integrator.integrate1d(img, npt=10)
46+
img_a = np.tile(img, (4, 1, 1))
47+
img_a[3, ...] = img - 1000
48+
q10_a, s10_a = integrator.integrate1d(img_a, npt=10)
49+
np.testing.assert_array_equal(q10_a, q10)
50+
for i in range(3):
51+
np.testing.assert_array_equal(s10_a[i], s10)
52+
_, s10_2 = integrator.integrate1d(img - 1000, npt=10)
53+
np.testing.assert_array_equal(s10_a[3], s10_2)
4254

4355
q100, s100 = integrator.integrate1d(img, npt=999)
4456

src/extra_foam/f_azimuthal_integrator.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,19 @@ void declareAzimuthalIntegrator(py::module& m)
4141
AZIMUTHAL_INTEGRATE1D(float)
4242
AZIMUTHAL_INTEGRATE1D(uint16_t)
4343
AZIMUTHAL_INTEGRATE1D(int16_t)
44+
45+
#define AZIMUTHAL_INTEGRATE1D_PARA(DTYPE) \
46+
cls.def("integrate1d", (std::pair<foam::ReducedVectorTypeFromArray<xt::pytensor<value_type, 3>>, \
47+
foam::ReducedImageType<xt::pytensor<value_type, 3>>> \
48+
(Integrator::*)(const xt::pytensor<DTYPE, 3>&, size_t, size_t, \
49+
foam::AzimuthalIntegrationMethod)) \
50+
&Integrator::template integrate1d<const xt::pytensor<DTYPE, 3>&>, \
51+
py::arg("src").noconvert(), py::arg("npt"), py::arg("min_count")=1, \
52+
py::arg("method")=foam::AzimuthalIntegrationMethod::HISTOGRAM);
53+
54+
AZIMUTHAL_INTEGRATE1D_PARA(float)
55+
AZIMUTHAL_INTEGRATE1D_PARA(uint16_t)
56+
AZIMUTHAL_INTEGRATE1D_PARA(int16_t)
4457
}
4558

4659
void declareConcentricRingsFinder(py::module& m)

src/extra_foam/include/f_azimuthal_integrator.hpp

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ namespace foam
3232
namespace
3333
{
3434

35-
template<typename T, typename E, EnableIf<std::decay_t<E>, IsImage> = false>
35+
template<typename T, typename E>
3636
auto computeGeometry(E&& src, T poni1, T poni2, T pixel1, T pixel2)
3737
{
3838
auto shape = src.shape();
@@ -51,20 +51,13 @@ auto computeGeometry(E&& src, T poni1, T poni2, T pixel1, T pixel2)
5151
return geometry;
5252
}
5353

54-
template<typename T, typename E, EnableIf<std::decay_t<E>, IsImage> = false>
55-
auto histogramAI(E&& src, const xt::xtensor<T, 2>& geometry, T q_min, T q_max,
56-
size_t n_bins, size_t min_count=1)
54+
template<typename E1, typename E2, typename E3, typename T>
55+
void histogramAI(E1&& src, const E2& geometry, E3& hist, T q_min, T q_max, size_t n_bins, size_t min_count)
5756
{
58-
auto shape = src.shape();
59-
60-
using vector_type = ReducedVectorType<E, T>;
61-
6257
T norm = T(1) / (q_max - q_min);
63-
64-
vector_type edges = xt::linspace<T>(q_min, q_max, n_bins + 1);
65-
vector_type hist = xt::zeros<T>({ n_bins });
6658
xt::xtensor<size_t, 1> counts = xt::zeros<size_t>({ n_bins });
6759

60+
auto shape = src.shape();
6861
for (size_t i = 0; i < shape[0]; ++i)
6962
{
7063
for (size_t j = 0; j < shape[1]; ++j)
@@ -103,7 +96,21 @@ auto histogramAI(E&& src, const xt::xtensor<T, 2>& geometry, T q_min, T q_max,
10396
else
10497
hist(i) /= counts(i);
10598
}
99+
}
100+
101+
template<typename T, typename E, EnableIf<std::decay_t<E>, IsImage> = false>
102+
auto histogramAI(E&& src, const xt::xtensor<T, 2>& geometry, T q_min, T q_max,
103+
size_t n_bins, size_t min_count=1)
104+
{
105+
auto shape = src.shape();
106106

107+
using vector_type = ReducedVectorType<E, T>;
108+
109+
vector_type hist = xt::zeros<T>({ n_bins });
110+
111+
histogramAI(std::forward<E>(src), geometry, hist, q_min, q_max, n_bins, min_count);
112+
113+
vector_type edges = xt::linspace<T>(q_min, q_max, n_bins + 1);
107114
auto&& centers = 0.5 * (xt::view(edges, xt::range(0, -1)) + xt::view(edges, xt::range(1, xt::placeholders::_)));
108115

109116
return std::make_pair<vector_type, vector_type>(centers, std::move(hist));
@@ -119,6 +126,43 @@ auto histogramAI(E&& src, T poni1, T poni2, T pixel1, T pixel2, size_t npt, size
119126
return histogramAI<T>(std::forward<E>(src), geometry, bounds[0], bounds[1], npt, min_count);
120127
}
121128

129+
template<typename T, typename E, EnableIf<std::decay_t<E>, IsImageArray> = false>
130+
auto histogramAI(E&& src, const xt::xtensor<T, 2>& geometry, T q_min, T q_max,
131+
size_t n_bins, size_t min_count=1)
132+
{
133+
size_t np = src.shape()[0];
134+
135+
using vector_type = ReducedVectorTypeFromArray<E, T>;
136+
using image_type = ReducedImageType<E, T>;
137+
138+
image_type hist = xt::zeros<T>({ np, n_bins });
139+
140+
#if defined(FOAM_USE_TBB)
141+
tbb::parallel_for(tbb::blocked_range<int>(0, np),
142+
[&src, &geometry, &hist, q_min, q_max, n_bins, min_count]
143+
(const tbb::blocked_range<int> &block)
144+
{
145+
for(int k=block.begin(); k != block.end(); ++k)
146+
{
147+
#else
148+
for (size_t k = 0; k < np; ++k)
149+
{
150+
#endif
151+
auto hist_view = xt::view(hist, k, xt::all());
152+
histogramAI(xt::view(src, k, xt::all(), xt::all()), geometry, hist_view,
153+
q_min, q_max, n_bins, min_count);
154+
}
155+
#if defined(FOAM_USE_TBB)
156+
}
157+
);
158+
#endif
159+
160+
vector_type edges = xt::linspace<T>(q_min, q_max, n_bins + 1);
161+
auto&& centers = 0.5 * (xt::view(edges, xt::range(0, -1)) + xt::view(edges, xt::range(1, xt::placeholders::_)));
162+
163+
return std::make_pair<vector_type, image_type>(centers, std::move(hist));
164+
}
165+
122166
} //namespace
123167

124168
enum class AzimuthalIntegrationMethod
@@ -186,6 +230,10 @@ class AzimuthalIntegrator
186230
template<typename E, EnableIf<std::decay_t<E>, IsImage> = false>
187231
auto integrate1d(E&& src, size_t npt, size_t min_count=1,
188232
AzimuthalIntegrationMethod method=AzimuthalIntegrationMethod::HISTOGRAM);
233+
234+
template<typename E, EnableIf<std::decay_t<E>, IsImageArray> = false>
235+
auto integrate1d(E&& src, size_t npt, size_t min_count=1,
236+
AzimuthalIntegrationMethod method=AzimuthalIntegrationMethod::HISTOGRAM);
189237
};
190238

191239
template<typename T>
@@ -249,6 +297,34 @@ auto AzimuthalIntegrator<T>::integrate1d(E&& src,
249297
}
250298
}
251299

300+
template<typename T>
301+
template<typename E, EnableIf<std::decay_t<E>, IsImageArray>>
302+
auto AzimuthalIntegrator<T>::integrate1d(E&& src,
303+
size_t npt,
304+
size_t min_count,
305+
AzimuthalIntegrationMethod method)
306+
{
307+
if (npt == 0) npt = 1;
308+
309+
auto src_shape = src.shape();
310+
std::array<size_t, 2> q_shape = q_.shape();
311+
if (!initialized_ || src_shape[1] != q_shape[0] || src_shape[2] != q_shape[1])
312+
{
313+
initQ(xt::view(src, 0, xt::all(), xt::all()));
314+
initialized_ = true;
315+
}
316+
317+
switch(method)
318+
{
319+
case AzimuthalIntegrationMethod::HISTOGRAM:
320+
{
321+
return histogramAI(std::forward<E>(src), q_, q_min_, q_max_, npt, min_count);
322+
}
323+
default:
324+
throw std::runtime_error("Unknown azimuthal integration method");
325+
}
326+
}
327+
252328
/**
253329
* class for finding the center of concentric rings in an image.
254330
*/

src/extra_foam/include/f_traits.hpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ using ReducedVectorType = decltype(xt::eval(xt::sum<std::conditional_t<std::is_s
8282
typename std::decay_t<E>::value_type,
8383
T>>(std::declval<E>(), {0})));
8484

85+
template<typename E, typename T = void, EnableIf<std::decay_t<E>, IsImageArray> = false>
86+
using ReducedVectorTypeFromArray = decltype(xt::eval(xt::sum<std::conditional_t<std::is_same<T, void>::value,
87+
typename std::decay_t<E>::value_type,
88+
T>>(std::declval<E>(), {0, 1})));
89+
8590
template<typename E, typename T = void, EnableIf<std::decay_t<E>, IsImageArray> = false>
8691
using ReducedImageType = decltype(xt::eval(xt::sum<std::conditional_t<std::is_same<T, void>::value,
8792
typename std::decay_t<E>::value_type,

test/test_azimuthal_integrator.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ TEST(TestAzimuthalIntegrator, TestDataType)
4949
TEST(TestAzimuthalIntegrator, TestIntegrator1D)
5050
{
5151
xt::xtensor<float, 2> src = xt::arange(1024).reshape({16, 128});
52+
xt::xtensor<float, 2> src2 = src - 100;
53+
auto src_a = xt::xtensor<float, 3>::from_shape({4, 16, 128});
54+
for (size_t i = 0; i < 3; ++i) xt::view(src_a, i, xt::all(), xt::all()) = src;
55+
xt::view(src_a, 3, xt::all(), xt::all()) = src2;
5256

5357
double distance = 0.2;
5458
double pixel1 = 1e-4;
@@ -69,12 +73,19 @@ TEST(TestAzimuthalIntegrator, TestIntegrator1D)
6973
EXPECT_EQ(ret10.first, ret10_cut.first);
7074
EXPECT_THAT(ret10_cut.second, Each(Eq(0.)));
7175

76+
// test integrate an array of images
77+
auto ret10_a = itgt.integrate1d(src_a, 10);
78+
EXPECT_EQ(ret10.first, ret10_a.first);
79+
for (size_t i = 0; i < 3; ++i) EXPECT_EQ(ret10.second, xt::view(ret10_a.second, i, xt::all()));
80+
auto ret10_2 = itgt.integrate1d(src2, 10);
81+
EXPECT_EQ(ret10_2.second, xt::view(ret10_a.second, 3, xt::all()));
82+
7283
// big npt
7384
itgt.integrate1d(src, 999);
7485

7586
// data has a single value
76-
xt::xtensor<double, 2> src2 = xt::ones<double>({16, 128});
77-
itgt.integrate1d(src2, 10);
87+
xt::xtensor<double, 2> src_single_value = xt::ones<double>({16, 128});
88+
itgt.integrate1d(src_single_value, 10);
7889

7990
// integral source value type
8091
xt::xtensor<uint16_t, 2> src_int = xt::arange(1024).reshape({16, 128});

0 commit comments

Comments
 (0)