1 // Generic simd conversions -*- C++ -*-
2
3 // Copyright (C) 2020-2023 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24
25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_
27
28 #if __cplusplus >= 201703L
29
30 _GLIBCXX_SIMD_BEGIN_NAMESPACE
31 // _SimdConverter scalar -> scalar {{{
32 template <typename _From, typename _To>
33 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::scalar,
34 enable_if_t<!is_same_v<_From, _To>>>
35 {
36 _GLIBCXX_SIMD_INTRINSIC constexpr _To operator()(_From __a) const noexcept
37 { return static_cast<_To>(__a); }
38 };
39
40 // }}}
41 // _SimdConverter scalar -> "native" {{{
42 template <typename _From, typename _To, typename _Abi>
43 struct _SimdConverter<_From, simd_abi::scalar, _To, _Abi,
44 enable_if_t<!is_same_v<_Abi, simd_abi::scalar>>>
45 {
46 using _Ret = typename _Abi::template __traits<_To>::_SimdMember;
47
48 template <typename... _More>
49 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
50 operator()(_From __a, _More... __more) const noexcept
51 {
52 static_assert(sizeof...(_More) + 1 == _Abi::template _S_size<_To>);
53 static_assert(conjunction_v<is_same<_From, _More>...>);
54 return __make_vector<_To>(__a, __more...);
55 }
56 };
57
58 // }}}
59 // _SimdConverter "native 1" -> "native 2" {{{
60 template <typename _From, typename _To, typename _AFrom, typename _ATo>
61 struct _SimdConverter<
62 _From, _AFrom, _To, _ATo,
63 enable_if_t<!disjunction_v<
64 __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>,
65 is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>,
66 conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>>>>
67 {
68 using _Arg = typename _AFrom::template __traits<_From>::_SimdMember;
69 using _Ret = typename _ATo::template __traits<_To>::_SimdMember;
70 using _V = __vector_type_t<_To, simd_size_v<_To, _ATo>>;
71
72 template <typename... _More>
73 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
74 operator()(_Arg __a, _More... __more) const noexcept
75 { return __vector_convert<_V>(__a, __more...); }
76 };
77
78 // }}}
79 // _SimdConverter scalar -> fixed_size<1> {{{1
80 template <typename _From, typename _To>
81 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::fixed_size<1>,
82 void>
83 {
84 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_To, simd_abi::scalar>
85 operator()(_From __x) const noexcept
86 { return {static_cast<_To>(__x)}; }
87 };
88
89 // _SimdConverter fixed_size<1> -> scalar {{{1
90 template <typename _From, typename _To>
91 struct _SimdConverter<_From, simd_abi::fixed_size<1>, _To, simd_abi::scalar,
92 void>
93 {
94 _GLIBCXX_SIMD_INTRINSIC constexpr _To
95 operator()(_SimdTuple<_From, simd_abi::scalar> __x) const noexcept
96 { return {static_cast<_To>(__x.first)}; }
97 };
98
99 // _SimdConverter fixed_size<_Np> -> fixed_size<_Np> {{{1
100 template <typename _From, typename _To, int _Np>
101 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To,
102 simd_abi::fixed_size<_Np>,
103 enable_if_t<!is_same_v<_From, _To>>>
104 {
105 using _Ret = __fixed_size_storage_t<_To, _Np>;
106 using _Arg = __fixed_size_storage_t<_From, _Np>;
107
108 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
109 operator()(const _Arg& __x) const noexcept
110 {
111 if constexpr (is_same_v<_From, _To>)
112 return __x;
113
114 // special case (optimize) int signedness casts
115 else if constexpr (sizeof(_From) == sizeof(_To)
116 && is_integral_v<_From> && is_integral_v<_To>)
117 return __bit_cast<_Ret>(__x);
118
119 // special case if all ABI tags in _Ret are scalar
120 else if constexpr (__is_scalar_abi<typename _Ret::_FirstAbi>())
121 {
122 return __call_with_subscripts(
123 __x, make_index_sequence<_Np>(),
124 [](auto... __values) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Ret {
125 return __make_simd_tuple<_To, decltype((void) __values,
126 simd_abi::scalar())...>(
127 static_cast<_To>(__values)...);
128 });
129 }
130
131 // from one vector to one vector
132 else if constexpr (_Arg::_S_first_size == _Ret::_S_first_size)
133 {
134 _SimdConverter<_From, typename _Arg::_FirstAbi, _To,
135 typename _Ret::_FirstAbi>
136 __native_cvt;
137 if constexpr (_Arg::_S_tuple_size == 1)
138 return {__native_cvt(__x.first)};
139 else
140 {
141 constexpr size_t _NRemain = _Np - _Arg::_S_first_size;
142 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To,
143 simd_abi::fixed_size<_NRemain>>
144 __remainder_cvt;
145 return {__native_cvt(__x.first), __remainder_cvt(__x.second)};
146 }
147 }
148
149 // from one vector to multiple vectors
150 else if constexpr (_Arg::_S_first_size > _Ret::_S_first_size)
151 {
152 const auto __multiple_return_chunks
153 = __convert_all<__vector_type_t<_To, _Ret::_S_first_size>>(
154 __x.first);
155 constexpr auto __converted = __multiple_return_chunks.size()
156 * _Ret::_FirstAbi::template _S_size<_To>;
157 constexpr auto __remaining = _Np - __converted;
158 if constexpr (_Arg::_S_tuple_size == 1 && __remaining == 0)
159 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks);
160 else if constexpr (_Arg::_S_tuple_size == 1)
161 { // e.g. <int, 3> -> <double, 2, 1> or <short, 7> -> <double, 4, 2,
162 // 1>
163 using _RetRem
164 = __remove_cvref_t<decltype(__simd_tuple_pop_front<__converted>(
165 _Ret()))>;
166 const auto __return_chunks2
167 = __convert_all<__vector_type_t<_To, _RetRem::_S_first_size>, 0,
168 __converted>(__x.first);
169 constexpr auto __converted2
170 = __converted
171 + __return_chunks2.size() * _RetRem::_S_first_size;
172 if constexpr (__converted2 == _Np)
173 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks,
174 __return_chunks2);
175 else
176 {
177 using _RetRem2 = __remove_cvref_t<
178 decltype(__simd_tuple_pop_front<__return_chunks2.size()
179 * _RetRem::_S_first_size>(
180 _RetRem()))>;
181 const auto __return_chunks3 = __convert_all<
182 __vector_type_t<_To, _RetRem2::_S_first_size>, 0,
183 __converted2>(__x.first);
184 constexpr auto __converted3
185 = __converted2
186 + __return_chunks3.size() * _RetRem2::_S_first_size;
187 if constexpr (__converted3 == _Np)
188 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks,
189 __return_chunks2,
190 __return_chunks3);
191 else
192 {
193 using _RetRem3
194 = __remove_cvref_t<decltype(__simd_tuple_pop_front<
195 __return_chunks3.size()
196 * _RetRem2::_S_first_size>(
197 _RetRem2()))>;
198 const auto __return_chunks4 = __convert_all<
199 __vector_type_t<_To, _RetRem3::_S_first_size>, 0,
200 __converted3>(__x.first);
201 constexpr auto __converted4
202 = __converted3
203 + __return_chunks4.size() * _RetRem3::_S_first_size;
204 if constexpr (__converted4 == _Np)
205 return __to_simd_tuple<_To, _Np>(
206 __multiple_return_chunks, __return_chunks2,
207 __return_chunks3, __return_chunks4);
208 else
209 __assert_unreachable<_To>();
210 }
211 }
212 }
213 else
214 {
215 constexpr size_t _NRemain = _Np - _Arg::_S_first_size;
216 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To,
217 simd_abi::fixed_size<_NRemain>>
218 __remainder_cvt;
219 return __simd_tuple_concat(
220 __to_simd_tuple<_To, _Arg::_S_first_size>(
221 __multiple_return_chunks),
222 __remainder_cvt(__x.second));
223 }
224 }
225
226 // from multiple vectors to one vector
227 // _Arg::_S_first_size < _Ret::_S_first_size
228 // a) heterogeneous input at the end of the tuple (possible with partial
229 // native registers in _Ret)
230 else if constexpr (_Ret::_S_tuple_size == 1
231 && _Np % _Arg::_S_first_size != 0)
232 {
233 static_assert(_Ret::_FirstAbi::template _S_is_partial<_To>);
234 return _Ret{__generate_from_n_evaluations<
235 _Np, typename _VectorTraits<typename _Ret::_FirstType>::type>(
236 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
237 return static_cast<_To>(__x[__i]);
238 })};
239 }
240 else
241 {
242 static_assert(_Arg::_S_tuple_size > 1);
243 constexpr auto __n
244 = __div_roundup(_Ret::_S_first_size, _Arg::_S_first_size);
245 return __call_with_n_evaluations<__n>(
246 [&__x](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
247 // assuming _Arg Abi tags for all __i are _Arg::_FirstAbi
248 _SimdConverter<_From, typename _Arg::_FirstAbi, _To,
249 typename _Ret::_FirstAbi>
250 __native_cvt;
251 if constexpr (_Ret::_S_tuple_size == 1)
252 return _Ret{__native_cvt(__uncvted...)};
253 else
254 return _Ret{
255 __native_cvt(__uncvted...),
256 _SimdConverter<
257 _From, simd_abi::fixed_size<_Np - _Ret::_S_first_size>, _To,
258 simd_abi::fixed_size<_Np - _Ret::_S_first_size>>()(
259 __simd_tuple_pop_front<_Ret::_S_first_size>(__x))};
260 }, [&__x](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
261 return __get_tuple_at<__i>(__x);
262 });
263 }
264 }
265 };
266
267 // _SimdConverter "native" -> fixed_size<_Np> {{{1
268 // i.e. 1 register to ? registers
269 template <typename _From, typename _Ap, typename _To, int _Np>
270 struct _SimdConverter<_From, _Ap, _To, simd_abi::fixed_size<_Np>,
271 enable_if_t<!__is_fixed_size_abi_v<_Ap>>>
272 {
273 static_assert(
274 _Np == simd_size_v<_From, _Ap>,
275 "_SimdConverter to fixed_size only works for equal element counts");
276
277 using _Ret = __fixed_size_storage_t<_To, _Np>;
278
279 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
280 operator()(typename _SimdTraits<_From, _Ap>::_SimdMember __x) const noexcept
281 {
282 if constexpr (_Ret::_S_tuple_size == 1)
283 return {__vector_convert<typename _Ret::_FirstType::_BuiltinType>(__x)};
284 else
285 {
286 using _FixedNp = simd_abi::fixed_size<_Np>;
287 _SimdConverter<_From, _FixedNp, _To, _FixedNp> __fixed_cvt;
288 using _FromFixedStorage = __fixed_size_storage_t<_From, _Np>;
289 if constexpr (_FromFixedStorage::_S_tuple_size == 1)
290 return __fixed_cvt(_FromFixedStorage{__x});
291 else if constexpr (_FromFixedStorage::_S_tuple_size == 2)
292 {
293 _FromFixedStorage __tmp;
294 static_assert(sizeof(__tmp) <= sizeof(__x));
295 __builtin_memcpy(&__tmp.first, &__x, sizeof(__tmp.first));
296 __builtin_memcpy(&__tmp.second.first,
297 reinterpret_cast<const char*>(&__x)
298 + sizeof(__tmp.first),
299 sizeof(__tmp.second.first));
300 return __fixed_cvt(__tmp);
301 }
302 else
303 __assert_unreachable<_From>();
304 }
305 }
306 };
307
308 // _SimdConverter fixed_size<_Np> -> "native" {{{1
309 // i.e. ? register to 1 registers
310 template <typename _From, int _Np, typename _To, typename _Ap>
311 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, _Ap,
312 enable_if_t<!__is_fixed_size_abi_v<_Ap>>>
313 {
314 static_assert(
315 _Np == simd_size_v<_To, _Ap>,
316 "_SimdConverter to fixed_size only works for equal element counts");
317
318 using _Arg = __fixed_size_storage_t<_From, _Np>;
319
320 _GLIBCXX_SIMD_INTRINSIC constexpr
321 typename _SimdTraits<_To, _Ap>::_SimdMember
322 operator()(const _Arg& __x) const noexcept
323 {
324 if constexpr (_Arg::_S_tuple_size == 1)
325 return __vector_convert<__vector_type_t<_To, _Np>>(__x.first);
326 else if constexpr (_Arg::_S_is_homogeneous)
327 return __call_with_n_evaluations<_Arg::_S_tuple_size>(
328 [](auto... __members) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
329 if constexpr ((is_convertible_v<decltype(__members), _To> && ...))
330 return __vector_type_t<_To, _Np>{static_cast<_To>(__members)...};
331 else
332 return __vector_convert<__vector_type_t<_To, _Np>>(__members...);
333 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
334 return __get_tuple_at<__i>(__x);
335 });
336 else if constexpr (__fixed_size_storage_t<_To, _Np>::_S_tuple_size == 1)
337 {
338 _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To,
339 simd_abi::fixed_size<_Np>>
340 __fixed_cvt;
341 return __fixed_cvt(__x).first;
342 }
343 else
344 {
345 const _SimdWrapper<_From, _Np> __xv
346 = __generate_from_n_evaluations<_Np, __vector_type_t<_From, _Np>>(
347 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
348 return __vector_convert<__vector_type_t<_To, _Np>>(__xv);
349 }
350 }
351 };
352
353 // }}}1
354 _GLIBCXX_SIMD_END_NAMESPACE
355 #endif // __cplusplus >= 201703L
356 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_
357
358 // vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80