1 /** 2 * SSE intrinsics. 3 * https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#techs=SSE 4 * 5 * Copyright: Copyright Guillaume Piolat 2016-2020. 6 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 7 */ 8 module inteli.xmmintrin; 9 10 public import inteli.types; 11 12 import inteli.internals; 13 14 import inteli.mmx; 15 import inteli.emmintrin; 16 17 import core.stdc.stdlib: malloc, free; 18 import core.stdc.string: memcpy; 19 import core.exception: onOutOfMemoryError; 20 21 version(D_InlineAsm_X86) 22 version = InlineX86Asm; 23 else version(D_InlineAsm_X86_64) 24 version = InlineX86Asm; 25 26 27 // SSE1 28 29 nothrow @nogc: 30 31 32 enum int _MM_EXCEPT_INVALID = 0x0001; /// MXCSR Exception states. 33 enum int _MM_EXCEPT_DENORM = 0x0002; ///ditto 34 enum int _MM_EXCEPT_DIV_ZERO = 0x0004; ///ditto 35 enum int _MM_EXCEPT_OVERFLOW = 0x0008; ///ditto 36 enum int _MM_EXCEPT_UNDERFLOW = 0x0010; ///ditto 37 enum int _MM_EXCEPT_INEXACT = 0x0020; ///ditto 38 enum int _MM_EXCEPT_MASK = 0x003f; /// MXCSR Exception states mask. 39 40 enum int _MM_MASK_INVALID = 0x0080; /// MXCSR Exception masks. 41 enum int _MM_MASK_DENORM = 0x0100; ///ditto 42 enum int _MM_MASK_DIV_ZERO = 0x0200; ///ditto 43 enum int _MM_MASK_OVERFLOW = 0x0400; ///ditto 44 enum int _MM_MASK_UNDERFLOW = 0x0800; ///ditto 45 enum int _MM_MASK_INEXACT = 0x1000; ///ditto 46 enum int _MM_MASK_MASK = 0x1f80; /// MXCSR Exception masks mask. 47 48 enum int _MM_ROUND_NEAREST = 0x0000; /// MXCSR Rounding mode. 49 enum int _MM_ROUND_DOWN = 0x2000; ///ditto 50 enum int _MM_ROUND_UP = 0x4000; ///ditto 51 enum int _MM_ROUND_TOWARD_ZERO = 0x6000; ///ditto 52 enum int _MM_ROUND_MASK = 0x6000; /// MXCSR Rounding mode mask. 53 54 enum int _MM_FLUSH_ZERO_MASK = 0x8000; /// MXCSR Denormal flush to zero mask. 55 enum int _MM_FLUSH_ZERO_ON = 0x8000; /// MXCSR Denormal flush to zero modes. 56 enum int _MM_FLUSH_ZERO_OFF = 0x0000; ///ditto 57 58 /// Add packed single-precision (32-bit) floating-point elements in `a` and `b`. 59 __m128 _mm_add_ps(__m128 a, __m128 b) pure @safe 60 { 61 pragma(inline, true); 62 return a + b; 63 } 64 unittest 65 { 66 __m128 a = [1, 2, 3, 4]; 67 a = _mm_add_ps(a, a); 68 assert(a.array[0] == 2); 69 assert(a.array[1] == 4); 70 assert(a.array[2] == 6); 71 assert(a.array[3] == 8); 72 } 73 74 /// Add the lower single-precision (32-bit) floating-point element 75 /// in `a` and `b`, store the result in the lower element of result, 76 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 77 __m128 _mm_add_ss(__m128 a, __m128 b) pure @safe 78 { 79 static if (GDC_with_SSE) 80 { 81 return __builtin_ia32_addss(a, b); 82 } 83 else static if (DMD_with_DSIMD) 84 { 85 return cast(__m128) __simd(XMM.ADDSS, a, b); 86 } 87 else 88 { 89 a[0] += b[0]; 90 return a; 91 } 92 } 93 unittest 94 { 95 __m128 a = [1, 2, 3, 4]; 96 a = _mm_add_ss(a, a); 97 assert(a.array == [2.0f, 2, 3, 4]); 98 } 99 100 /// Compute the bitwise AND of packed single-precision (32-bit) floating-point elements in `a` and `b`. 101 __m128 _mm_and_ps (__m128 a, __m128 b) pure @safe 102 { 103 pragma(inline, true); 104 return cast(__m128)(cast(__m128i)a & cast(__m128i)b); 105 } 106 unittest 107 { 108 float a = 4.32f; 109 float b = -78.99f; 110 int correct = (*cast(int*)(&a)) & (*cast(int*)(&b)); 111 __m128 A = _mm_set_ps(a, b, a, b); 112 __m128 B = _mm_set_ps(b, a, b, a); 113 int4 R = cast(int4)( _mm_and_ps(A, B) ); 114 assert(R.array[0] == correct); 115 assert(R.array[1] == correct); 116 assert(R.array[2] == correct); 117 assert(R.array[3] == correct); 118 } 119 120 /// Compute the bitwise NOT of packed single-precision (32-bit) floating-point elements in `a` and then AND with `b`. 121 __m128 _mm_andnot_ps (__m128 a, __m128 b) pure @safe 122 { 123 static if (DMD_with_DSIMD) 124 return cast(__m128) __simd(XMM.ANDNPS, a, b); 125 else 126 return cast(__m128)( (~cast(__m128i)a) & cast(__m128i)b ); 127 } 128 unittest 129 { 130 float a = 4.32f; 131 float b = -78.99f; 132 int correct = ~(*cast(int*)(&a)) & (*cast(int*)(&b)); 133 int correct2 = (*cast(int*)(&a)) & ~(*cast(int*)(&b)); 134 __m128 A = _mm_set_ps(a, b, a, b); 135 __m128 B = _mm_set_ps(b, a, b, a); 136 int4 R = cast(int4)( _mm_andnot_ps(A, B) ); 137 assert(R.array[0] == correct2); 138 assert(R.array[1] == correct); 139 assert(R.array[2] == correct2); 140 assert(R.array[3] == correct); 141 } 142 143 /// Average packed unsigned 16-bit integers in ``a` and `b`. 144 __m64 _mm_avg_pu16 (__m64 a, __m64 b) pure @safe 145 { 146 return to_m64(_mm_avg_epu16(to_m128i(a), to_m128i(b))); 147 } 148 149 /// Average packed unsigned 8-bit integers in ``a` and `b`. 150 __m64 _mm_avg_pu8 (__m64 a, __m64 b) pure @safe 151 { 152 return to_m64(_mm_avg_epu8(to_m128i(a), to_m128i(b))); 153 } 154 155 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for equality. 156 __m128 _mm_cmpeq_ps (__m128 a, __m128 b) pure @safe 157 { 158 static if (DMD_with_DSIMD) 159 return cast(__m128) __simd(XMM.CMPPS, a, b, 0); 160 else 161 return cast(__m128) cmpps!(FPComparison.oeq)(a, b); 162 } 163 unittest 164 { 165 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 166 __m128 B = _mm_setr_ps(3.0f, 2.0f, float.nan, float.nan); 167 __m128i R = cast(__m128i) _mm_cmpeq_ps(A, B); 168 int[4] correct = [0, -1, 0, 0]; 169 assert(R.array == correct); 170 } 171 172 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for equality, 173 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 174 __m128 _mm_cmpeq_ss (__m128 a, __m128 b) pure @safe 175 { 176 static if (DMD_with_DSIMD) 177 return cast(__m128) __simd(XMM.CMPSS, a, b, 0); 178 else 179 return cast(__m128) cmpss!(FPComparison.oeq)(a, b); 180 } 181 unittest 182 { 183 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 184 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 185 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 186 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 187 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 188 __m128i R1 = cast(__m128i) _mm_cmpeq_ss(A, B); 189 __m128i R2 = cast(__m128i) _mm_cmpeq_ss(A, C); 190 __m128i R3 = cast(__m128i) _mm_cmpeq_ss(A, D); 191 __m128i R4 = cast(__m128i) _mm_cmpeq_ss(A, E); 192 int[4] correct1 = [-1, 0, 0, 0]; 193 int[4] correct2 = [0, 0, 0, 0]; 194 int[4] correct3 = [0, 0, 0, 0]; 195 int[4] correct4 = [0, 0, 0, 0]; 196 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 197 } 198 199 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for greater-than-or-equal. 200 __m128 _mm_cmpge_ps (__m128 a, __m128 b) pure @safe 201 { 202 static if (DMD_with_DSIMD) 203 return cast(__m128) __simd(XMM.CMPPS, b, a, 2); 204 else 205 return cast(__m128) cmpps!(FPComparison.oge)(a, b); 206 } 207 unittest 208 { 209 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 210 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 211 __m128i R = cast(__m128i) _mm_cmpge_ps(A, B); 212 int[4] correct = [0, -1,-1, 0]; 213 assert(R.array == correct); 214 } 215 216 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for greater-than-or-equal, 217 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 218 __m128 _mm_cmpge_ss (__m128 a, __m128 b) pure @safe 219 { 220 static if (DMD_with_DSIMD) 221 { 222 __m128 c = cast(__m128) __simd(XMM.CMPSS, b, a, 2); 223 a[0] = c[0]; 224 return a; 225 } 226 else 227 return cast(__m128) cmpss!(FPComparison.oge)(a, b); 228 } 229 unittest 230 { 231 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 232 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 233 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 234 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 235 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 236 __m128i R1 = cast(__m128i) _mm_cmpge_ss(A, B); 237 __m128i R2 = cast(__m128i) _mm_cmpge_ss(A, C); 238 __m128i R3 = cast(__m128i) _mm_cmpge_ss(A, D); 239 __m128i R4 = cast(__m128i) _mm_cmpge_ss(A, E); 240 int[4] correct1 = [-1, 0, 0, 0]; 241 int[4] correct2 = [-1, 0, 0, 0]; 242 int[4] correct3 = [0, 0, 0, 0]; 243 int[4] correct4 = [0, 0, 0, 0]; 244 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 245 } 246 247 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for greater-than. 248 __m128 _mm_cmpgt_ps (__m128 a, __m128 b) pure @safe 249 { 250 static if (DMD_with_DSIMD) 251 return cast(__m128) __simd(XMM.CMPPS, b, a, 1); 252 else 253 return cast(__m128) cmpps!(FPComparison.ogt)(a, b); 254 } 255 unittest 256 { 257 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 258 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 259 __m128i R = cast(__m128i) _mm_cmpgt_ps(A, B); 260 int[4] correct = [0, 0,-1, 0]; 261 assert(R.array == correct); 262 } 263 264 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for greater-than, 265 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 266 __m128 _mm_cmpgt_ss (__m128 a, __m128 b) pure @safe 267 { 268 static if (DMD_with_DSIMD) 269 { 270 __m128 c = cast(__m128) __simd(XMM.CMPSS, b, a, 1); 271 a[0] = c[0]; 272 return a; 273 } 274 else 275 return cast(__m128) cmpss!(FPComparison.ogt)(a, b); 276 } 277 unittest 278 { 279 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 280 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 281 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 282 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 283 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 284 __m128i R1 = cast(__m128i) _mm_cmpgt_ss(A, B); 285 __m128i R2 = cast(__m128i) _mm_cmpgt_ss(A, C); 286 __m128i R3 = cast(__m128i) _mm_cmpgt_ss(A, D); 287 __m128i R4 = cast(__m128i) _mm_cmpgt_ss(A, E); 288 int[4] correct1 = [0, 0, 0, 0]; 289 int[4] correct2 = [-1, 0, 0, 0]; 290 int[4] correct3 = [0, 0, 0, 0]; 291 int[4] correct4 = [0, 0, 0, 0]; 292 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 293 } 294 295 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for less-than-or-equal. 296 __m128 _mm_cmple_ps (__m128 a, __m128 b) pure @safe 297 { 298 static if (DMD_with_DSIMD) 299 return cast(__m128) __simd(XMM.CMPPS, a, b, 2); 300 else 301 return cast(__m128) cmpps!(FPComparison.ole)(a, b); 302 } 303 unittest 304 { 305 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 306 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 307 __m128i R = cast(__m128i) _mm_cmple_ps(A, B); 308 int[4] correct = [-1, -1, 0, 0]; 309 assert(R.array == correct); 310 } 311 312 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for less-than-or-equal, 313 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 314 __m128 _mm_cmple_ss (__m128 a, __m128 b) pure @safe 315 { 316 static if (DMD_with_DSIMD) 317 return cast(__m128) __simd(XMM.CMPSS, a, b, 2); 318 else 319 return cast(__m128) cmpss!(FPComparison.ole)(a, b); 320 } 321 unittest 322 { 323 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 324 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 325 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 326 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 327 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 328 __m128i R1 = cast(__m128i) _mm_cmple_ss(A, B); 329 __m128i R2 = cast(__m128i) _mm_cmple_ss(A, C); 330 __m128i R3 = cast(__m128i) _mm_cmple_ss(A, D); 331 __m128i R4 = cast(__m128i) _mm_cmple_ss(A, E); 332 int[4] correct1 = [-1, 0, 0, 0]; 333 int[4] correct2 = [0, 0, 0, 0]; 334 int[4] correct3 = [0, 0, 0, 0]; 335 int[4] correct4 = [-1, 0, 0, 0]; 336 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 337 } 338 339 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for less-than. 340 __m128 _mm_cmplt_ps (__m128 a, __m128 b) pure @safe 341 { 342 static if (DMD_with_DSIMD) 343 return cast(__m128) __simd(XMM.CMPPS, a, b, 1); 344 else 345 return cast(__m128) cmpps!(FPComparison.olt)(a, b); 346 } 347 unittest 348 { 349 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 350 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 351 __m128i R = cast(__m128i) _mm_cmplt_ps(A, B); 352 int[4] correct = [-1, 0, 0, 0]; 353 assert(R.array == correct); 354 } 355 356 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for less-than, 357 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 358 __m128 _mm_cmplt_ss (__m128 a, __m128 b) pure @safe 359 { 360 static if (DMD_with_DSIMD) 361 return cast(__m128) __simd(XMM.CMPSS, a, b, 1); 362 else 363 return cast(__m128) cmpss!(FPComparison.olt)(a, b); 364 } 365 unittest 366 { 367 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 368 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 369 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 370 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 371 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 372 __m128i R1 = cast(__m128i) _mm_cmplt_ss(A, B); 373 __m128i R2 = cast(__m128i) _mm_cmplt_ss(A, C); 374 __m128i R3 = cast(__m128i) _mm_cmplt_ss(A, D); 375 __m128i R4 = cast(__m128i) _mm_cmplt_ss(A, E); 376 int[4] correct1 = [0, 0, 0, 0]; 377 int[4] correct2 = [0, 0, 0, 0]; 378 int[4] correct3 = [0, 0, 0, 0]; 379 int[4] correct4 = [-1, 0, 0, 0]; 380 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 381 } 382 383 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for not-equal. 384 __m128 _mm_cmpneq_ps (__m128 a, __m128 b) pure @safe 385 { 386 static if (DMD_with_DSIMD) 387 return cast(__m128) __simd(XMM.CMPPS, a, b, 4); 388 else 389 return cast(__m128) cmpps!(FPComparison.une)(a, b); 390 } 391 unittest 392 { 393 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 394 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 395 __m128i R = cast(__m128i) _mm_cmpneq_ps(A, B); 396 int[4] correct = [-1, 0, -1, -1]; 397 assert(R.array == correct); 398 } 399 400 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for not-equal, 401 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 402 __m128 _mm_cmpneq_ss (__m128 a, __m128 b) pure @safe 403 { 404 static if (DMD_with_DSIMD) 405 return cast(__m128) __simd(XMM.CMPSS, a, b, 4); 406 else 407 return cast(__m128) cmpss!(FPComparison.une)(a, b); 408 } 409 unittest 410 { 411 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 412 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 413 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 414 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 415 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 416 __m128i R1 = cast(__m128i) _mm_cmpneq_ss(A, B); 417 __m128i R2 = cast(__m128i) _mm_cmpneq_ss(A, C); 418 __m128i R3 = cast(__m128i) _mm_cmpneq_ss(A, D); 419 __m128i R4 = cast(__m128i) _mm_cmpneq_ss(A, E); 420 int[4] correct1 = [0, 0, 0, 0]; 421 int[4] correct2 = [-1, 0, 0, 0]; 422 int[4] correct3 = [-1, 0, 0, 0]; 423 int[4] correct4 = [-1, 0, 0, 0]; 424 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 425 } 426 427 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for not-greater-than-or-equal. 428 __m128 _mm_cmpnge_ps (__m128 a, __m128 b) pure @safe 429 { 430 static if (DMD_with_DSIMD) 431 return cast(__m128) __simd(XMM.CMPPS, b, a, 6); 432 else 433 return cast(__m128) cmpps!(FPComparison.ult)(a, b); 434 } 435 unittest 436 { 437 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 438 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 439 __m128i R = cast(__m128i) _mm_cmpnge_ps(A, B); 440 int[4] correct = [-1, 0, 0, -1]; 441 assert(R.array == correct); 442 } 443 444 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for not-greater-than-or-equal, 445 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 446 __m128 _mm_cmpnge_ss (__m128 a, __m128 b) pure @safe 447 { 448 static if (DMD_with_DSIMD) 449 { 450 __m128 c = cast(__m128) __simd(XMM.CMPSS, b, a, 6); 451 a[0] = c[0]; 452 return a; 453 } 454 else 455 return cast(__m128) cmpss!(FPComparison.ult)(a, b); 456 } 457 unittest 458 { 459 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 460 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 461 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 462 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 463 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 464 __m128i R1 = cast(__m128i) _mm_cmpnge_ss(A, B); 465 __m128i R2 = cast(__m128i) _mm_cmpnge_ss(A, C); 466 __m128i R3 = cast(__m128i) _mm_cmpnge_ss(A, D); 467 __m128i R4 = cast(__m128i) _mm_cmpnge_ss(A, E); 468 int[4] correct1 = [0, 0, 0, 0]; 469 int[4] correct2 = [0, 0, 0, 0]; 470 int[4] correct3 = [-1, 0, 0, 0]; 471 int[4] correct4 = [-1, 0, 0, 0]; 472 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 473 } 474 475 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for not-greater-than. 476 __m128 _mm_cmpngt_ps (__m128 a, __m128 b) pure @safe 477 { 478 static if (DMD_with_DSIMD) 479 return cast(__m128) __simd(XMM.CMPPS, b, a, 5); 480 else 481 return cast(__m128) cmpps!(FPComparison.ule)(a, b); 482 } 483 unittest 484 { 485 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 486 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 487 __m128i R = cast(__m128i) _mm_cmpngt_ps(A, B); 488 int[4] correct = [-1, -1, 0, -1]; 489 assert(R.array == correct); 490 } 491 492 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for not-greater-than, 493 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 494 __m128 _mm_cmpngt_ss (__m128 a, __m128 b) pure @safe 495 { 496 static if (DMD_with_DSIMD) 497 { 498 __m128 c = cast(__m128) __simd(XMM.CMPSS, b, a, 5); 499 a[0] = c[0]; 500 return a; 501 } 502 else 503 return cast(__m128) cmpss!(FPComparison.ule)(a, b); 504 } 505 unittest 506 { 507 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 508 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 509 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 510 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 511 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 512 __m128i R1 = cast(__m128i) _mm_cmpngt_ss(A, B); 513 __m128i R2 = cast(__m128i) _mm_cmpngt_ss(A, C); 514 __m128i R3 = cast(__m128i) _mm_cmpngt_ss(A, D); 515 __m128i R4 = cast(__m128i) _mm_cmpngt_ss(A, E); 516 int[4] correct1 = [-1, 0, 0, 0]; 517 int[4] correct2 = [0, 0, 0, 0]; 518 int[4] correct3 = [-1, 0, 0, 0]; 519 int[4] correct4 = [-1, 0, 0, 0]; 520 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 521 } 522 523 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for not-less-than-or-equal. 524 __m128 _mm_cmpnle_ps (__m128 a, __m128 b) pure @safe 525 { 526 static if (DMD_with_DSIMD) 527 return cast(__m128) __simd(XMM.CMPPS, a, b, 6); 528 else 529 return cast(__m128) cmpps!(FPComparison.ugt)(a, b); 530 } 531 unittest 532 { 533 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 534 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 535 __m128i R = cast(__m128i) _mm_cmpnle_ps(A, B); 536 int[4] correct = [0, 0, -1, -1]; 537 assert(R.array == correct); 538 } 539 540 541 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for not-less-than-or-equal, 542 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 543 __m128 _mm_cmpnle_ss (__m128 a, __m128 b) pure @safe 544 { 545 static if (DMD_with_DSIMD) 546 return cast(__m128) __simd(XMM.CMPSS, a, b, 6); 547 else 548 return cast(__m128) cmpss!(FPComparison.ugt)(a, b); 549 } 550 unittest 551 { 552 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 553 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 554 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 555 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 556 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 557 __m128i R1 = cast(__m128i) _mm_cmpnle_ss(A, B); 558 __m128i R2 = cast(__m128i) _mm_cmpnle_ss(A, C); 559 __m128i R3 = cast(__m128i) _mm_cmpnle_ss(A, D); 560 __m128i R4 = cast(__m128i) _mm_cmpnle_ss(A, E); 561 int[4] correct1 = [0, 0, 0, 0]; 562 int[4] correct2 = [-1, 0, 0, 0]; 563 int[4] correct3 = [-1, 0, 0, 0]; 564 int[4] correct4 = [0, 0, 0, 0]; 565 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 566 } 567 568 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` for not-less-than. 569 __m128 _mm_cmpnlt_ps (__m128 a, __m128 b) pure @safe 570 { 571 static if (DMD_with_DSIMD) 572 return cast(__m128) __simd(XMM.CMPPS, a, b, 5); 573 else 574 return cast(__m128) cmpps!(FPComparison.uge)(a, b); 575 } 576 unittest 577 { 578 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 579 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 580 __m128i R = cast(__m128i) _mm_cmpnlt_ps(A, B); 581 int[4] correct = [0, -1, -1, -1]; 582 assert(R.array == correct); 583 } 584 585 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` for not-less-than, 586 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 587 __m128 _mm_cmpnlt_ss (__m128 a, __m128 b) pure @safe 588 { 589 static if (DMD_with_DSIMD) 590 return cast(__m128) __simd(XMM.CMPSS, a, b, 5); 591 else 592 return cast(__m128) cmpss!(FPComparison.uge)(a, b); 593 } 594 unittest 595 { 596 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 597 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 598 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 599 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 600 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 601 __m128i R1 = cast(__m128i) _mm_cmpnlt_ss(A, B); 602 __m128i R2 = cast(__m128i) _mm_cmpnlt_ss(A, C); 603 __m128i R3 = cast(__m128i) _mm_cmpnlt_ss(A, D); 604 __m128i R4 = cast(__m128i) _mm_cmpnlt_ss(A, E); 605 int[4] correct1 = [-1, 0, 0, 0]; 606 int[4] correct2 = [-1, 0, 0, 0]; 607 int[4] correct3 = [-1, 0, 0, 0]; 608 int[4] correct4 = [0, 0, 0, 0]; 609 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 610 } 611 612 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` to see if neither is NaN. 613 __m128 _mm_cmpord_ps (__m128 a, __m128 b) pure @safe 614 { 615 static if (DMD_with_DSIMD) 616 return cast(__m128) __simd(XMM.CMPPS, a, b, 7); 617 else 618 return cast(__m128) cmpps!(FPComparison.ord)(a, b); 619 } 620 unittest 621 { 622 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 623 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 624 __m128i R = cast(__m128i) _mm_cmpord_ps(A, B); 625 int[4] correct = [-1, -1, -1, 0]; 626 assert(R.array == correct); 627 } 628 629 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` to see if neither is NaN, 630 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 631 __m128 _mm_cmpord_ss (__m128 a, __m128 b) pure @safe 632 { 633 static if (DMD_with_DSIMD) 634 return cast(__m128) __simd(XMM.CMPSS, a, b, 7); 635 else 636 return cast(__m128) cmpss!(FPComparison.ord)(a, b); 637 } 638 unittest 639 { 640 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 641 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 642 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 643 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 644 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 645 __m128i R1 = cast(__m128i) _mm_cmpord_ss(A, B); 646 __m128i R2 = cast(__m128i) _mm_cmpord_ss(A, C); 647 __m128i R3 = cast(__m128i) _mm_cmpord_ss(A, D); 648 __m128i R4 = cast(__m128i) _mm_cmpord_ss(A, E); 649 int[4] correct1 = [-1, 0, 0, 0]; 650 int[4] correct2 = [-1, 0, 0, 0]; 651 int[4] correct3 = [0, 0, 0, 0]; 652 int[4] correct4 = [-1, 0, 0, 0]; 653 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 654 } 655 656 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b` to see if either is NaN. 657 __m128 _mm_cmpunord_ps (__m128 a, __m128 b) pure @safe 658 { 659 static if (DMD_with_DSIMD) 660 return cast(__m128) __simd(XMM.CMPPS, a, b, 3); 661 else 662 return cast(__m128) cmpps!(FPComparison.uno)(a, b); 663 } 664 unittest 665 { 666 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, float.nan); 667 __m128 B = _mm_setr_ps(3.0f, 2.0f, 1.0f, float.nan); 668 __m128i R = cast(__m128i) _mm_cmpunord_ps(A, B); 669 int[4] correct = [0, 0, 0, -1]; 670 assert(R.array == correct); 671 } 672 673 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b` to see if either is NaN. 674 /// and copy the upper 3 packed elements from `a` to the upper elements of result. 675 __m128 _mm_cmpunord_ss (__m128 a, __m128 b) pure @safe 676 { 677 static if (DMD_with_DSIMD) 678 return cast(__m128) __simd(XMM.CMPSS, a, b, 3); 679 else return cast(__m128) cmpss!(FPComparison.uno)(a, b); 680 } 681 unittest 682 { 683 __m128 A = _mm_setr_ps(3.0f, 0, 0, 0); 684 __m128 B = _mm_setr_ps(3.0f, float.nan, float.nan, float.nan); 685 __m128 C = _mm_setr_ps(2.0f, float.nan, float.nan, float.nan); 686 __m128 D = _mm_setr_ps(float.nan, float.nan, float.nan, float.nan); 687 __m128 E = _mm_setr_ps(4.0f, float.nan, float.nan, float.nan); 688 __m128i R1 = cast(__m128i) _mm_cmpunord_ss(A, B); 689 __m128i R2 = cast(__m128i) _mm_cmpunord_ss(A, C); 690 __m128i R3 = cast(__m128i) _mm_cmpunord_ss(A, D); 691 __m128i R4 = cast(__m128i) _mm_cmpunord_ss(A, E); 692 int[4] correct1 = [0, 0, 0, 0]; 693 int[4] correct2 = [0, 0, 0, 0]; 694 int[4] correct3 = [-1, 0, 0, 0]; 695 int[4] correct4 = [0, 0, 0, 0]; 696 assert(R1.array == correct1 && R2.array == correct2 && R3.array == correct3 && R4.array == correct4); 697 } 698 699 700 /// Compare the lower single-precision (32-bit) floating-point element in `a` and `b` for equality, 701 /// and return the boolean result (0 or 1). 702 int _mm_comieq_ss (__m128 a, __m128 b) pure @safe 703 { 704 return a.array[0] == b.array[0]; 705 } 706 unittest 707 { 708 assert(1 == _mm_comieq_ss(_mm_set_ss(78.0f), _mm_set_ss(78.0f))); 709 assert(0 == _mm_comieq_ss(_mm_set_ss(78.0f), _mm_set_ss(-78.0f))); 710 assert(0 == _mm_comieq_ss(_mm_set_ss(78.0f), _mm_set_ss(float.nan))); 711 assert(0 == _mm_comieq_ss(_mm_set_ss(float.nan), _mm_set_ss(-4.22f))); 712 assert(1 == _mm_comieq_ss(_mm_set_ss(0.0), _mm_set_ss(-0.0))); 713 } 714 715 /// Compare the lower single-precision (32-bit) floating-point element in `a` and `b` for greater-than-or-equal, 716 /// and return the boolean result (0 or 1). 717 int _mm_comige_ss (__m128 a, __m128 b) pure @safe 718 { 719 return a.array[0] >= b.array[0]; 720 } 721 unittest 722 { 723 assert(1 == _mm_comige_ss(_mm_set_ss(78.0f), _mm_set_ss(78.0f))); 724 assert(1 == _mm_comige_ss(_mm_set_ss(78.0f), _mm_set_ss(-78.0f))); 725 assert(0 == _mm_comige_ss(_mm_set_ss(-78.0f), _mm_set_ss(78.0f))); 726 assert(0 == _mm_comige_ss(_mm_set_ss(78.0f), _mm_set_ss(float.nan))); 727 assert(0 == _mm_comige_ss(_mm_set_ss(float.nan), _mm_set_ss(-4.22f))); 728 assert(1 == _mm_comige_ss(_mm_set_ss(-0.0f), _mm_set_ss(0.0f))); 729 } 730 731 /// Compare the lower single-precision (32-bit) floating-point element in `a` and `b` for greater-than, 732 /// and return the boolean result (0 or 1). 733 int _mm_comigt_ss (__m128 a, __m128 b) pure @safe // comiss + seta 734 { 735 return a.array[0] > b.array[0]; 736 } 737 unittest 738 { 739 assert(0 == _mm_comigt_ss(_mm_set_ss(78.0f), _mm_set_ss(78.0f))); 740 assert(1 == _mm_comigt_ss(_mm_set_ss(78.0f), _mm_set_ss(-78.0f))); 741 assert(0 == _mm_comigt_ss(_mm_set_ss(78.0f), _mm_set_ss(float.nan))); 742 assert(0 == _mm_comigt_ss(_mm_set_ss(float.nan), _mm_set_ss(-4.22f))); 743 assert(0 == _mm_comigt_ss(_mm_set_ss(0.0f), _mm_set_ss(-0.0f))); 744 } 745 746 /// Compare the lower single-precision (32-bit) floating-point element in `a` and `b` for less-than-or-equal, 747 /// and return the boolean result (0 or 1). 748 int _mm_comile_ss (__m128 a, __m128 b) pure @safe // comiss + setbe 749 { 750 return a.array[0] <= b.array[0]; 751 } 752 unittest 753 { 754 assert(1 == _mm_comile_ss(_mm_set_ss(78.0f), _mm_set_ss(78.0f))); 755 assert(0 == _mm_comile_ss(_mm_set_ss(78.0f), _mm_set_ss(-78.0f))); 756 assert(1 == _mm_comile_ss(_mm_set_ss(-78.0f), _mm_set_ss(78.0f))); 757 assert(0 == _mm_comile_ss(_mm_set_ss(78.0f), _mm_set_ss(float.nan))); 758 assert(0 == _mm_comile_ss(_mm_set_ss(float.nan), _mm_set_ss(-4.22f))); 759 assert(1 == _mm_comile_ss(_mm_set_ss(0.0f), _mm_set_ss(-0.0f))); 760 } 761 762 /// Compare the lower single-precision (32-bit) floating-point element in `a` and `b` for less-than, 763 /// and return the boolean result (0 or 1). 764 int _mm_comilt_ss (__m128 a, __m128 b) pure @safe // comiss + setb 765 { 766 return a.array[0] < b.array[0]; 767 } 768 unittest 769 { 770 assert(0 == _mm_comilt_ss(_mm_set_ss(78.0f), _mm_set_ss(78.0f))); 771 assert(0 == _mm_comilt_ss(_mm_set_ss(78.0f), _mm_set_ss(-78.0f))); 772 assert(1 == _mm_comilt_ss(_mm_set_ss(-78.0f), _mm_set_ss(78.0f))); 773 assert(0 == _mm_comilt_ss(_mm_set_ss(78.0f), _mm_set_ss(float.nan))); 774 assert(0 == _mm_comilt_ss(_mm_set_ss(float.nan), _mm_set_ss(-4.22f))); 775 assert(0 == _mm_comilt_ss(_mm_set_ss(-0.0f), _mm_set_ss(0.0f))); 776 } 777 778 /// Compare the lower single-precision (32-bit) floating-point element in `a` and `b` for not-equal, 779 /// and return the boolean result (0 or 1). 780 int _mm_comineq_ss (__m128 a, __m128 b) pure @safe // comiss + setne 781 { 782 return a.array[0] != b.array[0]; 783 } 784 unittest 785 { 786 assert(0 == _mm_comineq_ss(_mm_set_ss(78.0f), _mm_set_ss(78.0f))); 787 assert(1 == _mm_comineq_ss(_mm_set_ss(78.0f), _mm_set_ss(-78.0f))); 788 assert(1 == _mm_comineq_ss(_mm_set_ss(78.0f), _mm_set_ss(float.nan))); 789 assert(1 == _mm_comineq_ss(_mm_set_ss(float.nan), _mm_set_ss(-4.22f))); 790 assert(0 == _mm_comineq_ss(_mm_set_ss(0.0f), _mm_set_ss(-0.0f))); 791 } 792 793 /// Convert packed signed 32-bit integers in `b` to packed single-precision (32-bit) 794 /// floating-point elements, store the results in the lower 2 elements, 795 /// and copy the upper 2 packed elements from `a` to the upper elements of result. 796 alias _mm_cvt_pi2ps = _mm_cvtpi32_ps; 797 798 /// Convert 2 lower packed single-precision (32-bit) floating-point elements in `a` 799 /// to packed 32-bit integers. 800 __m64 _mm_cvt_ps2pi (__m128 a) @safe 801 { 802 return to_m64(_mm_cvtps_epi32(a)); 803 } 804 805 /// Convert the signed 32-bit integer `b` to a single-precision (32-bit) floating-point element, 806 /// store the result in the lower element, and copy the upper 3 packed elements from `a` to the 807 /// upper elements of the result. 808 __m128 _mm_cvt_si2ss (__m128 v, int x) pure @trusted 809 { 810 v.ptr[0] = cast(float)x; 811 return v; 812 } 813 unittest 814 { 815 __m128 a = _mm_cvt_si2ss(_mm_set1_ps(0.0f), 42); 816 assert(a.array == [42f, 0, 0, 0]); 817 } 818 819 /// Convert packed 16-bit integers in `a` to packed single-precision (32-bit) floating-point elements. 820 __m128 _mm_cvtpi16_ps (__m64 a) pure @safe 821 { 822 __m128i ma = to_m128i(a); 823 ma = _mm_unpacklo_epi16(ma, _mm_setzero_si128()); // Zero-extend to 32-bit 824 ma = _mm_srai_epi32(_mm_slli_epi32(ma, 16), 16); // Replicate sign bit 825 return _mm_cvtepi32_ps(ma); 826 } 827 unittest 828 { 829 __m64 A = _mm_setr_pi16(-1, 2, -3, 4); 830 __m128 R = _mm_cvtpi16_ps(A); 831 float[4] correct = [-1.0f, 2.0f, -3.0f, 4.0f]; 832 assert(R.array == correct); 833 } 834 835 /// Convert packed signed 32-bit integers in `b` to packed single-precision (32-bit) 836 /// floating-point elements, store the results in the lower 2 elements, 837 /// and copy the upper 2 packed elements from `a` to the upper elements of result. 838 __m128 _mm_cvtpi32_ps (__m128 a, __m64 b) pure @trusted 839 { 840 __m128 fb = _mm_cvtepi32_ps(to_m128i(b)); 841 a.ptr[0] = fb.array[0]; 842 a.ptr[1] = fb.array[1]; 843 return a; 844 } 845 unittest 846 { 847 __m128 R = _mm_cvtpi32_ps(_mm_set1_ps(4.0f), _mm_setr_pi32(1, 2)); 848 float[4] correct = [1.0f, 2.0f, 4.0f, 4.0f]; 849 assert(R.array == correct); 850 } 851 852 /// Convert packed signed 32-bit integers in `a` to packed single-precision (32-bit) floating-point elements, 853 /// store the results in the lower 2 elements, then covert the packed signed 32-bit integers in `b` to 854 /// single-precision (32-bit) floating-point element, and store the results in the upper 2 elements. 855 __m128 _mm_cvtpi32x2_ps (__m64 a, __m64 b) pure @trusted 856 { 857 long2 l; 858 l.ptr[0] = a.array[0]; 859 l.ptr[1] = b.array[0]; 860 return _mm_cvtepi32_ps(cast(__m128i)l); 861 } 862 unittest 863 { 864 __m64 A = _mm_setr_pi32(-45, 128); 865 __m64 B = _mm_setr_pi32(0, 1000); 866 __m128 R = _mm_cvtpi32x2_ps(A, B); 867 float[4] correct = [-45.0f, 128.0f, 0.0f, 1000.0f]; 868 assert(R.array == correct); 869 } 870 871 /// Convert the lower packed 8-bit integers in `a` to packed single-precision (32-bit) floating-point elements. 872 __m128 _mm_cvtpi8_ps (__m64 a) pure @safe 873 { 874 __m128i b = to_m128i(a); 875 876 // Zero extend to 32-bit 877 b = _mm_unpacklo_epi8(b, _mm_setzero_si128()); 878 b = _mm_unpacklo_epi16(b, _mm_setzero_si128()); 879 880 // Replicate sign bit 881 b = _mm_srai_epi32(_mm_slli_epi32(b, 24), 24); // Replicate sign bit 882 return _mm_cvtepi32_ps(b); 883 } 884 unittest 885 { 886 __m64 A = _mm_setr_pi8(-1, 2, -3, 4, 0, 0, 0, 0); 887 __m128 R = _mm_cvtpi8_ps(A); 888 float[4] correct = [-1.0f, 2.0f, -3.0f, 4.0f]; 889 assert(R.array == correct); 890 } 891 892 /// Convert packed single-precision (32-bit) floating-point elements in `a` to packed 16-bit integers. 893 /// Note: this intrinsic will generate 0x7FFF, rather than 0x8000, for input values between 0x7FFF and 0x7FFFFFFF. 894 __m64 _mm_cvtps_pi16 (__m128 a) @safe 895 { 896 // The C++ version of this intrinsic convert to 32-bit float, then use packssdw 897 // Which means the 16-bit integers should be saturated 898 __m128i b = _mm_cvtps_epi32(a); 899 b = _mm_packs_epi32(b, b); 900 return to_m64(b); 901 } 902 unittest 903 { 904 __m128 A = _mm_setr_ps(-1.0f, 2.0f, -33000.0f, 70000.0f); 905 short4 R = cast(short4) _mm_cvtps_pi16(A); 906 short[4] correct = [-1, 2, -32768, 32767]; 907 assert(R.array == correct); 908 } 909 910 /// Convert packed single-precision (32-bit) floating-point elements in `a` to packed 32-bit integers. 911 __m64 _mm_cvtps_pi32 (__m128 a) @safe 912 { 913 return to_m64(_mm_cvtps_epi32(a)); 914 } 915 unittest 916 { 917 __m128 A = _mm_setr_ps(-33000.0f, 70000.0f, -1.0f, 2.0f, ); 918 int2 R = cast(int2) _mm_cvtps_pi32(A); 919 int[2] correct = [-33000, 70000]; 920 assert(R.array == correct); 921 } 922 923 /// Convert packed single-precision (32-bit) floating-point elements in `a` to packed 8-bit integers, 924 /// and store the results in lower 4 elements. 925 /// Note: this intrinsic will generate 0x7F, rather than 0x80, for input values between 0x7F and 0x7FFFFFFF. 926 __m64 _mm_cvtps_pi8 (__m128 a) @safe 927 { 928 // The C++ version of this intrinsic convert to 32-bit float, then use packssdw + packsswb 929 // Which means the 8-bit integers should be saturated 930 __m128i b = _mm_cvtps_epi32(a); 931 b = _mm_packs_epi32(b, _mm_setzero_si128()); 932 b = _mm_packs_epi16(b, _mm_setzero_si128()); 933 return to_m64(b); 934 } 935 unittest 936 { 937 __m128 A = _mm_setr_ps(-1.0f, 2.0f, -129.0f, 128.0f); 938 byte8 R = cast(byte8) _mm_cvtps_pi8(A); 939 byte[8] correct = [-1, 2, -128, 127, 0, 0, 0, 0]; 940 assert(R.array == correct); 941 } 942 943 /// Convert packed unsigned 16-bit integers in `a` to packed single-precision (32-bit) floating-point elements. 944 __m128 _mm_cvtpu16_ps (__m64 a) pure @safe 945 { 946 __m128i ma = to_m128i(a); 947 ma = _mm_unpacklo_epi16(ma, _mm_setzero_si128()); // Zero-extend to 32-bit 948 return _mm_cvtepi32_ps(ma); 949 } 950 unittest 951 { 952 __m64 A = _mm_setr_pi16(-1, 2, -3, 4); 953 __m128 R = _mm_cvtpu16_ps(A); 954 float[4] correct = [65535.0f, 2.0f, 65533.0f, 4.0f]; 955 assert(R.array == correct); 956 } 957 958 /// Convert the lower packed unsigned 8-bit integers in `a` to packed single-precision (32-bit) floating-point element. 959 __m128 _mm_cvtpu8_ps (__m64 a) pure @safe 960 { 961 __m128i b = to_m128i(a); 962 963 // Zero extend to 32-bit 964 b = _mm_unpacklo_epi8(b, _mm_setzero_si128()); 965 b = _mm_unpacklo_epi16(b, _mm_setzero_si128()); 966 return _mm_cvtepi32_ps(b); 967 } 968 unittest 969 { 970 __m64 A = _mm_setr_pi8(-1, 2, -3, 4, 0, 0, 0, 0); 971 __m128 R = _mm_cvtpu8_ps(A); 972 float[4] correct = [255.0f, 2.0f, 253.0f, 4.0f]; 973 assert(R.array == correct); 974 } 975 976 /// Convert the signed 32-bit integer `b` to a single-precision (32-bit) floating-point element, 977 /// store the result in the lower element, and copy the upper 3 packed elements from `a` to the 978 /// upper elements of result. 979 __m128 _mm_cvtsi32_ss(__m128 v, int x) pure @trusted 980 { 981 v.ptr[0] = cast(float)x; 982 return v; 983 } 984 unittest 985 { 986 __m128 a = _mm_cvtsi32_ss(_mm_set1_ps(0.0f), 42); 987 assert(a.array == [42.0f, 0, 0, 0]); 988 } 989 990 991 /// Convert the signed 64-bit integer `b` to a single-precision (32-bit) floating-point element, 992 /// store the result in the lower element, and copy the upper 3 packed elements from `a` to the 993 /// upper elements of result. 994 __m128 _mm_cvtsi64_ss(__m128 v, long x) pure @trusted 995 { 996 v.ptr[0] = cast(float)x; 997 return v; 998 } 999 unittest 1000 { 1001 __m128 a = _mm_cvtsi64_ss(_mm_set1_ps(0.0f), 42); 1002 assert(a.array == [42.0f, 0, 0, 0]); 1003 } 1004 1005 /// Take the lower single-precision (32-bit) floating-point element of `a`. 1006 float _mm_cvtss_f32(__m128 a) pure @safe 1007 { 1008 return a.array[0]; 1009 } 1010 1011 /// Convert the lower single-precision (32-bit) floating-point element in `a` to a 32-bit integer. 1012 int _mm_cvtss_si32 (__m128 a) @safe // PERF GDC 1013 { 1014 static if (GDC_with_SSE) 1015 { 1016 return __builtin_ia32_cvtss2si(a); 1017 } 1018 else static if (LDC_with_SSE1) 1019 { 1020 return __builtin_ia32_cvtss2si(a); 1021 } 1022 else static if (DMD_with_DSIMD) 1023 { 1024 __m128 b; 1025 __m128i r = cast(__m128i) __simd(XMM.CVTPS2DQ, a); // Note: converts 4 integers. 1026 return r.array[0]; 1027 } 1028 else 1029 { 1030 return convertFloatToInt32UsingMXCSR(a.array[0]); 1031 } 1032 } 1033 unittest 1034 { 1035 assert(1 == _mm_cvtss_si32(_mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f))); 1036 } 1037 1038 /// Convert the lower single-precision (32-bit) floating-point element in `a` to a 64-bit integer. 1039 long _mm_cvtss_si64 (__m128 a) @safe 1040 { 1041 version(LDC) 1042 { 1043 version(X86_64) 1044 { 1045 return __builtin_ia32_cvtss2si64(a); 1046 } 1047 else 1048 { 1049 // Note: In 32-bit x86, there is no way to convert from float/double to 64-bit integer 1050 // using SSE instructions only. So the builtin doesn't exit for this arch. 1051 return convertFloatToInt64UsingMXCSR(a.array[0]); 1052 } 1053 } 1054 else 1055 { 1056 return convertFloatToInt64UsingMXCSR(a.array[0]); 1057 } 1058 } 1059 unittest 1060 { 1061 assert(1 == _mm_cvtss_si64(_mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f))); 1062 1063 uint savedRounding = _MM_GET_ROUNDING_MODE(); 1064 1065 _MM_SET_ROUNDING_MODE(_MM_ROUND_NEAREST); 1066 assert(-86186 == _mm_cvtss_si64(_mm_set1_ps(-86186.49f))); 1067 1068 _MM_SET_ROUNDING_MODE(_MM_ROUND_DOWN); 1069 assert(-86187 == _mm_cvtss_si64(_mm_set1_ps(-86186.1f))); 1070 1071 _MM_SET_ROUNDING_MODE(_MM_ROUND_UP); 1072 assert(86187 == _mm_cvtss_si64(_mm_set1_ps(86186.1f))); 1073 1074 _MM_SET_ROUNDING_MODE(_MM_ROUND_TOWARD_ZERO); 1075 assert(-86186 == _mm_cvtss_si64(_mm_set1_ps(-86186.9f))); 1076 1077 _MM_SET_ROUNDING_MODE(savedRounding); 1078 } 1079 1080 1081 /// Convert the lower single-precision (32-bit) floating-point element in `a` to a 32-bit 1082 /// integer with truncation. 1083 int _mm_cvtt_ss2si (__m128 a) pure @safe 1084 { 1085 // x86: cvttss2si always generated, even in -O0 1086 return cast(int)(a.array[0]); 1087 } 1088 alias _mm_cvttss_si32 = _mm_cvtt_ss2si; ///ditto 1089 unittest 1090 { 1091 assert(1 == _mm_cvtt_ss2si(_mm_setr_ps(1.9f, 2.0f, 3.0f, 4.0f))); 1092 } 1093 1094 1095 /// Convert packed single-precision (32-bit) floating-point elements in `a` to packed 32-bit 1096 /// integers with truncation. 1097 __m64 _mm_cvtt_ps2pi (__m128 a) pure @safe 1098 { 1099 return to_m64(_mm_cvttps_epi32(a)); 1100 } 1101 1102 /// Convert the lower single-precision (32-bit) floating-point element in `a` to a 64-bit 1103 /// integer with truncation. 1104 long _mm_cvttss_si64 (__m128 a) pure @safe 1105 { 1106 return cast(long)(a.array[0]); 1107 } 1108 unittest 1109 { 1110 assert(1 == _mm_cvttss_si64(_mm_setr_ps(1.9f, 2.0f, 3.0f, 4.0f))); 1111 } 1112 1113 /// Divide packed single-precision (32-bit) floating-point elements in `a` by packed elements in `b`. 1114 __m128 _mm_div_ps(__m128 a, __m128 b) pure @safe 1115 { 1116 pragma(inline, true); 1117 return a / b; 1118 } 1119 unittest 1120 { 1121 __m128 a = [1.5f, -2.0f, 3.0f, 1.0f]; 1122 a = _mm_div_ps(a, a); 1123 float[4] correct = [1.0f, 1.0f, 1.0f, 1.0f]; 1124 assert(a.array == correct); 1125 } 1126 1127 /// Divide the lower single-precision (32-bit) floating-point element in `a` by the lower 1128 /// single-precision (32-bit) floating-point element in `b`, store the result in the lower 1129 /// element of result, and copy the upper 3 packed elements from `a` to the upper elements of result. 1130 __m128 _mm_div_ss(__m128 a, __m128 b) pure @safe 1131 { 1132 static if (DMD_with_DSIMD) 1133 return cast(__m128) __simd(XMM.DIVSS, a, b); 1134 else static if (GDC_with_SSE) 1135 return __builtin_ia32_divss(a, b); 1136 else 1137 { 1138 a[0] /= b[0]; 1139 return a; 1140 } 1141 } 1142 unittest 1143 { 1144 __m128 a = [1.5f, -2.0f, 3.0f, 1.0f]; 1145 a = _mm_div_ss(a, a); 1146 float[4] correct = [1.0f, -2.0, 3.0f, 1.0f]; 1147 assert(a.array == correct); 1148 } 1149 1150 /// Extract a 16-bit unsigned integer from `a`, selected with `imm8`. Zero-extended. 1151 int _mm_extract_pi16 (__m64 a, int imm8) 1152 { 1153 short4 sa = cast(short4)a; 1154 return cast(ushort)(sa.array[imm8]); 1155 } 1156 unittest 1157 { 1158 __m64 A = _mm_setr_pi16(-1, 6, 0, 4); 1159 assert(_mm_extract_pi16(A, 0) == 65535); 1160 assert(_mm_extract_pi16(A, 1) == 6); 1161 assert(_mm_extract_pi16(A, 2) == 0); 1162 assert(_mm_extract_pi16(A, 3) == 4); 1163 } 1164 1165 /// Free aligned memory that was allocated with `_mm_malloc`. 1166 void _mm_free(void * mem_addr) @trusted 1167 { 1168 // support for free(NULL) 1169 if (mem_addr is null) 1170 return; 1171 1172 // Technically we don't need to store size and alignement in the chunk, but we do in case we 1173 // have to implement _mm_realloc 1174 1175 size_t pointerSize = (void*).sizeof; 1176 void** rawLocation = cast(void**)(cast(char*)mem_addr - size_t.sizeof); 1177 size_t* alignmentLocation = cast(size_t*)(cast(char*)mem_addr - 3 * pointerSize); 1178 size_t alignment = *alignmentLocation; 1179 assert(alignment != 0); 1180 assert(isPointerAligned(mem_addr, alignment)); 1181 free(*rawLocation); 1182 } 1183 1184 /// Get the exception mask bits from the MXCSR control and status register. 1185 /// The exception mask may contain any of the following flags: `_MM_MASK_INVALID`, 1186 /// `_MM_MASK_DIV_ZERO`, `_MM_MASK_DENORM`, `_MM_MASK_OVERFLOW`, `_MM_MASK_UNDERFLOW`, `_MM_MASK_INEXACT`. 1187 /// Note: won't correspond to reality on non-x86, where MXCSR this is emulated. 1188 uint _MM_GET_EXCEPTION_MASK() @safe 1189 { 1190 return _mm_getcsr() & _MM_MASK_MASK; 1191 } 1192 1193 /// Get the exception state bits from the MXCSR control and status register. 1194 /// The exception state may contain any of the following flags: `_MM_EXCEPT_INVALID`, 1195 /// `_MM_EXCEPT_DIV_ZERO`, `_MM_EXCEPT_DENORM`, `_MM_EXCEPT_OVERFLOW`, `_MM_EXCEPT_UNDERFLOW`, `_MM_EXCEPT_INEXACT`. 1196 /// Note: won't correspond to reality on non-x86, where MXCSR this is emulated. No exception reported. 1197 uint _MM_GET_EXCEPTION_STATE() @safe 1198 { 1199 return _mm_getcsr() & _MM_EXCEPT_MASK; 1200 } 1201 1202 /// Get the flush zero bits from the MXCSR control and status register. 1203 /// The flush zero may contain any of the following flags: `_MM_FLUSH_ZERO_ON` or `_MM_FLUSH_ZERO_OFF` 1204 uint _MM_GET_FLUSH_ZERO_MODE() @safe 1205 { 1206 return _mm_getcsr() & _MM_FLUSH_ZERO_MASK; 1207 } 1208 1209 /// Get the rounding mode bits from the MXCSR control and status register. The rounding mode may 1210 /// contain any of the following flags: `_MM_ROUND_NEAREST, `_MM_ROUND_DOWN`, `_MM_ROUND_UP`, `_MM_ROUND_TOWARD_ZERO`. 1211 uint _MM_GET_ROUNDING_MODE() @safe 1212 { 1213 return _mm_getcsr() & _MM_ROUND_MASK; 1214 } 1215 1216 /// Get the unsigned 32-bit value of the MXCSR control and status register. 1217 /// Note: this is emulated on ARM, because there is no MXCSR register then. 1218 uint _mm_getcsr() @trusted 1219 { 1220 static if (LDC_with_ARM) 1221 { 1222 // Note: we convert the ARM FPSCR into a x86 SSE control word. 1223 // However, only rounding mode and flush to zero are actually set. 1224 // The returned control word will have all exceptions masked, and no exception detected. 1225 1226 uint fpscr = arm_get_fpcr(); 1227 1228 uint cw = 0; // No exception detected 1229 if (fpscr & _MM_FLUSH_ZERO_MASK_ARM) 1230 { 1231 // ARM has one single flag for ARM. 1232 // It does both x86 bits. 1233 // https://developer.arm.com/documentation/dui0473/c/neon-and-vfp-programming/the-effects-of-using-flush-to-zero-mode 1234 cw |= _MM_FLUSH_ZERO_ON; 1235 cw |= 0x40; // set "denormals are zeros" 1236 } 1237 cw |= _MM_MASK_MASK; // All exception maske 1238 1239 // Rounding mode 1240 switch(fpscr & _MM_ROUND_MASK_ARM) 1241 { 1242 default: 1243 case _MM_ROUND_NEAREST_ARM: cw |= _MM_ROUND_NEAREST; break; 1244 case _MM_ROUND_DOWN_ARM: cw |= _MM_ROUND_DOWN; break; 1245 case _MM_ROUND_UP_ARM: cw |= _MM_ROUND_UP; break; 1246 case _MM_ROUND_TOWARD_ZERO_ARM: cw |= _MM_ROUND_TOWARD_ZERO; break; 1247 } 1248 return cw; 1249 } 1250 else version(GNU) 1251 { 1252 static if (GDC_with_SSE) 1253 { 1254 return __builtin_ia32_stmxcsr(); 1255 } 1256 else version(X86) 1257 { 1258 uint sseRounding = 0; 1259 asm pure nothrow @nogc @trusted 1260 { 1261 "stmxcsr %0;\n" 1262 : "=m" (sseRounding) 1263 : 1264 : ; 1265 } 1266 return sseRounding; 1267 } 1268 else 1269 static assert(false); 1270 } 1271 else version (InlineX86Asm) 1272 { 1273 uint controlWord; 1274 asm nothrow @nogc pure @safe 1275 { 1276 stmxcsr controlWord; 1277 } 1278 return controlWord; 1279 } 1280 else 1281 static assert(0, "Not yet supported"); 1282 } 1283 unittest 1284 { 1285 uint csr = _mm_getcsr(); 1286 } 1287 1288 /// Insert a 16-bit integer `i` inside `a` at the location specified by `imm8`. 1289 __m64 _mm_insert_pi16 (__m64 v, int i, int imm8) pure @trusted 1290 { 1291 short4 r = cast(short4)v; 1292 r.ptr[imm8 & 3] = cast(short)i; 1293 return cast(__m64)r; 1294 } 1295 unittest 1296 { 1297 __m64 A = _mm_set_pi16(3, 2, 1, 0); 1298 short4 R = cast(short4) _mm_insert_pi16(A, 42, 1 | 4); 1299 short[4] correct = [0, 42, 2, 3]; 1300 assert(R.array == correct); 1301 } 1302 1303 /// Load 128-bits (composed of 4 packed single-precision (32-bit) floating-point elements) from memory. 1304 // `p` must be aligned on a 16-byte boundary or a general-protection exception may be generated. 1305 __m128 _mm_load_ps(const(float)*p) pure @trusted // TODO shouldn't be trusted 1306 { 1307 pragma(inline, true); 1308 return *cast(__m128*)p; 1309 } 1310 unittest 1311 { 1312 static immutable align(16) float[4] correct = [1.0f, 2.0f, 3.0f, 4.0f]; 1313 __m128 A = _mm_load_ps(correct.ptr); 1314 assert(A.array == correct); 1315 } 1316 1317 /// Load a single-precision (32-bit) floating-point element from memory into all elements. 1318 __m128 _mm_load_ps1(const(float)*p) pure @trusted 1319 { 1320 return __m128(*p); 1321 } 1322 unittest 1323 { 1324 float n = 2.5f; 1325 float[4] correct = [2.5f, 2.5f, 2.5f, 2.5f]; 1326 __m128 A = _mm_load_ps1(&n); 1327 assert(A.array == correct); 1328 } 1329 1330 /// Load a single-precision (32-bit) floating-point element from memory into the lower of dst, and zero the upper 3 1331 /// elements. `mem_addr` does not need to be aligned on any particular boundary. 1332 __m128 _mm_load_ss (const(float)* mem_addr) pure @trusted 1333 { 1334 pragma(inline, true); 1335 static if (DMD_with_DSIMD) 1336 { 1337 return cast(__m128)__simd(XMM.LODSS, *cast(__m128*)mem_addr); 1338 } 1339 else 1340 { 1341 __m128 r; 1342 r.ptr[0] = *mem_addr; 1343 r.ptr[1] = 0; 1344 r.ptr[2] = 0; 1345 r.ptr[3] = 0; 1346 return r; 1347 } 1348 } 1349 unittest 1350 { 1351 float n = 2.5f; 1352 float[4] correct = [2.5f, 0.0f, 0.0f, 0.0f]; 1353 __m128 A = _mm_load_ss(&n); 1354 assert(A.array == correct); 1355 } 1356 1357 /// Load a single-precision (32-bit) floating-point element from memory into all elements. 1358 alias _mm_load1_ps = _mm_load_ps1; 1359 1360 /// Load 2 single-precision (32-bit) floating-point elements from memory into the upper 2 elements of result, 1361 /// and copy the lower 2 elements from `a` to result. `mem_addr does` not need to be aligned on any particular boundary. 1362 __m128 _mm_loadh_pi (__m128 a, const(__m64)* mem_addr) pure @trusted 1363 { 1364 pragma(inline, true); 1365 static if (DMD_with_DSIMD) 1366 { 1367 return cast(__m128) __simd(XMM.LODHPS, a, *cast(const(__m128)*)mem_addr); 1368 } 1369 else 1370 { 1371 // x86: movlhps generated since LDC 1.9.0 -O1 1372 long2 la = cast(long2)a; 1373 la.ptr[1] = (*mem_addr).array[0]; 1374 return cast(__m128)la; 1375 } 1376 } 1377 unittest 1378 { 1379 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 1380 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 1381 __m64 M = to_m64(cast(__m128i)B); 1382 __m128 R = _mm_loadh_pi(A, &M); 1383 float[4] correct = [1.0f, 2.0f, 5.0f, 6.0f]; 1384 assert(R.array == correct); 1385 } 1386 1387 /// Load 2 single-precision (32-bit) floating-point elements from memory into the lower 2 elements of result, 1388 /// and copy the upper 2 elements from `a` to result. `mem_addr` does not need to be aligned on any particular boundary. 1389 __m128 _mm_loadl_pi (__m128 a, const(__m64)* mem_addr) pure @trusted 1390 { 1391 pragma(inline, true); 1392 1393 // Disabled because of https://issues.dlang.org/show_bug.cgi?id=23046 1394 /* 1395 static if (DMD_with_DSIMD) 1396 { 1397 return cast(__m128) __simd(XMM.LODLPS, a, *cast(const(__m128)*)mem_addr); 1398 } 1399 else */ 1400 { 1401 // x86: movlpd/movlps generated with all LDC -01 1402 long2 la = cast(long2)a; 1403 la.ptr[0] = (*mem_addr).array[0]; 1404 return cast(__m128)la; 1405 } 1406 } 1407 unittest 1408 { 1409 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 1410 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 1411 __m64 M = to_m64(cast(__m128i)B); 1412 __m128 R = _mm_loadl_pi(A, &M); 1413 float[4] correct = [5.0f, 6.0f, 3.0f, 4.0f]; 1414 assert(R.array == correct); 1415 } 1416 1417 /// Load 4 single-precision (32-bit) floating-point elements from memory in reverse order. 1418 /// `mem_addr` must be aligned on a 16-byte boundary or a general-protection exception may be generated. 1419 __m128 _mm_loadr_ps (const(float)* mem_addr) pure @trusted // TODO shouldn't be trusted 1420 { 1421 __m128* aligned = cast(__m128*)mem_addr; // x86: movaps + shups since LDC 1.0.0 -O1 1422 __m128 a = *aligned; 1423 static if (DMD_with_DSIMD) 1424 { 1425 return cast(__m128) __simd(XMM.SHUFPS, a, a, 27); 1426 } 1427 else 1428 { 1429 __m128 r; 1430 r.ptr[0] = a.array[3]; 1431 r.ptr[1] = a.array[2]; 1432 r.ptr[2] = a.array[1]; 1433 r.ptr[3] = a.array[0]; 1434 return r; 1435 } 1436 } 1437 unittest 1438 { 1439 align(16) static immutable float[4] arr = [ 1.0f, 2.0f, 3.0f, 8.0f ]; 1440 __m128 A = _mm_loadr_ps(arr.ptr); 1441 float[4] correct = [ 8.0f, 3.0f, 2.0f, 1.0f ]; 1442 assert(A.array == correct); 1443 } 1444 1445 /// Load 128-bits (composed of 4 packed single-precision (32-bit) floating-point elements) from memory. 1446 /// `mem_addr` does not need to be aligned on any particular boundary. 1447 __m128 _mm_loadu_ps(const(float)* mem_addr) pure @trusted 1448 { 1449 pragma(inline, true); 1450 static if (GDC_with_SSE2) 1451 { 1452 return __builtin_ia32_loadups(mem_addr); 1453 } 1454 else version(LDC) 1455 { 1456 return loadUnaligned!(__m128)(mem_addr); 1457 } 1458 else version(DigitalMars) 1459 { 1460 static if (DMD_with_DSIMD) 1461 { 1462 return cast(__m128)__simd(XMM.LODUPS, *cast(const(float4*))mem_addr); 1463 } 1464 else static if (SSESizedVectorsAreEmulated) 1465 { 1466 // Since this vector is emulated, it doesn't have alignement constraints 1467 // and as such we can just cast it. 1468 return *cast(__m128*)(mem_addr); 1469 } 1470 else 1471 { 1472 __m128 result; 1473 result.ptr[0] = mem_addr[0]; 1474 result.ptr[1] = mem_addr[1]; 1475 result.ptr[2] = mem_addr[2]; 1476 result.ptr[3] = mem_addr[3]; 1477 return result; 1478 } 1479 } 1480 else 1481 { 1482 __m128 result; 1483 result.ptr[0] = mem_addr[0]; 1484 result.ptr[1] = mem_addr[1]; 1485 result.ptr[2] = mem_addr[2]; 1486 result.ptr[3] = mem_addr[3]; 1487 return result; 1488 } 1489 } 1490 unittest 1491 { 1492 align(16) static immutable float[5] arr = [ 1.0f, 2.0f, 3.0f, 8.0f, 9.0f ]; // force unaligned load 1493 __m128 A = _mm_loadu_ps(&arr[1]); 1494 float[4] correct = [ 2.0f, 3.0f, 8.0f, 9.0f ]; 1495 assert(A.array == correct); 1496 } 1497 1498 /// Load unaligned 16-bit integer from memory into the first element, fill with zeroes otherwise. 1499 __m128i _mm_loadu_si16(const(void)* mem_addr) pure @trusted 1500 { 1501 static if (DMD_with_DSIMD) 1502 { 1503 int r = *cast(short*)(mem_addr); 1504 return cast(__m128i) __simd(XMM.LODD, *cast(__m128i*)&r); 1505 } 1506 else version(DigitalMars) 1507 { 1508 // Workaround issue: https://issues.dlang.org/show_bug.cgi?id=21672 1509 // DMD cannot handle the below code... 1510 align(16) short[8] r = [0, 0, 0, 0, 0, 0, 0, 0]; 1511 r[0] = *cast(short*)(mem_addr); 1512 return *cast(int4*)(r.ptr); 1513 } 1514 else 1515 { 1516 short r = *cast(short*)(mem_addr); 1517 short8 result = [0, 0, 0, 0, 0, 0, 0, 0]; 1518 result.ptr[0] = r; 1519 return cast(__m128i)result; 1520 } 1521 } 1522 unittest 1523 { 1524 short r = 13; 1525 short8 A = cast(short8) _mm_loadu_si16(&r); 1526 short[8] correct = [13, 0, 0, 0, 0, 0, 0, 0]; 1527 assert(A.array == correct); 1528 } 1529 1530 /// Load unaligned 64-bit integer from memory into the first element of result. 1531 /// Upper 64-bit is zeroed. 1532 __m128i _mm_loadu_si64(const(void)* mem_addr) pure @trusted 1533 { 1534 pragma(inline, true); 1535 static if (DMD_with_DSIMD) 1536 { 1537 return cast(__m128i) __simd(XMM.LODQ, *cast(__m128i*)mem_addr); 1538 } 1539 else 1540 { 1541 long r = *cast(long*)(mem_addr); 1542 long2 result = [0, 0]; 1543 result.ptr[0] = r; 1544 return cast(__m128i)result; 1545 } 1546 } 1547 unittest 1548 { 1549 long r = 446446446446; 1550 long2 A = cast(long2) _mm_loadu_si64(&r); 1551 long[2] correct = [446446446446, 0]; 1552 assert(A.array == correct); 1553 } 1554 1555 /// Allocate size bytes of memory, aligned to the alignment specified in align, 1556 /// and return a pointer to the allocated memory. `_mm_free` should be used to free 1557 /// memory that is allocated with `_mm_malloc`. 1558 void* _mm_malloc(size_t size, size_t alignment) @trusted 1559 { 1560 assert(alignment != 0); 1561 size_t request = requestedSize(size, alignment); 1562 void* raw = malloc(request); 1563 if (request > 0 && raw == null) // malloc(0) can validly return anything 1564 onOutOfMemoryError(); 1565 return storeRawPointerPlusInfo(raw, size, alignment); // PERF: no need to store size 1566 } 1567 1568 /// Conditionally store 8-bit integer elements from a into memory using mask (elements are not stored when the highest 1569 /// bit is not set in the corresponding element) and a non-temporal memory hint. 1570 void _mm_maskmove_si64 (__m64 a, __m64 mask, char* mem_addr) @trusted 1571 { 1572 // this works since mask is zero-extended 1573 return _mm_maskmoveu_si128 (to_m128i(a), to_m128i(mask), mem_addr); 1574 } 1575 1576 deprecated("Use _mm_maskmove_si64 instead") alias _m_maskmovq = _mm_maskmove_si64;/// 1577 1578 /// Compare packed signed 16-bit integers in `a` and `b`, and return packed maximum value. 1579 __m64 _mm_max_pi16 (__m64 a, __m64 b) pure @safe 1580 { 1581 return to_m64(_mm_max_epi16(to_m128i(a), to_m128i(b))); 1582 } 1583 1584 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b`, and return packed maximum values. 1585 __m128 _mm_max_ps(__m128 a, __m128 b) pure @safe 1586 { 1587 static if (DMD_with_DSIMD) 1588 { 1589 return cast(__m128) __simd(XMM.MAXPS, a, b); 1590 } 1591 else static if (GDC_with_SSE) 1592 { 1593 return __builtin_ia32_maxps(a, b); 1594 } 1595 else static if (LDC_with_SSE1) 1596 { 1597 return __builtin_ia32_maxps(a, b); 1598 } 1599 else 1600 { 1601 // ARM: Optimized into fcmgt + bsl since LDC 1.8 -02 1602 __m128 r; 1603 r[0] = (a[0] > b[0]) ? a[0] : b[0]; 1604 r[1] = (a[1] > b[1]) ? a[1] : b[1]; 1605 r[2] = (a[2] > b[2]) ? a[2] : b[2]; 1606 r[3] = (a[3] > b[3]) ? a[3] : b[3]; 1607 return r; 1608 } 1609 } 1610 unittest 1611 { 1612 __m128 A = _mm_setr_ps(1, 2, float.nan, 4); 1613 __m128 B = _mm_setr_ps(4, 1, 4, float.nan); 1614 __m128 M = _mm_max_ps(A, B); 1615 assert(M.array[0] == 4); 1616 assert(M.array[1] == 2); 1617 assert(M.array[2] == 4); // in case of NaN, second operand prevails (as it seems) 1618 assert(M.array[3] != M.array[3]); // in case of NaN, second operand prevails (as it seems) 1619 } 1620 1621 /// Compare packed unsigned 8-bit integers in `a` and `b`, and return packed maximum values. 1622 __m64 _mm_max_pu8 (__m64 a, __m64 b) pure @safe 1623 { 1624 return to_m64(_mm_max_epu8(to_m128i(a), to_m128i(b))); 1625 } 1626 1627 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b`, store the maximum value in the 1628 /// lower element of result, and copy the upper 3 packed elements from `a` to the upper element of result. 1629 __m128 _mm_max_ss(__m128 a, __m128 b) pure @safe 1630 { 1631 static if (DMD_with_DSIMD) 1632 { 1633 return cast(__m128) __simd(XMM.MAXSS, a, b); 1634 } 1635 else static if (GDC_with_SSE) 1636 { 1637 return __builtin_ia32_maxss(a, b); 1638 } 1639 else static if (LDC_with_SSE1) 1640 { 1641 return __builtin_ia32_maxss(a, b); 1642 } 1643 else 1644 { 1645 __m128 r = a; 1646 r[0] = (a[0] > b[0]) ? a[0] : b[0]; 1647 return r; 1648 } 1649 } 1650 unittest 1651 { 1652 __m128 A = _mm_setr_ps(1, 2, 3, 4); 1653 __m128 B = _mm_setr_ps(4, 1, 4, 1); 1654 __m128 C = _mm_setr_ps(float.nan, 1, 4, 1); 1655 __m128 M = _mm_max_ss(A, B); 1656 assert(M.array[0] == 4); 1657 assert(M.array[1] == 2); 1658 assert(M.array[2] == 3); 1659 assert(M.array[3] == 4); 1660 M = _mm_max_ps(A, C); // in case of NaN, second operand prevails 1661 assert(M.array[0] != M.array[0]); 1662 M = _mm_max_ps(C, A); // in case of NaN, second operand prevails 1663 assert(M.array[0] == 1); 1664 } 1665 1666 /// Compare packed signed 16-bit integers in a and b, and return packed minimum values. 1667 __m64 _mm_min_pi16 (__m64 a, __m64 b) pure @safe 1668 { 1669 return to_m64(_mm_min_epi16(to_m128i(a), to_m128i(b))); 1670 } 1671 1672 /// Compare packed single-precision (32-bit) floating-point elements in `a` and `b`, and return packed maximum values. 1673 __m128 _mm_min_ps(__m128 a, __m128 b) pure @safe 1674 { 1675 static if (DMD_with_DSIMD) 1676 { 1677 return cast(__m128) __simd(XMM.MINPS, a, b); 1678 } 1679 else static if (GDC_with_SSE) 1680 { 1681 return __builtin_ia32_minps(a, b); 1682 } 1683 else static if (LDC_with_SSE1) 1684 { 1685 // not technically needed, but better perf in debug mode 1686 return __builtin_ia32_minps(a, b); 1687 } 1688 else 1689 { 1690 // ARM: Optimized into fcmgt + bsl since LDC 1.8 -02 1691 __m128 r; 1692 r[0] = (a[0] < b[0]) ? a[0] : b[0]; 1693 r[1] = (a[1] < b[1]) ? a[1] : b[1]; 1694 r[2] = (a[2] < b[2]) ? a[2] : b[2]; 1695 r[3] = (a[3] < b[3]) ? a[3] : b[3]; 1696 return r; 1697 } 1698 } 1699 unittest 1700 { 1701 __m128 A = _mm_setr_ps(1, 2, float.nan, 4); 1702 __m128 B = _mm_setr_ps(4, 1, 4, float.nan); 1703 __m128 M = _mm_min_ps(A, B); 1704 assert(M.array[0] == 1); 1705 assert(M.array[1] == 1); 1706 assert(M.array[2] == 4); // in case of NaN, second operand prevails (as it seems) 1707 assert(M.array[3] != M.array[3]); // in case of NaN, second operand prevails (as it seems) 1708 } 1709 1710 /// Compare packed unsigned 8-bit integers in `a` and `b`, and return packed minimum values. 1711 __m64 _mm_min_pu8 (__m64 a, __m64 b) pure @safe 1712 { 1713 return to_m64(_mm_min_epu8(to_m128i(a), to_m128i(b))); 1714 } 1715 1716 /// Compare the lower single-precision (32-bit) floating-point elements in `a` and `b`, store the minimum value in the 1717 /// lower element of result, and copy the upper 3 packed elements from `a` to the upper element of result. 1718 __m128 _mm_min_ss(__m128 a, __m128 b) pure @safe 1719 { 1720 static if (DMD_with_DSIMD) 1721 { 1722 return cast(__m128) __simd(XMM.MINSS, a, b); 1723 } 1724 else static if (GDC_with_SSE) 1725 { 1726 return __builtin_ia32_minss(a, b); 1727 } 1728 else static if (LDC_with_SSE1) 1729 { 1730 return __builtin_ia32_minss(a, b); 1731 } 1732 else 1733 { 1734 // Generates minss since LDC 1.3 -O1 1735 __m128 r = a; 1736 r[0] = (a[0] < b[0]) ? a[0] : b[0]; 1737 return r; 1738 } 1739 } 1740 unittest 1741 { 1742 __m128 A = _mm_setr_ps(1, 2, 3, 4); 1743 __m128 B = _mm_setr_ps(4, 1, 4, 1); 1744 __m128 C = _mm_setr_ps(float.nan, 1, 4, 1); 1745 __m128 M = _mm_min_ss(A, B); 1746 assert(M.array[0] == 1); 1747 assert(M.array[1] == 2); 1748 assert(M.array[2] == 3); 1749 assert(M.array[3] == 4); 1750 M = _mm_min_ps(A, C); // in case of NaN, second operand prevails 1751 assert(M.array[0] != M.array[0]); 1752 M = _mm_min_ps(C, A); // in case of NaN, second operand prevails 1753 assert(M.array[0] == 1); 1754 } 1755 1756 /// Move the lower single-precision (32-bit) floating-point element from `b` to the lower element of result, and copy 1757 /// the upper 3 packed elements from `a` to the upper elements of result. 1758 __m128 _mm_move_ss (__m128 a, __m128 b) pure @trusted 1759 { 1760 // Workaround https://issues.dlang.org/show_bug.cgi?id=21673 1761 // inlining of this function fails. 1762 version(DigitalMars) asm nothrow @nogc pure { nop; } 1763 1764 a.ptr[0] = b.array[0]; 1765 return a; 1766 } 1767 unittest 1768 { 1769 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 1770 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 1771 __m128 R = _mm_move_ss(A, B); 1772 float[4] correct = [5.0f, 2.0f, 3.0f, 4.0f]; 1773 assert(R.array == correct); 1774 } 1775 1776 /// Move the upper 2 single-precision (32-bit) floating-point elements from `b` to the lower 2 elements of result, and 1777 /// copy the upper 2 elements from `a` to the upper 2 elements of dst. 1778 __m128 _mm_movehl_ps (__m128 a, __m128 b) pure @trusted 1779 { 1780 // Disabled because of https://issues.dlang.org/show_bug.cgi?id=19443 1781 /* 1782 static if (DMD_with_DSIMD) 1783 { 1784 1785 return cast(__m128) __simd(XMM.MOVHLPS, a, b); 1786 } 1787 else */ 1788 { 1789 a.ptr[0] = b.array[2]; 1790 a.ptr[1] = b.array[3]; 1791 return a; 1792 } 1793 } 1794 unittest 1795 { 1796 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 1797 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 1798 __m128 R = _mm_movehl_ps(A, B); 1799 float[4] correct = [7.0f, 8.0f, 3.0f, 4.0f]; 1800 assert(R.array == correct); 1801 } 1802 1803 /// Move the lower 2 single-precision (32-bit) floating-point elements from `b` to the upper 2 elements of result, and 1804 /// copy the lower 2 elements from `a` to the lower 2 elements of result 1805 __m128 _mm_movelh_ps (__m128 a, __m128 b) pure @trusted 1806 { 1807 // Disabled because of https://issues.dlang.org/show_bug.cgi?id=19443 1808 /* 1809 static if (DMD_with_DSIMD) 1810 { 1811 return cast(__m128) __simd(XMM.MOVLHPS, a, b); 1812 } 1813 else 1814 */ 1815 { 1816 a.ptr[2] = b.array[0]; 1817 a.ptr[3] = b.array[1]; 1818 return a; 1819 } 1820 } 1821 unittest 1822 { 1823 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 1824 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 1825 __m128 R = _mm_movelh_ps(A, B); 1826 float[4] correct = [1.0f, 2.0f, 5.0f, 6.0f]; 1827 assert(R.array == correct); 1828 } 1829 1830 /// Create mask from the most significant bit of each 8-bit element in `a`. 1831 int _mm_movemask_pi8 (__m64 a) pure @safe 1832 { 1833 return _mm_movemask_epi8(to_m128i(a)); 1834 } 1835 unittest 1836 { 1837 assert(0x9C == _mm_movemask_pi8(_mm_set_pi8(-1, 0, 0, -1, -1, -1, 0, 0))); 1838 } 1839 1840 /// Set each bit of result based on the most significant bit of the corresponding packed single-precision (32-bit) 1841 /// floating-point element in `a`. 1842 int _mm_movemask_ps (__m128 a) pure @trusted 1843 { 1844 // PERF: Not possible in D_SIMD because of https://issues.dlang.org/show_bug.cgi?id=8047 1845 static if (GDC_with_SSE) 1846 { 1847 return __builtin_ia32_movmskps(a); 1848 } 1849 else static if (LDC_with_SSE1) 1850 { 1851 return __builtin_ia32_movmskps(a); 1852 } 1853 else static if (LDC_with_ARM) 1854 { 1855 int4 ai = cast(int4)a; 1856 int4 shift31 = [31, 31, 31, 31]; 1857 ai = ai >>> shift31; 1858 int4 shift = [0, 1, 2, 3]; 1859 ai = ai << shift; // 4-way shift, only efficient on ARM. 1860 int r = ai.array[0] + (ai.array[1]) + (ai.array[2]) + (ai.array[3]); 1861 return r; 1862 } 1863 else 1864 { 1865 int4 ai = cast(int4)a; 1866 int r = 0; 1867 if (ai.array[0] < 0) r += 1; 1868 if (ai.array[1] < 0) r += 2; 1869 if (ai.array[2] < 0) r += 4; 1870 if (ai.array[3] < 0) r += 8; 1871 return r; 1872 } 1873 } 1874 unittest 1875 { 1876 int4 A = [-1, 0, -43, 0]; 1877 assert(5 == _mm_movemask_ps(cast(float4)A)); 1878 } 1879 1880 /// Multiply packed single-precision (32-bit) floating-point elements in `a` and `b`. 1881 __m128 _mm_mul_ps(__m128 a, __m128 b) pure @safe 1882 { 1883 pragma(inline, true); 1884 return a * b; 1885 } 1886 unittest 1887 { 1888 __m128 a = [1.5f, -2.0f, 3.0f, 1.0f]; 1889 a = _mm_mul_ps(a, a); 1890 float[4] correct = [2.25f, 4.0f, 9.0f, 1.0f]; 1891 assert(a.array == correct); 1892 } 1893 1894 /// Multiply the lower single-precision (32-bit) floating-point element in `a` and `b`, store the result in the lower 1895 /// element of result, and copy the upper 3 packed elements from `a` to the upper elements of result. 1896 __m128 _mm_mul_ss(__m128 a, __m128 b) pure @safe 1897 { 1898 static if (DMD_with_DSIMD) 1899 return cast(__m128) __simd(XMM.MULSS, a, b); 1900 else static if (GDC_with_SSE) 1901 return __builtin_ia32_mulss(a, b); 1902 else 1903 { 1904 a[0] *= b[0]; 1905 return a; 1906 } 1907 } 1908 unittest 1909 { 1910 __m128 a = [1.5f, -2.0f, 3.0f, 1.0f]; 1911 a = _mm_mul_ss(a, a); 1912 float[4] correct = [2.25f, -2.0f, 3.0f, 1.0f]; 1913 assert(a.array == correct); 1914 } 1915 1916 /// Multiply the packed unsigned 16-bit integers in `a` and `b`, producing intermediate 32-bit integers, 1917 /// and return the high 16 bits of the intermediate integers. 1918 __m64 _mm_mulhi_pu16 (__m64 a, __m64 b) pure @safe 1919 { 1920 return to_m64(_mm_mulhi_epu16(to_m128i(a), to_m128i(b))); 1921 } 1922 unittest 1923 { 1924 __m64 A = _mm_setr_pi16(0, -16, 2, 3); 1925 __m64 B = _mm_set1_pi16(16384); 1926 short4 R = cast(short4)_mm_mulhi_pu16(A, B); 1927 short[4] correct = [0, 0x3FFC, 0, 0]; 1928 assert(R.array == correct); 1929 } 1930 1931 /// Compute the bitwise OR of packed single-precision (32-bit) floating-point elements in `a` and `b`, and 1932 /// return the result. 1933 __m128 _mm_or_ps (__m128 a, __m128 b) pure @safe 1934 { 1935 static if (DMD_with_DSIMD) 1936 return cast(__m128)__simd(XMM.ORPS, a, b); 1937 else 1938 return cast(__m128)(cast(__m128i)a | cast(__m128i)b); 1939 } 1940 // TODO unittest and force inline 1941 1942 deprecated("Use _mm_avg_pu8 instead") alias _m_pavgb = _mm_avg_pu8;/// 1943 deprecated("Use _mm_avg_pu16 instead") alias _m_pavgw = _mm_avg_pu16;/// 1944 deprecated("Use _mm_extract_pi16 instead") alias _m_pextrw = _mm_extract_pi16;/// 1945 deprecated("Use _mm_insert_pi16 instead") alias _m_pinsrw = _mm_insert_pi16;/// 1946 deprecated("Use _mm_max_pi16 instead") alias _m_pmaxsw = _mm_max_pi16;/// 1947 deprecated("Use _mm_max_pu8 instead") alias _m_pmaxub = _mm_max_pu8;/// 1948 deprecated("Use _mm_min_pi16 instead") alias _m_pminsw = _mm_min_pi16;/// 1949 deprecated("Use _mm_min_pu8 instead") alias _m_pminub = _mm_min_pu8;/// 1950 deprecated("Use _mm_movemask_pi8 instead") alias _m_pmovmskb = _mm_movemask_pi8;/// 1951 deprecated("Use _mm_mulhi_pu16 instead") alias _m_pmulhuw = _mm_mulhi_pu16;/// 1952 1953 enum _MM_HINT_T0 = 3; /// 1954 enum _MM_HINT_T1 = 2; /// 1955 enum _MM_HINT_T2 = 1; /// 1956 enum _MM_HINT_NTA = 0; /// 1957 1958 1959 version(LDC) 1960 { 1961 // Starting with LLVM 10, it seems llvm.prefetch has changed its name. 1962 // Was reported at: https://github.com/ldc-developers/ldc/issues/3397 1963 static if (__VERSION__ >= 2091) 1964 { 1965 pragma(LDC_intrinsic, "llvm.prefetch.p0i8") // was "llvm.prefetch" 1966 void llvm_prefetch_fixed(void* ptr, uint rw, uint locality, uint cachetype) pure @safe; 1967 } 1968 } 1969 1970 /// Fetch the line of data from memory that contains address `p` to a location in the 1971 /// cache hierarchy specified by the locality hint i. 1972 /// 1973 /// Warning: `locality` is a compile-time parameter, unlike in Intel Intrinsics API. 1974 void _mm_prefetch(int locality)(const(void)* p) pure @trusted 1975 { 1976 static if (GDC_with_SSE) 1977 { 1978 return __builtin_prefetch(p, (locality & 0x4) >> 2, locality & 0x3); 1979 } 1980 else static if (DMD_with_DSIMD) 1981 { 1982 enum bool isWrite = (locality & 0x4) != 0; 1983 enum level = locality & 3; 1984 return prefetch!(isWrite, level)(p); 1985 } 1986 else version(LDC) 1987 { 1988 static if (__VERSION__ >= 2091) 1989 { 1990 // const_cast here. `llvm_prefetch` wants a mutable pointer 1991 llvm_prefetch_fixed( cast(void*)p, 0, locality, 1); 1992 } 1993 else 1994 { 1995 // const_cast here. `llvm_prefetch` wants a mutable pointer 1996 llvm_prefetch( cast(void*)p, 0, locality, 1); 1997 } 1998 } 1999 else version(D_InlineAsm_X86_64) 2000 { 2001 static if (locality == _MM_HINT_NTA) 2002 { 2003 asm pure nothrow @nogc @trusted 2004 { 2005 mov RAX, p; 2006 prefetchnta [RAX]; 2007 } 2008 } 2009 else static if (locality == _MM_HINT_T0) 2010 { 2011 asm pure nothrow @nogc @trusted 2012 { 2013 mov RAX, p; 2014 prefetcht0 [RAX]; 2015 } 2016 } 2017 else static if (locality == _MM_HINT_T1) 2018 { 2019 asm pure nothrow @nogc @trusted 2020 { 2021 mov RAX, p; 2022 prefetcht1 [RAX]; 2023 } 2024 } 2025 else static if (locality == _MM_HINT_T2) 2026 { 2027 asm pure nothrow @nogc @trusted 2028 { 2029 mov RAX, p; 2030 prefetcht2 [RAX]; 2031 } 2032 } 2033 else 2034 assert(false); // invalid locality hint 2035 } 2036 else version(D_InlineAsm_X86) 2037 { 2038 static if (locality == _MM_HINT_NTA) 2039 { 2040 asm pure nothrow @nogc @trusted 2041 { 2042 mov EAX, p; 2043 prefetchnta [EAX]; 2044 } 2045 } 2046 else static if (locality == _MM_HINT_T0) 2047 { 2048 asm pure nothrow @nogc @trusted 2049 { 2050 mov EAX, p; 2051 prefetcht0 [EAX]; 2052 } 2053 } 2054 else static if (locality == _MM_HINT_T1) 2055 { 2056 asm pure nothrow @nogc @trusted 2057 { 2058 mov EAX, p; 2059 prefetcht1 [EAX]; 2060 } 2061 } 2062 else static if (locality == _MM_HINT_T2) 2063 { 2064 asm pure nothrow @nogc @trusted 2065 { 2066 mov EAX, p; 2067 prefetcht2 [EAX]; 2068 } 2069 } 2070 else 2071 assert(false); // invalid locality hint 2072 } 2073 else 2074 { 2075 // Generic version: do nothing. From bitter experience, 2076 // it's unlikely you get ANY speed-up with manual prefetching. 2077 // Prefetching or not doesn't change program behaviour. 2078 } 2079 } 2080 unittest 2081 { 2082 // From Intel documentation: 2083 // "The amount of data prefetched is also processor implementation-dependent. It will, however, be a minimum of 2084 // 32 bytes." 2085 ubyte[256] cacheline; // though it seems it cannot generate GP fault 2086 _mm_prefetch!_MM_HINT_T0(cacheline.ptr); 2087 _mm_prefetch!_MM_HINT_T1(cacheline.ptr); 2088 _mm_prefetch!_MM_HINT_T2(cacheline.ptr); 2089 _mm_prefetch!_MM_HINT_NTA(cacheline.ptr); 2090 } 2091 2092 deprecated("Use _mm_sad_pu8 instead") alias _m_psadbw = _mm_sad_pu8;/// 2093 deprecated("Use _mm_shuffle_pi16 instead") alias _m_pshufw = _mm_shuffle_pi16;/// 2094 2095 2096 /// Compute the approximate reciprocal of packed single-precision (32-bit) floating-point elements in a`` , 2097 /// and return the results. The maximum relative error for this approximation is less than 1.5*2^-12. 2098 __m128 _mm_rcp_ps (__m128 a) pure @trusted 2099 { 2100 static if (DMD_with_DSIMD) 2101 { 2102 return cast(__m128) __simd(XMM.RCPPS, a); 2103 } 2104 else static if (GDC_with_SSE) 2105 { 2106 return __builtin_ia32_rcpps(a); 2107 } 2108 else static if (LDC_with_SSE1) 2109 { 2110 return __builtin_ia32_rcpps(a); 2111 } 2112 else 2113 { 2114 a.ptr[0] = 1.0f / a.array[0]; 2115 a.ptr[1] = 1.0f / a.array[1]; 2116 a.ptr[2] = 1.0f / a.array[2]; 2117 a.ptr[3] = 1.0f / a.array[3]; 2118 return a; 2119 } 2120 } 2121 unittest 2122 { 2123 __m128 A = _mm_setr_ps(2.34f, -70000.0f, 0.00001f, 345.5f); 2124 __m128 groundTruth = _mm_set1_ps(1.0f) / A; 2125 __m128 result = _mm_rcp_ps(A); 2126 foreach(i; 0..4) 2127 { 2128 double relError = (cast(double)(groundTruth.array[i]) / result.array[i]) - 1; 2129 assert(abs_double(relError) < 0.00037); // 1.5*2^-12 is 0.00036621093 2130 } 2131 } 2132 2133 /// Compute the approximate reciprocal of the lower single-precision (32-bit) floating-point element in `a`, store it 2134 /// in the lower element of the result, and copy the upper 3 packed elements from `a` to the upper elements of result. 2135 /// The maximum relative error for this approximation is less than 1.5*2^-12. 2136 __m128 _mm_rcp_ss (__m128 a) pure @trusted 2137 { 2138 // Disabled, see https://issues.dlang.org/show_bug.cgi?id=23049 2139 /*static if (DMD_with_DSIMD) 2140 { 2141 return cast(__m128) __simd(XMM.RCPSS, a); 2142 } 2143 else*/ 2144 static if (GDC_with_SSE) 2145 { 2146 return __builtin_ia32_rcpss(a); 2147 } 2148 else static if (LDC_with_SSE1) 2149 { 2150 return __builtin_ia32_rcpss(a); 2151 } 2152 else 2153 { 2154 a.ptr[0] = 1.0f / a.array[0]; 2155 return a; 2156 } 2157 } 2158 unittest 2159 { 2160 __m128 A = _mm_setr_ps(2.34f, -70000.0f, 0.00001f, 345.5f); 2161 __m128 correct = _mm_setr_ps(1 / 2.34f, -70000.0f, 0.00001f, 345.5f); 2162 __m128 R = _mm_rcp_ss(A); 2163 double relError = (cast(double)(correct.array[0]) / R.array[0]) - 1; 2164 assert(abs_double(relError) < 0.00037); // 1.5*2^-12 is 0.00036621093 2165 assert(R.array[1] == correct.array[1]); 2166 assert(R.array[2] == correct.array[2]); 2167 assert(R.array[3] == correct.array[3]); 2168 } 2169 2170 /// Reallocate `size` bytes of memory, aligned to the alignment specified in `alignment`, 2171 /// and return a pointer to the newly allocated memory. 2172 /// `_mm_free` or `alignedRealloc` with size 0 should be used to free memory that is 2173 /// allocated with `_mm_malloc` or `_mm_realloc`. 2174 /// Previous data is preserved. 2175 void* _mm_realloc(void* aligned, size_t size, size_t alignment) nothrow @nogc // #BONUS 2176 { 2177 return alignedReallocImpl!true(aligned, size, alignment); 2178 } 2179 unittest 2180 { 2181 enum NALLOC = 8; 2182 enum size_t[8] ALIGNMENTS = [1, 2, 4, 8, 16, 32, 64, 128]; 2183 2184 void*[NALLOC] alloc; 2185 2186 foreach(t; 0..100) 2187 { 2188 foreach(n; 0..NALLOC) 2189 { 2190 size_t alignment = ALIGNMENTS[n]; 2191 size_t s = ( (n + t * 69096) & 0xffff ); 2192 alloc[n] = _mm_realloc(alloc[n], s, alignment); 2193 assert(isPointerAligned(alloc[n], alignment)); 2194 foreach(b; 0..s) 2195 (cast(ubyte*)alloc[n])[b] = cast(ubyte)n; 2196 } 2197 } 2198 foreach(n; 0..NALLOC) 2199 { 2200 alloc[n] = _mm_realloc(alloc[n], 0, ALIGNMENTS[n]); 2201 } 2202 } 2203 2204 /// Reallocate `size` bytes of memory, aligned to the alignment specified in `alignment`, 2205 /// and return a pointer to the newly allocated memory. 2206 /// `_mm_free` or `alignedRealloc` with size 0 should be used to free memory that is 2207 /// allocated with `_mm_malloc` or `_mm_realloc`. 2208 /// Previous data is discarded. 2209 void* _mm_realloc_discard(void* aligned, size_t size, size_t alignment) nothrow @nogc // #BONUS 2210 { 2211 return alignedReallocImpl!false(aligned, size, alignment); 2212 } 2213 2214 /// Compute the approximate reciprocal square root of packed single-precision (32-bit) floating-point elements in `a`. 2215 /// The maximum relative error for this approximation is less than 1.5*2^-12. 2216 __m128 _mm_rsqrt_ps (__m128 a) pure @trusted 2217 { 2218 static if (DMD_with_DSIMD) 2219 { 2220 return cast(__m128) __simd(XMM.RSQRTPS, a); 2221 } 2222 else static if (GDC_with_SSE) 2223 { 2224 return __builtin_ia32_rsqrtps(a); 2225 } 2226 else static if (LDC_with_SSE1) 2227 { 2228 return __builtin_ia32_rsqrtps(a); 2229 } 2230 else version(LDC) 2231 { 2232 a[0] = 1.0f / llvm_sqrt(a[0]); 2233 a[1] = 1.0f / llvm_sqrt(a[1]); 2234 a[2] = 1.0f / llvm_sqrt(a[2]); 2235 a[3] = 1.0f / llvm_sqrt(a[3]); 2236 return a; 2237 } 2238 else 2239 { 2240 a.ptr[0] = 1.0f / sqrt(a.array[0]); 2241 a.ptr[1] = 1.0f / sqrt(a.array[1]); 2242 a.ptr[2] = 1.0f / sqrt(a.array[2]); 2243 a.ptr[3] = 1.0f / sqrt(a.array[3]); 2244 return a; 2245 } 2246 } 2247 unittest 2248 { 2249 __m128 A = _mm_setr_ps(2.34f, 70000.0f, 0.00001f, 345.5f); 2250 __m128 groundTruth = _mm_setr_ps(0.65372045f, 0.00377964473f, 316.227766f, 0.05379921937f); 2251 __m128 result = _mm_rsqrt_ps(A); 2252 foreach(i; 0..4) 2253 { 2254 double relError = (cast(double)(groundTruth.array[i]) / result.array[i]) - 1; 2255 assert(abs_double(relError) < 0.00037); // 1.5*2^-12 is 0.00036621093 2256 } 2257 } 2258 2259 /// Compute the approximate reciprocal square root of the lower single-precision (32-bit) floating-point element in `a`, 2260 /// store the result in the lower element. Copy the upper 3 packed elements from `a` to the upper elements of result. 2261 /// The maximum relative error for this approximation is less than 1.5*2^-12. 2262 __m128 _mm_rsqrt_ss (__m128 a) pure @trusted 2263 { 2264 static if (DMD_with_DSIMD) 2265 { 2266 return cast(__m128) __simd(XMM.RSQRTSS, a); 2267 } 2268 else static if (GDC_with_SSE) 2269 { 2270 return __builtin_ia32_rsqrtss(a); 2271 } 2272 else static if (LDC_with_SSE1) 2273 { 2274 return __builtin_ia32_rsqrtss(a); 2275 } 2276 else version(LDC) 2277 { 2278 a[0] = 1.0f / llvm_sqrt(a[0]); 2279 return a; 2280 } 2281 else 2282 { 2283 a[0] = 1.0f / sqrt(a[0]); 2284 return a; 2285 } 2286 } 2287 unittest // this one test 4 different intrinsics: _mm_rsqrt_ss, _mm_rsqrt_ps, _mm_rcp_ps, _mm_rcp_ss 2288 { 2289 double maxRelativeError = 0.000245; // -72 dB, stuff is apparently more precise than said in the doc? 2290 void testApproximateSSE(float number) nothrow @nogc 2291 { 2292 __m128 A = _mm_set1_ps(number); 2293 2294 // test _mm_rcp_ps 2295 __m128 B = _mm_rcp_ps(A); 2296 foreach(i; 0..4) 2297 { 2298 double exact = 1.0f / A.array[i]; 2299 double ratio = cast(double)(B.array[i]) / cast(double)(exact); 2300 assert(abs_double(ratio - 1) <= maxRelativeError); 2301 } 2302 2303 // test _mm_rcp_ss 2304 { 2305 B = _mm_rcp_ss(A); 2306 double exact = 1.0f / A.array[0]; 2307 double ratio = cast(double)(B.array[0]) / cast(double)(exact); 2308 assert(abs_double(ratio - 1) <= maxRelativeError); 2309 } 2310 2311 // test _mm_rsqrt_ps 2312 B = _mm_rsqrt_ps(A); 2313 foreach(i; 0..4) 2314 { 2315 double exact = 1.0f / sqrt(A.array[i]); 2316 double ratio = cast(double)(B.array[i]) / cast(double)(exact); 2317 assert(abs_double(ratio - 1) <= maxRelativeError); 2318 } 2319 2320 // test _mm_rsqrt_ss 2321 { 2322 B = _mm_rsqrt_ss(A); 2323 double exact = 1.0f / sqrt(A.array[0]); 2324 double ratio = cast(double)(B.array[0]) / cast(double)(exact); 2325 assert(abs_double(ratio - 1) <= maxRelativeError); 2326 } 2327 } 2328 2329 testApproximateSSE(0.00001f); 2330 testApproximateSSE(1.1f); 2331 testApproximateSSE(345.0f); 2332 testApproximateSSE(2.45674864151f); 2333 testApproximateSSE(700000.0f); 2334 testApproximateSSE(10000000.0f); 2335 testApproximateSSE(27841456468.0f); 2336 } 2337 2338 /// Compute the absolute differences of packed unsigned 8-bit integers in `a` and `b`, then horizontally sum each 2339 /// consecutive 8 differences to produce four unsigned 16-bit integers, and pack these unsigned 16-bit integers in the 2340 /// low 16 bits of result. 2341 __m64 _mm_sad_pu8 (__m64 a, __m64 b) pure @safe 2342 { 2343 return to_m64(_mm_sad_epu8(to_m128i(a), to_m128i(b))); 2344 } 2345 2346 /// Set the exception mask bits of the MXCSR control and status register to the value in unsigned 32-bit integer 2347 /// `_MM_MASK_xxxx`. The exception mask may contain any of the following flags: `_MM_MASK_INVALID`, `_MM_MASK_DIV_ZERO`, 2348 /// `_MM_MASK_DENORM`, `_MM_MASK_OVERFLOW`, `_MM_MASK_UNDERFLOW`, `_MM_MASK_INEXACT`. 2349 void _MM_SET_EXCEPTION_MASK(int _MM_MASK_xxxx) @safe 2350 { 2351 // Note: unsupported on ARM 2352 _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | _MM_MASK_xxxx); 2353 } 2354 2355 /// Set the exception state bits of the MXCSR control and status register to the value in unsigned 32-bit integer 2356 /// `_MM_EXCEPT_xxxx`. The exception state may contain any of the following flags: `_MM_EXCEPT_INVALID`, 2357 /// `_MM_EXCEPT_DIV_ZERO`, `_MM_EXCEPT_DENORM`, `_MM_EXCEPT_OVERFLOW`, `_MM_EXCEPT_UNDERFLOW`, `_MM_EXCEPT_INEXACT`. 2358 void _MM_SET_EXCEPTION_STATE(int _MM_EXCEPT_xxxx) @safe 2359 { 2360 // Note: unsupported on ARM 2361 _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | _MM_EXCEPT_xxxx); 2362 } 2363 2364 /// Set the flush zero bits of the MXCSR control and status register to the value in unsigned 32-bit integer 2365 /// `_MM_FLUSH_xxxx`. The flush zero may contain any of the following flags: `_MM_FLUSH_ZERO_ON` or `_MM_FLUSH_ZERO_OFF`. 2366 void _MM_SET_FLUSH_ZERO_MODE(int _MM_FLUSH_xxxx) @safe 2367 { 2368 _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | _MM_FLUSH_xxxx); 2369 } 2370 2371 /// Set packed single-precision (32-bit) floating-point elements with the supplied values. 2372 __m128 _mm_set_ps (float e3, float e2, float e1, float e0) pure @trusted 2373 { 2374 // Note: despite appearances, generates sensible code, 2375 // inlines correctly and is constant folded 2376 float[4] result = [e0, e1, e2, e3]; 2377 return loadUnaligned!(float4)(result.ptr); 2378 } 2379 unittest 2380 { 2381 __m128 A = _mm_set_ps(3, 2, 1, 546); 2382 float[4] correct = [546.0f, 1.0f, 2.0f, 3.0f]; 2383 assert(A.array == correct); 2384 } 2385 2386 deprecated("Use _mm_set1_ps instead") alias _mm_set_ps1 = _mm_set1_ps; /// 2387 2388 /// Set the rounding mode bits of the MXCSR control and status register to the value in unsigned 32-bit integer 2389 /// `_MM_ROUND_xxxx`. The rounding mode may contain any of the following flags: `_MM_ROUND_NEAREST`, `_MM_ROUND_DOWN`, 2390 /// `_MM_ROUND_UP`, `_MM_ROUND_TOWARD_ZERO`. 2391 void _MM_SET_ROUNDING_MODE(int _MM_ROUND_xxxx) @safe 2392 { 2393 // Work-around for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98607 2394 version(GNU) asm @trusted { "" : : : "memory"; } 2395 _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | _MM_ROUND_xxxx); 2396 } 2397 2398 /// Copy single-precision (32-bit) floating-point element `a` to the lower element of result, and zero the upper 3 elements. 2399 __m128 _mm_set_ss (float a) pure @trusted 2400 { 2401 static if (DMD_with_DSIMD) 2402 { 2403 return cast(__m128) __simd(XMM.LODSS, a); 2404 } 2405 else 2406 { 2407 __m128 r = _mm_setzero_ps(); 2408 r.ptr[0] = a; 2409 return r; 2410 } 2411 } 2412 unittest 2413 { 2414 float[4] correct = [42.0f, 0.0f, 0.0f, 0.0f]; 2415 __m128 A = _mm_set_ss(42.0f); 2416 assert(A.array == correct); 2417 } 2418 2419 /// Broadcast single-precision (32-bit) floating-point value `a` to all elements. 2420 __m128 _mm_set1_ps (float a) pure @trusted 2421 { 2422 pragma(inline, true); 2423 __m128 r = a; 2424 return r; 2425 } 2426 unittest 2427 { 2428 float[4] correct = [42.0f, 42.0f, 42.0f, 42.0f]; 2429 __m128 A = _mm_set1_ps(42.0f); 2430 assert(A.array == correct); 2431 } 2432 2433 /// Set the MXCSR control and status register with the value in unsigned 32-bit integer `controlWord`. 2434 void _mm_setcsr(uint controlWord) @trusted 2435 { 2436 static if (LDC_with_ARM) 2437 { 2438 // Convert from SSE to ARM control word. This is done _partially_ 2439 // and only support rounding mode changes. 2440 2441 // "To alter some bits of a VFP system register without 2442 // affecting other bits, use a read-modify-write procedure" 2443 uint fpscr = arm_get_fpcr(); 2444 2445 // Bits 23 to 22 are rounding modes, however not used in NEON 2446 fpscr = fpscr & ~_MM_ROUND_MASK_ARM; 2447 switch(controlWord & _MM_ROUND_MASK) 2448 { 2449 default: 2450 case _MM_ROUND_NEAREST: fpscr |= _MM_ROUND_NEAREST_ARM; break; 2451 case _MM_ROUND_DOWN: fpscr |= _MM_ROUND_DOWN_ARM; break; 2452 case _MM_ROUND_UP: fpscr |= _MM_ROUND_UP_ARM; break; 2453 case _MM_ROUND_TOWARD_ZERO: fpscr |= _MM_ROUND_TOWARD_ZERO_ARM; break; 2454 } 2455 fpscr = fpscr & ~_MM_FLUSH_ZERO_MASK_ARM; 2456 if (controlWord & _MM_FLUSH_ZERO_MASK) 2457 fpscr |= _MM_FLUSH_ZERO_MASK_ARM; 2458 arm_set_fpcr(fpscr); 2459 } 2460 else version(GNU) 2461 { 2462 static if (GDC_with_SSE) 2463 { 2464 // Work-around for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98607 2465 version(GNU) asm @trusted { "" : : : "memory"; } 2466 __builtin_ia32_ldmxcsr(controlWord); 2467 } 2468 else version(X86) 2469 { 2470 asm nothrow @nogc @trusted 2471 { 2472 "ldmxcsr %0;\n" 2473 : 2474 : "m" (controlWord) 2475 : ; 2476 } 2477 } 2478 else 2479 static assert(false); 2480 } 2481 else version (InlineX86Asm) 2482 { 2483 asm nothrow @nogc @safe 2484 { 2485 ldmxcsr controlWord; 2486 } 2487 } 2488 else 2489 static assert(0, "Not yet supported"); 2490 } 2491 unittest 2492 { 2493 _mm_setcsr(_mm_getcsr()); 2494 } 2495 2496 /// Set packed single-precision (32-bit) floating-point elements with the supplied values in reverse order. 2497 __m128 _mm_setr_ps (float e3, float e2, float e1, float e0) pure @trusted 2498 { 2499 pragma(inline, true); 2500 version(LDC) 2501 { 2502 float[4] result = [e3, e2, e1, e0]; 2503 return loadUnaligned!(float4)(result.ptr); 2504 } 2505 else 2506 { 2507 __m128 r; 2508 r.ptr[0] = e3; 2509 r.ptr[1] = e2; 2510 r.ptr[2] = e1; 2511 r.ptr[3] = e0; 2512 return r; 2513 } 2514 } 2515 unittest 2516 { 2517 __m128 A = _mm_setr_ps(3, 2, 1, 546); 2518 float[4] correct = [3.0f, 2.0f, 1.0f, 546.0f]; 2519 assert(A.array == correct); 2520 assert(A.array[0] == 3.0f); 2521 assert(A.array[1] == 2.0f); 2522 assert(A.array[2] == 1.0f); 2523 assert(A.array[3] == 546.0f); 2524 } 2525 2526 /// Return vector of type `__m128` with all elements set to zero. 2527 __m128 _mm_setzero_ps() pure @trusted 2528 { 2529 pragma(inline, true); 2530 float4 r; 2531 r = 0.0f; 2532 return r; 2533 } 2534 unittest 2535 { 2536 __m128 R = _mm_setzero_ps(); 2537 float[4] correct = [0.0f, 0, 0, 0]; 2538 assert(R.array == correct); 2539 } 2540 2541 /// Perform a serializing operation on all store-to-memory instructions that were issued prior 2542 /// to this instruction. Guarantees that every store instruction that precedes, in program order, 2543 /// is globally visible before any store instruction which follows the fence in program order. 2544 void _mm_sfence() @trusted 2545 { 2546 version(GNU) 2547 { 2548 static if (GDC_with_SSE) 2549 { 2550 __builtin_ia32_sfence(); 2551 } 2552 else version(X86) 2553 { 2554 asm pure nothrow @nogc @trusted 2555 { 2556 "sfence;\n" : : : ; 2557 } 2558 } 2559 else 2560 static assert(false); 2561 } 2562 else static if (LDC_with_SSE1) 2563 { 2564 __builtin_ia32_sfence(); 2565 } 2566 else static if (DMD_with_asm) 2567 { 2568 asm nothrow @nogc pure @safe 2569 { 2570 sfence; 2571 } 2572 } 2573 else version(LDC) 2574 { 2575 llvm_memory_fence(); // PERF: this generates mfence instead of sfence 2576 } 2577 else 2578 static assert(false); 2579 } 2580 unittest 2581 { 2582 _mm_sfence(); 2583 } 2584 2585 /// Warning: the immediate shuffle value `imm8` is given at compile-time instead of runtime. 2586 __m64 _mm_shuffle_pi16(int imm8)(__m64 a) pure @safe 2587 { 2588 return cast(__m64) shufflevector!(short4, ( (imm8 >> 0) & 3 ), 2589 ( (imm8 >> 2) & 3 ), 2590 ( (imm8 >> 4) & 3 ), 2591 ( (imm8 >> 6) & 3 ))(cast(short4)a, cast(short4)a); 2592 } 2593 unittest 2594 { 2595 __m64 A = _mm_setr_pi16(0, 1, 2, 3); 2596 enum int SHUFFLE = _MM_SHUFFLE(0, 1, 2, 3); 2597 short4 B = cast(short4) _mm_shuffle_pi16!SHUFFLE(A); 2598 short[4] expectedB = [ 3, 2, 1, 0 ]; 2599 assert(B.array == expectedB); 2600 } 2601 2602 /// Warning: the immediate shuffle value `imm8` is given at compile-time instead of runtime. 2603 __m128 _mm_shuffle_ps(ubyte imm)(__m128 a, __m128 b) pure @safe 2604 { 2605 return shufflevector!(__m128, imm & 3, (imm>>2) & 3, 4 + ((imm>>4) & 3), 4 + ((imm>>6) & 3) )(a, b); 2606 } 2607 2608 /// Compute the square root of packed single-precision (32-bit) floating-point elements in `a`. 2609 __m128 _mm_sqrt_ps(__m128 a) @trusted 2610 { 2611 static if (GDC_with_SSE) 2612 { 2613 return __builtin_ia32_sqrtps(a); 2614 } 2615 else version(LDC) 2616 { 2617 // Disappeared with LDC 1.11 2618 static if (__VERSION__ < 2081) 2619 return __builtin_ia32_sqrtps(a); 2620 else 2621 { 2622 a[0] = llvm_sqrt(a[0]); 2623 a[1] = llvm_sqrt(a[1]); 2624 a[2] = llvm_sqrt(a[2]); 2625 a[3] = llvm_sqrt(a[3]); 2626 return a; 2627 } 2628 } 2629 else 2630 { 2631 a.ptr[0] = sqrt(a.array[0]); 2632 a.ptr[1] = sqrt(a.array[1]); 2633 a.ptr[2] = sqrt(a.array[2]); 2634 a.ptr[3] = sqrt(a.array[3]); 2635 return a; 2636 } 2637 } 2638 unittest 2639 { 2640 __m128 A = _mm_sqrt_ps(_mm_set1_ps(4.0f)); 2641 assert(A.array[0] == 2.0f); 2642 assert(A.array[1] == 2.0f); 2643 assert(A.array[2] == 2.0f); 2644 assert(A.array[3] == 2.0f); 2645 } 2646 2647 /// Compute the square root of the lower single-precision (32-bit) floating-point element in `a`, store it in the lower 2648 /// element, and copy the upper 3 packed elements from `a` to the upper elements of result. 2649 __m128 _mm_sqrt_ss(__m128 a) @trusted 2650 { 2651 static if (GDC_with_SSE) 2652 { 2653 return __builtin_ia32_sqrtss(a); 2654 } 2655 else version(LDC) 2656 { 2657 a.ptr[0] = llvm_sqrt(a.array[0]); 2658 return a; 2659 } 2660 else 2661 { 2662 a.ptr[0] = sqrt(a.array[0]); 2663 return a; 2664 } 2665 } 2666 unittest 2667 { 2668 __m128 A = _mm_sqrt_ss(_mm_set1_ps(4.0f)); 2669 assert(A.array[0] == 2.0f); 2670 assert(A.array[1] == 4.0f); 2671 assert(A.array[2] == 4.0f); 2672 assert(A.array[3] == 4.0f); 2673 } 2674 2675 /// Store 128-bits (composed of 4 packed single-precision (32-bit) floating-point elements) from `a` into memory. 2676 /// `mem_addr` must be aligned on a 16-byte boundary or a general-protection exception may be generated. 2677 void _mm_store_ps (float* mem_addr, __m128 a) pure 2678 { 2679 pragma(inline, true); 2680 __m128* aligned = cast(__m128*)mem_addr; 2681 *aligned = a; 2682 } 2683 2684 deprecated("Use _mm_store1_ps instead") alias _mm_store_ps1 = _mm_store1_ps; /// 2685 2686 /// Store the lower single-precision (32-bit) floating-point element from `a` into memory. 2687 /// `mem_addr` does not need to be aligned on any particular boundary. 2688 void _mm_store_ss (float* mem_addr, __m128 a) pure @safe 2689 { 2690 pragma(inline, true); 2691 *mem_addr = a.array[0]; 2692 } 2693 unittest 2694 { 2695 float a; 2696 _mm_store_ss(&a, _mm_set_ps(3, 2, 1, 546)); 2697 assert(a == 546); 2698 } 2699 2700 /// Store the lower single-precision (32-bit) floating-point element from `a` into 4 contiguous elements in memory. 2701 /// `mem_addr` must be aligned on a 16-byte boundary or a general-protection exception may be generated. 2702 void _mm_store1_ps(float* mem_addr, __m128 a) pure @trusted // TODO: shouldn't be trusted 2703 { 2704 __m128* aligned = cast(__m128*)mem_addr; 2705 __m128 r; 2706 r.ptr[0] = a.array[0]; 2707 r.ptr[1] = a.array[0]; 2708 r.ptr[2] = a.array[0]; 2709 r.ptr[3] = a.array[0]; 2710 *aligned = r; 2711 } 2712 unittest 2713 { 2714 align(16) float[4] A; 2715 _mm_store1_ps(A.ptr, _mm_set_ss(42.0f)); 2716 float[4] correct = [42.0f, 42, 42, 42]; 2717 assert(A == correct); 2718 } 2719 2720 /// Store the upper 2 single-precision (32-bit) floating-point elements from `a` into memory. 2721 void _mm_storeh_pi(__m64* p, __m128 a) pure @trusted 2722 { 2723 pragma(inline, true); 2724 long2 la = cast(long2)a; 2725 (*p).ptr[0] = la.array[1]; 2726 } 2727 unittest 2728 { 2729 __m64 R = _mm_setzero_si64(); 2730 long2 A = [13, 25]; 2731 _mm_storeh_pi(&R, cast(__m128)A); 2732 assert(R.array[0] == 25); 2733 } 2734 2735 /// Store the lower 2 single-precision (32-bit) floating-point elements from `a` into memory. 2736 void _mm_storel_pi(__m64* p, __m128 a) pure @trusted 2737 { 2738 pragma(inline, true); 2739 long2 la = cast(long2)a; 2740 (*p).ptr[0] = la.array[0]; 2741 } 2742 unittest 2743 { 2744 __m64 R = _mm_setzero_si64(); 2745 long2 A = [13, 25]; 2746 _mm_storel_pi(&R, cast(__m128)A); 2747 assert(R.array[0] == 13); 2748 } 2749 2750 /// Store 4 single-precision (32-bit) floating-point elements from `a` into memory in reverse order. 2751 /// `mem_addr` must be aligned on a 16-byte boundary or a general-protection exception may be generated. 2752 void _mm_storer_ps(float* mem_addr, __m128 a) pure @trusted // TODO should not be trusted 2753 { 2754 __m128* aligned = cast(__m128*)mem_addr; 2755 __m128 r; 2756 r.ptr[0] = a.array[3]; 2757 r.ptr[1] = a.array[2]; 2758 r.ptr[2] = a.array[1]; 2759 r.ptr[3] = a.array[0]; 2760 *aligned = r; 2761 } 2762 unittest 2763 { 2764 align(16) float[4] A; 2765 _mm_storer_ps(A.ptr, _mm_setr_ps(1.0f, 2, 3, 4)); 2766 float[4] correct = [4.0f, 3.0f, 2.0f, 1.0f]; 2767 assert(A == correct); 2768 } 2769 2770 /// Store 128-bits (composed of 4 packed single-precision (32-bit) floating-point elements) from `a` into memory. 2771 /// `mem_addr` does not need to be aligned on any particular boundary. 2772 void _mm_storeu_ps(float* mem_addr, __m128 a) pure @safe // TODO should not be trusted 2773 { 2774 pragma(inline, true); 2775 storeUnaligned!(float4)(a, mem_addr); 2776 } 2777 2778 /// Store 64-bits of integer data from `a` into memory using a non-temporal memory hint. 2779 void _mm_stream_pi (__m64* mem_addr, __m64 a) 2780 { 2781 // BUG see `_mm_stream_ps` for an explanation why we don't implement non-temporal moves 2782 *mem_addr = a; // it's a regular move instead 2783 } 2784 2785 /// Store 128-bits (composed of 4 packed single-precision (32-bit) floating-point elements) from `a`s into memory using 2786 /// a non-temporal memory hint. mem_addr must be aligned on a 16-byte boundary or a general-protection exception may be 2787 /// generated. 2788 void _mm_stream_ps (float* mem_addr, __m128 a) 2789 { 2790 // BUG: can't implement non-temporal store with LDC inlineIR since !nontemporal 2791 // needs some IR outside this function that would say: 2792 // 2793 // !0 = !{ i32 1 } 2794 // 2795 // It's a LLVM IR metadata description. 2796 __m128* dest = cast(__m128*)mem_addr; 2797 *dest = a; // it's a regular move instead 2798 } 2799 unittest 2800 { 2801 align(16) float[4] A; 2802 _mm_stream_ps(A.ptr, _mm_set1_ps(78.0f)); 2803 assert(A[0] == 78.0f && A[1] == 78.0f && A[2] == 78.0f && A[3] == 78.0f); 2804 } 2805 2806 /// Subtract packed single-precision (32-bit) floating-point elements in `b` from packed single-precision (32-bit) 2807 /// floating-point elements in `a`. 2808 __m128 _mm_sub_ps(__m128 a, __m128 b) pure @safe 2809 { 2810 pragma(inline, true); 2811 return a - b; 2812 } 2813 unittest 2814 { 2815 __m128 a = [1.5f, -2.0f, 3.0f, 1.0f]; 2816 a = _mm_sub_ps(a, a); 2817 float[4] correct = [0.0f, 0.0f, 0.0f, 0.0f]; 2818 assert(a.array == correct); 2819 } 2820 2821 /// Subtract the lower single-precision (32-bit) floating-point element in `b` from the lower single-precision (32-bit) 2822 /// floating-point element in `a`, store the subtration result in the lower element of result, and copy the upper 3 2823 /// packed elements from a to the upper elements of result. 2824 __m128 _mm_sub_ss(__m128 a, __m128 b) pure @safe 2825 { 2826 static if (DMD_with_DSIMD) 2827 return cast(__m128) __simd(XMM.SUBSS, a, b); 2828 else static if (GDC_with_SSE) 2829 return __builtin_ia32_subss(a, b); 2830 else 2831 { 2832 a[0] -= b[0]; 2833 return a; 2834 } 2835 } 2836 unittest 2837 { 2838 __m128 a = [1.5f, -2.0f, 3.0f, 1.0f]; 2839 a = _mm_sub_ss(a, a); 2840 float[4] correct = [0.0f, -2.0, 3.0f, 1.0f]; 2841 assert(a.array == correct); 2842 } 2843 2844 /// Transpose the 4x4 matrix formed by the 4 rows of single-precision (32-bit) floating-point elements in row0, row1, 2845 /// row2, and row3, and store the transposed matrix in these vectors (row0 now contains column 0, etc.). 2846 void _MM_TRANSPOSE4_PS (ref __m128 row0, ref __m128 row1, ref __m128 row2, ref __m128 row3) pure @safe 2847 { 2848 __m128 tmp3, tmp2, tmp1, tmp0; 2849 tmp0 = _mm_unpacklo_ps(row0, row1); 2850 tmp2 = _mm_unpacklo_ps(row2, row3); 2851 tmp1 = _mm_unpackhi_ps(row0, row1); 2852 tmp3 = _mm_unpackhi_ps(row2, row3); 2853 row0 = _mm_movelh_ps(tmp0, tmp2); 2854 row1 = _mm_movehl_ps(tmp2, tmp0); 2855 row2 = _mm_movelh_ps(tmp1, tmp3); 2856 row3 = _mm_movehl_ps(tmp3, tmp1); 2857 } 2858 unittest 2859 { 2860 __m128 l0 = _mm_setr_ps(0, 1, 2, 3); 2861 __m128 l1 = _mm_setr_ps(4, 5, 6, 7); 2862 __m128 l2 = _mm_setr_ps(8, 9, 10, 11); 2863 __m128 l3 = _mm_setr_ps(12, 13, 14, 15); 2864 _MM_TRANSPOSE4_PS(l0, l1, l2, l3); 2865 float[4] r0 = [0.0f, 4, 8, 12]; 2866 float[4] r1 = [1.0f, 5, 9, 13]; 2867 float[4] r2 = [2.0f, 6, 10, 14]; 2868 float[4] r3 = [3.0f, 7, 11, 15]; 2869 assert(l0.array == r0); 2870 assert(l1.array == r1); 2871 assert(l2.array == r2); 2872 assert(l3.array == r3); 2873 } 2874 2875 // Note: the only difference between these intrinsics is the signalling 2876 // behaviour of quiet NaNs. This is incorrect but the case where 2877 // you would want to differentiate between qNaN and sNaN and then 2878 // treat them differently on purpose seems extremely rare. 2879 alias _mm_ucomieq_ss = _mm_comieq_ss; 2880 alias _mm_ucomige_ss = _mm_comige_ss; 2881 alias _mm_ucomigt_ss = _mm_comigt_ss; 2882 alias _mm_ucomile_ss = _mm_comile_ss; 2883 alias _mm_ucomilt_ss = _mm_comilt_ss; 2884 alias _mm_ucomineq_ss = _mm_comineq_ss; 2885 2886 /// Return vector of type `__m128` with undefined elements. 2887 __m128 _mm_undefined_ps() pure @safe 2888 { 2889 pragma(inline, true); 2890 __m128 undef = void; 2891 return undef; 2892 } 2893 2894 /// Unpack and interleave single-precision (32-bit) floating-point elements from the high half `a` and `b`. 2895 __m128 _mm_unpackhi_ps (__m128 a, __m128 b) pure @trusted 2896 { 2897 version(LDC) 2898 { 2899 // x86: plain version generates unpckhps with LDC 1.0.0 -O1, but shufflevector 8 less instructions in -O0 2900 return shufflevectorLDC!(__m128, 2, 6, 3, 7)(a, b); 2901 } 2902 else 2903 { 2904 __m128 r; 2905 r.ptr[0] = a.array[2]; 2906 r.ptr[1] = b.array[2]; 2907 r.ptr[2] = a.array[3]; 2908 r.ptr[3] = b.array[3]; 2909 return r; 2910 } 2911 } 2912 unittest 2913 { 2914 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 2915 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 2916 __m128 R = _mm_unpackhi_ps(A, B); 2917 float[4] correct = [3.0f, 7.0f, 4.0f, 8.0f]; 2918 assert(R.array == correct); 2919 } 2920 2921 /// Unpack and interleave single-precision (32-bit) floating-point elements from the low half of `a` and `b`. 2922 __m128 _mm_unpacklo_ps (__m128 a, __m128 b) pure @trusted 2923 { 2924 version(LDC) 2925 { 2926 // x86: plain version generates unpckhps with LDC 1.0.0 -O1, but shufflevector 8 less instructions in -O0 2927 return shufflevectorLDC!(__m128, 0, 4, 1, 5)(a, b); 2928 } 2929 else 2930 { 2931 __m128 r; 2932 r.ptr[0] = a.array[0]; 2933 r.ptr[1] = b.array[0]; 2934 r.ptr[2] = a.array[1]; 2935 r.ptr[3] = b.array[1]; 2936 return r; 2937 } 2938 } 2939 unittest 2940 { 2941 __m128 A = _mm_setr_ps(1.0f, 2.0f, 3.0f, 4.0f); 2942 __m128 B = _mm_setr_ps(5.0f, 6.0f, 7.0f, 8.0f); 2943 __m128 R = _mm_unpacklo_ps(A, B); 2944 float[4] correct = [1.0f, 5.0f, 2.0f, 6.0f]; 2945 assert(R.array == correct); 2946 } 2947 2948 /// Compute the bitwise XOR of packed single-precision (32-bit) floating-point elements in `a` and `b`. 2949 __m128 _mm_xor_ps (__m128 a, __m128 b) pure @safe 2950 { 2951 return cast(__m128)(cast(__m128i)a ^ cast(__m128i)b); 2952 } 2953 // TODO unittest and force inline 2954 2955 private 2956 { 2957 // Returns: `true` if the pointer is suitably aligned. 2958 bool isPointerAligned(void* p, size_t alignment) pure 2959 { 2960 assert(alignment != 0); 2961 return ( cast(size_t)p & (alignment - 1) ) == 0; 2962 } 2963 2964 // Returns: next pointer aligned with alignment bytes. 2965 void* nextAlignedPointer(void* start, size_t alignment) pure 2966 { 2967 return cast(void*)nextMultipleOf(cast(size_t)(start), alignment); 2968 } 2969 2970 // Returns number of bytes to actually allocate when asking 2971 // for a particular alignment 2972 @nogc size_t requestedSize(size_t askedSize, size_t alignment) pure 2973 { 2974 enum size_t pointerSize = size_t.sizeof; 2975 return askedSize + alignment - 1 + pointerSize * 3; 2976 } 2977 2978 // Store pointer given by malloc + size + alignment 2979 @nogc void* storeRawPointerPlusInfo(void* raw, size_t size, size_t alignment) pure 2980 { 2981 enum size_t pointerSize = size_t.sizeof; 2982 char* start = cast(char*)raw + pointerSize * 3; 2983 void* aligned = nextAlignedPointer(start, alignment); 2984 void** rawLocation = cast(void**)(cast(char*)aligned - pointerSize); 2985 *rawLocation = raw; 2986 size_t* sizeLocation = cast(size_t*)(cast(char*)aligned - 2 * pointerSize); 2987 *sizeLocation = size; 2988 size_t* alignmentLocation = cast(size_t*)(cast(char*)aligned - 3 * pointerSize); 2989 *alignmentLocation = alignment; 2990 assert( isPointerAligned(aligned, alignment) ); 2991 return aligned; 2992 } 2993 2994 // Returns: x, multiple of powerOfTwo, so that x >= n. 2995 @nogc size_t nextMultipleOf(size_t n, size_t powerOfTwo) pure nothrow 2996 { 2997 // check power-of-two 2998 assert( (powerOfTwo != 0) && ((powerOfTwo & (powerOfTwo - 1)) == 0)); 2999 3000 size_t mask = ~(powerOfTwo - 1); 3001 return (n + powerOfTwo - 1) & mask; 3002 } 3003 3004 void* alignedReallocImpl(bool PreserveDataIfResized)(void* aligned, size_t size, size_t alignment) 3005 { 3006 if (aligned is null) 3007 return _mm_malloc(size, alignment); 3008 3009 assert(alignment != 0); 3010 assert(isPointerAligned(aligned, alignment)); 3011 3012 size_t previousSize = *cast(size_t*)(cast(char*)aligned - size_t.sizeof * 2); 3013 size_t prevAlignment = *cast(size_t*)(cast(char*)aligned - size_t.sizeof * 3); 3014 3015 // It is illegal to change the alignment across calls. 3016 assert(prevAlignment == alignment); 3017 3018 void* raw = *cast(void**)(cast(char*)aligned - size_t.sizeof); 3019 size_t request = requestedSize(size, alignment); 3020 size_t previousRequest = requestedSize(previousSize, alignment); 3021 assert(previousRequest - request == previousSize - size); 3022 3023 // Heuristic: if a requested size is within 50% to 100% of what is already allocated 3024 // then exit with the same pointer 3025 // PERF it seems like `realloc` should do that, not us. 3026 if ( (previousRequest < request * 4) && (request <= previousRequest) ) 3027 return aligned; 3028 3029 void* newRaw = malloc(request); 3030 if (request > 0 && newRaw == null) // realloc(0) can validly return anything 3031 onOutOfMemoryError(); 3032 3033 void* newAligned = storeRawPointerPlusInfo(newRaw, size, alignment); 3034 3035 static if (PreserveDataIfResized) 3036 { 3037 size_t minSize = size < previousSize ? size : previousSize; 3038 memcpy(newAligned, aligned, minSize); 3039 } 3040 3041 // Free previous data 3042 _mm_free(aligned); 3043 assert(isPointerAligned(newAligned, alignment)); 3044 return newAligned; 3045 } 3046 } 3047 3048 unittest 3049 { 3050 assert(nextMultipleOf(0, 4) == 0); 3051 assert(nextMultipleOf(1, 4) == 4); 3052 assert(nextMultipleOf(2, 4) == 4); 3053 assert(nextMultipleOf(3, 4) == 4); 3054 assert(nextMultipleOf(4, 4) == 4); 3055 assert(nextMultipleOf(5, 4) == 8); 3056 3057 { 3058 void* p = _mm_malloc(23, 16); 3059 assert(p !is null); 3060 assert(((cast(size_t)p) & 0xf) == 0); 3061 _mm_free(p); 3062 } 3063 3064 void* nullAlloc = _mm_malloc(0, 32); 3065 assert(nullAlloc != null); 3066 _mm_free(nullAlloc); 3067 } 3068 3069 // For some reason, order of declaration is important for this one 3070 // so it is misplaced. 3071 // Note: is just another name for _mm_cvtss_si32 3072 alias _mm_cvt_ss2si = _mm_cvtss_si32;