| /* sha512-armv7-neon.S - ARM/NEON assembly implementation of SHA-512 transform |
| * |
| * Copyright © 2013-2014 Jussi Kivilinna <jussi.kivilinna@iki.fi> |
| * |
| * This program is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License as published by the Free |
| * Software Foundation; either version 2 of the License, or (at your option) |
| * any later version. |
| */ |
| |
| #include <linux/linkage.h> |
| |
| |
| .syntax unified |
| .code 32 |
| .fpu neon |
| |
| .text |
| |
| /* structure of SHA512_CONTEXT */ |
| #define hd_a 0 |
| #define hd_b ((hd_a) + 8) |
| #define hd_c ((hd_b) + 8) |
| #define hd_d ((hd_c) + 8) |
| #define hd_e ((hd_d) + 8) |
| #define hd_f ((hd_e) + 8) |
| #define hd_g ((hd_f) + 8) |
| |
| /* register macros */ |
| #define RK %r2 |
| |
| #define RA d0 |
| #define RB d1 |
| #define RC d2 |
| #define RD d3 |
| #define RE d4 |
| #define RF d5 |
| #define RG d6 |
| #define RH d7 |
| |
| #define RT0 d8 |
| #define RT1 d9 |
| #define RT2 d10 |
| #define RT3 d11 |
| #define RT4 d12 |
| #define RT5 d13 |
| #define RT6 d14 |
| #define RT7 d15 |
| |
| #define RT01q q4 |
| #define RT23q q5 |
| #define RT45q q6 |
| #define RT67q q7 |
| |
| #define RW0 d16 |
| #define RW1 d17 |
| #define RW2 d18 |
| #define RW3 d19 |
| #define RW4 d20 |
| #define RW5 d21 |
| #define RW6 d22 |
| #define RW7 d23 |
| #define RW8 d24 |
| #define RW9 d25 |
| #define RW10 d26 |
| #define RW11 d27 |
| #define RW12 d28 |
| #define RW13 d29 |
| #define RW14 d30 |
| #define RW15 d31 |
| |
| #define RW01q q8 |
| #define RW23q q9 |
| #define RW45q q10 |
| #define RW67q q11 |
| #define RW89q q12 |
| #define RW1011q q13 |
| #define RW1213q q14 |
| #define RW1415q q15 |
| |
| /*********************************************************************** |
| * ARM assembly implementation of sha512 transform |
| ***********************************************************************/ |
| #define rounds2_0_63(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, rw01q, rw2, \ |
| rw23q, rw1415q, rw9, rw10, interleave_op, arg1) \ |
| /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ |
| vshr.u64 RT2, re, #14; \ |
| vshl.u64 RT3, re, #64 - 14; \ |
| interleave_op(arg1); \ |
| vshr.u64 RT4, re, #18; \ |
| vshl.u64 RT5, re, #64 - 18; \ |
| vld1.64 {RT0}, [RK]!; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, re, #41; \ |
| vshl.u64 RT5, re, #64 - 41; \ |
| vadd.u64 RT0, RT0, rw0; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vmov.64 RT7, re; \ |
| veor.64 RT1, RT2, RT3; \ |
| vbsl.64 RT7, rf, rg; \ |
| \ |
| vadd.u64 RT1, RT1, rh; \ |
| vshr.u64 RT2, ra, #28; \ |
| vshl.u64 RT3, ra, #64 - 28; \ |
| vadd.u64 RT1, RT1, RT0; \ |
| vshr.u64 RT4, ra, #34; \ |
| vshl.u64 RT5, ra, #64 - 34; \ |
| vadd.u64 RT1, RT1, RT7; \ |
| \ |
| /* h = Sum0 (a) + Maj (a, b, c); */ \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, ra, #39; \ |
| vshl.u64 RT5, ra, #64 - 39; \ |
| veor.64 RT0, ra, rb; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vbsl.64 RT0, rc, rb; \ |
| vadd.u64 rd, rd, RT1; /* d+=t1; */ \ |
| veor.64 rh, RT2, RT3; \ |
| \ |
| /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ |
| vshr.u64 RT2, rd, #14; \ |
| vshl.u64 RT3, rd, #64 - 14; \ |
| vadd.u64 rh, rh, RT0; \ |
| vshr.u64 RT4, rd, #18; \ |
| vshl.u64 RT5, rd, #64 - 18; \ |
| vadd.u64 rh, rh, RT1; /* h+=t1; */ \ |
| vld1.64 {RT0}, [RK]!; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, rd, #41; \ |
| vshl.u64 RT5, rd, #64 - 41; \ |
| vadd.u64 RT0, RT0, rw1; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vmov.64 RT7, rd; \ |
| veor.64 RT1, RT2, RT3; \ |
| vbsl.64 RT7, re, rf; \ |
| \ |
| vadd.u64 RT1, RT1, rg; \ |
| vshr.u64 RT2, rh, #28; \ |
| vshl.u64 RT3, rh, #64 - 28; \ |
| vadd.u64 RT1, RT1, RT0; \ |
| vshr.u64 RT4, rh, #34; \ |
| vshl.u64 RT5, rh, #64 - 34; \ |
| vadd.u64 RT1, RT1, RT7; \ |
| \ |
| /* g = Sum0 (h) + Maj (h, a, b); */ \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, rh, #39; \ |
| vshl.u64 RT5, rh, #64 - 39; \ |
| veor.64 RT0, rh, ra; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vbsl.64 RT0, rb, ra; \ |
| vadd.u64 rc, rc, RT1; /* c+=t1; */ \ |
| veor.64 rg, RT2, RT3; \ |
| \ |
| /* w[0] += S1 (w[14]) + w[9] + S0 (w[1]); */ \ |
| /* w[1] += S1 (w[15]) + w[10] + S0 (w[2]); */ \ |
| \ |
| /**** S0(w[1:2]) */ \ |
| \ |
| /* w[0:1] += w[9:10] */ \ |
| /* RT23q = rw1:rw2 */ \ |
| vext.u64 RT23q, rw01q, rw23q, #1; \ |
| vadd.u64 rw0, rw9; \ |
| vadd.u64 rg, rg, RT0; \ |
| vadd.u64 rw1, rw10;\ |
| vadd.u64 rg, rg, RT1; /* g+=t1; */ \ |
| \ |
| vshr.u64 RT45q, RT23q, #1; \ |
| vshl.u64 RT67q, RT23q, #64 - 1; \ |
| vshr.u64 RT01q, RT23q, #8; \ |
| veor.u64 RT45q, RT45q, RT67q; \ |
| vshl.u64 RT67q, RT23q, #64 - 8; \ |
| veor.u64 RT45q, RT45q, RT01q; \ |
| vshr.u64 RT01q, RT23q, #7; \ |
| veor.u64 RT45q, RT45q, RT67q; \ |
| \ |
| /**** S1(w[14:15]) */ \ |
| vshr.u64 RT23q, rw1415q, #6; \ |
| veor.u64 RT01q, RT01q, RT45q; \ |
| vshr.u64 RT45q, rw1415q, #19; \ |
| vshl.u64 RT67q, rw1415q, #64 - 19; \ |
| veor.u64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT45q, rw1415q, #61; \ |
| veor.u64 RT23q, RT23q, RT67q; \ |
| vshl.u64 RT67q, rw1415q, #64 - 61; \ |
| veor.u64 RT23q, RT23q, RT45q; \ |
| vadd.u64 rw01q, RT01q; /* w[0:1] += S(w[1:2]) */ \ |
| veor.u64 RT01q, RT23q, RT67q; |
| #define vadd_RT01q(rw01q) \ |
| /* w[0:1] += S(w[14:15]) */ \ |
| vadd.u64 rw01q, RT01q; |
| |
| #define dummy(_) /*_*/ |
| |
| #define rounds2_64_79(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, \ |
| interleave_op1, arg1, interleave_op2, arg2) \ |
| /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ |
| vshr.u64 RT2, re, #14; \ |
| vshl.u64 RT3, re, #64 - 14; \ |
| interleave_op1(arg1); \ |
| vshr.u64 RT4, re, #18; \ |
| vshl.u64 RT5, re, #64 - 18; \ |
| interleave_op2(arg2); \ |
| vld1.64 {RT0}, [RK]!; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, re, #41; \ |
| vshl.u64 RT5, re, #64 - 41; \ |
| vadd.u64 RT0, RT0, rw0; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vmov.64 RT7, re; \ |
| veor.64 RT1, RT2, RT3; \ |
| vbsl.64 RT7, rf, rg; \ |
| \ |
| vadd.u64 RT1, RT1, rh; \ |
| vshr.u64 RT2, ra, #28; \ |
| vshl.u64 RT3, ra, #64 - 28; \ |
| vadd.u64 RT1, RT1, RT0; \ |
| vshr.u64 RT4, ra, #34; \ |
| vshl.u64 RT5, ra, #64 - 34; \ |
| vadd.u64 RT1, RT1, RT7; \ |
| \ |
| /* h = Sum0 (a) + Maj (a, b, c); */ \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, ra, #39; \ |
| vshl.u64 RT5, ra, #64 - 39; \ |
| veor.64 RT0, ra, rb; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vbsl.64 RT0, rc, rb; \ |
| vadd.u64 rd, rd, RT1; /* d+=t1; */ \ |
| veor.64 rh, RT2, RT3; \ |
| \ |
| /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ |
| vshr.u64 RT2, rd, #14; \ |
| vshl.u64 RT3, rd, #64 - 14; \ |
| vadd.u64 rh, rh, RT0; \ |
| vshr.u64 RT4, rd, #18; \ |
| vshl.u64 RT5, rd, #64 - 18; \ |
| vadd.u64 rh, rh, RT1; /* h+=t1; */ \ |
| vld1.64 {RT0}, [RK]!; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, rd, #41; \ |
| vshl.u64 RT5, rd, #64 - 41; \ |
| vadd.u64 RT0, RT0, rw1; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vmov.64 RT7, rd; \ |
| veor.64 RT1, RT2, RT3; \ |
| vbsl.64 RT7, re, rf; \ |
| \ |
| vadd.u64 RT1, RT1, rg; \ |
| vshr.u64 RT2, rh, #28; \ |
| vshl.u64 RT3, rh, #64 - 28; \ |
| vadd.u64 RT1, RT1, RT0; \ |
| vshr.u64 RT4, rh, #34; \ |
| vshl.u64 RT5, rh, #64 - 34; \ |
| vadd.u64 RT1, RT1, RT7; \ |
| \ |
| /* g = Sum0 (h) + Maj (h, a, b); */ \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vshr.u64 RT4, rh, #39; \ |
| vshl.u64 RT5, rh, #64 - 39; \ |
| veor.64 RT0, rh, ra; \ |
| veor.64 RT23q, RT23q, RT45q; \ |
| vbsl.64 RT0, rb, ra; \ |
| vadd.u64 rc, rc, RT1; /* c+=t1; */ \ |
| veor.64 rg, RT2, RT3; |
| #define vadd_rg_RT0(rg) \ |
| vadd.u64 rg, rg, RT0; |
| #define vadd_rg_RT1(rg) \ |
| vadd.u64 rg, rg, RT1; /* g+=t1; */ |
| |
| .align 3 |
| ENTRY(sha512_transform_neon) |
| /* Input: |
| * %r0: SHA512_CONTEXT |
| * %r1: data |
| * %r2: u64 k[] constants |
| * %r3: nblks |
| */ |
| push {%lr}; |
| |
| mov %lr, #0; |
| |
| /* Load context to d0-d7 */ |
| vld1.64 {RA-RD}, [%r0]!; |
| vld1.64 {RE-RH}, [%r0]; |
| sub %r0, #(4*8); |
| |
| /* Load input to w[16], d16-d31 */ |
| /* NOTE: Assumes that on ARMv7 unaligned accesses are always allowed. */ |
| vld1.64 {RW0-RW3}, [%r1]!; |
| vld1.64 {RW4-RW7}, [%r1]!; |
| vld1.64 {RW8-RW11}, [%r1]!; |
| vld1.64 {RW12-RW15}, [%r1]!; |
| #ifdef __ARMEL__ |
| /* byteswap */ |
| vrev64.8 RW01q, RW01q; |
| vrev64.8 RW23q, RW23q; |
| vrev64.8 RW45q, RW45q; |
| vrev64.8 RW67q, RW67q; |
| vrev64.8 RW89q, RW89q; |
| vrev64.8 RW1011q, RW1011q; |
| vrev64.8 RW1213q, RW1213q; |
| vrev64.8 RW1415q, RW1415q; |
| #endif |
| |
| /* EABI says that d8-d15 must be preserved by callee. */ |
| /*vpush {RT0-RT7};*/ |
| |
| .Loop: |
| rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, |
| RW23q, RW1415q, RW9, RW10, dummy, _); |
| b .Lenter_rounds; |
| |
| .Loop_rounds: |
| rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, |
| RW23q, RW1415q, RW9, RW10, vadd_RT01q, RW1415q); |
| .Lenter_rounds: |
| rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, RW23q, RW4, |
| RW45q, RW01q, RW11, RW12, vadd_RT01q, RW01q); |
| rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, RW45q, RW6, |
| RW67q, RW23q, RW13, RW14, vadd_RT01q, RW23q); |
| rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, RW67q, RW8, |
| RW89q, RW45q, RW15, RW0, vadd_RT01q, RW45q); |
| rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, RW89q, RW10, |
| RW1011q, RW67q, RW1, RW2, vadd_RT01q, RW67q); |
| rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, RW1011q, RW12, |
| RW1213q, RW89q, RW3, RW4, vadd_RT01q, RW89q); |
| add %lr, #16; |
| rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, RW1213q, RW14, |
| RW1415q, RW1011q, RW5, RW6, vadd_RT01q, RW1011q); |
| cmp %lr, #64; |
| rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, RW1415q, RW0, |
| RW01q, RW1213q, RW7, RW8, vadd_RT01q, RW1213q); |
| bne .Loop_rounds; |
| |
| subs %r3, #1; |
| |
| rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, |
| vadd_RT01q, RW1415q, dummy, _); |
| rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, |
| vadd_rg_RT0, RG, vadd_rg_RT1, RG); |
| beq .Lhandle_tail; |
| vld1.64 {RW0-RW3}, [%r1]!; |
| rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, |
| vadd_rg_RT0, RE, vadd_rg_RT1, RE); |
| rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, |
| vadd_rg_RT0, RC, vadd_rg_RT1, RC); |
| #ifdef __ARMEL__ |
| vrev64.8 RW01q, RW01q; |
| vrev64.8 RW23q, RW23q; |
| #endif |
| vld1.64 {RW4-RW7}, [%r1]!; |
| rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, |
| vadd_rg_RT0, RA, vadd_rg_RT1, RA); |
| rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, |
| vadd_rg_RT0, RG, vadd_rg_RT1, RG); |
| #ifdef __ARMEL__ |
| vrev64.8 RW45q, RW45q; |
| vrev64.8 RW67q, RW67q; |
| #endif |
| vld1.64 {RW8-RW11}, [%r1]!; |
| rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, |
| vadd_rg_RT0, RE, vadd_rg_RT1, RE); |
| rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, |
| vadd_rg_RT0, RC, vadd_rg_RT1, RC); |
| #ifdef __ARMEL__ |
| vrev64.8 RW89q, RW89q; |
| vrev64.8 RW1011q, RW1011q; |
| #endif |
| vld1.64 {RW12-RW15}, [%r1]!; |
| vadd_rg_RT0(RA); |
| vadd_rg_RT1(RA); |
| |
| /* Load context */ |
| vld1.64 {RT0-RT3}, [%r0]!; |
| vld1.64 {RT4-RT7}, [%r0]; |
| sub %r0, #(4*8); |
| |
| #ifdef __ARMEL__ |
| vrev64.8 RW1213q, RW1213q; |
| vrev64.8 RW1415q, RW1415q; |
| #endif |
| |
| vadd.u64 RA, RT0; |
| vadd.u64 RB, RT1; |
| vadd.u64 RC, RT2; |
| vadd.u64 RD, RT3; |
| vadd.u64 RE, RT4; |
| vadd.u64 RF, RT5; |
| vadd.u64 RG, RT6; |
| vadd.u64 RH, RT7; |
| |
| /* Store the first half of context */ |
| vst1.64 {RA-RD}, [%r0]!; |
| sub RK, $(8*80); |
| vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ |
| mov %lr, #0; |
| sub %r0, #(4*8); |
| |
| b .Loop; |
| |
| .Lhandle_tail: |
| rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, |
| vadd_rg_RT0, RE, vadd_rg_RT1, RE); |
| rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, |
| vadd_rg_RT0, RC, vadd_rg_RT1, RC); |
| rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, |
| vadd_rg_RT0, RA, vadd_rg_RT1, RA); |
| rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, |
| vadd_rg_RT0, RG, vadd_rg_RT1, RG); |
| rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, |
| vadd_rg_RT0, RE, vadd_rg_RT1, RE); |
| rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, |
| vadd_rg_RT0, RC, vadd_rg_RT1, RC); |
| |
| /* Load context to d16-d23 */ |
| vld1.64 {RW0-RW3}, [%r0]!; |
| vadd_rg_RT0(RA); |
| vld1.64 {RW4-RW7}, [%r0]; |
| vadd_rg_RT1(RA); |
| sub %r0, #(4*8); |
| |
| vadd.u64 RA, RW0; |
| vadd.u64 RB, RW1; |
| vadd.u64 RC, RW2; |
| vadd.u64 RD, RW3; |
| vadd.u64 RE, RW4; |
| vadd.u64 RF, RW5; |
| vadd.u64 RG, RW6; |
| vadd.u64 RH, RW7; |
| |
| /* Store the first half of context */ |
| vst1.64 {RA-RD}, [%r0]!; |
| |
| /* Clear used registers */ |
| /* d16-d31 */ |
| veor.u64 RW01q, RW01q; |
| veor.u64 RW23q, RW23q; |
| veor.u64 RW45q, RW45q; |
| veor.u64 RW67q, RW67q; |
| vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ |
| veor.u64 RW89q, RW89q; |
| veor.u64 RW1011q, RW1011q; |
| veor.u64 RW1213q, RW1213q; |
| veor.u64 RW1415q, RW1415q; |
| /* d8-d15 */ |
| /*vpop {RT0-RT7};*/ |
| /* d0-d7 (q0-q3) */ |
| veor.u64 %q0, %q0; |
| veor.u64 %q1, %q1; |
| veor.u64 %q2, %q2; |
| veor.u64 %q3, %q3; |
| |
| pop {%pc}; |
| ENDPROC(sha512_transform_neon) |
