;---------------------
;
;   This code implements two interfaces of SHA-1 update function: 1) working on a single 
;   64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks 
;   version of code is software pipelined and faster overall, it is a default. Assemble
;   with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
;
;   C++ prototypes of implemented functions are below:
;
;   #ifndef INTEL_SHA1_SINGLEBLOCK
;      // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consecutive 64-byte blocks
;      extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
;   #else
;      // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
;      extern "C" void sha1_update_intel(int *hash, const char* input);
;   #endif
;
;   Function name 'sha1_update_intel' can be changed in the source or via macro:
;     -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
;
;   It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
;
;   Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
;   and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs 
;   is also required, the default (e.g. one being replaced) function can be provided for 
;   dispatch on such CPUs, the name of old function can be changed in the source or via macro:
;      -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name 
; 
;   Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
;

%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
;; can be replaced with a default SHA-1 update function name
%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH  sha1_intel_non_ssse3_cpu_stub_  
%else 
extern  INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
%endif

;; provide alternative SHA-1 update function's name here
%ifndef INTEL_SHA1_UPDATE_FUNCNAME
%define INTEL_SHA1_UPDATE_FUNCNAME     sha1_update_intel
%endif

global INTEL_SHA1_UPDATE_FUNCNAME


%ifndef INTEL_SHA1_SINGLEBLOCK
%assign multiblock 1
%else
%assign multiblock 0
%endif


bits 64
default rel

%ifdef WIN_ABI
%xdefine arg1 rcx
%xdefine arg2 rdx
%xdefine arg3 r8
%else
%xdefine arg1 rdi
%xdefine arg2 rsi
%xdefine arg3 rdx
%endif

%xdefine ctx arg1
%xdefine buf arg2
%xdefine cnt arg3

%macro REGALLOC 0
%xdefine A ecx
%xdefine B esi
%xdefine C edi
%xdefine D ebp
%xdefine E edx

%xdefine T1 eax
%xdefine T2 ebx
%endmacro

%xdefine K_BASE     r8
%xdefine HASH_PTR   r9
%xdefine BUFFER_PTR r10
%xdefine BUFFER_END r11

%xdefine W_TMP  xmm0
%xdefine W_TMP2 xmm9

%xdefine W0  xmm1
%xdefine W4  xmm2
%xdefine W8  xmm3
%xdefine W12 xmm4
%xdefine W16 xmm5
%xdefine W20 xmm6
%xdefine W24 xmm7
%xdefine W28 xmm8

%xdefine XMM_SHUFB_BSWAP xmm10

;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
%xdefine WK(t) (rsp + (t & 15)*4)

;------------------------------------------------------------------------------
;
; macro implements SHA-1 function's body for single or several 64-byte blocks
; first param: function's name
; second param: =0 - function implements single 64-byte block hash
;               =1 - function implements multiple64-byte blocks hash
;                    3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
;
%macro  SHA1_VECTOR_ASM  2
align 4096
%1:
push rbx
push rbp

%ifdef WIN_ABI
push rdi
push rsi

%xdefine stack_size (16*4 + 16*5 + 8)
%else
%xdefine stack_size (16*4 + 8)
%endif    

sub     rsp, stack_size

%ifdef WIN_ABI
%xdefine xmm_save_base (rsp + 16*4)

xmm_mov [xmm_save_base + 0*16], xmm6
xmm_mov [xmm_save_base + 1*16], xmm7
xmm_mov [xmm_save_base + 2*16], xmm8
xmm_mov [xmm_save_base + 3*16], xmm9
xmm_mov [xmm_save_base + 4*16], xmm10
%endif

mov     HASH_PTR, ctx
mov     BUFFER_PTR, buf

%if (%2 == 1)
shl     cnt, 6           ;; mul by 64
add     cnt, buf
mov     BUFFER_END, cnt
%endif

lea     K_BASE, [K_XMM_AR]
xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]

SHA1_PIPELINED_MAIN_BODY %2

%ifdef WIN_ABI
xmm_mov xmm6, [xmm_save_base + 0*16]
xmm_mov xmm7, [xmm_save_base + 1*16]
xmm_mov xmm8, [xmm_save_base + 2*16]
xmm_mov xmm9, [xmm_save_base + 3*16]
xmm_mov xmm10,[xmm_save_base + 4*16]
%endif

add rsp, stack_size

%ifdef WIN_ABI
pop rsi
pop rdi
%endif

pop rbp
pop rbx

ret
%endmacro

;--------------------------------------------
; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
; macro param: =0 - process single 64-byte block
;              =1 - multiple blocks
;
%macro SHA1_PIPELINED_MAIN_BODY 1

REGALLOC

mov A, [HASH_PTR   ]
mov B, [HASH_PTR+ 4]
mov C, [HASH_PTR+ 8]
mov D, [HASH_PTR+12]

mov E, [HASH_PTR+16]

%assign i 0
%rep    W_PRECALC_AHEAD
W_PRECALC i
%assign i i+1
%endrep

%xdefine F F1

%if (%1 == 1)                         ;; code loops through more than one block
%%_loop:
cmp BUFFER_PTR, K_BASE          ;; we use K_BASE value as a signal of a last block,
jne %%_begin                    ;; it is set below by: cmovae BUFFER_PTR, K_BASE
jmp %%_end

align 32
%%_begin:
%endif
RR A,B,C,D,E,0
RR D,E,A,B,C,2
RR B,C,D,E,A,4
RR E,A,B,C,D,6
RR C,D,E,A,B,8

RR A,B,C,D,E,10
RR D,E,A,B,C,12
RR B,C,D,E,A,14
RR E,A,B,C,D,16
RR C,D,E,A,B,18

%xdefine F F2

RR A,B,C,D,E,20 
RR D,E,A,B,C,22 
RR B,C,D,E,A,24 
RR E,A,B,C,D,26 
RR C,D,E,A,B,28 

RR A,B,C,D,E,30 
RR D,E,A,B,C,32 
RR B,C,D,E,A,34 
RR E,A,B,C,D,36 
RR C,D,E,A,B,38 

%xdefine F F3

RR A,B,C,D,E,40 
RR D,E,A,B,C,42 
RR B,C,D,E,A,44 
RR E,A,B,C,D,46 
RR C,D,E,A,B,48 

RR A,B,C,D,E,50 
RR D,E,A,B,C,52 
RR B,C,D,E,A,54 
RR E,A,B,C,D,56 
RR C,D,E,A,B,58 

%xdefine F F4

%if (%1 == 1)                         ;; if code loops through more than one block
add   BUFFER_PTR, 64            ;; move to next 64-byte block
cmp   BUFFER_PTR, BUFFER_END    ;; check if current block is the last one
cmovae BUFFER_PTR, K_BASE       ;; smart way to signal the last iteration
%else
%xdefine W_NO_TAIL_PRECALC 1    ;; no software pipelining for single block interface
%endif

RR A,B,C,D,E,60
RR D,E,A,B,C,62
RR B,C,D,E,A,64
RR E,A,B,C,D,66
RR C,D,E,A,B,68

RR A,B,C,D,E,70
RR D,E,A,B,C,72
RR B,C,D,E,A,74
RR E,A,B,C,D,76
RR C,D,E,A,B,78

UPDATE_HASH [HASH_PTR   ],A
UPDATE_HASH [HASH_PTR+ 4],B
UPDATE_HASH [HASH_PTR+ 8],C
UPDATE_HASH [HASH_PTR+12],D
UPDATE_HASH [HASH_PTR+16],E

%if (%1 == 1)
jmp %%_loop

align 32  
%%_end:
%endif


%xdefine W_NO_TAIL_PRECALC 0
%xdefine F %error

%endmacro


%macro F1 3
mov T1,%2
xor T1,%3
and T1,%1
xor T1,%3
%endmacro

%macro F2 3
mov T1,%3
xor T1,%2
xor T1,%1
%endmacro

%macro F3 3
mov T1,%2
mov T2,%1
or  T1,%1
and T2,%2
and T1,%3
or  T1,T2
%endmacro

%define F4 F2

%macro UPDATE_HASH 2
add %2, %1
mov %1, %2
%endmacro 


%macro W_PRECALC 1
%xdefine i (%1)

%if (i < 20)
%xdefine K_XMM  0
%elif (i < 40)
%xdefine K_XMM  16
%elif (i < 60)
%xdefine K_XMM  32
%else
%xdefine K_XMM  48
%endif

%if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))

%if (W_NO_TAIL_PRECALC == 0)

%xdefine i ((%1) % 80)        ;; pre-compute for the next iteration

%if (i == 0)
W_PRECALC_RESET
%endif


W_PRECALC_00_15
%endif

%elif (i < 32)
W_PRECALC_16_31
%elif (i < 80)   ;; rounds 32-79
W_PRECALC_32_79
%endif
%endmacro

%macro W_PRECALC_RESET 0
%xdefine    W             W0
%xdefine    W_minus_04    W4
%xdefine    W_minus_08    W8
%xdefine    W_minus_12    W12
%xdefine    W_minus_16    W16
%xdefine    W_minus_20    W20
%xdefine    W_minus_24    W24
%xdefine    W_minus_28    W28
%xdefine    W_minus_32    W
%endmacro

%macro W_PRECALC_ROTATE 0
%xdefine    W_minus_32    W_minus_28
%xdefine    W_minus_28    W_minus_24
%xdefine    W_minus_24    W_minus_20
%xdefine    W_minus_20    W_minus_16
%xdefine    W_minus_16    W_minus_12
%xdefine    W_minus_12    W_minus_08
%xdefine    W_minus_08    W_minus_04
%xdefine    W_minus_04    W
%xdefine    W             W_minus_32
%endmacro

%xdefine W_PRECALC_AHEAD   16
%xdefine W_NO_TAIL_PRECALC 0


%xdefine xmm_mov            movdqa

%macro W_PRECALC_00_15 0
;; message scheduling pre-compute for rounds 0-15
%if ((i & 3) == 0)       ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
movdqu W_TMP, [BUFFER_PTR + (i * 4)]
%elif ((i & 3) == 1)
pshufb W_TMP, XMM_SHUFB_BSWAP
movdqa W, W_TMP
%elif ((i & 3) == 2)
paddd  W_TMP, [K_BASE]
%elif ((i & 3) == 3)
movdqa  [WK(i&~3)], W_TMP

W_PRECALC_ROTATE
%endif
%endmacro

%macro W_PRECALC_16_31 0
;; message scheduling pre-compute for rounds 16-31
;; calculating last 32 w[i] values in 8 XMM registers
;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
;;
;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
;;
%if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
movdqa  W, W_minus_12
palignr W, W_minus_16, 8       ;; w[i-14]
movdqa  W_TMP, W_minus_04
psrldq  W_TMP, 4               ;; w[i-3]
pxor    W, W_minus_08
%elif ((i & 3) == 1)
pxor    W_TMP, W_minus_16
pxor    W, W_TMP
movdqa  W_TMP2, W
movdqa  W_TMP, W
pslldq  W_TMP2, 12
%elif ((i & 3) == 2)
psrld   W, 31
pslld   W_TMP, 1
por     W_TMP, W
movdqa  W, W_TMP2
psrld   W_TMP2, 30
pslld   W, 2
%elif ((i & 3) == 3)
pxor    W_TMP, W
pxor    W_TMP, W_TMP2
movdqa  W, W_TMP
paddd   W_TMP, [K_BASE + K_XMM]
movdqa  [WK(i&~3)],W_TMP

W_PRECALC_ROTATE
%endif
%endmacro

%macro W_PRECALC_32_79 0
;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
;; instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
;;
%if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
movdqa  W_TMP, W_minus_04
pxor    W, W_minus_28         ;; W is W_minus_32 before xor
palignr W_TMP, W_minus_08, 8
%elif ((i & 3) == 1)
pxor    W, W_minus_16
pxor    W, W_TMP
movdqa  W_TMP, W
%elif ((i & 3) == 2)
psrld   W, 30
pslld   W_TMP, 2
por     W_TMP, W
%elif ((i & 3) == 3)
movdqa  W, W_TMP
paddd   W_TMP, [K_BASE + K_XMM]
movdqa  [WK(i&~3)],W_TMP

W_PRECALC_ROTATE
%endif
%endmacro 

%macro RR 6             ;; RR does two rounds of SHA-1 back to back with W pre-calculation

;;     TEMP = A
;;     A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
;;     C = ROTATE_LEFT( B, 30 )
;;     D = C
;;     E = D
;;     B = TEMP

W_PRECALC (%6 + W_PRECALC_AHEAD)
F    %2, %3, %4     ;; F returns result in T1
add  %5, [WK(%6)]
rol  %2, 30
mov  T2, %1
add  %4, [WK(%6 + 1)]
rol  T2, 5
add  %5, T1

W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
add  T2, %5
mov  %5, T2
rol  T2, 5
add  %4, T2
F    %1, %2, %3    ;; F returns result in T1
add  %4, T1
rol  %1, 30

;; write:  %1, %2
;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
%endmacro



;;----------------------
section .data
align 128

%xdefine K1 0x5a827999
%xdefine K2 0x6ed9eba1
%xdefine K3 0x8f1bbcdc
%xdefine K4 0xca62c1d6

align 128
K_XMM_AR:
DD K1, K1, K1, K1
DD K2, K2, K2, K2
DD K3, K3, K3, K3
DD K4, K4, K4, K4

align 16
bswap_shufb_ctl:
DD 00010203h
DD 04050607h
DD 08090a0bh
DD 0c0d0e0fh

;; dispatch pointer, points to the init routine for the first invocation
sha1_update_intel_dispatched:
DQ  sha1_update_intel_init_

;;----------------------
section .text
align 4096

SHA1_VECTOR_ASM     sha1_update_intel_ssse3_, multiblock

align 32
sha1_update_intel_init_:       ;; we get here with the first time invocation
call    sha1_update_intel_dispacth_init_
INTEL_SHA1_UPDATE_FUNCNAME:    ;; we get here after init
jmp     qword [sha1_update_intel_dispatched]

;; CPUID feature flag based dispatch
sha1_update_intel_dispacth_init_:
push    rax
push    rbx
push    rcx
push    rdx
push    rsi

lea     rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]

mov     eax, 1
cpuid

test    ecx, 0200h          ;; SSSE3 support, CPUID.1.ECX[bit 9]
jz      _done

lea     rsi, [sha1_update_intel_ssse3_]

_done:
mov     [sha1_update_intel_dispatched], rsi

pop     rsi
pop     rdx
pop     rcx
pop     rbx
pop     rax
ret

;;----------------------
;; in the case a default SHA-1 update function implementation was not provided
;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this 
;; failure in a safest way - jumps to the stub function with UD2 instruction below
sha1_intel_non_ssse3_cpu_stub_:
ud2     ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
ret

; END
;----------------------
