VGF2P8MULB Y2, Y7, K1, Y31                         // 62624529cffa
	VGF2P8MULB Y21, Y7, K1, Y31                        // 62224529cffd
	VGF2P8MULB Y12, Y7, K1, Y31                        // 62424529cffc
	VGF2P8MULB (R8), Y7, K1, Y31                       // 62424529cf38
	VGF2P8MULB 15(DX)(BX*2), Y7, K1, Y31               // 62624529cfbc5a0f000000

	VPMADD52HUQ 15(DX)(BX*2), Y31, K7, Y7              // 62f28527b5bc5a0f000000
	VPMADD52HUQ Y28, Y8, K7, Y7                        // 6292bd2fb5fc
	VPMADD52HUQ Y13, Y8, K7, Y7                        // 62d2bd2fb5fd
	VPMADD52HUQ Y7, Y8, K7, Y7                         // 62f2bd2fb5ff
	VPMADD52HUQ (R8), Y8, K7, Y7                       // 62d2bd2fb538

	VPSHRDD $97, Y6, Y1, K3, Y7                        // 62f3752b73fe61
	VPSHRDD $97, 15(R8)(R14*1), Y1, K3, Y7             // 6293752b73bc300f00000061
	VPSHRDD $97, 15(R8)(R14*2), Y1, K3, Y7             // 6293752b73bc700f00000061
	VPSHRDD $97, Y31, Y30, K3, Y7                      // 62930d2373ff61
	VPSHRDD $97, Y22, Y30, K3, Y7                      // 62b30d2373fe61

	VPERMT2B Y14, Y7, K6, Y18                          // 62c2452e7dd6
	VPERMT2B Y8, Y7, K6, Y18                           // 62c2452e7dd0
	VPERMT2B Y20, Y7, K6, Y18                          // 62a2452e7dd4
	VPERMT2B 7(SI)(DI*4), Y7, K6, Y18                  // 62e2452e7d94be07000000
	VPERMT2B -7(DI)(R8*2), Y7, K6, Y18                 // 62a2452e7d9447f9ffffff

		y1 := s[8*4] << 8
		y2 := s[8*6]
		y3 := s[8*2]
		y4 := s[8*1]
		y5 := s[8*7]
		y6 := s[8*5]
		y7 := s[8*3]

		// Stage 1.
		y8 := w7*(y4+y5) + 4
		y4 = (y8 + w1mw7*y4) >> 3
		y5 = (y8 - w1pw7*y5) >> 3
		y8 = w3*(y6+y7) + 4
		y6 = (y8 - w3mw5*y6) >> 3
		y7 = (y8 - w3pw5*y7) >> 3

		// Stage 2.
		y8 = y0 + y1
		y0 -= y1
		y1 = w6*(y3+y2) + 4
		y2 = (y1 - w2pw6*y2) >> 3

	VPOPCNTW 7(SI)(DI*2), K3, Y21                      // 62e2fd2b54ac7e07000000
	VPOPCNTW Y13, K3, Y7                               // 62d2fd2b54fd
	VPOPCNTW Y18, K3, Y7                               // 62b2fd2b54fa
	VPOPCNTW Y24, K3, Y7                               // 6292fd2b54f8
	VPOPCNTW (SI), K3, Y7                              // 62f2fd2b543e
	VPOPCNTW 7(SI)(DI*2), K3, Y7                       // 62f2fd2b54bc7e07000000

	VANDPS Y6, Y1, K2, Y7                              // 62f1742a54fe
	VANDPS 15(R8)(R14*4), Y1, K2, Y7                   // 6291742a54bcb00f000000
	VANDPS -7(CX)(DX*4), Y1, K2, Y7                    // 62f1742a54bc91f9ffffff
	VANDPS Y31, Y30, K2, Y7                            // 62910c2254ff
	VANDPS Y22, Y30, K2, Y7                            // 62b10c2254fe

	VPCONFLICTD Y14, K3, Y7                            // 62d27d2bc4fe
	VPCONFLICTD Y8, K3, Y7                             // 62d27d2bc4f8
	VPCONFLICTD Y20, K3, Y7                            // 62b27d2bc4fc
	VPCONFLICTD -7(CX), K3, Y7                         // 62f27d2bc4b9f9ffffff
	VPCONFLICTD 15(DX)(BX*4), K3, Y7                   // 62f27d2bc4bc9a0f000000

	PRECALC_00_15(0x20,Y13) \
	PRECALC_00_15(0x30,Y12) \
	PRECALC_16_31(Y8,Y12,Y13,Y14,Y15,0,0x80) \
	PRECALC_16_31(Y7,Y8,Y12,Y13,Y14,0x20,0xa0) \
	PRECALC_16_31(Y5,Y7,Y8,Y12,Y13,0x20,0xc0) \
	PRECALC_16_31(Y3,Y5,Y7,Y8,Y12,0x20,0xe0) \
	PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x20,0x100) \
	PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x20,0x120) \
	PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x40,0x140) \
	PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x40,0x160) \

	ADDQ R14, BX
	ADDQ R15, R9
	ADDQ R15, BX

	ADDQ DI, BX

	VPSRLQ $19, Y7, Y3
	VPSLLQ $(64-19), Y7, Y1
	VPOR   Y1, Y3, Y3
	VPXOR  Y3, Y8, Y8
	VPSRLQ $61, Y7, Y3
	VPSLLQ $(64-61), Y7, Y1
	VPOR   Y1, Y3, Y3
	VPXOR  Y3, Y8, Y8

	VPADDQ Y8, Y0, Y2

	VPBLENDD $0xF0, Y2, Y7, Y7

	MOVQ  BX, DI
	RORXQ $41, R9, R13
	RORXQ $18, R9, R14
	ADDQ  3*8+frame_YFER(SP), AX