[PATCH 2/6] rijndael/ppc: implement single-block mode, and implement OCB block cipher

Jussi Kivilinna jussi.kivilinna at iki.fi
Fri Aug 23 18:52:05 CEST 2019


From: Shawn Landden <shawn at git.icu>

* cipher/rijndael-ppc.c: New implementation of single-block mode, and
implementation of OCB mode.
--

[jk: split rijndael-ppc.c from patch 'rijndael/ppc: reimplement
     single-block mode, and implement OCB block cipher' for basis
     of new PowerPC vector crypto implementation of AES:
     https://lists.gnupg.org/pipermail/gcrypt-devel/2019-July/004765.html]
[jk: coding-style fixes]
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 0 files changed

diff --git a/cipher/rijndael-ppc.c b/cipher/rijndael-ppc.c
new file mode 100644
index 000000000..2e5dd2f89
--- /dev/null
+++ b/cipher/rijndael-ppc.c
@@ -0,0 +1,676 @@
+/* Rijndael (AES) for GnuPG - PowerPC Vector Crypto AES
+ * Copyright (C) 2019 Shawn Landden <shawn at git.icu>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Alternatively, this code may be used in OpenSSL from The OpenSSL Project,
+ * and Cryptogams by Andy Polyakov, and if made part of a release of either
+ * or both projects, is thereafter dual-licensed under the license said project
+ * is released under.
+ */
+
+#include <config.h>
+
+/* PPC AES extensions */
+#include <altivec.h>
+#include "rijndael-internal.h"
+#include "cipher-internal.h"
+
+typedef vector unsigned char block;
+static const vector unsigned char backwards =
+  { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
+
+#ifdef __LITTLE_ENDIAN__
+#define swap_if_le(a) \
+  vec_perm(a, a, backwards)
+#elif __BIG_ENDIAN__
+#define swap_if_le(a) (a)
+#else
+#error "What endianness?"
+#endif
+
+/* Passes in AltiVec registers (big-endian)
+ * sadly compilers don't know how to unroll outer loops into
+ * inner loops with more registers on static functions,
+ * so that this can be properly optimized for OOO multi-issue
+ * without having to hand-unroll.
+ */
+static block _gcry_aes_ppc8_encrypt_altivec (const RIJNDAEL_context *ctx,
+                                             block a)
+{
+  int r;
+  int rounds = ctx->rounds;
+  block *rk = (block*)ctx->keyschenc;
+
+  a = rk[0] ^ a;
+  for (r = 1;r < rounds;r++)
+    {
+      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+        :"+v" (a)
+        :"v" (rk[r])
+      );
+    }
+  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+    :"+v" (a)
+    :"v" (rk[r])
+  );
+  return a;
+}
+
+
+static block _gcry_aes_ppc8_decrypt_altivec (const RIJNDAEL_context *ctx,
+                                             block a)
+{
+  int r;
+  int rounds = ctx->rounds;
+  block *rk = (block*)ctx->keyschdec;
+
+  a = rk[0] ^ a;
+  for (r = 1;r < rounds;r++)
+    {
+      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+        :"+v" (a)
+        :"v" (rk[r])
+      );
+    }
+  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+    :"+v" (a)
+    :"v" (rk[r])
+  );
+  return a;
+}
+
+unsigned int _gcry_aes_ppc8_encrypt (const RIJNDAEL_context *ctx,
+				     unsigned char *b,
+				     const unsigned char *a)
+{
+  uintptr_t zero = 0;
+  block sa;
+
+  if ((uintptr_t)a % 16 == 0)
+    {
+      sa = vec_ld (0, a);
+    }
+  else
+    {
+      block unalignedprev, unalignedcur;
+      unalignedprev = vec_ld (0, a);
+      unalignedcur = vec_ld (16, a);
+      sa = vec_perm (unalignedprev, unalignedcur, vec_lvsl(0, a));
+    }
+
+  sa = swap_if_le(sa);
+  sa = _gcry_aes_ppc8_encrypt_altivec(ctx, sa);
+
+  __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+    :
+    : "wa" (sa), "r" (zero), "r" ((uintptr_t)b));
+
+  return 0; /* does not use stack */
+}
+
+unsigned int _gcry_aes_ppc8_decrypt (const RIJNDAEL_context *ctx,
+				     unsigned char *b,
+				     const unsigned char *a)
+{
+  uintptr_t zero = 0;
+  block sa, unalignedprev, unalignedcur;
+
+  if ((uintptr_t)a % 16 == 0)
+    {
+      sa = vec_ld(0, a);
+    }
+  else
+    {
+      unalignedprev = vec_ld (0, a);
+      unalignedcur = vec_ld (16, a);
+      sa = vec_perm (unalignedprev, unalignedcur, vec_lvsl(0, a));
+    }
+
+  sa = swap_if_le (sa);
+  sa = _gcry_aes_ppc8_decrypt_altivec  (ctx, sa);
+
+  if ((uintptr_t)b % 16 == 0)
+    {
+      vec_vsx_st(swap_if_le(sa), 0, b);
+    }
+  else
+    {
+      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+	:
+	: "wa" (sa), "r" (zero), "r" ((uintptr_t)b));
+    }
+  return 0; /* does not use stack */
+}
+
+size_t _gcry_aes_ppc8_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
+                                            const void *inbuf_arg, size_t nblocks,
+                                            int encrypt)
+{
+  RIJNDAEL_context *ctx = (void *)&c->context.c;
+  unsigned char *outbuf = outbuf_arg;
+  const unsigned char *inbuf = inbuf_arg;
+  block *in = (block*)inbuf;
+  block *out = (block*)outbuf;
+  uintptr_t zero = 0;
+  int r;
+  int rounds = ctx->rounds;
+
+  if (encrypt)
+    {
+      const int unroll = 8;
+      block unalignedprev, ctr, iv;
+
+      if (((uintptr_t)inbuf % 16) != 0)
+	{
+	  unalignedprev = vec_ld(0, in++);
+	}
+
+      iv = vec_ld (0, (block*)&c->u_iv.iv);
+      ctr = vec_ld (0, (block*)&c->u_ctr.ctr);
+
+      for ( ;nblocks >= unroll; nblocks -= unroll)
+	{
+	  u64 i = c->u_mode.ocb.data_nblocks + 1;
+	  block l0, l1, l2, l3, l4, l5, l6, l7;
+	  block b0, b1, b2, b3, b4, b5, b6, b7;
+	  block iv0, iv1, iv2, iv3, iv4, iv5, iv6, iv7;
+	  const block *rk = (block*)&ctx->keyschenc;
+
+	  c->u_mode.ocb.data_nblocks += unroll;
+
+	  iv0 = iv;
+	  if ((uintptr_t)inbuf % 16 == 0)
+	    {
+	      b0 = vec_ld (0, in++);
+	      b1 = vec_ld (0, in++);
+	      b2 = vec_ld (0, in++);
+	      b3 = vec_ld (0, in++);
+	      b4 = vec_ld (0, in++);
+	      b5 = vec_ld (0, in++);
+	      b6 = vec_ld (0, in++);
+	      b7 = vec_ld (0, in++);
+	    }
+	  else
+	    {
+	      block unaligned0, unaligned1, unaligned2,
+		unaligned3, unaligned4, unaligned5, unaligned6;
+	      unaligned0 = vec_ld (0, in++);
+	      unaligned1 = vec_ld (0, in++);
+	      unaligned2 = vec_ld (0, in++);
+	      unaligned3 = vec_ld (0, in++);
+	      unaligned4 = vec_ld (0, in++);
+	      unaligned5 = vec_ld (0, in++);
+	      unaligned6 = vec_ld (0, in++);
+	      b0 = vec_perm (unalignedprev, unaligned0, vec_lvsl (0, inbuf));
+	      unalignedprev = vec_ld (0, in++);
+	      b1 = vec_perm(unaligned0, unaligned1, vec_lvsl (0, inbuf));
+	      b2 = vec_perm(unaligned1, unaligned2, vec_lvsl (0, inbuf));
+	      b3 = vec_perm(unaligned2, unaligned3, vec_lvsl (0, inbuf));
+	      b4 = vec_perm(unaligned3, unaligned4, vec_lvsl (0, inbuf));
+	      b5 = vec_perm(unaligned4, unaligned5, vec_lvsl (0, inbuf));
+	      b6 = vec_perm(unaligned5, unaligned6, vec_lvsl (0, inbuf));
+	      b7 = vec_perm(unaligned6, unalignedprev, vec_lvsl (0, inbuf));
+	    }
+
+	  l0 = *(block*)ocb_get_l (c, i++);
+	  l1 = *(block*)ocb_get_l (c, i++);
+	  l2 = *(block*)ocb_get_l (c, i++);
+	  l3 = *(block*)ocb_get_l (c, i++);
+	  l4 = *(block*)ocb_get_l (c, i++);
+	  l5 = *(block*)ocb_get_l (c, i++);
+	  l6 = *(block*)ocb_get_l (c, i++);
+	  l7 = *(block*)ocb_get_l (c, i++);
+
+	  ctr ^= b0 ^ b1 ^ b2 ^ b3 ^ b4 ^ b5 ^ b6 ^ b7;
+
+	  iv0 ^= l0;
+	  b0 ^= iv0;
+	  iv1 = iv0 ^ l1;
+	  b1 ^= iv1;
+	  iv2 = iv1 ^ l2;
+	  b2 ^= iv2;
+	  iv3 = iv2 ^ l3;
+	  b3 ^= iv3;
+	  iv4 = iv3 ^ l4;
+	  b4 ^= iv4;
+	  iv5 = iv4 ^ l5;
+	  b5 ^= iv5;
+	  iv6 = iv5 ^ l6;
+	  b6 ^= iv6;
+	  iv7 = iv6 ^ l7;
+	  b7 ^= iv7;
+
+	  b0 = swap_if_le (b0);
+	  b1 = swap_if_le (b1);
+	  b2 = swap_if_le (b2);
+	  b3 = swap_if_le (b3);
+	  b4 = swap_if_le (b4);
+	  b5 = swap_if_le (b5);
+	  b6 = swap_if_le (b6);
+	  b7 = swap_if_le (b7);
+
+	  b0 ^= rk[0];
+	  b1 ^= rk[0];
+	  b2 ^= rk[0];
+	  b3 ^= rk[0];
+	  b4 ^= rk[0];
+	  b5 ^= rk[0];
+	  b6 ^= rk[0];
+	  b7 ^= rk[0];
+
+	  for (r = 1;r < rounds;r++)
+	    {
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b0)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b1)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b2)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b3)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b4)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b5)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b6)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vcipher %0, %0, %1\n\t"
+		:"+v" (b7)
+		:"v" (rk[r]));
+	    }
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b0)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b1)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b2)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b3)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b4)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b5)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b6)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vcipherlast %0, %0, %1\n\t"
+	    :"+v" (b7)
+	    :"v" (rk[r]));
+
+	  iv = iv7;
+
+	  /* The unaligned store stxvb16x writes big-endian,
+	     so in the unaligned case we swap the iv instead of the bytes */
+	  if ((uintptr_t)outbuf % 16 == 0)
+	    {
+	      vec_vsx_st (swap_if_le (b0) ^ iv0, 0, out++);
+	      vec_vsx_st (swap_if_le (b1) ^ iv1, 0, out++);
+	      vec_vsx_st (swap_if_le (b2) ^ iv2, 0, out++);
+	      vec_vsx_st (swap_if_le (b3) ^ iv3, 0, out++);
+	      vec_vsx_st (swap_if_le (b4) ^ iv4, 0, out++);
+	      vec_vsx_st (swap_if_le (b5) ^ iv5, 0, out++);
+	      vec_vsx_st (swap_if_le (b6) ^ iv6, 0, out++);
+	      vec_vsx_st (swap_if_le (b7) ^ iv7, 0, out++);
+	    }
+	  else
+	    {
+	      b0 ^= swap_if_le (iv0);
+	      b1 ^= swap_if_le (iv1);
+	      b2 ^= swap_if_le (iv2);
+	      b3 ^= swap_if_le (iv3);
+	      b4 ^= swap_if_le (iv4);
+	      b5 ^= swap_if_le (iv5);
+	      b6 ^= swap_if_le (iv6);
+	      b7 ^= swap_if_le (iv7);
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b0), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b1), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b2), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b3), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b4), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b5), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b6), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b7), "r" (zero), "r" ((uintptr_t)(out++)));
+	    }
+	}
+
+      for ( ;nblocks; nblocks-- )
+	{
+	  block b;
+	  u64 i = ++c->u_mode.ocb.data_nblocks;
+	  const block l = *(block*)ocb_get_l (c, i);
+
+	  /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	  iv ^= l;
+	  if ((uintptr_t)in % 16 == 0)
+	    {
+	      b = vec_ld (0, in++);
+	    }
+	  else
+	    {
+	      block unalignedprevprev;
+	      unalignedprevprev = unalignedprev;
+	      unalignedprev = vec_ld (0, in++);
+	      b = vec_perm (unalignedprevprev, unalignedprev, vec_lvsl (0, inbuf));
+	    }
+
+	  /* Checksum_i = Checksum_{i-1} xor P_i  */
+	  ctr ^= b;
+	  /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
+	  b ^= iv;
+	  b = swap_if_le (b);
+	  b = _gcry_aes_ppc8_encrypt_altivec (ctx, b);
+	  if ((uintptr_t)out % 16 == 0)
+	    {
+	      vec_vsx_st (swap_if_le (b) ^ iv, 0, out++);
+	    }
+	  else
+	    {
+	      b ^= swap_if_le (iv);
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+	        :
+	        : "wa" (b), "r" (zero), "r" ((uintptr_t)out++));
+	    }
+	}
+
+      /* We want to store iv and ctr big-endian and the unaligned
+         store stxvb16x stores them little endian, so we have to swap them. */
+      iv = swap_if_le (iv);
+      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+	:: "wa" (iv), "r" (zero), "r" ((uintptr_t)&c->u_iv.iv));
+      ctr = swap_if_le (ctr);
+      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+	:: "wa" (ctr), "r" (zero), "r" ((uintptr_t)&c->u_ctr.ctr));
+    }
+  else
+    {
+      const int unroll = 8;
+      block unalignedprev, ctr, iv;
+      if (((uintptr_t)inbuf % 16) != 0)
+	{
+	  unalignedprev = vec_ld (0, in++);
+	}
+
+      iv = vec_ld (0, (block*)&c->u_iv.iv);
+      ctr = vec_ld (0, (block*)&c->u_ctr.ctr);
+
+      for ( ;nblocks >= unroll; nblocks -= unroll)
+	{
+	  u64 i = c->u_mode.ocb.data_nblocks + 1;
+	  block l0, l1, l2, l3, l4, l5, l6, l7;
+	  block b0, b1, b2, b3, b4, b5, b6, b7;
+	  block iv0, iv1, iv2, iv3, iv4, iv5, iv6, iv7;
+	  const block *rk = (block*)&ctx->keyschdec;
+
+	  c->u_mode.ocb.data_nblocks += unroll;
+
+	  iv0 = iv;
+	  if ((uintptr_t)inbuf % 16 == 0)
+	    {
+	      b0 = vec_ld (0, in++);
+	      b1 = vec_ld (0, in++);
+	      b2 = vec_ld (0, in++);
+	      b3 = vec_ld (0, in++);
+	      b4 = vec_ld (0, in++);
+	      b5 = vec_ld (0, in++);
+	      b6 = vec_ld (0, in++);
+	      b7 = vec_ld (0, in++);
+	    }
+	  else
+	    {
+	      block unaligned0, unaligned1, unaligned2,
+		unaligned3, unaligned4, unaligned5, unaligned6;
+	      unaligned0 = vec_ld (0, in++);
+	      unaligned1 = vec_ld (0, in++);
+	      unaligned2 = vec_ld (0, in++);
+	      unaligned3 = vec_ld (0, in++);
+	      unaligned4 = vec_ld (0, in++);
+	      unaligned5 = vec_ld (0, in++);
+	      unaligned6 = vec_ld (0, in++);
+	      b0 = vec_perm (unalignedprev, unaligned0, vec_lvsl (0, inbuf));
+	      unalignedprev = vec_ld (0, in++);
+	      b1 = vec_perm (unaligned0, unaligned1, vec_lvsl (0, inbuf));
+	      b2 = vec_perm (unaligned1, unaligned2, vec_lvsl (0, inbuf));
+	      b3 = vec_perm (unaligned2, unaligned3, vec_lvsl (0, inbuf));
+	      b4 = vec_perm (unaligned3, unaligned4, vec_lvsl (0, inbuf));
+	      b5 = vec_perm (unaligned4, unaligned5, vec_lvsl (0, inbuf));
+	      b6 = vec_perm (unaligned5, unaligned6, vec_lvsl (0, inbuf));
+	      b7 = vec_perm (unaligned6, unalignedprev, vec_lvsl (0, inbuf));
+	    }
+
+	  l0 = *(block*)ocb_get_l (c, i++);
+	  l1 = *(block*)ocb_get_l (c, i++);
+	  l2 = *(block*)ocb_get_l (c, i++);
+	  l3 = *(block*)ocb_get_l (c, i++);
+	  l4 = *(block*)ocb_get_l (c, i++);
+	  l5 = *(block*)ocb_get_l (c, i++);
+	  l6 = *(block*)ocb_get_l (c, i++);
+	  l7 = *(block*)ocb_get_l (c, i++);
+
+	  iv0 ^= l0;
+	  b0 ^= iv0;
+	  iv1 = iv0 ^ l1;
+	  b1 ^= iv1;
+	  iv2 = iv1 ^ l2;
+	  b2 ^= iv2;
+	  iv3 = iv2 ^ l3;
+	  b3 ^= iv3;
+	  iv4 = iv3 ^ l4;
+	  b4 ^= iv4;
+	  iv5 = iv4 ^ l5;
+	  b5 ^= iv5;
+	  iv6 = iv5 ^ l6;
+	  b6 ^= iv6;
+	  iv7 = iv6 ^ l7;
+	  b7 ^= iv7;
+
+	  b0 = swap_if_le (b0);
+	  b1 = swap_if_le (b1);
+	  b2 = swap_if_le (b2);
+	  b3 = swap_if_le (b3);
+	  b4 = swap_if_le (b4);
+	  b5 = swap_if_le (b5);
+	  b6 = swap_if_le (b6);
+	  b7 = swap_if_le (b7);
+
+	  b0 ^= rk[0];
+	  b1 ^= rk[0];
+	  b2 ^= rk[0];
+	  b3 ^= rk[0];
+	  b4 ^= rk[0];
+	  b5 ^= rk[0];
+	  b6 ^= rk[0];
+	  b7 ^= rk[0];
+
+	  for (r = 1;r < rounds;r++)
+	    {
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b0)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b1)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b2)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b3)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b4)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b5)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b6)
+		:"v" (rk[r]));
+	      __asm__ volatile ("vncipher %0, %0, %1\n\t"
+		:"+v" (b7)
+		:"v" (rk[r]));
+	    }
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b0)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b1)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b2)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b3)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b4)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b5)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b6)
+	    :"v" (rk[r]));
+	  __asm__ volatile ("vncipherlast %0, %0, %1\n\t"
+	    :"+v" (b7)
+	    :"v" (rk[r]));
+
+	  iv = iv7;
+
+	  b0 = swap_if_le (b0) ^ iv0;
+	  b1 = swap_if_le (b1) ^ iv1;
+	  b2 = swap_if_le (b2) ^ iv2;
+	  b3 = swap_if_le (b3) ^ iv3;
+	  b4 = swap_if_le (b4) ^ iv4;
+	  b5 = swap_if_le (b5) ^ iv5;
+	  b6 = swap_if_le (b6) ^ iv6;
+	  b7 = swap_if_le (b7) ^ iv7;
+
+	  ctr ^= b0 ^ b1 ^ b2 ^ b3 ^ b4 ^ b5 ^ b6 ^ b7;
+
+	  /* The unaligned store stxvb16x writes big-endian */
+	  if ((uintptr_t)outbuf % 16 == 0)
+	    {
+	      vec_vsx_st (b0, 0, out++);
+	      vec_vsx_st (b1, 0, out++);
+	      vec_vsx_st (b2, 0, out++);
+	      vec_vsx_st (b3, 0, out++);
+	      vec_vsx_st (b4, 0, out++);
+	      vec_vsx_st (b5, 0, out++);
+	      vec_vsx_st (b6, 0, out++);
+	      vec_vsx_st (b7, 0, out++);
+	    }
+	  else
+	    {
+	      b0 = swap_if_le (b0);
+	      b1 = swap_if_le (b1);
+	      b2 = swap_if_le (b2);
+	      b3 = swap_if_le (b3);
+	      b4 = swap_if_le (b4);
+	      b5 = swap_if_le (b5);
+	      b6 = swap_if_le (b6);
+	      b7 = swap_if_le (b7);
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b0), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b1), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b2), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b3), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b4), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b5), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b6), "r" (zero), "r" ((uintptr_t)(out++)));
+	      __asm__ ("stxvb16x %x0, %1, %2\n\t"
+		:: "wa" (b7), "r" (zero), "r" ((uintptr_t)(out++)));
+	    }
+	}
+
+      for ( ;nblocks; nblocks-- )
+	{
+	  block b;
+	  u64 i = ++c->u_mode.ocb.data_nblocks;
+	  const block l = *(block*)ocb_get_l (c, i);
+
+	  /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	  iv ^= l;
+	  if ((uintptr_t)in % 16 == 0)
+	    {
+	      b = vec_ld (0, in++);
+	    }
+	  else
+	    {
+	      block unalignedprevprev;
+	      unalignedprevprev = unalignedprev;
+	      unalignedprev = vec_ld (0, in++);
+	      b = vec_perm (unalignedprevprev, unalignedprev, vec_lvsl (0, inbuf));
+	    }
+
+	  /* Checksum_i = Checksum_{i-1} xor P_i  */
+	  /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
+	  b ^= iv;
+	  b = swap_if_le (b);
+	  b = _gcry_aes_ppc8_decrypt_altivec (ctx, b);
+	  b = swap_if_le (b) ^ iv;
+	  ctr ^= b;
+	  if ((uintptr_t)out % 16 == 0)
+	    {
+	      vec_vsx_st (b, 0, out++);
+	    }
+	  else
+	    {
+	      b = swap_if_le (b);
+	      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+		:
+		: "wa" (b), "r" (zero), "r" ((uintptr_t)out++));
+	    }
+	}
+
+      /* We want to store iv and ctr big-endian and the unaligned
+         store stxvb16x stores them little endian, so we have to swap them. */
+      iv = swap_if_le (iv);
+      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+	:: "wa" (iv), "r" (zero), "r" ((uintptr_t)&c->u_iv.iv));
+      ctr = swap_if_le(ctr);
+      __asm__ volatile ("stxvb16x %x0, %1, %2\n\t"
+	:: "wa" (ctr), "r" (zero), "r" ((uintptr_t)&c->u_ctr.ctr));
+    }
+  return 0;
+}
+




More information about the Gcrypt-devel mailing list