sha1 hash using libgcrypt different from what returns sha1sum

[Werner Koch]

On Wed, 13 Nov 2013 22:05, dkg at fifthhorseman.net said:
> On 11/13/2013 02:00 PM, Werner Koch wrote:
>> On Wed, 13 Nov 2013 15:57, dkg at fifthhorseman.net said:

> yep, agreed, that would be pretty obnoxious for a regression suite.
> maybe we should consider a separate "extended regression suite" annex
> for people with CPU to burn?  I'm not sure how else to really test this
> sort of codepath without testing it.

Could be done with a configure option.  Actually we already have such an
option for the PKITS tests.

> Maybe we could save and store intermediate digest state in git and make
> the test suite load that intermediate state and restart the digest from
> most-of-the-way-through?  that kind of seems like cheating though.

Right, that is not a real test.  In particular because there is no API
for intermediate values.

> hm, I was just offering reasonable and clearly-understood test vectors
> that are easily available.  I'm not sure non-compressability is a
> characteristic we need care about for a test vectors to avoid a

Weel, we could also use -z 0 to disable compression.  Given that it is
an algorithm error, we may not need to test the entire gpg output but
just the plain hashing (ie. --print-md).

Meanwhile I started with a tests program for Libgcrypt and now I only
need to wait for test vectors.  I am currently using the program below
to generate data and run sha1sum on it:

 ./genhashdata --gigs 256 --bytes -64 | sha1sum
 ./genhashdata --gigs 256 --bytes -1  | sha1sum
 ./genhashdata --gigs 256 --bytes 0   | sha1sum
 ./genhashdata --gigs 256 --bytes 1   | sha1sum

Libgcrypt's new hash test program outputs 4 values by taking copies of
the hash context and thus not requiring 4 indivudal runs.  Example:

$ ./hashtest --gigs 1 --verbose sha1
hashtest: 1 GiB hashed
hashtest: 1 GiB -64 SHA1       dd636d1d217b368e9cdf02f001580aa7e1e69324
hashtest: 1 GiB -1  SHA1       108e2e62b787deb94d64a7e4c4ec32f6ecb8f876
hashtest: 1 GiB +0  SHA1       ecebf8a78d57368378471ce3d7046702ed865e92
hashtest: 1 GiB +1  SHA1       48544b31ab4b4963f219a8c821081176ba7d1269


Should be done for all algorithms, though.  I guess some help running
shaXXXX will be needed.


Salam-Shalom,

   Werner


/* genhashdata.c - Create data for hash tests
 * Copyright (C) 2013 g10 Code GmbH
 *
 * This file is part of Libgcrypt.
 *
 * Libgcrypt is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * Libgcrypt is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

#define PGM "genhashdata"

static void
die (const char *format, ...)
{
  va_list arg_ptr ;

  fflush (stdout);
  fprintf (stderr, "%s: ", PGM);
  va_start (arg_ptr, format ) ;
  vfprintf (stderr, format, arg_ptr );
  va_end(arg_ptr);
  if (*format && format[strlen(format)-1] != '\n')
    putc ('\n', stderr);
  exit (1);
}

int
main (int argc, char **argv)
{
  int last_argc = -1;
  int gigs = 0;
  int bytes = 0;
  char pattern[1024];
  int i, g;

  if (argc)
    { argc--; argv++; }

  while (argc && last_argc != argc )
    {
      last_argc = argc;
      if (!strcmp (*argv, "--"))
        {
          argc--; argv++;
          break;
        }
      else if (!strcmp (*argv, "--help"))
        {
          fputs ("usage: " PGM " [options]\n"
                 "Options:\n"
                 "  --gigs  N     Emit N GiB of test bytes\n"
                 "  --bytes DIFF  Stop DIFF bytes earlier or later\n",
                 stdout);
          exit (0);
        }
      else if (!strcmp (*argv, "--gigs"))
        {
          argc--; argv++;
          if (argc)
            {
              gigs = atoi (*argv);
              argc--; argv++;
            }
        }
      else if (!strcmp (*argv, "--bytes"))
        {
          argc--; argv++;
          if (argc)
            {
              bytes = atoi (*argv);
              argc--; argv++;
            }
        }
      else if (!strncmp (*argv, "--", 2))
        die ("unknown option '%s'", *argv);
    }

  if (gigs < 0 || gigs > 1024*1024)
    die ("value for --gigs must be in the range 0 to %d", 1024*1024);
  if (bytes < -1024 || bytes > 1024)
      die ("value for --bytes must be in the range -1024 to 1024");
  if (sizeof pattern != 1024)
    die ("internal error");

  if (argc > 1)
    die ("arguments are not expected");

  memset (pattern, 'a', sizeof pattern);

  for (g=0; g < gigs; g++)
    {
      if (g + 1 == gigs && bytes < 0)
        {
          for (i = 0; i < 1024*1023; i++)
            if (fwrite (pattern, sizeof pattern, 1, stdout) != 1)
              die ("writing to stdout failed: %s", strerror (errno));
          for (i = 0; i < 1023; i++)
            if (fwrite (pattern, sizeof pattern, 1, stdout) != 1)
              die ("writing to stdout failed: %s", strerror (errno));
          if (fwrite (pattern, sizeof pattern + bytes, 1, stdout) != 1)
            die ("writing to stdout failed: %s", strerror (errno));
        }
      else
        {
          for (i = 0; i < 1024*1024; i++)
            if (fwrite (pattern, sizeof pattern, 1, stdout) != 1)
              die ("writing to stdout failed: %s", strerror (errno));
        }
    }
  if (bytes > 0)
    if (fwrite (pattern, bytes, 1, stdout) != 1)
      die ("writing to stdout failed: %s", strerror (errno));
  if (fflush (stdout))
    die ("writing to stdout failed: %s", strerror (errno));

  return 0;
}

-- 
Die Gedanken sind frei.  Ausnahmen regelt ein Bundesgesetz.