[git] GnuPG - branch, master, updated. gnupg-2.2.3-128-gdb7661b

by Werner Koch cvs at cvs.gnupg.org
Wed Jan 24 18:45:53 CET 2018


This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "The GNU Privacy Guard".

The branch, master has been updated
       via  db7661b5a297a58c95fa9873d43f31d697b8feb3 (commit)
       via  bfc11816444512b4ebcc6617d3c3b5988e753de3 (commit)
      from  ff1bdc23d9f1693c1add7c1fe8d218b7bf743e31 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit db7661b5a297a58c95fa9873d43f31d697b8feb3
Author: Werner Koch <wk at gnupg.org>
Date:   Wed Jan 24 18:29:08 2018 +0100

    gpg: New maintainer option --debug-set-iobuf-size.
    
    * g10/gpg.c (opts): Add new option.
    (opt_set_iobuf_size): New var.
    (set_debug): Set the option.
    * tests/openpgp/armor.scm: Use this option to revert the buffer size
    to the one which used to exhibit the tested bugs.
    
    Signed-off-by: Werner Koch <wk at gnupg.org>

diff --git a/common/iobuf.c b/common/iobuf.c
index bf59778..02c9b49 100644
--- a/common/iobuf.c
+++ b/common/iobuf.c
@@ -59,9 +59,7 @@
 
 /*-- Begin configurable part.  --*/
 
-/* The size of the internal buffers.
-   NOTE: If you change this value you MUST also adjust the regression
-   test "armored_key_8192" in armor.test! */
+/* The standard size of the internal buffers.  */
 #define DEFAULT_IOBUF_BUFFER_SIZE  (64*1024)
 
 /* To avoid a potential DoS with compression packets we better limit
diff --git a/doc/gpg.texi b/doc/gpg.texi
index 1e6ea59..83b234e 100644
--- a/doc/gpg.texi
+++ b/doc/gpg.texi
@@ -2859,6 +2859,12 @@ Set all useful debugging flags.
 Set stdout into line buffered mode.  This option is only honored when
 given on the command line.
 
+ at item --debug-set-iobuf-size @var{n}
+ at opindex debug-iolbf
+Change the buffer size of the IOBUFs to @var{n} kilobyte.  Using 0
+prints the current size.  Note well: This is a maintainer only option
+and may thus be changed or removed at any time without notice.
+
 @item --faked-system-time @var{epoch}
 @opindex faked-system-time
 This option is only useful for testing; it sets the system time back or
diff --git a/g10/gpg.c b/g10/gpg.c
index 283de20..cbac967 100644
--- a/g10/gpg.c
+++ b/g10/gpg.c
@@ -223,6 +223,7 @@ enum cmd_and_opt_values
     oDebugLevel,
     oDebugAll,
     oDebugIOLBF,
+    oDebugSetIobufSize,
     oStatusFD,
     oStatusFile,
     oAttributeFD,
@@ -642,6 +643,7 @@ static ARGPARSE_OPTS opts[] = {
   ARGPARSE_s_s (oDebugLevel, "debug-level", "@"),
   ARGPARSE_s_n (oDebugAll, "debug-all", "@"),
   ARGPARSE_s_n (oDebugIOLBF, "debug-iolbf", "@"),
+  ARGPARSE_s_u (oDebugSetIobufSize, "debug-set-iobuf-size", "@"),
   ARGPARSE_s_i (oStatusFD, "status-fd", "@"),
   ARGPARSE_s_s (oStatusFile, "status-file", "@"),
   ARGPARSE_s_i (oAttributeFD, "attribute-fd", "@"),
@@ -956,6 +958,8 @@ int g10_errors_seen = 0;
 
 static int utf8_strings = 0;
 static int maybe_setuid = 1;
+static unsigned int opt_set_iobuf_size;
+static unsigned int opt_set_iobuf_size_used;
 
 static char *build_list( const char *text, char letter,
 			 const char *(*mapf)(int), int (*chkf)(int) );
@@ -1276,6 +1280,10 @@ set_debug (const char *level)
 
   if (opt.debug)
     parse_debug_flag (NULL, &opt.debug, debug_flags);
+
+  if (opt_set_iobuf_size || opt_set_iobuf_size_used)
+    log_debug ("iobuf buffer size is %uk\n",
+               iobuf_set_buffer_size (opt_set_iobuf_size));
 }
 
 
@@ -2742,6 +2750,11 @@ main (int argc, char **argv)
 
           case oDebugIOLBF: break; /* Already set in pre-parse step.  */
 
+          case oDebugSetIobufSize:
+            opt_set_iobuf_size = pargs.r.ret_ulong;
+            opt_set_iobuf_size_used = 1;
+            break;
+
 	  case oStatusFD:
             set_status_fd ( translate_sys2libc_fd_int (pargs.r.ret_int, 1) );
             break;
diff --git a/tests/openpgp/armor.scm b/tests/openpgp/armor.scm
index 3c117dd..a1b0aa9 100755
--- a/tests/openpgp/armor.scm
+++ b/tests/openpgp/armor.scm
@@ -191,7 +191,7 @@ nW1ff9rt1YcTH9LiiE4EGBECAAYFAjnKLe0AEgkQ3uyMCd5BWw4HZUdQRwABAZeBAKDsa7tc
 (info "Checking armored_key_8192")
 (pipe:do
  (pipe:echo armored_key_8192)
- (pipe:gpg '(--import)))
+ (pipe:gpg '(--debug-set-iobuf-size=8 --import)))
 
 (define nopad_armored_msg "-----BEGIN PGP MESSAGE-----
 Version: GnuPG v1.4.11-svn5139 (GNU/Linux)
@@ -758,10 +758,10 @@ wg7Md81a5RI3F2FG8747t9gX
 (info "Importing alpha_seckey")
 (pipe:do
  (pipe:echo alpha_seckey)
- (pipe:gpg '(--import)))
+ (pipe:gpg '(--debug-set-iobuf-size=8 --import)))
 
 (info "Checking for bug #1179")
 (tr:do
  (tr:pipe-do
   (pipe:echo nopad_armored_msg)
-  (pipe:gpg '(--decrypt))))
+  (pipe:gpg '(--debug-set-iobuf-size=8 --decrypt))))

commit bfc11816444512b4ebcc6617d3c3b5988e753de3
Author: Werner Koch <wk at gnupg.org>
Date:   Wed Jan 24 18:37:55 2018 +0100

    iobuf: Increase the size of the buffer.  Add iobuf_set_buffer_size.
    
    * common/iobuf.c (IOBUF_BUFFER_SIZE): Rename to
    DEFAULT_IOBUF_BUFFER_SIZE and increase to 64k.
    (iobuf_buffer_size): New var.  Always use this instead of the macro.
    (iobuf_set_buffer_size): New.
    (struct file_filter_ctx_t): Add field delayed_rc.
    (file_filter) [!W32]: Try to fill the supplied buffer.
    --
    
    I did some test to see whether this has an effect.  A test program
    piped 100 million random bytes to gpg to symmetric encryption only w/0
    compression.  Single read means the old behaviour, multi read the new
    behaviour which fills up the buffer when the read(2) returned only 4k
    in once call.
    
    8k buffer single read
            User time (seconds): 0.09
            System time (seconds): 0.04
            Percent of CPU this job got: 6%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.04
    
    8k buffer multi read
           User time (seconds): 0.08
           System time (seconds): 0.05
           Percent of CPU this job got: 6%
           Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.04
    
    64k buffer single read
            User time (seconds): 0.09
            System time (seconds): 0.06
            Percent of CPU this job got: 6%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.20
    
    64k buffer multi read
            User time (seconds): 0.11
            System time (seconds): 0.06
            Percent of CPU this job got: 8%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.01
    
    128k buffer single read
            User time (seconds): 0.09
            System time (seconds): 0.05
            Percent of CPU this job got: 7%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.05
    
    128k buffer multi read
            User time (seconds): 0.11
            System time (seconds): 0.05
            Percent of CPU this job got: 8%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.01
    
    512k buffer single read:
            User time (seconds): 0.08
            System time (seconds): 0.08
            Percent of CPU this job got: 7%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.21
    
    512k buffer multi read:
            User time (seconds): 0.10
            System time (seconds): 0.06
            Percent of CPU this job got: 7%
            Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.05
    
    Does not make much of a difference :-(.  Maybe it changes depending on
    the type of used filters.
    
    Signed-off-by: Werner Koch <wk at gnupg.org>

diff --git a/common/iobuf.c b/common/iobuf.c
index 5a9fd7c..bf59778 100644
--- a/common/iobuf.c
+++ b/common/iobuf.c
@@ -62,7 +62,7 @@
 /* The size of the internal buffers.
    NOTE: If you change this value you MUST also adjust the regression
    test "armored_key_8192" in armor.test! */
-#define IOBUF_BUFFER_SIZE  8192
+#define DEFAULT_IOBUF_BUFFER_SIZE  (64*1024)
 
 /* To avoid a potential DoS with compression packets we better limit
    the number of filters in a chain.  */
@@ -70,6 +70,10 @@
 
 /*-- End configurable part.  --*/
 
+/* The size of the iobuffers.  This can be chnages using the
+ * iobuf_set_buffer_size fucntion.  */
+static unsigned int iobuf_buffer_size = DEFAULT_IOBUF_BUFFER_SIZE;
+
 
 #ifdef HAVE_W32_SYSTEM
 # ifdef HAVE_W32CE_SYSTEM
@@ -92,6 +96,7 @@ typedef struct
   int keep_open;
   int no_cache;
   int eof_seen;
+  int delayed_rc;
   int print_only_name; /* Flags indicating that fname is not a real file.  */
   char fname[1];       /* Name of the file.  */
 } file_filter_ctx_t;
@@ -167,7 +172,7 @@ static int translate_file_handle (int fd, int for_write);
    to be sent to A's filter function.
 
    If A is a IOBUF_OUTPUT_TEMP filter, then this also enlarges the
-   buffer by IOBUF_BUFFER_SIZE.
+   buffer by iobuf_buffer_size.
 
    May only be called on an IOBUF_OUTPUT or IOBUF_OUTPUT_TEMP filters.  */
 static int filter_flush (iobuf_t a);
@@ -451,12 +456,20 @@ file_filter (void *opaque, int control, iobuf_t chain, byte * buf,
 
   if (control == IOBUFCTRL_UNDERFLOW)
     {
-      assert (size); /* We need a buffer.  */
+      log_assert (size); /* We need a buffer.  */
       if (a->eof_seen)
 	{
 	  rc = -1;
 	  *ret_len = 0;
 	}
+      else if (a->delayed_rc)
+        {
+          rc = a->delayed_rc;
+          a->delayed_rc = 0;
+          if (rc == -1)
+            a->eof_seen = -1;
+	  *ret_len = 0;
+        }
       else
 	{
 #ifdef HAVE_W32_SYSTEM
@@ -487,29 +500,39 @@ file_filter (void *opaque, int control, iobuf_t chain, byte * buf,
 	  int n;
 
 	  nbytes = 0;
-	  do
-	    {
-	      n = read (f, buf, size);
-	    }
-	  while (n == -1 && errno == EINTR);
-	  if (n == -1)
-	    {			/* error */
-	      if (errno != EPIPE)
-		{
-		  rc = gpg_error_from_syserror ();
-		  log_error ("%s: read error: %s\n",
-			     a->fname, strerror (errno));
-		}
-	    }
-	  else if (!n)
-	    {			/* eof */
-	      a->eof_seen = 1;
-	      rc = -1;
-	    }
-	  else
-	    {
-	      nbytes = n;
-	    }
+        read_more:
+          do
+            {
+              n = read (f, buf + nbytes, size - nbytes);
+            }
+          while (n == -1 && errno == EINTR);
+          if (n > 0)
+            {
+              nbytes += n;
+              if (nbytes < size)
+                goto read_more;
+            }
+          else if (!n) /* eof */
+            {
+              if (nbytes)
+                a->delayed_rc = -1;
+              else
+                {
+                  a->eof_seen = 1;
+                  rc = -1;
+                }
+            }
+          else /* error */
+            {
+              rc = gpg_error_from_syserror ();
+              if (gpg_err_code (rc) != GPG_ERR_EPIPE)
+                log_error ("%s: read error: %s\n", a->fname, gpg_strerror (rc));
+              if (nbytes)
+                {
+                  a->delayed_rc = rc;
+                  rc = 0;
+                }
+            }
 #endif
 	  *ret_len = nbytes;
 	}
@@ -569,6 +592,7 @@ file_filter (void *opaque, int control, iobuf_t chain, byte * buf,
   else if (control == IOBUFCTRL_INIT)
     {
       a->eof_seen = 0;
+      a->delayed_rc = 0;
       a->keep_open = 0;
       a->no_cache = 0;
     }
@@ -1053,6 +1077,30 @@ block_filter (void *opaque, int control, iobuf_t chain, byte * buffer,
   return rc;
 }
 
+
+/* Change the default size for all IOBUFs to KILOBYTE.  This needs to
+ * be called before any iobufs are used and can only be used once.
+ * Returns the current value.  Using 0 has no effect except for
+ * returning the current value.  */
+unsigned int
+iobuf_set_buffer_size (unsigned int kilobyte)
+{
+  static int used;
+
+  if (!used && kilobyte)
+    {
+      if (kilobyte < 4)
+        kilobyte = 4;
+      else if (kilobyte > 16*1024)
+        kilobyte = 16*1024;
+
+      iobuf_buffer_size = kilobyte * 1024;
+      used = 1;
+    }
+  return iobuf_buffer_size / 1024;
+}
+
+
 #define MAX_IOBUF_DESC 32
 /*
  * Fill the buffer by the description of iobuf A.
@@ -1105,7 +1153,7 @@ iobuf_alloc (int use, size_t bufsize)
   if (bufsize == 0)
     {
       log_bug ("iobuf_alloc() passed a bufsize of 0!\n");
-      bufsize = IOBUF_BUFFER_SIZE;
+      bufsize = iobuf_buffer_size;
     }
 
   a = xcalloc (1, sizeof *a);
@@ -1213,7 +1261,7 @@ iobuf_cancel (iobuf_t a)
 iobuf_t
 iobuf_temp (void)
 {
-  return iobuf_alloc (IOBUF_OUTPUT_TEMP, IOBUF_BUFFER_SIZE);
+  return iobuf_alloc (IOBUF_OUTPUT_TEMP, iobuf_buffer_size);
 }
 
 iobuf_t
@@ -1288,7 +1336,7 @@ do_open (const char *fname, int special_filenames,
 	return NULL;
     }
 
-  a = iobuf_alloc (use, IOBUF_BUFFER_SIZE);
+  a = iobuf_alloc (use, iobuf_buffer_size);
   fcx = xmalloc (sizeof *fcx + strlen (fname));
   fcx->fp = fp;
   fcx->print_only_name = print_only;
@@ -1335,7 +1383,7 @@ do_iobuf_fdopen (int fd, const char *mode, int keep_open)
   fp = INT2FD (fd);
 
   a = iobuf_alloc (strchr (mode, 'w') ? IOBUF_OUTPUT : IOBUF_INPUT,
-		   IOBUF_BUFFER_SIZE);
+		   iobuf_buffer_size);
   fcx = xmalloc (sizeof *fcx + 20);
   fcx->fp = fp;
   fcx->print_only_name = 1;
@@ -1373,7 +1421,7 @@ iobuf_esopen (estream_t estream, const char *mode, int keep_open)
   size_t len = 0;
 
   a = iobuf_alloc (strchr (mode, 'w') ? IOBUF_OUTPUT : IOBUF_INPUT,
-		   IOBUF_BUFFER_SIZE);
+		   iobuf_buffer_size);
   fcx = xtrymalloc (sizeof *fcx + 30);
   fcx->fp = estream;
   fcx->print_only_name = 1;
@@ -1398,7 +1446,7 @@ iobuf_sockopen (int fd, const char *mode)
   size_t len;
 
   a = iobuf_alloc (strchr (mode, 'w') ? IOBUF_OUTPUT : IOBUF_INPUT,
-		   IOBUF_BUFFER_SIZE);
+		   iobuf_buffer_size);
   scx = xmalloc (sizeof *scx + 25);
   scx->sock = fd;
   scx->print_only_name = 1;
@@ -1599,13 +1647,13 @@ iobuf_push_filter2 (iobuf_t a,
 	 increased accordingly.  We don't need to allocate a 10 MB
 	 buffer for a non-terminal filter.  Just use the default
 	 size.  */
-      a->d.size = IOBUF_BUFFER_SIZE;
+      a->d.size = iobuf_buffer_size;
     }
   else if (a->use == IOBUF_INPUT_TEMP)
     /* Same idea as above.  */
     {
       a->use = IOBUF_INPUT;
-      a->d.size = IOBUF_BUFFER_SIZE;
+      a->d.size = iobuf_buffer_size;
     }
 
   /* The new filter (A) gets a new buffer.
@@ -1922,7 +1970,7 @@ filter_flush (iobuf_t a)
 
   if (a->use == IOBUF_OUTPUT_TEMP)
     {				/* increase the temp buffer */
-      size_t newsize = a->d.size + IOBUF_BUFFER_SIZE;
+      size_t newsize = a->d.size + iobuf_buffer_size;
 
       if (DBG_IOBUF)
 	log_debug ("increasing temp iobuf from %lu to %lu\n",
diff --git a/common/iobuf.h b/common/iobuf.h
index 22e02da..1615638 100644
--- a/common/iobuf.h
+++ b/common/iobuf.h
@@ -259,6 +259,12 @@ struct iobuf_struct
 EXTERN_UNLESS_MAIN_MODULE int iobuf_debug_mode;
 
 
+/* Change the default size for all IOBUFs to KILOBYTE.  This needs to
+ * be called before any iobufs are used and can only be used once.
+ * Returns the current value.  Using 0 has no effect except for
+ * returning the current value.  */
+unsigned int iobuf_set_buffer_size (unsigned int kilobyte);
+
 /* Returns whether the specified filename corresponds to a pipe.  In
    particular, this function checks if FNAME is "-" and, if special
    filenames are enabled (see check_special_filename), whether

-----------------------------------------------------------------------

Summary of changes:
 common/iobuf.c          | 122 +++++++++++++++++++++++++++++++++---------------
 common/iobuf.h          |   6 +++
 doc/gpg.texi            |   6 +++
 g10/gpg.c               |  13 ++++++
 tests/openpgp/armor.scm |   6 +--
 5 files changed, 112 insertions(+), 41 deletions(-)


hooks/post-receive
-- 
The GNU Privacy Guard
http://git.gnupg.org




More information about the Gnupg-commits mailing list