[PATCH] sha1: add gnutls as a sha1 provider

[PATCH] sha1: add gnutls as a sha1 provider

[Shawn Landden]

GNUTLS uses the same cryptograms SHA1 routines (Cryptograms)
by Andy Polyakov <appro@openssl.org> as OpenSSL, but with a license
that is acceptable for downstream packagers.

This is not the cleanest way to use the GNUTLS library,
as it is reallocating the context every time, and GNUTLS itsself
fudges an OpenSSL CTX to use the cryptograms code, HOWEVER
in my benchmarks the code performs as well as both the OpenSSL library,
and my own integration of cryptograms with git.

I think this is preferrable to bringing the assembly routines into
the git code-base, as a way of getting access to these high-performance
routines to a git available in Debian, Ubuntu, or Fedora (which
all use BLK_SHA1=1 due to GPLv2 + OpenSSL license considerations,
see Debian Bug #879459).

I struggle with autotools, and I suspect something is wrong with that
part of the patch.

This laptop is ancient, Intel(R) Core(TM) i5 CPU M 520.
When I get arm64 hardware in a week I will update with new benchmarks.
Builtin (BLK_SHA1=1):
~/git/git$ time git fsck
Checking object directories: 100% (256/256), done.
Checking objects: 100% (238410/238410), done.
Checking connectivity: 236605, done.

real	0m25.806s
user	0m25.187s
sys	0m0.579s

This patch:
~/git/git$ time ./git fsck
Checking object directories: 100% (256/256), done.
Checking objects: 100% (238410/238410), done.
Checking connectivity: 236606, done.

real	0m22.368s
user	0m21.790s
sys	0m0.539s

Signed-off-by: Shawn Landden <slandden@gmail.com>
---
 Makefile           | 10 ++++++++++
 configure.ac       | 31 +++++++++++++++++++++++++++++++
 gnutls-sha1/sha1.c | 25 +++++++++++++++++++++++++
 gnutls-sha1/sha1.h | 12 ++++++++++++
 hash.h             |  2 ++
 5 files changed, 80 insertions(+)
 create mode 100644 gnutls-sha1/sha1.c
 create mode 100644 gnutls-sha1/sha1.h

diff --git a/Makefile b/Makefile
index cd7598599..e23648dbd 100644
--- a/Makefile
+++ b/Makefile
@@ -1252,7 +1252,9 @@ ifndef NO_OPENSSL
 	endif
 else
 	BASIC_CFLAGS += -DNO_OPENSSL
+ifndef GNUTLS_SHA1
 	BLK_SHA1 = 1
+endif
 	OPENSSL_LIBSSL =
 endif
 ifdef NO_OPENSSL
@@ -1481,6 +1483,11 @@ ifdef BLK_SHA1
 	LIB_OBJS += block-sha1/sha1.o
 	BASIC_CFLAGS += -DSHA1_BLK
 else
+ifdef GNUTLS_SHA1
+	LIB_OBJS += gnutls-sha1/sha1.o
+	BASIC_CFLAGS += -DSHA1_GNUTLS
+	EXTLIBS += -lgnutls
+endif
 ifdef PPC_SHA1
 	LIB_OBJS += ppc/sha1.o ppc/sha1ppc.o
 	BASIC_CFLAGS += -DSHA1_PPC
@@ -1488,6 +1495,8 @@ else
 ifdef APPLE_COMMON_CRYPTO
 	COMPAT_CFLAGS += -DCOMMON_DIGEST_FOR_OPENSSL
 	BASIC_CFLAGS += -DSHA1_APPLE
+else
+ifdef GNUTLS_SHA1
 else
 	DC_SHA1 := YesPlease
 	BASIC_CFLAGS += -DSHA1_DC
@@ -1506,6 +1515,7 @@ ifdef DC_SHA1_SUBMODULE
 else
 	LIB_OBJS += sha1dc/sha1.o
 	LIB_OBJS += sha1dc/ubc_check.o
+endif
 endif
 	BASIC_CFLAGS += \
 		-DSHA1DC_NO_STANDARD_INCLUDES \
diff --git a/configure.ac b/configure.ac
index 2f55237e6..109c4758d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -250,6 +250,23 @@ AS_HELP_STRING([--with-openssl],[use OpenSSL library (default is YES)])
 AS_HELP_STRING([],              [ARG can be prefix for openssl library and headers]),
 GIT_PARSE_WITH([openssl]))
 
+# Define GNUTLS_SHA1 if you have and want to use libgnutls. This offers
+# similar sha1 routines as openssl.
+AC_ARG_WITH(gnutls,
+AS_HELP_STRING([--with-gnutls],[use GNUTLS library (default is YES)]),
+    if test "$withval" = "no"; then
+        USE_GNUTLS=
+    elif test "$withval" = "yes"; then
+	USE_GNUTLS=YesPlease
+    else
+	USE_GNUTLS=YesPlease
+	LIBGNUTLSDIR=$withval
+	AC_MSG_NOTICE([Setting LIBGNUTLSDIR to $LIBGNUTLSDIR])
+        dnl USE_LIBGNUTLS can still be modified below, so don't substitute
+        dnl it yet.
+	GIT_CONF_SUBST([LIBGNUTLSDIR])
+    fi)
+
 # Define USE_LIBPCRE if you have and want to use libpcre. Various
 # commands such as log and grep offer runtime options to use
 # Perl-compatible regular expressions instead of standard or extended
@@ -540,6 +557,20 @@ GIT_UNSTASH_FLAGS($OPENSSLDIR)
 GIT_CONF_SUBST([NEEDS_SSL_WITH_CRYPTO])
 GIT_CONF_SUBST([NO_OPENSSL])
 
+#
+# Handle USE_GNUTLS from above
+#
+if test -n "$USE_GNUTLS"; then
+
+GIT_STASH_FLAGS($LIBGNUTLSDIR)
+
+AC_CHECK_LIB([gnutls], [gnutls_hash_init],
+[GNUTLS_SHA1=YesPlease],
+[GNUTLS_SHA1=])
+
+GIT_UNSTASH_FLAGS($LIBGNUTLSDIR)
+
+fi
 #
 # Handle the USE_LIBPCRE1 and USE_LIBPCRE2 options potentially set
 # above.
diff --git a/gnutls-sha1/sha1.c b/gnutls-sha1/sha1.c
new file mode 100644
index 000000000..f7ede4ddf
--- /dev/null
+++ b/gnutls-sha1/sha1.c
@@ -0,0 +1,25 @@
+/* this is only to get definitions for memcpy(), ntohl() and htonl() */
+#include "../git-compat-util.h"
+
+#include <gnutls/gnutls.h>
+#include <gnutls/crypto.h>
+
+#include "sha1.h"
+
+void gnutls_SHA1_Init(gnutls_SHA_CTX *ctx)
+{
+	int ret;
+	ret = gnutls_hash_init((void *) &ctx->handle, GNUTLS_DIG_SHA1);
+	if (ret < 0)
+		abort();
+}
+
+void gnutls_SHA1_Update(gnutls_SHA_CTX *ctx, const void *data, unsigned long len)
+{
+	gnutls_hash(ctx->handle, data, len);
+}
+
+void gnutls_SHA1_Final(unsigned char hashout[20], gnutls_SHA_CTX *ctx)
+{
+	gnutls_hash_deinit(ctx->handle, hashout);
+}
diff --git a/gnutls-sha1/sha1.h b/gnutls-sha1/sha1.h
new file mode 100644
index 000000000..ade60fcbe
--- /dev/null
+++ b/gnutls-sha1/sha1.h
@@ -0,0 +1,12 @@
+typedef struct {
+	void *handle;
+} gnutls_SHA_CTX;
+
+void gnutls_SHA1_Init(gnutls_SHA_CTX *ctx);
+void gnutls_SHA1_Update(gnutls_SHA_CTX *ctx, const void *dataIn, unsigned long len);
+void gnutls_SHA1_Final(unsigned char hashout[20], gnutls_SHA_CTX *ctx);
+
+#define platform_SHA_CTX	gnutls_SHA_CTX
+#define platform_SHA1_Init	gnutls_SHA1_Init
+#define platform_SHA1_Update	gnutls_SHA1_Update
+#define platform_SHA1_Final	gnutls_SHA1_Final
diff --git a/hash.h b/hash.h
index 024d0d3d5..7e5e307fc 100644
--- a/hash.h
+++ b/hash.h
@@ -9,6 +9,8 @@
 #include <openssl/sha.h>
 #elif defined(SHA1_DC)
 #include "sha1dc_git.h"
+#elif defined(SHA1_GNUTLS)
+#include "gnutls-sha1/sha1.h"
 #else /* SHA1_BLK */
 #include "block-sha1/sha1.h"
 #endif
-- 
2.15.0.rc2

Re: [PATCH] sha1: add gnutls as a sha1 provider

[Todd Zullinger]

Hi Shawn,

Shawn Landden wrote:
> I think this is preferrable to bringing the assembly routines into 
> the git code-base, as a way of getting access to these high-performance 
> routines to a git available in Debian, Ubuntu, or Fedora (which 
> all use BLK_SHA1=1 due to GPLv2 + OpenSSL license considerations, 
> see Debian Bug #879459).

While it seems like it could be useful to have the choice of using the 
fast SHA1 implementation without concern about licensing issues, 
there's a few details I thought were worth mentioning.

Fedora moved from OpenSSL SHA1 to BLK_SHA1 to reduce the size of the 
binaries and dependencies, not due to licensing issues (Fedora 
considers OpenSSL a system library and allows linking GPLv2 code).

Fedora now uses the default DC_SHA1 (the collision-detecting SHA1 
implementation).  DC_SHA1 is not, as far as I know, as fast as the 
OpenSSL/GnuTLS SHA1, but it's safer given the increasingly successful 
attacks against SHA1.  I don't envision changing that to gain 
performance.  (And, of course, the speed of SHA1 should become less of 
an issue once git moves to a new, stronger hash.)

It looks like the Debian packages use the default DC_SHA1 
implementation as well.  Regardless of the licensing concerns 
regarding OpenSSL in Debian, I suspect they'll want to use the 
default, collision-detecting SHA1 implementation.  That doesn't mean a 
patch to add the option of GnuTLS isn't useful though.

Fedora does link with OpenSSL's libcrypto and libssl in Fedora for the 
remote-curl helpers and imap-send.  I believe the remote-curl helpers 
just link with curl, which happens to use OpenSSL on Fedora and could 
use GnuTLS instead.  The imap-send command might also use curl and 
whatever crypto library curl is built with too, but I'm not terribly 
familiar with imap-send. (I think those are the only uses of libcrypto 
or libssl in Fedora's packages, but I could be mistaken).

That's a lot of text without having anything to say about the actual 
patch.  Hopefully it's at least mildly useful to you or others. :)

-- 
Todd
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When we remember we are all mad, the mysteries of life disappear and
life stands explained.
    -- Mark Twain

Re: [PATCH] sha1: add gnutls as a sha1 provider

[Shawn Landden]

On Tue, Nov 14, 2017 at 11:47 AM, Todd Zullinger <tmz@pobox.com> wrote:
>
> Hi Shawn,
>
> Shawn Landden wrote:
>>
>> I think this is preferrable to bringing the assembly routines into the git code-base, as a way of getting access to these high-performance routines to a git available in Debian, Ubuntu, or Fedora (which all use BLK_SHA1=1 due to GPLv2 + OpenSSL license considerations, see Debian Bug #879459).
>
>
> While it seems like it could be useful to have the choice of using the fast SHA1 implementation without concern about licensing issues, there's a few details I thought were worth mentioning.
>
> Fedora moved from OpenSSL SHA1 to BLK_SHA1 to reduce the size of the binaries and dependencies, not due to licensing issues (Fedora considers OpenSSL a system library and allows linking GPLv2 code).
>
> Fedora now uses the default DC_SHA1 (the collision-detecting SHA1 implementation).  DC_SHA1 is not, as far as I know, as fast as the OpenSSL/GnuTLS SHA1, but it's safer given the increasingly successful attacks against SHA1.  I don't envision changing that to gain performance.  (And, of course, the speed of SHA1 should become less of an issue once git moves to a new, stronger hash.)
>
> It looks like the Debian packages use the default DC_SHA1 implementation as well.  Regardless of the licensing concerns regarding OpenSSL in Debian, I suspect they'll want to use the default, collision-detecting SHA1 implementation.  That doesn't mean a patch to add the option of GnuTLS isn't useful though.
>
> Fedora does link with OpenSSL's libcrypto and libssl in Fedora for the remote-curl helpers and imap-send.  I believe the remote-curl helpers just link with curl, which happens to use OpenSSL on Fedora and could use GnuTLS instead.  The imap-send command might also use curl and whatever crypto library curl is built with too, but I'm not terribly familiar with imap-send. (I think those are the only uses of libcrypto or libssl in Fedora's packages, but I could be mistaken).
>
> That's a lot of text without having anything to say about the actual patch.  Hopefully it's at least mildly useful to you or others. :)
It is all appreciated. I just want to make note that I am still
interested in getting this patch in.