Add a compile-time option to use the new, very fast sha1dc Rust crate

Cargo.toml

@@ -8,3 +8,7 @@ rust-version = "1.49.0"
 crate-type = ["staticlib"]
 
 [dependencies]
+sha1dc = { version = "0.1.3", optional = true }
+
+[features]
+sha1dc-rs = ["sha1dc"]

Documentation/config/core.adoc

@@ -382,6 +382,11 @@ core.repositoryFormatVersion::
 	Internal variable identifying the repository format and layout
 	version. See linkgit:gitrepository-layout[5].
 
+core.sha1dcBackend::
+	Select the collision-detecting SHA-1 implementation when Git is built
+	with `DC_SHA1_RS`: valid values are `rust` (the default) and `c` (the C
+	fallback). This setting has no effect with other SHA-1 backends.
+
 core.sharedRepository::
 	When 'group' (or 'true'), the repository is made shareable between
 	several users in a group (making sure all the files and objects are

Makefile

@@ -567,6 +567,10 @@ include shared.mak
 # by the git project to migrate to using sha1collisiondetection as a
 # submodule.
 #
+# Define DC_SHA1_RS to use the sha1dc Rust crate by default, with the C
+# implementation available via core.sha1dcBackend=c. This requires Rust
+# 1.87 or newer.
+#
 # === SHA-256 backend ===
 #
 # ==== Security ====
@@ -2137,6 +2141,23 @@ ifdef PPC_SHA1
 $(error the PPC_SHA1 flag has been removed along with the PowerPC-specific SHA-1 implementation.)
 endif
 
+ifdef DC_SHA1_RS
+ifdef NO_RUST
+$(error DC_SHA1_RS requires Rust support)
+endif
+ifneq ($(strip $(OPENSSL_SHA1)$(BLK_SHA1)$(APPLE_COMMON_CRYPTO_SHA1)),)
+$(error DC_SHA1_RS cannot be combined with another SHA-1 backend)
+endif
+ifdef DC_SHA1_EXTERNAL
+$(error Only set DC_SHA1_RS or DC_SHA1_EXTERNAL, not both)
+endif
+ifdef DC_SHA1_SUBMODULE
+ifneq ($(DC_SHA1_SUBMODULE),auto)
+$(error Only set DC_SHA1_RS or DC_SHA1_SUBMODULE, not both)
+endif
+endif
+endif
+
 ifdef OPENSSL_SHA1
 	EXTLIBS += $(LIB_4_CRYPTO)
 	BASIC_CFLAGS += -DSHA1_OPENSSL
@@ -2151,6 +2172,14 @@ ifdef APPLE_COMMON_CRYPTO_SHA1
 else
 	BASIC_CFLAGS += -DSHA1_DC
 	LIB_OBJS += sha1dc_git.o
+ifdef DC_SHA1_RS
+	BASIC_CFLAGS += -DDC_SHA1_RS
+	CARGO_ARGS += --features sha1dc-rs
+	RUST_SOURCES += src/sha1dc_rs.rs
+ifeq ($(uname_S),MINGW)
+	EXTLIBS += -luserenv
+endif
+endif
 ifdef DC_SHA1_EXTERNAL
         ifdef DC_SHA1_SUBMODULE
                 ifneq ($(DC_SHA1_SUBMODULE),auto)

compat/win32/pthread.c

@@ -60,6 +60,22 @@ pthread_t pthread_self(void)
 	return t;
 }
 
+static BOOL CALLBACK win32_pthread_once(PINIT_ONCE once UNUSED,
+					PVOID parameter,
+					PVOID *context UNUSED)
+{
+	(*(void (**)(void))parameter)();
+	return TRUE;
+}
+
+int pthread_once(pthread_once_t *once_control, void (*init_routine)(void))
+{
+	if (!InitOnceExecuteOnce(once_control, win32_pthread_once,
+				 &init_routine, NULL))
+		return err_win_to_posix(GetLastError());
+	return 0;
+}
+
 int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
 {
 	if (SleepConditionVariableCS(cond, mutex, INFINITE) == 0)

compat/win32/pthread.h

@@ -26,6 +26,11 @@ static inline int return_0(int i UNUSED) {
 #define pthread_mutex_lock EnterCriticalSection
 #define pthread_mutex_unlock LeaveCriticalSection
 
+typedef INIT_ONCE pthread_once_t;
+#define PTHREAD_ONCE_INIT INIT_ONCE_STATIC_INIT
+
+int pthread_once(pthread_once_t *once_control, void (*init_routine)(void));
+
 typedef int pthread_mutexattr_t;
 #define pthread_mutexattr_init(a) (*(a) = 0)
 #define pthread_mutexattr_destroy(a) do {} while (0)

hash.h

@@ -12,8 +12,13 @@
 #    include "sha1/openssl.h"
 #  endif
 #elif defined(SHA1_DC)
+#ifdef DC_SHA1_RS
+#define SHA1_BACKEND "SHA1_DC-rs"
+#include "sha1dc_rs.h"
+#else
 #define SHA1_BACKEND "SHA1_DC"
 #include "sha1dc_git.h"
+#endif
 #else /* SHA1_BLK */
 #define SHA1_BACKEND "SHA1_BLK (No collision detection)"
 #include "block-sha1/sha1.h"

sha1dc_git.c

@@ -1,6 +1,14 @@
+#ifdef DC_SHA1_RS
+#define USE_THE_REPOSITORY_VARIABLE
+#endif
 #include "git-compat-util.h"
-#include "sha1dc_git.h"
 #include "hex.h"
+#include "sha1dc_git.h"
+#ifdef DC_SHA1_RS
+#include "config.h"
+#include "repository.h"
+#include "thread-utils.h"
+#endif
 
 #ifdef DC_SHA1_EXTERNAL
 /*
@@ -16,12 +24,13 @@ void git_SHA1DCInit(SHA1_CTX *ctx)
 /*
  * Same as SHA1DCFinal, but convert collision attack case into a verbose die().
  */
-void git_SHA1DCFinal(unsigned char hash[20], SHA1_CTX *ctx)
+void git_SHA1DCFinal(unsigned char hash[20], SHA1_CTX *ctx,
+		     void (*die_fn)(const char *, ...))
 {
 	if (!SHA1DCFinal(hash, ctx))
 		return;
-	die("SHA-1 appears to be part of a collision attack: %s",
-	    hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1]));
+	die_fn("SHA-1 appears to be part of a collision attack: %s",
+	       hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1]));
 }
 
 /*
@@ -37,3 +46,95 @@ void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *vdata, size_t len)
 	}
 	SHA1DCUpdate(ctx, data, len);
 }
+
+#ifdef DC_SHA1_RS
+static void sha1dc_c_clone(SHA1_CTX *dst, const SHA1_CTX *src)
+{
+	*dst = *src;
+}
+
+static void sha1dc_c_discard(SHA1_CTX *ctx UNUSED)
+{
+	/* The C context owns no resources. */
+}
+
+/* The first SHA-1 initialization must precede concurrent hashing. */
+static void initial_init(SHA1_CTX *);
+static void initial_clone(SHA1_CTX *, const SHA1_CTX *);
+static void initial_update(SHA1_CTX *, const void *, size_t);
+static void initial_final(unsigned char [20], SHA1_CTX *,
+			  void (*die_fn)(const char *, ...));
+static void initial_discard(SHA1_CTX *ctx);
+
+void (*sha1dc_init)(SHA1_CTX *) = initial_init;
+void (*sha1dc_clone)(SHA1_CTX *, const SHA1_CTX *) = initial_clone;
+void (*sha1dc_update)(SHA1_CTX *, const void *, size_t) = initial_update;
+void (*sha1dc_final)(unsigned char [20], SHA1_CTX *,
+		     void (*die_fn)(const char *, ...)) = initial_final;
+void (*sha1dc_discard)(SHA1_CTX *) = initial_discard;
+
+static void sha1dc_choose(void)
+{
+	const char *backend;
+	int use_c = 0;
+
+	if (!repo_config_get_string_tmp(the_repository, "core.sha1dcbackend",
+					&backend)) {
+		if (!strcasecmp(backend, "c"))
+			use_c = 1;
+		else if (strcasecmp(backend, "rust"))
+			die("invalid value for core.sha1dcBackend: '%s'",
+			    backend);
+	}
+
+	sha1dc_clone = use_c ? sha1dc_c_clone : sha1dc_rs_clone;
+	sha1dc_update = use_c ? git_SHA1DCUpdate : sha1dc_rs_update;
+	sha1dc_final = use_c ? git_SHA1DCFinal : sha1dc_rs_final;
+	sha1dc_discard = use_c ? sha1dc_c_discard : sha1dc_rs_discard;
+	sha1dc_init = use_c ? git_SHA1DCInit : sha1dc_rs_init;
+}
+
+static pthread_once_t once = PTHREAD_ONCE_INIT;
+
+static void initial_init(SHA1_CTX *ctx)
+{
+	int ret = pthread_once(&once, sha1dc_choose);
+	if (ret)
+		die("cannot initialize SHA-1 backend: %s", strerror(ret));
+	sha1dc_init(ctx);
+}
+
+static void initial_clone(SHA1_CTX *dst, const SHA1_CTX *src)
+{
+	int ret = pthread_once(&once, sha1dc_choose);
+	if (ret)
+		die("cannot initialize SHA-1 backend: %s", strerror(ret));
+	sha1dc_clone(dst, src);
+}
+
+static void initial_update(SHA1_CTX *ctx, const void *buf, size_t len)
+{
+	int ret = pthread_once(&once, sha1dc_choose);
+	if (ret)
+		die("cannot initialize SHA-1 backend: %s", strerror(ret));
+	sha1dc_update(ctx, buf, len);
+}
+
+static void initial_final(unsigned char hash[20], SHA1_CTX *ctx,
+			  void (*die_fn)(const char *, ...))
+{
+	int ret = pthread_once(&once, sha1dc_choose);
+	if (ret)
+		die("cannot initialize SHA-1 backend: %s", strerror(ret));
+	sha1dc_final(hash, ctx, die_fn);
+}
+
+static void initial_discard(SHA1_CTX *ctx)
+{
+	int ret = pthread_once(&once, sha1dc_choose);
+	if (ret)
+		die("cannot initialize SHA-1 backend: %s", strerror(ret));
+	sha1dc_discard(ctx);
+}
+
+#endif

sha1dc_git.h

@@ -14,7 +14,8 @@ void git_SHA1DCInit(SHA1_CTX *);
 #define git_SHA1DCInit	SHA1DCInit
 #endif
 
-void git_SHA1DCFinal(unsigned char [20], SHA1_CTX *);
+void git_SHA1DCFinal(unsigned char [20], SHA1_CTX *,
+		    void (*die_fn)(const char *, ...));
 void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *data, size_t len);
 
 #define platform_SHA_IS_SHA1DC /* used by "test-tool sha1-is-sha1dc" */
@@ -23,5 +24,5 @@ void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *data, size_t len);
 #define platform_SHA_CTX SHA1_CTX
 #define platform_SHA1_Init git_SHA1DCInit
 #define platform_SHA1_Update git_SHA1DCUpdate
-#define platform_SHA1_Final git_SHA1DCFinal
+#define platform_SHA1_Final(hash, ctx) git_SHA1DCFinal((hash), (ctx), die)
 #endif

sha1dc_rs.h

@@ -0,0 +1,37 @@
+#ifndef SHA1DC_RS_H
+#define SHA1DC_RS_H
+
+#define platform_SHA_CTX union sha1dc_ctx
+#include "sha1dc_git.h"
+
+typedef struct sha1dc_rs_hasher *sha1dc_rs_ctx;
+
+union sha1dc_ctx {
+	SHA1_CTX c;
+	sha1dc_rs_ctx rs;
+};
+
+void sha1dc_rs_init(SHA1_CTX *);
+void sha1dc_rs_clone(SHA1_CTX *, const SHA1_CTX *);
+void sha1dc_rs_update(SHA1_CTX *, const void *, size_t);
+void sha1dc_rs_final(unsigned char [20], SHA1_CTX *,
+		     void (*die_fn)(const char *, ...));
+void sha1dc_rs_discard(SHA1_CTX *);
+
+extern void (*sha1dc_init)(SHA1_CTX *);
+extern void (*sha1dc_clone)(SHA1_CTX *, const SHA1_CTX *);
+extern void (*sha1dc_update)(SHA1_CTX *, const void *, size_t);
+extern void (*sha1dc_final)(unsigned char [20], SHA1_CTX *,
+			    void (*die_fn)(const char *, ...));
+extern void (*sha1dc_discard)(SHA1_CTX *);
+
+#define platform_SHA1_Init(ctx) sha1dc_init(&(ctx)->c)
+#define platform_SHA1_Update(ctx, data, len) \
+	sha1dc_update(&(ctx)->c, (data), (len))
+#define platform_SHA1_Final(hash, ctx) \
+	sha1dc_final((hash), &(ctx)->c, die)
+#define SHA1_NEEDS_CLONE_HELPER
+#define platform_SHA1_Clone(dst, src) sha1dc_clone(&(dst)->c, &(src)->c)
+#define platform_SHA1_Discard(ctx) sha1dc_discard(&(ctx)->c)
+
+#endif

src/lib.rs

@@ -1,4 +1,6 @@
 pub mod csum_file;
 pub mod hash;
 pub mod loose;
+#[cfg(feature = "sha1dc-rs")]
+mod sha1dc_rs;
 pub mod varint;

src/sha1dc_rs.rs

@@ -0,0 +1,78 @@
+use sha1dc::Hasher;
+use std::ffi::{c_void, CString};
+use std::os::raw::c_char;
+use std::{ptr, slice};
+
+/// Initialize a collision-detecting SHA-1 context.
+///
+/// # Safety
+/// `ctx` must point to an uninitialized SHA-1 context.
+#[no_mangle]
+pub unsafe extern "C" fn sha1dc_rs_init(ctx: *mut c_void) {
+    let ctx = ctx.cast::<*mut Hasher>();
+    *ctx = Box::into_raw(Box::new(Hasher::new()));
+}
+
+/// Replace a SHA-1 context with a clone of another.
+///
+/// # Safety
+/// Both contexts must be initialized.
+#[no_mangle]
+pub unsafe extern "C" fn sha1dc_rs_clone(dst: *mut c_void, src: *const c_void) {
+    let dst = dst.cast::<*mut Hasher>();
+    let src = src.cast::<*mut Hasher>();
+    let hasher = Box::new((**src).clone());
+    drop(Box::from_raw(*dst));
+    *dst = Box::into_raw(hasher);
+}
+
+/// Update the SHA-1 hasher with the given bytes.
+///
+/// # Safety
+/// `ctx` must be initialized and `data` must point to `len` bytes unless
+/// `len` is zero.
+#[no_mangle]
+pub unsafe extern "C" fn sha1dc_rs_update(ctx: *mut c_void, data: *const c_void, len: usize) {
+    let ctx = ctx.cast::<*mut Hasher>();
+    if len != 0 {
+        (**ctx).update(slice::from_raw_parts(data.cast::<u8>(), len));
+    }
+}
+
+/// Finalize SHA-1, reporting detected collisions through `die`.
+///
+/// # Safety
+/// `ctx` must be initialized, `hash` must point to at least 20 bytes, and
+/// `die` must be a non-returning C variadic function.
+#[no_mangle]
+pub unsafe extern "C" fn sha1dc_rs_final(
+    hash: *mut u8,
+    ctx: *mut c_void,
+    die: unsafe extern "C" fn(*const c_char, ...) -> !,
+) {
+    let ctx = ctx.cast::<*mut Hasher>();
+    let hasher = *Box::from_raw(*ctx);
+    *ctx = ptr::null_mut();
+    match hasher.finalize() {
+        Ok(digest) => ptr::copy_nonoverlapping(digest.as_bytes().as_ptr(), hash, 20),
+        Err(collision) => {
+            let message = CString::new(format!(
+                "SHA-1 appears to be part of a collision attack: {}",
+                collision.digest()
+            ))
+            .expect("collision message contains no NUL");
+            die(message.as_ptr());
+        }
+    }
+}
+
+/// Discard a SHA-1 context without producing a digest.
+///
+/// # Safety
+/// `ctx` must be initialized.
+#[no_mangle]
+pub unsafe extern "C" fn sha1dc_rs_discard(ctx: *mut c_void) {
+    let ctx = ctx.cast::<*mut Hasher>();
+    drop(Box::from_raw(*ctx));
+    *ctx = ptr::null_mut();
+}