Offload SHA256 to FreeBSD's OCF.

include/os/freebsd/zfs/sys/crypto_os.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2021 Jeremy Faulkner <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef	_ZFS_CRYPTO_OS_H
+#define	_ZFS_CRYPTO_OS_H
+
+#include <sys/freebsd_crypto.h>
+
+#define	zfs_offload_hash(checksum, abd, size, zcp)	\
+	freebsd_offload_hash_to_ocf(checksum, abd, size, zcp)
+
+#endif /* _ZFS_CRYPTO_OS_H */

include/os/freebsd/zfs/sys/freebsd_crypto.h

@@ -37,6 +37,7 @@
 #include <opencrypto/cryptodev.h>
 #include <crypto/sha2/sha256.h>
 #include <crypto/sha2/sha512.h>
+#include <sys/zio_checksum.h>
 
 #define	SUN_CKM_AES_CCM	"CKM_AES_CCM"
 #define	SUN_CKM_AES_GCM	"CKM_AES_GCM"
@@ -88,6 +89,14 @@ int freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
     const struct zio_crypt_info *, crypto_key_t *);
 void freebsd_crypt_freesession(freebsd_crypt_session_t *sessp);
 
+int freebsd_hash_newsession(freebsd_crypt_session_t *sessionp,
+    size_t checksum);
+int freebsd_hash(freebsd_crypt_session_t *input_sessionp,
+    size_t checksum, uint8_t *buf, uint64_t size, uint8_t *obuf,
+    uint64_t osize);
+int freebsd_offload_hash_to_ocf(uint64_t checksum, abd_t *abd,
+    uint64_t size, zio_cksum_t *zcp);
+
 int freebsd_crypt_uio(boolean_t, freebsd_crypt_session_t *,
 	const struct zio_crypt_info *, zfs_uio_t *, crypto_key_t *, uint8_t *,
 	size_t, size_t);

include/os/linux/zfs/sys/crypto_os.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021 Jeremy Faulkner <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef	_ZFS_CRYPTO_OS_H
+#define	_ZFS_CRYPTO_OS_H
+
+#define	zfs_offload_hash(checksum, abd, size, zcp)	EOPNOTSUPP
+
+#endif /* _ZFS_CRYPTO_OS_H */

module/os/freebsd/zfs/crypto_os.c

@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include <opencrypto/xform.h>
 #endif
 
+#include <sys/zio_checksum.h>
 #include <sys/zio_crypt.h>
 #include <sys/fs/zfs.h>
 #include <sys/zio.h>
@@ -321,7 +322,8 @@ freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
 	 * CPU-accelerated drivers such as aesni(4) register themselves as
 	 * hardware drivers.
 	 */
-	error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
+	error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE |
+	    CRYPTOCAP_F_ACCEL_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 	mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
 	    NULL, MTX_DEF);
 	crypt_sessions++;
@@ -333,6 +335,110 @@ freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
 	return (error);
 }
 
+int
+freebsd_hash_newsession(freebsd_crypt_session_t *sessionp, size_t checksum)
+{
+	int error = 0;
+	struct crypto_session_params csp;
+
+	bzero(&csp, sizeof (struct crypto_session_params));
+	csp.csp_mode = CSP_MODE_DIGEST;
+	csp.csp_auth_alg = checksum;
+	csp.csp_flags = CSP_F_SEPARATE_OUTPUT;
+	csp.csp_auth_mlen = 0;
+
+	error = crypto_newsession(&sessionp->fs_sid, &csp,
+	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_ACCEL_SOFTWARE);
+	mtx_init(&sessionp->fs_lock, "FreeBSD Cryptographic Session Lock",
+	    NULL, MTX_DEF);
+	sessionp->fs_done = false;
+
+	return (error);
+}
+
+int
+freebsd_hash(freebsd_crypt_session_t *input_sessionp, size_t checksum,
+    uint8_t *buf, uint64_t size, uint8_t *obuf, uint64_t osize)
+{
+	int error = 0;
+	freebsd_crypt_session_t *session = NULL;
+	struct cryptop *crp;
+
+	if (input_sessionp == NULL) {
+		session = kmem_zalloc(sizeof (freebsd_crypt_session_t),
+		    KM_SLEEP);
+		error = freebsd_hash_newsession(session, checksum);
+		if (error)
+			goto out;
+	} else {
+		session = input_sessionp;
+	}
+
+	crp = crypto_getreq(session->fs_sid, M_WAITOK);
+	crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_CBIMM;
+	crp->crp_digest_start = 0;
+	crp->crp_payload_start = 0;
+	crp->crp_payload_length = size;
+	crp->crp_payload_output_start = 0;
+	crypto_use_buf(crp, buf, size);
+	crypto_use_output_buf(crp, obuf, osize);
+
+	error = zfs_crypto_dispatch(session, crp);
+
+	if (!session->fs_done || !(crp->crp_flags & CRYPTO_F_DONE) ||
+	    (crp->crp_etype))
+		error = EINVAL;
+
+	crypto_freereq(crp);
+
+out:
+	if (input_sessionp == NULL) {
+		freebsd_crypt_freesession(session);
+		kmem_free(session, sizeof (freebsd_crypt_session_t));
+	}
+
+#ifdef FCRYPTO_DEBUG
+	if (error)
+		printf("\n%s: returning error %d\n", __FUNCTION__, error);
+
+	printf("\nInput buffer size is: %lu\n", size);
+
+	printf("\nOutput buffer is:\n");
+	for (int i = 0; i < osize; ++i)
+		printf("%02X", obuf[i]);
+#endif
+
+	return (error);
+}
+
+int
+freebsd_offload_hash_to_ocf(uint64_t checksum, abd_t *abd, uint64_t size,
+    zio_cksum_t *zcp)
+{
+	int error = 1;
+	freebsd_crypt_session_t *session = NULL;
+
+	session = kmem_zalloc(sizeof (freebsd_crypt_session_t), KM_SLEEP);
+
+	if (session != NULL)
+		error = freebsd_hash_newsession(session, checksum);
+
+	if (error == 0) {
+		uint8_t *buf = abd_borrow_buf_copy(abd, size);
+		error = freebsd_hash(session, checksum, buf, size,
+		    (uint8_t *)zcp, sizeof (zio_cksum_t));
+		abd_return_buf(abd, buf, size);
+	}
+
+	if (session != NULL) {
+		freebsd_crypt_freesession(session);
+		kmem_free(session, sizeof (freebsd_crypt_session_t));
+	}
+
+	return (error);
+}
+
 int
 freebsd_crypt_uio(boolean_t encrypt,
     freebsd_crypt_session_t *input_sessionp,
@@ -393,7 +499,7 @@ freebsd_crypt_uio(boolean_t encrypt,
 	return (error);
 }
 
-#else
+#else /* __FreeBSD_version >= 1300087 */
 int
 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
     const struct zio_crypt_info *c_info, crypto_key_t *key)
@@ -632,4 +738,6 @@ freebsd_crypt_uio(boolean_t encrypt,
 #endif
 	return (error);
 }
+
+#define	freebsd_offload_hash_to_ocf(checksum, abd, size, zcp)	EOPNOTSUPP
 #endif

module/zfs/sha256.c

@@ -33,6 +33,10 @@
 #include <sys/abd.h>
 #include <sys/qat.h>
 
+#ifdef _KERNEL
+#include <sys/crypto_os.h>
+#endif
+
 static int
 sha_incremental(void *buf, size_t size, void *arg)
 {
@@ -50,6 +54,11 @@ abd_checksum_SHA256(abd_t *abd, uint64_t size,
 	SHA2_CTX ctx;
 	zio_cksum_t tmp;
 
+#ifdef _KERNEL
+	if (zfs_offload_hash(CRYPTO_SHA2_256, abd, size, &tmp) == 0)
+		goto bswap;
+#endif
+
 	if (qat_checksum_use_accel(size)) {
 		uint8_t *buf = abd_borrow_buf_copy(abd, size);
 		ret = qat_checksum(ZIO_CHECKSUM_SHA256, buf, size, &tmp);