Skip to content

Commit

Permalink
Merge pull request #512 from SeeSpotRun/blake3_local
Browse files Browse the repository at this point in the history 
checksums: implement blake3
  • Loading branch information
@SeeSpotRun
SeeSpotRun authored May 28, 2021
2 parents da0577b + 4ab29be commit bdb591f
Show file tree
Hide file tree
Showing 28 changed files with 31,210 additions and 5 deletions.
47 changes: 47 additions & 0 deletions SConstruct
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,29 @@ def check_builtin_cpu_supports(context):
return rc


def check_cpu_extensions(context):
l_cpuInfo = {'flags' : []}
print('==> Checking CPU checksum and vector extensions...')
rc = 1
if ARGUMENTS.get('CPU_EXTENSIONS') != '0':
try:
import sys
sys.path.append('submodules/py-cpuinfo')
from cpuinfo import get_cpu_info
l_cpuInfo = get_cpu_info()
except:
print(' Unable to get cpuinfo, maybe install py-cpuinfo')
rc = 0

for ext in ['AVX512F', 'AVX512VL', 'AVX2', 'SSE4_1', 'SSE2']:
have_ext = 1 if ext.lower() in l_cpuInfo['flags'] else 0
conf.env['HAVE_' + ext] = have_ext
print(' {}: {}'.format(ext, have_ext))

context.did_show_result = True
context.Result(rc)
return rc

def create_uninstall_target(env, path):
env.Command("uninstall-" + path, path, [
Delete("$SOURCE"),
Expand Down Expand Up @@ -608,6 +631,7 @@ conf = Configure(env, custom_tests={
'check_uname': check_uname,
'check_cygwin': check_cygwin,
'check_mm_crc32_u64': check_mm_crc32_u64,
'check_cpu_extensions': check_cpu_extensions,
'check_builtin_cpu_supports': check_builtin_cpu_supports,
'check_sysmacro_h': check_sysmacro_h
})
Expand Down Expand Up @@ -710,6 +734,7 @@ conf.env.Append(CCFLAGS=[
'-Wno-implicit-fallthrough',
])


env.ParseConfig(pkg_config + ' --cflags --libs ' + ' '.join(packages))


Expand All @@ -731,10 +756,24 @@ conf.check_btrfs_h()
conf.check_linux_fs_h()
conf.check_uname()
conf.check_sysmacro_h()
conf.check_cpu_extensions()

if conf.env['HAVE_LIBELF']:
conf.env.Append(_LIBFLAGS=['-lelf'])

if conf.env['HAVE_AVX2']:
conf.env.Append(CCFLAGS=['-mavx2'])

if conf.env['HAVE_AVX512F'] and conf.env['HAVE_AVX512VL']:
conf.env.Append(CCFLAGS=['-mavx512f', '-mavx512vl'])

if conf.env['HAVE_SSE4_1']:
conf.env.Append(CCFLAGS=['-msse4.1'])

if conf.env['HAVE_SSE2']:
conf.env.Append(CCFLAGS=['-msse2'])


if ARGUMENTS.get('GDB') == '1':
ARGUMENTS['DEBUG'] = '1'
ARGUMENTS['SYMBOLS'] = '1'
Expand Down Expand Up @@ -908,6 +947,10 @@ if 'config' in COMMAND_LINE_TARGETS:
Find non-stripped binaries (needs libelf) : {libelf}
Optimize using ioctl(FS_IOC_FIEMAP) (needs linux) : {fiemap}
Support for SHA512 (needs glib >= 2.31) : {sha512}
AVX512F and AVX512VL cpu extensions : {avx512}
AVX2 cpu extensions : {avx2}
SSE4.1 cpu extensions : {sse41}
SSE2 cpu extensions : {sse2}
Build manpage from docs/rmlint.1.rst : {sphinx}
Support for caching checksums in file's xattr : {xattr}
Checking for proper support of big files >= 4GB : {bigfiles}
Expand Down Expand Up @@ -945,6 +988,10 @@ Type 'scons' to actually compile rmlint now. Good luck.
blkid=yesno(env['HAVE_BLKID']),
fiemap=yesno(env['HAVE_FIEMAP']),
sha512=yesno(env['HAVE_SHA512']),
avx512=yesno(env['HAVE_AVX512F'] and env['HAVE_AVX512VL']),
avx2=yesno(env['HAVE_AVX2']),
sse41=yesno(env['HAVE_SSE4_1']),
sse2=yesno(env['HAVE_SSE2']),
bigfiles=yesno(env['HAVE_BIGFILES']),
bigofft=yesno(env['HAVE_BIG_OFF_T']),
bigstat=yesno(env['HAVE_BIG_STAT']),
Expand Down
27 changes: 27 additions & 0 deletions lib/SConscript
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ def build_config_template(target, source, env):
HAVE_LINUX_FS_H=env['HAVE_LINUX_FS_H'],
HAVE_BTRFS_H=env['HAVE_BTRFS_H'],
HAVE_MM_CRC32_U64=env['HAVE_MM_CRC32_U64'],
HAVE_AVX2=env['HAVE_AVX2'],
HAVE_AVX512F=env['HAVE_AVX512F'],
HAVE_AVX512VL=env['HAVE_AVX512VL'],
HAVE_SSE4_1=env['HAVE_SSE4_1'],
HAVE_BUILTIN_CPU_SUPPORTS=env['HAVE_BUILTIN_CPU_SUPPORTS'],
HAVE_UNAME=env['HAVE_UNAME'],
HAVE_SYSMACROS_H=env['HAVE_SYSMACROS_H'],
Expand Down Expand Up @@ -89,12 +93,35 @@ sh_formatter = env.Command(
'formats/sh.c', 'formats/sh.c.in', build_sh_formatter
)

blake3 = Glob('checksums/blake3/*.[cS]')
if not env['HAVE_AVX2']:
print('Not compiling blake3 avx2 options')
blake3 = [f for f in blake3 if 'avx2' not in f.path]
env.Append(CCFLAGS=['-DBLAKE3_NO_AVX2'])


if not env['HAVE_AVX512F'] or not env['HAVE_AVX512VL']:
print('Not compiling blake3 avx512 options')
blake3 = [f for f in blake3 if 'avx512' not in f.path]
env.Append(CCFLAGS=['-DBLAKE3_NO_AVX512'])

if not env['HAVE_SSE4_1']:
print('Not compiling blake3 sse41 options')
blake3 = [f for f in blake3 if 'sse41' not in f.path]
env.Append(CCFLAGS=['-DBLAKE3_NO_SSE41'])

if not env['HAVE_SSE2']:
print('Not compiling blake3 sse2 options')
blake3 = [f for f in blake3 if 'sse41' not in f.path]
env.Append(CCFLAGS=['-DBLAKE3_NO_SSE2'])

library = env.Library(
'../rmlint',
Glob('*.c') +
Glob('checksums/*.c') +
Glob('checksums/xxhash/*.c') +
Glob('checksums/blake2/*.c') +
blake3 +
Glob('checksums/sha3/*.c') +
Glob('formats/*.c') +
Glob('fts/*.c')
Expand Down
61 changes: 56 additions & 5 deletions lib/checksum.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@

#include "checksums/highwayhash.h"
#include "checksums/blake2/blake2.h"
#include "checksums/blake3/blake3.h"
#include "checksums/metrohash.h"
#include "checksums/murmur3.h"
#include "checksums/sha3/sha3.h"
Expand Down Expand Up @@ -563,7 +564,7 @@ RM_DIGEST_DEFINE_SHA3(512)
// blake hashes //
///////////////////////////

#define CREATE_BLAKE_INTERFACE(ALGO, ALGO_BIG) \
#define CREATE_BLAKE2_INTERFACE(ALGO, ALGO_BIG) \
\
static ALGO##_state *rm_digest_##ALGO##_new(void) { \
ALGO##_state *state = g_slice_new(ALGO##_state); \
Expand Down Expand Up @@ -595,10 +596,58 @@ RM_DIGEST_DEFINE_SHA3(512)
.copy = (RmDigestCopyFunc)rm_digest_##ALGO##_copy, \
.steal = (RmDigestStealFunc)rm_digest_##ALGO##_steal};

CREATE_BLAKE_INTERFACE(blake2b, BLAKE2B);
CREATE_BLAKE_INTERFACE(blake2bp, BLAKE2B);
CREATE_BLAKE_INTERFACE(blake2s, BLAKE2S);
CREATE_BLAKE_INTERFACE(blake2sp, BLAKE2S);
CREATE_BLAKE2_INTERFACE(blake2b, BLAKE2B);
CREATE_BLAKE2_INTERFACE(blake2bp, BLAKE2B);
CREATE_BLAKE2_INTERFACE(blake2s, BLAKE2S);
CREATE_BLAKE2_INTERFACE(blake2sp, BLAKE2S);

static blake3_hasher *rm_digest_blake3_new(void) {
blake3_hasher *hasher = g_slice_new(blake3_hasher);
blake3_hasher_init(hasher);
return hasher;
}

static void rm_digest_blake3_free(blake3_hasher *hasher) {
g_slice_free(blake3_hasher, hasher);
}

static blake3_hasher *rm_digest_blake3_copy(blake3_hasher *hasher) {
blake3_hasher *copy = g_slice_copy(sizeof(blake3_hasher), hasher);
return copy;
}

static void rm_digest_blake3_steal(blake3_hasher *hasher, guint8 *result) {
blake3_hasher *copy = rm_digest_blake3_copy(hasher);
blake3_hasher_finalize(copy, result, BLAKE3_OUT_LEN);
rm_digest_blake3_free(copy);
}

static void rm_digest_blake3_512_steal(blake3_hasher *hasher, guint8 *result) {
blake3_hasher *copy = rm_digest_blake3_copy(hasher);
blake3_hasher_finalize(copy, result, 64);
rm_digest_blake3_free(copy);
}

static const RmDigestInterface blake3_interface = {
.name = "blake3",
.bits = 8 * BLAKE3_OUT_LEN,
.len = NULL,
.new = (RmDigestNewFunc)rm_digest_blake3_new,
.free = (RmDigestFreeFunc)rm_digest_blake3_free,
.update = (RmDigestUpdateFunc)blake3_hasher_update,
.copy = (RmDigestCopyFunc)rm_digest_blake3_copy,
.steal = (RmDigestStealFunc)rm_digest_blake3_steal};

static const RmDigestInterface blake3_512_interface = {
.name = "blake3_512",
.bits = 512,
.len = NULL,
.new = (RmDigestNewFunc)rm_digest_blake3_new,
.free = (RmDigestFreeFunc)rm_digest_blake3_free,
.update = (RmDigestUpdateFunc)blake3_hasher_update,
.copy = (RmDigestCopyFunc)rm_digest_blake3_copy,
.steal = (RmDigestStealFunc)rm_digest_blake3_512_steal};


///////////////////////////
// ext hash //
Expand Down Expand Up @@ -826,6 +875,8 @@ static const RmDigestInterface *rm_digest_get_interface(RmDigestType type) {
[RM_DIGEST_BLAKE2B] = &blake2b_interface,
[RM_DIGEST_BLAKE2SP] = &blake2sp_interface,
[RM_DIGEST_BLAKE2BP] = &blake2bp_interface,
[RM_DIGEST_BLAKE3] = &blake3_interface,
[RM_DIGEST_BLAKE3_512] = &blake3_512_interface,
[RM_DIGEST_EXT] = &ext_interface,
[RM_DIGEST_CUMULATIVE] = &cumulative_interface,
[RM_DIGEST_PARANOID] = &paranoid_interface,
Expand Down
2 changes: 2 additions & 0 deletions lib/checksum.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ typedef enum RmDigestType {
RM_DIGEST_BLAKE2B,
RM_DIGEST_BLAKE2SP /* Parallel version of BLAKE2P */,
RM_DIGEST_BLAKE2BP /* Parallel version of BLAKE2S */,
RM_DIGEST_BLAKE3,
RM_DIGEST_BLAKE3_512,
RM_DIGEST_XXHASH,
RM_DIGEST_HIGHWAY64,
RM_DIGEST_HIGHWAY128,
Expand Down
Loading

0 comments on commit bdb591f

Please sign in to comment.