Skip to content

Commit f7ebbbf

Browse files
magnumrippermagnumripper
committed
configure/make: Improve plugin handling
Move the plugin logic back to Makefile where it belongs, so we can add or remove plugins without re-running configure. Add logic for fast detection of added or removed plugins. See #3566
1 parent 79b652b commit f7ebbbf

File tree

8 files changed

+40
-245
lines changed

8 files changed

+40
-245
lines changed

‎.gitignore‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*.sam
1717

1818
*.o
19+
*.a
1920
*.exe
2021
*.dSYM
2122
.DS_Store
@@ -78,7 +79,7 @@ run/zip2john
7879
.deps/
7980

8081
src/.gdbinit
81-
src/mbedtls/aes.a
82+
src/.plugin_fmt_list
8283
src/arch.h
8384
src/all_tests.lst
8485
src/autoconfig-stamp-h
@@ -98,9 +99,6 @@ src/fmt_registers.h
9899
src/generic.h
99100
src/john_build_rule.h
100101
src/memdbg_defines.h
101-
src/secp256k1/secp256k1.a
102-
src/ed25519-donna/ed25519-donna.a
103-
src/poly1305-donna/poly1305-donna.a
104102
src/stamp-h1
105103
src/version.h
106104
src/version.h.new

‎src/Makefile.in‎

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ TR = tr
3232
SED = @SED@
3333
FIND = @FIND@
3434
NULL = /dev/null
35-
SORT = @SORT@
35+
SORT = LC_ALL=C @SORT@
3636
STRIP = @STRIP@
3737
PEFLAGS = peflags --dynamicbase=true --nxcompat=true
3838

@@ -72,7 +72,11 @@ OPT_NORMAL = @OPT_NORMAL_FLAGS@
7272
OPT_INLINE = @OPT_INLINE_FLAGS@
7373
#
7474

75-
PLUGFORMATS_OBJS = @PLUGFORMATS_OBJS@
75+
PLUGFORMATS_SRCS := $(filter-out opencl_%_fmt_plug.c, $(wildcard *_fmt_plug.c))
76+
PLUGIN_OBJS := $(filter-out opencl_%_plug.c, $(wildcard *_plug.c))
77+
PLUGIN_OBJS := $(PLUGIN_OBJS:.c=.o)
78+
OPENCL_PLUGIN_OBJS := $(wildcard opencl*_plug.c)
79+
OPENCL_PLUGIN_OBJS := $(OPENCL_PLUGIN_OBJS:.c=.o)
7680

7781
JOHN_OBJS = \
7882
jumbo.o john_mpi.o \
@@ -100,7 +104,7 @@ JOHN_OBJS = \
100104
@UNRAR_OBJS@ \
101105
rar2john.o \
102106
zip2john.o pkzip.o \
103-
$(PLUGFORMATS_OBJS) \
107+
$(PLUGIN_OBJS) \
104108
dyna_salt.o dummy.o \
105109
gost.o \
106110
gpu_common.o \
@@ -125,9 +129,7 @@ JOHN_OBJS = \
125129
showformats.o \
126130
color.o
127131

128-
OCL_OBJS = opencl_common.o opencl_dynamic_loader.o opencl_autotune.o bt.o bt_hash_type_64.o bt_hash_type_128.o bt_hash_type_192.o bt_twister.o
129-
130-
OPENCL_PLUGFORMATS_OBJS = @OPENCL_PLUGFORMATS_OBJS@
132+
OPENCL_OBJS = opencl_common.o opencl_dynamic_loader.o opencl_autotune.o bt.o bt_hash_type_64.o bt_hash_type_128.o bt_hash_type_192.o bt_twister.o
131133

132134
ZTEX_OBJS = ztex_descrypt.o ztex_bcrypt.o ztex_sha512crypt.o ztex_drupal7.o ztex_sha256crypt.o ztex_md5crypt.o ztex_phpass.o ztex_common.o
133135

@@ -154,7 +156,8 @@ WITH_OPENCL=@CL_LIBS@
154156
ifdef WITH_OPENCL
155157
CFLAGS += -DHAVE_OPENCL @CL_CFLAGS@
156158
CFLAGS_MAIN += -DHAVE_OPENCL @CL_CFLAGS@
157-
JOHN_OBJS += $(OCL_OBJS) $(OPENCL_PLUGFORMATS_OBJS)
159+
PLUGFORMATS_SRCS += $(wildcard opencl_*_fmt_plug.c)
160+
JOHN_OBJS += $(OPENCL_OBJS) $(OPENCL_PLUGIN_OBJS)
158161

159162
# core OpenCL dependencies
160163
CL_DEVICE_HEADER = ../run/opencl/opencl_device_info.h
@@ -191,7 +194,6 @@ endif
191194
#
192195
#########################################################
193196
default:
194-
$(MAKE) find_version
195197
@$(MAKE) $(PROJ) \
196198
JOHN_OBJS="$(JOHN_OBJS) @CC_ASM_OBJS@"
197199

@@ -234,15 +236,30 @@ options.o: options.c options.h version.h
234236

235237
listconf.o: listconf.c listconf.h version.h
236238

237-
version.h: find_version
239+
# Phony without the side-effects
240+
FORCE:
238241

239-
find_version:
240-
echo "#define JTR_GIT_VERSION $(JTR_GIT_VERSION)" > version.h.new
241-
diff >/dev/null 2>/dev/null version.h.new version.h && $(RM) version.h.new || $(MV) version.h.new version.h
242+
version.h.new: FORCE
243+
echo "#define JTR_GIT_VERSION $(JTR_GIT_VERSION)" > $@
242244

243-
SUBDIRS = mbedtls secp256k1 ed25519-donna poly1305-donna @ZTEX_SUBDIRS@
245+
version.h: version.h.new
246+
cmp -s $@.new $@ && $(RM) $@.new || $(MV) $@.new $@
247+
248+
# Rebuild the list of plugins, but don't bump timestamp unless content changed.
249+
.plugin_fmt_list: FORCE
250+
@echo $(PLUGFORMATS_SRCS) > $@.new
251+
cmp -s $@.new $@ && $(RM) $@.new || $(MV) $@.new $@
252+
253+
# These are SLOW nowadays, lots of plugins.
254+
fmt_externs.h: .plugin_fmt_list
255+
$(CC) -DFMT_EXTERNS_H -E -P $(CFLAGS) $(PLUGFORMATS_SRCS) | grep -F "extern struct fmt_main" | $(SORT) -f > $@
244256

245-
.PHONY: subdirs $(SUBDIRS) find_version
257+
fmt_registers.h: .plugin_fmt_list
258+
$(CC) -DFMT_REGISTERS_H -E -P $(CFLAGS) $(PLUGFORMATS_SRCS) | grep -F "john_register_one" | $(SORT) -f > $@
259+
260+
.PHONY: subdirs $(SUBDIRS)
261+
262+
SUBDIRS = mbedtls secp256k1 ed25519-donna poly1305-donna @ZTEX_SUBDIRS@
246263

247264
subdirs: $(SUBDIRS)
248265

@@ -415,7 +432,7 @@ poly1305-donna/poly1305-donna.a:
415432
../run/tgtsnarf@EXE_EXT@: tgtsnarf.o
416433
$(LD) tgtsnarf.o $(LDFLAGS) @OPENMP_CFLAGS@ -o $@
417434

418-
john.o: john.c autoconfig.h os.h os-autoconf.h jumbo.h arch.h params.h openssl_local_overrides.h misc.h path.h memory.h list.h tty.h signals.h common.h idle.h formats.h dyna_salt.h loader.h logger.h status.h recovery.h options.h getopt.h config.h bench.h fuzz.h charset.h single.h wordlist.h prince.h inc.h mask.h mkv.h mkvlib.h external.h compiler.h batch.h dynamic.h simd-intrinsics.h pseudo_intrinsics.h aligned.h simd-intrinsics-load-flags.h dynamic_compiler.h fake_salts.h listconf.h crc32.h john_mpi.h regex.h unicode.h $(CL_COMMON_HEADER) $(CL_DEVICE_HEADER) john_build_rule.h fmt_externs.h fmt_registers.h subsets.h
435+
john.o: john.c john.h fmt_externs.h fmt_registers.h
419436
$(CC) $(CFLAGS_MAIN) $(OPT_NORMAL) -O1 $*.c
420437

421438
path.o: path.c path.h autoconfig.h arch.h params.h misc.h memory.h
@@ -695,11 +712,11 @@ distclean: clean kernel-cache-clean-john
695712
(cd $$dir && $(MAKE) distclean) \
696713
|| case "$(MFLAGS)" in *k*) fail=yes;; *) exit 1;; esac; \
697714
done && test -z "$$fail"
698-
$(RM) Makefile autoconfig.h config.status config.cache \
699-
config.log autoconfig-stamp-h autoconfig-stamp-h-in stamp-h1
715+
$(RM) Makefile autoconfig.h config.status config.cache arch.h \
716+
config.log autoconfig-stamp-h autoconfig-stamp-h-in stamp-h1 \
717+
fmt_registers.h fmt_externs.h dynamic_big_crypt.c john_build_rule.h \
718+
version.h .plugin_fmt_list
700719
$(LN) Makefile.stub Makefile
701-
$(RM) arch.h
702-
$(RM) fmt_registers.h fmt_externs.h dynamic_big_crypt.c john_build_rule.h version.h version.h.new
703720

704721
strip: default
705722
@echo Stripping executables.

‎src/configure‎

Lines changed: 0 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -625,9 +625,6 @@ ac_includes_default="\
625625

626626
ac_subst_vars='LTLIBOBJS
627627
HOST_OS
628-
PLUGFORMATS_DEPS
629-
OPENCL_PLUGFORMATS_OBJS
630-
PLUGFORMATS_OBJS
631628
M4_INCLUDES
632629
HAVE_LIBFUZZER
633630
HAVE_FUZZ
@@ -755,7 +752,6 @@ enable_asan
755752
enable_ubsan
756753
enable_ubsantrap
757754
enable_simd
758-
enable_plugin_dependencies
759755
enable_werror
760756
enable_openmp_for_fast_formats
761757
enable_mpi
@@ -1384,9 +1380,6 @@ Optional Features:
13841380
--disable-simd, --enable-simd=foo
13851381
* Build forcing SIMD logic to be disabled, or force
13861382
use of "foo" for SIMD (eg. --enable-simd=altivec)
1387-
--disable-plugin-dependencies
1388-
Do not create best-effort Makefile dependencies for
1389-
plugins
13901383
--enable-werror Treat warnings as errors
13911384
--enable-openmp-for-fast-formats
13921385
Enable OpenMP for fast formats (usually a bad idea -
@@ -3462,13 +3455,6 @@ else
34623455
simd=yes
34633456
fi
34643457

3465-
# Check whether --enable-plugin-dependencies was given.
3466-
if test "${enable_plugin_dependencies+set}" = set; then :
3467-
enableval=$enable_plugin_dependencies; plug_deps=$enableval
3468-
else
3469-
plug_deps=yes
3470-
fi
3471-
34723458

34733459
# Check whether --enable-werror was given.
34743460
if test "${enable_werror+set}" = set; then :
@@ -16611,27 +16597,6 @@ if test x$with_flock != xno; then
1661116597
fi
1661216598

1661316599

16614-
if test x$using_cl = xyes ; then
16615-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating *_plug.c and OpenCL object rules" >&5
16616-
$as_echo "$as_me: creating *_plug.c and OpenCL object rules" >&6;}
16617-
else
16618-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating *_plug.c rules" >&5
16619-
$as_echo "$as_me: creating *_plug.c rules" >&6;}
16620-
fi
16621-
if test "`echo *_plug.c`" != "*_plug.c"; then
16622-
PLUGFORMATS_OBJS=`echo *_plug.c | sed 's/opencl[A-Za-z0-9_\-]*\.c //g;s/\.c/.o/g'`
16623-
16624-
OPENCL_PLUGFORMATS_OBJS=`echo opencl*_plug.c | sed 's/\.c/.o/g'`
16625-
16626-
if test "x$plug_deps" = xyes -a "x$PERL" != x ; then
16627-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating Makefile dependencies" >&5
16628-
$as_echo "$as_me: creating Makefile dependencies" >&6;}
16629-
16630-
PLUGFORMATS_DEPS=`$PERL ./plugin_deps.pl 2>/dev/null *_plug.c`
16631-
16632-
fi
16633-
fi
16634-
1663516600
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating ./john_build_rule.h" >&5
1663616601
$as_echo "$as_me: creating ./john_build_rule.h" >&6;}
1663716602
echo "#define JOHN_BLD \"${host_os} ${CPU_BIT_STR}-bit${using_x32} ${host_cpu} ${SIMD_NAME} AC\"" > john_build_rule.h
@@ -18019,43 +17984,6 @@ fi
1801917984

1802017985
CFLAGS_EXTRA=$CFLAGS_EXTRA_BACKUP
1802117986

18022-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating ./fmt_externs.h" >&5
18023-
$as_echo "$as_me: creating ./fmt_externs.h" >&6;}
18024-
rm -f fmt_externs.h
18025-
18026-
CFLAGS_EX2=
18027-
if test "x$simd" = xno ; then
18028-
CFLAGS_EX2=-DJOHN_NO_SIMD
18029-
fi
18030-
18031-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
18032-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_EXTERNS_H *_fmt_plug.c | LC_ALL=C $GREP "extern struct fmt_main" | LC_ALL=C $SORT -f > fmt_externs.h
18033-
fi
18034-
if test -f fmt_externs.h -a ! -s fmt_externs.h; then
18035-
cp Makefile.stub Makefile
18036-
as_fn_error $? "The build of fmt_externs.h failed. Can not continue!" "$LINENO" 5
18037-
fi
18038-
if test ! -f fmt_externs.h; then
18039-
echo >fmt_externs.h
18040-
fi
18041-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating ./fmt_registers.h" >&5
18042-
$as_echo "$as_me: creating ./fmt_registers.h" >&6;}
18043-
rm -f fmt_registers.h
18044-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
18045-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H *_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f > fmt_registers.h
18046-
fi
18047-
if test -f fmt_registers.h -a ! -s fmt_registers.h; then
18048-
cp Makefile.stub Makefile
18049-
as_fn_error $? "The build of fmt_registers.h failed. Can not continue!" "$LINENO" 5
18050-
fi
18051-
if test ! -f fmt_registers.h; then
18052-
echo >fmt_registers.h
18053-
fi
18054-
18055-
if test "`echo opencl_*_fmt_plug.c`" != "opencl_*_fmt_plug.c"; then
18056-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H opencl_*_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f >> fmt_registers.h
18057-
fi
18058-
1805917987

1806017988
if test "x$ac_cv_prog_c_openmp" != x -a "x$enable_openmp" != xno -a "x$ac_cv_prog_c_openmp" != xunsupported ; then
1806117989
if test "x$ompfast" = "xyes"; then

‎src/configure.ac‎

Lines changed: 0 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ AC_ARG_ENABLE([asan], [AS_HELP_STRING([--enable-asan], [* Build with AddressSani
6464
AC_ARG_ENABLE([ubsan], [AS_HELP_STRING([--enable-ubsan], [* Build with UndefinedBehaviorSanitizer])], [ubsan=$enableval], [ubsan=no])
6565
AC_ARG_ENABLE([ubsantrap], [AS_HELP_STRING([--enable-ubsantrap], [* Build with UndefinedBehaviorSanitizer, crashes on failure])], [ubsantrap=$enableval], [ubsantrap=no])
6666
AC_ARG_ENABLE([simd], [AS_HELP_STRING([--disable-simd, --enable-simd=foo], [* Build forcing SIMD logic to be disabled, or force use of "foo" for SIMD (eg. --enable-simd=altivec)])], [simd=$enableval], [simd=yes])
67-
AC_ARG_ENABLE([plugin-dependencies], [AS_HELP_STRING([--disable-plugin-dependencies], [Do not create best-effort Makefile dependencies for plugins])], [plug_deps=$enableval], [plug_deps=yes])
6867

6968
AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], [Treat warnings as errors])], [werror=$enableval], [werror=no])
7069

@@ -1040,30 +1039,6 @@ dnl AC_DEFINE([DYNAMIC_DISABLED], 1, [Disables Dynamic format completely])
10401039
dnl fi
10411040
dnl fi
10421041

1043-
dnl ======================================================================
1044-
dnl Code to create @PLUGFORMATS_OBJS@ and
1045-
dnl john_buildrule.h moved from Makefile.in to here.
1046-
dnl don't build opencl_*plug.c unless an opencl build
1047-
dnl ======================================================================
1048-
if test x$using_cl = xyes ; then
1049-
AC_MSG_NOTICE([creating *_plug.c and OpenCL object rules])
1050-
else
1051-
AC_MSG_NOTICE([creating *_plug.c rules])
1052-
fi
1053-
if test "`echo *_plug.c`" != "*_plug.c"; then
1054-
AC_SUBST([PLUGFORMATS_OBJS],[`echo *_plug.c | sed 's/opencl[[A-Za-z0-9_\-]]*\.c //g;s/\.c/.o/g'`])
1055-
AC_SUBST([OPENCL_PLUGFORMATS_OBJS],[`echo opencl*_plug.c | sed 's/\.c/.o/g'`])
1056-
if test "x$plug_deps" = xyes -a "x$PERL" != x ; then
1057-
AC_MSG_NOTICE([creating Makefile dependencies])
1058-
1059-
dnl The "plugin_deps.pl" script isn't clever enough to detect when
1060-
dnl #include statements are ifdef'ed out, so we direct the warnings
1061-
dnl to /dev/null. The mbedTLS directory has lots of inactive code
1062-
dnl which otherwise triggering warnings for "missing" files.
1063-
AC_SUBST([PLUGFORMATS_DEPS],[`$PERL ./plugin_deps.pl 2>/dev/null *_plug.c`])
1064-
fi
1065-
fi
1066-
10671042
AC_MSG_NOTICE([creating ./john_build_rule.h])
10681043
echo "#define JOHN_BLD \"${host_os} ${CPU_BIT_STR}-bit${using_x32} ${host_cpu} ${SIMD_NAME} AC\"" > john_build_rule.h
10691044

@@ -1085,50 +1060,6 @@ AC_OUTPUT(,echo "timestamp from configure.in" > autoconfig-stamp-h)
10851060

10861061
CFLAGS_EXTRA=$CFLAGS_EXTRA_BACKUP
10871062

1088-
dnl ======================================================================
1089-
dnl Create fmt_externs.h and fmt_registers.h
1090-
dnl This needs arch.h to be linked so can't happen before
1091-
dnl AC_OUTPUT
1092-
dnl ======================================================================
1093-
AC_MSG_NOTICE([creating ./fmt_externs.h])
1094-
rm -f fmt_externs.h
1095-
1096-
CFLAGS_EX2=
1097-
if test "x$simd" = xno ; then
1098-
CFLAGS_EX2=-DJOHN_NO_SIMD
1099-
fi
1100-
1101-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
1102-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_EXTERNS_H *_fmt_plug.c | LC_ALL=C $GREP "extern struct fmt_main" | LC_ALL=C $SORT -f > fmt_externs.h
1103-
fi
1104-
dnl if the file was not created, then bail.
1105-
if test -f fmt_externs.h -a ! -s fmt_externs.h; then
1106-
cp Makefile.stub Makefile
1107-
AC_MSG_ERROR([The build of fmt_externs.h failed. Can not continue!])
1108-
fi
1109-
dnl if file does not exist, then create a blank one (i.e. no *_plug.c files)
1110-
if test ! -f fmt_externs.h; then
1111-
echo >fmt_externs.h
1112-
fi
1113-
AC_MSG_NOTICE([creating ./fmt_registers.h])
1114-
rm -f fmt_registers.h
1115-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
1116-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H *_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f > fmt_registers.h
1117-
fi
1118-
dnl if the file was not created, then bail.
1119-
if test -f fmt_registers.h -a ! -s fmt_registers.h; then
1120-
cp Makefile.stub Makefile
1121-
AC_MSG_ERROR([The build of fmt_registers.h failed. Can not continue!])
1122-
fi
1123-
dnl if file does not exist, then create a blank one (i.e. no *_plug.c files)
1124-
if test ! -f fmt_registers.h; then
1125-
echo >fmt_registers.h
1126-
fi
1127-
1128-
if test "`echo opencl_*_fmt_plug.c`" != "opencl_*_fmt_plug.c"; then
1129-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H opencl_*_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f >> fmt_registers.h
1130-
fi
1131-
11321063
dnl ======================================================================
11331064
dnl ONLY _COSMETICAL_ OUTPUT STUFF BELOW THIS LINE
11341065
dnl ======================================================================

‎src/opencl_rar_fmt_plug.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ static size_t max_blob_size;
129129
static int salt_single;
130130

131131
#define RAR_OPENCL_FORMAT
132-
#include "rar_common.c"
132+
#include "rar_common.h"
133133

134134
static void release_clobj(void);
135135

0 commit comments

Comments
 (0)