From a667a7f02066ce1a6de568d75f1062f525b3c2f0 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 26 Sep 2024 16:27:10 +0200 Subject: [PATCH 001/277] wget: fix compile warnings when WGET_FTP is not selected Signed-off-by: Denys Vlasenko --- networking/wget.c | 9 ++++++--- util-linux/hwclock.c | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/networking/wget.c b/networking/wget.c index 512bebfc2f..ec3767793d 100644 --- a/networking/wget.c +++ b/networking/wget.c @@ -1178,7 +1178,9 @@ static void download_one_url(const char *url) /*G.content_len = 0; - redundant, got_clen = 0 is enough */ G.got_clen = 0; G.chunked = 0; - if (use_proxy || target.protocol[0] != 'f' /*not ftp[s]*/) { + if (!ENABLE_FEATURE_WGET_FTP + || use_proxy || target.protocol[0] != 'f' /*not ftp[s]*/ + ) { /* * HTTP session */ @@ -1447,14 +1449,15 @@ However, in real world it was observed that some web servers /* For HTTP, data is pumped over the same connection */ dfp = sfp; - } else { + } #if ENABLE_FEATURE_WGET_FTP + else { /* * FTP session */ sfp = prepare_ftp_session(&dfp, &target, lsa); -#endif } +#endif free(lsa); diff --git a/util-linux/hwclock.c b/util-linux/hwclock.c index e6f0043d05..c3fd0eb575 100644 --- a/util-linux/hwclock.c +++ b/util-linux/hwclock.c @@ -366,7 +366,7 @@ static void set_rtc_param(const char **pp_rtcname, char *rtc_param) /* handle param name */ eq = strchr(rtc_param, '='); if (!eq) - bb_error_msg_and_die("expected ="); + bb_simple_error_msg_and_die("expected ="); *eq = '\0'; param.param = resolve_rtc_param_alias(rtc_param); *eq = '='; From be7d1b7b1701d225379bc1665487ed0871b592a5 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 26 Sep 2024 23:31:15 +0200 Subject: [PATCH 002/277] Bump version to 1.37.0 Signed-off-by: Denys Vlasenko --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e67553d5fc..1ec8ac3fd0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION = 1 PATCHLEVEL = 37 SUBLEVEL = 0 -EXTRAVERSION = .git +EXTRAVERSION = NAME = Unnamed # *DOCUMENTATION* From a023f392655d55e6e5042fedc57b89a89bf8abd4 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 26 Sep 2024 23:33:06 +0200 Subject: [PATCH 003/277] Start 1.38.0 development cycle Signed-off-by: Denys Vlasenko --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1ec8ac3fd0..9550c391a4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION = 1 -PATCHLEVEL = 37 +PATCHLEVEL = 38 SUBLEVEL = 0 -EXTRAVERSION = +EXTRAVERSION = .git NAME = Unnamed # *DOCUMENTATION* From e5b0547ac834dcd34c72df9aee1cf9c326c14b6f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 27 Sep 2024 19:35:50 +0200 Subject: [PATCH 004/277] lineedit: make save_history() FAST_FUNC function old new delta save_history 267 266 -1 hush_exit 98 97 -1 exitshell 140 138 -2 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/3 up/down: 0/-4) Total: -4 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 10 +++++----- libbb/lineedit.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 01cdb1bdc8..67e7bf7b99 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1998,11 +1998,11 @@ enum { FOR_SHELL = DO_HISTORY | TAB_COMPLETION | USERNAME_COMPLETION | LI_INTERRUPTIBLE, }; line_input_t *new_line_input_t(int flags) FAST_FUNC; -#if ENABLE_FEATURE_EDITING_SAVEHISTORY +# if ENABLE_FEATURE_EDITING_SAVEHISTORY void free_line_input_t(line_input_t *n) FAST_FUNC; -#else -# define free_line_input_t(n) free(n) -#endif +# else +# define free_line_input_t(n) free(n) +# endif /* * maxsize must be >= 2. * Returns: @@ -2013,7 +2013,7 @@ void free_line_input_t(line_input_t *n) FAST_FUNC; int read_line_input(line_input_t *st, const char *prompt, char *command, int maxsize) FAST_FUNC; void show_history(const line_input_t *st) FAST_FUNC; # if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT -void save_history(line_input_t *st); +void save_history(line_input_t *st) FAST_FUNC; # endif #else #define MAX_HISTORY 0 diff --git a/libbb/lineedit.c b/libbb/lineedit.c index 543a3f11c3..314fedf5c2 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -1555,7 +1555,7 @@ static void load_history(line_input_t *st_parm) } # if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT -void save_history(line_input_t *st) +void FAST_FUNC save_history(line_input_t *st) { FILE *fp; From 2c232f1e60fc42230c4e14bc7c548ac0e6593173 Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Tue, 9 Apr 2024 09:50:59 +0100 Subject: [PATCH 005/277] libbb: use full_write1_str() to shrink busybox_main() There are two calls to dup2() in busybox_main(). These were introduced to coerce full_write2_str() into writing to stdout. The relevant commits were: 21278dff7 (busybox: do not print help to fd 2, print it to fd 1) and 5a7c72015 (busybox --list option. +140 bytes. Rob wanted it.) Later, in commit 729ecb87b (bbconfig: make it independent from printf functions), the function full_write1_str() was added. Using this in busybox_main() allows us to drop the dup2() calls. function old new delta run_applet_and_exit 796 760 -36 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-36) Total: -36 bytes Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- libbb/appletlib.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/libbb/appletlib.c b/libbb/appletlib.c index d9cc48423b..ad373ae1c2 100644 --- a/libbb/appletlib.c +++ b/libbb/appletlib.c @@ -776,10 +776,9 @@ int busybox_main(int argc UNUSED_PARAM, char **argv) help: output_width = get_terminal_width(2); - dup2(1, 2); - full_write2_str(bb_banner); /* reuse const string */ - full_write2_str(" multi-call binary.\n"); /* reuse */ - full_write2_str( + full_write1_str(bb_banner); /* reuse const string */ + full_write1_str(" multi-call binary.\n"); /* reuse */ + full_write1_str( "BusyBox is copyrighted by many authors between 1998-2015.\n" "Licensed under GPLv2. See source distribution for detailed\n" "copyright notices.\n" @@ -817,20 +816,20 @@ int busybox_main(int argc UNUSED_PARAM, char **argv) while (*a) { int len2 = strlen(a) + 2; if (col >= (int)output_width - len2) { - full_write2_str(",\n"); + full_write1_str(",\n"); col = 0; } if (col == 0) { col = 6; - full_write2_str("\t"); + full_write1_str("\t"); } else { - full_write2_str(", "); + full_write1_str(", "); } - full_write2_str(a); + full_write1_str(a); col += len2; a += len2 - 1; } - full_write2_str("\n"); + full_write1_str("\n"); return 0; } @@ -850,14 +849,13 @@ int busybox_main(int argc UNUSED_PARAM, char **argv) if (is_prefixed_with(argv[1], "--list")) { unsigned i = 0; const char *a = applet_names; - dup2(1, 2); while (*a) { # if ENABLE_FEATURE_INSTALLER if (argv[1][6]) /* --list-full? */ - full_write2_str(install_dir[APPLET_INSTALL_LOC(i)] + 1); + full_write1_str(install_dir[APPLET_INSTALL_LOC(i)] + 1); # endif - full_write2_str(a); - full_write2_str("\n"); + full_write1_str(a); + full_write1_str("\n"); i++; while (*a++ != '\0') continue; From dbd14c4a42454b61a5474f6243865a6ec113e60b Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Tue, 9 Apr 2024 09:51:48 +0100 Subject: [PATCH 006/277] libbb: send usage messages to correct stream POSIX generally requires normal output to go to stdout and diagnostic (i.e. error) output to go to stderr. When usage messages for BusyBox applets are specifically requested by the user (e.g. 'find --help') they should go to stdout; when they're emitted due to an error they should go to stderr. function old new delta bb_show_usage 148 146 -2 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-2) Total: -2 bytes Signed-off-by: Avi Halachmi Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- libbb/appletlib.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/libbb/appletlib.c b/libbb/appletlib.c index ad373ae1c2..d2e5900b5f 100644 --- a/libbb/appletlib.c +++ b/libbb/appletlib.c @@ -127,17 +127,19 @@ static const char packed_usage[] ALIGN1 = { PACKED_USAGE }; void FAST_FUNC bb_show_usage(void) { if (ENABLE_SHOW_USAGE) { + ssize_t FAST_FUNC (*full_write_fn)(const char *) = + xfunc_error_retval ? full_write2_str : full_write1_str; #ifdef SINGLE_APPLET_STR /* Imagine that this applet is "true". Dont link in printf! */ const char *usage_string = unpack_usage_messages(); if (usage_string) { if (*usage_string == '\b') { - full_write2_str("No help available\n"); + full_write_fn("No help available\n"); } else { - full_write2_str("Usage: "SINGLE_APPLET_STR" "); - full_write2_str(usage_string); - full_write2_str("\n"); + full_write_fn("Usage: "SINGLE_APPLET_STR" "); + full_write_fn(usage_string); + full_write_fn("\n"); } if (ENABLE_FEATURE_CLEAN_UP) dealloc_usage_messages((char*)usage_string); @@ -153,19 +155,19 @@ void FAST_FUNC bb_show_usage(void) while (*p++) continue; ap--; } - full_write2_str(bb_banner); - full_write2_str(" multi-call binary.\n"); /* common string */ + full_write_fn(bb_banner); + full_write_fn(" multi-call binary.\n"); /* common string */ if (*p == '\b') - full_write2_str("\nNo help available\n"); + full_write_fn("\nNo help available\n"); else { - full_write2_str("\nUsage: "); - full_write2_str(applet_name); + full_write_fn("\nUsage: "); + full_write_fn(applet_name); if (p[0]) { if (p[0] != '\n') - full_write2_str(" "); - full_write2_str(p); + full_write_fn(" "); + full_write_fn(p); } - full_write2_str("\n"); + full_write_fn("\n"); } if (ENABLE_FEATURE_CLEAN_UP) dealloc_usage_messages((char*)usage_string); @@ -268,8 +270,10 @@ void lbb_prepare(const char *applet && !(ENABLE_TRUE && strcmp(applet_name, "true") == 0) && !(ENABLE_FALSE && strcmp(applet_name, "false") == 0) && !(ENABLE_ECHO && strcmp(applet_name, "echo") == 0) - ) + ) { + xfunc_error_retval = 0; bb_show_usage(); + } } #endif } From 480a07bd6828285628abbbe3fe8e5e3b25ce1a92 Mon Sep 17 00:00:00 2001 From: Sam James Date: Tue, 23 Apr 2024 21:10:18 +0100 Subject: [PATCH 007/277] fixdep: add fstat error handling When `fstat` fails, `st` is left uninitialised. In our case, Ben Kohler noticed our release media builds were failing in Gentoo on x86 when building busybox with occasional SIGBUS. This turned out to be EOVERFLOW (from 32-bit ino_t) which wasn't being reported because nothing was checking the return value from `fstat`. Fix that to avoid UB (use of uninit var) and to give a more friendly error to the user. This actually turns out to be fixed already in the kernel from back in 2010 [0] and 2016 [1]. [0] https://github.com/torvalds/linux/commit/a3ba81131aca243bfecfa78c42edec0cd69f72d6 [1] https://github.com/torvalds/linux/commit/46fe94ad18aa7ce6b3dad8c035fb538942020f2b Reported-by: Ben Kohler Signed-off-by: Sam James Signed-off-by: Denys Vlasenko --- scripts/basic/fixdep.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 66be73aadd..071c3b4073 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -105,6 +105,7 @@ #include #include #include +#include #include #include #include @@ -292,7 +293,10 @@ void do_config_file(char *filename) perror(filename); exit(2); } - fstat(fd, &st); + if (fstat(fd, &st) < 0) { + fprintf(stderr, "fixdep: fstat %s %s\n", filename, strerror(errno)); + exit(2); + } if (st.st_size == 0) { close(fd); return; @@ -368,7 +372,10 @@ void print_deps(void) perror(depfile); exit(2); } - fstat(fd, &st); + if (fstat(fd, &st) < 0) { + fprintf(stderr, "fixdep: fstat %s %s\n", depfile, strerror(errno)); + exit(2); + } if (st.st_size == 0) { fprintf(stderr,"fixdep: %s is empty\n",depfile); close(fd); From 26895db35d4b9160ff5f2fe1f942e1ddbf58747d Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Thu, 18 Jul 2024 14:59:30 +0100 Subject: [PATCH 008/277] ed: fix line insertion before current line. Closes 15081 When text is inserted by insertLine() the lines following the insertion are moved down and the insertion point is made the new current line. To avoid too much scanning of the linked list of lines setCurNum() may use the position of the old current line to determine the location of the new current line. If the insertion point is before the old current line in the file the latter will have been moved down, so its line pointer needs to be adjusted. function old new delta insertLine 162 180 +18 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/0 up/down: 18/0) Total: 18 bytes Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- editors/ed.c | 2 ++ testsuite/ed.tests | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100755 testsuite/ed.tests diff --git a/editors/ed.c b/editors/ed.c index 8ec23d07fc..a02634ec71 100644 --- a/editors/ed.c +++ b/editors/ed.c @@ -345,6 +345,8 @@ static int insertLine(int num, const char *data, int len) lp->prev->next = newLp; lp->prev = newLp; + if (num <= curNum) + curLine = curLine->prev; lastNum++; dirty = TRUE; return setCurNum(num); diff --git a/testsuite/ed.tests b/testsuite/ed.tests new file mode 100755 index 0000000000..475fdc244f --- /dev/null +++ b/testsuite/ed.tests @@ -0,0 +1,21 @@ +#!/bin/sh + +. ./testing.sh + +# testing "test name" "command" "expected result" "file input" "stdin" + +testing "ed insert text before current line" \ + "ed input" "8\n1\ntext\n2\n3\n4\n13\n" "1\n2\n3\n4\n" '2i +text +. +,p +w +q +' + +testing "ed read text before current line" \ + "ed input" "8\n8\n1\n2\n1\n2\n3\n4\n3\n4\n16\n" "1\n2\n3\n4\n" '2r input +,p +w +q +' From 39c5c08b28f14e02611d5c79e357fb7432da1e25 Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Sat, 27 Jul 2024 07:51:35 +0100 Subject: [PATCH 009/277] lineedit: use stdout for shell history builtin Commit fd47f0567 (lineedit: print prompt and editing operations to stderr) changed various print statements to output to stderr. The change in show_history() caused the shell history builtin to send its output to stderr. Revert that part of the commit. function old new delta show_history 47 42 -5 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-5) Total: -5 bytes Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- libbb/lineedit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbb/lineedit.c b/libbb/lineedit.c index 314fedf5c2..151208c1c1 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -1463,7 +1463,7 @@ void FAST_FUNC show_history(const line_input_t *st) if (!st) return; for (i = 0; i < st->cnt_history; i++) - fprintf(stderr, "%4d %s\n", i, st->history[i]); + printf("%4d %s\n", i, st->history[i]); } # if ENABLE_FEATURE_EDITING_SAVEHISTORY From 24aa93d5382f2365c35d6e804b748e0a1febf2f6 Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Mon, 23 Sep 2024 12:44:28 +0100 Subject: [PATCH 010/277] ash: reject unknown long options Commit 64f70cc755 (Add --login support) added code in options() to handle the bash-compatible '--login' option. In doing so it committed BusyBox ash to silently accepting all other long options. Restore compatibility with other ash variants by rejecting unknown long options. function old new delta options 589 624 +35 Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- shell/ash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shell/ash.c b/shell/ash.c index bbd7307703..8e029765d2 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -11502,11 +11502,12 @@ options(int *login_sh) if (val && (c == '-')) { /* long options */ if (strcmp(p, "login") == 0) { *login_sh = 1; + break; } /* TODO: --noprofile: e.g. if I want to run emergency shell from sulogin, * I want minimal/no shell init scripts - but it insists on running it as "-ash"... */ - break; + ash_msg_and_raise_error("bad option '%s'", p - 2); } } if (c == 'o') { From dff444bc375e6eb4c6775dbfc753ba71b46e9797 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 28 Sep 2024 03:52:44 +0200 Subject: [PATCH 011/277] hush: whitespace and comment fixes Signed-off-by: Denys Vlasenko --- shell/hush.c | 70 ++++++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/shell/hush.c b/shell/hush.c index 707b77c9cc..6b6ec7c6bd 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -390,7 +390,6 @@ #define BASH_TEST2 (ENABLE_HUSH_BASH_COMPAT && ENABLE_HUSH_TEST) #define BASH_READ_D ENABLE_HUSH_BASH_COMPAT - /* Build knobs */ #define LEAK_HUNTING 0 #define BUILD_AS_NOMMU 0 @@ -411,7 +410,6 @@ */ #define ENABLE_HUSH_DOLLAR_OPS 1 - #if BUILD_AS_NOMMU # undef BB_MMU # undef USE_FOR_NOMMU @@ -640,7 +638,6 @@ typedef enum redir_type { HEREDOC_QUOTED = 2, } redir_type; - struct command { pid_t pid; /* 0 if exited */ unsigned assignment_cnt; /* how many argv[i] are assignments? */ @@ -798,7 +795,6 @@ struct function { }; #endif - /* set -/+o OPT support. (TODO: make it optional) * bash supports the following opts: * allexport off @@ -1046,7 +1042,6 @@ struct globals { G.sa.sa_flags = SA_RESTART; \ } while (0) - /* Function prototypes for builtins */ static int builtin_cd(char **argv) FAST_FUNC; #if ENABLE_HUSH_ECHO @@ -1248,8 +1243,8 @@ static const struct built_in_command bltins2[] ALIGN_PTR = { #endif }; - -/* Debug printouts. +/* + * Debug printouts. */ #if HUSH_DEBUG >= 2 /* prevent disasters with G.debug_indent < 0 */ @@ -1342,8 +1337,8 @@ static void debug_print_strings(const char *prefix, char **vv) # define debug_print_strings(prefix, vv) ((void)0) #endif - -/* Leak hunting. Use hush_leaktool.sh for post-processing. +/* + * Leak hunting. Use hush_leaktool.sh for post-processing. */ #if LEAK_HUNTING static void *xxmalloc(int lineno, size_t size) @@ -1375,8 +1370,8 @@ static void xxfree(void *ptr) # define free(p) xxfree(p) #endif - -/* Syntax and runtime errors. They always abort scripts. +/* + * Syntax and runtime errors. They always abort scripts. * In interactive use they usually discard unparsed and/or unexecuted commands * and return to the prompt. * HUSH_DEBUG >= 2 prints line number in this file where it was detected. @@ -1469,8 +1464,8 @@ static void syntax_error_unexpected_ch(unsigned lineno UNUSED_PARAM, int ch) # define syntax_error_unexpected_ch(ch) syntax_error_unexpected_ch(__LINE__, ch) #endif - -/* Utility functions +/* + * Utility functions */ /* Replace each \x with x in place, return ptr past NUL. */ static char *unbackslash(char *src) @@ -1616,8 +1611,9 @@ static int xdup_CLOEXEC_and_close(int fd, int avoid_fd) return newfd; } - -/* Manipulating HFILEs */ +/* + * Manipulating HFILEs + */ static HFILE *hfopen(const char *name) { HFILE *fp; @@ -1763,8 +1759,8 @@ static int fd_in_HFILEs(int fd) return 0; } - -/* Helpers for setting new $n and restoring them back +/* + * Helpers for setting new $n and restoring them back */ typedef struct save_arg_t { char *sv_argv0; @@ -1804,8 +1800,8 @@ static void restore_G_args(save_arg_t *sv, char **argv) IF_HUSH_SET(G.global_args_malloced = sv->sv_g_malloced;) } - -/* Basic theory of signal handling in shell +/* + * Basic theory of signal handling in shell * ======================================== * This does not describe what hush does, rather, it is current understanding * what it _should_ do. If it doesn't, it's a bug. @@ -2246,7 +2242,9 @@ static int check_and_run_traps(void) return last_sig; } - +/* + * Shell and environment variable support + */ static const char *get_cwd(int force) { if (force || G.cwd == NULL) { @@ -2261,10 +2259,6 @@ static const char *get_cwd(int force) return G.cwd; } - -/* - * Shell and environment variable support - */ static struct variable **get_ptr_to_local_var(const char *name) { struct variable **pp; @@ -2538,7 +2532,6 @@ static int unset_local_var(const char *name) } #endif - /* * Helpers for "var1=val1 var2=val2 cmd" feature */ @@ -2612,7 +2605,6 @@ static void set_vars_and_save_old(char **strings) free(strings); } - /* * Unicode helper */ @@ -2971,7 +2963,6 @@ static void setup_string_in_str(struct in_str *i, const char *s) i->p = s; } - /* * o_string support */ @@ -3703,9 +3694,9 @@ static void free_pipe_list(struct pipe *pi) } } - -/*** Parsing routines ***/ - +/* + * Parsing routines + */ #ifndef debug_print_tree static void debug_print_tree(struct pipe *pi, int lvl) { @@ -4339,7 +4330,6 @@ static int done_word(struct parse_context *ctx) return 0; } - /* Peek ahead in the input to find out if we have a "&n" construct, * as in "2>&1", that represents duplicating a file descriptor. * Return: @@ -4635,7 +4625,6 @@ static int fetch_heredocs(o_string *as_string, struct pipe *pi, int heredoc_cnt, return heredoc_cnt; } - static int run_list(struct pipe *pi); #if BB_MMU #define parse_stream(pstring, heredoc_cnt_ptr, input, end_trigger) \ @@ -6082,9 +6071,9 @@ static struct pipe *parse_stream(char **pstring, } } - -/*** Execution routines ***/ - +/* + * Execution routines + */ /* Expansion can recurse, need forward decls: */ #if !BASH_PATTERN_SUBST && !ENABLE_HUSH_CASE #define expand_string_to_string(str, EXP_flags, do_unbackslash) \ @@ -7347,7 +7336,6 @@ static char **expand_assignments(char **argv, int count) return p; } - static void switch_off_special_sigs(unsigned mask) { unsigned sig = 0; @@ -7565,7 +7553,6 @@ static void re_execute_shell(char ***to_free, const char *s, } #endif /* !BB_MMU */ - static int run_and_free_list(struct pipe *pi); /* Executing from string: eval, sh -c '...' @@ -7796,7 +7783,6 @@ static int process_command_subs(o_string *dest, const char *s) } #endif /* ENABLE_HUSH_TICK */ - static void setup_heredoc(struct redir_struct *redir) { struct fd_pair pair; @@ -8524,7 +8510,6 @@ static int run_function(const struct function *funcp, char **argv) } #endif /* ENABLE_HUSH_FUNCTIONS */ - #if BB_MMU #define exec_builtin(to_free, x, argv) \ exec_builtin(x, argv) @@ -8558,7 +8543,6 @@ static void exec_builtin(char ***to_free, #endif } - static void execvp_or_die(char **argv) NORETURN; static void execvp_or_die(char **argv) { @@ -10184,7 +10168,9 @@ static int run_and_free_list(struct pipe *pi) return rcode; } - +/* + * Initialization and main + */ static void install_sighandlers(unsigned mask) { sighandler_t old_handler; @@ -10861,7 +10847,6 @@ int hush_main(int argc, char **argv) hush_exit(G.last_exitcode); } - /* * Built-ins */ @@ -12340,7 +12325,6 @@ static int FAST_FUNC builtin_memleak(char **argv UNUSED_PARAM) if (l < (unsigned long)p) l = (unsigned long)p; free(p); - # if 0 /* debug */ { struct mallinfo mi = mallinfo(); From bf57f732a5b6842f6fa3e0f90385f039e5d6a92c Mon Sep 17 00:00:00 2001 From: Rudi Heitbaum Date: Sun, 29 Sep 2024 17:57:55 +0000 Subject: [PATCH 012/277] libbb: fix sha1 on !x86 if CONFIG_SHA1_HWACCEL=y fixes non i386 and x86 builds libbb/hash_md5_sha.c: In function 'sha1_end': libbb/hash_md5_sha.c:1316:35: error: 'sha1_process_block64_shaNI' undeclared 1316 | || ctx->process_block == sha1_process_block64_shaNI Signed-off-by: Rudi Heitbaum Signed-off-by: Denys Vlasenko --- libbb/hash_md5_sha.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libbb/hash_md5_sha.c b/libbb/hash_md5_sha.c index 57a801459e..75a61c32c5 100644 --- a/libbb/hash_md5_sha.c +++ b/libbb/hash_md5_sha.c @@ -1313,7 +1313,9 @@ unsigned FAST_FUNC sha1_end(sha1_ctx_t *ctx, void *resbuf) hash_size = 8; if (ctx->process_block == sha1_process_block64 #if ENABLE_SHA1_HWACCEL +# if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) || ctx->process_block == sha1_process_block64_shaNI +# endif #endif ) { hash_size = 5; From 8953429a8253176adda4dcfbed4738e55ee3b8ea Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Fri, 4 Oct 2024 15:32:55 +0200 Subject: [PATCH 013/277] networking/libiproute/iplink.c: fix support for older kernels - The CAN netlink interface has been added in Linux v2.6.31 with only 3 options [1]: CAN_CTRLMODE_LOOPBACK 0x1 /* Loopback mode */ CAN_CTRLMODE_LISTENONLY 0x2 /* Listen-only mode */ CAN_CTRLMODE_3_SAMPLES 0x4 /* Triple sampling mode */ So define the other options. - IFLA_CAN_TERMINATION has been added in Linux 4.11 [2], define it for older kernels. [1] https://github.com/torvalds/linux/blob/v2.6.31/include/linux/can/netlink.h#L80-L82 [2] https://github.com/torvalds/linux/commit/12a6075cabc0d9ffbc0366b44daa22f278606312 Signed-off-by: Thomas Devoogdt Signed-off-by: Denys Vlasenko --- networking/libiproute/iplink.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/networking/libiproute/iplink.c b/networking/libiproute/iplink.c index 37ed114bc9..67602a4663 100644 --- a/networking/libiproute/iplink.c +++ b/networking/libiproute/iplink.c @@ -51,6 +51,27 @@ struct ifla_vlan_flags { }; #endif +#if ENABLE_FEATURE_IP_LINK_CAN +# ifndef CAN_CTRLMODE_ONE_SHOT +# define CAN_CTRLMODE_ONE_SHOT 0x08 /* One-Shot mode */ +# endif +# ifndef CAN_CTRLMODE_BERR_REPORTING +# define CAN_CTRLMODE_BERR_REPORTING 0x10 /* Bus-error reporting */ +# endif +# ifndef CAN_CTRLMODE_FD +# define CAN_CTRLMODE_FD 0x20 /* CAN FD mode */ +# endif +# ifndef CAN_CTRLMODE_PRESUME_ACK +# define CAN_CTRLMODE_PRESUME_ACK 0x40 /* Ignore missing CAN ACKs */ +# endif +# ifndef CAN_CTRLMODE_FD_NON_ISO +# define CAN_CTRLMODE_FD_NON_ISO 0x80 /* CAN FD in non-ISO mode */ +# endif +# ifndef IFLA_CAN_TERMINATION +# define IFLA_CAN_TERMINATION 11 +# endif +#endif + /* taken from linux/sockios.h */ #define SIOCSIFNAME 0x8923 /* set interface name */ From 222802e83362eabd4c631897ff7db5e85955d7c5 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Oct 2024 19:08:10 +0200 Subject: [PATCH 014/277] test: code shrink function old new delta nexpr 813 800 -13 Signed-off-by: Denys Vlasenko --- coreutils/test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreutils/test.c b/coreutils/test.c index 008d90b254..7df7d0fc83 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -760,7 +760,7 @@ static int filstat(char *nm, enum token mode) return ((s.st_mode & i) != 0); } if (mode == FILGZ) - return s.st_size > 0L; + return s.st_size != 0L; /* shorter than "> 0" test */ if (mode == FILUID) return s.st_uid == geteuid(); if (mode == FILGID) From bb5525613ec109aa30d2cb1db84e18aa0b084576 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Oct 2024 19:24:06 +0200 Subject: [PATCH 015/277] test: Invert return value of test_eaccess and rename it to test_st_mode From dash: From: herbert Date: Wed, 2 Mar 2005 22:14:54 +1100 Invert return value of test_eaccess and rename it to test_st_mode. function old new delta nexpr 800 766 -34 Signed-off-by: Denys Vlasenko --- coreutils/test.c | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/coreutils/test.c b/coreutils/test.c index 7df7d0fc83..c02c927450 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -665,34 +665,29 @@ static int is_a_group_member(gid_t gid) return 0; } - -/* Do the same thing access(2) does, but use the effective uid and gid, - and don't make the mistake of telling root that any file is - executable. */ -static int test_eaccess(struct stat *st, int mode) +/* + * Similar to what access(2) does, but uses the effective uid and gid. + * Doesn't make the mistake of telling root that any file is executable. + * Returns non-zero if the file is accessible. + */ +static int test_st_mode(struct stat *st, int mode) { unsigned int euid = geteuid(); if (euid == 0) { /* Root can read or write any file. */ if (mode != X_OK) - return 0; + return 1; /* Root can execute any file that has any one of the execute * bits set. */ - if (st->st_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) - return 0; - } - - if (st->st_uid == euid) /* owner */ + mode = S_IXUSR | S_IXGRP | S_IXOTH; + } else if (st->st_uid == euid) /* owner */ mode <<= 6; else if (is_a_group_member(st->st_gid)) mode <<= 3; - if (st->st_mode & mode) - return 0; - - return -1; + return st->st_mode & mode; } @@ -722,7 +717,7 @@ static int filstat(char *nm, enum token mode) i = W_OK; if (mode == FILEX) i = X_OK; - return test_eaccess(&s, i) == 0; + return test_st_mode(&s, i); } if (is_file_type(mode)) { if (mode == FILREG) From 748b1681549067f2e27ab2b9102ef9352cfa8a4c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 01:20:43 +0200 Subject: [PATCH 016/277] libbb: move is_in_supplementary_groups() from test to libbb function old new delta is_in_supplementary_groups - 54 +54 nexpr 766 721 -45 ------------------------------------------------------------------------------ (add/remove: 1/0 grow/shrink: 0/1 up/down: 54/-45) Total: 9 bytes Signed-off-by: Denys Vlasenko --- coreutils/test.c | 20 +------------------- include/libbb.h | 5 +++++ libbb/bb_getgroups.c | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/coreutils/test.c b/coreutils/test.c index c02c927450..874285704d 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -637,32 +637,14 @@ static int binop(void) /*return 1; - NOTREACHED */ } -static void initialize_group_array(void) -{ - group_array = bb_getgroups(&ngroups, NULL); -} - /* Return non-zero if GID is one that we have in our groups list. */ -//XXX: FIXME: duplicate of existing libbb function? -// see toplevel TODO file: -// possible code duplication ingroup() and is_a_group_member() static int is_a_group_member(gid_t gid) { - int i; - /* Short-circuit if possible, maybe saving a call to getgroups(). */ if (gid == getgid() || gid == getegid()) return 1; - if (ngroups == 0) - initialize_group_array(); - - /* Search through the list looking for GID. */ - for (i = 0; i < ngroups; i++) - if (gid == group_array[i]) - return 1; - - return 0; + return is_in_supplementary_groups(&ngroups, &group_array, gid); } /* diff --git a/include/libbb.h b/include/libbb.h index 67e7bf7b99..e06aef08e6 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1201,6 +1201,11 @@ void die_if_bad_username(const char* name) FAST_FUNC; * Dies on errors (on Linux, only xrealloc can cause this, not internal getgroups call). */ gid_t *bb_getgroups(int *ngroups, gid_t *group_array) FAST_FUNC; +/* + * True if GID is in our getgroups() result. + * getgroups() is cached in group_array[], to makse successive calls faster. + */ +int FAST_FUNC is_in_supplementary_groups(int *pngroups, gid_t **pgroup_array, gid_t gid); #if ENABLE_FEATURE_UTMP void FAST_FUNC write_new_utmp(pid_t pid, int new_type, const char *tty_name, const char *username, const char *hostname); diff --git a/libbb/bb_getgroups.c b/libbb/bb_getgroups.c index 5d83c729a9..f030d5eac9 100644 --- a/libbb/bb_getgroups.c +++ b/libbb/bb_getgroups.c @@ -45,3 +45,23 @@ gid_t* FAST_FUNC bb_getgroups(int *ngroups, gid_t *group_array) *ngroups = n; return group_array; } + +/* Return non-zero if GID is in our supplementary group list. */ +int FAST_FUNC is_in_supplementary_groups(int *pngroups, gid_t **pgroup_array, gid_t gid) +{ + int i; + int ngroups; + gid_t *group_array; + + if (*pngroups == 0) + *pgroup_array = bb_getgroups(pngroups, NULL); + ngroups = *pngroups; + group_array = *pgroup_array; + + /* Search through the list looking for GID. */ + for (i = 0; i < ngroups; i++) + if (gid == group_array[i]) + return 1; + + return 0; +} From 860b3d066f6aaa12dfa0cd2351559e05288cf9b5 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 05:46:31 +0200 Subject: [PATCH 017/277] ash: command -v CMD must skip (go to next path) when CMD exists, but is not executable Upstream commit: Date: Fri, 5 Apr 2024 17:55:46 +0800 exec: Check executable bit when searching path Andrej Shadura wrote: ... > https://bugs.debian.org/874264 > -------- Forwarded Message -------- > Subject: dash: 'command -v' mistakenly returns a shell script whose > executable is not set > Date: Mon, 04 Sep 2017 10:45:48 -0400 > From: Norman Ramsey > To: Debian Bug Tracking System ... > I tracked a build bug in s-nail to a problem with dash. Symptom: > building s-nail tries to run /home/nr/bin/clang, a script whose > executable bit is not set. We tracked the problem to the result of > running `command -v clang` with /bin/sh: ... This is inherited from NetBSD. There is even a commented-out block of code that tried to fix this. Anyway, we now have faccessat so we can simply use it. function old new delta test_exec - 125 +125 find_command 911 918 +7 ------------------------------------------------------------------------------ (add/remove: 1/0 grow/shrink: 1/0 up/down: 132/0) Total: 132 bytes Signed-off-by: Denys Vlasenko --- shell/ash.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/shell/ash.c b/shell/ash.c index 8e029765d2..984a71f07e 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -491,6 +491,11 @@ struct globals_misc { char **trap_ptr; /* used only by "trap hack" */ /* Rarely referenced stuff */ + + /* Cached supplementary group array (for testing executable'ity of files) */ + int ngroups; + gid_t *group_array; + #if ENABLE_ASH_RANDOM_SUPPORT random_t random_gen; #endif @@ -523,6 +528,8 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc; #define may_have_traps (G_misc.may_have_traps ) #define trap (G_misc.trap ) #define trap_ptr (G_misc.trap_ptr ) +#define ngroups (G_misc.ngroups ) +#define group_array (G_misc.group_array) #define random_gen (G_misc.random_gen ) #define backgndpid (G_misc.backgndpid ) #define INIT_G_misc() do { \ @@ -10171,6 +10178,7 @@ static int FAST_FUNC printfcmd(int argc, char **argv) { return printf_main(argc, #endif #if ENABLE_ASH_TEST || BASH_TEST2 static int FAST_FUNC testcmd(int argc, char **argv) { return test_main(argc, argv); } +// TODO: pass &ngroups and &group_array addresses to test_main to use cached supplementary groups #endif #if ENABLE_ASH_SLEEP static int FAST_FUNC sleepcmd(int argc, char **argv) { return sleep_main(argc, argv); } @@ -13782,6 +13790,43 @@ readcmdfile(char *name) /* ============ find_command inplementation */ +static int test_exec(/*const char *fullname,*/ struct stat *statb) +{ + /* + * TODO: use faccessat(AT_FDCWD, fullname, X_OK, AT_EACCESS) + * instead: executability may depend on ACLs, capabilities + * and who knows what else, not just mode bits. + * (faccessat2 syscall was added to Linux in May 14 2020) + */ + mode_t stmode; + uid_t euid; + enum { ANY_IX = S_IXUSR | S_IXGRP | S_IXOTH }; + + /* Do we already know with no extra syscalls? */ + if (!S_ISREG(statb->st_mode)) + return 0; /* not a regular file */ + if ((statb->st_mode & ANY_IX) == 0) + return 0; /* no one can execute */ + if ((statb->st_mode & ANY_IX) == ANY_IX) + return 1; /* anyone can execute */ + + /* Executability depends on our euid/egid/supplementary groups */ + stmode = S_IXOTH; + euid = geteuid(); +//TODO: cache euid? + if (euid == 0) + /* for root user, any X bit is good enough */ + stmode = ANY_IX; + else if (statb->st_uid == euid) + stmode = S_IXUSR; + else if (statb->st_gid == getegid()) + stmode = S_IXGRP; + else if (is_in_supplementary_groups(&ngroups, &group_array, statb->st_gid)) + stmode = S_IXGRP; + + return statb->st_mode & stmode; +} + /* * Resolve a command name. If you change this routine, you may have to * change the shellexec routine as well. @@ -13808,9 +13853,12 @@ find_command(char *name, struct cmdentry *entry, int act, const char *path) if (errno == EINTR) continue; #endif + absfail: entry->cmdtype = CMDUNKNOWN; return; } + if (!test_exec(/*name,*/ &statb)) + goto absfail; } entry->cmdtype = CMDNORMAL; return; @@ -13923,9 +13971,6 @@ find_command(char *name, struct cmdentry *entry, int act, const char *path) e = errno; goto loop; } - e = EACCES; /* if we fail, this will be the error */ - if (!S_ISREG(statb.st_mode)) - continue; if (lpathopt) { /* this is a %func directory */ stalloc(len); /* NB: stalloc will return space pointed by fullname @@ -13938,6 +13983,9 @@ find_command(char *name, struct cmdentry *entry, int act, const char *path) stunalloc(fullname); goto success; } + e = EACCES; /* if we fail, this will be the error */ + if (!test_exec(/*fullname,*/ &statb)) + continue; TRACE(("searchexec \"%s\" returns \"%s\"\n", name, fullname)); if (!updatetbl) { entry->cmdtype = CMDNORMAL; From 4c1d645c86f4e7a380d96f9ba962f8b270f595dc Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 06:36:00 +0200 Subject: [PATCH 018/277] libbb: simplify parameter passing in is_in_supplementary_groups() function old new delta is_in_supplementary_groups 54 52 -2 nexpr 721 718 -3 test_exec 125 119 -6 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/3 up/down: 0/-11) Total: -11 bytes Signed-off-by: Denys Vlasenko --- coreutils/test.c | 26 ++++++++++++++++++-------- include/libbb.h | 10 ++++++++-- libbb/bb_getgroups.c | 10 +++++----- shell/ash.c | 8 +++----- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/coreutils/test.c b/coreutils/test.c index 874285704d..96462d3a23 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -426,8 +426,7 @@ struct test_statics { /* set only by check_operator(), either to bogus struct * or points to matching operator_t struct. Never NULL. */ const struct operator_t *last_operator; - gid_t *group_array; - int ngroups; + struct cached_groupinfo groupinfo; #if BASH_TEST2 bool bash_test2; #endif @@ -440,8 +439,7 @@ extern struct test_statics *BB_GLOBAL_CONST test_ptr_to_statics; #define S (*test_ptr_to_statics) #define args (S.args ) #define last_operator (S.last_operator) -#define group_array (S.group_array ) -#define ngroups (S.ngroups ) +#define groupinfo (S.groupinfo ) #define bash_test2 (S.bash_test2 ) #define leaving (S.leaving ) @@ -449,7 +447,7 @@ extern struct test_statics *BB_GLOBAL_CONST test_ptr_to_statics; XZALLOC_CONST_PTR(&test_ptr_to_statics, sizeof(S)); \ } while (0) #define DEINIT_S() do { \ - free(group_array); \ + free(groupinfo.supplementary_array); \ free(test_ptr_to_statics); \ } while (0) @@ -644,7 +642,7 @@ static int is_a_group_member(gid_t gid) if (gid == getgid() || gid == getegid()) return 1; - return is_in_supplementary_groups(&ngroups, &group_array, gid); + return is_in_supplementary_groups(&groupinfo, gid); } /* @@ -654,8 +652,20 @@ static int is_a_group_member(gid_t gid) */ static int test_st_mode(struct stat *st, int mode) { - unsigned int euid = geteuid(); - + enum { ANY_IX = S_IXUSR | S_IXGRP | S_IXOTH }; + unsigned euid; + +//TODO if (mode == X_OK) { +// /* Do we already know with no extra syscalls? */ +// if (!S_ISREG(st->st_mode)) +// return 0; /* not a regular file */ +// if ((st->st_mode & ANY_IX) == 0) +// return 0; /* no one can execute */ +// if ((st->st_mode & ANY_IX) == ANY_IX) +// return 1; /* anyone can execute */ +// } + + euid = geteuid(); if (euid == 0) { /* Root can read or write any file. */ if (mode != X_OK) diff --git a/include/libbb.h b/include/libbb.h index e06aef08e6..99bbf623b9 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1203,9 +1203,15 @@ void die_if_bad_username(const char* name) FAST_FUNC; gid_t *bb_getgroups(int *ngroups, gid_t *group_array) FAST_FUNC; /* * True if GID is in our getgroups() result. - * getgroups() is cached in group_array[], to makse successive calls faster. + * getgroups() is cached in supplementary_array[], to make successive calls faster. */ -int FAST_FUNC is_in_supplementary_groups(int *pngroups, gid_t **pgroup_array, gid_t gid); +struct cached_groupinfo { + //TODO? gid_t egid; + int ngroups; + gid_t *supplementary_array; +}; +//TODO? int FAST_FUNC get_cached_egid(gid_t *egid); +int FAST_FUNC is_in_supplementary_groups(struct cached_groupinfo *groupinfo, gid_t gid); #if ENABLE_FEATURE_UTMP void FAST_FUNC write_new_utmp(pid_t pid, int new_type, const char *tty_name, const char *username, const char *hostname); diff --git a/libbb/bb_getgroups.c b/libbb/bb_getgroups.c index f030d5eac9..d9bbe95c3a 100644 --- a/libbb/bb_getgroups.c +++ b/libbb/bb_getgroups.c @@ -47,16 +47,16 @@ gid_t* FAST_FUNC bb_getgroups(int *ngroups, gid_t *group_array) } /* Return non-zero if GID is in our supplementary group list. */ -int FAST_FUNC is_in_supplementary_groups(int *pngroups, gid_t **pgroup_array, gid_t gid) +int FAST_FUNC is_in_supplementary_groups(struct cached_groupinfo *groupinfo, gid_t gid) { int i; int ngroups; gid_t *group_array; - if (*pngroups == 0) - *pgroup_array = bb_getgroups(pngroups, NULL); - ngroups = *pngroups; - group_array = *pgroup_array; + if (groupinfo->ngroups == 0) + groupinfo->supplementary_array = bb_getgroups(&groupinfo->ngroups, NULL); + ngroups = groupinfo->ngroups; + group_array = groupinfo->supplementary_array; /* Search through the list looking for GID. */ for (i = 0; i < ngroups; i++) diff --git a/shell/ash.c b/shell/ash.c index 984a71f07e..fa57511a71 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -493,8 +493,7 @@ struct globals_misc { /* Rarely referenced stuff */ /* Cached supplementary group array (for testing executable'ity of files) */ - int ngroups; - gid_t *group_array; + struct cached_groupinfo groupinfo; #if ENABLE_ASH_RANDOM_SUPPORT random_t random_gen; @@ -528,8 +527,7 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc; #define may_have_traps (G_misc.may_have_traps ) #define trap (G_misc.trap ) #define trap_ptr (G_misc.trap_ptr ) -#define ngroups (G_misc.ngroups ) -#define group_array (G_misc.group_array) +#define groupinfo (G_misc.groupinfo ) #define random_gen (G_misc.random_gen ) #define backgndpid (G_misc.backgndpid ) #define INIT_G_misc() do { \ @@ -13821,7 +13819,7 @@ static int test_exec(/*const char *fullname,*/ struct stat *statb) stmode = S_IXUSR; else if (statb->st_gid == getegid()) stmode = S_IXGRP; - else if (is_in_supplementary_groups(&ngroups, &group_array, statb->st_gid)) + else if (is_in_supplementary_groups(&groupinfo, statb->st_gid)) stmode = S_IXGRP; return statb->st_mode & stmode; From d26e9587252fdf4774a05e351b86d3a18c46384e Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 07:14:27 +0200 Subject: [PATCH 019/277] ash: make "test -x" use cached groupinfo function old new delta test_main2 - 407 +407 testcmd 10 23 +13 test_main 418 56 -362 ------------------------------------------------------------------------------ (add/remove: 1/0 grow/shrink: 1/1 up/down: 420/-362) Total: 58 bytes Signed-off-by: Denys Vlasenko --- coreutils/test.c | 21 +++++++++++++++++---- include/libbb.h | 1 + shell/ash.c | 3 +-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/coreutils/test.c b/coreutils/test.c index 96462d3a23..ad777953f0 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -426,7 +426,7 @@ struct test_statics { /* set only by check_operator(), either to bogus struct * or points to matching operator_t struct. Never NULL. */ const struct operator_t *last_operator; - struct cached_groupinfo groupinfo; + struct cached_groupinfo *groupinfo; #if BASH_TEST2 bool bash_test2; #endif @@ -447,7 +447,6 @@ extern struct test_statics *BB_GLOBAL_CONST test_ptr_to_statics; XZALLOC_CONST_PTR(&test_ptr_to_statics, sizeof(S)); \ } while (0) #define DEINIT_S() do { \ - free(groupinfo.supplementary_array); \ free(test_ptr_to_statics); \ } while (0) @@ -642,7 +641,7 @@ static int is_a_group_member(gid_t gid) if (gid == getgid() || gid == getegid()) return 1; - return is_in_supplementary_groups(&groupinfo, gid); + return is_in_supplementary_groups(groupinfo, gid); } /* @@ -878,7 +877,7 @@ static number_t primary(enum token n) } -int test_main(int argc, char **argv) +int FAST_FUNC test_main2(struct cached_groupinfo *pgroupinfo, int argc, char **argv) { int res; const char *arg0; @@ -911,6 +910,7 @@ int test_main(int argc, char **argv) /* We must do DEINIT_S() prior to returning */ INIT_S(); + groupinfo = pgroupinfo; #if BASH_TEST2 bash_test2 = bt2; @@ -1013,3 +1013,16 @@ int test_main(int argc, char **argv) DEINIT_S(); return res; } + +int test_main(int argc, char **argv) +{ + struct cached_groupinfo info; + int r; + + info.ngroups = 0; + info.supplementary_array = NULL; + r = test_main2(&info, argc, argv); + free(info.supplementary_array); + + return r; +} diff --git a/include/libbb.h b/include/libbb.h index 99bbf623b9..8748464ed8 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1550,6 +1550,7 @@ int test_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE #endif ; +int FAST_FUNC test_main2(struct cached_groupinfo *pgroupinfo, int argc, char **argv); int kill_main(int argc, char **argv) #if ENABLE_KILL || ENABLE_KILLALL || ENABLE_KILLALL5 MAIN_EXTERNALLY_VISIBLE diff --git a/shell/ash.c b/shell/ash.c index fa57511a71..a6bb9894c0 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -10175,8 +10175,7 @@ static int FAST_FUNC echocmd(int argc, char **argv) { return echo_main(argc, a static int FAST_FUNC printfcmd(int argc, char **argv) { return printf_main(argc, argv); } #endif #if ENABLE_ASH_TEST || BASH_TEST2 -static int FAST_FUNC testcmd(int argc, char **argv) { return test_main(argc, argv); } -// TODO: pass &ngroups and &group_array addresses to test_main to use cached supplementary groups +static int FAST_FUNC testcmd(int argc, char **argv) { return test_main2(&groupinfo, argc, argv); } #endif #if ENABLE_ASH_SLEEP static int FAST_FUNC sleepcmd(int argc, char **argv) { return sleep_main(argc, argv); } From 96b0607302500ed201a7816282efbaa8f990aa33 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 07:28:44 +0200 Subject: [PATCH 020/277] ash: cache more of uid/gid syscalls Testcase: setuidgid 1:1 strace ash -c 'test -x TODO; test -x TODO; echo $?' should show that second "test -x" does not query ids again. function old new delta ash_main 1236 1256 +20 get_cached_euid - 19 +19 get_cached_egid - 19 +19 test_main 56 72 +16 test_exec 119 135 +16 is_in_supplementary_groups 52 57 +5 nexpr 718 702 -16 ------------------------------------------------------------------------------ (add/remove: 2/0 grow/shrink: 4/1 up/down: 95/-16) Total: 79 bytes Signed-off-by: Denys Vlasenko --- coreutils/test.c | 11 +++++++---- include/libbb.h | 6 ++++-- libbb/bb_getgroups.c | 14 ++++++++++++++ shell/ash.c | 9 +++++---- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/coreutils/test.c b/coreutils/test.c index ad777953f0..6085d66631 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -638,7 +638,7 @@ static int binop(void) static int is_a_group_member(gid_t gid) { /* Short-circuit if possible, maybe saving a call to getgroups(). */ - if (gid == getgid() || gid == getegid()) + if (gid == get_cached_egid(&groupinfo->egid)) return 1; return is_in_supplementary_groups(groupinfo, gid); @@ -656,15 +656,16 @@ static int test_st_mode(struct stat *st, int mode) //TODO if (mode == X_OK) { // /* Do we already know with no extra syscalls? */ -// if (!S_ISREG(st->st_mode)) -// return 0; /* not a regular file */ +// //if (!S_ISREG(st->st_mode)) +// // return 0; /* not a regular file */ +// // ^^^ bash does not check this // if ((st->st_mode & ANY_IX) == 0) // return 0; /* no one can execute */ // if ((st->st_mode & ANY_IX) == ANY_IX) // return 1; /* anyone can execute */ // } - euid = geteuid(); + euid = get_cached_euid(&groupinfo->euid); if (euid == 0) { /* Root can read or write any file. */ if (mode != X_OK) @@ -1019,6 +1020,8 @@ int test_main(int argc, char **argv) struct cached_groupinfo info; int r; + info.euid = -1; + info.egid = -1; info.ngroups = 0; info.supplementary_array = NULL; r = test_main2(&info, argc, argv); diff --git a/include/libbb.h b/include/libbb.h index 8748464ed8..f5f8e1635f 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1206,11 +1206,13 @@ gid_t *bb_getgroups(int *ngroups, gid_t *group_array) FAST_FUNC; * getgroups() is cached in supplementary_array[], to make successive calls faster. */ struct cached_groupinfo { - //TODO? gid_t egid; + uid_t euid; + gid_t egid; int ngroups; gid_t *supplementary_array; }; -//TODO? int FAST_FUNC get_cached_egid(gid_t *egid); +uid_t FAST_FUNC get_cached_euid(uid_t *euid); +gid_t FAST_FUNC get_cached_egid(gid_t *egid); int FAST_FUNC is_in_supplementary_groups(struct cached_groupinfo *groupinfo, gid_t gid); #if ENABLE_FEATURE_UTMP diff --git a/libbb/bb_getgroups.c b/libbb/bb_getgroups.c index d9bbe95c3a..31cff2b41d 100644 --- a/libbb/bb_getgroups.c +++ b/libbb/bb_getgroups.c @@ -46,6 +46,20 @@ gid_t* FAST_FUNC bb_getgroups(int *ngroups, gid_t *group_array) return group_array; } +uid_t FAST_FUNC get_cached_euid(uid_t *euid) +{ + if (*euid == (uid_t)-1) + *euid = geteuid(); + return *euid; +} + +gid_t FAST_FUNC get_cached_egid(gid_t *egid) +{ + if (*egid == (gid_t)-1) + *egid = getegid(); + return *egid; +} + /* Return non-zero if GID is in our supplementary group list. */ int FAST_FUNC is_in_supplementary_groups(struct cached_groupinfo *groupinfo, gid_t gid) { diff --git a/shell/ash.c b/shell/ash.c index a6bb9894c0..9173b8608e 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -536,6 +536,8 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc; curdir = nullstr; \ physdir = nullstr; \ trap_ptr = trap; \ + groupinfo.euid = -1; \ + groupinfo.egid = -1; \ } while (0) @@ -2319,7 +2321,7 @@ initvar(void) #if ENABLE_FEATURE_EDITING && ENABLE_FEATURE_EDITING_FANCY_PROMPT vps1.var_text = "PS1=\\w \\$ "; #else - if (!geteuid()) + if (!get_cached_euid(&groupinfo.euid)); vps1.var_text = "PS1=# "; #endif vp = varinit; @@ -13809,14 +13811,13 @@ static int test_exec(/*const char *fullname,*/ struct stat *statb) /* Executability depends on our euid/egid/supplementary groups */ stmode = S_IXOTH; - euid = geteuid(); -//TODO: cache euid? + euid = get_cached_euid(&groupinfo.euid); if (euid == 0) /* for root user, any X bit is good enough */ stmode = ANY_IX; else if (statb->st_uid == euid) stmode = S_IXUSR; - else if (statb->st_gid == getegid()) + else if (statb->st_gid == get_cached_egid(&groupinfo.egid)) stmode = S_IXGRP; else if (is_in_supplementary_groups(&groupinfo, statb->st_gid)) stmode = S_IXGRP; From 0929a129fc75c556de67877491281e0bc3ef3edd Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 07:33:58 +0200 Subject: [PATCH 021/277] test: -x can return 0/1 early if all X bits are the same function old new delta nexpr 702 725 +23 Signed-off-by: Denys Vlasenko --- coreutils/test.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/coreutils/test.c b/coreutils/test.c index 6085d66631..b63e33cc0e 100644 --- a/coreutils/test.c +++ b/coreutils/test.c @@ -654,16 +654,16 @@ static int test_st_mode(struct stat *st, int mode) enum { ANY_IX = S_IXUSR | S_IXGRP | S_IXOTH }; unsigned euid; -//TODO if (mode == X_OK) { -// /* Do we already know with no extra syscalls? */ -// //if (!S_ISREG(st->st_mode)) -// // return 0; /* not a regular file */ -// // ^^^ bash does not check this -// if ((st->st_mode & ANY_IX) == 0) -// return 0; /* no one can execute */ -// if ((st->st_mode & ANY_IX) == ANY_IX) -// return 1; /* anyone can execute */ -// } + if (mode == X_OK) { + /* Do we already know with no extra syscalls? */ + //if (!S_ISREG(st->st_mode)) + // return 0; /* not a regular file */ + // ^^^ bash 5.2.15 "test -x" does not check this! + if ((st->st_mode & ANY_IX) == 0) + return 0; /* no one can execute */ + if ((st->st_mode & ANY_IX) == ANY_IX) + return 1; /* anyone can execute */ + } euid = get_cached_euid(&groupinfo->euid); if (euid == 0) { From baa49bdc1b7b992bfb8298589cd33c04353c6b57 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 15:18:45 +0200 Subject: [PATCH 022/277] hush: make "test -x" use cached groupinfo While at it, correct "type" to skip non-executable files in PATH function old new delta builtin_source 211 316 +105 builtin_test 10 32 +22 hush_main 1150 1170 +20 builtin_type 122 137 +15 if_command_vV_print_and_exit 120 114 -6 find_in_path 131 - -131 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 4/1 up/down: 162/-137) Total: 25 bytes Signed-off-by: Denys Vlasenko --- shell/hush.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/shell/hush.c b/shell/hush.c index 6b6ec7c6bd..074f35f2b6 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -1021,6 +1021,13 @@ struct globals { #endif #if HUSH_DEBUG >= 2 int debug_indent; +#endif +#if ENABLE_HUSH_TEST || BASH_TEST2 + /* Cached supplementary group array (for testing executable'ity of files) */ + struct cached_groupinfo groupinfo; +# define GROUPINFO_INIT { G.groupinfo.euid = -1; G.groupinfo.egid = -1; } +#else +# define GROUPINFO_INIT /* nothing */ #endif struct sigaction sa; char optstring_buf[sizeof("eixcs")]; @@ -1040,6 +1047,7 @@ struct globals { /* memset(&G.sa, 0, sizeof(G.sa)); */ \ sigfillset(&G.sa.sa_mask); \ G.sa.sa_flags = SA_RESTART; \ + GROUPINFO_INIT; \ } while (0) /* Function prototypes for builtins */ @@ -8193,6 +8201,8 @@ static int setup_redirects(struct command *prog, struct squirrel **sqp) return 0; } +/* Find a file in PATH, not necessarily executable */ +//TODO: shares code with find_executable() in libbb, factor out? static char *find_in_path(const char *arg) { char *ret = NULL; @@ -8635,7 +8645,7 @@ static void if_command_vV_print_and_exit(char opt_vV, char *cmd, const char *exp to_free = NULL; if (!explanation) { - char *path = getenv("PATH"); + char *path = (char*)get_local_var_value("PATH"); explanation = to_free = find_executable(cmd, &path); /* path == NULL is ok */ if (!explanation) _exit(1); /* PROG was not found */ @@ -10870,7 +10880,8 @@ static NOINLINE int run_applet_main(char **argv, int (*applet_main_func)(int arg #if ENABLE_HUSH_TEST || BASH_TEST2 static int FAST_FUNC builtin_test(char **argv) { - return run_applet_main(argv, test_main); + int argc = string_array_len(argv); + return test_main2(&G.groupinfo, argc, argv); } #endif #if ENABLE_HUSH_ECHO @@ -11063,12 +11074,16 @@ static int FAST_FUNC builtin_type(char **argv) # endif else if (find_builtin(*argv)) type = "a shell builtin"; - else if ((path = find_in_path(*argv)) != NULL) - type = path; else { - bb_error_msg("type: %s: not found", *argv); - ret = EXIT_FAILURE; - continue; + char *pathvar = (char*)get_local_var_value("PATH"); + path = find_executable(*argv, &pathvar); + if (path) + type = path; + else { + bb_error_msg("type: %s: not found", *argv); + ret = EXIT_FAILURE; + continue; + } } printf("%s is %s\n", *argv, type); From 8c4bccb83e5e2594f16310b7cbe07bf05fc9f13a Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Oct 2024 22:00:16 +0200 Subject: [PATCH 023/277] hush: fix "type ./cat" and "command -v ./cat" to not scan PATH function old new delta find_executable_in_PATH - 67 +67 if_command_vV_print_and_exit 114 116 +2 .rodata 105712 105710 -2 builtin_type 137 128 -9 ------------------------------------------------------------------------------ (add/remove: 1/0 grow/shrink: 1/2 up/down: 69/-11) Total: 58 bytes Signed-off-by: Denys Vlasenko --- shell/hush.c | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/shell/hush.c b/shell/hush.c index 074f35f2b6..4e477d05aa 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -8201,27 +8201,29 @@ static int setup_redirects(struct command *prog, struct squirrel **sqp) return 0; } -/* Find a file in PATH, not necessarily executable */ +/* Find a file in PATH, not necessarily executable + * Name is known to not contain '/'. + */ //TODO: shares code with find_executable() in libbb, factor out? -static char *find_in_path(const char *arg) +static char *find_in_PATH(const char *name) { - char *ret = NULL; + char *ret; const char *PATH = get_local_var_value("PATH"); if (!PATH) return NULL; - + ret = NULL; while (1) { const char *end = strchrnul(PATH, ':'); int sz = end - PATH; /* must be int! */ free(ret); if (sz != 0) { - ret = xasprintf("%.*s/%s", sz, PATH, arg); + ret = xasprintf("%.*s/%s", sz, PATH, name); } else { /* We have xxx::yyyy in $PATH, * it means "use current dir" */ - ret = xstrdup(arg); + ret = xstrdup(name); } if (access(ret, F_OK) == 0) break; @@ -8236,6 +8238,24 @@ static char *find_in_path(const char *arg) return ret; } +#if ENABLE_HUSH_TYPE || ENABLE_HUSH_COMMAND +static char *find_executable_in_PATH(const char *name) +{ + char *PATH; + if (strchr(name, '/')) { + /* Name with '/' is tested verbatim, with no PATH traversal: + * "cd /bin; type ./cat" should print "./cat is ./cat", + * NOT "./cat is /bin/./cat" + */ + if (file_is_executable(name)) + return xstrdup(name); + return NULL; + } + PATH = (char*)get_local_var_value("PATH"); + return find_executable(name, &PATH); /* path == NULL is ok */ +} +#endif + static const struct built_in_command *find_builtin_helper(const char *name, const struct built_in_command *x, const struct built_in_command *end) @@ -8645,10 +8665,11 @@ static void if_command_vV_print_and_exit(char opt_vV, char *cmd, const char *exp to_free = NULL; if (!explanation) { - char *path = (char*)get_local_var_value("PATH"); - explanation = to_free = find_executable(cmd, &path); /* path == NULL is ok */ - if (!explanation) + explanation = to_free = find_executable_in_PATH(cmd); + if (!explanation) { + bb_error_msg("%s: %s: not found", "command", cmd); _exit(1); /* PROG was not found */ + } if (opt_vV != 'V') cmd = to_free; /* -v PROG prints "/path/to/PROG" */ } @@ -11075,12 +11096,11 @@ static int FAST_FUNC builtin_type(char **argv) else if (find_builtin(*argv)) type = "a shell builtin"; else { - char *pathvar = (char*)get_local_var_value("PATH"); - path = find_executable(*argv, &pathvar); + path = find_executable_in_PATH(*argv); if (path) type = path; else { - bb_error_msg("type: %s: not found", *argv); + bb_error_msg("%s: %s: not found", "type", *argv); ret = EXIT_FAILURE; continue; } @@ -11710,7 +11730,7 @@ static int FAST_FUNC builtin_source(char **argv) } arg_path = NULL; if (!strchr(filename, '/')) { - arg_path = find_in_path(filename); + arg_path = find_in_PATH(filename); if (arg_path) filename = arg_path; else if (!ENABLE_HUSH_BASH_SOURCE_CURDIR) { From 49d9e06fbab0b02a71deed57610edb0c8f4fb20c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Oct 2024 04:03:17 +0200 Subject: [PATCH 024/277] libbb: modify find_executable() to not temporarily write to PATH This allows to simplify "which" applet code function old new delta find_executable 93 111 +18 which_main 191 177 -14 builtin_source 316 294 -22 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/2 up/down: 18/-36) Total: -18 bytes Signed-off-by: Denys Vlasenko --- debianutils/which.c | 9 +++----- include/libbb.h | 2 +- libbb/executable.c | 56 ++++++++++++++++++++++++--------------------- shell/hush.c | 35 ++++++++++++---------------- 4 files changed, 48 insertions(+), 54 deletions(-) diff --git a/debianutils/which.c b/debianutils/which.c index 1f547919f2..a7d55a2159 100644 --- a/debianutils/which.c +++ b/debianutils/which.c @@ -31,15 +31,12 @@ int which_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int which_main(int argc UNUSED_PARAM, char **argv) { - char *env_path; + const char *env_path; int status = 0; - /* This sizeof(): bb_default_root_path is shorter than BB_PATH_ROOT_PATH */ - char buf[sizeof(BB_PATH_ROOT_PATH)]; env_path = getenv("PATH"); if (!env_path) - /* env_path must be writable, and must not alloc, so... */ - env_path = strcpy(buf, bb_default_root_path); + env_path = bb_default_root_path; getopt32(argv, "^" "a" "\0" "-1"/*at least one arg*/); argv += optind; @@ -54,7 +51,7 @@ int which_main(int argc UNUSED_PARAM, char **argv) puts(*argv); } } else { - char *path; + const char *path; char *p; path = env_path; diff --git a/include/libbb.h b/include/libbb.h index f5f8e1635f..4d6193795e 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1227,7 +1227,7 @@ void FAST_FUNC update_utmp_DEAD_PROCESS(pid_t pid); int file_is_executable(const char *name) FAST_FUNC; -char *find_executable(const char *filename, char **PATHp) FAST_FUNC; +char *find_executable(const char *filename, const char **PATHp) FAST_FUNC; int executable_exists(const char *filename) FAST_FUNC; /* BB_EXECxx always execs (it's not doing NOFORK/NOEXEC stuff), diff --git a/libbb/executable.c b/libbb/executable.c index a033b74d99..09bed1eaf0 100644 --- a/libbb/executable.c +++ b/libbb/executable.c @@ -21,14 +21,11 @@ int FAST_FUNC file_is_executable(const char *name) /* search (*PATHp) for an executable file; * return allocated string containing full path if found; * PATHp points to the component after the one where it was found - * (or NULL), + * (or NULL if found in last component), * you may call find_executable again with this PATHp to continue - * (if it's not NULL). - * return NULL otherwise; (PATHp is undefined) - * in all cases (*PATHp) contents are temporarily modified - * but are restored on return (s/:/NUL/ and back). + * return NULL otherwise (PATHp is undefined) */ -char* FAST_FUNC find_executable(const char *filename, char **PATHp) +char* FAST_FUNC find_executable(const char *name, const char **PATHp) { /* About empty components in $PATH: * http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap08.html @@ -38,38 +35,45 @@ char* FAST_FUNC find_executable(const char *filename, char **PATHp) * initial colon preceding the rest of the list, or as a trailing colon * following the rest of the list. */ - char *p, *n; + char *p = (char*) *PATHp; - p = *PATHp; - while (p) { - int ex; + if (!p) + return NULL; + while (1) { + const char *end = strchrnul(p, ':'); + int sz = end - p; - n = strchr(p, ':'); - if (n) *n = '\0'; - p = concat_path_file( - p[0] ? p : ".", /* handle "::" case */ - filename - ); - ex = file_is_executable(p); - if (n) *n++ = ':'; - if (ex) { - *PATHp = n; + if (sz != 0) { + p = xasprintf("%.*s/%s", sz, p, name); + } else { + /* We have xxx::yyy in $PATH, + * it means "use current dir" */ + p = xstrdup(name); +// A bit of discrepancy wrt the path used if file is found here. +// bash 5.2.15 "type" returns "./NAME". +// GNU which v2.21 returns "/CUR/DIR/NAME". +// With -a, both skip over all colons: xxx::::yyy is the same as xxx::yyy, +// current dir is not tried the second time. + } + if (file_is_executable(p)) { + *PATHp = (*end ? end+1 : NULL); return p; } free(p); - p = n; - } /* on loop exit p == NULL */ - return p; + if (*end == '\0') + return NULL; + p = (char *) end + 1; + } } /* search $PATH for an executable file; * return 1 if found; * return 0 otherwise; */ -int FAST_FUNC executable_exists(const char *filename) +int FAST_FUNC executable_exists(const char *name) { - char *path = getenv("PATH"); - char *ret = find_executable(filename, &path); + const char *path = getenv("PATH"); + char *ret = find_executable(name, &path); free(ret); return ret != NULL; } diff --git a/shell/hush.c b/shell/hush.c index 4e477d05aa..04dda0734f 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -8207,41 +8207,34 @@ static int setup_redirects(struct command *prog, struct squirrel **sqp) //TODO: shares code with find_executable() in libbb, factor out? static char *find_in_PATH(const char *name) { - char *ret; - const char *PATH = get_local_var_value("PATH"); + char *p = (char*) get_local_var_value("PATH"); - if (!PATH) + if (!p) return NULL; - ret = NULL; while (1) { - const char *end = strchrnul(PATH, ':'); - int sz = end - PATH; /* must be int! */ + const char *end = strchrnul(p, ':'); + int sz = end - p; /* must be int! */ - free(ret); if (sz != 0) { - ret = xasprintf("%.*s/%s", sz, PATH, name); + p = xasprintf("%.*s/%s", sz, p, name); } else { - /* We have xxx::yyyy in $PATH, + /* We have xxx::yyy in $PATH, * it means "use current dir" */ - ret = xstrdup(name); + p = xstrdup(name); } - if (access(ret, F_OK) == 0) - break; - - if (*end == '\0') { - free(ret); + if (access(p, F_OK) == 0) + return p; + free(p); + if (*end == '\0') return NULL; - } - PATH = end + 1; + p = (char *) end + 1; } - - return ret; } #if ENABLE_HUSH_TYPE || ENABLE_HUSH_COMMAND static char *find_executable_in_PATH(const char *name) { - char *PATH; + const char *PATH; if (strchr(name, '/')) { /* Name with '/' is tested verbatim, with no PATH traversal: * "cd /bin; type ./cat" should print "./cat is ./cat", @@ -8251,7 +8244,7 @@ static char *find_executable_in_PATH(const char *name) return xstrdup(name); return NULL; } - PATH = (char*)get_local_var_value("PATH"); + PATH = get_local_var_value("PATH"); return find_executable(name, &PATH); /* path == NULL is ok */ } #endif From 75ca8d074bacb6896d770993b93161c40aa31b9f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Oct 2024 04:23:29 +0200 Subject: [PATCH 025/277] hexdump: accept hex numbers in -n, closes 16195 function old new delta hexdump_main 366 383 +17 Signed-off-by: Denys Vlasenko --- util-linux/hexdump.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/util-linux/hexdump.c b/util-linux/hexdump.c index c46ffeca70..5a938e7830 100644 --- a/util-linux/hexdump.c +++ b/util-linux/hexdump.c @@ -127,15 +127,20 @@ int hexdump_main(int argc, char **argv) if (ch == 'f') { bb_dump_addfile(dumper, optarg); } /* else */ - if (ch == 'n') { - dumper->dump_length = xatoi_positive(optarg); + if (ch == 'n') { /* compat: -n accepts hex numbers too */ + dumper->dump_length = xstrtou_range_sfx( + optarg, + /*base:*/ 0, + /*lo:*/ 0, /*hi:*/ INT_MAX, + kmg_i_suffixes + ); } /* else */ if (ch == 's') { /* compat: -s accepts hex numbers too */ dumper->dump_skip = xstrtoull_range_sfx( optarg, /*base:*/ 0, /*lo:*/ 0, /*hi:*/ OFF_T_MAX, - bkm_suffixes + kmg_i_suffixes ); } /* else */ if (ch == 'v') { From 23db6896830d6b37fc3bcc3ad5a7f725eb97d2e1 Mon Sep 17 00:00:00 2001 From: Hajime Tazaki Date: Mon, 25 Nov 2024 14:08:36 +0900 Subject: [PATCH 026/277] hush: fix G.argv0_for_re_execing to avoid endless loop When the busybox is used as /sbin/init and the inittab file contains below: ::respawn:-/bin/sh /sbin/init spawns hush for the first time with the argv[0] contains '-', and hush treats it as login shell. Then it reads /etc/profile and if the file contains the command execution like below, it invokes hush as login shell because the argv[0] argument is still '-/bin/sh' and reads /etc/profile again. This will last until some failure (e.g., memory failure) happens. [ "$(id -u)" -eq 0 ] && PS1="${PS1}# " || PS1="${PS1}\$ " This commit fixes this issues by adding an offset (+1) to the G.argv0_for_re_execing variable. This issue happens on our out-of-tree UML (use mode linux) with nommu configuration. Link: https://lore.kernel.org/all/cover.1731290567.git.thehajime@gmail.com/ Signed-off-by: Hajime Tazaki Signed-off-by: Denys Vlasenko --- shell/hush.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shell/hush.c b/shell/hush.c index 04dda0734f..4a97293ccb 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -10361,6 +10361,9 @@ int hush_main(int argc, char **argv) _exit(0); } G.argv0_for_re_execing = argv[0]; + if (G.argv0_for_re_execing[0] == '-') + /* reexeced hush should never be a login shell */ + G.argv0_for_re_execing++; #endif #if ENABLE_HUSH_TRAP # if ENABLE_HUSH_FUNCTIONS From e6f3a3b381fd32fc711b410aff9c9a37b75d353f Mon Sep 17 00:00:00 2001 From: Christian Franke Date: Sun, 1 Dec 2024 13:40:16 +0100 Subject: [PATCH 027/277] chpst: fix error check of nice(2) call Check errno instead of return value because -1 is a valid return value also on success. Signed-off-by: Christian Franke Signed-off-by: Denys Vlasenko --- runit/chpst.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runit/chpst.c b/runit/chpst.c index 2be1a57754..4e3d613b7d 100644 --- a/runit/chpst.c +++ b/runit/chpst.c @@ -466,7 +466,8 @@ int chpst_main(int argc UNUSED_PARAM, char **argv) /* nice should be done before xsetuid */ if (opt & OPT_n) { errno = 0; - if (nice(xatoi(nicestr)) == -1) + nice(xatoi(nicestr)); + if (errno) bb_simple_perror_msg_and_die("nice"); } From 175b8dda19022f5ca5b81f87248924817e7ea1cf Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Wed, 6 Nov 2024 15:14:41 +0000 Subject: [PATCH 028/277] libbb: tidy argument checks in getopt32() When getopt32() has complementary options it's possible to specify the minimum and maximum number of arguments allowed. Checking these values was inconsistent: - '?' correctly checked that it was followed by a digit but set the otherwise unused spec_flgs variable on error. - '=' failed to check that it was followed by a digit. function old new delta vgetopt32 1307 1319 +12 Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- libbb/getopt32.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/libbb/getopt32.c b/libbb/getopt32.c index e861d05676..a8dd85159a 100644 --- a/libbb/getopt32.c +++ b/libbb/getopt32.c @@ -348,9 +348,6 @@ vgetopt32(char **argv, const char *applet_opts, const char *applet_long_options, unsigned trigger; int min_arg = 0; int max_arg = -1; - int spec_flgs = 0; - -#define SHOW_USAGE_IF_ERROR 1 on_off = complementary; memset(on_off, 0, sizeof(complementary)); @@ -449,9 +446,7 @@ vgetopt32(char **argv, const char *applet_opts, const char *applet_long_options, continue; c = s[1]; if (*s == '?') { - if (c < '0' || c > '9') { - spec_flgs |= SHOW_USAGE_IF_ERROR; - } else { + if (c >= '0' && c <= '9') { max_arg = c - '0'; s++; } @@ -465,8 +460,10 @@ vgetopt32(char **argv, const char *applet_opts, const char *applet_long_options, continue; } if (*s == '=') { - min_arg = max_arg = c - '0'; - s++; + if (c >= '0' && c <= '9') { + min_arg = max_arg = c - '0'; + s++; + } continue; } for (on_off = complementary; on_off->opt_char; on_off++) From 87e60dcf0f7ef917b73353d8605188a420bd91f9 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Mon, 28 Oct 2024 15:26:21 +0100 Subject: [PATCH 029/277] hexdump: fix regression with -n4 -e '"%u"' Fix bug introduced in busybox 1.37.0 that broke kernel builds. Fixes commit e2287f99fe6f (od: for !DESKTOP, match output more closely to GNU coreutils 9.1, implement -s) function old new delta rewrite 967 976 +9 Signed-off-by: Natanael Copa Signed-off-by: Denys Vlasenko --- libbb/dump.c | 6 ++++-- testsuite/hexdump.tests | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/libbb/dump.c b/libbb/dump.c index b406a24283..2ca9919da4 100644 --- a/libbb/dump.c +++ b/libbb/dump.c @@ -198,9 +198,11 @@ static NOINLINE void rewrite(priv_dumper_t *dumper, FS *fs) if (!e) goto DO_BAD_CONV_CHAR; pr->flags = F_INT; - if (e > int_convs + 1) /* not d or i? */ - pr->flags = F_UINT; byte_count_str = "\010\004\002\001"; + if (e > int_convs + 1) { /* not d or i? */ + pr->flags = F_UINT; + byte_count_str++; + } goto DO_BYTE_COUNT; } else if (strchr(int_convs, *p1)) { /* %d etc */ diff --git a/testsuite/hexdump.tests b/testsuite/hexdump.tests index be0379cfc2..517ec508bb 100755 --- a/testsuite/hexdump.tests +++ b/testsuite/hexdump.tests @@ -82,4 +82,10 @@ testing "hexdump -e /2 %d" \ "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"\ "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"\ +testing "hexdump -n4 -e '\"%u\"'" \ + "hexdump -n4 -e '\"%u\"'" \ + "12345678" \ + "" \ + "\x4e\x61\xbc\x00AAAA" + exit $FAILCOUNT From 50e2b59370542097eb0efd79cc8d3d39ee52bf82 Mon Sep 17 00:00:00 2001 From: Colin McAllister Date: Wed, 17 Jul 2024 16:33:35 -0500 Subject: [PATCH 030/277] cut: fix "-s" flag to omit blank lines Using cut with the delimiter flag ("-d") with the "-s" flag to only output lines containing the delimiter will print blank lines. This is deviant behavior from cut provided by GNU Coreutils. Blank lines should be omitted if "-s" is used with "-d". This change introduces a somewhat naiive, yet efficient solution, where line length is checked before looping though bytes. If line length is zero and the "-s" flag is used, the code will jump to parsing the next line to avoid printing a newline character. function old new delta cut_main 1196 1185 -11 Signed-off-by: Colin McAllister Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 20 +++++++++++++++----- testsuite/cut.tests | 9 +++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index d129f9b9dd..b7fe11126c 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -152,11 +152,18 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, unsigned uu = 0, start = 0, end = 0, out = 0; int dcount = 0; + /* Blank line? Check -s (later check for -s does not catch empty lines) */ + if (linelen == 0) { + if (option_mask32 & CUT_OPT_SUPPRESS_FLGS) + goto next_line; + } + /* Loop through bytes, finding next delimiter */ for (;;) { /* End of current range? */ if (end == linelen || dcount > cut_lists[cl_pos].endpos) { - if (++cl_pos >= nlists) break; + if (++cl_pos >= nlists) + break; if (option_mask32 & CUT_OPT_NOSORT_FLGS) start = dcount = uu = 0; end = 0; @@ -175,15 +182,18 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (shoe) { regmatch_t rr = {-1, -1}; - if (!regexec(®, line+uu, 1, &rr, REG_NOTBOL|REG_NOTEOL)) { + if (!regexec(®, line + uu, 1, &rr, REG_NOTBOL|REG_NOTEOL)) { end = uu + rr.rm_so; uu += rr.rm_eo; } else { uu = linelen; continue; } - } else if (line[end = uu++] != *delim) - continue; + } else { + end = uu++; + if (line[end] != *delim) + continue; + } /* Got delimiter. Loop if not yet within range. */ if (dcount++ < cut_lists[cl_pos].startpos) { @@ -192,7 +202,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } } if (end != start || !shoe) - printf("%s%.*s", out++ ? odelim : "", end-start, line + start); + printf("%s%.*s", out++ ? odelim : "", end - start, line + start); start = uu; if (!dcount) break; diff --git a/testsuite/cut.tests b/testsuite/cut.tests index 2458c019c4..0b401bc006 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -65,6 +65,15 @@ testing "cut with -d -f( ) -s" "cut -d' ' -f3 -s input && echo yes" "yes\n" "$in testing "cut with -d -f(a) -s" "cut -da -f3 -s input" "n\nsium:Jim\n\ncion:Ed\n" "$input" "" testing "cut with -d -f(a) -s -n" "cut -da -f3 -s -n input" "n\nsium:Jim\n\ncion:Ed\n" "$input" "" +input="\ + +foo bar baz + +bing bong boop + +" +testing "cut with -d -s omits blank lines" "cut -d' ' -f2 -s input" "bar\nbong\n" "$input" "" + # substitute for awk optional FEATURE_CUT_REGEX testing "cut -DF" "cut -DF 2,7,5" \ From f02041441344389b05d10fe6ba8759b6670b8e10 Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Sun, 3 Nov 2024 12:47:27 +0000 Subject: [PATCH 031/277] cut: improve detection of invalid ranges Commit 0068ce2fa (cut: add toybox-compatible options -O OUTSEP, -D, -F LIST) added detection of reversed ranges. Further improvements are possible. - The test for reversed ranges compared the start after it had been decremented with the end before decrement. It thus missed ranges of the form 2-1. - Zero isn't a valid start value for a range. (Nor is it a valid end value, but that's caught by the test for a reversed range.) - The code if (!*ltok) e = INT_MAX; duplicates a check that's already been made. - Display the actual range in the error message to make it easier to find which range was at fault. function old new delta .rodata 100273 100287 +14 cut_main 1239 1237 -2 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 14/-2) Total: 12 bytes Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 19 ++++++++----------- testsuite/cut.tests | 5 ++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index b7fe11126c..f68bbbad5c 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -278,29 +278,26 @@ int cut_main(int argc UNUSED_PARAM, char **argv) if (!ntok[0]) { s = 0; } else { - s = xatoi_positive(ntok); /* account for the fact that arrays are zero based, while * the user expects the first char on the line to be char #1 */ - if (s != 0) - s--; + s = xatoi_positive(ntok) - 1; } /* get the end pos */ if (ltok == NULL) { e = s; } else if (!ltok[0]) { + /* if the user specified no end position, + * that means "til the end of the line" */ e = INT_MAX; } else { - e = xatoi_positive(ltok); - /* if the user specified and end position of 0, - * that means "til the end of the line" */ - if (!*ltok) - e = INT_MAX; - else if (e < s) - bb_error_msg_and_die("%d<%d", e, s); - e--; /* again, arrays are zero based, lines are 1 based */ + /* again, arrays are zero based, lines are 1 based */ + e = xatoi_positive(ltok) - 1; } + if (s < 0 || e < s) + bb_error_msg_and_die("invalid range %s-%s", ntok, ltok ?: ntok); + /* add the new list */ cut_lists = xrealloc_vector(cut_lists, 4, nlists); /* NB: startpos is always >= 0 */ diff --git a/testsuite/cut.tests b/testsuite/cut.tests index 0b401bc006..c335f824b4 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -31,7 +31,10 @@ testing "-b encapsulated" "cut -b 3-8,4-6 input" "e:two:\npha:be\ne quic\n" \ #testing "cut -bO overlaps" \ # "cut --output-delimiter ' ' -b 1-3,2-5,7-9,9-10 input" \ # "one:t o:th\nalpha beta\nthe q ick \n" "$abc" "" -testing "cut high-low error" "cut -b 8-3 abc.txt 2>/dev/null || echo err" "err\n" \ +testing "cut high-low error" "cut -b 8-3 input 2>/dev/null || echo err" "err\n" \ + "$abc" "" + +testing "cut -b 2-1 error" "cut -b 2-1 input 2>/dev/null || echo err" "err\n" \ "$abc" "" testing "cut -c a-b" "cut -c 4-10 input" ":two:th\nha:beta\n quick \n" "$abc" "" From 73e9d25d7503896e94b5c00093a77b33d1a17a0d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 01:38:31 +0100 Subject: [PATCH 032/277] cut: simplify OPT_ names, eliminate one variable Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 51 ++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index f68bbbad5c..48f3656b4e 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -65,14 +65,14 @@ typedef struct { int rm_eo, rm_so; } regmatch_t; /* option vars */ #define OPT_STR "b:c:f:d:O:sD"IF_FEATURE_CUT_REGEX("F:")"n" -#define CUT_OPT_BYTE_FLGS (1 << 0) -#define CUT_OPT_CHAR_FLGS (1 << 1) -#define CUT_OPT_FIELDS_FLGS (1 << 2) -#define CUT_OPT_DELIM_FLGS (1 << 3) -#define CUT_OPT_ODELIM_FLGS (1 << 4) -#define CUT_OPT_SUPPRESS_FLGS (1 << 5) -#define CUT_OPT_NOSORT_FLGS (1 << 6) -#define CUT_OPT_REGEX_FLGS ((1 << 7) * ENABLE_FEATURE_CUT_REGEX) +#define OPT_BYTE (1 << 0) +#define OPT_CHAR (1 << 1) +#define OPT_FIELDS (1 << 2) +#define OPT_DELIM (1 << 3) +#define OPT_ODELIM (1 << 4) +#define OPT_SUPPRESS (1 << 5) +#define OPT_NOSORT (1 << 6) +#define OPT_REGEX ((1 << 7) * ENABLE_FEATURE_CUT_REGEX) struct cut_list { int startpos; @@ -88,12 +88,14 @@ static int cmpfunc(const void *a, const void *b) static void cut_file(FILE *file, const char *delim, const char *odelim, const struct cut_list *cut_lists, unsigned nlists) { +#define opt_REGEX (option_mask32 & OPT_REGEX) char *line; unsigned linenum = 0; /* keep these zero-based to be consistent */ regex_t reg; - int spos, shoe = option_mask32 & CUT_OPT_REGEX_FLGS; + int spos; - if (shoe) xregcomp(®, delim, REG_EXTENDED); + if (opt_REGEX) + xregcomp(®, delim, REG_EXTENDED); /* go through every line in the file */ while ((line = xmalloc_fgetline(file)) != NULL) { @@ -105,7 +107,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, unsigned cl_pos = 0; /* cut based on chars/bytes XXX: only works when sizeof(char) == byte */ - if (option_mask32 & (CUT_OPT_CHAR_FLGS | CUT_OPT_BYTE_FLGS)) { + if (option_mask32 & (OPT_CHAR | OPT_BYTE)) { /* print the chars specified in each cut list */ for (; cl_pos < nlists; cl_pos++) { for (spos = cut_lists[cl_pos].startpos; spos < linelen;) { @@ -154,7 +156,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* Blank line? Check -s (later check for -s does not catch empty lines) */ if (linelen == 0) { - if (option_mask32 & CUT_OPT_SUPPRESS_FLGS) + if (option_mask32 & OPT_SUPPRESS) goto next_line; } @@ -164,22 +166,22 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (end == linelen || dcount > cut_lists[cl_pos].endpos) { if (++cl_pos >= nlists) break; - if (option_mask32 & CUT_OPT_NOSORT_FLGS) + if (option_mask32 & OPT_NOSORT) start = dcount = uu = 0; end = 0; } /* End of current line? */ if (uu == linelen) { /* If we've seen no delimiters, check -s */ - if (!cl_pos && !dcount && !shoe) { - if (option_mask32 & CUT_OPT_SUPPRESS_FLGS) + if (!cl_pos && !dcount && !opt_REGEX) { + if (option_mask32 & OPT_SUPPRESS) goto next_line; } else if (dcount < cut_lists[cl_pos].startpos) start = linelen; end = linelen; } else { /* Find next delimiter */ - if (shoe) { + if (opt_REGEX) { regmatch_t rr = {-1, -1}; if (!regexec(®, line + uu, 1, &rr, REG_NOTBOL|REG_NOTEOL)) { @@ -201,7 +203,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, continue; } } - if (end != start || !shoe) + if (end != start || !opt_REGEX) printf("%s%.*s", out++ ? odelim : "", end - start, line + start); start = uu; if (!dcount) @@ -215,6 +217,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, free(printed); free(orig_line); } +#undef opt_REGEX } int cut_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; @@ -235,23 +238,23 @@ int cut_main(int argc UNUSED_PARAM, char **argv) &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) ); if (!delim || !*delim) - delim = (opt & CUT_OPT_REGEX_FLGS) ? "[[:space:]]+" : "\t"; - if (!odelim) odelim = (opt & CUT_OPT_REGEX_FLGS) ? " " : delim; + delim = (opt & OPT_REGEX) ? "[[:space:]]+" : "\t"; + if (!odelim) odelim = (opt & OPT_REGEX) ? " " : delim; // argc -= optind; argv += optind; - if (!(opt & (CUT_OPT_BYTE_FLGS | CUT_OPT_CHAR_FLGS | CUT_OPT_FIELDS_FLGS | CUT_OPT_REGEX_FLGS))) + if (!(opt & (OPT_BYTE | OPT_CHAR | OPT_FIELDS | OPT_REGEX))) bb_simple_error_msg_and_die("expected a list of bytes, characters, or fields"); /* non-field (char or byte) cutting has some special handling */ - if (!(opt & (CUT_OPT_FIELDS_FLGS|CUT_OPT_REGEX_FLGS))) { + if (!(opt & (OPT_FIELDS|OPT_REGEX))) { static const char _op_on_field[] ALIGN1 = " only when operating on fields"; - if (opt & CUT_OPT_SUPPRESS_FLGS) { + if (opt & OPT_SUPPRESS) { bb_error_msg_and_die ("suppressing non-delimited lines makes sense%s", _op_on_field); } - if (opt & CUT_OPT_DELIM_FLGS) { + if (opt & OPT_DELIM) { bb_error_msg_and_die ("a delimiter may be specified%s", _op_on_field); } @@ -313,7 +316,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) /* now that the lists are parsed, we need to sort them to make life * easier on us when it comes time to print the chars / fields / lines */ - if (!(opt & CUT_OPT_NOSORT_FLGS)) + if (!(opt & OPT_NOSORT)) qsort(cut_lists, nlists, sizeof(cut_lists[0]), cmpfunc); } From ad12ab439b5d0383ac4ebe41479b694df0b2e70d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 01:44:13 +0100 Subject: [PATCH 033/277] cut: localize 'spos' variable, convert !NUMVAR to NUMVAR == 0 This imporves readability Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 48f3656b4e..e81c6fecbf 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -92,7 +92,6 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, char *line; unsigned linenum = 0; /* keep these zero-based to be consistent */ regex_t reg; - int spos; if (opt_REGEX) xregcomp(®, delim, REG_EXTENDED); @@ -110,6 +109,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (option_mask32 & (OPT_CHAR | OPT_BYTE)) { /* print the chars specified in each cut list */ for (; cl_pos < nlists; cl_pos++) { + int spos; for (spos = cut_lists[cl_pos].startpos; spos < linelen;) { if (!printed[spos]) { printed[spos] = 'X'; @@ -121,7 +121,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } } } else if (*delim == '\n') { /* cut by lines */ - spos = cut_lists[cl_pos].startpos; + int spos = cut_lists[cl_pos].startpos; /* get out if we have no more lists to process or if the lines * are lower than what we're interested in */ @@ -173,7 +173,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* End of current line? */ if (uu == linelen) { /* If we've seen no delimiters, check -s */ - if (!cl_pos && !dcount && !opt_REGEX) { + if (cl_pos == 0 && dcount == 0 && !opt_REGEX) { if (option_mask32 & OPT_SUPPRESS) goto next_line; } else if (dcount < cut_lists[cl_pos].startpos) @@ -206,7 +206,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (end != start || !opt_REGEX) printf("%s%.*s", out++ ? odelim : "", end - start, line + start); start = uu; - if (!dcount) + if (dcount == 0) break; } } @@ -239,7 +239,8 @@ int cut_main(int argc UNUSED_PARAM, char **argv) ); if (!delim || !*delim) delim = (opt & OPT_REGEX) ? "[[:space:]]+" : "\t"; - if (!odelim) odelim = (opt & OPT_REGEX) ? " " : delim; + if (!odelim) + odelim = (opt & OPT_REGEX) ? " " : delim; // argc -= optind; argv += optind; From 478ac90f2c55cf66c13aa9805bde69bc705647c3 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 01:50:58 +0100 Subject: [PATCH 034/277] cut: allocate "printed" only if OPT_CHAR or OPT_BYTE Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index e81c6fecbf..ca2408f977 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -101,12 +101,13 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* set up a list so we can keep track of what's been printed */ int linelen = strlen(line); - char *printed = xzalloc(linelen + 1); char *orig_line = line; unsigned cl_pos = 0; /* cut based on chars/bytes XXX: only works when sizeof(char) == byte */ if (option_mask32 & (OPT_CHAR | OPT_BYTE)) { + char *printed = xzalloc(linelen + 1); + /* print the chars specified in each cut list */ for (; cl_pos < nlists; cl_pos++) { int spos; @@ -120,6 +121,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } } } + free(printed); } else if (*delim == '\n') { /* cut by lines */ int spos = cut_lists[cl_pos].startpos; @@ -214,9 +216,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, putchar('\n'); next_line: linenum++; - free(printed); free(orig_line); - } + } /* while (got line) */ #undef opt_REGEX } From 808155ebf3e731538643e2fa840daeb76aac5435 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 01:55:25 +0100 Subject: [PATCH 035/277] cut: "orig_line" is redundant, remove it Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index ca2408f977..72a6f2b80f 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -101,7 +101,6 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* set up a list so we can keep track of what's been printed */ int linelen = strlen(line); - char *orig_line = line; unsigned cl_pos = 0; /* cut based on chars/bytes XXX: only works when sizeof(char) == byte */ @@ -216,7 +215,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, putchar('\n'); next_line: linenum++; - free(orig_line); + free(line); } /* while (got line) */ #undef opt_REGEX } From 1220b1519d6bb46b2aba0559969bac7377f15794 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 02:15:25 +0100 Subject: [PATCH 036/277] cut: rename some variables to hopefully better names Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 72a6f2b80f..0fbeff8ea3 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -152,7 +152,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, puts(line); goto next_line; } else { /* cut by fields */ - unsigned uu = 0, start = 0, end = 0, out = 0; + unsigned next = 0, start = 0, end = 0; + int first_print = 1; int dcount = 0; /* Blank line? Check -s (later check for -s does not catch empty lines) */ @@ -168,11 +169,11 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (++cl_pos >= nlists) break; if (option_mask32 & OPT_NOSORT) - start = dcount = uu = 0; + start = dcount = next = 0; end = 0; } /* End of current line? */ - if (uu == linelen) { + if (next == linelen) { /* If we've seen no delimiters, check -s */ if (cl_pos == 0 && dcount == 0 && !opt_REGEX) { if (option_mask32 & OPT_SUPPRESS) @@ -185,31 +186,36 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (opt_REGEX) { regmatch_t rr = {-1, -1}; - if (!regexec(®, line + uu, 1, &rr, REG_NOTBOL|REG_NOTEOL)) { - end = uu + rr.rm_so; - uu += rr.rm_eo; + if (!regexec(®, line + next, 1, &rr, REG_NOTBOL|REG_NOTEOL)) { + end = next + rr.rm_so; + next += rr.rm_eo; } else { - uu = linelen; + next = linelen; continue; } } else { - end = uu++; + end = next++; if (line[end] != *delim) continue; } /* Got delimiter. Loop if not yet within range. */ if (dcount++ < cut_lists[cl_pos].startpos) { - start = uu; + start = next; continue; } } - if (end != start || !opt_REGEX) - printf("%s%.*s", out++ ? odelim : "", end - start, line + start); - start = uu; + if (end != start || !opt_REGEX) { + if (first_print) { + first_print = 0; + printf("%.*s", end - start, line + start); + } else + printf("%s%.*s", odelim, end - start, line + start); + } + start = next; if (dcount == 0) break; - } + } /* byte loop */ } /* if we printed anything, finish with newline */ putchar('\n'); From 55fc6a18da068a67b1854e4ca6fbb8d92e3af745 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 02:29:48 +0100 Subject: [PATCH 037/277] cut: rename "cut_lists" to "cut_list" Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 0fbeff8ea3..33aeff6ea5 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -86,7 +86,7 @@ static int cmpfunc(const void *a, const void *b) } static void cut_file(FILE *file, const char *delim, const char *odelim, - const struct cut_list *cut_lists, unsigned nlists) + const struct cut_list *cut_list, unsigned nlists) { #define opt_REGEX (option_mask32 & OPT_REGEX) char *line; @@ -110,19 +110,19 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* print the chars specified in each cut list */ for (; cl_pos < nlists; cl_pos++) { int spos; - for (spos = cut_lists[cl_pos].startpos; spos < linelen;) { + for (spos = cut_list[cl_pos].startpos; spos < linelen;) { if (!printed[spos]) { printed[spos] = 'X'; putchar(line[spos]); } - if (++spos > cut_lists[cl_pos].endpos) { + if (++spos > cut_list[cl_pos].endpos) { break; } } } free(printed); } else if (*delim == '\n') { /* cut by lines */ - int spos = cut_lists[cl_pos].startpos; + int spos = cut_list[cl_pos].startpos; /* get out if we have no more lists to process or if the lines * are lower than what we're interested in */ @@ -134,12 +134,12 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, while (spos < (int)linenum) { spos++; /* go to the next list if we're at the end of this one */ - if (spos > cut_lists[cl_pos].endpos) { + if (spos > cut_list[cl_pos].endpos) { cl_pos++; /* get out if there's no more lists to process */ if (cl_pos >= nlists) goto next_line; - spos = cut_lists[cl_pos].startpos; + spos = cut_list[cl_pos].startpos; /* get out if the current line is lower than the one * we just became interested in */ if ((int)linenum < spos) @@ -153,8 +153,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, goto next_line; } else { /* cut by fields */ unsigned next = 0, start = 0, end = 0; + int dcount = 0; /* Nth delimiter we saw (0 - didn't see any yet) */ int first_print = 1; - int dcount = 0; /* Blank line? Check -s (later check for -s does not catch empty lines) */ if (linelen == 0) { @@ -165,12 +165,12 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* Loop through bytes, finding next delimiter */ for (;;) { /* End of current range? */ - if (end == linelen || dcount > cut_lists[cl_pos].endpos) { + if (end == linelen || dcount > cut_list[cl_pos].endpos) { if (++cl_pos >= nlists) break; if (option_mask32 & OPT_NOSORT) start = dcount = next = 0; - end = 0; + end = 0; /* (why?) */ } /* End of current line? */ if (next == linelen) { @@ -178,8 +178,9 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (cl_pos == 0 && dcount == 0 && !opt_REGEX) { if (option_mask32 & OPT_SUPPRESS) goto next_line; - } else if (dcount < cut_lists[cl_pos].startpos) - start = linelen; + /* else: will print entire line */ + } else if (dcount < cut_list[cl_pos].startpos) + start = linelen; /* do not print */ end = linelen; } else { /* Find next delimiter */ @@ -200,7 +201,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } /* Got delimiter. Loop if not yet within range. */ - if (dcount++ < cut_lists[cl_pos].startpos) { + if (dcount++ < cut_list[cl_pos].startpos) { start = next; continue; } @@ -230,7 +231,7 @@ int cut_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int cut_main(int argc UNUSED_PARAM, char **argv) { /* growable array holding a series of lists */ - struct cut_list *cut_lists = NULL; + struct cut_list *cut_list = NULL; unsigned nlists = 0; /* number of elements in above list */ char *sopt, *ltok; const char *delim = NULL; @@ -309,10 +310,10 @@ int cut_main(int argc UNUSED_PARAM, char **argv) bb_error_msg_and_die("invalid range %s-%s", ntok, ltok ?: ntok); /* add the new list */ - cut_lists = xrealloc_vector(cut_lists, 4, nlists); + cut_list = xrealloc_vector(cut_list, 4, nlists); /* NB: startpos is always >= 0 */ - cut_lists[nlists].startpos = s; - cut_lists[nlists].endpos = e; + cut_list[nlists].startpos = s; + cut_list[nlists].endpos = e; nlists++; } @@ -324,7 +325,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) * easier on us when it comes time to print the chars / fields / lines */ if (!(opt & OPT_NOSORT)) - qsort(cut_lists, nlists, sizeof(cut_lists[0]), cmpfunc); + qsort(cut_list, nlists, sizeof(cut_list[0]), cmpfunc); } { @@ -339,12 +340,12 @@ int cut_main(int argc UNUSED_PARAM, char **argv) retval = EXIT_FAILURE; continue; } - cut_file(file, delim, odelim, cut_lists, nlists); + cut_file(file, delim, odelim, cut_list, nlists); fclose_if_not_stdin(file); } while (*++argv); if (ENABLE_FEATURE_CLEAN_UP) - free(cut_lists); + free(cut_list); fflush_stdout_and_exit(retval); } } From 470f00955212368cb688832e2e4b1fdd165e9ec6 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 02:36:59 +0100 Subject: [PATCH 038/277] cut: with -F, do not regcomp() pattern for every file function old new delta cut_main 1218 1228 +10 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 33aeff6ea5..1eb4968d95 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -34,6 +34,7 @@ //usage: "\n -c LIST Output only characters from LIST" //usage: "\n -d SEP Field delimiter for input (default -f TAB, -F run of whitespace)" //usage: "\n -O SEP Field delimeter for output (default = -d for -f, one space for -F)" +//TODO: --output-delimiter=SEP //usage: "\n -D Don't sort/collate sections or match -fF lines without delimeter" //usage: "\n -f LIST Print only these fields (-d is single char)" //usage: IF_FEATURE_CUT_REGEX( @@ -53,11 +54,6 @@ #if ENABLE_FEATURE_CUT_REGEX #include "xregex.h" -#else -#define regex_t int -typedef struct { int rm_eo, rm_so; } regmatch_t; -#define xregcomp(x, ...) *(x) = 0 -#define regexec(...) 0 #endif /* This is a NOEXEC applet. Be very careful! */ @@ -74,6 +70,8 @@ typedef struct { int rm_eo, rm_so; } regmatch_t; #define OPT_NOSORT (1 << 6) #define OPT_REGEX ((1 << 7) * ENABLE_FEATURE_CUT_REGEX) +#define opt_REGEX (option_mask32 & OPT_REGEX) + struct cut_list { int startpos; int endpos; @@ -88,13 +86,8 @@ static int cmpfunc(const void *a, const void *b) static void cut_file(FILE *file, const char *delim, const char *odelim, const struct cut_list *cut_list, unsigned nlists) { -#define opt_REGEX (option_mask32 & OPT_REGEX) char *line; unsigned linenum = 0; /* keep these zero-based to be consistent */ - regex_t reg; - - if (opt_REGEX) - xregcomp(®, delim, REG_EXTENDED); /* go through every line in the file */ while ((line = xmalloc_fgetline(file)) != NULL) { @@ -121,7 +114,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } } free(printed); - } else if (*delim == '\n') { /* cut by lines */ + } else if (!opt_REGEX && *delim == '\n') { /* cut by lines */ int spos = cut_list[cl_pos].startpos; /* get out if we have no more lists to process or if the lines @@ -181,20 +174,24 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* else: will print entire line */ } else if (dcount < cut_list[cl_pos].startpos) start = linelen; /* do not print */ - end = linelen; + end = linelen; /* print up to end */ } else { /* Find next delimiter */ +#if ENABLE_FEATURE_CUT_REGEX if (opt_REGEX) { regmatch_t rr = {-1, -1}; + regex_t *reg = (void*) delim; - if (!regexec(®, line + next, 1, &rr, REG_NOTBOL|REG_NOTEOL)) { - end = next + rr.rm_so; - next += rr.rm_eo; - } else { + if (regexec(reg, line + next, 1, &rr, REG_NOTBOL|REG_NOTEOL) != 0) { + /* not found, go to "end of line" logic */ next = linelen; continue; } - } else { + end = next + rr.rm_so; + next += rr.rm_eo; + } else +#endif + { end = next++; if (line[end] != *delim) continue; @@ -224,7 +221,6 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, linenum++; free(line); } /* while (got line) */ -#undef opt_REGEX } int cut_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; @@ -237,6 +233,9 @@ int cut_main(int argc UNUSED_PARAM, char **argv) const char *delim = NULL; const char *odelim = NULL; unsigned opt; +#if ENABLE_FEATURE_CUT_REGEX + regex_t reg; +#endif #define ARG "bcf"IF_FEATURE_CUT_REGEX("F") opt = getopt32(argv, "^" @@ -328,6 +327,13 @@ int cut_main(int argc UNUSED_PARAM, char **argv) qsort(cut_list, nlists, sizeof(cut_list[0]), cmpfunc); } +#if ENABLE_FEATURE_CUT_REGEX + if (opt & OPT_REGEX) { + xregcomp(®, delim, REG_EXTENDED); + delim = (void*) ® + } +#endif + { exitcode_t retval = EXIT_SUCCESS; From 103139d0e6e97c188a647adeb5c71eb39c308c26 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 17:21:30 +0100 Subject: [PATCH 039/277] cut: fix "echo 1.2 | cut -d. -f1,3" (print "1", not "1.") function old new delta cut_main 1228 1201 -27 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 23 ++++++++++++++++++----- testsuite/cut.tests | 11 +++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 1eb4968d95..2511befc89 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -114,7 +114,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } } free(printed); - } else if (!opt_REGEX && *delim == '\n') { /* cut by lines */ + /* Cut by lines */ + } else if (!opt_REGEX && *delim == '\n') { int spos = cut_list[cl_pos].startpos; /* get out if we have no more lists to process or if the lines @@ -144,7 +145,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, * looking for, so print it */ puts(line); goto next_line; - } else { /* cut by fields */ + /* Cut by fields */ + } else { unsigned next = 0, start = 0, end = 0; int dcount = 0; /* Nth delimiter we saw (0 - didn't see any yet) */ int first_print = 1; @@ -159,22 +161,33 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, for (;;) { /* End of current range? */ if (end == linelen || dcount > cut_list[cl_pos].endpos) { + end_of_range: if (++cl_pos >= nlists) break; if (option_mask32 & OPT_NOSORT) start = dcount = next = 0; end = 0; /* (why?) */ + //bb_error_msg("End of current range"); } /* End of current line? */ if (next == linelen) { + end = linelen; /* print up to end */ /* If we've seen no delimiters, check -s */ if (cl_pos == 0 && dcount == 0 && !opt_REGEX) { if (option_mask32 & OPT_SUPPRESS) goto next_line; /* else: will print entire line */ - } else if (dcount < cut_list[cl_pos].startpos) - start = linelen; /* do not print */ - end = linelen; /* print up to end */ + } else if (dcount < cut_list[cl_pos].startpos) { + /* echo 1.2 | cut -d. -f1,3: prints "1", not "1." */ + //break; + /* ^^^ this fails a case with -D: + * echo 1 2 | cut -DF 1,3,2: + * do not end line processing when didn't find field#3 + */ + //if (option_mask32 & OPT_NOSORT) - no, just do it always + goto end_of_range; + } + //bb_error_msg("End of current line: s:%d e:%d", start, end); } else { /* Find next delimiter */ #if ENABLE_FEATURE_CUT_REGEX diff --git a/testsuite/cut.tests b/testsuite/cut.tests index c335f824b4..46ef545d74 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -90,7 +90,14 @@ Subcalifragilisticexpialidocious. Auntie Em: Hate you, hate Kansas. Took the dog. Dorothy." SKIP= -testing "cut empty field" "cut -d ':' -f 1-3" "a::b\n" "" "a::b\n" -testing "cut empty field 2" "cut -d ':' -f 3-5" "b::c\n" "" "a::b::c:d\n" +testing "cut empty field" "cut -d ':' -f 1-3" \ + "a::b\n" \ + "" "a::b\n" +testing "cut empty field 2" "cut -d ':' -f 3-5" \ + "b::c\n" \ + "" "a::b::c:d\n" +testing "cut non-existing field" "cut -d ':' -f1,3" \ + "1\n" \ + "" "1:\n" exit $FAILCOUNT From 38a3c0ad324359904ba3a97b097d8f94ef0cd863 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 17:50:29 +0100 Subject: [PATCH 040/277] cut: comment out code which seems to be not needed Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 2511befc89..54f74fc7ef 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -33,14 +33,14 @@ //usage: "\n -b LIST Output only bytes from LIST" //usage: "\n -c LIST Output only characters from LIST" //usage: "\n -d SEP Field delimiter for input (default -f TAB, -F run of whitespace)" +//usage: "\n -s Drop lines with no delimiter" //usage: "\n -O SEP Field delimeter for output (default = -d for -f, one space for -F)" //TODO: --output-delimiter=SEP -//usage: "\n -D Don't sort/collate sections or match -fF lines without delimeter" //usage: "\n -f LIST Print only these fields (-d is single char)" //usage: IF_FEATURE_CUT_REGEX( //usage: "\n -F LIST Print only these fields (-d is regex)" //usage: ) -//usage: "\n -s Output only lines containing delimiter" +//usage: "\n -D Don't sort/collate sections or match -fF lines without delimeter" //usage: "\n -n Ignored" //(manpage:-n with -b: don't split multibyte characters) //usage: @@ -148,7 +148,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* Cut by fields */ } else { unsigned next = 0, start = 0, end = 0; - int dcount = 0; /* Nth delimiter we saw (0 - didn't see any yet) */ + int dcount = 0; /* we saw Nth delimiter (0 - didn't see any yet) */ int first_print = 1; /* Blank line? Check -s (later check for -s does not catch empty lines) */ @@ -209,9 +209,9 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (line[end] != *delim) continue; } - - /* Got delimiter. Loop if not yet within range. */ + /* Got delimiter */ if (dcount++ < cut_list[cl_pos].startpos) { + /* Not yet within range - loop */ start = next; continue; } @@ -224,8 +224,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, printf("%s%.*s", odelim, end - start, line + start); } start = next; - if (dcount == 0) - break; + //if (dcount == 0) + // break; - why? } /* byte loop */ } /* if we printed anything, finish with newline */ From b402b13fab027185e460a4020ee0a0b897bba441 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 18:04:13 +0100 Subject: [PATCH 041/277] cut: with -F disabled in .config, don't show it in --help function old new delta packed_usage 34897 34849 -48 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 54f74fc7ef..f4d930db03 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -32,15 +32,23 @@ //usage: "Print selected fields from FILEs to stdout\n" //usage: "\n -b LIST Output only bytes from LIST" //usage: "\n -c LIST Output only characters from LIST" -//usage: "\n -d SEP Field delimiter for input (default -f TAB, -F run of whitespace)" -//usage: "\n -s Drop lines with no delimiter" -//usage: "\n -O SEP Field delimeter for output (default = -d for -f, one space for -F)" -//TODO: --output-delimiter=SEP +//usage: IF_FEATURE_CUT_REGEX( +//usage: "\n -d SEP Input field delimiter (default -f TAB, -F run of whitespace)" +//usage: ) IF_NOT_FEATURE_CUT_REGEX( +//usage: "\n -d SEP Input field delimiter (default TAB)" +//usage: ) //usage: "\n -f LIST Print only these fields (-d is single char)" //usage: IF_FEATURE_CUT_REGEX( //usage: "\n -F LIST Print only these fields (-d is regex)" //usage: ) -//usage: "\n -D Don't sort/collate sections or match -fF lines without delimeter" +//usage: "\n -s Drop lines with no delimiter (else print them in full)" +//usage: "\n -D Don't sort/collate sections or match -f"IF_FEATURE_CUT_REGEX("F")" lines without delimeter" +//usage: IF_FEATURE_CUT_REGEX( +//usage: "\n -O SEP Output field delimeter (default = -d for -f, one space for -F)" +//usage: ) IF_NOT_FEATURE_CUT_REGEX( +//usage: "\n -O SEP Output field delimeter (default = -d)" +//usage: ) +//TODO: --output-delimiter=SEP //usage: "\n -n Ignored" //(manpage:-n with -b: don't split multibyte characters) //usage: From a208fa03de8467d1f51821f874cbf142aaad74fa Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 18:49:10 +0100 Subject: [PATCH 042/277] cut: implement --output-delimiter function old new delta cut_main 1204 1261 +57 static.cut_longopts - 20 +20 ------------------------------------------------------------------------------ (add/remove: 1/0 grow/shrink: 1/0 up/down: 77/0) Total: 77 bytes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 27 +++++++++++++++++++++++++-- testsuite/cut.tests | 32 ++++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index f4d930db03..9f5b649d84 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -107,6 +107,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* cut based on chars/bytes XXX: only works when sizeof(char) == byte */ if (option_mask32 & (OPT_CHAR | OPT_BYTE)) { char *printed = xzalloc(linelen + 1); + int need_odelim = 0; /* print the chars specified in each cut list */ for (; cl_pos < nlists; cl_pos++) { @@ -114,9 +115,14 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, for (spos = cut_list[cl_pos].startpos; spos < linelen;) { if (!printed[spos]) { printed[spos] = 'X'; + if (need_odelim && spos != 0 && !printed[spos-1]) { + need_odelim = 0; + fputs_stdout(odelim); + } putchar(line[spos]); } if (++spos > cut_list[cl_pos].endpos) { + need_odelim = (odelim && odelim[0]); /* will print OSEP (if not empty) */ break; } } @@ -165,6 +171,9 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, goto next_line; } + if (!odelim) + odelim = "\t"; + /* Loop through bytes, finding next delimiter */ for (;;) { /* End of current range? */ @@ -257,17 +266,31 @@ int cut_main(int argc UNUSED_PARAM, char **argv) #if ENABLE_FEATURE_CUT_REGEX regex_t reg; #endif +#if ENABLE_LONG_OPTS + static const char cut_longopts[] ALIGN1 = + "output-delimiter\0" Required_argument "O" + ; +#endif #define ARG "bcf"IF_FEATURE_CUT_REGEX("F") +#if !ENABLE_LONG_OPTS opt = getopt32(argv, "^" OPT_STR // = "b:c:f:d:O:sD"IF_FEATURE_CUT_REGEX("F:")"n" "\0" "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX("F--"ARG), &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) ); - if (!delim || !*delim) - delim = (opt & OPT_REGEX) ? "[[:space:]]+" : "\t"; +#else + opt = getopt32long(argv, "^" + OPT_STR // = "b:c:f:d:O:sD"IF_FEATURE_CUT_REGEX("F:")"n" + "\0" "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX("F--"ARG), + cut_longopts, + &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) + ); +#endif if (!odelim) odelim = (opt & OPT_REGEX) ? " " : delim; + if (!delim || !*delim) + delim = (opt & OPT_REGEX) ? "[[:space:]]+" : "\t"; // argc -= optind; argv += optind; diff --git a/testsuite/cut.tests b/testsuite/cut.tests index 46ef545d74..ba5f88d60f 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -23,14 +23,30 @@ the quick brown fox jumps over the lazy dog testing "cut -b a,a,a" "cut -b 3,3,3 input" "e\np\ne\n" "$abc" "" -testing "cut -b overlaps" "cut -b 1-3,2-5,7-9,9-10 input" \ - "one:to:th\nalphabeta\nthe qick \n" "$abc" "" -testing "-b encapsulated" "cut -b 3-8,4-6 input" "e:two:\npha:be\ne quic\n" \ - "$abc" "" -# --output-delimiter not implemnted (yet?) -#testing "cut -bO overlaps" \ -# "cut --output-delimiter ' ' -b 1-3,2-5,7-9,9-10 input" \ -# "one:t o:th\nalpha beta\nthe q ick \n" "$abc" "" +testing "cut -b overlaps" \ + "cut -b 1-3,2-5,7-9,9-10 input" \ + "\ +one:to:th +alphabeta +the qick \n" \ + "$abc" "" +testing "-b encapsulated" \ + "cut -b 3-8,4-6 input" \ + "\ +e:two: +pha:be +e quic\n" \ + "$abc" "" +optional LONG_OPTS +testing "cut -b --output-delimiter overlaps" \ + "cut --output-delimiter='^' -b 1-3,2-5,7-9,9-10 input" \ + "\ +one:t^o:th +alpha^beta +the q^ick \n" \ + "$abc" "" +SKIP= + testing "cut high-low error" "cut -b 8-3 input 2>/dev/null || echo err" "err\n" \ "$abc" "" From 9e364b16d1020cb7b8f8f4d4f3fe1833496b7a12 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 10 Dec 2024 21:01:52 +0100 Subject: [PATCH 043/277] cut: fix -d$'\n' --output-delimiter=@@ behavior function old new delta cut_main 1261 1353 +92 packed_usage 34925 34901 -24 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 92/-24) Total: 68 bytes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 34 +++++++++++++++++++++++++++------- testsuite/cut.tests | 26 ++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 9f5b649d84..2d0a6237cc 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -43,11 +43,19 @@ //usage: ) //usage: "\n -s Drop lines with no delimiter (else print them in full)" //usage: "\n -D Don't sort/collate sections or match -f"IF_FEATURE_CUT_REGEX("F")" lines without delimeter" +//usage: IF_LONG_OPTS( +//usage: IF_FEATURE_CUT_REGEX( +//usage: "\n --output-delimiter SEP Output field delimeter (default = -d for -f, one space for -F)" +//usage: ) IF_NOT_FEATURE_CUT_REGEX( +//usage: "\n --output-delimiter SEP Output field delimeter (default = -d)" +//usage: ) +//usage: ) IF_NOT_LONG_OPTS( //usage: IF_FEATURE_CUT_REGEX( //usage: "\n -O SEP Output field delimeter (default = -d for -f, one space for -F)" //usage: ) IF_NOT_FEATURE_CUT_REGEX( //usage: "\n -O SEP Output field delimeter (default = -d)" //usage: ) +//usage: ) //TODO: --output-delimiter=SEP //usage: "\n -n Ignored" //(manpage:-n with -b: don't split multibyte characters) @@ -96,6 +104,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, { char *line; unsigned linenum = 0; /* keep these zero-based to be consistent */ + int first_print = 1; /* go through every line in the file */ while ((line = xmalloc_fgetline(file)) != NULL) { @@ -130,16 +139,16 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, free(printed); /* Cut by lines */ } else if (!opt_REGEX && *delim == '\n') { - int spos = cut_list[cl_pos].startpos; + unsigned spos = cut_list[cl_pos].startpos; /* get out if we have no more lists to process or if the lines * are lower than what we're interested in */ - if (((int)linenum < spos) || (cl_pos >= nlists)) + if ((linenum < spos) || (cl_pos >= nlists)) goto next_line; /* if the line we're looking for is lower than the one we were * passed, it means we displayed it already, so move on */ - while (spos < (int)linenum) { + while (spos < linenum) { spos++; /* go to the next list if we're at the end of this one */ if (spos > cut_list[cl_pos].endpos) { @@ -150,20 +159,23 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, spos = cut_list[cl_pos].startpos; /* get out if the current line is lower than the one * we just became interested in */ - if ((int)linenum < spos) + if (linenum < spos) goto next_line; } } /* If we made it here, it means we've found the line we're * looking for, so print it */ - puts(line); + if (first_print) { + first_print = 0; + fputs_stdout(line); + } else + printf("%s%s", odelim, line); goto next_line; /* Cut by fields */ } else { unsigned next = 0, start = 0, end = 0; int dcount = 0; /* we saw Nth delimiter (0 - didn't see any yet) */ - int first_print = 1; /* Blank line? Check -s (later check for -s does not catch empty lines) */ if (linelen == 0) { @@ -173,6 +185,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, if (!odelim) odelim = "\t"; + first_print = 1; /* Loop through bytes, finding next delimiter */ for (;;) { @@ -233,7 +246,10 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, continue; } } - if (end != start || !opt_REGEX) { +#if ENABLE_FEATURE_CUT_REGEX + if (end != start || !opt_REGEX) +#endif + { if (first_print) { first_print = 0; printf("%.*s", end - start, line + start); @@ -251,6 +267,10 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, linenum++; free(line); } /* while (got line) */ + + /* For -d$'\n' --output-delimiter=^, the overall output is still terminated with \n, not ^ */ + if (!opt_REGEX && *delim == '\n' && !first_print) + putchar('\n'); } int cut_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; diff --git a/testsuite/cut.tests b/testsuite/cut.tests index ba5f88d60f..8da390cd7a 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -116,4 +116,30 @@ testing "cut non-existing field" "cut -d ':' -f1,3" \ "1\n" \ "" "1:\n" +# cut -d$'\n' has a special meaning: "select input lines". +# I didn't find any documentation for this feature. +testing "cut -dNEWLINE" \ + "cut -d' +' -f4,2,6-8" \ + "2\n4\n6\n7\n" \ + "" "1\n2\n3\n4\n5\n6\n7" + +testing "cut -dNEWLINE --output-delimiter" \ + "cut -d' +' -O@@ -f4,2,6-8" \ + "2@@4@@6@@7\n" \ + "" "1\n2\n3\n4\n5\n6\n7" + +testing "cut -dNEWLINE --output-delimiter 2" \ + "cut -d' +' -O@@ -f4,2,6-8" \ + "2@@4@@6@@7\n" \ + "" "1\n2\n3\n4\n5\n6\n7\n" + +testing "cut -dNEWLINE --output-delimiter EMPTY_INPUT" \ + "cut -d' +' -O@@ -f4,2,6-8" \ + "" \ + "" "" + exit $FAILCOUNT From a4894eaf713f0e452c272db1c5dc2a459e05808f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 11 Dec 2024 01:10:05 +0100 Subject: [PATCH 044/277] cut: tweak --help function old new delta packed_usage 34901 34934 +33 cut_main 1353 1339 -14 .rodata 105724 105685 -39 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/2 up/down: 33/-53) Total: -20 bytes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 17 ++++++++--------- libbb/getopt32.c | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 2d0a6237cc..1e9867858f 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -27,7 +27,8 @@ //kbuild:lib-$(CONFIG_CUT) += cut.o //usage:#define cut_trivial_usage -//usage: "[OPTIONS] [FILE]..." +//usage: "{-b|c LIST | -f"IF_FEATURE_CUT_REGEX("|F")" LIST [-d SEP] [-s]} [-D] [-O SEP] [FILE]..." +// --output-delimiter SEP is too long to fit into 80 char-wide help ----------------^^^^^^^^ //usage:#define cut_full_usage "\n\n" //usage: "Print selected fields from FILEs to stdout\n" //usage: "\n -b LIST Output only bytes from LIST" @@ -44,11 +45,7 @@ //usage: "\n -s Drop lines with no delimiter (else print them in full)" //usage: "\n -D Don't sort/collate sections or match -f"IF_FEATURE_CUT_REGEX("F")" lines without delimeter" //usage: IF_LONG_OPTS( -//usage: IF_FEATURE_CUT_REGEX( -//usage: "\n --output-delimiter SEP Output field delimeter (default = -d for -f, one space for -F)" -//usage: ) IF_NOT_FEATURE_CUT_REGEX( -//usage: "\n --output-delimiter SEP Output field delimeter (default = -d)" -//usage: ) +//usage: "\n --output-delimiter SEP Output field delimeter" //usage: ) IF_NOT_LONG_OPTS( //usage: IF_FEATURE_CUT_REGEX( //usage: "\n -O SEP Output field delimeter (default = -d for -f, one space for -F)" @@ -302,7 +299,8 @@ int cut_main(int argc UNUSED_PARAM, char **argv) #else opt = getopt32long(argv, "^" OPT_STR // = "b:c:f:d:O:sD"IF_FEATURE_CUT_REGEX("F:")"n" - "\0" "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX("F--"ARG), + "\0" "b:c:f:"IF_FEATURE_CUT_REGEX("F:") /* one of -bcfF is required */ + "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX(":F--"ARG), /* they are mutually exclusive */ cut_longopts, &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) ); @@ -314,8 +312,9 @@ int cut_main(int argc UNUSED_PARAM, char **argv) // argc -= optind; argv += optind; - if (!(opt & (OPT_BYTE | OPT_CHAR | OPT_FIELDS | OPT_REGEX))) - bb_simple_error_msg_and_die("expected a list of bytes, characters, or fields"); + //if (!(opt & (OPT_BYTE | OPT_CHAR | OPT_FIELDS | OPT_REGEX))) + // bb_simple_error_msg_and_die("expected a list of bytes, characters, or fields"); + // ^^^ handled by getopt32 /* non-field (char or byte) cutting has some special handling */ if (!(opt & (OPT_FIELDS|OPT_REGEX))) { diff --git a/libbb/getopt32.c b/libbb/getopt32.c index a8dd85159a..b5efa19ac5 100644 --- a/libbb/getopt32.c +++ b/libbb/getopt32.c @@ -93,7 +93,7 @@ getopt32(char **argv, const char *applet_opts, ...) "!" If the first character in the applet_opts string is a '!', report bad options, missing required options, - inconsistent options with all-ones return value (instead of abort. + inconsistent options with all-ones return value instead of aborting. "+" If the first character in the applet_opts string is a plus, then option processing will stop as soon as a non-option is @@ -265,7 +265,7 @@ Special characters: for "long options only" cases, such as tar --exclude=PATTERN, wget --header=HDR cases. - "a?b" A "?" between an option and a group of options means that + "a?bc" A "?" between an option and a group of options means that at least one of them is required to occur if the first option occurs in preceding command line arguments. From 9adafbc1184a413999e7c8bbfc2de85bda3e0b97 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 13 Dec 2024 19:06:58 +0100 Subject: [PATCH 045/277] cut: prevent infinite loop if -F REGEX matches empty delimiter function old new delta cut_main 1339 1348 +9 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 1e9867858f..e12c567329 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -228,7 +228,9 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, continue; } end = next + rr.rm_so; - next += rr.rm_eo; + next += (rr.rm_eo ? rr.rm_eo : 1); + /* ^^^ advancing by at least 1 prevents infinite loops */ + /* testcase: echo "no at sign" | cut -d'@*' -F 1- */ } else #endif { From 96a230f17ba6cb1a7ad1e383d595424da02e87fc Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 13 Dec 2024 19:31:41 +0100 Subject: [PATCH 046/277] cut: fix handling of -d '' Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 2 +- testsuite/cut.tests | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index e12c567329..e33626d7f8 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -309,7 +309,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) #endif if (!odelim) odelim = (opt & OPT_REGEX) ? " " : delim; - if (!delim || !*delim) + if (!delim) delim = (opt & OPT_REGEX) ? "[[:space:]]+" : "\t"; // argc -= optind; diff --git a/testsuite/cut.tests b/testsuite/cut.tests index 8da390cd7a..52d05fde39 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -142,4 +142,16 @@ testing "cut -dNEWLINE --output-delimiter EMPTY_INPUT" \ "" \ "" "" +# This seems to work as if delimiter is never found. +# We test here that -d '' does *not* operate as if there was no -d +# and delimiter has defaulted to TAB: +testing "cut -d EMPTY" \ + "cut -d '' -f2-" \ + "1 2\t3 4 5\n" \ + "" "1 2\t3 4 5\n" +testing "cut -d EMPTY -s" \ + "cut -d '' -f2- -s" \ + "" \ + "" "1 2\t3 4 5\n" + exit $FAILCOUNT From ed6561685c8b37141a17459655774ec3386dd021 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 13 Dec 2024 19:39:34 +0100 Subject: [PATCH 047/277] cut: tweak comments Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index e33626d7f8..e624ab4a58 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -53,9 +53,8 @@ //usage: "\n -O SEP Output field delimeter (default = -d)" //usage: ) //usage: ) -//TODO: --output-delimiter=SEP //usage: "\n -n Ignored" -//(manpage:-n with -b: don't split multibyte characters) +//(manpage:-n with -b: don't split multibyte characters) //usage: //usage:#define cut_example_usage //usage: "$ echo \"Hello world\" | cut -f 1 -d ' '\n" @@ -318,7 +317,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) // bb_simple_error_msg_and_die("expected a list of bytes, characters, or fields"); // ^^^ handled by getopt32 - /* non-field (char or byte) cutting has some special handling */ + /* non-field (char or byte) cutting has some special handling */ if (!(opt & (OPT_FIELDS|OPT_REGEX))) { static const char _op_on_field[] ALIGN1 = " only when operating on fields"; From f4f8dc68645c061d4ccec5481bc3a659b02bc84f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 13 Dec 2024 19:43:53 +0100 Subject: [PATCH 048/277] cut: whitespace fixes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index e624ab4a58..6eac7793f4 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -285,9 +285,9 @@ int cut_main(int argc UNUSED_PARAM, char **argv) regex_t reg; #endif #if ENABLE_LONG_OPTS - static const char cut_longopts[] ALIGN1 = - "output-delimiter\0" Required_argument "O" - ; + static const char cut_longopts[] ALIGN1 = + "output-delimiter\0" Required_argument "O" + ; #endif #define ARG "bcf"IF_FEATURE_CUT_REGEX("F") From 7624077772878db25d8221fc4d6f731e29ebcdba Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 13 Dec 2024 20:22:05 +0100 Subject: [PATCH 049/277] cut: "it's legal to pass an empty list" seems to be untrue function old new delta cut_main 1344 1339 -5 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 90 ++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 6eac7793f4..9a99ad05c3 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -336,61 +336,59 @@ int cut_main(int argc UNUSED_PARAM, char **argv) * valid list formats: N, N-, N-M, -M * more than one list can be separated by commas */ - { + /* take apart the lists, one by one (they are separated with commas) */ + while ((ltok = strsep(&sopt, ",")) != NULL) { char *ntok; - int s = 0, e = 0; - - /* take apart the lists, one by one (they are separated with commas) */ - while ((ltok = strsep(&sopt, ",")) != NULL) { - - /* it's actually legal to pass an empty list */ - if (!ltok[0]) - continue; + int s, e; - /* get the start pos */ - ntok = strsep(<ok, "-"); - if (!ntok[0]) { - s = 0; - } else { - /* account for the fact that arrays are zero based, while - * the user expects the first char on the line to be char #1 */ - s = xatoi_positive(ntok) - 1; - } - - /* get the end pos */ - if (ltok == NULL) { - e = s; - } else if (!ltok[0]) { - /* if the user specified no end position, - * that means "til the end of the line" */ - e = INT_MAX; - } else { - /* again, arrays are zero based, lines are 1 based */ - e = xatoi_positive(ltok) - 1; - } + /* it's actually legal to pass an empty list */ + //if (!ltok[0]) + // continue; + //^^^ testcase? - if (s < 0 || e < s) - bb_error_msg_and_die("invalid range %s-%s", ntok, ltok ?: ntok); + /* get the start pos */ + ntok = strsep(<ok, "-"); + if (!ntok[0]) { + s = 0; + } else { + /* account for the fact that arrays are zero based, while + * the user expects the first char on the line to be char #1 */ + s = xatoi_positive(ntok) - 1; + } - /* add the new list */ - cut_list = xrealloc_vector(cut_list, 4, nlists); - /* NB: startpos is always >= 0 */ - cut_list[nlists].startpos = s; - cut_list[nlists].endpos = e; - nlists++; + /* get the end pos */ + if (ltok == NULL) { + e = s; + } else if (!ltok[0]) { + /* if the user specified no end position, + * that means "til the end of the line" */ + e = INT_MAX; + } else { + /* again, arrays are zero based, lines are 1 based */ + e = xatoi_positive(ltok) - 1; } - /* make sure we got some cut positions out of all that */ - if (nlists == 0) - bb_simple_error_msg_and_die("missing list of positions"); + if (s < 0 || e < s) + bb_error_msg_and_die("invalid range %s-%s", ntok, ltok ?: ntok); - /* now that the lists are parsed, we need to sort them to make life - * easier on us when it comes time to print the chars / fields / lines - */ - if (!(opt & OPT_NOSORT)) - qsort(cut_list, nlists, sizeof(cut_list[0]), cmpfunc); + /* add the new list */ + cut_list = xrealloc_vector(cut_list, 4, nlists); + /* NB: startpos is always >= 0 */ + cut_list[nlists].startpos = s; + cut_list[nlists].endpos = e; + nlists++; } + /* make sure we got some cut positions out of all that */ + if (nlists == 0) + bb_simple_error_msg_and_die("missing list of positions"); + + /* now that the lists are parsed, we need to sort them to make life + * easier on us when it comes time to print the chars / fields / lines + */ + if (!(opt & OPT_NOSORT)) + qsort(cut_list, nlists, sizeof(cut_list[0]), cmpfunc); + #if ENABLE_FEATURE_CUT_REGEX if (opt & OPT_REGEX) { xregcomp(®, delim, REG_EXTENDED); From d48400d8fb266526059eb43b318cb95132f61fb3 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 13 Dec 2024 20:35:28 +0100 Subject: [PATCH 050/277] cut: simplify getopt32 code Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 9a99ad05c3..f4cf5401b1 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -291,21 +291,18 @@ int cut_main(int argc UNUSED_PARAM, char **argv) #endif #define ARG "bcf"IF_FEATURE_CUT_REGEX("F") -#if !ENABLE_LONG_OPTS - opt = getopt32(argv, "^" - OPT_STR // = "b:c:f:d:O:sD"IF_FEATURE_CUT_REGEX("F:")"n" - "\0" "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX("F--"ARG), - &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) - ); +#if ENABLE_LONG_OPTS + opt = getopt32long #else - opt = getopt32long(argv, "^" + opt = getopt32 +#endif + (argv, "^" OPT_STR // = "b:c:f:d:O:sD"IF_FEATURE_CUT_REGEX("F:")"n" - "\0" "b:c:f:"IF_FEATURE_CUT_REGEX("F:") /* one of -bcfF is required */ + "\0" "b:c:f:" IF_FEATURE_CUT_REGEX("F:") /* one of -bcfF is required */ "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX(":F--"ARG), /* they are mutually exclusive */ - cut_longopts, + IF_LONG_OPTS(cut_longopts,) &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) - ); -#endif + ); if (!odelim) odelim = (opt & OPT_REGEX) ? " " : delim; if (!delim) From b25ea3f156c6b5927b358f9b12b90f1852f30311 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 14 Dec 2024 13:58:49 +0100 Subject: [PATCH 051/277] cut: fix -F n-m to match toybox function old new delta cut_main 1339 1391 +52 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 10 ++++++++++ testsuite/cut.tests | 21 ++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index f4cf5401b1..3abebe7ad1 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -243,6 +243,16 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, start = next; continue; } + /* -F N-M preserves intermediate delimiters: */ + //printf "1 2 3 4 5 6 7\n" | toybox cut -O: -F2,4-6,7 + //2:4 5 6:7 + if (opt_REGEX && dcount <= cut_list[cl_pos].endpos) + continue; +// NB: toybox does the above for -f too, but it's a compatibility bug: +//printf "1 2 3 4 5 6 7 8\n" | toybox cut -d' ' -O: -f2,4-6,7 +//2:4 5 6:7 // WRONG! +//printf "1 2 3 4 5 6 7 8\n" | cut -d' ' --output-delimiter=: -f2,4-6,7 +//2:4:5:6:7 // GNU coreutils 9.1 } #if ENABLE_FEATURE_CUT_REGEX if (end != start || !opt_REGEX) diff --git a/testsuite/cut.tests b/testsuite/cut.tests index 52d05fde39..e57b028ac4 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -104,6 +104,19 @@ Weather forecast for tonight : dark. Apple: you can buy better, but you can't pay more. Subcalifragilisticexpialidocious. Auntie Em: Hate you, hate Kansas. Took the dog. Dorothy." + +optional FEATURE_CUT_REGEX LONG_OPTS +testing "cut -F preserves intermediate delimiters" \ + "cut --output-delimiter=: -F2,4-6,7" \ + "2:4 5 6:7\n" \ + "" "1 2 3 4\t\t5 6 7 8\n" +SKIP= + +optional LONG_OPTS +testing "cut -f does not preserve intermediate delimiters" \ + "cut --output-delimiter=: -d' ' -f2,4-6,7" \ + "2:4:5:6:7\n" \ + "" "1 2 3 4 5 6 7 8\n" SKIP= testing "cut empty field" "cut -d ':' -f 1-3" \ @@ -124,23 +137,25 @@ testing "cut -dNEWLINE" \ "2\n4\n6\n7\n" \ "" "1\n2\n3\n4\n5\n6\n7" +optional LONG_OPTS testing "cut -dNEWLINE --output-delimiter" \ "cut -d' -' -O@@ -f4,2,6-8" \ +' --output-delimiter=@@ -f4,2,6-8" \ "2@@4@@6@@7\n" \ "" "1\n2\n3\n4\n5\n6\n7" testing "cut -dNEWLINE --output-delimiter 2" \ "cut -d' -' -O@@ -f4,2,6-8" \ +' --output-delimiter=@@ -f4,2,6-8" \ "2@@4@@6@@7\n" \ "" "1\n2\n3\n4\n5\n6\n7\n" testing "cut -dNEWLINE --output-delimiter EMPTY_INPUT" \ "cut -d' -' -O@@ -f4,2,6-8" \ +' --output-delimiter=@@ -f4,2,6-8" \ "" \ "" "" +SKIP= # This seems to work as if delimiter is never found. # We test here that -d '' does *not* operate as if there was no -d From e8622f0d8524baf8a08a734b7d0fb6aac31b5ee2 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 15 Dec 2024 12:26:07 +0100 Subject: [PATCH 052/277] cut: disallow -f '' and -f '-' function old new delta cut_main 1391 1410 +19 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 3abebe7ad1..20138075cc 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -85,8 +85,8 @@ #define opt_REGEX (option_mask32 & OPT_REGEX) struct cut_list { - int startpos; - int endpos; + unsigned startpos; + unsigned endpos; }; static int cmpfunc(const void *a, const void *b) @@ -116,7 +116,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* print the chars specified in each cut list */ for (; cl_pos < nlists; cl_pos++) { - int spos; + unsigned spos; for (spos = cut_list[cl_pos].startpos; spos < linelen;) { if (!printed[spos]) { printed[spos] = 'X'; @@ -171,7 +171,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* Cut by fields */ } else { unsigned next = 0, start = 0, end = 0; - int dcount = 0; /* we saw Nth delimiter (0 - didn't see any yet) */ + unsigned dcount = 0; /* we saw Nth delimiter (0 - didn't see any yet) */ /* Blank line? Check -s (later check for -s does not catch empty lines) */ if (linelen == 0) { @@ -340,10 +340,10 @@ int cut_main(int argc UNUSED_PARAM, char **argv) /* * parse list and put values into startpos and endpos. - * valid list formats: N, N-, N-M, -M - * more than one list can be separated by commas + * valid range formats: N, N-, N-M, -M + * more than one range can be separated by commas */ - /* take apart the lists, one by one (they are separated with commas) */ + /* take apart the ranges, one by one (separated with commas) */ while ((ltok = strsep(&sopt, ",")) != NULL) { char *ntok; int s, e; @@ -356,22 +356,26 @@ int cut_main(int argc UNUSED_PARAM, char **argv) /* get the start pos */ ntok = strsep(<ok, "-"); if (!ntok[0]) { - s = 0; + if (!ltok) /* testcase: -f '' */ + bb_show_usage(); + if (!ltok[0]) /* testcase: -f - */ + bb_show_usage(); + s = 0; /* "-M" means "1-M" */ } else { - /* account for the fact that arrays are zero based, while - * the user expects the first char on the line to be char #1 */ + /* "N" or "N-[M]" */ + /* arrays are zero based, while the user expects + * the first field/char on the line to be char #1 */ s = xatoi_positive(ntok) - 1; } /* get the end pos */ - if (ltok == NULL) { - e = s; + if (!ltok) { + e = s; /* "N" means "N-N" */ } else if (!ltok[0]) { - /* if the user specified no end position, - * that means "til the end of the line" */ + /* "N-" means "until the end of the line" */ e = INT_MAX; } else { - /* again, arrays are zero based, lines are 1 based */ + /* again, arrays are zero based, fields are 1 based */ e = xatoi_positive(ltok) - 1; } From ba9651b803f0eb2a8cba0205ae72c75ab773adaf Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 16 Dec 2024 00:19:43 +0100 Subject: [PATCH 053/277] cut: we can't get empty cut_list[], remove the check for that function old new delta .rodata 105685 105659 -26 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 20138075cc..3d9f2b373f 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -287,7 +287,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) /* growable array holding a series of lists */ struct cut_list *cut_list = NULL; unsigned nlists = 0; /* number of elements in above list */ - char *sopt, *ltok; + char *LIST, *ltok; const char *delim = NULL; const char *odelim = NULL; unsigned opt; @@ -311,7 +311,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) "\0" "b:c:f:" IF_FEATURE_CUT_REGEX("F:") /* one of -bcfF is required */ "b--"ARG":c--"ARG":f--"ARG IF_FEATURE_CUT_REGEX(":F--"ARG), /* they are mutually exclusive */ IF_LONG_OPTS(cut_longopts,) - &sopt, &sopt, &sopt, &delim, &odelim IF_FEATURE_CUT_REGEX(, &sopt) + &LIST, &LIST, &LIST, &delim, &odelim IF_FEATURE_CUT_REGEX(, &LIST) ); if (!odelim) odelim = (opt & OPT_REGEX) ? " " : delim; @@ -322,7 +322,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) argv += optind; //if (!(opt & (OPT_BYTE | OPT_CHAR | OPT_FIELDS | OPT_REGEX))) // bb_simple_error_msg_and_die("expected a list of bytes, characters, or fields"); - // ^^^ handled by getopt32 + //^^^ handled by getopt32 /* non-field (char or byte) cutting has some special handling */ if (!(opt & (OPT_FIELDS|OPT_REGEX))) { @@ -344,7 +344,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) * more than one range can be separated by commas */ /* take apart the ranges, one by one (separated with commas) */ - while ((ltok = strsep(&sopt, ",")) != NULL) { + while ((ltok = strsep(&LIST, ",")) != NULL) { char *ntok; int s, e; @@ -382,17 +382,20 @@ int cut_main(int argc UNUSED_PARAM, char **argv) if (s < 0 || e < s) bb_error_msg_and_die("invalid range %s-%s", ntok, ltok ?: ntok); - /* add the new list */ + /* add the new range */ cut_list = xrealloc_vector(cut_list, 4, nlists); - /* NB: startpos is always >= 0 */ + /* NB: s is always >= 0 */ cut_list[nlists].startpos = s; cut_list[nlists].endpos = e; nlists++; } /* make sure we got some cut positions out of all that */ - if (nlists == 0) - bb_simple_error_msg_and_die("missing list of positions"); + //if (nlists == 0) + // bb_simple_error_msg_and_die("missing list of positions"); + //^^^ this is impossible since one of -bcfF is required, + // they populate LIST with non-empty string and when it is parsed, + // cut_list[] gets at least one element. /* now that the lists are parsed, we need to sort them to make life * easier on us when it comes time to print the chars / fields / lines From 0bd84c94720b5a68d97c05975220e6482c455623 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 16 Dec 2024 00:32:57 +0100 Subject: [PATCH 054/277] cut: terminate cut_list[] so that we don't need "size of the array" variable function old new delta cut_main 1410 1404 -6 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 3d9f2b373f..0913c4b688 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -84,19 +84,22 @@ #define opt_REGEX (option_mask32 & OPT_REGEX) -struct cut_list { +struct cut_range { unsigned startpos; unsigned endpos; }; static int cmpfunc(const void *a, const void *b) { - return (((struct cut_list *) a)->startpos - - ((struct cut_list *) b)->startpos); + return (((struct cut_range *) a)->startpos - + ((struct cut_range *) b)->startpos); } +#define END_OF_LIST(list_elem) ((list_elem).startpos == UINT_MAX) +#define NOT_END_OF_LIST(list_elem) ((list_elem).startpos != UINT_MAX) + static void cut_file(FILE *file, const char *delim, const char *odelim, - const struct cut_list *cut_list, unsigned nlists) + const struct cut_range *cut_list) { char *line; unsigned linenum = 0; /* keep these zero-based to be consistent */ @@ -115,7 +118,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, int need_odelim = 0; /* print the chars specified in each cut list */ - for (; cl_pos < nlists; cl_pos++) { + for (; NOT_END_OF_LIST(cut_list[cl_pos]); cl_pos++) { unsigned spos; for (spos = cut_list[cl_pos].startpos; spos < linelen;) { if (!printed[spos]) { @@ -137,9 +140,9 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } else if (!opt_REGEX && *delim == '\n') { unsigned spos = cut_list[cl_pos].startpos; - /* get out if we have no more lists to process or if the lines + /* get out if we have no more ranges to process or if the lines * are lower than what we're interested in */ - if ((linenum < spos) || (cl_pos >= nlists)) + if ((linenum < spos) || END_OF_LIST(cut_list[cl_pos])) goto next_line; /* if the line we're looking for is lower than the one we were @@ -149,8 +152,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* go to the next list if we're at the end of this one */ if (spos > cut_list[cl_pos].endpos) { cl_pos++; - /* get out if there's no more lists to process */ - if (cl_pos >= nlists) + /* get out if there's no more ranges to process */ + if (END_OF_LIST(cut_list[cl_pos])) goto next_line; spos = cut_list[cl_pos].startpos; /* get out if the current line is lower than the one @@ -188,7 +191,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* End of current range? */ if (end == linelen || dcount > cut_list[cl_pos].endpos) { end_of_range: - if (++cl_pos >= nlists) + cl_pos++; + if (END_OF_LIST(cut_list[cl_pos])) break; if (option_mask32 & OPT_NOSORT) start = dcount = next = 0; @@ -284,9 +288,9 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, int cut_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int cut_main(int argc UNUSED_PARAM, char **argv) { - /* growable array holding a series of lists */ - struct cut_list *cut_list = NULL; - unsigned nlists = 0; /* number of elements in above list */ + /* growable array holding a series of ranges */ + struct cut_range *cut_list = NULL; + unsigned nranges = 0; /* number of elements in above list */ char *LIST, *ltok; const char *delim = NULL; const char *odelim = NULL; @@ -383,15 +387,16 @@ int cut_main(int argc UNUSED_PARAM, char **argv) bb_error_msg_and_die("invalid range %s-%s", ntok, ltok ?: ntok); /* add the new range */ - cut_list = xrealloc_vector(cut_list, 4, nlists); + cut_list = xrealloc_vector(cut_list, 4, nranges); /* NB: s is always >= 0 */ - cut_list[nlists].startpos = s; - cut_list[nlists].endpos = e; - nlists++; + cut_list[nranges].startpos = s; + cut_list[nranges].endpos = e; + nranges++; } + cut_list[nranges].startpos = UINT_MAX; /* end indicator */ /* make sure we got some cut positions out of all that */ - //if (nlists == 0) + //if (nranges == 0) // bb_simple_error_msg_and_die("missing list of positions"); //^^^ this is impossible since one of -bcfF is required, // they populate LIST with non-empty string and when it is parsed, @@ -401,7 +406,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) * easier on us when it comes time to print the chars / fields / lines */ if (!(opt & OPT_NOSORT)) - qsort(cut_list, nlists, sizeof(cut_list[0]), cmpfunc); + qsort(cut_list, nranges, sizeof(cut_list[0]), cmpfunc); #if ENABLE_FEATURE_CUT_REGEX if (opt & OPT_REGEX) { @@ -422,7 +427,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) retval = EXIT_FAILURE; continue; } - cut_file(file, delim, odelim, cut_list, nlists); + cut_file(file, delim, odelim, cut_list); fclose_if_not_stdin(file); } while (*++argv); From ee8b94acbf9ac9a6a48831a244cfef897affe82e Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 16 Dec 2024 00:51:31 +0100 Subject: [PATCH 055/277] cut: shorten error messages on bad syntax We don't need to mimic GNU cut error messages. $ cut -d@ -b3 cut: -d DELIM makes sense only with -f or -F $ cut -s -b3 cut: -s makes sense only with -f or -F function old new delta static._op_on_field 31 32 +1 .rodata 105659 105598 -61 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 1/-61) Total: -60 bytes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 0913c4b688..af2fd9fd4e 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -330,15 +330,15 @@ int cut_main(int argc UNUSED_PARAM, char **argv) /* non-field (char or byte) cutting has some special handling */ if (!(opt & (OPT_FIELDS|OPT_REGEX))) { - static const char _op_on_field[] ALIGN1 = " only when operating on fields"; + static const char _op_on_field[] ALIGN1 = " makes sense only with -f"IF_FEATURE_CUT_REGEX(" or -F"); if (opt & OPT_SUPPRESS) { bb_error_msg_and_die - ("suppressing non-delimited lines makes sense%s", _op_on_field); + ("-s%s", _op_on_field); } if (opt & OPT_DELIM) { bb_error_msg_and_die - ("a delimiter may be specified%s", _op_on_field); + ("-d DELIM%s", _op_on_field); } } From e2304d47a98a2eff65a3a0a13c53df2cc69529b1 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 16 Dec 2024 01:10:55 +0100 Subject: [PATCH 056/277] cut: shorten error messages on bad syntax even more $ cut -s -b3 cut: -s requires -f or -F $ cut -d@ -b3 cut: -d DELIM requires -f or -F function old new delta static.requires_f - 19 +19 static._op_on_field 32 - -32 ------------------------------------------------------------------------------ (add/remove: 1/1 grow/shrink: 0/0 up/down: 19/-32) Total: -13 bytes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index af2fd9fd4e..74a704c8fb 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -330,16 +330,12 @@ int cut_main(int argc UNUSED_PARAM, char **argv) /* non-field (char or byte) cutting has some special handling */ if (!(opt & (OPT_FIELDS|OPT_REGEX))) { - static const char _op_on_field[] ALIGN1 = " makes sense only with -f"IF_FEATURE_CUT_REGEX(" or -F"); - - if (opt & OPT_SUPPRESS) { - bb_error_msg_and_die - ("-s%s", _op_on_field); - } - if (opt & OPT_DELIM) { - bb_error_msg_and_die - ("-d DELIM%s", _op_on_field); - } + static const char requires_f[] ALIGN1 = " requires -f" + IF_FEATURE_CUT_REGEX(" or -F"); + if (opt & OPT_SUPPRESS) + bb_error_msg_and_die("-s%s", requires_f); + if (opt & OPT_DELIM) + bb_error_msg_and_die("-d DELIM%s", requires_f); } /* From dd40b40ee59e1eae6a70265741657bc55d960cce Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 20 Dec 2024 21:46:32 +0100 Subject: [PATCH 057/277] cut: remove unnecessary initialization of regmatch_t function old new delta cut_main 1404 1388 -16 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 74a704c8fb..a766db40f5 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -222,7 +222,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* Find next delimiter */ #if ENABLE_FEATURE_CUT_REGEX if (opt_REGEX) { - regmatch_t rr = {-1, -1}; + regmatch_t rr; regex_t *reg = (void*) delim; if (regexec(reg, line + next, 1, &rr, REG_NOTBOL|REG_NOTEOL) != 0) { From b03f5162ac239c3743cfac246b3760b0020f4d23 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 20 Dec 2024 22:12:33 +0100 Subject: [PATCH 058/277] cut: fix up -D/-s behavior with -F function old new delta cut_main 1388 1402 +14 packed_usage 34934 34933 -1 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 14/-1) Total: 13 bytes Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 6 +++--- testsuite/cut.tests | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index a766db40f5..65e0e5c30b 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -43,7 +43,7 @@ //usage: "\n -F LIST Print only these fields (-d is regex)" //usage: ) //usage: "\n -s Drop lines with no delimiter (else print them in full)" -//usage: "\n -D Don't sort/collate sections or match -f"IF_FEATURE_CUT_REGEX("F")" lines without delimeter" +//usage: "\n -D Don't sort ranges; line without delimiters has one field" //usage: IF_LONG_OPTS( //usage: "\n --output-delimiter SEP Output field delimeter" //usage: ) IF_NOT_LONG_OPTS( @@ -202,8 +202,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* End of current line? */ if (next == linelen) { end = linelen; /* print up to end */ - /* If we've seen no delimiters, check -s */ - if (cl_pos == 0 && dcount == 0 && !opt_REGEX) { + /* If we've seen no delimiters, and no -D, check -s */ + if (!(option_mask32 & OPT_NOSORT) && cl_pos == 0 && dcount == 0) { if (option_mask32 & OPT_SUPPRESS) goto next_line; /* else: will print entire line */ diff --git a/testsuite/cut.tests b/testsuite/cut.tests index e57b028ac4..21cfea809f 100755 --- a/testsuite/cut.tests +++ b/testsuite/cut.tests @@ -95,8 +95,16 @@ testing "cut with -d -s omits blank lines" "cut -d' ' -f2 -s input" "bar\nbong\n # substitute for awk optional FEATURE_CUT_REGEX -testing "cut -DF" "cut -DF 2,7,5" \ - "said and your\nare\nis demand. supply\nforecast :\nyou you better,\n\nEm: Took hate\n" "" \ +testing "cut -DF unordered" "cut -DF 2,7,5" \ + "\ +said and your +are +is demand. supply +forecast : +you you better, + +Em: Took hate +" "" \ "Bother, said Pooh. It's your husband, and he has a gun. Cheerios are donut seeds. Talk is cheap because supply exceeds demand. @@ -105,6 +113,29 @@ Apple: you can buy better, but you can't pay more. Subcalifragilisticexpialidocious. Auntie Em: Hate you, hate Kansas. Took the dog. Dorothy." +# No delimiter found: print entire line regardless of -F RANGES +testing "cut -F1" "cut -d: -F1" \ + "the_only_field\n" "" \ + "the_only_field\n" +testing "cut -F2" "cut -d: -F2" \ + "the_only_field\n" "" \ + "the_only_field\n" +# No delimiter found and -s: skip entire line +testing "cut -sF1" "cut -d: -sF1" \ + "" "" \ + "the_only_field\n" +#^^^ the above is probably mishandled by toybox, it prints the line +testing "cut -sF2" "cut -d: -sF2" \ + "" "" \ + "the_only_field\n" +# -D disables special handling of lines with no delimiters, the line is treated as the 1st field +testing "cut -DF1" "cut -d: -DF1" \ + "the_only_field\n" "" \ + "the_only_field\n" +testing "cut -DF2" "cut -d: -DF2" \ + "\n" "" \ + "the_only_field\n" + optional FEATURE_CUT_REGEX LONG_OPTS testing "cut -F preserves intermediate delimiters" \ "cut --output-delimiter=: -F2,4-6,7" \ From 1ea89fa98a7c4b1b6924f136963c91caf5a5dccb Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 21 Dec 2024 00:24:30 +0100 Subject: [PATCH 059/277] cut: code shrink This change eliminates one temporary: - if (dcount++ < cut_list[cl_pos].startpos) + dcount++; + if (dcount <= cut_list[cl_pos].startpos) function old new delta cut_main 1402 1373 -29 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 65e0e5c30b..93b58b4936 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -91,8 +91,9 @@ struct cut_range { static int cmpfunc(const void *a, const void *b) { - return (((struct cut_range *) a)->startpos - - ((struct cut_range *) b)->startpos); + const struct cut_range *aa = a; + const struct cut_range *bb = b; + return aa->startpos - bb->startpos; } #define END_OF_LIST(list_elem) ((list_elem).startpos == UINT_MAX) @@ -109,18 +110,18 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, while ((line = xmalloc_fgetline(file)) != NULL) { /* set up a list so we can keep track of what's been printed */ - int linelen = strlen(line); + unsigned linelen = strlen(line); unsigned cl_pos = 0; - /* cut based on chars/bytes XXX: only works when sizeof(char) == byte */ + /* Cut based on chars/bytes XXX: only works when sizeof(char) == byte */ if (option_mask32 & (OPT_CHAR | OPT_BYTE)) { char *printed = xzalloc(linelen + 1); int need_odelim = 0; /* print the chars specified in each cut list */ for (; NOT_END_OF_LIST(cut_list[cl_pos]); cl_pos++) { - unsigned spos; - for (spos = cut_list[cl_pos].startpos; spos < linelen;) { + unsigned spos = cut_list[cl_pos].startpos; + while (spos < linelen) { if (!printed[spos]) { printed[spos] = 'X'; if (need_odelim && spos != 0 && !printed[spos-1]) { @@ -129,8 +130,10 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } putchar(line[spos]); } - if (++spos > cut_list[cl_pos].endpos) { - need_odelim = (odelim && odelim[0]); /* will print OSEP (if not empty) */ + spos++; + if (spos > cut_list[cl_pos].endpos) { + /* will print OSEP (if not empty) */ + need_odelim = (odelim && odelim[0]); break; } } @@ -242,7 +245,8 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, continue; } /* Got delimiter */ - if (dcount++ < cut_list[cl_pos].startpos) { + dcount++; + if (dcount <= cut_list[cl_pos].startpos) { /* Not yet within range - loop */ start = next; continue; From 14f57f5357cb674b88e7cdaff6267bf9d84c6b80 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 21 Dec 2024 00:43:45 +0100 Subject: [PATCH 060/277] cut: code shrink move "linenum" manipulations to the one place where it is used. function old new delta cut_main 1373 1360 -13 Signed-off-by: Denys Vlasenko --- coreutils/cut.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/coreutils/cut.c b/coreutils/cut.c index 93b58b4936..d81f36bcd9 100644 --- a/coreutils/cut.c +++ b/coreutils/cut.c @@ -143,15 +143,15 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, } else if (!opt_REGEX && *delim == '\n') { unsigned spos = cut_list[cl_pos].startpos; + linenum++; /* get out if we have no more ranges to process or if the lines * are lower than what we're interested in */ - if ((linenum < spos) || END_OF_LIST(cut_list[cl_pos])) + if (linenum <= spos || END_OF_LIST(cut_list[cl_pos])) goto next_line; /* if the line we're looking for is lower than the one we were * passed, it means we displayed it already, so move on */ - while (spos < linenum) { - spos++; + while (++spos < linenum) { /* go to the next list if we're at the end of this one */ if (spos > cut_list[cl_pos].endpos) { cl_pos++; @@ -161,7 +161,7 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, spos = cut_list[cl_pos].startpos; /* get out if the current line is lower than the one * we just became interested in */ - if (linenum < spos) + if (linenum <= spos) goto next_line; } } @@ -280,7 +280,6 @@ static void cut_file(FILE *file, const char *delim, const char *odelim, /* if we printed anything, finish with newline */ putchar('\n'); next_line: - linenum++; free(line); } /* while (got line) */ @@ -399,7 +398,7 @@ int cut_main(int argc UNUSED_PARAM, char **argv) //if (nranges == 0) // bb_simple_error_msg_and_die("missing list of positions"); //^^^ this is impossible since one of -bcfF is required, - // they populate LIST with non-empty string and when it is parsed, + // they populate LIST with non-NULL string and when it is parsed, // cut_list[] gets at least one element. /* now that the lists are parsed, we need to sort them to make life From ef21820dc9dd7520cc89469ef2b9d7387c7be78b Mon Sep 17 00:00:00 2001 From: Laurent Bercot Date: Sun, 9 Feb 2025 12:50:41 +0100 Subject: [PATCH 061/277] udhcpc6: improvements Several small improvements to udhcpc6. - Remove usage text for the nonexistent -B option. - Fix a segfault when renewing an IA_PD lease without IA_NA (which means the client hasn't been assigned an ip, so we cannot locally bind to it). - Fix NAK management: check the option length, and print the status code and status message - Add a -m option to always send renew requests as multicast. These last two changes are useful to deal with hopelessly broken DHCPv6 servers such as the one from the Orange Livebox (one of the main French ISPs) which I'm currently having the displeasure to have to talk to, hence the patch. function old new delta static.send_d6_renew - 126 +126 .rodata 105598 105649 +51 udhcpc6_main 2607 2650 +43 packed_usage 34933 34953 +20 d6_send_kernel_packet_from_client_data_ifindex 266 282 +16 send_d6_renew 174 - -174 ------------------------------------------------------------------------------ (add/remove: 1/1 grow/shrink: 4/0 up/down: 256/-174) Total: 82 bytes Signed-off-by: Laurent Bercot Signed-off-by: Denys Vlasenko --- networking/udhcp/d6_dhcpc.c | 88 ++++++++++++++++++++++++++---------- networking/udhcp/d6_packet.c | 16 ++++--- 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/networking/udhcp/d6_dhcpc.c b/networking/udhcp/d6_dhcpc.c index 79cef19993..706377be8f 100644 --- a/networking/udhcp/d6_dhcpc.c +++ b/networking/udhcp/d6_dhcpc.c @@ -148,10 +148,11 @@ enum { OPT_o = 1 << 12, OPT_x = 1 << 13, OPT_f = 1 << 14, - OPT_l = 1 << 15, - OPT_d = 1 << 16, + OPT_m = 1 << 15, + OPT_l = 1 << 16, + OPT_d = 1 << 17, /* The rest has variable bit positions, need to be clever */ - OPTBIT_d = 16, + OPTBIT_d = 17, USE_FOR_MMU( OPTBIT_b,) ///IF_FEATURE_UDHCPC_ARPING(OPTBIT_a,) IF_FEATURE_UDHCP_PORT( OPTBIT_P,) @@ -268,6 +269,23 @@ static void option_to_env(const uint8_t *option, const uint8_t *option_end) //case D6_OPT_SERVERID: case D6_OPT_IA_NA: case D6_OPT_IA_PD: +/* 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | OPTION_IA_PD | option-length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | IAID (4 octets) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | T1 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | T2 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * . . + * . IA_PD-options . + * . . + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + /* recurse to handle "IA_PD-options" field */ option_to_env(option + 16, option + 4 + option[3]); break; //case D6_OPT_IA_TA: @@ -1131,12 +1149,11 @@ static void client_background(void) //usage:#endif //usage:#define udhcpc6_trivial_usage //usage: "[-fbq"IF_UDHCP_VERBOSE("v")"R] [-t N] [-T SEC] [-A SEC|-n] [-i IFACE] [-s PROG]\n" -//usage: " [-p PIDFILE]"IF_FEATURE_UDHCP_PORT(" [-P PORT]")" [-ldo] [-r IPv6] [-x OPT:VAL]... [-O OPT]..." +//usage: " [-p PIDFILE]"IF_FEATURE_UDHCP_PORT(" [-P PORT]")" [-mldo] [-r IPv6] [-x OPT:VAL]... [-O OPT]..." //usage:#define udhcpc6_full_usage "\n" //usage: "\n -i IFACE Interface to use (default "CONFIG_UDHCPC_DEFAULT_INTERFACE")" //usage: "\n -p FILE Create pidfile" //usage: "\n -s PROG Run PROG at DHCP events (default "CONFIG_UDHCPC6_DEFAULT_SCRIPT")" -//usage: "\n -B Request broadcast replies" //usage: "\n -t N Send up to N discover packets" //usage: "\n -T SEC Pause between packets (default 3)" //usage: "\n -A SEC Wait if lease is not obtained (default 20)" @@ -1154,6 +1171,7 @@ static void client_background(void) ////usage: IF_FEATURE_UDHCPC_ARPING( ////usage: "\n -a Use arping to validate offered address" ////usage: ) +//usage: "\n -m Send multicast renew requests rather than unicast ones" //usage: "\n -l Send 'information request' instead of 'solicit'" //usage: "\n (used for servers which do not assign IPv6 addresses)" //usage: "\n -r IPv6 Request this address ('no' to not request any IP)" @@ -1211,7 +1229,7 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) /* Parse command line */ opt = getopt32long(argv, "^" /* O,x: list; -T,-t,-A take numeric param */ - "i:np:qRr:s:T:+t:+SA:+O:*ox:*fld" + "i:np:qRr:s:T:+t:+SA:+O:*ox:*fmld" USE_FOR_MMU("b") ///IF_FEATURE_UDHCPC_ARPING("a") IF_FEATURE_UDHCP_PORT("P:") @@ -1464,7 +1482,7 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) if (opt & OPT_l) send_d6_info_request(); else - send_d6_renew(&srv6_buf, requested_ipv6); + send_d6_renew(OPT_m ? NULL : &srv6_buf, requested_ipv6); timeout = discover_timeout; packet_num++; continue; @@ -1617,8 +1635,7 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) address_timeout = 0; prefix_timeout = 0; option = d6_find_option(packet.d6_options, packet_end, D6_OPT_STATUS_CODE); - if (option && (option->data[0] | option->data[1]) != 0) { -///FIXME: + if (option) { // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | OPTION_STATUS_CODE | option-len | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -1626,21 +1643,28 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | // . status-message . // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// so why do we think it's NAK if data[0] is zero but data[1] is not? That's wrong... -// we should also check that option->len is ok (i.e. not 0), right? - /* return to init state */ - bb_info_msg("received DHCP NAK (%u)", option->data[4]); - d6_run_script(packet.d6_options, - packet_end, "nak"); - if (client_data.state != REQUESTING) - d6_run_script_no_option("deconfig"); - sleep(3); /* avoid excessive network traffic */ - client_data.state = INIT_SELECTING; - client_data.first_secs = 0; /* make secs field count from 0 */ - requested_ipv6 = NULL; - timeout = 0; - packet_num = 0; - continue; + unsigned len, status; + len = ((unsigned)option->len_hi << 8) + option->len; + if (len < 2) { + bb_simple_error_msg("invalid OPTION_STATUS_CODE, ignoring packet"); + continue; + } + status = ((unsigned)option->data[0] << 8) + option->data[1]; + if (status != 0) { +//TODO: handle status == 5 (UseMulticast)? + /* return to init state */ + bb_info_msg("received DHCP NAK: %u '%.*s'", status, len - 2, option->data + 2); + d6_run_script(packet.d6_options, packet_end, "nak"); + if (client_data.state != REQUESTING) + d6_run_script_no_option("deconfig"); + sleep(3); /* avoid excessive network traffic */ + client_data.state = INIT_SELECTING; + client_data.first_secs = 0; /* make secs field count from 0 */ + requested_ipv6 = NULL; + timeout = 0; + packet_num = 0; + continue; + } } option = d6_copy_option(packet.d6_options, packet_end, D6_OPT_SERVERID); if (!option) { @@ -1661,7 +1685,6 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) packet_num = 0; continue; } - /* It's a D6_MSG_REPLY */ /* * RFC 3315 18.1.8. Receipt of Reply Messages * @@ -1790,6 +1813,21 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) free(client6_data.ia_pd); client6_data.ia_pd = d6_copy_option(packet.d6_options, packet_end, D6_OPT_IA_PD); +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | OPTION_IA_PD | option-length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | IAID (4 octets) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | T1 | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | T2 | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// . . +// . IA_PD-options . +// . . +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ if (!client6_data.ia_pd) { bb_info_msg("no %s option%s", "IA_PD", ", ignoring packet"); continue; diff --git a/networking/udhcp/d6_packet.c b/networking/udhcp/d6_packet.c index 142de9b435..1d7541948b 100644 --- a/networking/udhcp/d6_packet.c +++ b/networking/udhcp/d6_packet.c @@ -153,13 +153,15 @@ int FAST_FUNC d6_send_kernel_packet_from_client_data_ifindex( } setsockopt_reuseaddr(fd); - memset(&sa, 0, sizeof(sa)); - sa.sin6_family = AF_INET6; - sa.sin6_port = htons(source_port); - sa.sin6_addr = *src_ipv6; /* struct copy */ - if (bind(fd, (struct sockaddr *)&sa, sizeof(sa)) == -1) { - msg = "bind(%s)"; - goto ret_close; + if (src_ipv6) { + memset(&sa, 0, sizeof(sa)); + sa.sin6_family = AF_INET6; + sa.sin6_port = htons(source_port); + sa.sin6_addr = *src_ipv6; /* struct copy */ + if (bind(fd, (struct sockaddr *)&sa, sizeof(sa)) == -1) { + msg = "bind(%s)"; + goto ret_close; + } } memset(&sa, 0, sizeof(sa)); From 779cca899fbf0cd398fdcff266fec693b507854f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 9 Feb 2025 12:54:50 +0100 Subject: [PATCH 062/277] udhcpc6: move block comment, no code changes Signed-off-by: Denys Vlasenko --- networking/udhcp/d6_dhcpc.c | 122 ++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/networking/udhcp/d6_dhcpc.c b/networking/udhcp/d6_dhcpc.c index 706377be8f..d639a9f80d 100644 --- a/networking/udhcp/d6_dhcpc.c +++ b/networking/udhcp/d6_dhcpc.c @@ -1624,67 +1624,6 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) case RENEW_REQUESTED: case REBINDING: if (packet.d6_msg_type == D6_MSG_REPLY) { - unsigned start; - uint32_t lease_seconds; - struct d6_option *option; - unsigned address_timeout; - unsigned prefix_timeout; - type_is_ok: - change_listen_mode(LISTEN_NONE); - - address_timeout = 0; - prefix_timeout = 0; - option = d6_find_option(packet.d6_options, packet_end, D6_OPT_STATUS_CODE); - if (option) { -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | OPTION_STATUS_CODE | option-len | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | status-code | | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | -// . status-message . -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - unsigned len, status; - len = ((unsigned)option->len_hi << 8) + option->len; - if (len < 2) { - bb_simple_error_msg("invalid OPTION_STATUS_CODE, ignoring packet"); - continue; - } - status = ((unsigned)option->data[0] << 8) + option->data[1]; - if (status != 0) { -//TODO: handle status == 5 (UseMulticast)? - /* return to init state */ - bb_info_msg("received DHCP NAK: %u '%.*s'", status, len - 2, option->data + 2); - d6_run_script(packet.d6_options, packet_end, "nak"); - if (client_data.state != REQUESTING) - d6_run_script_no_option("deconfig"); - sleep(3); /* avoid excessive network traffic */ - client_data.state = INIT_SELECTING; - client_data.first_secs = 0; /* make secs field count from 0 */ - requested_ipv6 = NULL; - timeout = 0; - packet_num = 0; - continue; - } - } - option = d6_copy_option(packet.d6_options, packet_end, D6_OPT_SERVERID); - if (!option) { - bb_simple_info_msg("no server ID, ignoring packet"); - continue; - /* still selecting - this server looks bad */ - } -//Note: we do not bother comparing server IDs in Advertise and Reply msgs. -//server_id variable is used solely for creation of proper server_id option -//in outgoing packets. (why DHCPv6 even introduced it is a mystery). - free(client6_data.server_id); - client6_data.server_id = option; - if (packet.d6_msg_type == D6_MSG_ADVERTISE) { - /* enter requesting state */ - change_listen_mode(LISTEN_RAW); - client_data.state = REQUESTING; - timeout = 0; - packet_num = 0; - continue; - } /* * RFC 3315 18.1.8. Receipt of Reply Messages * @@ -1770,6 +1709,67 @@ int udhcpc6_main(int argc UNUSED_PARAM, char **argv) * . . * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ + unsigned start; + uint32_t lease_seconds; + struct d6_option *option; + unsigned address_timeout; + unsigned prefix_timeout; + type_is_ok: + change_listen_mode(LISTEN_NONE); + + address_timeout = 0; + prefix_timeout = 0; + option = d6_find_option(packet.d6_options, packet_end, D6_OPT_STATUS_CODE); + if (option) { +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | OPTION_STATUS_CODE | option-len | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | status-code | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | +// . status-message . +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + unsigned len, status; + len = ((unsigned)option->len_hi << 8) + option->len; + if (len < 2) { + bb_simple_error_msg("invalid OPTION_STATUS_CODE, ignoring packet"); + continue; + } + status = ((unsigned)option->data[0] << 8) + option->data[1]; + if (status != 0) { +//TODO: handle status == 5 (UseMulticast)? + /* return to init state */ + bb_info_msg("received DHCP NAK: %u '%.*s'", status, len - 2, option->data + 2); + d6_run_script(packet.d6_options, packet_end, "nak"); + if (client_data.state != REQUESTING) + d6_run_script_no_option("deconfig"); + sleep(3); /* avoid excessive network traffic */ + client_data.state = INIT_SELECTING; + client_data.first_secs = 0; /* make secs field count from 0 */ + requested_ipv6 = NULL; + timeout = 0; + packet_num = 0; + continue; + } + } + option = d6_copy_option(packet.d6_options, packet_end, D6_OPT_SERVERID); + if (!option) { + bb_simple_info_msg("no server ID, ignoring packet"); + continue; + /* still selecting - this server looks bad */ + } +//Note: we do not bother comparing server IDs in Advertise and Reply msgs. +//server_id variable is used solely for creation of proper server_id option +//in outgoing packets. (why DHCPv6 even introduced it is a mystery). + free(client6_data.server_id); + client6_data.server_id = option; + if (packet.d6_msg_type == D6_MSG_ADVERTISE) { + /* enter requesting state */ + change_listen_mode(LISTEN_RAW); + client_data.state = REQUESTING; + timeout = 0; + packet_num = 0; + continue; + } if (option_mask32 & OPT_r) { struct d6_option *iaaddr; From 411543949605fd7e9c996a0a2aadf8b344a4ced0 Mon Sep 17 00:00:00 2001 From: Zhou Siqi Date: Sun, 9 Feb 2025 13:50:18 +0100 Subject: [PATCH 063/277] udhcpc6: generate a consistent IAID Currently, udhcpc6 does not meet the requirements for Identity Association in RFC 3315. This is a specific explanation in RFC 3315 protocol: https://datatracker.ietf.org/doc/html/rfc3315#section-10 "The IAID uniquely identifies the IA and must be chosen to be unique among the IAIDs on the client. The IAID is chosen by the client. For any given use of an IA by the client, the IAID for that IA MUST be consistent across restarts of the DHCP client." This patch makes the client generate a consistent IAID based on the MAC address. function old new delta send_d6_discover 285 270 -15 Signed-off-by: Zhou Siqi Signed-off-by: Denys Vlasenko --- networking/udhcp/d6_dhcpc.c | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/networking/udhcp/d6_dhcpc.c b/networking/udhcp/d6_dhcpc.c index d639a9f80d..733eb533cf 100644 --- a/networking/udhcp/d6_dhcpc.c +++ b/networking/udhcp/d6_dhcpc.c @@ -622,6 +622,31 @@ static NOINLINE int send_d6_info_request(void) return d6_mcast_from_client_data_ifindex(&packet, opt_ptr); } +/* + * RFC 3315 10. Identity Association + * + * An "identity-association" (IA) is a construct through which a server + * and a client can identify, group, and manage a set of related IPv6 + * addresses. Each IA consists of an IAID and associated configuration + * information. + * + * A client must associate at least one distinct IA with each of its + * network interfaces for which it is to request the assignment of IPv6 + * addresses from a DHCP server. The client uses the IAs assigned to an + * interface to obtain configuration information from a server for that + * interface. Each IA must be associated with exactly one interface. + * + * The IAID uniquely identifies the IA and must be chosen to be unique + * among the IAIDs on the client. The IAID is chosen by the client. + * For any given use of an IA by the client, the IAID for that IA MUST + * be consistent across restarts of the DHCP client... + */ +/* Generate IAID. We base it on our MAC address' last 4 bytes */ +static void generate_iaid(uint8_t *iaid) +{ + memcpy(iaid, &client_data.client_mac[2], 4); +} + /* Multicast a DHCPv6 Solicit packet to the network, with an optionally requested IP. * * RFC 3315 17.1.1. Creation of Solicit Messages @@ -721,7 +746,7 @@ static NOINLINE int send_d6_discover(struct in6_addr *requested_ipv6) client6_data.ia_na = xzalloc(len); client6_data.ia_na->code = D6_OPT_IA_NA; client6_data.ia_na->len = len - 4; - *(bb__aliased_uint32_t*)client6_data.ia_na->data = rand(); /* IAID */ + generate_iaid(client6_data.ia_na->data); /* IAID */ if (requested_ipv6) { struct d6_option *iaaddr = (void*)(client6_data.ia_na->data + 4+4+4); iaaddr->code = D6_OPT_IAADDR; @@ -739,7 +764,7 @@ static NOINLINE int send_d6_discover(struct in6_addr *requested_ipv6) client6_data.ia_pd = xzalloc(len); client6_data.ia_pd->code = D6_OPT_IA_PD; client6_data.ia_pd->len = len - 4; - *(bb__aliased_uint32_t*)client6_data.ia_pd->data = rand(); /* IAID */ + generate_iaid(client6_data.ia_na->data); /* IAID */ opt_ptr = mempcpy(opt_ptr, client6_data.ia_pd, len); } From 887295686dbc69e5b2d530f0e6a8dbf94094ac9f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 9 Feb 2025 15:09:15 +0100 Subject: [PATCH 064/277] udhcpc6: fix copy-paste error in "generate a consistent IAID" commit Signed-off-by: Denys Vlasenko --- networking/udhcp/d6_dhcpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking/udhcp/d6_dhcpc.c b/networking/udhcp/d6_dhcpc.c index 733eb533cf..19c961d5c4 100644 --- a/networking/udhcp/d6_dhcpc.c +++ b/networking/udhcp/d6_dhcpc.c @@ -764,7 +764,7 @@ static NOINLINE int send_d6_discover(struct in6_addr *requested_ipv6) client6_data.ia_pd = xzalloc(len); client6_data.ia_pd->code = D6_OPT_IA_PD; client6_data.ia_pd->len = len - 4; - generate_iaid(client6_data.ia_na->data); /* IAID */ + generate_iaid(client6_data.ia_pd->data); /* IAID */ opt_ptr = mempcpy(opt_ptr, client6_data.ia_pd, len); } From f9274e8d6e15d1c3cc0368aa55974de84bbbb002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Parisot?= Date: Wed, 2 Apr 2025 17:43:54 +0200 Subject: [PATCH 065/277] init: improve log message when a process exits: show exit code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit function old new delta .rodata 105649 105680 +31 init_main 776 804 +28 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 59/0) Total: 59 bytes Signed-off-by: Sébastien Parisot Signed-off-by: Denys Vlasenko --- init/init.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/init/init.c b/init/init.c index 2ee1e4cde5..797e0a0ebc 100644 --- a/init/init.c +++ b/init/init.c @@ -1198,17 +1198,29 @@ int init_main(int argc UNUSED_PARAM, char **argv) /* Wait for any child process(es) to exit */ while (1) { pid_t wpid; + int status; struct init_action *a; - wpid = waitpid(-1, NULL, WNOHANG); + wpid = waitpid(-1, &status, WNOHANG); if (wpid <= 0) break; a = mark_terminated(wpid); if (a) { - message(L_LOG, "process '%s' (pid %u) exited. " + const char *s = "killed, signal"; + int ex = WTERMSIG(status); + /* "if (!WIFSIGNALED(status))" generates more code: + * on linux, WIFEXITED(status) is "WTERMSIG(status) == 0" + * and WTERMSIG(status) is known, so compiler optimizes. + */ + if (WIFEXITED(status)) { + s = "exited, exitcode"; + ex = WEXITSTATUS(status); + } + message(L_LOG, "process '%s' (pid %u) %s:%d. " "Scheduling for restart.", - a->command, (unsigned)wpid); + a->command, (unsigned)wpid, + s, ex); } } From a8349b115d17025ee3a29f898b873906e3570d39 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Apr 2025 12:13:16 +0200 Subject: [PATCH 066/277] Makefile: fix passing of EXTRA_LDLIBS Signed-off-by: Denys Vlasenko --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9550c391a4..dcf7cd5859 100644 --- a/Makefile +++ b/Makefile @@ -612,8 +612,12 @@ quiet_cmd_busybox__ ?= LINK $@ "$(core-y)" \ "$(libs-y)" \ "$(LDLIBS)" \ - "$(CONFIG_EXTRA_LDLIBS)" \ + $(CONFIG_EXTRA_LDLIBS) \ && $(srctree)/scripts/generate_BUFSIZ.sh --post include/common_bufsiz.h +# ^^^ note: CONFIG_xyz strings already have double quotes: their value +# is '"LIB LIB2"', therefore $(CONFIG_EXTRA_LDLIBS) above must NOT be written +# as "$(CONFIG_EXTRA_LDLIBS)", it would be passed as ""LIB LIB2"", +# and LIB2 would end up in $9, not $8 (and lost or misinterpreted). # Generate System.map quiet_cmd_sysmap = SYSMAP From d2d23c848acd6f336d9b0edae613eef443a6de20 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Apr 2025 02:46:39 +0200 Subject: [PATCH 067/277] udhcpd: send DHCPOFFERs as unicast (unless clients specifically asks for bcast) RFC 2131 says we should do that. Evidently, since for so many years no one complained, sending them broadcast works too, but finally we've got someone who wants RFC-compliand behavior. function old new delta send_packet 141 179 +38 .rodata 105680 105681 +1 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 39/0) Total: 39 bytes Signed-off-by: Denys Vlasenko --- networking/udhcp/dhcpd.c | 58 +++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/networking/udhcp/dhcpd.c b/networking/udhcp/dhcpd.c index 2904119e54..b9cbd6464a 100644 --- a/networking/udhcp/dhcpd.c +++ b/networking/udhcp/dhcpd.c @@ -575,29 +575,51 @@ static void send_packet_to_client(struct dhcp_packet *dhcp_pkt, int force_broadc const uint8_t *chaddr; uint32_t ciaddr; - // Was: + // Logic: //if (force_broadcast) { /* broadcast */ } //else if (dhcp_pkt->ciaddr) { /* unicast to dhcp_pkt->ciaddr */ } + // ^^^ dhcp_pkt->ciaddr comes from client's request packet. + // We expect such clients to have an UDP socket listening on that IP. //else if (dhcp_pkt->flags & htons(BROADCAST_FLAG)) { /* broadcast */ } //else { /* unicast to dhcp_pkt->yiaddr */ } - // But this is wrong: yiaddr is _our_ idea what client's IP is - // (for example, from lease file). Client may not know that, - // and may not have UDP socket listening on that IP! - // We should never unicast to dhcp_pkt->yiaddr! - // dhcp_pkt->ciaddr, OTOH, comes from client's request packet, - // and can be used. - - if (force_broadcast - || (dhcp_pkt->flags & htons(BROADCAST_FLAG)) - || dhcp_pkt->ciaddr == 0 + // ^^^ The last case is confusing, but *should* work. + // It's a case where client have sent a DISCOVER + // and does not have a kernel UDP socket listening on the IP + // we are offering in yiaddr (it does not know the IP yet)! + // This *should* work because client *should* listen on a raw socket + // instead at this time (IOW: it should examine ALL IPv4 packets + // "by hand", not relying on kernel's UDP stack.) + + chaddr = dhcp_pkt->chaddr; + + if (dhcp_pkt->ciaddr == 0 + || force_broadcast /* sending DHCPNAK pkt? */ ) { - log1s("broadcasting packet to client"); - ciaddr = INADDR_BROADCAST; - chaddr = MAC_BCAST_ADDR; + if (dhcp_pkt->flags & htons(BROADCAST_FLAG) + || force_broadcast /* sending DHCPNAK pkt? */ + ) { +// RFC 2131: +// If 'giaddr' is zero and 'ciaddr' is zero, and the broadcast bit is +// set, then the server broadcasts DHCPOFFER and DHCPACK messages to +// 0xffffffff. ... +// In all cases, when 'giaddr' is zero, the server broadcasts any DHCPNAK +// messages to 0xffffffff. + ciaddr = INADDR_BROADCAST; + chaddr = MAC_BCAST_ADDR; + log1s("broadcasting packet to client"); + } else { +// If the broadcast bit is not set and 'giaddr' is zero and +// 'ciaddr' is zero, then the server unicasts DHCPOFFER and DHCPACK +// messages to the client's hardware address and 'yiaddr' address. + ciaddr = dhcp_pkt->yiaddr; + log1("unicasting packet to client %ciaddr", 'y'); + } } else { - log1s("unicasting packet to client ciaddr"); +// If the 'giaddr' +// field is zero and the 'ciaddr' field is nonzero, then the server +// unicasts DHCPOFFER and DHCPACK messages to the address in 'ciaddr'. ciaddr = dhcp_pkt->ciaddr; - chaddr = dhcp_pkt->chaddr; + log1("unicasting packet to client %ciaddr", 'c'); } udhcp_send_raw_packet(dhcp_pkt, @@ -624,6 +646,10 @@ static void send_packet_to_relay(struct dhcp_packet *dhcp_pkt) static void send_packet(struct dhcp_packet *dhcp_pkt, int force_broadcast) { if (dhcp_pkt->gateway_nip) +// RFC 2131: +// If the 'giaddr' field in a DHCP message from a client is non-zero, +// the server sends any return messages to the 'DHCP server' port on the +// BOOTP relay agent whose address appears in 'giaddr'. send_packet_to_relay(dhcp_pkt); else send_packet_to_client(dhcp_pkt, force_broadcast); From fc466720b5e8611f485bc574c0114d5037525f92 Mon Sep 17 00:00:00 2001 From: Grant Erickson Date: Tue, 8 Apr 2025 15:39:30 +0200 Subject: [PATCH 068/277] syslogd: Fix 'OPT_locallog' check regression in 'syslogd_main' In the recent refactoring of 'syslogd_main', a regression was introduced in handling the manual bitwise OR of 'OPT_locallog' as follows: if (ENABLE_FEATURE_REMOTE_LOG && !(opts & OPT_remotelog)) // -R option_mask32 |= OPT_locallog; 'opts' represents the locally-scoped output of 'getopt32' and 'option_mask32' represents the globally-scoped state of the same. Consequently, the above performs a bitwise OR to include 'OPT_locallog' of the globally-scoped option state, which 'opts' will not reflect locally. Manipulating the global, rather than local, state is correct as 'timestamp_and_log_internal' will later need to check 'OPT_locallog'. However, when the aforementioned refactor occurred, the following regressing change was made: - if (!ENABLE_FEATURE_REMOTE_LOG || (option_mask32 & OPT_locallog)) { + if (!ENABLE_FEATURE_REMOTE_LOG || (opts & OPT_locallog)) { breaking the spatially- and temporally-removed check in 'timestamp_and_log_internal'. Fixes: 02378ce20c6d2 ("syslogd: decrease stack usage, ~50 bytes") function old new delta syslogd_init 1140 1137 -3 Signed-off-by: Grant Erickson Signed-off-by: Denys Vlasenko --- sysklogd/syslogd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysklogd/syslogd.c b/sysklogd/syslogd.c index 7558051f0e..2cbb22b6d1 100644 --- a/sysklogd/syslogd.c +++ b/sysklogd/syslogd.c @@ -1045,7 +1045,7 @@ static int NOINLINE syslogd_init(char **argv) #endif /* If they have not specified remote logging, then log locally */ if (ENABLE_FEATURE_REMOTE_LOG && !(opts & OPT_remotelog)) // -R - option_mask32 |= OPT_locallog; + option_mask32 = (opts |= OPT_locallog); #if ENABLE_FEATURE_SYSLOGD_CFG parse_syslogdcfg(opt_f); #endif From 9a8796436b9b0641e13480811902ea2ac57881d3 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 2 Oct 2024 10:12:05 +0200 Subject: [PATCH 069/277] archival: disallow path traversals (CVE-2023-39810) Create new configure option for archival/libarchive based extractions to disallow path traversals. As this is a paranoid option and might introduce backward incompatibility, default it to no. Fixes: CVE-2023-39810 Based on the patch by Peter Kaestle function old new delta data_extract_all 921 945 +24 strip_unsafe_prefix 101 102 +1 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 25/0) Total: 25 bytes Signed-off-by: Denys Vlasenko --- archival/Config.src | 11 +++++++++++ archival/libarchive/data_extract_all.c | 8 ++++++++ archival/libarchive/unsafe_prefix.c | 6 +++++- scripts/kconfig/lxdialog/check-lxdialog.sh | 2 +- testsuite/cpio.tests | 23 ++++++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/archival/Config.src b/archival/Config.src index 6f4f30c43d..cbcd7217ce 100644 --- a/archival/Config.src +++ b/archival/Config.src @@ -35,4 +35,15 @@ config FEATURE_LZMA_FAST This option reduces decompression time by about 25% at the cost of a 1K bigger binary. +config FEATURE_PATH_TRAVERSAL_PROTECTION + bool "Prevent extraction of filenames with /../ path component" + default n + help + busybox tar and unzip remove "PREFIX/../" (if it exists) + from extracted names. + This option enables this behavior for all other unpacking applets, + such as cpio, ar, rpm. + GNU cpio 2.15 has NO such sanity check. +# try other archivers and document their behavior? + endmenu diff --git a/archival/libarchive/data_extract_all.c b/archival/libarchive/data_extract_all.c index 049c2c1563..8a69711c1f 100644 --- a/archival/libarchive/data_extract_all.c +++ b/archival/libarchive/data_extract_all.c @@ -65,6 +65,14 @@ void FAST_FUNC data_extract_all(archive_handle_t *archive_handle) } while (--n != 0); } #endif +#if ENABLE_FEATURE_PATH_TRAVERSAL_PROTECTION + /* Strip leading "/" and up to last "/../" path component */ + dst_name = (char *)strip_unsafe_prefix(dst_name); +#endif +// ^^^ This may be a problem if some applets do need to extract absolute names. +// (Probably will need to invent ARCHIVE_ALLOW_UNSAFE_NAME flag). +// You might think that rpm needs it, but in my tests rpm's internal cpio +// archive has names like "./usr/bin/FOO", not "/usr/bin/FOO". if (archive_handle->ah_flags & ARCHIVE_CREATE_LEADING_DIRS) { char *slash = strrchr(dst_name, '/'); diff --git a/archival/libarchive/unsafe_prefix.c b/archival/libarchive/unsafe_prefix.c index 33e487bf94..667081195f 100644 --- a/archival/libarchive/unsafe_prefix.c +++ b/archival/libarchive/unsafe_prefix.c @@ -14,7 +14,11 @@ const char* FAST_FUNC strip_unsafe_prefix(const char *str) cp++; continue; } - if (is_prefixed_with(cp, "/../"+1)) { + /* We are called lots of times. + * is_prefixed_with(cp, "../") is slower than open-coding it, + * with minimal code growth (~few bytes). + */ + if (cp[0] == '.' && cp[1] == '.' && cp[2] == '/') { cp += 3; continue; } diff --git a/scripts/kconfig/lxdialog/check-lxdialog.sh b/scripts/kconfig/lxdialog/check-lxdialog.sh index 5075ebf2d3..910ca1f7ce 100755 --- a/scripts/kconfig/lxdialog/check-lxdialog.sh +++ b/scripts/kconfig/lxdialog/check-lxdialog.sh @@ -47,7 +47,7 @@ trap "rm -f $tmp" 0 1 2 3 15 check() { $cc -x c - -o $tmp 2>/dev/null <<'EOF' #include CURSES_LOC -main() {} +int main() { return 0; } EOF if [ $? != 0 ]; then echo " *** Unable to find the ncurses libraries or the" 1>&2 diff --git a/testsuite/cpio.tests b/testsuite/cpio.tests index 85e7465891..a4462c53e0 100755 --- a/testsuite/cpio.tests +++ b/testsuite/cpio.tests @@ -154,6 +154,29 @@ testing "cpio -R with extract" \ " "" "" SKIP= +# Create an archive containing a file with "../dont_write" filename. +# See that it will not be allowed to unpack. +# NB: GNU cpio 2.15 DOES NOT do such checks. +optional FEATURE_PATH_TRAVERSAL_PROTECTION +rm -rf cpio.testdir +mkdir -p cpio.testdir/prepare/inner +echo "file outside of destination was written" > cpio.testdir/prepare/dont_write +echo "data" > cpio.testdir/prepare/inner/to_extract +mkdir -p cpio.testdir/extract +testing "cpio extract file outside of destination" "\ +(cd cpio.testdir/prepare/inner && echo -e '../dont_write\nto_extract' | cpio -o -H newc) | (cd cpio.testdir/extract && cpio -vi 2>&1) +echo \$? +ls cpio.testdir/dont_write 2>&1" \ +"\ +cpio: removing leading '../' from member names +../dont_write +to_extract +1 blocks +0 +ls: cpio.testdir/dont_write: No such file or directory +" "" "" +SKIP= + # Clean up rm -rf cpio.testdir cpio.testdir2 2>/dev/null From 636315ccb9b9e7e6cb040a9626282b7892b20df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Neusch=C3=A4fer?= Date: Mon, 14 Apr 2025 20:40:43 +0200 Subject: [PATCH 070/277] runit: fix setuidgid help text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: J. Neuschäfer Signed-off-by: Denys Vlasenko --- runit/chpst.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runit/chpst.c b/runit/chpst.c index 4e3d613b7d..3d04ee50d4 100644 --- a/runit/chpst.c +++ b/runit/chpst.c @@ -38,7 +38,8 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //config: bool "setuidgid (4.2 kb)" //config: default y //config: help -//config: Sets soft resource limits as specified by options +//config: Sets UID and GID to those of the given account, and execs +//config: specified program. //config: //config:config ENVUIDGID //config: bool "envuidgid (4.1 kb)" From c61fdadf974205104f6690ad360117bdf86f0ba6 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 20 Apr 2025 23:43:19 +0200 Subject: [PATCH 071/277] libbb/archival: make setup_unzip_on_fd() return bytes read if not compressed setup_unzip_on_fd() does not return the transformer structure, so the user does not know how much to seek back (or alternatively what the signature was) when compressor signature is not detected. Currently not needed (the only user is tar which dies anyway). However, rpm2cpio may need this if we extend it to extract the internal .cpio even if cpio's compressions algo is not known. function old new delta setup_unzip_on_fd 53 59 +6 Signed-off-by: Denys Vlasenko --- archival/bbunzip.c | 6 +++--- archival/libarchive/open_transformer.c | 26 ++++++++++++++------------ archival/rpm.c | 4 ++-- archival/tar.c | 2 +- include/libbb.h | 6 +++--- miscutils/fbsplash.c | 2 +- miscutils/man.c | 2 +- 7 files changed, 25 insertions(+), 23 deletions(-) diff --git a/archival/bbunzip.c b/archival/bbunzip.c index b7944a62ad..42b4baf886 100644 --- a/archival/bbunzip.c +++ b/archival/bbunzip.c @@ -71,8 +71,8 @@ int FAST_FUNC bbunpack(char **argv, goto err; } else { /* "clever zcat" with FILE */ - /* fail_if_not_compressed because zcat refuses uncompressed input */ - int fd = open_zipped(filename, /*fail_if_not_compressed:*/ 1); + /* die_if_not_compressed because zcat refuses uncompressed input */ + int fd = open_zipped(filename, /*die_if_not_compressed:*/ 1); if (fd < 0) goto err_name; xmove_fd(fd, STDIN_FILENO); @@ -80,7 +80,7 @@ int FAST_FUNC bbunpack(char **argv, } else if (option_mask32 & BBUNPK_SEAMLESS_MAGIC) { /* "clever zcat" on stdin */ - if (setup_unzip_on_fd(STDIN_FILENO, /*fail_if_not_compressed*/ 1)) + if (setup_unzip_on_fd(STDIN_FILENO, /*die_if_not_compressed*/ 1)) goto err; } diff --git a/archival/libarchive/open_transformer.c b/archival/libarchive/open_transformer.c index 44715ef252..353f682170 100644 --- a/archival/libarchive/open_transformer.c +++ b/archival/libarchive/open_transformer.c @@ -157,7 +157,7 @@ void FAST_FUNC fork_transformer(int fd, const char *transform_prog) /* Used by e.g. rpm which gives us a fd without filename, * thus we can't guess the format from filename's extension. */ -static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_compressed) +static transformer_state_t *setup_transformer_on_fd(int fd, int die_if_not_compressed) { transformer_state_t *xstate; @@ -204,7 +204,7 @@ static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_comp } /* No known magic seen */ - if (fail_if_not_compressed) + if (die_if_not_compressed) bb_simple_error_msg_and_die("no gzip" IF_FEATURE_SEAMLESS_BZ2("/bzip2") IF_FEATURE_SEAMLESS_XZ("/xz") @@ -240,13 +240,15 @@ static void fork_transformer_and_free(transformer_state_t *xstate) /* Used by e.g. rpm which gives us a fd without filename, * thus we can't guess the format from filename's extension. */ -int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) +int FAST_FUNC setup_unzip_on_fd(int fd, int die_if_not_compressed) { - transformer_state_t *xstate = setup_transformer_on_fd(fd, fail_if_not_compressed); + transformer_state_t *xstate = setup_transformer_on_fd(fd, die_if_not_compressed); if (!xstate->xformer) { + /* Not compressed */ + int retval = xstate->signature_skipped; /* never zero */ free(xstate); - return 1; + return retval; } fork_transformer_and_free(xstate); @@ -264,7 +266,7 @@ void FAST_FUNC setup_lzma_on_fd(int fd) } #endif -static transformer_state_t *open_transformer(const char *fname, int fail_if_not_compressed) +static transformer_state_t *open_transformer(const char *fname, int die_if_not_compressed) { transformer_state_t *xstate; int fd; @@ -284,18 +286,18 @@ static transformer_state_t *open_transformer(const char *fname, int fail_if_not_ } } - xstate = setup_transformer_on_fd(fd, fail_if_not_compressed); + xstate = setup_transformer_on_fd(fd, die_if_not_compressed); return xstate; } -int FAST_FUNC open_zipped(const char *fname, int fail_if_not_compressed) +int FAST_FUNC open_zipped(const char *fname, int die_if_not_compressed) { int fd; transformer_state_t *xstate; - xstate = open_transformer(fname, fail_if_not_compressed); - if (!xstate) + xstate = open_transformer(fname, die_if_not_compressed); + if (!xstate) /* open error */ return -1; fd = xstate->src_fd; @@ -326,7 +328,7 @@ void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_ transformer_state_t *xstate; char *image; - xstate = open_transformer(fname, /*fail_if_not_compressed:*/ 0); + xstate = open_transformer(fname, /*die_if_not_compressed:*/ 0); if (!xstate) /* file open error */ return NULL; @@ -371,7 +373,7 @@ void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_ int fd; char *image; - fd = open_zipped(fname, /*fail_if_not_compressed:*/ 0); + fd = open_zipped(fname, /*die_if_not_compressed:*/ 0); if (fd < 0) return NULL; diff --git a/archival/rpm.c b/archival/rpm.c index af8db99a6a..7fd2a2b46c 100644 --- a/archival/rpm.c +++ b/archival/rpm.c @@ -316,7 +316,7 @@ static void extract_cpio(int fd, const char *source_rpm) archive_handle->src_fd = fd; /*archive_handle->offset = 0; - init_handle() did it */ - setup_unzip_on_fd(archive_handle->src_fd, /*fail_if_not_compressed:*/ 1); + setup_unzip_on_fd(archive_handle->src_fd, /*die_if_not_compressed:*/ 1); while (get_header_cpio(archive_handle) == EXIT_SUCCESS) continue; } @@ -541,7 +541,7 @@ int rpm2cpio_main(int argc UNUSED_PARAM, char **argv) // set up decompressor without detection setup_lzma_on_fd(rpm_fd); } else { - setup_unzip_on_fd(rpm_fd, /*fail_if_not_compressed:*/ 1); + setup_unzip_on_fd(rpm_fd, /*die_if_not_compressed:*/ 1); } if (bb_copyfd_eof(rpm_fd, STDOUT_FILENO) < 0) diff --git a/archival/tar.c b/archival/tar.c index d6ca6c1e0d..38906ba024 100644 --- a/archival/tar.c +++ b/archival/tar.c @@ -1164,7 +1164,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv) * on e.g. tarball with 1st file named "BZh5". */ ) { - tar_handle->src_fd = open_zipped(tar_filename, /*fail_if_not_compressed:*/ 0); + tar_handle->src_fd = open_zipped(tar_filename, /*die_if_not_compressed:*/ 0); if (tar_handle->src_fd < 0) bb_perror_msg_and_die("can't open '%s'", tar_filename); } else { diff --git a/include/libbb.h b/include/libbb.h index 4d6193795e..558f3c6272 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1011,13 +1011,13 @@ unsigned bb_clk_tck(void) FAST_FUNC; #if SEAMLESS_COMPRESSION /* Autodetects gzip/bzip2 formats. fd may be in the middle of the file! */ -int setup_unzip_on_fd(int fd, int fail_if_not_compressed) FAST_FUNC; +int setup_unzip_on_fd(int fd, int die_if_not_compressed) FAST_FUNC; /* Autodetects .gz etc */ -extern int open_zipped(const char *fname, int fail_if_not_compressed) FAST_FUNC; +extern int open_zipped(const char *fname, int die_if_not_compressed) FAST_FUNC; extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC; #else # define setup_unzip_on_fd(...) (0) -# define open_zipped(fname, fail_if_not_compressed) open((fname), O_RDONLY); +# define open_zipped(fname, die_if_not_compressed) open((fname), O_RDONLY); # define xmalloc_open_zipped_read_close(fname, maxsz_p) xmalloc_open_read_close((fname), (maxsz_p)) #endif /* lzma has no signature, need a little helper. NB: exist only for ENABLE_FEATURE_SEAMLESS_LZMA=y */ diff --git a/miscutils/fbsplash.c b/miscutils/fbsplash.c index 2934d8eb7b..912a501a31 100644 --- a/miscutils/fbsplash.c +++ b/miscutils/fbsplash.c @@ -382,7 +382,7 @@ static void fb_drawimage(void) if (LONE_DASH(G.image_filename)) { theme_file = stdin; } else { - int fd = open_zipped(G.image_filename, /*fail_if_not_compressed:*/ 0); + int fd = open_zipped(G.image_filename, /*die_if_not_compressed:*/ 0); if (fd < 0) bb_simple_perror_msg_and_die(G.image_filename); theme_file = xfdopen_for_read(fd); diff --git a/miscutils/man.c b/miscutils/man.c index deaf9e5abb..6fa1fbfdcc 100644 --- a/miscutils/man.c +++ b/miscutils/man.c @@ -143,7 +143,7 @@ static int run_pipe(char *man_filename, int man, int level) ordinary_manpage: close(STDIN_FILENO); - open_zipped(man_filename, /*fail_if_not_compressed:*/ 0); /* guaranteed to use fd 0 (STDIN_FILENO) */ + open_zipped(man_filename, /*die_if_not_compressed:*/ 0); /* guaranteed to use fd 0 (STDIN_FILENO) */ if (man) { int w = get_terminal_width(-1); if (w > 10) From f13f68288f45708ff12e0f6853def5910acd32c7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 20 Apr 2025 23:59:38 +0200 Subject: [PATCH 072/277] rpm2cpio: extract cpio even if compression is not known This is more useful than failing outright. function old new delta rpm2cpio_main 110 144 +34 .rodata 105681 105715 +34 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 68/0) Total: 68 bytes Signed-off-by: Denys Vlasenko --- archival/rpm.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/archival/rpm.c b/archival/rpm.c index 7fd2a2b46c..95a8c79b6b 100644 --- a/archival/rpm.c +++ b/archival/rpm.c @@ -533,6 +533,7 @@ int rpm2cpio_main(int argc UNUSED_PARAM, char **argv) // /* We need to know whether child (gzip/bzip/etc) exits abnormally */ // signal(SIGCHLD, check_errors_in_children); + str = NULL; if (ENABLE_FEATURE_SEAMLESS_LZMA && (str = rpm_getstr0(TAG_PAYLOADCOMPRESSOR)) != NULL && strcmp(str, "lzma") == 0 @@ -541,7 +542,11 @@ int rpm2cpio_main(int argc UNUSED_PARAM, char **argv) // set up decompressor without detection setup_lzma_on_fd(rpm_fd); } else { - setup_unzip_on_fd(rpm_fd, /*die_if_not_compressed:*/ 1); + int signature_bytes = setup_unzip_on_fd(rpm_fd, /*die_if_not_compressed:*/ 0); + if (signature_bytes != 0) { + xlseek(rpm_fd, - signature_bytes, SEEK_CUR); + bb_error_msg("warning, unknown compression '%s'", str); + } } if (bb_copyfd_eof(rpm_fd, STDOUT_FILENO) < 0) From 5f07327251c93184dfcfc8d978fc35705930ec53 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 21 Apr 2025 04:41:50 +0200 Subject: [PATCH 073/277] cpio: error out if the file to be achived is >=4GB function old new delta .rodata 105715 105751 +36 cpio_o 1145 1167 +22 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 58/0) Total: 58 bytes Signed-off-by: Denys Vlasenko --- archival/cpio.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/archival/cpio.c b/archival/cpio.c index f0d9900485..7b01ee3a24 100644 --- a/archival/cpio.c +++ b/archival/cpio.c @@ -350,6 +350,12 @@ static NOINLINE int cpio_o(void) st.st_dev = st.st_rdev = 0; #endif + if (sizeof(st.st_size) > 4 + && st.st_size > (off_t)0xffffffff + ) { + bb_error_msg_and_die("error: file '%s' is larger than 4GB", name); + } + bytes += printf("070701" "%08X%08X%08X%08X%08X%08X%08X" "%08X%08X%08X%08X" /* GNU cpio uses uppercase hex */ From 37abfb919dfa92132bc7a14c583016a46026729b Mon Sep 17 00:00:00 2001 From: Ahmad Fatoum Date: Tue, 15 Apr 2025 23:45:54 +0200 Subject: [PATCH 074/277] lineedit: fix printing lines during tab completion Indentation and escape characters are all output to stderr now, but the matches themself remained on stdout leading to garbled output on Tab completion. Print the results to stderr as well to fix this. Fixes: fd47f056765a ("lineedit: print prompt and editing operations to stderr") Signed-off-by: Ahmad Fatoum Signed-off-by: Denys Vlasenko --- libbb/lineedit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libbb/lineedit.c b/libbb/lineedit.c index 151208c1c1..db95fbb004 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -1170,9 +1170,10 @@ static void showfiles(void) ); } if (ENABLE_UNICODE_SUPPORT) - puts(printable_string(matches[n])); + fputs(printable_string(matches[n]), stderr); else - puts(matches[n]); + fputs(matches[n], stderr); + bb_putchar_stderr('\n'); } } From 952a7b9cececa314b2f5a9827dac8213b1a74460 Mon Sep 17 00:00:00 2001 From: Ahmad Fatoum Date: Tue, 15 Apr 2025 23:45:55 +0200 Subject: [PATCH 075/277] lineedit: fix left-over print to stdout There's code printing to stderr both before and after the single call to puts inside put_cur_glyph_and_inc_cursor(). This is likely an oversight as we want everything to go through the same file descriptor, so switch it over to fputs. Fixes: fd47f056765a ("lineedit: print prompt and editing operations to stderr") Signed-off-by: Ahmad Fatoum Signed-off-by: Denys Vlasenko --- libbb/lineedit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbb/lineedit.c b/libbb/lineedit.c index db95fbb004..ea216e16c9 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -451,7 +451,7 @@ static void put_cur_glyph_and_inc_cursor(void) * have automargin (IOW: it is moving cursor to next line * by itself (which is wrong for VT-10x terminals)), * this will break things: there will be one extra empty line */ - puts("\r"); /* + implicit '\n' */ + fputs("\r\n", stderr); #else /* VT-10x terminals don't wrap cursor to next line when last char * on the line is printed - cursor stays "over" this char. From 58adc7b5278f1d68dca0d5e303fe48e015f7b7bd Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 1 Jul 2025 12:54:00 +0200 Subject: [PATCH 076/277] ip route: make cache printing a bit more similar to upstream function old new delta print_route 1573 1565 -8 Signed-off-by: Denys Vlasenko --- networking/libiproute/iproute.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/networking/libiproute/iproute.c b/networking/libiproute/iproute.c index cd77f642ff..a30f070eb4 100644 --- a/networking/libiproute/iproute.c +++ b/networking/libiproute/iproute.c @@ -302,24 +302,22 @@ static int FAST_FUNC print_route(const struct sockaddr_nl *who UNUSED_PARAM, printf("notify "); } - if (r->rtm_family == AF_INET6) { - struct rta_cacheinfo *ci = NULL; - if (tb[RTA_CACHEINFO]) { - ci = RTA_DATA(tb[RTA_CACHEINFO]); - } - if ((r->rtm_flags & RTM_F_CLONED) || (ci && ci->rta_expires)) { + if (r->rtm_family == AF_INET || r->rtm_family == AF_INET6) { + if (r->rtm_family == AF_INET) { if (r->rtm_flags & RTM_F_CLONED) { printf("%c cache ", _SL_); + /* upstream: print_cache_flags() prints more here */ } + } + if (tb[RTA_CACHEINFO]) { + struct rta_cacheinfo *ci = RTA_DATA(tb[RTA_CACHEINFO]); if (ci->rta_expires) { printf(" expires %dsec", ci->rta_expires / get_hz()); } if (ci->rta_error != 0) { printf(" error %d", ci->rta_error); } - } else if (ci) { - if (ci->rta_error != 0) - printf(" error %d", ci->rta_error); + /* upstream: print_rta_cacheinfo() prints more here */ } } if (tb[RTA_IIF] && G_filter.iif == 0) { From 73c67d75ef6197a5e722ea1f19c30fa79f4de569 Mon Sep 17 00:00:00 2001 From: Michael Glembotzki Date: Tue, 17 Jun 2025 12:20:02 +0200 Subject: [PATCH 077/277] ntpd: Raise MAXDIST to 3s for better tolerance and consistency with chrony function old new delta select_and_cluster 1088 1093 +5 Signed-off-by: Michael Glembotzki Signed-off-by: Denys Vlasenko --- networking/ntpd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking/ntpd.c b/networking/ntpd.c index dcbdb8e600..4c4842cb29 100644 --- a/networking/ntpd.c +++ b/networking/ntpd.c @@ -205,7 +205,7 @@ #define MINDISP 0.01 /* minimum dispersion (sec) */ #define MAXDISP 16 /* maximum dispersion (sec) */ #define MAXSTRAT 16 /* maximum stratum (infinity metric) */ -#define MAXDIST 1 /* distance threshold (sec) */ +#define MAXDIST 3 /* distance threshold (sec): do not use peers who are farther away */ #define MIN_SELECTED 1 /* minimum intersection survivors */ #define MIN_CLUSTERED 3 /* minimum cluster survivors */ From f5c7cae55fc3e19d074198bc12152486067ea8c7 Mon Sep 17 00:00:00 2001 From: Radoslav Kolev Date: Thu, 24 Apr 2025 00:45:25 +0300 Subject: [PATCH 078/277] hexdump: fix regression for uint16 on big endian systems Commit 34751d8bf introduced a bug in the handling of uint16 values on big endian systems not considered safe for unaligned access when falling back to memcpy. Signed-off-by: Radoslav Kolev Signed-off-by: Denys Vlasenko --- libbb/dump.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libbb/dump.c b/libbb/dump.c index 2ca9919da4..ac5d47d9ea 100644 --- a/libbb/dump.c +++ b/libbb/dump.c @@ -667,15 +667,21 @@ static NOINLINE void display(priv_dumper_t* dumper) conv_u(pr, bp); break; case F_UINT: { + union { + uint16_t uval16; + uint32_t uval32; + } u; unsigned value = (unsigned char)*bp; switch (pr->bcnt) { case 1: break; case 2: - move_from_unaligned16(value, bp); + move_from_unaligned16(u.uval16, bp); + value = u.uval16; break; case 4: - move_from_unaligned32(value, bp); + move_from_unaligned32(u.uval32, bp); + value = u.uval32; break; /* case 8: no users yet */ } From b8ce2084cdb95bd77afc93fe0ccb270ba12139f8 Mon Sep 17 00:00:00 2001 From: Radoslav Kolev Date: Thu, 24 Apr 2025 00:45:26 +0300 Subject: [PATCH 079/277] od: make -B test little endian only, add variant for big endian Signed-off-by: Radoslav Kolev Signed-off-by: Denys Vlasenko --- testsuite/od.tests | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/testsuite/od.tests b/testsuite/od.tests index 4f245a7e80..c863bf2e87 100755 --- a/testsuite/od.tests +++ b/testsuite/od.tests @@ -61,7 +61,8 @@ testing "od -a (DESKTOP)" \ "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" SKIP= -testing "od -B" \ +$little_endian || SKIP=1 +testing "od -B (little-endian)" \ "od -B" \ "\ 0000000 001001 005003 041101 177103 @@ -70,6 +71,16 @@ testing "od -B" \ "" "$input" SKIP= +$little_endian && SKIP=1 +testing "od -B (big-endian)" \ + "od -B" \ +"\ +0000000 000402 001412 040502 041776 +0000010 +" \ + "" "$input" +SKIP= + $little_endian || SKIP=1 testing "od -o (little-endian)" \ "od -o" \ From e6317e72dd7d169d20836df6384eb021cb529464 Mon Sep 17 00:00:00 2001 From: Radoslav Kolev Date: Thu, 24 Apr 2025 00:45:27 +0300 Subject: [PATCH 080/277] hexdump: add tests for %x, handle little/big endian properly Signed-off-by: Radoslav Kolev Signed-off-by: Denys Vlasenko --- testsuite/hexdump.tests | 91 +++++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 22 deletions(-) diff --git a/testsuite/hexdump.tests b/testsuite/hexdump.tests index 517ec508bb..b2f6a2201c 100755 --- a/testsuite/hexdump.tests +++ b/testsuite/hexdump.tests @@ -5,6 +5,17 @@ . ./testing.sh +input=\ +"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"\ +"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"\ +"\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"\ +"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"\ +"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + +little_endian=false +{ printf '\0\1' | hexdump -d | grep -q 256; } && little_endian=true +readonly little_endian + # testing "description" "command" "result" "infile" "stdin" testing 'hexdump -C with four NULs' \ 'hexdump -C' \ @@ -43,12 +54,7 @@ testing "hexdump -e %3_u" \ 80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff " \ - "" \ -"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"\ -"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"\ -"\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"\ -"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"\ -"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"\ + "" "$input" testing "hexdump -e /1 %d" \ "hexdump -e '16/1 \" %4d\" \"\n\"'" \ @@ -59,33 +65,74 @@ testing "hexdump -e /1 %d" \ -128 -127 -126 -125 -124 -123 -122 -121 -120 -119 -118 -117 -116 -115 -114 -113 -16 -15 -14 -13 -12 -11 -10 -9 -8 -7 -6 -5 -4 -3 -2 -1 " \ - "" \ -"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"\ -"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"\ -"\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"\ -"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"\ -"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"\ + "" "$input" -testing "hexdump -e /2 %d" \ - "hexdump -e '8/2 \" %6d\" \"\n\"'" \ - "\ +$little_endian || SKIP=1 +testing "hexdump -e /2 %d (little endian)" \ + "hexdump -e '8/2 \" %6d\" \"\n\"'" \ + "\ 256 770 1284 1798 2312 2826 3340 3854 4368 4882 5396 5910 6424 6938 7452 7966 29040 29554 30068 30582 31096 31610 32124 32638 -32384 -31870 -31356 -30842 -30328 -29814 -29300 -28786 -3600 -3086 -2572 -2058 -1544 -1030 -516 -2 " \ - "" \ -"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"\ -"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"\ -"\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"\ -"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"\ -"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"\ + "" "$input" +SKIP= + +$little_endian && SKIP=1 +testing "hexdump -e /2 %d (big endian)" \ + "hexdump -e '8/2 \" %6d\" \"\n\"'" \ + "\ + 1 515 1029 1543 2057 2571 3085 3599 + 4113 4627 5141 5655 6169 6683 7197 7711 + 28785 29299 29813 30327 30841 31355 31869 32383 + -32639 -32125 -31611 -31097 -30583 -30069 -29555 -29041 + -3855 -3341 -2827 -2313 -1799 -1285 -771 -257 +" \ + "" "$input" +SKIP= -testing "hexdump -n4 -e '\"%u\"'" \ +$little_endian || SKIP=1 +testing "hexdump -e /2 %x (little endian)" \ + "hexdump -e '8/2 \" %6x\" \"\n\"'" \ + "\ + 100 302 504 706 908 b0a d0c f0e + 1110 1312 1514 1716 1918 1b1a 1d1c 1f1e + 7170 7372 7574 7776 7978 7b7a 7d7c 7f7e + 8180 8382 8584 8786 8988 8b8a 8d8c 8f8e + f1f0 f3f2 f5f4 f7f6 f9f8 fbfa fdfc fffe +" \ + "" "$input" +SKIP= + +$little_endian && SKIP=1 +testing "hexdump -e /2 %x (big endian)" \ + "hexdump -e '8/2 \" %6x\" \"\n\"'" \ + "\ + 1 203 405 607 809 a0b c0d e0f + 1011 1213 1415 1617 1819 1a1b 1c1d 1e1f + 7071 7273 7475 7677 7879 7a7b 7c7d 7e7f + 8081 8283 8485 8687 8889 8a8b 8c8d 8e8f + f0f1 f2f3 f4f5 f6f7 f8f9 fafb fcfd feff +" \ + "" "$input" +SKIP= + +$little_endian || SKIP=1 +testing "hexdump -n4 -e '\"%u\"' (little endian)" \ "hexdump -n4 -e '\"%u\"'" \ "12345678" \ "" \ "\x4e\x61\xbc\x00AAAA" +SKIP= + +$little_endian && SKIP=1 +testing "hexdump -n4 -e '\"%u\"' (big endian)" \ + "hexdump -n4 -e '\"%u\"'" \ + "1315027968" \ + "" \ + "\x4e\x61\xbc\x00AAAA" +SKIP= exit $FAILCOUNT From 4b1f1da358e0cf18bd7c8265ea1a9121c2801a21 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 1 Jul 2025 19:26:50 +0200 Subject: [PATCH 081/277] df: -k overrides block size to 1k even if $POSIXLY_CORRECT function old new delta df_main 1060 1070 +10 Signed-off-by: Denys Vlasenko --- coreutils/df.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/coreutils/df.c b/coreutils/df.c index 03aa781485..fba23246fe 100644 --- a/coreutils/df.c +++ b/coreutils/df.c @@ -155,6 +155,9 @@ int df_main(int argc UNUSED_PARAM, char **argv) , &opt_t IF_FEATURE_DF_FANCY(, &chp) ); + /* -k overrides $POSIXLY_CORRECT: */ + if (opt & OPT_KILO) + df_disp_hr = 1024; if (opt & OPT_MEGA) df_disp_hr = 1024*1024; @@ -185,8 +188,8 @@ int df_main(int argc UNUSED_PARAM, char **argv) if (disp_units_hdr == NULL) { #if ENABLE_FEATURE_HUMAN_READABLE disp_units_hdr = xasprintf("%s-blocks", - /* print df_disp_hr, show no fractionals, - * use suffixes if OPT_POSIX is set in opt */ + /* print df_disp_hr; show no fractionals; + * if -P, unit=1 (print it in full, no KMG suffixes) */ make_human_readable_str(df_disp_hr, 0, !!(opt & OPT_POSIX)) ); #else From dcd8df258a402658ad1fdc018a245b06a610ca8d Mon Sep 17 00:00:00 2001 From: Ron Yorston Date: Tue, 1 Jul 2025 09:47:14 +0100 Subject: [PATCH 082/277] shell: improve bash compatibility of read built-in Make the read built-in more compatible with bash: - Return an exit code of 142 on timeout. - When the timeout expires before a newline is detected in the input bash captures the partial input. This behaviour is new since bash version 4.4. BusyBox shells had the pre-4.4 behaviour where the input was lost. Update the tests to suit and fix a couple of compiler errors in the testsuite. function old new delta builtin_read 154 174 +20 readcmd 213 228 +15 shell_builtin_read 1364 1370 +6 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 3/0 up/down: 41/0) Total: 41 bytes Signed-off-by: Ron Yorston Signed-off-by: Denys Vlasenko --- shell/ash.c | 5 +++++ shell/ash_test/ash-read/read_ifs2.right | 9 +++++++++ shell/ash_test/ash-read/read_ifs2.tests | 9 +++++++++ shell/ash_test/ash-read/read_t.right | 8 ++++---- shell/ash_test/ash-read/read_t.tests | 18 +++++++++--------- shell/ash_test/printenv.c | 4 +--- shell/ash_test/recho.c | 2 +- shell/hush.c | 5 +++++ shell/hush_test/hush-read/read_t.right | 8 ++++---- shell/hush_test/hush-read/read_t.tests | 18 +++++++++--------- shell/shell_common.c | 10 +++++++--- 11 files changed, 63 insertions(+), 33 deletions(-) create mode 100644 shell/ash_test/ash-read/read_ifs2.right create mode 100755 shell/ash_test/ash-read/read_ifs2.tests diff --git a/shell/ash.c b/shell/ash.c index 9173b8608e..92b1df5f8a 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -14395,6 +14395,11 @@ readcmd(int argc UNUSED_PARAM, char **argv UNUSED_PARAM) goto again; } + if ((uintptr_t)r == 2) /* -t SEC timeout? */ + /* bash: "The exit status is greater than 128 if the timeout is exceeded." */ + /* The actual value observed with bash 5.2.15: */ + return 128 + SIGALRM; + if ((uintptr_t)r > 1) ash_msg_and_raise_error(r); diff --git a/shell/ash_test/ash-read/read_ifs2.right b/shell/ash_test/ash-read/read_ifs2.right new file mode 100644 index 0000000000..797137dae9 --- /dev/null +++ b/shell/ash_test/ash-read/read_ifs2.right @@ -0,0 +1,9 @@ +|X|Y:Z:| +|X|Y:Z| +|X|Y| +|X|Y| +|X|| +|X|| +||| +Whitespace should be trimmed too: +|X|Y| diff --git a/shell/ash_test/ash-read/read_ifs2.tests b/shell/ash_test/ash-read/read_ifs2.tests new file mode 100755 index 0000000000..f01a68978f --- /dev/null +++ b/shell/ash_test/ash-read/read_ifs2.tests @@ -0,0 +1,9 @@ +echo "X:Y:Z:" | (IFS=": " read x y; echo "|$x|$y|") +echo "X:Y:Z" | (IFS=": " read x y; echo "|$x|$y|") +echo "X:Y:" | (IFS=": " read x y; echo "|$x|$y|") +echo "X:Y" | (IFS=": " read x y; echo "|$x|$y|") +echo "X:" | (IFS=": " read x y; echo "|$x|$y|") +echo "X" | (IFS=": " read x y; echo "|$x|$y|") +echo "" | (IFS=": " read x y; echo "|$x|$y|") +echo Whitespace should be trimmed too: +echo "X:Y : " | (IFS=": " read x y; echo "|$x|$y|") diff --git a/shell/ash_test/ash-read/read_t.right b/shell/ash_test/ash-read/read_t.right index 04126cbe60..3eedae2756 100644 --- a/shell/ash_test/ash-read/read_t.right +++ b/shell/ash_test/ash-read/read_t.right @@ -1,4 +1,4 @@ ->< ->< ->test< ->test< +>te:142< +>:142< +>test:0< +>test:0< diff --git a/shell/ash_test/ash-read/read_t.tests b/shell/ash_test/ash-read/read_t.tests index d65f1aeaa0..9fbeec5172 100755 --- a/shell/ash_test/ash-read/read_t.tests +++ b/shell/ash_test/ash-read/read_t.tests @@ -1,10 +1,10 @@ -# bash 3.2 outputs: +# bash 5.2 outputs: -# >< -{ echo -n 'te'; sleep 2; echo 'st'; } | (read -t 1 reply; echo ">$reply<") -# >< -{ sleep 2; echo 'test'; } | (read -t 1 reply; echo ">$reply<") -# >test< -{ echo -n 'te'; sleep 1; echo 'st'; } | (read -t 2 reply; echo ">$reply<") -# >test< -{ sleep 1; echo 'test'; } | (read -t 2 reply; echo ">$reply<") +# >te:142< +{ echo -n 'te'; sleep 2; echo 'st'; } | (read -t 1 reply; echo ">$reply:$?<") +# >:142< +{ sleep 2; echo 'test'; } | (read -t 1 reply; echo ">$reply:$?<") +# >test:0< +{ echo -n 'te'; sleep 1; echo 'st'; } | (read -t 2 reply; echo ">$reply:$?<") +# >test:0< +{ sleep 1; echo 'test'; } | (read -t 2 reply; echo ">$reply:$?<") diff --git a/shell/ash_test/printenv.c b/shell/ash_test/printenv.c index c86308d3b6..f0f41984d3 100644 --- a/shell/ash_test/printenv.c +++ b/shell/ash_test/printenv.c @@ -31,9 +31,7 @@ extern char **environ; int -main (argc, argv) - int argc; - char **argv; +main (int argc, char **argv) { register char **envp, *eval; int len; diff --git a/shell/ash_test/recho.c b/shell/ash_test/recho.c index 42a5feafd6..7e96b14cc4 100644 --- a/shell/ash_test/recho.c +++ b/shell/ash_test/recho.c @@ -27,7 +27,7 @@ #include #include -void strprint(); +void strprint(char *); int main(int argc, char **argv) { diff --git a/shell/hush.c b/shell/hush.c index 4a97293ccb..37cfecc083 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -11175,6 +11175,11 @@ static int FAST_FUNC builtin_read(char **argv) goto again; } + if ((uintptr_t)r == 2) /* -t SEC timeout? */ + /* bash: "The exit status is greater than 128 if the timeout is exceeded." */ + /* The actual value observed with bash 5.2.15: */ + return 128 + SIGALRM; + if ((uintptr_t)r > 1) { bb_simple_error_msg(r); r = (char*)(uintptr_t)1; diff --git a/shell/hush_test/hush-read/read_t.right b/shell/hush_test/hush-read/read_t.right index 04126cbe60..3eedae2756 100644 --- a/shell/hush_test/hush-read/read_t.right +++ b/shell/hush_test/hush-read/read_t.right @@ -1,4 +1,4 @@ ->< ->< ->test< ->test< +>te:142< +>:142< +>test:0< +>test:0< diff --git a/shell/hush_test/hush-read/read_t.tests b/shell/hush_test/hush-read/read_t.tests index d65f1aeaa0..9fbeec5172 100755 --- a/shell/hush_test/hush-read/read_t.tests +++ b/shell/hush_test/hush-read/read_t.tests @@ -1,10 +1,10 @@ -# bash 3.2 outputs: +# bash 5.2 outputs: -# >< -{ echo -n 'te'; sleep 2; echo 'st'; } | (read -t 1 reply; echo ">$reply<") -# >< -{ sleep 2; echo 'test'; } | (read -t 1 reply; echo ">$reply<") -# >test< -{ echo -n 'te'; sleep 1; echo 'st'; } | (read -t 2 reply; echo ">$reply<") -# >test< -{ sleep 1; echo 'test'; } | (read -t 2 reply; echo ">$reply<") +# >te:142< +{ echo -n 'te'; sleep 2; echo 'st'; } | (read -t 1 reply; echo ">$reply:$?<") +# >:142< +{ sleep 2; echo 'test'; } | (read -t 1 reply; echo ">$reply:$?<") +# >test:0< +{ echo -n 'te'; sleep 1; echo 'st'; } | (read -t 2 reply; echo ">$reply:$?<") +# >test:0< +{ sleep 1; echo 'test'; } | (read -t 2 reply; echo ">$reply:$?<") diff --git a/shell/shell_common.c b/shell/shell_common.c index e5c2cefb3c..9a03f72655 100644 --- a/shell/shell_common.c +++ b/shell/shell_common.c @@ -204,8 +204,8 @@ shell_builtin_read(struct builtin_read_params *params) * 32-bit unix time wrapped (year 2038+). */ if (timeout <= 0) { /* already late? */ - retval = (const char *)(uintptr_t)1; - goto ret; + retval = (const char *)(uintptr_t)2; + break; } } @@ -217,8 +217,12 @@ shell_builtin_read(struct builtin_read_params *params) pfd[0].events = POLLIN; //TODO race with a signal arriving just before the poll! if (poll(pfd, 1, timeout) <= 0) { - /* timed out, or EINTR */ + /* timed out, or some error */ err = errno; + if (!err) { + retval = (const char *)(uintptr_t)2; + break; + } retval = (const char *)(uintptr_t)1; goto ret; } From 244f5c47954010d45b7bf19dc4c7c9c041bf6807 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Sun, 1 Jun 2025 12:26:06 +0800 Subject: [PATCH 083/277] libbb: Prevent clang from reading the G pointer before it is assigned It was observed that getty crashes on RISC-V 64-bit target, with the busybox binary compiled by clang/LLVM 17 with -O2. Not only getty, but also some other applets like syslogd/vi are broken too. Commit 5156b245 ("Make const ptr assign as function call in clang") introduced XZALLOC_CONST_PTR() to defeat the compiler optimization, however it only fixed a small number of broken places when compiling busybox with clang/LLVM. A large number of places remain broken. This commit treats ASSIGN_CONST_PTR() the same way as XZALLOC_CONST_PTR(). Signed-off-by: Bin Meng Signed-off-by: Denys Vlasenko --- include/libbb.h | 22 +++------------------- libbb/const_hack.c | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 558f3c6272..287103756d 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2337,26 +2337,10 @@ extern struct globals *BB_GLOBAL_CONST ptr_to_globals; #define barrier() asm volatile ("":::"memory") #if defined(__clang_major__) && __clang_major__ >= 9 -/* Clang/llvm drops assignment to "constant" storage. Silently. - * Needs serious convincing to not eliminate the store. - */ -static ALWAYS_INLINE void* not_const_pp(const void *p) -{ - void *pp; - asm volatile ( - "# forget that p points to const" - : /*outputs*/ "=r" (pp) - : /*inputs*/ "0" (p) - ); - return pp; -} -# define ASSIGN_CONST_PTR(pptr, v) do { \ - *(void**)not_const_pp(pptr) = (void*)(v); \ - barrier(); \ -} while (0) -/* XZALLOC_CONST_PTR() is an out-of-line function to prevent - * clang from reading pointer before it is assigned. +/* {ASSIGN,XZALLOC}_CONST_PTR() are out-of-line functions + * to prevent clang from reading pointer before it is assigned. */ +void ASSIGN_CONST_PTR(const void *pptr, void *v) FAST_FUNC; void XZALLOC_CONST_PTR(const void *pptr, size_t size) FAST_FUNC; #else # define ASSIGN_CONST_PTR(pptr, v) do { \ diff --git a/libbb/const_hack.c b/libbb/const_hack.c index 9575e6d676..1d175481b0 100644 --- a/libbb/const_hack.c +++ b/libbb/const_hack.c @@ -9,8 +9,27 @@ #include "libbb.h" #if defined(__clang_major__) && __clang_major__ >= 9 +/* Clang/llvm drops assignment to "constant" storage. Silently. + * Needs serious convincing to not eliminate the store. + */ +static ALWAYS_INLINE void* not_const_pp(const void *p) +{ + void *pp; + asm volatile ( + "# forget that p points to const" + : /*outputs*/ "=r" (pp) + : /*inputs*/ "0" (p) + ); + return pp; +} +void FAST_FUNC ASSIGN_CONST_PTR(const void *pptr, void *v) +{ + *(void**)not_const_pp(pptr) = v; + barrier(); +} void FAST_FUNC XZALLOC_CONST_PTR(const void *pptr, size_t size) { - ASSIGN_CONST_PTR(pptr, xzalloc(size)); + *(void**)not_const_pp(pptr) = xzalloc(size); + barrier(); } #endif From 8c835540efb88f46a66ee28c74f031b74ad07b26 Mon Sep 17 00:00:00 2001 From: Sertonix Date: Tue, 8 Apr 2025 09:32:26 +0000 Subject: [PATCH 084/277] cpio: map -F to --file long option function old new delta .rodata 100437 100444 +7 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/0 up/down: 7/0) Total: 7 bytes Signed-off-by: Denys Vlasenko --- archival/cpio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/archival/cpio.c b/archival/cpio.c index 7b01ee3a24..b033b37337 100644 --- a/archival/cpio.c +++ b/archival/cpio.c @@ -427,6 +427,7 @@ int cpio_main(int argc UNUSED_PARAM, char **argv) #endif #endif "owner\0" Required_argument "R" + "file\0" Required_argument "F" "verbose\0" No_argument "v" "null\0" No_argument "0" "quiet\0" No_argument "\xff" From bbaa56f7378240fca3db9165dfd02a4504d95ff1 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 2 Jul 2025 21:55:42 +0200 Subject: [PATCH 085/277] use pollfd[1] array for poll() argument Signed-off-by: Denys Vlasenko --- libbb/read_key.c | 11 ++++++----- shell/shell_common.c | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/libbb/read_key.c b/libbb/read_key.c index cf8ed411e8..1bea75fcf1 100644 --- a/libbb/read_key.c +++ b/libbb/read_key.c @@ -11,7 +11,7 @@ int64_t FAST_FUNC read_key(int fd, char *buffer, int timeout) { - struct pollfd pfd; + struct pollfd pfd[1]; const char *seq; int n; @@ -112,8 +112,8 @@ int64_t FAST_FUNC read_key(int fd, char *buffer, int timeout) 0 }; - pfd.fd = fd; - pfd.events = POLLIN; + pfd->fd = fd; + pfd->events = POLLIN; buffer++; /* saved chars counter is in buffer[-1] now */ @@ -135,6 +135,7 @@ int64_t FAST_FUNC read_key(int fd, char *buffer, int timeout) return -1; } } + /* It is tempting to read more than one byte here, * but it breaks pasting. Example: at shell prompt, * user presses "c","a","t" and then pastes "\nline\n". @@ -173,7 +174,7 @@ int64_t FAST_FUNC read_key(int fd, char *buffer, int timeout) * so if we block for long it's not really an escape sequence. * Timeout is needed to reconnect escape sequences * split up by transmission over a serial console. */ - if (safe_poll(&pfd, 1, 50) == 0) { + if (safe_poll(pfd, 1, 50) == 0) { /* No more data! * Array is sorted from shortest to longest, * we can't match anything later in array - @@ -222,7 +223,7 @@ int64_t FAST_FUNC read_key(int fd, char *buffer, int timeout) * n = bytes read. Try to read more until we time out. */ while (n < KEYCODE_BUFFER_SIZE-1) { /* 1 for count byte at buffer[-1] */ - if (safe_poll(&pfd, 1, 50) == 0) { + if (safe_poll(pfd, 1, 50) == 0) { /* No more data! */ break; } diff --git a/shell/shell_common.c b/shell/shell_common.c index 9a03f72655..2baa9d3a80 100644 --- a/shell/shell_common.c +++ b/shell/shell_common.c @@ -55,7 +55,7 @@ const char* FAST_FUNC shell_builtin_read(struct builtin_read_params *params) { struct pollfd pfd[1]; -#define fd (pfd[0].fd) /* -u FD */ +#define fd (pfd->fd) /* -u FD */ unsigned err; unsigned end_ms; /* -t TIMEOUT */ int nchars; /* -n NUM */ @@ -142,7 +142,7 @@ shell_builtin_read(struct builtin_read_params *params) * bash seems to ignore -p PROMPT for this use case. */ int r; - pfd[0].events = POLLIN; + pfd->events = POLLIN; r = poll(pfd, 1, /*timeout:*/ 0); /* Return 0 only if poll returns 1 ("one fd ready"), else return 1: */ return (const char *)(uintptr_t)(r <= 0); From 36ac283682bb176987c0c09e5f8c7e7e96fb1651 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 2 Jul 2025 22:42:47 +0200 Subject: [PATCH 086/277] shell: fix race between signal handlers setting bb_got_signal and poll() function old new delta __ppoll_time64 - 211 +211 check_got_signal_and_poll - 164 +164 read_key 607 601 -6 shell_builtin_read 1328 1318 -10 ------------------------------------------------------------------------------ (add/remove: 4/0 grow/shrink: 0/2 up/down: 375/-16) Total: 359 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 1 + libbb/lineedit.c | 1 - libbb/poll_with_signals.c | 48 +++++++++++++++++++++++++++++++++++++++ libbb/read_key.c | 14 ++++++++---- miscutils/less.c | 2 +- shell/ash.c | 2 +- shell/hush.c | 2 +- shell/shell_common.c | 9 ++++---- 8 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 libbb/poll_with_signals.c diff --git a/include/libbb.h b/include/libbb.h index 287103756d..801f072faa 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1951,6 +1951,7 @@ int64_t read_key(int fd, char *buffer, int timeout) FAST_FUNC; int64_t safe_read_key(int fd, char *buffer, int timeout) FAST_FUNC; void read_key_ungets(char *buffer, const char *str, unsigned len) FAST_FUNC; +int check_got_signal_and_poll(struct pollfd pfd[1], int timeout) FAST_FUNC; #if ENABLE_FEATURE_EDITING /* It's NOT just ENABLEd or disabled. It's a number: */ diff --git a/libbb/lineedit.c b/libbb/lineedit.c index ea216e16c9..10a83bcb7e 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -2195,7 +2195,6 @@ static int lineedit_read_key(char *read_key_buffer, int timeout) errno = EINTR; return -1; } -//FIXME: still races here with signals, but small window to poll() inside read_key IF_FEATURE_EDITING_WINCH(S.ok_to_redraw = 1;) /* errno = 0; - read_key does this itself */ ic = read_key(STDIN_FILENO, read_key_buffer, timeout); diff --git a/libbb/poll_with_signals.c b/libbb/poll_with_signals.c new file mode 100644 index 0000000000..47419f240f --- /dev/null +++ b/libbb/poll_with_signals.c @@ -0,0 +1,48 @@ +/* vi: set sw=4 ts=4: */ +/* + * Utility routines. + * + * Copyright (C) 2025 Denys Vlasenko + * + * Licensed under GPLv2, see file LICENSE in this source tree. + */ +//kbuild:lib-y += poll_with_signals.o + +#include "libbb.h" + +/* Shells, for example, need their line input and "read" builtin + * to be interruptible, and the naive handling of it a-la: + * if (bb_got_signal) { + * errno = EINTR; + * return -1; + * } + * poll(pfd, 1, -1); // signal here would set EINTR + * is racy. + * This is a bit heavy-handed, but safe wrt races: + */ +int FAST_FUNC check_got_signal_and_poll(struct pollfd pfd[1], int timeout) +{ + int n; + struct timespec tv; + sigset_t orig_mask; + + if (bb_got_signal) /* optimization */ + goto eintr; + + if (timeout >= 0) { + tv.tv_sec = timeout / 1000; + tv.tv_nsec = (timeout % 1000) * 1000000; + } + /* test bb_got_signal, then poll(), atomically wrt signals */ + sigfillset(&orig_mask); + sigprocmask2(SIG_BLOCK, &orig_mask); + if (bb_got_signal) { + sigprocmask2(SIG_SETMASK, &orig_mask); + eintr: + errno = EINTR; /* inform the caller that we got a signal */ + return -1; + } + n = ppoll(pfd, 1, timeout >= 0 ? &tv : NULL, &orig_mask); + sigprocmask2(SIG_SETMASK, &orig_mask); + return n; +} diff --git a/libbb/read_key.c b/libbb/read_key.c index 1bea75fcf1..3df9769f75 100644 --- a/libbb/read_key.c +++ b/libbb/read_key.c @@ -121,12 +121,16 @@ int64_t FAST_FUNC read_key(int fd, char *buffer, int timeout) errno = 0; n = (unsigned char)buffer[-1]; if (n == 0) { - /* If no data, wait for input. - * If requested, wait TIMEOUT ms. TIMEOUT = -1 is useful - * if fd can be in non-blocking mode. - */ + /* No data. Wait for input. */ + + /* timeout == -2 means "do not poll". Else: */ if (timeout >= -1) { - n = poll(&pfd, 1, timeout); + /* We must poll even if timeout == -1: + * we want to be interrupted if signal arrives, + * regardless of SA_RESTART-ness of that signal! + */ + /* test bb_got_signal, then poll(), atomically wrt signals */ + n = check_got_signal_and_poll(pfd, timeout); if (n < 0 && errno == EINTR) return n; if (n == 0) { diff --git a/miscutils/less.c b/miscutils/less.c index 8a0525cb7b..2204b1cecb 100644 --- a/miscutils/less.c +++ b/miscutils/less.c @@ -1139,7 +1139,7 @@ static int64_t getch_nowait(void) /* We have kbd_fd in O_NONBLOCK mode, read inside safe_read_key() * would not block even if there is no input available */ - key64 = safe_read_key(kbd_fd, kbd_input, /*timeout off:*/ -2); + key64 = safe_read_key(kbd_fd, kbd_input, /*do not poll:*/ -2); if ((int)key64 == -1) { if (errno == EAGAIN) { /* No keyboard input available. Since poll() did return, diff --git a/shell/ash.c b/shell/ash.c index 92b1df5f8a..16eb88a7b8 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -3687,7 +3687,7 @@ signal_handler(int signo) return; } #if ENABLE_FEATURE_EDITING - bb_got_signal = signo; /* for read_line_input: "we got a signal" */ + bb_got_signal = signo; /* for read_line_input / read builtin: "we got a signal" */ #endif gotsig[signo - 1] = 1; pending_sig = signo; diff --git a/shell/hush.c b/shell/hush.c index 37cfecc083..70b730f678 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -1973,7 +1973,7 @@ static void record_pending_signo(int sig) || (G_traps && G_traps[SIGCHLD] && G_traps[SIGCHLD][0]) /* ^^^ if SIGCHLD, interrupt line reading only if it has a trap */ ) { - bb_got_signal = sig; /* for read_line_input: "we got a signal" */ + bb_got_signal = sig; /* for read_line_input / read builtin: "we got a signal" */ } #endif #if ENABLE_HUSH_FAST diff --git a/shell/shell_common.c b/shell/shell_common.c index 2baa9d3a80..754fef34b5 100644 --- a/shell/shell_common.c +++ b/shell/shell_common.c @@ -214,12 +214,13 @@ shell_builtin_read(struct builtin_read_params *params) * regardless of SA_RESTART-ness of that signal! */ errno = 0; - pfd[0].events = POLLIN; -//TODO race with a signal arriving just before the poll! - if (poll(pfd, 1, timeout) <= 0) { + pfd->events = POLLIN; + + /* test bb_got_signal, then poll(), atomically wrt signals */ + if (check_got_signal_and_poll(pfd, timeout) <= 0) { /* timed out, or some error */ err = errno; - if (!err) { + if (!err) { /* timed out */ retval = (const char *)(uintptr_t)2; break; } From 9c46a0688576dd8c67d2fc24b68c07402da14fc8 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 3 Jul 2025 19:10:42 +0200 Subject: [PATCH 087/277] shell: update HISTFILESIZE code to be actually useful "HISTFILESIZE=0" in profile wasn't working as intended, "unset HISTFILE" wasn't preventing creation of history files Now: HISTSIZE=n allows to reduce in-memory history buffer HISTFILESIZE=n allows to reduce history file size (0: truncate it) unset HISTFILE allows to not save history file at all function old new delta exitshell 138 194 +56 hush_exit 97 143 +46 save_history 266 296 +30 hush_main 1170 1186 +16 .rodata 105762 105771 +9 load_history 246 254 +8 size_from_HISTFILESIZE 44 41 -3 read_line_input 2746 2712 -34 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 6/2 up/down: 165/-37) Total: 128 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 2 +- libbb/Config.src | 4 +- libbb/lineedit.c | 98 ++++++++++++++++++++++++++---------------------- shell/Config.src | 7 ++-- shell/ash.c | 26 ++++++++++++- shell/hush.c | 41 +++++++++++++++++--- 6 files changed, 120 insertions(+), 58 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 801f072faa..e765e18ebc 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1989,7 +1989,7 @@ typedef struct line_input_t { # if MAX_HISTORY int cnt_history; int cur_history; - int max_history; /* must never be <= 0 */ + int max_history; /* must never be < 0 */ # if ENABLE_FEATURE_EDITING_SAVEHISTORY /* meaning of this field depends on FEATURE_EDITING_SAVE_ON_EXIT: * if !FEATURE_EDITING_SAVE_ON_EXIT: "how many lines are diff --git a/libbb/Config.src b/libbb/Config.src index b980f19a92..55e670dcd7 100644 --- a/libbb/Config.src +++ b/libbb/Config.src @@ -182,8 +182,8 @@ config FEATURE_EDITING_VI config FEATURE_EDITING_HISTORY int "History size" # Don't allow way too big values here, code uses fixed "char *history[N]" struct member - range 0 9999 - default 255 + range 0 2000 + default 200 depends on FEATURE_EDITING help Specify command history size (0 - disable). diff --git a/libbb/lineedit.c b/libbb/lineedit.c index 10a83bcb7e..fe0dbc5b85 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -1406,8 +1406,8 @@ unsigned FAST_FUNC size_from_HISTFILESIZE(const char *hp) int size = MAX_HISTORY; if (hp) { size = atoi(hp); - if (size <= 0) - return 1; + if (size < 0) + return 0; if (size > MAX_HISTORY) return MAX_HISTORY; } @@ -1501,18 +1501,21 @@ static void load_history(line_input_t *st_parm) /* NB: do not trash old history if file can't be opened */ fp = fopen_for_read(st_parm->hist_file); - if (fp) { - /* clean up old history */ - for (idx = st_parm->cnt_history; idx > 0;) { - idx--; - free(st_parm->history[idx]); - st_parm->history[idx] = NULL; - } + if (!fp) + return; - /* fill temp_h[], retaining only last MAX_HISTORY lines */ - memset(temp_h, 0, sizeof(temp_h)); - idx = 0; - st_parm->cnt_history_in_file = 0; + /* clean up old history */ + for (idx = st_parm->cnt_history; idx > 0;) { + idx--; + free(st_parm->history[idx]); + st_parm->history[idx] = NULL; + } + + /* fill temp_h[], retaining only last max_history lines */ + memset(temp_h, 0, sizeof(temp_h)); + idx = 0; + st_parm->cnt_history_in_file = 0; + if (st_parm->max_history != 0) { while ((line = xmalloc_fgetline(fp)) != NULL) { if (line[0] == '\0') { free(line); @@ -1525,34 +1528,34 @@ static void load_history(line_input_t *st_parm) if (idx == st_parm->max_history) idx = 0; } - fclose(fp); - - /* find first non-NULL temp_h[], if any */ - if (st_parm->cnt_history_in_file) { - while (temp_h[idx] == NULL) { - idx++; - if (idx == st_parm->max_history) - idx = 0; - } - } + } + fclose(fp); - /* copy temp_h[] to st_parm->history[] */ - for (i = 0; i < st_parm->max_history;) { - line = temp_h[idx]; - if (!line) - break; + /* find first non-NULL temp_h[], if any */ + if (st_parm->cnt_history_in_file != 0) { + while (temp_h[idx] == NULL) { idx++; if (idx == st_parm->max_history) idx = 0; - line_len = strlen(line); - if (line_len >= MAX_LINELEN) - line[MAX_LINELEN-1] = '\0'; - st_parm->history[i++] = line; } - st_parm->cnt_history = i; - if (ENABLE_FEATURE_EDITING_SAVE_ON_EXIT) - st_parm->cnt_history_in_file = i; } + + /* copy temp_h[] to st_parm->history[] */ + for (i = 0; i < st_parm->max_history;) { + line = temp_h[idx]; + if (!line) + break; + idx++; + if (idx == st_parm->max_history) + idx = 0; + line_len = strlen(line); + if (line_len >= MAX_LINELEN) + line[MAX_LINELEN-1] = '\0'; + st_parm->history[i++] = line; + } + st_parm->cnt_history = i; + if (ENABLE_FEATURE_EDITING_SAVE_ON_EXIT) + st_parm->cnt_history_in_file = i; } # if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT @@ -1563,14 +1566,23 @@ void FAST_FUNC save_history(line_input_t *st) if (!st || !st->hist_file) return; if (st->cnt_history <= st->cnt_history_in_file) - return; + return; /* no new entries were added */ + /* note: if st->max_history is 0, we do not abort: we truncate the history to 0 lines */ - fp = fopen(st->hist_file, "a"); + fp = fopen(st->hist_file, (st->max_history == 0 ? "w" : "a")); if (fp) { int i, fd; char *new_name; line_input_t *st_temp; + /* max_history==0 needs special-casing in general code, + * just handle it in a simpler way: */ + if (st->max_history == 0) { + /* fopen("w") already truncated it */ + fclose(fp); + return; + } + for (i = st->cnt_history_in_file; i < st->cnt_history; i++) fprintf(fp, "%s\n", st->history[i]); fclose(fp); @@ -1580,6 +1592,8 @@ void FAST_FUNC save_history(line_input_t *st) st_temp = new_line_input_t(st->flags); st_temp->hist_file = st->hist_file; st_temp->max_history = st->max_history; + /* load no more than max_history last lines */ + /* (in unlikely case that file disappeared, st_temp gets empty history) */ load_history(st_temp); /* write out temp file and replace hist_file atomically */ @@ -1609,7 +1623,6 @@ static void save_history(char *str) fd = open(state->hist_file, O_WRONLY | O_CREAT | O_APPEND, 0600); if (fd < 0) return; - xlseek(fd, 0, SEEK_END); /* paranoia */ len = strlen(str); str[len] = '\n'; /* we (try to) do atomic write */ len2 = full_write(fd, str, len + 1); @@ -1664,13 +1677,10 @@ static void remember_in_history(char *str) if (str[0] == '\0') return; i = state->cnt_history; - /* Don't save dupes */ - if (i && strcmp(state->history[i-1], str) == 0) + /* Don't save dups */ + if (i != 0 && strcmp(state->history[i-1], str) == 0) return; - free(state->history[state->max_history]); /* redundant, paranoia */ - state->history[state->max_history] = NULL; /* redundant, paranoia */ - /* If history[] is full, remove the oldest command */ /* we need to keep history[state->max_history] empty, hence >=, not > */ if (i >= state->max_history) { @@ -1683,7 +1693,7 @@ static void remember_in_history(char *str) state->cnt_history_in_file--; # endif } - /* i <= state->max_history-1 */ + /* i < state->max_history */ state->history[i++] = xstrdup(str); /* i <= state->max_history */ state->cur_history = i; diff --git a/shell/Config.src b/shell/Config.src index 5efbf99959..5b3fe08f38 100644 --- a/shell/Config.src +++ b/shell/Config.src @@ -166,9 +166,10 @@ config FEATURE_SH_HISTFILESIZE default y depends on SHELL_ASH || SHELL_HUSH help - This option makes busybox shells to use $HISTFILESIZE variable - to set shell history size. Note that its max value is capped - by "History size" setting in library tuning section. + This option makes busybox shells to use $HISTSIZE and + $HISTFILESIZE variables to set shell history size. + Note that its max value is capped by "History size" setting + in library tuning section. config FEATURE_SH_EMBEDDED_SCRIPTS bool "Embed scripts in the binary" diff --git a/shell/ash.c b/shell/ash.c index 16eb88a7b8..18344767a1 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -14525,8 +14525,25 @@ exitshell(void) char *p; #if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT - save_history(line_input_state); /* may be NULL */ + if (line_input_state) { + const char *hp; +# if ENABLE_FEATURE_SH_HISTFILESIZE +// in bash: +// HISTFILESIZE controls the on-disk history file size (in lines, 0=no history): +// "When this variable is assigned a value, the history file is truncated, if necessary" +// but we do it only at exit, not on assignment: + /* Use HISTFILESIZE to limit file size */ + hp = lookupvar("HISTFILESIZE"); + if (hp) + line_input_state->max_history = size_from_HISTFILESIZE(hp); +# endif + /* HISTFILE: "If unset, the command history is not saved when a shell exits." */ + hp = lookupvar("HISTFILE"); + line_input_state->hist_file = hp; + save_history(line_input_state); /* no-op if hist_file is NULL */ + } #endif + savestatus = exitstatus; TRACE(("pid %d, exitshell(%d)\n", getpid(), savestatus)); if (setjmp(loc.loc)) @@ -14867,7 +14884,12 @@ int ash_main(int argc UNUSED_PARAM, char **argv) if (hp) line_input_state->hist_file = xstrdup(hp); # if ENABLE_FEATURE_SH_HISTFILESIZE - hp = lookupvar("HISTFILESIZE"); + hp = lookupvar("HISTSIZE"); + /* Using HISTFILESIZE above to limit max_history would be WRONG: + * users may set HISTFILESIZE=0 in their profile scripts + * to prevent _saving_ of history files, but still want to have + * non-zero history limit for in-memory list. + */ line_input_state->max_history = size_from_HISTFILESIZE(hp); # endif } diff --git a/shell/hush.c b/shell/hush.c index 70b730f678..68aca53a3b 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -2099,11 +2099,29 @@ static sighandler_t pick_sighandler(unsigned sig) return handler; } +static const char* FAST_FUNC get_local_var_value(const char *name); + /* Restores tty foreground process group, and exits. */ static void hush_exit(int exitcode) { #if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT - save_history(G.line_input_state); /* may be NULL */ + if (G.line_input_state) { + const char *hp; +# if ENABLE_FEATURE_SH_HISTFILESIZE +// in bash: +// HISTFILESIZE controls the on-disk history file size (in lines, 0=no history): +// "When this variable is assigned a value, the history file is truncated, if necessary" +// but we do it only at exit, not on every assignment: + /* Use HISTFILESIZE to limit file size */ + hp = get_local_var_value("HISTFILESIZE"); + if (hp) + G.line_input_state->max_history = size_from_HISTFILESIZE(hp); +# endif + /* HISTFILE: "If unset, the command history is not saved when a shell exits." */ + hp = get_local_var_value("HISTFILE"); + G.line_input_state->hist_file = hp; + save_history(G.line_input_state); /* no-op if hist_file is NULL */ + } #endif fflush_all(); @@ -10427,7 +10445,7 @@ int hush_main(int argc, char **argv) if (!get_local_var_value("PATH")) set_local_var_from_halves("PATH", bb_default_root_path); - /* PS1/PS2 are set later, if we determine that we are interactive */ + /* PS1/PS2/HISTFILE are set later, if we determine that we are interactive */ /* bash also exports SHLVL and _, * and sets (but doesn't export) the following variables: @@ -10449,7 +10467,6 @@ int hush_main(int argc, char **argv) * BASH_SOURCE=() * DIRSTACK=() * PIPESTATUS=([0]="0") - * HISTFILE=//.bash_history * HISTFILESIZE=500 * HISTSIZE=500 * MAILCHECK=60 @@ -10809,18 +10826,30 @@ int hush_main(int argc, char **argv) const char *hp = get_local_var_value("HISTFILE"); if (!hp) { hp = get_local_var_value("HOME"); - if (hp) + if (hp) { hp = concat_path_file(hp, ".hush_history"); + /* Make HISTFILE set on exit (else history won't be saved) */ + set_local_var_from_halves("HISTFILE", hp); + } } else { hp = xstrdup(hp); } if (hp) { G.line_input_state->hist_file = hp; - //set_local_var(xasprintf("HISTFILE=%s", ...)); } # if ENABLE_FEATURE_SH_HISTFILESIZE - hp = get_local_var_value("HISTFILESIZE"); + hp = get_local_var_value("HISTSIZE"); + /* Using HISTFILESIZE above to limit max_history would be WRONG: + * users may set HISTFILESIZE=0 in their profile scripts + * to prevent _saving_ of history files, but still want to have + * non-zero history limit for in-memory list. + */ +// in bash, runtime history size is controlled by HISTSIZE (0=no history), +// HISTFILESIZE controls on-disk history file size (in lines, 0=no history): G.line_input_state->max_history = size_from_HISTFILESIZE(hp); +// HISTFILESIZE: "The shell sets the default value to the value of HISTSIZE after reading any startup files." +// HISTSIZE: "The shell sets the default value to 500 after reading any startup files." +// (meaning: if the value wasn't set after startup files, the default value is set as described above) # endif } # endif From 8d3d0789172ab3f6580d04cd4bbafaef5e108ac7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 4 Jul 2025 12:47:11 +0200 Subject: [PATCH 088/277] shell: empty HISTFILE disables history saving, just as unset one did The rationale here is that unsetting HISTFILE in /etc/profile does not "stick": if it's unset, the default one is set later (after /etc/profile is executed) by the shell. But setting (and exporting, so it is inherited by all (grand)child shells) an empty one works. function old new delta save_history 296 316 +20 Signed-off-by: Denys Vlasenko --- libbb/lineedit.c | 6 ++++-- shell/ash.c | 2 +- shell/hush.c | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libbb/lineedit.c b/libbb/lineedit.c index fe0dbc5b85..43d1da35c9 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -1563,7 +1563,8 @@ void FAST_FUNC save_history(line_input_t *st) { FILE *fp; - if (!st || !st->hist_file) + /* bash compat: HISTFILE="" disables history saving */ + if (!st || !st->hist_file || !state->hist_file[0]) return; if (st->cnt_history <= st->cnt_history_in_file) return; /* no new entries were added */ @@ -1617,7 +1618,8 @@ static void save_history(char *str) int fd; int len, len2; - if (!state->hist_file) + /* bash compat: HISTFILE="" disables history saving */ + if (!state->hist_file || !state->hist_file[0]) return; fd = open(state->hist_file, O_WRONLY | O_CREAT | O_APPEND, 0600); diff --git a/shell/ash.c b/shell/ash.c index 18344767a1..1a85eec715 100644 --- a/shell/ash.c +++ b/shell/ash.c @@ -14540,7 +14540,7 @@ exitshell(void) /* HISTFILE: "If unset, the command history is not saved when a shell exits." */ hp = lookupvar("HISTFILE"); line_input_state->hist_file = hp; - save_history(line_input_state); /* no-op if hist_file is NULL */ + save_history(line_input_state); /* no-op if hist_file is NULL or "" */ } #endif diff --git a/shell/hush.c b/shell/hush.c index 68aca53a3b..0b2bc01f56 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -2120,7 +2120,7 @@ static void hush_exit(int exitcode) /* HISTFILE: "If unset, the command history is not saved when a shell exits." */ hp = get_local_var_value("HISTFILE"); G.line_input_state->hist_file = hp; - save_history(G.line_input_state); /* no-op if hist_file is NULL */ + save_history(G.line_input_state); /* no-op if hist_file is NULL or "" */ } #endif From 142ac2d796dddea50d8926d1747523182c12d097 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 4 Jul 2025 21:40:51 +0200 Subject: [PATCH 089/277] diff: code shrink function old new delta diffreg 1317 1307 -10 Signed-off-by: Denys Vlasenko --- editors/diff.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editors/diff.c b/editors/diff.c index 1adc4cbc78..6f4ed97120 100644 --- a/editors/diff.c +++ b/editors/diff.c @@ -741,7 +741,7 @@ static int diffreg(char *file[2]) fd = fd_tmp; xlseek(fd, 0, SEEK_SET); } - fp[i] = fdopen(fd, "r"); + fp[i] = xfdopen_for_read(fd); } setup_common_bufsiz(); From 84766710f420dd444e2a03d33a1915ce55661e67 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 5 Jul 2025 12:59:42 +0200 Subject: [PATCH 090/277] whitespace fix Signed-off-by: Denys Vlasenko --- networking/hostname.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/networking/hostname.c b/networking/hostname.c index 36cb708669..101b89e77c 100644 --- a/networking/hostname.c +++ b/networking/hostname.c @@ -25,8 +25,8 @@ //applet:IF_DNSDOMAINNAME(APPLET_NOEXEC(dnsdomainname, hostname, BB_DIR_BIN, BB_SUID_DROP, dnsdomainname)) //applet:IF_HOSTNAME( APPLET_NOEXEC(hostname, hostname, BB_DIR_BIN, BB_SUID_DROP, hostname )) -//kbuild: lib-$(CONFIG_HOSTNAME) += hostname.o -//kbuild: lib-$(CONFIG_DNSDOMAINNAME) += hostname.o +//kbuild:lib-$(CONFIG_HOSTNAME) += hostname.o +//kbuild:lib-$(CONFIG_DNSDOMAINNAME) += hostname.o //usage:#define hostname_trivial_usage //usage: "[-sidf] [HOSTNAME | -F FILE]" From 80e676664e1d7a0b07f14bff44f93d1fef709cf4 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 00:44:19 +0200 Subject: [PATCH 091/277] libbb: add yescrypt password hashing support It seems to work, but not at all optimized for size. The extra copy of sha256 code need to be removed. The yescrypt code in libbb/yescrypt/* is adapted from libxcrypt-4.4.38 with minimal edits, hopefully making it easier to track backports by resetting the tree to this commit, then comparing changes in upstream libxcrypt to the tree. function old new delta blockmix_xor_save - 7050 +7050 static.blockmix_xor - 6475 +6475 blockmix - 3390 +3390 SHA256_Transform - 3083 +3083 yescrypt_kdf_body - 1724 +1724 PBKDF2_SHA256 - 1003 +1003 smix1 - 960 +960 yescrypt_r - 890 +890 salsa20 - 804 +804 smix - 790 +790 smix2 - 659 +659 blockmix_salsa8_xor - 601 +601 yescrypt_kdf - 479 +479 blockmix_salsa8 - 415 +415 Krnd - 256 +256 _HMAC_SHA256_Init - 213 +213 _SHA256_Update - 198 +198 _SHA256_Final - 195 +195 decode64_uint32 - 166 +166 encode64 - 153 +153 decode64 - 136 +136 libcperciva_HMAC_SHA256_Buf - 132 +132 SHA256_Pad_Almost - 131 +131 salsa20_simd_unshuffle - 101 +101 salsa20_simd_shuffle - 101 +101 yes_crypt - 90 +90 libcperciva_SHA256_Buf - 86 +86 crypt_make_rand64encoded - 85 +85 static.atoi64_partial - 77 +77 alloc_region - 72 +72 ascii64 - 65 +65 PAD - 64 +64 _HMAC_SHA256_Final - 55 +55 static.cpu_to_be32_vect - 51 +51 free_region - 47 +47 libcperciva_SHA256_Init - 37 +37 yescrypt_init_local - 34 +34 crypt_make_pw_salt 92 125 +33 initial_state - 32 +32 .rodata 105771 105803 +32 atoi64 - 25 +25 explicit_bzero - 22 +22 pw_encrypt 920 941 +21 yescrypt_free_local - 9 +9 crypt_make_salt 85 - -85 ------------------------------------------------------------------------------ (add/remove: 43/1 grow/shrink: 3/0 up/down: 31042/-85) Total: 30957 bytes Signed-off-by: Denys Vlasenko --- Makefile | 1 + include/libbb.h | 22 +- libbb/pw_encrypt.c | 53 +- libbb/yescrypt/Kbuild.src | 9 + libbb/yescrypt/README | 4 + libbb/yescrypt/alg-sha256.c | 608 ++++++++++++++ libbb/yescrypt/alg-sha256.h | 113 +++ libbb/yescrypt/alg-yescrypt-common.c | 714 ++++++++++++++++ libbb/yescrypt/alg-yescrypt-kdf.c | 1070 ++++++++++++++++++++++++ libbb/yescrypt/alg-yescrypt-platform.c | 65 ++ libbb/yescrypt/alg-yescrypt.h | 355 ++++++++ libbb/yescrypt/y.c | 139 +++ loginutils/Config.src | 11 + networking/httpd.c | 2 +- testsuite/cryptpw.tests | 66 +- 15 files changed, 3206 insertions(+), 26 deletions(-) create mode 100644 libbb/yescrypt/Kbuild.src create mode 100644 libbb/yescrypt/README create mode 100644 libbb/yescrypt/alg-sha256.c create mode 100644 libbb/yescrypt/alg-sha256.h create mode 100644 libbb/yescrypt/alg-yescrypt-common.c create mode 100644 libbb/yescrypt/alg-yescrypt-kdf.c create mode 100644 libbb/yescrypt/alg-yescrypt-platform.c create mode 100644 libbb/yescrypt/alg-yescrypt.h create mode 100644 libbb/yescrypt/y.c diff --git a/Makefile b/Makefile index dcf7cd5859..3af11c0e69 100644 --- a/Makefile +++ b/Makefile @@ -476,6 +476,7 @@ libs-y := \ findutils/ \ init/ \ libbb/ \ + libbb/yescrypt/ \ libpwdgrp/ \ loginutils/ \ mailutils/ \ diff --git a/include/libbb.h b/include/libbb.h index e765e18ebc..9a0a2f9167 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1806,18 +1806,24 @@ extern char *pw_encrypt(const char *clear, const char *salt, int cleanup) FAST_F extern int obscure(const char *old, const char *newval, const struct passwd *pwdp) FAST_FUNC; /* * rnd is additional random input. New one is returned. - * Useful if you call crypt_make_salt many times in a row: - * rnd = crypt_make_salt(buf1, 4, 0); - * rnd = crypt_make_salt(buf2, 4, rnd); - * rnd = crypt_make_salt(buf3, 4, rnd); + * Useful if you call crypt_make_rand64encoded many times in a row: + * rnd = crypt_make_rand64encoded(buf1, 4, 0); + * rnd = crypt_make_rand64encoded(buf2, 4, rnd); + * rnd = crypt_make_rand64encoded(buf3, 4, rnd); * (otherwise we risk having same salt generated) */ -extern int crypt_make_salt(char *p, int cnt /*, int rnd*/) FAST_FUNC; -/* "$N$" + sha_salt_16_bytes + NUL */ -#define MAX_PW_SALT_LEN (3 + 16 + 1) +extern int crypt_make_rand64encoded(char *p, int cnt /*, int rnd*/) FAST_FUNC; +/* Size of char salt[] to hold randomly-generated salt string + * sha256/512: + * "$5$" + * "$6$" + * #define MAX_PW_SALT_LEN (3 + 16 + 1) + * yescrypt: + * "$y$j9T$" + */ +#define MAX_PW_SALT_LEN (7 + 24 + 1) extern char* crypt_make_pw_salt(char p[MAX_PW_SALT_LEN], const char *algo) FAST_FUNC; - /* Returns number of lines changed, or -1 on error */ #if !(ENABLE_FEATURE_ADDUSER_TO_GROUP || ENABLE_FEATURE_DEL_USER_FROM_GROUP) #define update_passwd(filename, username, data, member) \ diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 3463fd95ba..1d530974ef 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -31,7 +31,7 @@ static int i64c(int i) return ('a' - 38 + i); } -int FAST_FUNC crypt_make_salt(char *p, int cnt /*, int x */) +int FAST_FUNC crypt_make_rand64encoded(char *p, int cnt /*, int x */) { /* was: x += ... */ unsigned x = getpid() + monotonic_us(); @@ -68,11 +68,49 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) #if !ENABLE_USE_BB_CRYPT || ENABLE_USE_BB_CRYPT_SHA if ((algo[0]|0x20) == 's') { /* sha */ salt[1] = '5' + (strcasecmp(algo, "sha512") == 0); - len = 16/2; + len = 16 / 2; + } +#endif +#if !ENABLE_USE_BB_CRYPT || ENABLE_USE_BB_CRYPT_YES + if ((algo[0]|0x20) == 'y') { /* yescrypt */ + salt[1] = 'y'; + len = 24 / 2; +// The "j9T$" below is the default "yescrypt parameters" encoded by yescrypt_encode_params_r(): +// +//shadow-4.17.4/src/passwd.c +// salt = crypt_make_rand64encoded(NULL, NULL); +//shadow-4.17.4/lib/salt.c +//const char *crypt_make_rand64encoded(const char *meth, void *arg) +// if (streq(method, "YESCRYPT")) { +// MAGNUM(result, 'y'); +// salt_len = YESCRYPT_SALT_SIZE; // 24 +// rounds = YESCRYPT_get_salt_cost(arg); // always Y_COST_DEFAULT == 5 for NULL arg +// YESCRYPT_salt_cost_to_buf(result, rounds); // always "j9T$" +// char *retval = crypt_gensalt(result, rounds, NULL, 0); +//libxcrypt-4.4.38/lib/crypt-yescrypt.c +//void gensalt_yescrypt_rn (unsigned long count, +// const uint8_t *rbytes, size_t nrbytes, +// uint8_t *output, size_t o_size) +// yescrypt_params_t params = { +// .flags = YESCRYPT_DEFAULTS, +// .p = 1, +// }; +// if (count < 3) ... else +// params.r = 32; // N in 4KiB +// params.N = 1ULL << (count + 7); // 3 -> 1024, 4 -> 2048, ... 11 -> 262144 +// yescrypt_encode_params_r(¶ms, rbytes, nrbytes, outbuf, o_size) // always "$y$j9T$" + salt_ptr = stpcpy(salt_ptr, "j9T$"); + crypt_make_rand64encoded(salt_ptr, len); /* appends 2*len random chars */ + /* For "mkpasswd -m yescrypt PASS j9T$" use case, + * "j9T$" is considered part of salt, + * need to return pointer to 'j'. Without -4, + * we'd end up using "j9T$j9T$" as salt. + */ + return salt_ptr - 4; } #endif } - crypt_make_salt(salt_ptr, len); + crypt_make_rand64encoded(salt_ptr, len); /* appends 2*len random chars */ return salt_ptr; } @@ -99,6 +137,9 @@ to64(char *s, unsigned v, int n) #if ENABLE_USE_BB_CRYPT_SHA #include "pw_encrypt_sha.c" #endif +#if ENABLE_USE_BB_CRYPT_YES +#include "pw_encrypt_yes.c" +#endif /* Other advanced crypt ids (TODO?): */ /* $2$ or $2a$: Blowfish */ @@ -109,10 +150,14 @@ static struct des_ctx *des_ctx; /* my_crypt returns malloc'ed data */ static char *my_crypt(const char *key, const char *salt) { - /* MD5 or SHA? */ + /* "$x$...." string? */ if (salt[0] == '$' && salt[1] && salt[2] == '$') { if (salt[1] == '1') return md5_crypt(xzalloc(MD5_OUT_BUFSIZE), (unsigned char*)key, (unsigned char*)salt); +#if ENABLE_USE_BB_CRYPT_YES + if (salt[1] == 'y') + return yes_crypt(key, salt); +#endif #if ENABLE_USE_BB_CRYPT_SHA if (salt[1] == '5' || salt[1] == '6') return sha_crypt((char*)key, (char*)salt); diff --git a/libbb/yescrypt/Kbuild.src b/libbb/yescrypt/Kbuild.src new file mode 100644 index 0000000000..a61211a292 --- /dev/null +++ b/libbb/yescrypt/Kbuild.src @@ -0,0 +1,9 @@ +# Makefile for busybox +# +# Copyright (C) 2025 by Denys Vlasenko +# +# Licensed under GPLv2, see file LICENSE in this source tree. + +lib-y:= + +INSERT diff --git a/libbb/yescrypt/README b/libbb/yescrypt/README new file mode 100644 index 0000000000..c1011c56a3 --- /dev/null +++ b/libbb/yescrypt/README @@ -0,0 +1,4 @@ +The yescrypt code in this directory is adapted from libxcrypt-4.4.38 +with minimal edits, hopefully making it easier to track +backports by resetting the tree to the commit which created this file, +then comparing changes in upstream libxcrypt to the tree. diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c new file mode 100644 index 0000000000..0c1b846be1 --- /dev/null +++ b/libbb/yescrypt/alg-sha256.c @@ -0,0 +1,608 @@ +/*- + * Copyright 2005-2016 Colin Percival + * Copyright 2016-2018,2021 Alexander Peslyak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__GNUC__) +#define restrict __restrict +#else +#define restrict +#endif + +/* SHA256 round constants. */ +static const uint32_t Krnd[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +/* Elementary functions used by SHA256 */ +#define Ch(x, y, z) ((x & (y ^ z)) ^ z) +#if 1 /* Explicit caching/reuse of common subexpression between rounds */ +#define Maj(x, y, z) (y ^ ((x_xor_y = x ^ y) & y_xor_z)) +#else /* Let the compiler cache/reuse or not */ +#define Maj(x, y, z) (y ^ ((x ^ y) & (y ^ z))) +#endif +#define SHR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << (32 - n))) +#define S0(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define S1(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define s0(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHR(x, 3)) +#define s1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10)) + +/* SHA256 round function */ +#define RND(a, b, c, d, e, f, g, h, k) \ + h += S1(e) + Ch(e, f, g) + k; \ + d += h; \ + h += S0(a) + Maj(a, b, c); \ + y_xor_z = x_xor_y; + +/* Adjusted round function for rotating state */ +#define RNDr(S, W, i, ii) \ + RND(S[(64 - i) % 8], S[(65 - i) % 8], \ + S[(66 - i) % 8], S[(67 - i) % 8], \ + S[(68 - i) % 8], S[(69 - i) % 8], \ + S[(70 - i) % 8], S[(71 - i) % 8], \ + W[i + ii] + Krnd[i + ii]) + +/* Message schedule computation */ +#define MSCH(W, ii, i) \ + W[i + ii + 16] = s1(W[i + ii + 14]) + W[i + ii + 9] + s0(W[i + ii + 1]) + W[i + ii] + +/* + * SHA256 block compression function. The 256-bit state is transformed via + * the 512-bit input block to produce a new state. + */ +static void +SHA256_Transform(uint32_t state[static restrict 8], + const uint8_t block[static restrict 64], + uint32_t W[static restrict 64], uint32_t S[static restrict 8]) +{ + int i; + + /* 1. Prepare the first part of the message schedule W. */ + be32dec_vect(W, block, 16); + + /* 2. Initialize working variables. */ + memcpy(S, state, 32); + + /* 3. Mix. */ + for (i = 0; i <= 48; i += 16) { + uint32_t x_xor_y, y_xor_z = S[(65 - i) % 8] ^ S[(66 - i) % 8]; + RNDr(S, W, 0, i); + RNDr(S, W, 1, i); + RNDr(S, W, 2, i); + RNDr(S, W, 3, i); + RNDr(S, W, 4, i); + RNDr(S, W, 5, i); + RNDr(S, W, 6, i); + RNDr(S, W, 7, i); + RNDr(S, W, 8, i); + RNDr(S, W, 9, i); + RNDr(S, W, 10, i); + RNDr(S, W, 11, i); + RNDr(S, W, 12, i); + RNDr(S, W, 13, i); + RNDr(S, W, 14, i); + RNDr(S, W, 15, i); + + if (i == 48) + break; + + MSCH(W, 0, i); + MSCH(W, 1, i); + MSCH(W, 2, i); + MSCH(W, 3, i); + MSCH(W, 4, i); + MSCH(W, 5, i); + MSCH(W, 6, i); + MSCH(W, 7, i); + MSCH(W, 8, i); + MSCH(W, 9, i); + MSCH(W, 10, i); + MSCH(W, 11, i); + MSCH(W, 12, i); + MSCH(W, 13, i); + MSCH(W, 14, i); + MSCH(W, 15, i); + } + + /* 4. Mix local working variables into global state. */ + state[0] += S[0]; + state[1] += S[1]; + state[2] += S[2]; + state[3] += S[3]; + state[4] += S[4]; + state[5] += S[5]; + state[6] += S[6]; + state[7] += S[7]; +} + +static const uint8_t PAD[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* Add padding and terminating bit-count. */ +static void +SHA256_Pad(SHA256_CTX * ctx, uint32_t tmp32[static restrict 72]) +{ + size_t r; + + /* Figure out how many bytes we have buffered. */ + r = (ctx->count >> 3) & 0x3f; + + /* Pad to 56 mod 64, transforming if we finish a block en route. */ + if (r < 56) { + /* Pad to 56 mod 64. */ + memcpy(&ctx->buf[r], PAD, 56 - r); + } else { + /* Finish the current block and mix. */ + memcpy(&ctx->buf[r], PAD, 64 - r); + SHA256_Transform(ctx->state, ctx->buf, &tmp32[0], &tmp32[64]); + + /* The start of the final block is all zeroes. */ + memset(&ctx->buf[0], 0, 56); + } + + /* Add the terminating bit-count. */ + be64enc(&ctx->buf[56], ctx->count); + + /* Mix in the final block. */ + SHA256_Transform(ctx->state, ctx->buf, &tmp32[0], &tmp32[64]); +} + +/* Magic initialization constants. */ +static const uint32_t initial_state[8] = { + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 +}; + +/** + * SHA256_Init(ctx): + * Initialize the SHA256 context ${ctx}. + */ +void +SHA256_Init(SHA256_CTX * ctx) +{ + + /* Zero bits processed so far. */ + ctx->count = 0; + + /* Initialize state. */ + memcpy(ctx->state, initial_state, sizeof(initial_state)); +} + +/** + * SHA256_Update(ctx, in, len): + * Input ${len} bytes from ${in} into the SHA256 context ${ctx}. + */ +static void +_SHA256_Update(SHA256_CTX * ctx, const void * in, size_t len, + uint32_t tmp32[static restrict 72]) +{ + uint32_t r; + const uint8_t * src = in; + + /* Return immediately if we have nothing to do. */ + if (len == 0) + return; + + /* Number of bytes left in the buffer from previous updates. */ + r = (ctx->count >> 3) & 0x3f; + + /* Update number of bits. */ + ctx->count += (uint64_t)(len) << 3; + + /* Handle the case where we don't need to perform any transforms. */ + if (len < 64 - r) { + memcpy(&ctx->buf[r], src, len); + return; + } + + /* Finish the current block. */ + memcpy(&ctx->buf[r], src, 64 - r); + SHA256_Transform(ctx->state, ctx->buf, &tmp32[0], &tmp32[64]); + src += 64 - r; + len -= 64 - r; + + /* Perform complete blocks. */ + while (len >= 64) { + SHA256_Transform(ctx->state, src, &tmp32[0], &tmp32[64]); + src += 64; + len -= 64; + } + + /* Copy left over data into buffer. */ + memcpy(ctx->buf, src, len); +} + +/* Wrapper function for intermediate-values sanitization. */ +void +SHA256_Update(SHA256_CTX * ctx, const void * in, size_t len) +{ + uint32_t tmp32[72]; + + /* Call the real function. */ + _SHA256_Update(ctx, in, len, tmp32); + + /* Clean the stack. */ + explicit_bzero(tmp32, 288); +} + +/** + * SHA256_Final(digest, ctx): + * Output the SHA256 hash of the data input to the context ${ctx} into the + * buffer ${digest}. + */ +static void +_SHA256_Final(uint8_t digest[32], SHA256_CTX * ctx, + uint32_t tmp32[static restrict 72]) +{ + + /* Add padding. */ + SHA256_Pad(ctx, tmp32); + + /* Write the hash. */ + be32enc_vect(digest, ctx->state, 8); +} + +/* Wrapper function for intermediate-values sanitization. */ +void +SHA256_Final(uint8_t digest[32], SHA256_CTX * ctx) +{ + uint32_t tmp32[72]; + + /* Call the real function. */ + _SHA256_Final(digest, ctx, tmp32); + + /* Clear the context state. */ + explicit_bzero(ctx, sizeof(SHA256_CTX)); + + /* Clean the stack. */ + explicit_bzero(tmp32, 288); +} + +/** + * SHA256_Buf(in, len, digest): + * Compute the SHA256 hash of ${len} bytes from ${in} and write it to ${digest}. + */ +void +SHA256_Buf(const void * in, size_t len, uint8_t digest[32]) +{ + SHA256_CTX ctx; + uint32_t tmp32[72]; + + SHA256_Init(&ctx); + _SHA256_Update(&ctx, in, len, tmp32); + _SHA256_Final(digest, &ctx, tmp32); + + /* Clean the stack. */ + explicit_bzero(&ctx, sizeof(SHA256_CTX)); + explicit_bzero(tmp32, 288); +} + +/** + * HMAC_SHA256_Init(ctx, K, Klen): + * Initialize the HMAC-SHA256 context ${ctx} with ${Klen} bytes of key from + * ${K}. + */ +static void +_HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen, + uint32_t tmp32[static restrict 72], uint8_t pad[static restrict 64], + uint8_t khash[static restrict 32]) +{ + const uint8_t * K = _K; + size_t i; + + /* If Klen > 64, the key is really SHA256(K). */ + if (Klen > 64) { + SHA256_Init(&ctx->ictx); + _SHA256_Update(&ctx->ictx, K, Klen, tmp32); + _SHA256_Final(khash, &ctx->ictx, tmp32); + K = khash; + Klen = 32; + } + + /* Inner SHA256 operation is SHA256(K xor [block of 0x36] || data). */ + SHA256_Init(&ctx->ictx); + memset(pad, 0x36, 64); + for (i = 0; i < Klen; i++) + pad[i] ^= K[i]; + _SHA256_Update(&ctx->ictx, pad, 64, tmp32); + + /* Outer SHA256 operation is SHA256(K xor [block of 0x5c] || hash). */ + SHA256_Init(&ctx->octx); + memset(pad, 0x5c, 64); + for (i = 0; i < Klen; i++) + pad[i] ^= K[i]; + _SHA256_Update(&ctx->octx, pad, 64, tmp32); +} + +/* Wrapper function for intermediate-values sanitization. */ +void +HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen) +{ + uint32_t tmp32[72]; + uint8_t pad[64]; + uint8_t khash[32]; + + /* Call the real function. */ + _HMAC_SHA256_Init(ctx, _K, Klen, tmp32, pad, khash); + + /* Clean the stack. */ + explicit_bzero(tmp32, 288); + explicit_bzero(khash, 32); + explicit_bzero(pad, 64); +} + +/** + * HMAC_SHA256_Update(ctx, in, len): + * Input ${len} bytes from ${in} into the HMAC-SHA256 context ${ctx}. + */ +static void +_HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len, + uint32_t tmp32[static restrict 72]) +{ + + /* Feed data to the inner SHA256 operation. */ + _SHA256_Update(&ctx->ictx, in, len, tmp32); +} + +/* Wrapper function for intermediate-values sanitization. */ +void +HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len) +{ + uint32_t tmp32[72]; + + /* Call the real function. */ + _HMAC_SHA256_Update(ctx, in, len, tmp32); + + /* Clean the stack. */ + explicit_bzero(tmp32, 288); +} + +/** + * HMAC_SHA256_Final(digest, ctx): + * Output the HMAC-SHA256 of the data input to the context ${ctx} into the + * buffer ${digest}. + */ +static void +_HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx, + uint32_t tmp32[static restrict 72], uint8_t ihash[static restrict 32]) +{ + + /* Finish the inner SHA256 operation. */ + _SHA256_Final(ihash, &ctx->ictx, tmp32); + + /* Feed the inner hash to the outer SHA256 operation. */ + _SHA256_Update(&ctx->octx, ihash, 32, tmp32); + + /* Finish the outer SHA256 operation. */ + _SHA256_Final(digest, &ctx->octx, tmp32); +} + +/* Wrapper function for intermediate-values sanitization. */ +void +HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx) +{ + uint32_t tmp32[72]; + uint8_t ihash[32]; + + /* Call the real function. */ + _HMAC_SHA256_Final(digest, ctx, tmp32, ihash); + + /* Clear the context state. */ + explicit_bzero(ctx, sizeof(HMAC_SHA256_CTX)); + + /* Clean the stack. */ + explicit_bzero(tmp32, 288); + explicit_bzero(ihash, 32); +} + +/** + * HMAC_SHA256_Buf(K, Klen, in, len, digest): + * Compute the HMAC-SHA256 of ${len} bytes from ${in} using the key ${K} of + * length ${Klen}, and write the result to ${digest}. + */ +void +HMAC_SHA256_Buf(const void * K, size_t Klen, const void * in, size_t len, + uint8_t digest[32]) +{ + HMAC_SHA256_CTX ctx; + uint32_t tmp32[72]; + uint8_t tmp8[96]; + + _HMAC_SHA256_Init(&ctx, K, Klen, tmp32, &tmp8[0], &tmp8[64]); + _HMAC_SHA256_Update(&ctx, in, len, tmp32); + _HMAC_SHA256_Final(digest, &ctx, tmp32, &tmp8[0]); + + /* Clean the stack. */ + explicit_bzero(&ctx, sizeof(HMAC_SHA256_CTX)); + explicit_bzero(tmp32, 288); + explicit_bzero(tmp8, 96); +} + +/* Add padding and terminating bit-count, but don't invoke Transform yet. */ +static int +SHA256_Pad_Almost(SHA256_CTX * ctx, uint8_t len[static restrict 8], + uint32_t tmp32[static restrict 72]) +{ + uint32_t r; + + r = (ctx->count >> 3) & 0x3f; + if (r >= 56) + return -1; + + /* + * Convert length to a vector of bytes -- we do this now rather + * than later because the length will change after we pad. + */ + be64enc(len, ctx->count); + + /* Add 1--56 bytes so that the resulting length is 56 mod 64. */ + _SHA256_Update(ctx, PAD, 56 - r, tmp32); + + /* Add the terminating bit-count. */ + ctx->buf[63] = len[7]; + _SHA256_Update(ctx, len, 7, tmp32); + + return 0; +} + +/** + * PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, c, buf, dkLen): + * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and + * write the output to buf. The value dkLen must be at most 32 * (2^32 - 1). + */ +void +PBKDF2_SHA256(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, + size_t saltlen, uint64_t c, uint8_t * buf, size_t dkLen) +{ + HMAC_SHA256_CTX Phctx, PShctx, hctx; + uint32_t tmp32[72]; + union { + uint8_t tmp8[96]; + uint32_t state[8]; + } u; + size_t i; + uint8_t ivec[4]; + uint8_t U[32]; + uint8_t T[32]; + uint64_t j; + int k; + size_t clen; + + /* Sanity-check. */ + assert(dkLen <= 32 * (size_t)(UINT32_MAX)); + + if (c == 1 && (dkLen & 31) == 0 && (saltlen & 63) <= 51) { + uint32_t oldcount; + uint8_t * ivecp; + + /* Compute HMAC state after processing P and S. */ + _HMAC_SHA256_Init(&hctx, passwd, passwdlen, + tmp32, &u.tmp8[0], &u.tmp8[64]); + _HMAC_SHA256_Update(&hctx, salt, saltlen, tmp32); + + /* Prepare ictx padding. */ + oldcount = hctx.ictx.count & (0x3f << 3); + _HMAC_SHA256_Update(&hctx, "\0\0\0", 4, tmp32); + if ((hctx.ictx.count & (0x3f << 3)) < oldcount || + SHA256_Pad_Almost(&hctx.ictx, u.tmp8, tmp32)) + goto generic; /* Can't happen due to saltlen check */ + ivecp = hctx.ictx.buf + (oldcount >> 3); + + /* Prepare octx padding. */ + hctx.octx.count += 32 << 3; + SHA256_Pad_Almost(&hctx.octx, u.tmp8, tmp32); + + /* Iterate through the blocks. */ + for (i = 0; i * 32 < dkLen; i++) { + /* Generate INT(i + 1). */ + be32enc(ivecp, (uint32_t)(i + 1)); + + /* Compute U_1 = PRF(P, S || INT(i)). */ + memcpy(u.state, hctx.ictx.state, sizeof(u.state)); + SHA256_Transform(u.state, hctx.ictx.buf, + &tmp32[0], &tmp32[64]); + be32enc_vect(hctx.octx.buf, u.state, 8); + memcpy(u.state, hctx.octx.state, sizeof(u.state)); + SHA256_Transform(u.state, hctx.octx.buf, + &tmp32[0], &tmp32[64]); + be32enc_vect(&buf[i * 32], u.state, 8); + } + + goto cleanup; + } + +generic: + /* Compute HMAC state after processing P. */ + _HMAC_SHA256_Init(&Phctx, passwd, passwdlen, + tmp32, &u.tmp8[0], &u.tmp8[64]); + + /* Compute HMAC state after processing P and S. */ + memcpy(&PShctx, &Phctx, sizeof(HMAC_SHA256_CTX)); + _HMAC_SHA256_Update(&PShctx, salt, saltlen, tmp32); + + /* Iterate through the blocks. */ + for (i = 0; i * 32 < dkLen; i++) { + /* Generate INT(i + 1). */ + be32enc(ivec, (uint32_t)(i + 1)); + + /* Compute U_1 = PRF(P, S || INT(i)). */ + memcpy(&hctx, &PShctx, sizeof(HMAC_SHA256_CTX)); + _HMAC_SHA256_Update(&hctx, ivec, 4, tmp32); + _HMAC_SHA256_Final(T, &hctx, tmp32, u.tmp8); + + if (c > 1) { + /* T_i = U_1 ... */ + memcpy(U, T, 32); + + for (j = 2; j <= c; j++) { + /* Compute U_j. */ + memcpy(&hctx, &Phctx, sizeof(HMAC_SHA256_CTX)); + _HMAC_SHA256_Update(&hctx, U, 32, tmp32); + _HMAC_SHA256_Final(U, &hctx, tmp32, u.tmp8); + + /* ... xor U_j ... */ + for (k = 0; k < 32; k++) + T[k] ^= U[k]; + } + } + + /* Copy as many bytes as necessary into buf. */ + clen = dkLen - i * 32; + if (clen > 32) + clen = 32; + memcpy(&buf[i * 32], T, clen); + } + + /* Clean the stack. */ + explicit_bzero(&Phctx, sizeof(HMAC_SHA256_CTX)); + explicit_bzero(&PShctx, sizeof(HMAC_SHA256_CTX)); + explicit_bzero(U, 32); + explicit_bzero(T, 32); + +cleanup: + explicit_bzero(&hctx, sizeof(HMAC_SHA256_CTX)); + explicit_bzero(tmp32, 288); + explicit_bzero(&u, sizeof(u)); +} diff --git a/libbb/yescrypt/alg-sha256.h b/libbb/yescrypt/alg-sha256.h new file mode 100644 index 0000000000..1e75307d3b --- /dev/null +++ b/libbb/yescrypt/alg-sha256.h @@ -0,0 +1,113 @@ +/*- + * Copyright 2005-2016 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Use #defines in order to avoid namespace collisions with anyone else's + * SHA256 code (e.g., the code in OpenSSL). + */ +#define SHA256_Init libcperciva_SHA256_Init +#define SHA256_Update libcperciva_SHA256_Update +#define SHA256_Final libcperciva_SHA256_Final +#define SHA256_Buf libcperciva_SHA256_Buf +#define SHA256_CTX libcperciva_SHA256_CTX +#define HMAC_SHA256_Init libcperciva_HMAC_SHA256_Init +#define HMAC_SHA256_Update libcperciva_HMAC_SHA256_Update +#define HMAC_SHA256_Final libcperciva_HMAC_SHA256_Final +#define HMAC_SHA256_Buf libcperciva_HMAC_SHA256_Buf +#define HMAC_SHA256_CTX libcperciva_HMAC_SHA256_CTX + +/* Context structure for SHA256 operations. */ +typedef struct { + uint32_t state[8]; + uint64_t count; + uint8_t buf[64]; +} SHA256_CTX; + +/** + * SHA256_Init(ctx): + * Initialize the SHA256 context ${ctx}. + */ +extern void SHA256_Init(SHA256_CTX *); + +/** + * SHA256_Update(ctx, in, len): + * Input ${len} bytes from ${in} into the SHA256 context ${ctx}. + */ +extern void SHA256_Update(SHA256_CTX *, const void *, size_t); + +/** + * SHA256_Final(digest, ctx): + * Output the SHA256 hash of the data input to the context ${ctx} into the + * buffer ${digest}. + */ +extern void SHA256_Final(uint8_t[32], SHA256_CTX *); + +/** + * SHA256_Buf(in, len, digest): + * Compute the SHA256 hash of ${len} bytes from ${in} and write it to ${digest}. + */ +extern void SHA256_Buf(const void *, size_t, uint8_t[32]); + +/* Context structure for HMAC-SHA256 operations. */ +typedef struct { + SHA256_CTX ictx; + SHA256_CTX octx; +} HMAC_SHA256_CTX; + +/** + * HMAC_SHA256_Init(ctx, K, Klen): + * Initialize the HMAC-SHA256 context ${ctx} with ${Klen} bytes of key from + * ${K}. + */ +extern void HMAC_SHA256_Init(HMAC_SHA256_CTX *, const void *, size_t); + +/** + * HMAC_SHA256_Update(ctx, in, len): + * Input ${len} bytes from ${in} into the HMAC-SHA256 context ${ctx}. + */ +extern void HMAC_SHA256_Update(HMAC_SHA256_CTX *, const void *, size_t); + +/** + * HMAC_SHA256_Final(digest, ctx): + * Output the HMAC-SHA256 of the data input to the context ${ctx} into the + * buffer ${digest}. + */ +extern void HMAC_SHA256_Final(uint8_t[32], HMAC_SHA256_CTX *); + +/** + * HMAC_SHA256_Buf(K, Klen, in, len, digest): + * Compute the HMAC-SHA256 of ${len} bytes from ${in} using the key ${K} of + * length ${Klen}, and write the result to ${digest}. + */ +extern void HMAC_SHA256_Buf(const void *, size_t, const void *, size_t, uint8_t[32]); + +/** + * PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, c, buf, dkLen): + * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and + * write the output to buf. The value dkLen must be at most 32 * (2^32 - 1). + */ +extern void PBKDF2_SHA256(const uint8_t *, size_t, const uint8_t *, size_t, + uint64_t, uint8_t *, size_t); diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c new file mode 100644 index 0000000000..75b59d1cf4 --- /dev/null +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -0,0 +1,714 @@ +/*- + * Copyright 2013-2018 Alexander Peslyak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if 0 //UNUSED +static uint8_t *encode64_uint32(uint8_t *dst, size_t dstlen, + uint32_t src, uint32_t min) +{ + uint32_t start = 0, end = 47, chars = 1, bits = 0; + + if (src < min) + return NULL; + src -= min; + + do { + uint32_t count = (end + 1 - start) << bits; + if (src < count) + break; + if (start >= 63) + return NULL; + start = end + 1; + end = start + (62 - end) / 2; + src -= count; + chars++; + bits += 6; + } while (1); + + if (dstlen <= chars) /* require room for a NUL terminator */ + return NULL; + + *dst++ = itoa64[start + (src >> bits)]; + + while (--chars) { + bits -= 6; + *dst++ = itoa64[(src >> bits) & 0x3f]; + } + + *dst = 0; /* NUL terminate just in case */ + + return dst; +} +#endif //UNUSED + +static inline uint32_t atoi64(uint8_t src) +{ + static const uint8_t atoi64_partial[77] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 64, 64, 64, 64, 64, 64, 64, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + 64, 64, 64, 64, 64, 64, + 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 + }; + if (src >= '.' && src <= 'z') + return atoi64_partial[src - '.']; + return 64; +} + +static const uint8_t *decode64_uint32(uint32_t *dst, + const uint8_t *src, uint32_t min) +{ + uint32_t start = 0, end = 47, chars = 1, bits = 0; + uint32_t c; + + c = atoi64(*src++); + if (c > 63) + goto fail; + + *dst = min; + while (c > end) { + *dst += (end + 1 - start) << bits; + start = end + 1; + end = start + (62 - end) / 2; + chars++; + bits += 6; + } + + *dst += (c - start) << bits; + + while (--chars) { + c = atoi64(*src++); + if (c > 63) + goto fail; + bits -= 6; + *dst += c << bits; + } + + return src; + +fail: + *dst = 0; + return NULL; +} + +static uint8_t *encode64_uint32_fixed(uint8_t *dst, size_t dstlen, + uint32_t src, uint32_t srcbits) +{ + uint32_t bits; + + for (bits = 0; bits < srcbits; bits += 6) { + if (dstlen < 2) + return NULL; + *dst++ = itoa64[src & 0x3f]; + dstlen--; + src >>= 6; + } + + if (src || dstlen < 1) + return NULL; + + *dst = 0; /* NUL terminate just in case */ + + return dst; +} + +uint8_t *encode64(uint8_t *dst, size_t dstlen, + const uint8_t *src, size_t srclen) +{ + size_t i; + + for (i = 0; i < srclen; ) { + uint8_t *dnext; + uint32_t value = 0, bits = 0; + do { + value |= (uint32_t)src[i++] << bits; + bits += 8; + } while (bits < 24 && i < srclen); + dnext = encode64_uint32_fixed(dst, dstlen, value, bits); + if (!dnext) + return NULL; + dstlen -= dnext - dst; + dst = dnext; + } + + if (dstlen < 1) + return NULL; + + *dst = 0; /* NUL terminate just in case */ + + return dst; +} + +static const uint8_t *decode64_uint32_fixed(uint32_t *dst, uint32_t dstbits, + const uint8_t *src) +{ + uint32_t bits; + + *dst = 0; + for (bits = 0; bits < dstbits; bits += 6) { + uint32_t c = atoi64(*src++); + if (c > 63) { + *dst = 0; + return NULL; + } + *dst |= c << bits; + } + + return src; +} + +const uint8_t *decode64(uint8_t *dst, size_t *dstlen, + const uint8_t *src, size_t srclen) +{ + size_t dstpos = 0; + + while (dstpos <= *dstlen && srclen) { + uint32_t value = 0, bits = 0; + while (srclen--) { + uint32_t c = atoi64(*src); + if (c > 63) { + srclen = 0; + break; + } + src++; + value |= c << bits; + bits += 6; + if (bits >= 24) + break; + } + if (!bits) + break; + if (bits < 12) /* must have at least one full byte */ + goto fail; + while (dstpos++ < *dstlen) { + *dst++ = value; + value >>= 8; + bits -= 8; + if (bits < 8) { /* 2 or 4 */ + if (value) /* must be 0 */ + goto fail; + bits = 0; + break; + } + } + if (bits) + goto fail; + } + + if (!srclen && dstpos <= *dstlen) { + *dstlen = dstpos; + return src; + } + +fail: + *dstlen = 0; + return NULL; +} + +#if 0 //UNUSED //KEY: +typedef enum { ENC = 1, DEC = -1 } encrypt_dir_t; + +static void memxor(unsigned char *dst, unsigned char *src, size_t size) +{ + while (size--) + *dst++ ^= *src++; +} + +static void yescrypt_sha256_cipher(unsigned char *data, size_t datalen, + const yescrypt_binary_t *key, encrypt_dir_t dir) +{ + SHA256_CTX ctx; + unsigned char f[32 + 4]; + size_t halflen, which; + unsigned char mask, round, target; + + if (!datalen) + return; + if (datalen > 64) + datalen = 64; + + halflen = datalen >> 1; + + which = 0; /* offset to half we are working on (0 or halflen) */ + mask = 0x0f; /* current half's extra nibble mask if datalen is odd */ + + round = 0; + target = 5; /* 6 rounds due to Jacques Patarin's CRYPTO 2004 paper */ + + if (dir == DEC) { + which = halflen; /* even round count, so swap the halves */ + mask ^= 0xff; + + round = target; + target = 0; + } + + f[32] = 0; + f[33] = sizeof(*key); + f[34] = datalen; + + do { + SHA256_Init(&ctx); + f[35] = round; + SHA256_Update(&ctx, &f[32], 4); + SHA256_Update(&ctx, key, sizeof(*key)); + SHA256_Update(&ctx, &data[which], halflen); + if (datalen & 1) { + f[0] = data[datalen - 1] & mask; + SHA256_Update(&ctx, f, 1); + } + SHA256_Final(f, &ctx); + which ^= halflen; + memxor(&data[which], f, halflen); + if (datalen & 1) { + mask ^= 0xff; + data[datalen - 1] ^= f[halflen] & mask; + } + if (round == target) + break; + round += dir; + } while (1); + + /* ctx is presumably zeroized by SHA256_Final() */ + explicit_bzero(f, sizeof(f)); +} +#endif //UNUSED //KEY: + +uint8_t *yescrypt_r( + const yescrypt_shared_t *shared, + yescrypt_local_t *local, + const uint8_t *passwd, size_t passwdlen, + const uint8_t *setting, + //KEY const yescrypt_binary_t *key, + uint8_t *buf, size_t buflen) +{ + unsigned char saltbin[64], hashbin[32]; + const uint8_t *src, *saltstr, *salt; + uint8_t *dst; + size_t need, prefixlen, saltstrlen, saltlen; + yescrypt_params_t params = { .p = 1 }; + + if (setting[0] != '$' || + (setting[1] != '7' && setting[1] != 'y') || + setting[2] != '$') + return NULL; + src = setting + 3; + + if (setting[1] == '7') { + uint32_t N_log2 = atoi64(*src++); + if (N_log2 < 1 || N_log2 > 63) + return NULL; + params.N = (uint64_t)1 << N_log2; + + src = decode64_uint32_fixed(¶ms.r, 30, src); + if (!src) + return NULL; + + src = decode64_uint32_fixed(¶ms.p, 30, src); + if (!src) + return NULL; + + //KEY:if (key) + //KEY: return NULL; + } else { + uint32_t flavor, N_log2; + + src = decode64_uint32(&flavor, src, 0); + if (!src) + return NULL; + + if (flavor < YESCRYPT_RW) { + params.flags = flavor; + } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { + params.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); + } else { + return NULL; + } + + src = decode64_uint32(&N_log2, src, 1); + if (!src || N_log2 > 63) + return NULL; + params.N = (uint64_t)1 << N_log2; + + src = decode64_uint32(¶ms.r, src, 1); + if (!src) + return NULL; + + if (*src != '$') { + uint32_t have; + + src = decode64_uint32(&have, src, 1); + if (!src) + return NULL; + + if (have & 1) { + src = decode64_uint32(¶ms.p, src, 2); + if (!src) + return NULL; + } + + if (have & 2) { + src = decode64_uint32(¶ms.t, src, 1); + if (!src) + return NULL; + } + + if (have & 4) { + src = decode64_uint32(¶ms.g, src, 1); + if (!src) + return NULL; + } + + if (have & 8) { + uint32_t NROM_log2; + src = decode64_uint32(&NROM_log2, src, 1); + if (!src || NROM_log2 > 63) + return NULL; + params.NROM = (uint64_t)1 << NROM_log2; + } + } + + if (*src++ != '$') + return NULL; + } + + prefixlen = src - setting; + + saltstr = src; + src = (uint8_t *)strrchr((char *)saltstr, '$'); + if (src) + saltstrlen = src - saltstr; + else + saltstrlen = strlen((char *)saltstr); + + if (setting[1] == '7') { + salt = saltstr; + saltlen = saltstrlen; + } else { + const uint8_t *saltend; + + saltlen = sizeof(saltbin); + saltend = decode64(saltbin, &saltlen, saltstr, saltstrlen); + + if (!saltend || (size_t)(saltend - saltstr) != saltstrlen) + goto fail; + + salt = saltbin; + + //KEY:if (key) + //KEY: yescrypt_sha256_cipher(saltbin, saltlen, key, ENC); + } + + need = prefixlen + saltstrlen + 1 + HASH_LEN + 1; + if (need > buflen || need < saltstrlen) + goto fail; + + if (yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, + ¶ms, hashbin, sizeof(hashbin))) + goto fail; + + //KEY:if (key) { + //KEY: explicit_bzero(saltbin, sizeof(saltbin)); + //KEY: yescrypt_sha256_cipher(hashbin, sizeof(hashbin), key, ENC); + //KEY:} + + dst = buf; + memcpy(dst, setting, prefixlen + saltstrlen); + dst += prefixlen + saltstrlen; + *dst++ = '$'; + + dst = encode64(dst, buflen - (dst - buf), hashbin, sizeof(hashbin)); + explicit_bzero(hashbin, sizeof(hashbin)); + if (!dst || dst >= buf + buflen) + return NULL; + + *dst = 0; /* NUL termination */ + + return buf; + +fail: + explicit_bzero(saltbin, sizeof(saltbin)); + explicit_bzero(hashbin, sizeof(hashbin)); + return NULL; +} + + + + + + + + + +int yescrypt_free_shared(yescrypt_shared_t *shared) +{ + return free_region(shared); +} + +int yescrypt_init_local(yescrypt_local_t *local) +{ + init_region(local); + return 0; +} + +int yescrypt_free_local(yescrypt_local_t *local) +{ + return free_region(local); +} + +#if 0 //UNUSED +uint8_t *yescrypt(const uint8_t *passwd, const uint8_t *setting) +{ + /* prefix, '$', hash, NUL */ + static uint8_t buf[PREFIX_LEN + 1 + HASH_LEN + 1]; + yescrypt_local_t local; + uint8_t *retval; + + if (yescrypt_init_local(&local)) + return NULL; + retval = yescrypt_r( + /* const yescrypt_shared_t *shared */ NULL, + &local, + passwd, strlen((char *)passwd), + setting, + //KEY:/* const yescrypt_binary_t *key */ NULL, + buf, sizeof(buf)); + if (yescrypt_free_local(&local)) + return NULL; + return retval; +} + +uint8_t *yescrypt_reencrypt(uint8_t *hash, + const yescrypt_binary_t *from_key, + const yescrypt_binary_t *to_key) +{ + uint8_t *retval = NULL, *saltstart, *hashstart; + const uint8_t *hashend; + unsigned char saltbin[64], hashbin[32]; + size_t saltstrlen, saltlen = 0, hashlen; + + if (strncmp((char *)hash, "$y$", 3)) + return NULL; + + saltstart = NULL; + hashstart = (uint8_t *)strrchr((char *)hash, '$'); + if (hashstart) { + if (hashstart > (uint8_t *)hash) { + saltstart = hashstart - 1; + while (*saltstart != '$' && saltstart > hash) + saltstart--; + if (*saltstart == '$') + saltstart++; + } + hashstart++; + } else { + hashstart = hash; + } + saltstrlen = saltstart ? (hashstart - 1 - saltstart) : 0; + if (saltstrlen > BYTES2CHARS(64) || + strlen((char *)hashstart) != HASH_LEN) + return NULL; + + if (saltstrlen) { + const uint8_t *saltend; + saltlen = sizeof(saltbin); + saltend = decode64(saltbin, &saltlen, saltstart, saltstrlen); + if (!saltend || *saltend != '$' || saltlen < 1 || saltlen > 64) + goto out; + + if (from_key) + yescrypt_sha256_cipher(saltbin, saltlen, from_key, ENC); + if (to_key) + yescrypt_sha256_cipher(saltbin, saltlen, to_key, DEC); + } + + hashlen = sizeof(hashbin); + hashend = decode64(hashbin, &hashlen, hashstart, HASH_LEN); + if (!hashend || *hashend || hashlen != sizeof(hashbin)) + goto out; + + if (from_key) + yescrypt_sha256_cipher(hashbin, hashlen, from_key, DEC); + if (to_key) + yescrypt_sha256_cipher(hashbin, hashlen, to_key, ENC); + + if (saltstrlen) { + if (!encode64(saltstart, saltstrlen + 1, saltbin, saltlen)) + goto out; /* can't happen */ + *(saltstart + saltstrlen) = '$'; + } + + if (!encode64(hashstart, HASH_LEN + 1, hashbin, hashlen)) + goto out; /* can't happen */ + + retval = hash; + +out: + explicit_bzero(saltbin, sizeof(saltbin)); + explicit_bzero(hashbin, sizeof(hashbin)); + + return retval; +} + +static uint32_t N2log2(uint64_t N) +{ + uint32_t N_log2; + + if (N < 2) + return 0; + + N_log2 = 2; + while (N >> N_log2 != 0) + N_log2++; + N_log2--; + + if (N >> N_log2 != 1) + return 0; + + return N_log2; +} + +uint8_t *yescrypt_encode_params_r(const yescrypt_params_t *params, + const uint8_t *src, size_t srclen, + uint8_t *buf, size_t buflen) +{ + uint32_t flavor, N_log2, NROM_log2, have; + uint8_t *dst; + + if (srclen > SIZE_MAX / 16) + return NULL; + + if (params->flags < YESCRYPT_RW) { + flavor = params->flags; + } else if ((params->flags & YESCRYPT_MODE_MASK) == YESCRYPT_RW && + params->flags <= (YESCRYPT_RW | YESCRYPT_RW_FLAVOR_MASK)) { + flavor = YESCRYPT_RW + (params->flags >> 2); + } else { + return NULL; + } + + N_log2 = N2log2(params->N); + if (!N_log2) + return NULL; + + NROM_log2 = N2log2(params->NROM); + if (params->NROM && !NROM_log2) + return NULL; + + if ((uint64_t)params->r * (uint64_t)params->p >= (1U << 30)) + return NULL; + + dst = buf; + *dst++ = '$'; + *dst++ = 'y'; + *dst++ = '$'; + + dst = encode64_uint32(dst, buflen - (dst - buf), flavor, 0); + if (!dst) + return NULL; + + dst = encode64_uint32(dst, buflen - (dst - buf), N_log2, 1); + if (!dst) + return NULL; + + dst = encode64_uint32(dst, buflen - (dst - buf), params->r, 1); + if (!dst) + return NULL; + + have = 0; + if (params->p != 1) + have |= 1; + if (params->t) + have |= 2; + if (params->g) + have |= 4; + if (NROM_log2) + have |= 8; + + if (have) { + dst = encode64_uint32(dst, buflen - (dst - buf), have, 1); + if (!dst) + return NULL; + } + + if (params->p != 1) { + dst = encode64_uint32(dst, buflen - (dst - buf), params->p, 2); + if (!dst) + return NULL; + } + + if (params->t) { + dst = encode64_uint32(dst, buflen - (dst - buf), params->t, 1); + if (!dst) + return NULL; + } + + if (params->g) { + dst = encode64_uint32(dst, buflen - (dst - buf), params->g, 1); + if (!dst) + return NULL; + } + + if (NROM_log2) { + dst = encode64_uint32(dst, buflen - (dst - buf), NROM_log2, 1); + if (!dst) + return NULL; + } + + if (dst >= buf + buflen) + return NULL; + + *dst++ = '$'; + + dst = encode64(dst, buflen - (dst - buf), src, srclen); + if (!dst || dst >= buf + buflen) + return NULL; + + *dst = 0; /* NUL termination */ + + return buf; +} + +uint8_t *yescrypt_encode_params(const yescrypt_params_t *params, + const uint8_t *src, size_t srclen) +{ + /* prefix, NUL */ + static uint8_t buf[PREFIX_LEN + 1]; + return yescrypt_encode_params_r(params, src, srclen, buf, sizeof(buf)); +} + +int crypto_scrypt(const uint8_t *passwd, size_t passwdlen, + const uint8_t *salt, size_t saltlen, uint64_t N, uint32_t r, uint32_t p, + uint8_t *buf, size_t buflen) +{ + yescrypt_local_t local; + yescrypt_params_t params = { .flags = 0, .N = N, .r = r, .p = p }; + int retval; + + if (yescrypt_init_local(&local)) + return -1; + retval = yescrypt_kdf(NULL, &local, + passwd, passwdlen, salt, saltlen, ¶ms, buf, buflen); + if (yescrypt_free_local(&local)) + return -1; + return retval; +} +#endif //UNUSED diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c new file mode 100644 index 0000000000..3e98ffb0a0 --- /dev/null +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -0,0 +1,1070 @@ +/*- + * Copyright 2009 Colin Percival + * Copyright 2012-2018 Alexander Peslyak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file was originally written by Colin Percival as part of the Tarsnap + * online backup system. + */ + +#if __STDC_VERSION__ >= 199901L +/* Have restrict */ +#elif defined(__GNUC__) +#define restrict __restrict +#else +#define restrict +#endif + +#ifdef __GNUC__ +#define unlikely(exp) __builtin_expect(exp, 0) +#else +#define unlikely(exp) (exp) +#endif + +#if 0 //def __SSE__ +... +#else /* !defined(__SSE2__) */ + +typedef union { + uint32_t w[16]; + uint64_t d[8]; +} salsa20_blk_t; + +static inline void salsa20_simd_shuffle(const salsa20_blk_t *Bin, + salsa20_blk_t *Bout) +{ +#define COMBINE(out, in1, in2) \ + Bout->d[out] = Bin->w[in1 * 2] | ((uint64_t)Bin->w[in2 * 2 + 1] << 32); + COMBINE(0, 0, 2) + COMBINE(1, 5, 7) + COMBINE(2, 2, 4) + COMBINE(3, 7, 1) + COMBINE(4, 4, 6) + COMBINE(5, 1, 3) + COMBINE(6, 6, 0) + COMBINE(7, 3, 5) +#undef COMBINE +} + +static inline void salsa20_simd_unshuffle(const salsa20_blk_t *Bin, + salsa20_blk_t *Bout) +{ +#define UNCOMBINE(out, in1, in2) \ + Bout->w[out * 2] = Bin->d[in1]; \ + Bout->w[out * 2 + 1] = Bin->d[in2] >> 32; + UNCOMBINE(0, 0, 6) + UNCOMBINE(1, 5, 3) + UNCOMBINE(2, 2, 0) + UNCOMBINE(3, 7, 5) + UNCOMBINE(4, 4, 2) + UNCOMBINE(5, 1, 7) + UNCOMBINE(6, 6, 4) + UNCOMBINE(7, 3, 1) +#undef UNCOMBINE +} +#define DECL_X \ + salsa20_blk_t X; +#define DECL_Y \ + salsa20_blk_t Y; + +#define COPY(out, in) \ + (out).d[0] = (in).d[0]; \ + (out).d[1] = (in).d[1]; \ + (out).d[2] = (in).d[2]; \ + (out).d[3] = (in).d[3]; \ + (out).d[4] = (in).d[4]; \ + (out).d[5] = (in).d[5]; \ + (out).d[6] = (in).d[6]; \ + (out).d[7] = (in).d[7]; + +#define READ_X(in) COPY(X, in) +#define WRITE_X(out) COPY(out, X) + +/** + * salsa20(B): + * Apply the Salsa20 core to the provided block. + */ +static inline void salsa20(salsa20_blk_t *restrict B, + salsa20_blk_t *restrict Bout, uint32_t doublerounds) +{ + salsa20_blk_t X; +#define x X.w + + salsa20_simd_unshuffle(B, &X); + + do { +#define R(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) + /* Operate on columns */ + x[ 4] ^= R(x[ 0]+x[12], 7); x[ 8] ^= R(x[ 4]+x[ 0], 9); + x[12] ^= R(x[ 8]+x[ 4],13); x[ 0] ^= R(x[12]+x[ 8],18); + + x[ 9] ^= R(x[ 5]+x[ 1], 7); x[13] ^= R(x[ 9]+x[ 5], 9); + x[ 1] ^= R(x[13]+x[ 9],13); x[ 5] ^= R(x[ 1]+x[13],18); + + x[14] ^= R(x[10]+x[ 6], 7); x[ 2] ^= R(x[14]+x[10], 9); + x[ 6] ^= R(x[ 2]+x[14],13); x[10] ^= R(x[ 6]+x[ 2],18); + + x[ 3] ^= R(x[15]+x[11], 7); x[ 7] ^= R(x[ 3]+x[15], 9); + x[11] ^= R(x[ 7]+x[ 3],13); x[15] ^= R(x[11]+x[ 7],18); + + /* Operate on rows */ + x[ 1] ^= R(x[ 0]+x[ 3], 7); x[ 2] ^= R(x[ 1]+x[ 0], 9); + x[ 3] ^= R(x[ 2]+x[ 1],13); x[ 0] ^= R(x[ 3]+x[ 2],18); + + x[ 6] ^= R(x[ 5]+x[ 4], 7); x[ 7] ^= R(x[ 6]+x[ 5], 9); + x[ 4] ^= R(x[ 7]+x[ 6],13); x[ 5] ^= R(x[ 4]+x[ 7],18); + + x[11] ^= R(x[10]+x[ 9], 7); x[ 8] ^= R(x[11]+x[10], 9); + x[ 9] ^= R(x[ 8]+x[11],13); x[10] ^= R(x[ 9]+x[ 8],18); + + x[12] ^= R(x[15]+x[14], 7); x[13] ^= R(x[12]+x[15], 9); + x[14] ^= R(x[13]+x[12],13); x[15] ^= R(x[14]+x[13],18); +#undef R + } while (--doublerounds); +#undef x + + { + uint32_t i; + salsa20_simd_shuffle(&X, Bout); + for (i = 0; i < 16; i += 4) { + B->w[i] = Bout->w[i] += B->w[i]; + B->w[i + 1] = Bout->w[i + 1] += B->w[i + 1]; + B->w[i + 2] = Bout->w[i + 2] += B->w[i + 2]; + B->w[i + 3] = Bout->w[i + 3] += B->w[i + 3]; + } + } + +#if 0 + /* Too expensive */ + explicit_bzero(&X, sizeof(X)); +#endif +} + +/** + * Apply the Salsa20/2 core to the block provided in X. + */ +#define SALSA20_2(out) \ + salsa20(&X, &out, 1); + +#define XOR(out, in1, in2) \ + (out).d[0] = (in1).d[0] ^ (in2).d[0]; \ + (out).d[1] = (in1).d[1] ^ (in2).d[1]; \ + (out).d[2] = (in1).d[2] ^ (in2).d[2]; \ + (out).d[3] = (in1).d[3] ^ (in2).d[3]; \ + (out).d[4] = (in1).d[4] ^ (in2).d[4]; \ + (out).d[5] = (in1).d[5] ^ (in2).d[5]; \ + (out).d[6] = (in1).d[6] ^ (in2).d[6]; \ + (out).d[7] = (in1).d[7] ^ (in2).d[7]; + +#define XOR_X(in) XOR(X, X, in) +#define XOR_X_2(in1, in2) XOR(X, in1, in2) +#define XOR_X_WRITE_XOR_Y_2(out, in) \ + XOR(Y, out, in) \ + COPY(out, Y) \ + XOR(X, X, Y) + +/** + * Apply the Salsa20/8 core to the block provided in X ^ in. + */ +#define SALSA20_8_XOR_MEM(in, out) \ + XOR_X(in); \ + salsa20(&X, &out, 4); + +#define INTEGERIFY (uint32_t)X.d[0] + +#endif /* !defined(__SSE2__) */ + +/** + * blockmix_salsa8(Bin, Bout, r): + * Compute Bout = BlockMix_{salsa20/8, r}(Bin). The input Bin must be 128r + * bytes in length; the output Bout must also be the same size. + */ +static void blockmix_salsa8(const salsa20_blk_t *restrict Bin, + salsa20_blk_t *restrict Bout, size_t r) +{ + size_t i; + DECL_X + + READ_X(Bin[r * 2 - 1]) + for (i = 0; i < r; i++) { + SALSA20_8_XOR_MEM(Bin[i * 2], Bout[i]) + SALSA20_8_XOR_MEM(Bin[i * 2 + 1], Bout[r + i]) + } +} + +static uint32_t blockmix_salsa8_xor(const salsa20_blk_t *restrict Bin1, + const salsa20_blk_t *restrict Bin2, salsa20_blk_t *restrict Bout, + size_t r) +{ + size_t i; + DECL_X + +#ifdef PREFETCH + PREFETCH(&Bin2[r * 2 - 1], _MM_HINT_T0) + for (i = 0; i < r - 1; i++) { + PREFETCH(&Bin2[i * 2], _MM_HINT_T0) + PREFETCH(&Bin2[i * 2 + 1], _MM_HINT_T0) + } + PREFETCH(&Bin2[i * 2], _MM_HINT_T0) +#endif + + XOR_X_2(Bin1[r * 2 - 1], Bin2[r * 2 - 1]) + for (i = 0; i < r; i++) { + XOR_X(Bin1[i * 2]) + SALSA20_8_XOR_MEM(Bin2[i * 2], Bout[i]) + XOR_X(Bin1[i * 2 + 1]) + SALSA20_8_XOR_MEM(Bin2[i * 2 + 1], Bout[r + i]) + } + + return INTEGERIFY; +} + +/* This is tunable */ +#define Swidth 8 + +/* Not tunable in this implementation, hard-coded in a few places */ +#define PWXsimple 2 +#define PWXgather 4 + +/* Derived values. Not tunable except via Swidth above. */ +#define PWXbytes (PWXgather * PWXsimple * 8) +#define Sbytes (3 * (1 << Swidth) * PWXsimple * 8) +#define Smask (((1 << Swidth) - 1) * PWXsimple * 8) +#define Smask2 (((uint64_t)Smask << 32) | Smask) + +#define DECL_SMASK2REG /* empty */ +#define FORCE_REGALLOC_3 /* empty */ +#define MAYBE_MEMORY_BARRIER /* empty */ + +#if 0 //def __SSE2__ +... +#else /* !defined(__SSE2__) */ + +#define PWXFORM_SIMD(x0, x1) { \ + uint64_t x = x0 & Smask2; \ + uint64_t *p0 = (uint64_t *)(S0 + (uint32_t)x); \ + uint64_t *p1 = (uint64_t *)(S1 + (x >> 32)); \ + x0 = ((x0 >> 32) * (uint32_t)x0 + p0[0]) ^ p1[0]; \ + x1 = ((x1 >> 32) * (uint32_t)x1 + p0[1]) ^ p1[1]; \ +} + +#define PWXFORM_ROUND \ + PWXFORM_SIMD(X.d[0], X.d[1]) \ + PWXFORM_SIMD(X.d[2], X.d[3]) \ + PWXFORM_SIMD(X.d[4], X.d[5]) \ + PWXFORM_SIMD(X.d[6], X.d[7]) +#endif + +/* + * This offset helps address the 256-byte write block via the single-byte + * displacements encodable in x86(-64) instructions. It is needed because the + * displacements are signed. Without it, we'd get 4-byte displacements for + * half of the writes. Setting it to 0x80 instead of 0x7c would avoid needing + * a displacement for one of the writes, but then the LEA instruction would + * need a 4-byte displacement. + */ +#define PWXFORM_WRITE_OFFSET 0x7c + +#define PWXFORM_WRITE \ + WRITE_X(*(salsa20_blk_t *)(Sw - PWXFORM_WRITE_OFFSET)) \ + Sw += 64; + +#define PWXFORM { \ + uint8_t *Sw = S2 + w + PWXFORM_WRITE_OFFSET; \ + FORCE_REGALLOC_3 \ + MAYBE_MEMORY_BARRIER \ + PWXFORM_ROUND \ + PWXFORM_ROUND PWXFORM_WRITE \ + PWXFORM_ROUND PWXFORM_WRITE \ + PWXFORM_ROUND PWXFORM_WRITE \ + PWXFORM_ROUND PWXFORM_WRITE \ + PWXFORM_ROUND \ + w = (w + 64 * 4) & Smask2; \ + { \ + uint8_t *Stmp = S2; \ + S2 = S1; \ + S1 = S0; \ + S0 = Stmp; \ + } \ +} + +typedef struct { + uint8_t *S0, *S1, *S2; + size_t w; +} pwxform_ctx_t; + +#define Salloc (Sbytes + ((sizeof(pwxform_ctx_t) + 63) & ~63U)) + +/** + * blockmix_pwxform(Bin, Bout, r, S): + * Compute Bout = BlockMix_pwxform{salsa20/2, r, S}(Bin). The input Bin must + * be 128r bytes in length; the output Bout must also be the same size. + */ +static void blockmix(const salsa20_blk_t *restrict Bin, + salsa20_blk_t *restrict Bout, size_t r, pwxform_ctx_t *restrict ctx) +{ + uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; + size_t w = ctx->w; + size_t i; + DECL_X + + /* Convert count of 128-byte blocks to max index of 64-byte block */ + r = r * 2 - 1; + + READ_X(Bin[r]) + + DECL_SMASK2REG + + i = 0; + do { + XOR_X(Bin[i]) + PWXFORM + if (unlikely(i >= r)) + break; + WRITE_X(Bout[i]) + i++; + } while (1); + + ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; + ctx->w = w; + + SALSA20_2(Bout[i]) +} + +static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, + const salsa20_blk_t *restrict Bin2, salsa20_blk_t *Bout, + size_t r, int Bin2_in_ROM, pwxform_ctx_t *restrict ctx) +{ + uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; + size_t w = ctx->w; + size_t i; + DECL_X + + /* Convert count of 128-byte blocks to max index of 64-byte block */ + r = r * 2 - 1; + +#ifdef PREFETCH + if (Bin2_in_ROM) { + PREFETCH(&Bin2[r], _MM_HINT_NTA) + for (i = 0; i < r; i++) { + PREFETCH(&Bin2[i], _MM_HINT_NTA) + } + } else { + PREFETCH(&Bin2[r], _MM_HINT_T0) + for (i = 0; i < r; i++) { + PREFETCH(&Bin2[i], _MM_HINT_T0) + } + } +#else + (void)Bin2_in_ROM; /* unused */ +#endif + + XOR_X_2(Bin1[r], Bin2[r]) + + DECL_SMASK2REG + + i = 0; + r--; + do { + XOR_X(Bin1[i]) + XOR_X(Bin2[i]) + PWXFORM + WRITE_X(Bout[i]) + + XOR_X(Bin1[i + 1]) + XOR_X(Bin2[i + 1]) + PWXFORM + + if (unlikely(i >= r)) + break; + + WRITE_X(Bout[i + 1]) + + i += 2; + } while (1); + i++; + + ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; + ctx->w = w; + + SALSA20_2(Bout[i]) + + return INTEGERIFY; +} + +static uint32_t blockmix_xor_save(salsa20_blk_t *restrict Bin1out, + salsa20_blk_t *restrict Bin2, + size_t r, pwxform_ctx_t *restrict ctx) +{ + uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; + size_t w = ctx->w; + size_t i; + DECL_X + DECL_Y + + /* Convert count of 128-byte blocks to max index of 64-byte block */ + r = r * 2 - 1; + +#ifdef PREFETCH + PREFETCH(&Bin2[r], _MM_HINT_T0) + for (i = 0; i < r; i++) { + PREFETCH(&Bin2[i], _MM_HINT_T0) + } +#endif + + XOR_X_2(Bin1out[r], Bin2[r]) + + DECL_SMASK2REG + + i = 0; + r--; + do { + XOR_X_WRITE_XOR_Y_2(Bin2[i], Bin1out[i]) + PWXFORM + WRITE_X(Bin1out[i]) + + XOR_X_WRITE_XOR_Y_2(Bin2[i + 1], Bin1out[i + 1]) + PWXFORM + + if (unlikely(i >= r)) + break; + + WRITE_X(Bin1out[i + 1]) + + i += 2; + } while (1); + i++; + + ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; + ctx->w = w; + + SALSA20_2(Bin1out[i]) + + return INTEGERIFY; +} + +/** + * integerify(B, r): + * Return the result of parsing B_{2r-1} as a little-endian integer. + */ +static inline uint32_t integerify(const salsa20_blk_t *B, size_t r) +{ +/* + * Our 64-bit words are in host byte order, which is why we don't just read + * w[0] here (would be wrong on big-endian). Also, our 32-bit words are + * SIMD-shuffled (so the next 32 bits would be part of d[6]), but currently + * this does not matter as we only care about the least significant 32 bits. + */ + return (uint32_t)B[2 * r - 1].d[0]; +} + +/** + * smix1(B, r, N, flags, V, NROM, VROM, XY, ctx): + * Compute first loop of B = SMix_r(B, N). The input B must be 128r bytes in + * length; the temporary storage V must be 128rN bytes in length; the temporary + * storage XY must be 128r+64 bytes in length. N must be even and at least 4. + * The array V must be aligned to a multiple of 64 bytes, and arrays B and XY + * to a multiple of at least 16 bytes. + */ +static void smix1(uint8_t *B, size_t r, uint32_t N, yescrypt_flags_t flags, + salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, + salsa20_blk_t *XY, pwxform_ctx_t *ctx) +{ + size_t s = 2 * r; + salsa20_blk_t *X = V, *Y = &V[s]; + uint32_t i, j; + + for (i = 0; i < 2 * r; i++) { + const salsa20_blk_t *src = (salsa20_blk_t *)&B[i * 64]; + salsa20_blk_t *tmp = Y; + salsa20_blk_t *dst = &X[i]; + size_t k; + for (k = 0; k < 16; k++) + tmp->w[k] = le32dec((const uint8_t *) &src->w[k]); + salsa20_simd_shuffle(tmp, dst); + } + + if (VROM) { + uint32_t n; + const salsa20_blk_t *V_j; + + V_j = &VROM[(NROM - 1) * s]; + j = blockmix_xor(X, V_j, Y, r, 1, ctx) & (NROM - 1); + V_j = &VROM[j * s]; + X = Y + s; + j = blockmix_xor(Y, V_j, X, r, 1, ctx); + + for (n = 2; n < N; n <<= 1) { + uint32_t m = (n < N / 2) ? n : (N - 1 - n); + for (i = 1; i < m; i += 2) { + j &= n - 1; + j += i - 1; + V_j = &V[j * s]; + Y = X + s; + j = blockmix_xor(X, V_j, Y, r, 0, ctx) & (NROM - 1); + V_j = &VROM[j * s]; + X = Y + s; + j = blockmix_xor(Y, V_j, X, r, 1, ctx); + } + } + n >>= 1; + + j &= n - 1; + j += N - 2 - n; + V_j = &V[j * s]; + Y = X + s; + j = blockmix_xor(X, V_j, Y, r, 0, ctx) & (NROM - 1); + V_j = &VROM[j * s]; + blockmix_xor(Y, V_j, XY, r, 1, ctx); + } else if (flags & YESCRYPT_RW) { + uint32_t n; + salsa20_blk_t *V_j; + + blockmix(X, Y, r, ctx); + X = Y + s; + blockmix(Y, X, r, ctx); + j = integerify(X, r); + + for (n = 2; n < N; n <<= 1) { + uint32_t m = (n < N / 2) ? n : (N - 1 - n); + for (i = 1; i < m; i += 2) { + Y = X + s; + j &= n - 1; + j += i - 1; + V_j = &V[j * s]; + j = blockmix_xor(X, V_j, Y, r, 0, ctx); + j &= n - 1; + j += i; + V_j = &V[j * s]; + X = Y + s; + j = blockmix_xor(Y, V_j, X, r, 0, ctx); + } + } + n >>= 1; + + j &= n - 1; + j += N - 2 - n; + V_j = &V[j * s]; + Y = X + s; + j = blockmix_xor(X, V_j, Y, r, 0, ctx); + j &= n - 1; + j += N - 1 - n; + V_j = &V[j * s]; + blockmix_xor(Y, V_j, XY, r, 0, ctx); + } else { + N -= 2; + do { + blockmix_salsa8(X, Y, r); + X = Y + s; + blockmix_salsa8(Y, X, r); + Y = X + s; + } while ((N -= 2)); + + blockmix_salsa8(X, Y, r); + blockmix_salsa8(Y, XY, r); + } + + for (i = 0; i < 2 * r; i++) { + const salsa20_blk_t *src = &XY[i]; + salsa20_blk_t *tmp = &XY[s]; + salsa20_blk_t *dst = (salsa20_blk_t *)&B[i * 64]; + size_t k; + for (k = 0; k < 16; k++) + le32enc((uint8_t *)&tmp->w[k], src->w[k]); + salsa20_simd_unshuffle(tmp, dst); + } +} + +/** + * smix2(B, r, N, Nloop, flags, V, NROM, VROM, XY, ctx): + * Compute second loop of B = SMix_r(B, N). The input B must be 128r bytes in + * length; the temporary storage V must be 128rN bytes in length; the temporary + * storage XY must be 256r bytes in length. N must be a power of 2 and at + * least 2. Nloop must be even. The array V must be aligned to a multiple of + * 64 bytes, and arrays B and XY to a multiple of at least 16 bytes. + */ +static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, + yescrypt_flags_t flags, salsa20_blk_t *V, uint32_t NROM, + const salsa20_blk_t *VROM, salsa20_blk_t *XY, pwxform_ctx_t *ctx) +{ + size_t s = 2 * r; + salsa20_blk_t *X = XY, *Y = &XY[s]; + uint32_t i, j; + + if (Nloop == 0) + return; + + for (i = 0; i < 2 * r; i++) { + const salsa20_blk_t *src = (salsa20_blk_t *)&B[i * 64]; + salsa20_blk_t *tmp = Y; + salsa20_blk_t *dst = &X[i]; + size_t k; + for (k = 0; k < 16; k++) + tmp->w[k] = le32dec((const uint8_t *)&src->w[k]); + salsa20_simd_shuffle(tmp, dst); + } + + j = integerify(X, r) & (N - 1); + +/* + * Normally, VROM implies YESCRYPT_RW, but we check for these separately + * because our SMix resets YESCRYPT_RW for the smix2() calls operating on the + * entire V when p > 1. + */ + if (VROM && (flags & YESCRYPT_RW)) { + do { + salsa20_blk_t *V_j = &V[j * s]; + const salsa20_blk_t *VROM_j; + j = blockmix_xor_save(X, V_j, r, ctx) & (NROM - 1); + VROM_j = &VROM[j * s]; + j = blockmix_xor(X, VROM_j, X, r, 1, ctx) & (N - 1); + } while (Nloop -= 2); + } else if (VROM) { + do { + const salsa20_blk_t *V_j = &V[j * s]; + j = blockmix_xor(X, V_j, X, r, 0, ctx) & (NROM - 1); + V_j = &VROM[j * s]; + j = blockmix_xor(X, V_j, X, r, 1, ctx) & (N - 1); + } while (Nloop -= 2); + } else if (flags & YESCRYPT_RW) { + do { + salsa20_blk_t *V_j = &V[j * s]; + j = blockmix_xor_save(X, V_j, r, ctx) & (N - 1); + V_j = &V[j * s]; + j = blockmix_xor_save(X, V_j, r, ctx) & (N - 1); + } while (Nloop -= 2); + } else if (ctx) { + do { + const salsa20_blk_t *V_j = &V[j * s]; + j = blockmix_xor(X, V_j, X, r, 0, ctx) & (N - 1); + V_j = &V[j * s]; + j = blockmix_xor(X, V_j, X, r, 0, ctx) & (N - 1); + } while (Nloop -= 2); + } else { + do { + const salsa20_blk_t *V_j = &V[j * s]; + j = blockmix_salsa8_xor(X, V_j, Y, r) & (N - 1); + V_j = &V[j * s]; + j = blockmix_salsa8_xor(Y, V_j, X, r) & (N - 1); + } while (Nloop -= 2); + } + + for (i = 0; i < 2 * r; i++) { + const salsa20_blk_t *src = &X[i]; + salsa20_blk_t *tmp = Y; + salsa20_blk_t *dst = (salsa20_blk_t *)&B[i * 64]; + size_t k; + for (k = 0; k < 16; k++) + le32enc((uint8_t *)&tmp->w[k], src->w[k]); + salsa20_simd_unshuffle(tmp, dst); + } +} + +/** + * p2floor(x): + * Largest power of 2 not greater than argument. + */ +static uint64_t p2floor(uint64_t x) +{ + uint64_t y; + while ((y = x & (x - 1))) + x = y; + return x; +} + +/** + * smix(B, r, N, p, t, flags, V, NROM, VROM, XY, S, passwd): + * Compute B = SMix_r(B, N). The input B must be 128rp bytes in length; the + * temporary storage V must be 128rN bytes in length; the temporary storage + * XY must be 256r or 256rp bytes in length (the larger size is required with + * OpenMP-enabled builds). N must be a power of 2 and at least 4. The array V + * must be aligned to a multiple of 64 bytes, and arrays B and XY to a multiple + * of at least 16 bytes (aligning them to 64 bytes as well saves cache lines + * and helps avoid false sharing in OpenMP-enabled builds when p > 1, but it + * might also result in cache bank conflicts). + */ +static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, + yescrypt_flags_t flags, + salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, + salsa20_blk_t *XY, uint8_t *S, uint8_t *passwd) +{ + size_t s = 2 * r; + uint32_t Nchunk; + uint64_t Nloop_all, Nloop_rw; + uint32_t i; + + Nchunk = N / p; + Nloop_all = Nchunk; + if (flags & YESCRYPT_RW) { + if (t <= 1) { + if (t) + Nloop_all *= 2; /* 2/3 */ + Nloop_all = (Nloop_all + 2) / 3; /* 1/3, round up */ + } else { + Nloop_all *= t - 1; + } + } else if (t) { + if (t == 1) + Nloop_all += (Nloop_all + 1) / 2; /* 1.5, round up */ + Nloop_all *= t; + } + + Nloop_rw = 0; + if (flags & YESCRYPT_INIT_SHARED) + Nloop_rw = Nloop_all; + else if (flags & YESCRYPT_RW) + Nloop_rw = Nloop_all / p; + + Nchunk &= ~(uint32_t)1; /* round down to even */ + Nloop_all++; Nloop_all &= ~(uint64_t)1; /* round up to even */ + Nloop_rw++; Nloop_rw &= ~(uint64_t)1; /* round up to even */ + + for (i = 0; i < p; i++) { + uint32_t Vchunk = i * Nchunk; + uint32_t Np = (i < p - 1) ? Nchunk : (N - Vchunk); + uint8_t *Bp = &B[128 * r * i]; + salsa20_blk_t *Vp = &V[Vchunk * s]; + salsa20_blk_t *XYp = XY; + pwxform_ctx_t *ctx_i = NULL; + if (flags & YESCRYPT_RW) { + uint8_t *Si = S + i * Salloc; + smix1(Bp, 1, Sbytes / 128, 0 /* no flags */, + (salsa20_blk_t *)Si, 0, NULL, XYp, NULL); + ctx_i = (pwxform_ctx_t *)(Si + Sbytes); + ctx_i->S2 = Si; + ctx_i->S1 = Si + Sbytes / 3; + ctx_i->S0 = Si + Sbytes / 3 * 2; + ctx_i->w = 0; + if (i == 0) + HMAC_SHA256_Buf(Bp + (128 * r - 64), 64, + passwd, 32, passwd); + } + smix1(Bp, r, Np, flags, Vp, NROM, VROM, XYp, ctx_i); + smix2(Bp, r, p2floor(Np), Nloop_rw, flags, Vp, + NROM, VROM, XYp, ctx_i); + } + + if (Nloop_all > Nloop_rw) { + for (i = 0; i < p; i++) { + uint8_t *Bp = &B[128 * r * i]; + salsa20_blk_t *XYp = XY; + pwxform_ctx_t *ctx_i = NULL; + if (flags & YESCRYPT_RW) { + uint8_t *Si = S + i * Salloc; + ctx_i = (pwxform_ctx_t *)(Si + Sbytes); + } + smix2(Bp, r, N, Nloop_all - Nloop_rw, + flags & (yescrypt_flags_t)~YESCRYPT_RW, + V, NROM, VROM, XYp, ctx_i); + } + } +} + +/** + * yescrypt_kdf_body(shared, local, passwd, passwdlen, salt, saltlen, + * flags, N, r, p, t, NROM, buf, buflen): + * Compute scrypt(passwd[0 .. passwdlen - 1], salt[0 .. saltlen - 1], N, r, + * p, buflen), or a revision of scrypt as requested by flags and shared, and + * write the result into buf. + * + * shared and flags may request special modes as described in yescrypt.h. + * + * local is the thread-local data structure, allowing to preserve and reuse a + * memory allocation across calls, thereby reducing its overhead. + * + * t controls computation time while not affecting peak memory usage. + * + * Return 0 on success; or -1 on error. + * + * This optimized implementation currently limits N to the range from 4 to + * 2^31, but other implementations might not. + */ +static int yescrypt_kdf_body(const yescrypt_shared_t *shared, + yescrypt_local_t *local, + const uint8_t *passwd, size_t passwdlen, + const uint8_t *salt, size_t saltlen, + yescrypt_flags_t flags, uint64_t N, uint32_t r, uint32_t p, uint32_t t, + uint64_t NROM, + uint8_t *buf, size_t buflen) +{ + yescrypt_region_t tmp; + const salsa20_blk_t *VROM; + size_t B_size, V_size, XY_size, need; + uint8_t *B, *S; + salsa20_blk_t *V, *XY; + uint8_t sha256[32]; + uint8_t dk[sizeof(sha256)], *dkp = buf; + + /* Sanity-check parameters */ + switch (flags & YESCRYPT_MODE_MASK) { + case 0: /* classic scrypt - can't have anything non-standard */ + if (flags || t || NROM) + goto out_EINVAL; + break; + case YESCRYPT_WORM: + if (flags != YESCRYPT_WORM || NROM) + goto out_EINVAL; + break; + case YESCRYPT_RW: + if (flags != (flags & YESCRYPT_KNOWN_FLAGS)) + goto out_EINVAL; +#if PWXsimple == 2 && PWXgather == 4 && Sbytes == 12288 + if ((flags & YESCRYPT_RW_FLAVOR_MASK) == + (YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | + YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K)) + break; +#else +#error "Unsupported pwxform settings" +#endif + /* FALLTHRU */ + default: + goto out_EINVAL; + } +#if SIZE_MAX > UINT32_MAX + if (buflen > (((uint64_t)1 << 32) - 1) * 32) + goto out_EINVAL; +#endif + if ((uint64_t)r * (uint64_t)p >= 1 << 30) + goto out_EINVAL; + if (N > UINT32_MAX) + goto out_EINVAL; + if ((N & (N - 1)) != 0 || N <= 3 || r < 1 || p < 1) + goto out_EINVAL; + if (r > SIZE_MAX / 256 / p || + N > SIZE_MAX / 128 / r) + goto out_EINVAL; + if (flags & YESCRYPT_RW) { + /* p cannot be greater than SIZE_MAX/Salloc on 64-bit systems, + but it can on 32-bit systems. */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wtype-limits" + if (N / p <= 3 || p > SIZE_MAX / Salloc) + goto out_EINVAL; +#pragma GCC diagnostic pop + } + + VROM = NULL; + if (shared) { + uint64_t expected_size = (size_t)128 * r * NROM; + if ((NROM & (NROM - 1)) != 0 || + NROM <= 1 || NROM > UINT32_MAX || + shared->aligned_size < expected_size) + goto out_EINVAL; + if (!(flags & YESCRYPT_INIT_SHARED)) { + uint64_t *tag = (uint64_t *) + ((uint8_t *)shared->aligned + expected_size - 48); + if (tag[0] != YESCRYPT_ROM_TAG1 || tag[1] != YESCRYPT_ROM_TAG2) + goto out_EINVAL; + } + VROM = shared->aligned; + } else { + if (NROM) + goto out_EINVAL; + } + + /* Allocate memory */ + V = NULL; + V_size = (size_t)128 * r * N; + need = V_size; + if (flags & YESCRYPT_INIT_SHARED) { + if (local->aligned_size < need) { + if (local->base || local->aligned || + local->base_size || local->aligned_size) + goto out_EINVAL; + if (!alloc_region(local, need)) + return -1; + } + if (flags & YESCRYPT_ALLOC_ONLY) + return -2; /* expected "failure" */ + V = (salsa20_blk_t *)local->aligned; + need = 0; + } + B_size = (size_t)128 * r * p; + need += B_size; + if (need < B_size) + goto out_EINVAL; + XY_size = (size_t)256 * r; + need += XY_size; + if (need < XY_size) + goto out_EINVAL; + if (flags & YESCRYPT_RW) { + size_t S_size = (size_t)Salloc * p; + need += S_size; + if (need < S_size) + goto out_EINVAL; + } + if (flags & YESCRYPT_INIT_SHARED) { + if (!alloc_region(&tmp, need)) + return -1; + B = (uint8_t *)tmp.aligned; + XY = (salsa20_blk_t *)((uint8_t *)B + B_size); + } else { + init_region(&tmp); + if (local->aligned_size < need) { + if (free_region(local)) + return -1; + if (!alloc_region(local, need)) + return -1; + } + if (flags & YESCRYPT_ALLOC_ONLY) + return -3; /* expected "failure" */ + B = (uint8_t *)local->aligned; + V = (salsa20_blk_t *)((uint8_t *)B + B_size); + XY = (salsa20_blk_t *)((uint8_t *)V + V_size); + } + S = NULL; + if (flags & YESCRYPT_RW) + S = (uint8_t *)XY + XY_size; + + if (flags) { + HMAC_SHA256_Buf("yescrypt-prehash", + (flags & YESCRYPT_PREHASH) ? 16 : 8, + passwd, passwdlen, sha256); + passwd = sha256; + passwdlen = sizeof(sha256); + } + + PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, 1, B, B_size); + + if (flags) + memcpy(sha256, B, sizeof(sha256)); + + if (p == 1 || (flags & YESCRYPT_RW)) { + smix(B, r, N, p, t, flags, V, NROM, VROM, XY, S, sha256); + } else { + uint32_t i; + for (i = 0; i < p; i++) { + smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, V, + NROM, VROM, XY, NULL, NULL); + } + } + + dkp = buf; + if (flags && buflen < sizeof(dk)) { + PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, dk, sizeof(dk)); + dkp = dk; + } + + PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, buf, buflen); + + /* + * Except when computing classic scrypt, allow all computation so far + * to be performed on the client. The final steps below match those of + * SCRAM (RFC 5802), so that an extension of SCRAM (with the steps so + * far in place of SCRAM's use of PBKDF2 and with SHA-256 in place of + * SCRAM's use of SHA-1) would be usable with yescrypt hashes. + */ + if (flags && !(flags & YESCRYPT_PREHASH)) { + /* Compute ClientKey */ + HMAC_SHA256_Buf(dkp, sizeof(dk), "Client Key", 10, sha256); + /* Compute StoredKey */ + { + size_t clen = buflen; + if (clen > sizeof(dk)) + clen = sizeof(dk); + SHA256_Buf(sha256, sizeof(sha256), dk); + memcpy(buf, dk, clen); + } + } + + if (flags) { + explicit_bzero(sha256, sizeof(sha256)); + explicit_bzero(dk, sizeof(dk)); + } + + if (free_region(&tmp)) { + explicit_bzero(buf, buflen); /* must preserve errno */ + return -1; + } + + /* Success! */ + return 0; + +out_EINVAL: + errno = EINVAL; + return -1; +} + +/** + * yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, params, + * buf, buflen): + * Compute scrypt or its revision as requested by the parameters. The inputs + * to this function are the same as those for yescrypt_kdf_body() above, with + * the addition of g, which controls hash upgrades (0 for no upgrades so far). + */ +int yescrypt_kdf(const yescrypt_shared_t *shared, + yescrypt_local_t *local, + const uint8_t *passwd, size_t passwdlen, + const uint8_t *salt, size_t saltlen, + const yescrypt_params_t *params, + uint8_t *buf, size_t buflen) +{ + yescrypt_flags_t flags = params->flags; + uint64_t N = params->N; + uint32_t r = params->r; + uint32_t p = params->p; + uint32_t t = params->t; + uint32_t g = params->g; + uint64_t NROM = params->NROM; + uint8_t dk[32]; + int retval; + + /* Support for hash upgrades has been temporarily removed */ + if (g) { + errno = EINVAL; + return -1; + } + + if ((flags & (YESCRYPT_RW | YESCRYPT_INIT_SHARED)) == YESCRYPT_RW + && p >= 1 + && N / p >= 0x100 + && N / p * r >= 0x20000 + ) { + if (yescrypt_kdf_body(shared, local, + passwd, passwdlen, salt, saltlen, + flags | YESCRYPT_ALLOC_ONLY, N, r, p, t, NROM, + buf, buflen) != -3 + ) { + errno = EINVAL; + return -1; + } + retval = yescrypt_kdf_body(shared, local, + passwd, passwdlen, salt, saltlen, + flags | YESCRYPT_PREHASH, N >> 6, r, p, 0, NROM, + dk, sizeof(dk)); + if (retval) + return retval; + passwd = dk; + passwdlen = sizeof(dk); + } + + retval = yescrypt_kdf_body(shared, local, + passwd, passwdlen, salt, saltlen, + flags, N, r, p, t, NROM, buf, buflen); +#ifndef SKIP_MEMZERO + if (passwd == dk) + explicit_bzero(dk, sizeof(dk)); +#endif + return retval; +} diff --git a/libbb/yescrypt/alg-yescrypt-platform.c b/libbb/yescrypt/alg-yescrypt-platform.c new file mode 100644 index 0000000000..09809c4b0f --- /dev/null +++ b/libbb/yescrypt/alg-yescrypt-platform.c @@ -0,0 +1,65 @@ +/*- + * Copyright 2013-2018,2022 Alexander Peslyak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +static void *alloc_region(yescrypt_region_t *region, size_t size) +{ + size_t base_size = size; + uint8_t *base, *aligned; + +#if 0 //def MAP_ANON - use mmap, possibly +//(if defined(MAP_HUGETLB) && defined(MAP_HUGE_2MB)) using 2MB pages +#else /* mmap not available */ + base = aligned = NULL; + if (size + 63 < size) { + errno = ENOMEM; + } else { + base = malloc(size + 63); + if (base) { + aligned = base + 63; + aligned -= (uintptr_t)aligned & 63; + } + } +#endif + region->base = base; + region->aligned = aligned; + region->base_size = base ? base_size : 0; + region->aligned_size = base ? size : 0; + return aligned; +} + +static inline void init_region(yescrypt_region_t *region) +{ + region->base = region->aligned = NULL; + region->base_size = region->aligned_size = 0; +} + +static int free_region(yescrypt_region_t *region) +{ + if (region->base) { +#if 0 //def MAP_ANON + if (munmap(region->base, region->base_size)) + return -1; +#else + free(region->base); +#endif + } + init_region(region); + return 0; +} diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h new file mode 100644 index 0000000000..e3c7e63981 --- /dev/null +++ b/libbb/yescrypt/alg-yescrypt.h @@ -0,0 +1,355 @@ +/*- + * Copyright 2009 Colin Percival + * Copyright 2013-2018 Alexander Peslyak + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file was originally written by Colin Percival as part of the Tarsnap + * online backup system. + */ + +#if 0 //UNUSED +/** + * crypto_scrypt(passwd, passwdlen, salt, saltlen, N, r, p, buf, buflen): + * Compute scrypt(passwd[0 .. passwdlen - 1], salt[0 .. saltlen - 1], N, r, + * p, buflen) and write the result into buf. The parameters r, p, and buflen + * must satisfy r * p < 2^30 and buflen <= (2^32 - 1) * 32. The parameter N + * must be a power of 2 greater than 1. + * + * Return 0 on success; or -1 on error. + * + * MT-safe as long as buf is local to the thread. + */ +extern int crypto_scrypt(const uint8_t *passwd, size_t passwdlen, + const uint8_t *salt, size_t saltlen, + uint64_t N, uint32_t r, uint32_t p, uint8_t *buf, size_t buflen); +#endif + +/** + * Internal type used by the memory allocator. Please do not use it directly. + * Use yescrypt_shared_t and yescrypt_local_t as appropriate instead, since + * they might differ from each other in a future version. + */ +typedef struct { + void *base, *aligned; + size_t base_size, aligned_size; +} yescrypt_region_t; + +/** + * Types for shared (ROM) and thread-local (RAM) data structures. + */ +typedef yescrypt_region_t yescrypt_shared_t; +typedef yescrypt_region_t yescrypt_local_t; + +/** + * Two 64-bit tags placed 48 bytes to the end of a ROM in host byte endianness + * (and followed by 32 bytes of the ROM digest). + */ +#define YESCRYPT_ROM_TAG1 0x7470797263736579 /* "yescrypt" */ +#define YESCRYPT_ROM_TAG2 0x687361684d4f522d /* "-ROMhash" */ + +/** + * Type and possible values for the flags argument of yescrypt_kdf(), + * yescrypt_encode_params_r(), yescrypt_encode_params(). Most of these may be + * OR'ed together, except that YESCRYPT_WORM stands on its own. + * Please refer to the description of yescrypt_kdf() below for the meaning of + * these flags. + */ +typedef uint32_t yescrypt_flags_t; +/* Public */ +#define YESCRYPT_WORM 1 +#define YESCRYPT_RW 0x002 +#define YESCRYPT_ROUNDS_3 0x000 +#define YESCRYPT_ROUNDS_6 0x004 +#define YESCRYPT_GATHER_1 0x000 +#define YESCRYPT_GATHER_2 0x008 +#define YESCRYPT_GATHER_4 0x010 +#define YESCRYPT_GATHER_8 0x018 +#define YESCRYPT_SIMPLE_1 0x000 +#define YESCRYPT_SIMPLE_2 0x020 +#define YESCRYPT_SIMPLE_4 0x040 +#define YESCRYPT_SIMPLE_8 0x060 +#define YESCRYPT_SBOX_6K 0x000 +#define YESCRYPT_SBOX_12K 0x080 +#define YESCRYPT_SBOX_24K 0x100 +#define YESCRYPT_SBOX_48K 0x180 +#define YESCRYPT_SBOX_96K 0x200 +#define YESCRYPT_SBOX_192K 0x280 +#define YESCRYPT_SBOX_384K 0x300 +#define YESCRYPT_SBOX_768K 0x380 +/* Only valid for yescrypt_init_shared() */ +#define YESCRYPT_SHARED_PREALLOCATED 0x10000 +#ifdef YESCRYPT_INTERNAL +/* Private */ +#define YESCRYPT_MODE_MASK 0x003 +#define YESCRYPT_RW_FLAVOR_MASK 0x3fc +#define YESCRYPT_INIT_SHARED 0x01000000 +#define YESCRYPT_ALLOC_ONLY 0x08000000 +#define YESCRYPT_PREHASH 0x10000000 +#endif + +#define YESCRYPT_RW_DEFAULTS \ + (YESCRYPT_RW | \ + YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | \ + YESCRYPT_SBOX_12K) + +#define YESCRYPT_DEFAULTS YESCRYPT_RW_DEFAULTS + +#ifdef YESCRYPT_INTERNAL +#define YESCRYPT_KNOWN_FLAGS \ + (YESCRYPT_MODE_MASK | YESCRYPT_RW_FLAVOR_MASK | \ + YESCRYPT_SHARED_PREALLOCATED | \ + YESCRYPT_INIT_SHARED | YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH) +#endif + +/** + * yescrypt parameters combined into one struct. N, r, p are the same as in + * classic scrypt, except that the meaning of p changes when YESCRYPT_RW is + * set. flags, t, g, NROM are special to yescrypt. + */ +typedef struct { + yescrypt_flags_t flags; + uint64_t N; + uint32_t r, p, t, g; + uint64_t NROM; +} yescrypt_params_t; + +/** + * A 256-bit yescrypt hash, or a hash encryption key (which may itself have + * been derived as a yescrypt hash of a human-specified key string). + */ +typedef union { + unsigned char uc[32]; + uint64_t u64[4]; +} yescrypt_binary_t; + +/* How many chars base-64 encoded bytes require? */ +#define BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) +/* The /etc/passwd-style hash is "$" */ +/* + * "$y$", up to 8 params of up to 6 chars each, '$', salt + * Alternatively, but that's smaller: + * "$7$", 3 params encoded as 1+5+5 chars, salt + */ +#define PREFIX_LEN (3 + 8 * 6 + 1 + BYTES2CHARS(32)) + +#define HASH_SIZE sizeof(yescrypt_binary_t) /* bytes */ +#define HASH_LEN BYTES2CHARS(HASH_SIZE) + + +/** + * yescrypt_init_shared(shared, seed, seedlen, params): + * Optionally allocate memory for and initialize the shared (ROM) data + * structure. The parameters flags, NROM, r, p, and t specify how the ROM is + * to be initialized, and seed and seedlen specify the initial seed affecting + * the data with which the ROM is filled. + * + * Return 0 on success; or -1 on error. + * + * If bit YESCRYPT_SHARED_PREALLOCATED in flags is set, then memory for the + * ROM is assumed to have been preallocated by the caller, with shared->aligned + * being the start address of the ROM and shared->aligned_size being its size + * (which must be sufficient for NROM, r, p). This may be used e.g. when the + * ROM is to be placed in a SysV shared memory segment allocated by the caller. + * + * MT-safe as long as shared is local to the thread. + */ +extern int yescrypt_init_shared(yescrypt_shared_t *shared, + const uint8_t *seed, size_t seedlen, const yescrypt_params_t *params); + +/** + * yescrypt_digest_shared(shared): + * Extract the previously stored message digest of the provided yescrypt ROM. + * + * Return pointer to the message digest on success; or NULL on error. + * + * MT-unsafe. + */ +extern yescrypt_binary_t *yescrypt_digest_shared(yescrypt_shared_t *shared); + +/** + * yescrypt_free_shared(shared): + * Free memory that had been allocated with yescrypt_init_shared(). + * + * Return 0 on success; or -1 on error. + * + * MT-safe as long as shared is local to the thread. + */ +extern int yescrypt_free_shared(yescrypt_shared_t *shared); + +/** + * yescrypt_init_local(local): + * Initialize the thread-local (RAM) data structure. Actual memory allocation + * is currently fully postponed until a call to yescrypt_kdf() or yescrypt_r(). + * + * Return 0 on success; or -1 on error. + * + * MT-safe as long as local is local to the thread. + */ +extern int yescrypt_init_local(yescrypt_local_t *local); + +/** + * yescrypt_free_local(local): + * Free memory that may have been allocated for an initialized thread-local + * (RAM) data structure. + * + * Return 0 on success; or -1 on error. + * + * MT-safe as long as local is local to the thread. + */ +extern int yescrypt_free_local(yescrypt_local_t *local); + +/** + * yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, params, + * buf, buflen): + * Compute scrypt(passwd[0 .. passwdlen - 1], salt[0 .. saltlen - 1], N, r, + * p, buflen), or a revision of scrypt as requested by flags and shared, and + * write the result into buf. The parameters N, r, p, and buflen must satisfy + * the same conditions as with crypto_scrypt(). t controls computation time + * while not affecting peak memory usage (t = 0 is optimal unless higher N*r + * is not affordable while higher t is). g controls hash upgrades (g = 0 for + * no upgrades so far). shared and flags may request special modes. local is + * the thread-local data structure, allowing to preserve and reuse a memory + * allocation across calls, thereby reducing processing overhead. + * + * Return 0 on success; or -1 on error. + * + * Classic scrypt is available by setting shared = NULL, flags = 0, and t = 0. + * + * Setting YESCRYPT_WORM enables only minimal deviations from classic scrypt: + * support for the t parameter, and pre- and post-hashing. + * + * Setting YESCRYPT_RW fully enables yescrypt. As a side effect of differences + * between the algorithms, it also prevents p > 1 from growing the threads' + * combined processing time and memory allocation (like it did with classic + * scrypt and YESCRYPT_WORM), treating p as a divider rather than a multiplier. + * + * Passing a shared structure, with ROM contents previously computed by + * yescrypt_init_shared(), enables the use of ROM and requires YESCRYPT_RW. + * + * In order to allow for initialization of the ROM to be split into a separate + * program (or separate invocation of the same program), the shared->aligned + * and shared->aligned_size fields may optionally be set by the caller directly + * (e.g., to a mapped SysV shm segment), without using yescrypt_init_shared(). + * + * local must be initialized with yescrypt_init_local(). + * + * MT-safe as long as local and buf are local to the thread. + */ +extern int yescrypt_kdf(const yescrypt_shared_t *shared, + yescrypt_local_t *local, + const uint8_t *passwd, size_t passwdlen, + const uint8_t *salt, size_t saltlen, + const yescrypt_params_t *params, + uint8_t *buf, size_t buflen); + +/** + * yescrypt_r(shared, local, passwd, passwdlen, setting, key, buf, buflen): + * Compute and encode an scrypt or enhanced scrypt hash of passwd given the + * parameters and salt value encoded in setting. If shared is not NULL, a ROM + * is used and YESCRYPT_RW is required. Otherwise, whether to compute classic + * scrypt, YESCRYPT_WORM (a slight deviation from classic scrypt), or + * YESCRYPT_RW (time-memory tradeoff discouraging modification) is determined + * by the setting string. shared (if not NULL) and local must be initialized + * as described above for yescrypt_kdf(). buf must be large enough (as + * indicated by buflen) to hold the encoded hash string. + * + * Return the encoded hash string on success; or NULL on error. + * + * MT-safe as long as local and buf are local to the thread. + */ +extern uint8_t *yescrypt_r(const yescrypt_shared_t *shared, + yescrypt_local_t *local, + const uint8_t *passwd, size_t passwdlen, + const uint8_t *setting, + //KEY: const yescrypt_binary_t *key, + uint8_t *buf, size_t buflen); + +#if 0 //UNUSED +/** + * yescrypt(passwd, setting): + * Compute and encode an scrypt or enhanced scrypt hash of passwd given the + * parameters and salt value encoded in setting. Whether to compute classic + * scrypt, YESCRYPT_WORM (a slight deviation from classic scrypt), or + * YESCRYPT_RW (time-memory tradeoff discouraging modification) is determined + * by the setting string. + * + * Return the encoded hash string on success; or NULL on error. + * + * This is a crypt(3)-like interface, which is simpler to use than + * yescrypt_r(), but it is not MT-safe, it does not allow for the use of a ROM, + * and it is slower than yescrypt_r() for repeated calls because it allocates + * and frees memory on each call. + * + * MT-unsafe. + */ +extern uint8_t *yescrypt(const uint8_t *passwd, const uint8_t *setting); + +/** + * yescrypt_reencrypt(hash, from_key, to_key): + * Re-encrypt a yescrypt hash from one key to another. Either key may be NULL + * to indicate unencrypted hash. The encoded hash string is modified in-place. + * + * Return the hash pointer on success; or NULL on error (in which case the hash + * string is left unmodified). + * + * MT-safe as long as hash is local to the thread. + */ +extern uint8_t *yescrypt_reencrypt(uint8_t *hash, + const yescrypt_binary_t *from_key, + const yescrypt_binary_t *to_key); + +/** + * yescrypt_encode_params_r(params, src, srclen, buf, buflen): + * Generate a setting string for use with yescrypt_r() and yescrypt() by + * encoding into it the parameters flags, N, r, p, t, g, and a salt given by + * src (of srclen bytes). buf must be large enough (as indicated by buflen) + * to hold the setting string. + * + * Return the setting string on success; or NULL on error. + * + * MT-safe as long as buf is local to the thread. + */ +extern uint8_t *yescrypt_encode_params_r(const yescrypt_params_t *params, + const uint8_t *src, size_t srclen, + uint8_t *buf, size_t buflen); + + +/** + * yescrypt_encode_params(params, src, srclen): + * Generate a setting string for use with yescrypt_r() and yescrypt(). This + * function is the same as yescrypt_encode_params_r() except that it uses a + * static buffer and thus is not MT-safe. + * + * Return the setting string on success; or NULL on error. + * + * MT-unsafe. + */ +extern uint8_t *yescrypt_encode_params(const yescrypt_params_t *params, + const uint8_t *src, size_t srclen); +#endif + +extern const uint8_t *decode64(uint8_t *dst, size_t *dstlen, + const uint8_t *src, size_t srclen); +extern uint8_t *encode64(uint8_t *dst, size_t dstlen, + const uint8_t *src, size_t srclen); diff --git a/libbb/yescrypt/y.c b/libbb/yescrypt/y.c new file mode 100644 index 0000000000..042c439a0c --- /dev/null +++ b/libbb/yescrypt/y.c @@ -0,0 +1,139 @@ +//kbuild:lib-$(CONFIG_USE_BB_CRYPT_YES) += y.o + +#include + +#include + +static inline void +cpu_to_le32 (unsigned char *buf, uint32_t n) +{ + buf[0] = (unsigned char)((n & 0x000000FFu) >> 0); + buf[1] = (unsigned char)((n & 0x0000FF00u) >> 8); + buf[2] = (unsigned char)((n & 0x00FF0000u) >> 16); + buf[3] = (unsigned char)((n & 0xFF000000u) >> 24); +} +static inline uint32_t +le32_to_cpu (const unsigned char *buf) +{ + return ((((uint32_t)buf[0]) << 0) | + (((uint32_t)buf[1]) << 8) | + (((uint32_t)buf[2]) << 16) | + (((uint32_t)buf[3]) << 24) ); +} + +/* Alternative names used in code derived from Colin Percival's + cryptography libraries. */ +#define le32enc cpu_to_le32 +#define le32dec le32_to_cpu +#define le64enc cpu_to_le64 +#define le64dec le64_to_cpu + +#define be32enc cpu_to_be32 +#define be32dec be32_to_cpu +#define be64enc cpu_to_be64 +#define be64dec be64_to_cpu + +#define be32enc_vect cpu_to_be32_vect +#define be32dec_vect be32_to_cpu_vect +#define be64enc_vect cpu_to_be64_vect +#define be64dec_vect be64_to_cpu_vect + + +//USED ONY BY SHA256 for be32_to_cpu_vect(): +static inline void +cpu_to_be32(unsigned char *buf, uint32_t n) +{ + buf[0] = (unsigned char)((n & 0xFF000000u) >> 24); + buf[1] = (unsigned char)((n & 0x00FF0000u) >> 16); + buf[2] = (unsigned char)((n & 0x0000FF00u) >> 8); + buf[3] = (unsigned char)((n & 0x000000FFu) >> 0); +} +static inline void +cpu_to_be64 (unsigned char *buf, uint64_t n) +{ + buf[0] = (unsigned char)((n & 0xFF00000000000000ull) >> 56); + buf[1] = (unsigned char)((n & 0x00FF000000000000ull) >> 48); + buf[2] = (unsigned char)((n & 0x0000FF0000000000ull) >> 40); + buf[3] = (unsigned char)((n & 0x000000FF00000000ull) >> 32); + buf[4] = (unsigned char)((n & 0x00000000FF000000ull) >> 24); + buf[5] = (unsigned char)((n & 0x0000000000FF0000ull) >> 16); + buf[6] = (unsigned char)((n & 0x000000000000FF00ull) >> 8); + buf[7] = (unsigned char)((n & 0x00000000000000FFull) >> 0); +} +static inline uint32_t +be32_to_cpu (const unsigned char *buf) +{ + return ((((uint32_t)buf[0]) << 24) | + (((uint32_t)buf[1]) << 16) | + (((uint32_t)buf[2]) << 8) | + (((uint32_t)buf[3]) << 0) ); +} +static inline uint64_t +be64_to_cpu (const unsigned char *buf) +{ + return ((((uint64_t)buf[0]) << 56) | + (((uint64_t)buf[1]) << 48) | + (((uint64_t)buf[2]) << 40) | + (((uint64_t)buf[3]) << 32) | + (((uint64_t)buf[4]) << 24) | + (((uint64_t)buf[5]) << 16) | + (((uint64_t)buf[6]) << 8) | + (((uint64_t)buf[7]) << 0) ); +} +/* Template: Define a function named cpu_to__vect that + takes a vector SRC of LEN integers, each of type uint_t, and + writes them to the buffer DST in the endianness defined by END. + Caution: LEN is the number of vector elements, not the total size + of the buffers. */ +#define VECTOR_CPU_TO(end, bits) VECTOR_CPU_TO_(end##bits, uint##bits##_t) +#define VECTOR_CPU_TO_(prim, stype) \ + static inline void \ + cpu_to_##prim##_vect(uint8_t *dst, const stype *src, size_t len) \ + { \ + while (len) \ + { \ + cpu_to_##prim(dst, *src); \ + src += 1; \ + dst += sizeof(stype); \ + len -= 1; \ + } \ + } struct _swallow_semicolon +/* Template: Define a function named _to_cpu_vect that + reads a vector of LEN integers, each of type uint_t, from the + buffer SRC, in the endianness defined by END, and writes them to + the vector DST. Caution: LEN is the number of vector elements, not + the total size of the buffers. */ +#define VECTOR_TO_CPU(end, bits) VECTOR_TO_CPU_(end##bits, uint##bits##_t) +#define VECTOR_TO_CPU_(prim, dtype) \ + static inline void \ + prim##_to_cpu_vect(dtype *dst, const uint8_t *src, size_t len) \ + { \ + while (len) \ + { \ + *dst = prim##_to_cpu(src); \ + src += sizeof(dtype); \ + dst += 1; \ + len -= 1; \ + } \ + } struct _swallow_semicolon +/* These are the vectorized endianness-conversion functions that are + presently used. Add more as necessary. */ +VECTOR_CPU_TO(be,32); +VECTOR_CPU_TO(be,64); +VECTOR_TO_CPU(be,32); +VECTOR_TO_CPU(be,64); + + +const unsigned char ascii64[65] = + "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; +/* 0000000000111111111122222222223333333333444444444455555555556666 */ +/* 0123456789012345678901234567890123456789012345678901234567890123 */ +#define itoa64 ascii64 + +#define YESCRYPT_INTERNAL +#include "alg-sha256.h" +#include "alg-yescrypt.h" +#include "alg-sha256.c" +#include "alg-yescrypt-platform.c" +#include "alg-yescrypt-kdf.c" +#include "alg-yescrypt-common.c" diff --git a/loginutils/Config.src b/loginutils/Config.src index cbb09646bd..a7812bd32c 100644 --- a/loginutils/Config.src +++ b/loginutils/Config.src @@ -91,6 +91,17 @@ config USE_BB_CRYPT_SHA With this option off, login will fail password check for any user which has password encrypted with these algorithms. +config USE_BB_CRYPT_YES + bool "Enable yescrypt functions" + default y + depends on USE_BB_CRYPT + help + Enable this if you have passwords starting with "$y$" or + in your /etc/passwd or /etc/shadow files. These passwords + are hashed using yescrypt algorithms. + With this option off, login will fail password check for any + user which has password encrypted with these algorithms. + INSERT endmenu diff --git a/networking/httpd.c b/networking/httpd.c index ddcb03bca1..e1a447fa1a 100644 --- a/networking/httpd.c +++ b/networking/httpd.c @@ -2826,7 +2826,7 @@ int httpd_main(int argc UNUSED_PARAM, char **argv) salt[0] = '$'; salt[1] = '1'; salt[2] = '$'; - crypt_make_salt(salt + 3, 4); + crypt_make_rand64encoded(salt + 3, 8 / 2); /* 8 chars */ puts(pw_encrypt(pass, salt, /*cleanup:*/ 0)); return 0; } diff --git a/testsuite/cryptpw.tests b/testsuite/cryptpw.tests index 0dd91fe15f..a17123218e 100755 --- a/testsuite/cryptpw.tests +++ b/testsuite/cryptpw.tests @@ -22,21 +22,61 @@ testing "cryptpw des zz" \ #SKIP= optional USE_BB_CRYPT_SHA -testing "cryptpw sha256" \ - "cryptpw -m sha256 QWErty '123456789012345678901234567890'" \ - '$5$1234567890123456$5DxfOCmU4vRhtzfsbdK.6wSGMwwVbac7ZkWwusb8Si7\n' "" "" +# Note: mkpasswd-5.6.2 won't accept "-m sha256", wants "-m sha256crypt" +testing 'cryptpw sha256' \ + 'cryptpw -m sha256 QWErty 1234567890123456' \ + '$5$1234567890123456$5DxfOCmU4vRhtzfsbdK.6wSGMwwVbac7ZkWwusb8Si7\n' \ + '' '' +# mkpasswd-5.6.2 does not allow overlong salts, we truncate (at 16 chars for sha256) +testing 'cryptpw sha256 overlong' \ + 'cryptpw -m sha256 QWErty 123456789012345678901234567890' \ + '$5$1234567890123456$5DxfOCmU4vRhtzfsbdK.6wSGMwwVbac7ZkWwusb8Si7\n' \ + '' '' +testing 'cryptpw sha256 implicit' \ + 'cryptpw QWErty \$5\$1234567890123456' \ + '$5$1234567890123456$5DxfOCmU4vRhtzfsbdK.6wSGMwwVbac7ZkWwusb8Si7\n' \ + '' '' +testing 'cryptpw sha256 rounds=99999' \ + 'cryptpw -m sha256 QWErty rounds=99999\$123456789012345678901234567890' \ + '$5$rounds=99999$1234567890123456$aYellycJGZM6AKyVzaQsSrDBdTixubtMnM6J.MN0xM8\n' \ + '' '' +testing 'cryptpw sha256 rounds=99999 implicit' \ + 'cryptpw QWErty \$5\$rounds=99999\$123456789012345678901234567890' \ + '$5$rounds=99999$1234567890123456$aYellycJGZM6AKyVzaQsSrDBdTixubtMnM6J.MN0xM8\n' \ + '' '' -testing "cryptpw sha256 rounds=99999" \ - "cryptpw -m sha256 QWErty 'rounds=99999\$123456789012345678901234567890'" \ - '$5$rounds=99999$1234567890123456$aYellycJGZM6AKyVzaQsSrDBdTixubtMnM6J.MN0xM8\n' "" "" - -testing "cryptpw sha512" \ - "cryptpw -m sha512 QWErty '123456789012345678901234567890'" \ - '$6$1234567890123456$KB7QqxFyqmJSWyQYcCuGeFukgz1bPQoipWZf7.9L7z3k8UNTXa6UikbKcUGDc2ANn7DOGmDaroxDgpK16w/RE0\n' "" "" +testing 'cryptpw sha512' \ + 'cryptpw -m sha512 QWErty 123456789012345678901234567890' \ + '$6$1234567890123456$KB7QqxFyqmJSWyQYcCuGeFukgz1bPQoipWZf7.9L7z3k8UNTXa6UikbKcUGDc2ANn7DOGmDaroxDgpK16w/RE0\n' \ + '' '' +testing 'cryptpw sha512 rounds=99999' \ + 'cryptpw -m sha512 QWErty rounds=99999\$123456789012345678901234567890' \ + '$6$rounds=99999$1234567890123456$BfF6gD6ZjUmwawH5QaAglYAxtU./yvsz0fcQ464l49aMI2DZW3j5ri28CrxK7riPWNpLuUpfaIdY751SBYKUH.\n' \ + '' '' +SKIP= -testing "cryptpw sha512 rounds=99999" \ - "cryptpw -m sha512 QWErty 'rounds=99999\$123456789012345678901234567890'" \ - '$6$rounds=99999$1234567890123456$BfF6gD6ZjUmwawH5QaAglYAxtU./yvsz0fcQ464l49aMI2DZW3j5ri28CrxK7riPWNpLuUpfaIdY751SBYKUH.\n' "" "" +optional USE_BB_CRYPT_YES +testing 'cryptpw yescrypt' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123456789012345678901234' \ + '$y$j9T$123456789012345678901234$AKxw5OX/T4jD.v./IW.5tE/j7izNjw06fg3OvH1LsN9\n' \ + '' '' +testing 'cryptpw yescrypt with non-standard cost 4 instead of 5 (j8T instead of j9T)' \ + 'cryptpw -m yescrypt qweRTY123@-+ j8T\$123456789012345678901234' \ + '$y$j8T$123456789012345678901234$JQUUfopCxlfZNE8f.THJwbOkhy.XtB3GIjo9HUVioWB\n' \ + '' '' +# mkpasswd-5.6.2 allows short salts for yescrypt +testing 'cryptpw yescrypt with empty salt' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$' \ + '$y$j9T$$hpeksL94GXNRwnA00L3c8WFy0khFAUbCpBSak.N3Bp.\n' \ + '' '' +testing 'cryptpw yescrypt with 3-char salt' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123' \ + '$y$j9T$123$A34DMIGUbUIo3bjx66Wtk2IFoREMIw6d49it25KQh2D\n' \ + '' '' +testing 'cryptpw yescrypt implicit' \ + 'cryptpw qweRTY123@-+ \$y\$j9T\$123456789012345678901234' \ + '$y$j9T$123456789012345678901234$AKxw5OX/T4jD.v./IW.5tE/j7izNjw06fg3OvH1LsN9\n' \ + '' '' SKIP= exit $FAILCOUNT From 444a91abc93fe1b3d98e9a18a63f0a2056290208 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 00:55:08 +0200 Subject: [PATCH 092/277] libbb/yescrypt: remove obviously unused, commented-out parts No code changes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 364 --------------------------- libbb/yescrypt/alg-yescrypt.h | 81 ------ 2 files changed, 445 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 75b59d1cf4..7c519b01dc 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -18,45 +18,6 @@ * SUCH DAMAGE. */ -#if 0 //UNUSED -static uint8_t *encode64_uint32(uint8_t *dst, size_t dstlen, - uint32_t src, uint32_t min) -{ - uint32_t start = 0, end = 47, chars = 1, bits = 0; - - if (src < min) - return NULL; - src -= min; - - do { - uint32_t count = (end + 1 - start) << bits; - if (src < count) - break; - if (start >= 63) - return NULL; - start = end + 1; - end = start + (62 - end) / 2; - src -= count; - chars++; - bits += 6; - } while (1); - - if (dstlen <= chars) /* require room for a NUL terminator */ - return NULL; - - *dst++ = itoa64[start + (src >> bits)]; - - while (--chars) { - bits -= 6; - *dst++ = itoa64[(src >> bits) & 0x3f]; - } - - *dst = 0; /* NUL terminate just in case */ - - return dst; -} -#endif //UNUSED - static inline uint32_t atoi64(uint8_t src) { static const uint8_t atoi64_partial[77] = { @@ -223,81 +184,11 @@ const uint8_t *decode64(uint8_t *dst, size_t *dstlen, return NULL; } -#if 0 //UNUSED //KEY: -typedef enum { ENC = 1, DEC = -1 } encrypt_dir_t; - -static void memxor(unsigned char *dst, unsigned char *src, size_t size) -{ - while (size--) - *dst++ ^= *src++; -} - -static void yescrypt_sha256_cipher(unsigned char *data, size_t datalen, - const yescrypt_binary_t *key, encrypt_dir_t dir) -{ - SHA256_CTX ctx; - unsigned char f[32 + 4]; - size_t halflen, which; - unsigned char mask, round, target; - - if (!datalen) - return; - if (datalen > 64) - datalen = 64; - - halflen = datalen >> 1; - - which = 0; /* offset to half we are working on (0 or halflen) */ - mask = 0x0f; /* current half's extra nibble mask if datalen is odd */ - - round = 0; - target = 5; /* 6 rounds due to Jacques Patarin's CRYPTO 2004 paper */ - - if (dir == DEC) { - which = halflen; /* even round count, so swap the halves */ - mask ^= 0xff; - - round = target; - target = 0; - } - - f[32] = 0; - f[33] = sizeof(*key); - f[34] = datalen; - - do { - SHA256_Init(&ctx); - f[35] = round; - SHA256_Update(&ctx, &f[32], 4); - SHA256_Update(&ctx, key, sizeof(*key)); - SHA256_Update(&ctx, &data[which], halflen); - if (datalen & 1) { - f[0] = data[datalen - 1] & mask; - SHA256_Update(&ctx, f, 1); - } - SHA256_Final(f, &ctx); - which ^= halflen; - memxor(&data[which], f, halflen); - if (datalen & 1) { - mask ^= 0xff; - data[datalen - 1] ^= f[halflen] & mask; - } - if (round == target) - break; - round += dir; - } while (1); - - /* ctx is presumably zeroized by SHA256_Final() */ - explicit_bzero(f, sizeof(f)); -} -#endif //UNUSED //KEY: - uint8_t *yescrypt_r( const yescrypt_shared_t *shared, yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, - //KEY const yescrypt_binary_t *key, uint8_t *buf, size_t buflen) { unsigned char saltbin[64], hashbin[32]; @@ -325,9 +216,6 @@ uint8_t *yescrypt_r( src = decode64_uint32_fixed(¶ms.p, 30, src); if (!src) return NULL; - - //KEY:if (key) - //KEY: return NULL; } else { uint32_t flavor, N_log2; @@ -425,11 +313,6 @@ uint8_t *yescrypt_r( ¶ms, hashbin, sizeof(hashbin))) goto fail; - //KEY:if (key) { - //KEY: explicit_bzero(saltbin, sizeof(saltbin)); - //KEY: yescrypt_sha256_cipher(hashbin, sizeof(hashbin), key, ENC); - //KEY:} - dst = buf; memcpy(dst, setting, prefixlen + saltstrlen); dst += prefixlen + saltstrlen; @@ -450,14 +333,6 @@ uint8_t *yescrypt_r( return NULL; } - - - - - - - - int yescrypt_free_shared(yescrypt_shared_t *shared) { return free_region(shared); @@ -473,242 +348,3 @@ int yescrypt_free_local(yescrypt_local_t *local) { return free_region(local); } - -#if 0 //UNUSED -uint8_t *yescrypt(const uint8_t *passwd, const uint8_t *setting) -{ - /* prefix, '$', hash, NUL */ - static uint8_t buf[PREFIX_LEN + 1 + HASH_LEN + 1]; - yescrypt_local_t local; - uint8_t *retval; - - if (yescrypt_init_local(&local)) - return NULL; - retval = yescrypt_r( - /* const yescrypt_shared_t *shared */ NULL, - &local, - passwd, strlen((char *)passwd), - setting, - //KEY:/* const yescrypt_binary_t *key */ NULL, - buf, sizeof(buf)); - if (yescrypt_free_local(&local)) - return NULL; - return retval; -} - -uint8_t *yescrypt_reencrypt(uint8_t *hash, - const yescrypt_binary_t *from_key, - const yescrypt_binary_t *to_key) -{ - uint8_t *retval = NULL, *saltstart, *hashstart; - const uint8_t *hashend; - unsigned char saltbin[64], hashbin[32]; - size_t saltstrlen, saltlen = 0, hashlen; - - if (strncmp((char *)hash, "$y$", 3)) - return NULL; - - saltstart = NULL; - hashstart = (uint8_t *)strrchr((char *)hash, '$'); - if (hashstart) { - if (hashstart > (uint8_t *)hash) { - saltstart = hashstart - 1; - while (*saltstart != '$' && saltstart > hash) - saltstart--; - if (*saltstart == '$') - saltstart++; - } - hashstart++; - } else { - hashstart = hash; - } - saltstrlen = saltstart ? (hashstart - 1 - saltstart) : 0; - if (saltstrlen > BYTES2CHARS(64) || - strlen((char *)hashstart) != HASH_LEN) - return NULL; - - if (saltstrlen) { - const uint8_t *saltend; - saltlen = sizeof(saltbin); - saltend = decode64(saltbin, &saltlen, saltstart, saltstrlen); - if (!saltend || *saltend != '$' || saltlen < 1 || saltlen > 64) - goto out; - - if (from_key) - yescrypt_sha256_cipher(saltbin, saltlen, from_key, ENC); - if (to_key) - yescrypt_sha256_cipher(saltbin, saltlen, to_key, DEC); - } - - hashlen = sizeof(hashbin); - hashend = decode64(hashbin, &hashlen, hashstart, HASH_LEN); - if (!hashend || *hashend || hashlen != sizeof(hashbin)) - goto out; - - if (from_key) - yescrypt_sha256_cipher(hashbin, hashlen, from_key, DEC); - if (to_key) - yescrypt_sha256_cipher(hashbin, hashlen, to_key, ENC); - - if (saltstrlen) { - if (!encode64(saltstart, saltstrlen + 1, saltbin, saltlen)) - goto out; /* can't happen */ - *(saltstart + saltstrlen) = '$'; - } - - if (!encode64(hashstart, HASH_LEN + 1, hashbin, hashlen)) - goto out; /* can't happen */ - - retval = hash; - -out: - explicit_bzero(saltbin, sizeof(saltbin)); - explicit_bzero(hashbin, sizeof(hashbin)); - - return retval; -} - -static uint32_t N2log2(uint64_t N) -{ - uint32_t N_log2; - - if (N < 2) - return 0; - - N_log2 = 2; - while (N >> N_log2 != 0) - N_log2++; - N_log2--; - - if (N >> N_log2 != 1) - return 0; - - return N_log2; -} - -uint8_t *yescrypt_encode_params_r(const yescrypt_params_t *params, - const uint8_t *src, size_t srclen, - uint8_t *buf, size_t buflen) -{ - uint32_t flavor, N_log2, NROM_log2, have; - uint8_t *dst; - - if (srclen > SIZE_MAX / 16) - return NULL; - - if (params->flags < YESCRYPT_RW) { - flavor = params->flags; - } else if ((params->flags & YESCRYPT_MODE_MASK) == YESCRYPT_RW && - params->flags <= (YESCRYPT_RW | YESCRYPT_RW_FLAVOR_MASK)) { - flavor = YESCRYPT_RW + (params->flags >> 2); - } else { - return NULL; - } - - N_log2 = N2log2(params->N); - if (!N_log2) - return NULL; - - NROM_log2 = N2log2(params->NROM); - if (params->NROM && !NROM_log2) - return NULL; - - if ((uint64_t)params->r * (uint64_t)params->p >= (1U << 30)) - return NULL; - - dst = buf; - *dst++ = '$'; - *dst++ = 'y'; - *dst++ = '$'; - - dst = encode64_uint32(dst, buflen - (dst - buf), flavor, 0); - if (!dst) - return NULL; - - dst = encode64_uint32(dst, buflen - (dst - buf), N_log2, 1); - if (!dst) - return NULL; - - dst = encode64_uint32(dst, buflen - (dst - buf), params->r, 1); - if (!dst) - return NULL; - - have = 0; - if (params->p != 1) - have |= 1; - if (params->t) - have |= 2; - if (params->g) - have |= 4; - if (NROM_log2) - have |= 8; - - if (have) { - dst = encode64_uint32(dst, buflen - (dst - buf), have, 1); - if (!dst) - return NULL; - } - - if (params->p != 1) { - dst = encode64_uint32(dst, buflen - (dst - buf), params->p, 2); - if (!dst) - return NULL; - } - - if (params->t) { - dst = encode64_uint32(dst, buflen - (dst - buf), params->t, 1); - if (!dst) - return NULL; - } - - if (params->g) { - dst = encode64_uint32(dst, buflen - (dst - buf), params->g, 1); - if (!dst) - return NULL; - } - - if (NROM_log2) { - dst = encode64_uint32(dst, buflen - (dst - buf), NROM_log2, 1); - if (!dst) - return NULL; - } - - if (dst >= buf + buflen) - return NULL; - - *dst++ = '$'; - - dst = encode64(dst, buflen - (dst - buf), src, srclen); - if (!dst || dst >= buf + buflen) - return NULL; - - *dst = 0; /* NUL termination */ - - return buf; -} - -uint8_t *yescrypt_encode_params(const yescrypt_params_t *params, - const uint8_t *src, size_t srclen) -{ - /* prefix, NUL */ - static uint8_t buf[PREFIX_LEN + 1]; - return yescrypt_encode_params_r(params, src, srclen, buf, sizeof(buf)); -} - -int crypto_scrypt(const uint8_t *passwd, size_t passwdlen, - const uint8_t *salt, size_t saltlen, uint64_t N, uint32_t r, uint32_t p, - uint8_t *buf, size_t buflen) -{ - yescrypt_local_t local; - yescrypt_params_t params = { .flags = 0, .N = N, .r = r, .p = p }; - int retval; - - if (yescrypt_init_local(&local)) - return -1; - retval = yescrypt_kdf(NULL, &local, - passwd, passwdlen, salt, saltlen, ¶ms, buf, buflen); - if (yescrypt_free_local(&local)) - return -1; - return retval; -} -#endif //UNUSED diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index e3c7e63981..9755ac4206 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -28,23 +28,6 @@ * online backup system. */ -#if 0 //UNUSED -/** - * crypto_scrypt(passwd, passwdlen, salt, saltlen, N, r, p, buf, buflen): - * Compute scrypt(passwd[0 .. passwdlen - 1], salt[0 .. saltlen - 1], N, r, - * p, buflen) and write the result into buf. The parameters r, p, and buflen - * must satisfy r * p < 2^30 and buflen <= (2^32 - 1) * 32. The parameter N - * must be a power of 2 greater than 1. - * - * Return 0 on success; or -1 on error. - * - * MT-safe as long as buf is local to the thread. - */ -extern int crypto_scrypt(const uint8_t *passwd, size_t passwdlen, - const uint8_t *salt, size_t saltlen, - uint64_t N, uint32_t r, uint32_t p, uint8_t *buf, size_t buflen); -#endif - /** * Internal type used by the memory allocator. Please do not use it directly. * Use yescrypt_shared_t and yescrypt_local_t as appropriate instead, since @@ -285,70 +268,6 @@ extern uint8_t *yescrypt_r(const yescrypt_shared_t *shared, //KEY: const yescrypt_binary_t *key, uint8_t *buf, size_t buflen); -#if 0 //UNUSED -/** - * yescrypt(passwd, setting): - * Compute and encode an scrypt or enhanced scrypt hash of passwd given the - * parameters and salt value encoded in setting. Whether to compute classic - * scrypt, YESCRYPT_WORM (a slight deviation from classic scrypt), or - * YESCRYPT_RW (time-memory tradeoff discouraging modification) is determined - * by the setting string. - * - * Return the encoded hash string on success; or NULL on error. - * - * This is a crypt(3)-like interface, which is simpler to use than - * yescrypt_r(), but it is not MT-safe, it does not allow for the use of a ROM, - * and it is slower than yescrypt_r() for repeated calls because it allocates - * and frees memory on each call. - * - * MT-unsafe. - */ -extern uint8_t *yescrypt(const uint8_t *passwd, const uint8_t *setting); - -/** - * yescrypt_reencrypt(hash, from_key, to_key): - * Re-encrypt a yescrypt hash from one key to another. Either key may be NULL - * to indicate unencrypted hash. The encoded hash string is modified in-place. - * - * Return the hash pointer on success; or NULL on error (in which case the hash - * string is left unmodified). - * - * MT-safe as long as hash is local to the thread. - */ -extern uint8_t *yescrypt_reencrypt(uint8_t *hash, - const yescrypt_binary_t *from_key, - const yescrypt_binary_t *to_key); - -/** - * yescrypt_encode_params_r(params, src, srclen, buf, buflen): - * Generate a setting string for use with yescrypt_r() and yescrypt() by - * encoding into it the parameters flags, N, r, p, t, g, and a salt given by - * src (of srclen bytes). buf must be large enough (as indicated by buflen) - * to hold the setting string. - * - * Return the setting string on success; or NULL on error. - * - * MT-safe as long as buf is local to the thread. - */ -extern uint8_t *yescrypt_encode_params_r(const yescrypt_params_t *params, - const uint8_t *src, size_t srclen, - uint8_t *buf, size_t buflen); - - -/** - * yescrypt_encode_params(params, src, srclen): - * Generate a setting string for use with yescrypt_r() and yescrypt(). This - * function is the same as yescrypt_encode_params_r() except that it uses a - * static buffer and thus is not MT-safe. - * - * Return the setting string on success; or NULL on error. - * - * MT-unsafe. - */ -extern uint8_t *yescrypt_encode_params(const yescrypt_params_t *params, - const uint8_t *src, size_t srclen); -#endif - extern const uint8_t *decode64(uint8_t *dst, size_t *dstlen, const uint8_t *src, size_t srclen); extern uint8_t *encode64(uint8_t *dst, size_t dstlen, From be327bed9da573275e874af106fb52effb846dc1 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 01:03:49 +0200 Subject: [PATCH 093/277] libbb/yescrypt: remove unused yescrypt_shared_t function old new delta static.smix - 755 +755 static.smix1 - 631 +631 static.smix2 - 452 +452 yescrypt_free_local 9 49 +40 yes_crypt 90 87 -3 yescrypt_r 890 879 -11 yescrypt_kdf 479 449 -30 free_region 47 - -47 yescrypt_kdf_body 1724 1467 -257 smix2 659 - -659 smix 790 - -790 smix1 960 - -960 ------------------------------------------------------------------------------ (add/remove: 3/4 grow/shrink: 1/4 up/down: 1878/-2757) Total: -879 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 8 +---- libbb/yescrypt/alg-yescrypt-kdf.c | 29 ++++------------ libbb/yescrypt/alg-yescrypt.h | 50 ++-------------------------- 3 files changed, 10 insertions(+), 77 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 7c519b01dc..9d7231f09a 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -185,7 +185,6 @@ const uint8_t *decode64(uint8_t *dst, size_t *dstlen, } uint8_t *yescrypt_r( - const yescrypt_shared_t *shared, yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, @@ -309,7 +308,7 @@ uint8_t *yescrypt_r( if (need > buflen || need < saltstrlen) goto fail; - if (yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, + if (yescrypt_kdf(local, passwd, passwdlen, salt, saltlen, ¶ms, hashbin, sizeof(hashbin))) goto fail; @@ -333,11 +332,6 @@ uint8_t *yescrypt_r( return NULL; } -int yescrypt_free_shared(yescrypt_shared_t *shared) -{ - return free_region(shared); -} - int yescrypt_init_local(yescrypt_local_t *local) { init_region(local); diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 3e98ffb0a0..da23c1b59d 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -800,7 +800,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, * This optimized implementation currently limits N to the range from 4 to * 2^31, but other implementations might not. */ -static int yescrypt_kdf_body(const yescrypt_shared_t *shared, +static int yescrypt_kdf_body( yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, @@ -865,23 +865,8 @@ static int yescrypt_kdf_body(const yescrypt_shared_t *shared, } VROM = NULL; - if (shared) { - uint64_t expected_size = (size_t)128 * r * NROM; - if ((NROM & (NROM - 1)) != 0 || - NROM <= 1 || NROM > UINT32_MAX || - shared->aligned_size < expected_size) - goto out_EINVAL; - if (!(flags & YESCRYPT_INIT_SHARED)) { - uint64_t *tag = (uint64_t *) - ((uint8_t *)shared->aligned + expected_size - 48); - if (tag[0] != YESCRYPT_ROM_TAG1 || tag[1] != YESCRYPT_ROM_TAG2) - goto out_EINVAL; - } - VROM = shared->aligned; - } else { - if (NROM) - goto out_EINVAL; - } + if (NROM) + goto out_EINVAL; /* Allocate memory */ V = NULL; @@ -1013,7 +998,7 @@ static int yescrypt_kdf_body(const yescrypt_shared_t *shared, * to this function are the same as those for yescrypt_kdf_body() above, with * the addition of g, which controls hash upgrades (0 for no upgrades so far). */ -int yescrypt_kdf(const yescrypt_shared_t *shared, +int yescrypt_kdf( yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, @@ -1041,7 +1026,7 @@ int yescrypt_kdf(const yescrypt_shared_t *shared, && N / p >= 0x100 && N / p * r >= 0x20000 ) { - if (yescrypt_kdf_body(shared, local, + if (yescrypt_kdf_body(local, passwd, passwdlen, salt, saltlen, flags | YESCRYPT_ALLOC_ONLY, N, r, p, t, NROM, buf, buflen) != -3 @@ -1049,7 +1034,7 @@ int yescrypt_kdf(const yescrypt_shared_t *shared, errno = EINVAL; return -1; } - retval = yescrypt_kdf_body(shared, local, + retval = yescrypt_kdf_body(local, passwd, passwdlen, salt, saltlen, flags | YESCRYPT_PREHASH, N >> 6, r, p, 0, NROM, dk, sizeof(dk)); @@ -1059,7 +1044,7 @@ int yescrypt_kdf(const yescrypt_shared_t *shared, passwdlen = sizeof(dk); } - retval = yescrypt_kdf_body(shared, local, + retval = yescrypt_kdf_body(local, passwd, passwdlen, salt, saltlen, flags, N, r, p, t, NROM, buf, buflen); #ifndef SKIP_MEMZERO diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 9755ac4206..09638e3e11 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -41,7 +41,6 @@ typedef struct { /** * Types for shared (ROM) and thread-local (RAM) data structures. */ -typedef yescrypt_region_t yescrypt_shared_t; typedef yescrypt_region_t yescrypt_local_t; /** @@ -80,8 +79,6 @@ typedef uint32_t yescrypt_flags_t; #define YESCRYPT_SBOX_192K 0x280 #define YESCRYPT_SBOX_384K 0x300 #define YESCRYPT_SBOX_768K 0x380 -/* Only valid for yescrypt_init_shared() */ -#define YESCRYPT_SHARED_PREALLOCATED 0x10000 #ifdef YESCRYPT_INTERNAL /* Private */ #define YESCRYPT_MODE_MASK 0x003 @@ -101,7 +98,6 @@ typedef uint32_t yescrypt_flags_t; #ifdef YESCRYPT_INTERNAL #define YESCRYPT_KNOWN_FLAGS \ (YESCRYPT_MODE_MASK | YESCRYPT_RW_FLAVOR_MASK | \ - YESCRYPT_SHARED_PREALLOCATED | \ YESCRYPT_INIT_SHARED | YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH) #endif @@ -139,47 +135,6 @@ typedef union { #define HASH_SIZE sizeof(yescrypt_binary_t) /* bytes */ #define HASH_LEN BYTES2CHARS(HASH_SIZE) - -/** - * yescrypt_init_shared(shared, seed, seedlen, params): - * Optionally allocate memory for and initialize the shared (ROM) data - * structure. The parameters flags, NROM, r, p, and t specify how the ROM is - * to be initialized, and seed and seedlen specify the initial seed affecting - * the data with which the ROM is filled. - * - * Return 0 on success; or -1 on error. - * - * If bit YESCRYPT_SHARED_PREALLOCATED in flags is set, then memory for the - * ROM is assumed to have been preallocated by the caller, with shared->aligned - * being the start address of the ROM and shared->aligned_size being its size - * (which must be sufficient for NROM, r, p). This may be used e.g. when the - * ROM is to be placed in a SysV shared memory segment allocated by the caller. - * - * MT-safe as long as shared is local to the thread. - */ -extern int yescrypt_init_shared(yescrypt_shared_t *shared, - const uint8_t *seed, size_t seedlen, const yescrypt_params_t *params); - -/** - * yescrypt_digest_shared(shared): - * Extract the previously stored message digest of the provided yescrypt ROM. - * - * Return pointer to the message digest on success; or NULL on error. - * - * MT-unsafe. - */ -extern yescrypt_binary_t *yescrypt_digest_shared(yescrypt_shared_t *shared); - -/** - * yescrypt_free_shared(shared): - * Free memory that had been allocated with yescrypt_init_shared(). - * - * Return 0 on success; or -1 on error. - * - * MT-safe as long as shared is local to the thread. - */ -extern int yescrypt_free_shared(yescrypt_shared_t *shared); - /** * yescrypt_init_local(local): * Initialize the thread-local (RAM) data structure. Actual memory allocation @@ -239,7 +194,7 @@ extern int yescrypt_free_local(yescrypt_local_t *local); * * MT-safe as long as local and buf are local to the thread. */ -extern int yescrypt_kdf(const yescrypt_shared_t *shared, +extern int yescrypt_kdf( yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, @@ -261,11 +216,10 @@ extern int yescrypt_kdf(const yescrypt_shared_t *shared, * * MT-safe as long as local and buf are local to the thread. */ -extern uint8_t *yescrypt_r(const yescrypt_shared_t *shared, +extern uint8_t *yescrypt_r( yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, - //KEY: const yescrypt_binary_t *key, uint8_t *buf, size_t buflen); extern const uint8_t *decode64(uint8_t *dst, size_t *dstlen, From 10196929da47461d1d684cc65662cbaf591412fe Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 01:10:53 +0200 Subject: [PATCH 094/277] libbb/yescrypt: remove unreachable support for scrypt hash ("$7$...") function old new delta decode64 136 174 +38 decode64_uint32 166 201 +35 atoi64 25 - -25 yescrypt_r 879 692 -187 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 2/1 up/down: 73/-212) Total: -139 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 144 +++++++++------------------ 1 file changed, 49 insertions(+), 95 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 9d7231f09a..bf2934bc9b 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -118,24 +118,6 @@ uint8_t *encode64(uint8_t *dst, size_t dstlen, return dst; } -static const uint8_t *decode64_uint32_fixed(uint32_t *dst, uint32_t dstbits, - const uint8_t *src) -{ - uint32_t bits; - - *dst = 0; - for (bits = 0; bits < dstbits; bits += 6) { - uint32_t c = atoi64(*src++); - if (c > 63) { - *dst = 0; - return NULL; - } - *dst |= c << bits; - } - - return src; -} - const uint8_t *decode64(uint8_t *dst, size_t *dstlen, const uint8_t *src, size_t srclen) { @@ -192,91 +174,73 @@ uint8_t *yescrypt_r( { unsigned char saltbin[64], hashbin[32]; const uint8_t *src, *saltstr, *salt; + const uint8_t *saltend; uint8_t *dst; size_t need, prefixlen, saltstrlen, saltlen; + uint32_t flavor, N_log2; yescrypt_params_t params = { .p = 1 }; - if (setting[0] != '$' || - (setting[1] != '7' && setting[1] != 'y') || - setting[2] != '$') - return NULL; + /* we assume setting starts with "$y$" (caller must ensure this) */ src = setting + 3; - if (setting[1] == '7') { - uint32_t N_log2 = atoi64(*src++); - if (N_log2 < 1 || N_log2 > 63) - return NULL; - params.N = (uint64_t)1 << N_log2; - - src = decode64_uint32_fixed(¶ms.r, 30, src); - if (!src) - return NULL; + src = decode64_uint32(&flavor, src, 0); + if (!src) + return NULL; - src = decode64_uint32_fixed(¶ms.p, 30, src); - if (!src) - return NULL; + if (flavor < YESCRYPT_RW) { + params.flags = flavor; + } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { + params.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); } else { - uint32_t flavor, N_log2; + return NULL; + } - src = decode64_uint32(&flavor, src, 0); - if (!src) - return NULL; + src = decode64_uint32(&N_log2, src, 1); + if (!src || N_log2 > 63) + return NULL; + params.N = (uint64_t)1 << N_log2; - if (flavor < YESCRYPT_RW) { - params.flags = flavor; - } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { - params.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); - } else { - return NULL; - } + src = decode64_uint32(¶ms.r, src, 1); + if (!src) + return NULL; - src = decode64_uint32(&N_log2, src, 1); - if (!src || N_log2 > 63) - return NULL; - params.N = (uint64_t)1 << N_log2; + if (*src != '$') { + uint32_t have; - src = decode64_uint32(¶ms.r, src, 1); + src = decode64_uint32(&have, src, 1); if (!src) return NULL; - if (*src != '$') { - uint32_t have; - - src = decode64_uint32(&have, src, 1); + if (have & 1) { + src = decode64_uint32(¶ms.p, src, 2); if (!src) return NULL; + } - if (have & 1) { - src = decode64_uint32(¶ms.p, src, 2); - if (!src) - return NULL; - } - - if (have & 2) { - src = decode64_uint32(¶ms.t, src, 1); - if (!src) - return NULL; - } - - if (have & 4) { - src = decode64_uint32(¶ms.g, src, 1); - if (!src) - return NULL; - } + if (have & 2) { + src = decode64_uint32(¶ms.t, src, 1); + if (!src) + return NULL; + } - if (have & 8) { - uint32_t NROM_log2; - src = decode64_uint32(&NROM_log2, src, 1); - if (!src || NROM_log2 > 63) - return NULL; - params.NROM = (uint64_t)1 << NROM_log2; - } + if (have & 4) { + src = decode64_uint32(¶ms.g, src, 1); + if (!src) + return NULL; } - if (*src++ != '$') - return NULL; + if (have & 8) { + uint32_t NROM_log2; + src = decode64_uint32(&NROM_log2, src, 1); + if (!src || NROM_log2 > 63) + return NULL; + params.NROM = (uint64_t)1 << NROM_log2; + } } + if (*src++ != '$') + return NULL; + prefixlen = src - setting; saltstr = src; @@ -286,23 +250,13 @@ uint8_t *yescrypt_r( else saltstrlen = strlen((char *)saltstr); - if (setting[1] == '7') { - salt = saltstr; - saltlen = saltstrlen; - } else { - const uint8_t *saltend; - - saltlen = sizeof(saltbin); - saltend = decode64(saltbin, &saltlen, saltstr, saltstrlen); - - if (!saltend || (size_t)(saltend - saltstr) != saltstrlen) - goto fail; + saltlen = sizeof(saltbin); + saltend = decode64(saltbin, &saltlen, saltstr, saltstrlen); - salt = saltbin; + if (!saltend || (size_t)(saltend - saltstr) != saltstrlen) + goto fail; - //KEY:if (key) - //KEY: yescrypt_sha256_cipher(saltbin, saltlen, key, ENC); - } + salt = saltbin; need = prefixlen + saltstrlen + 1 + HASH_LEN + 1; if (need > buflen || need < saltstrlen) From bf8231d11ddd49abcded8b18b63ddeaea49c7fc8 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 01:29:14 +0200 Subject: [PATCH 095/277] libbb/yescrypt: remove YESCRYPT_INIT_SHARED flag and code using it It's only used by libxcrypt-4.4.38/lib/alg-yescrypt-opt.c code (optimized code version?) which wasn't ported to busybox function old new delta yescrypt_kdf 449 442 -7 static.smix 755 739 -16 alloc_region 72 - -72 yescrypt_kdf_body 1467 1239 -228 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 0/3 up/down: 0/-323) Total: -323 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 46 ++++++++----------------------- libbb/yescrypt/alg-yescrypt.h | 3 +- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index da23c1b59d..6c11085142 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -731,9 +731,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, } Nloop_rw = 0; - if (flags & YESCRYPT_INIT_SHARED) - Nloop_rw = Nloop_all; - else if (flags & YESCRYPT_RW) + if (flags & YESCRYPT_RW) Nloop_rw = Nloop_all / p; Nchunk &= ~(uint32_t)1; /* round down to even */ @@ -872,19 +870,6 @@ static int yescrypt_kdf_body( V = NULL; V_size = (size_t)128 * r * N; need = V_size; - if (flags & YESCRYPT_INIT_SHARED) { - if (local->aligned_size < need) { - if (local->base || local->aligned || - local->base_size || local->aligned_size) - goto out_EINVAL; - if (!alloc_region(local, need)) - return -1; - } - if (flags & YESCRYPT_ALLOC_ONLY) - return -2; /* expected "failure" */ - V = (salsa20_blk_t *)local->aligned; - need = 0; - } B_size = (size_t)128 * r * p; need += B_size; if (need < B_size) @@ -899,25 +884,18 @@ static int yescrypt_kdf_body( if (need < S_size) goto out_EINVAL; } - if (flags & YESCRYPT_INIT_SHARED) { - if (!alloc_region(&tmp, need)) + init_region(&tmp); + if (local->aligned_size < need) { + if (free_region(local)) + return -1; + if (!alloc_region(local, need)) return -1; - B = (uint8_t *)tmp.aligned; - XY = (salsa20_blk_t *)((uint8_t *)B + B_size); - } else { - init_region(&tmp); - if (local->aligned_size < need) { - if (free_region(local)) - return -1; - if (!alloc_region(local, need)) - return -1; - } - if (flags & YESCRYPT_ALLOC_ONLY) - return -3; /* expected "failure" */ - B = (uint8_t *)local->aligned; - V = (salsa20_blk_t *)((uint8_t *)B + B_size); - XY = (salsa20_blk_t *)((uint8_t *)V + V_size); } + if (flags & YESCRYPT_ALLOC_ONLY) + return -3; /* expected "failure" */ + B = (uint8_t *)local->aligned; + V = (salsa20_blk_t *)((uint8_t *)B + B_size); + XY = (salsa20_blk_t *)((uint8_t *)V + V_size); S = NULL; if (flags & YESCRYPT_RW) S = (uint8_t *)XY + XY_size; @@ -1021,7 +999,7 @@ int yescrypt_kdf( return -1; } - if ((flags & (YESCRYPT_RW | YESCRYPT_INIT_SHARED)) == YESCRYPT_RW + if ((flags & YESCRYPT_RW) && p >= 1 && N / p >= 0x100 && N / p * r >= 0x20000 diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 09638e3e11..fb791e8998 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -83,7 +83,6 @@ typedef uint32_t yescrypt_flags_t; /* Private */ #define YESCRYPT_MODE_MASK 0x003 #define YESCRYPT_RW_FLAVOR_MASK 0x3fc -#define YESCRYPT_INIT_SHARED 0x01000000 #define YESCRYPT_ALLOC_ONLY 0x08000000 #define YESCRYPT_PREHASH 0x10000000 #endif @@ -98,7 +97,7 @@ typedef uint32_t yescrypt_flags_t; #ifdef YESCRYPT_INTERNAL #define YESCRYPT_KNOWN_FLAGS \ (YESCRYPT_MODE_MASK | YESCRYPT_RW_FLAVOR_MASK | \ - YESCRYPT_INIT_SHARED | YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH) + YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH) #endif /** From 3d9b965b4af3f497560cc17a371f0f45d0feeb87 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 01:35:23 +0200 Subject: [PATCH 096/277] libbb/yescrypt: remove unused yescrypt_binary_t Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt.h | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index fb791e8998..bb30da8a83 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -112,15 +112,6 @@ typedef struct { uint64_t NROM; } yescrypt_params_t; -/** - * A 256-bit yescrypt hash, or a hash encryption key (which may itself have - * been derived as a yescrypt hash of a human-specified key string). - */ -typedef union { - unsigned char uc[32]; - uint64_t u64[4]; -} yescrypt_binary_t; - /* How many chars base-64 encoded bytes require? */ #define BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) /* The /etc/passwd-style hash is "$" */ @@ -131,7 +122,7 @@ typedef union { */ #define PREFIX_LEN (3 + 8 * 6 + 1 + BYTES2CHARS(32)) -#define HASH_SIZE sizeof(yescrypt_binary_t) /* bytes */ +#define HASH_SIZE 32 #define HASH_LEN BYTES2CHARS(HASH_SIZE) /** From e189464fa7dac7c6b6ae5ee80d55ab05dc67b6d7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 02:39:07 +0200 Subject: [PATCH 097/277] libbb/yescrypt: make some functions static function old new delta static.yescrypt_kdf_body - 1166 +1166 yescrypt_r 692 1342 +650 atoi64 - 25 +25 decode64_uint32 201 166 -35 encode64 153 - -153 decode64 174 - -174 yescrypt_kdf 442 - -442 yescrypt_kdf_body 1239 - -1239 ------------------------------------------------------------------------------ (add/remove: 2/4 grow/shrink: 1/1 up/down: 1841/-2043) Total: -202 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 27 ++++++++++++++++++--------- libbb/yescrypt/alg-yescrypt-kdf.c | 1 + libbb/yescrypt/alg-yescrypt.h | 9 +++------ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index bf2934bc9b..c04e074bdc 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -18,7 +18,12 @@ * SUCH DAMAGE. */ -static inline uint32_t atoi64(uint8_t src) +/* Not inlining: + * decode64 fuinctions are only used to read + * yescrypt_params_t field, and convert salt ti binary - + * both of these are negligible compared to main hashing operation + */ +static NOINLINE uint32_t atoi64(uint8_t src) { static const uint8_t atoi64_partial[77] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, @@ -34,8 +39,9 @@ static inline uint32_t atoi64(uint8_t src) return 64; } -static const uint8_t *decode64_uint32(uint32_t *dst, - const uint8_t *src, uint32_t min) +static NOINLINE const uint8_t *decode64_uint32( + uint32_t *dst, + const uint8_t *src, uint32_t min) { uint32_t start = 0, end = 47, chars = 1, bits = 0; uint32_t c; @@ -70,8 +76,9 @@ static const uint8_t *decode64_uint32(uint32_t *dst, return NULL; } -static uint8_t *encode64_uint32_fixed(uint8_t *dst, size_t dstlen, - uint32_t src, uint32_t srcbits) +static uint8_t *encode64_uint32_fixed( + uint8_t *dst, size_t dstlen, + uint32_t src, uint32_t srcbits) { uint32_t bits; @@ -91,8 +98,9 @@ static uint8_t *encode64_uint32_fixed(uint8_t *dst, size_t dstlen, return dst; } -uint8_t *encode64(uint8_t *dst, size_t dstlen, - const uint8_t *src, size_t srclen) +static uint8_t *encode64( + uint8_t *dst, size_t dstlen, + const uint8_t *src, size_t srclen) { size_t i; @@ -118,8 +126,9 @@ uint8_t *encode64(uint8_t *dst, size_t dstlen, return dst; } -const uint8_t *decode64(uint8_t *dst, size_t *dstlen, - const uint8_t *src, size_t srclen) +static const uint8_t *decode64( + uint8_t *dst, size_t *dstlen, + const uint8_t *src, size_t srclen) { size_t dstpos = 0; diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 6c11085142..93938e69c7 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -976,6 +976,7 @@ static int yescrypt_kdf_body( * to this function are the same as those for yescrypt_kdf_body() above, with * the addition of g, which controls hash upgrades (0 for no upgrades so far). */ +static int yescrypt_kdf( yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index bb30da8a83..7f2b7f471b 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -184,12 +184,14 @@ extern int yescrypt_free_local(yescrypt_local_t *local); * * MT-safe as long as local and buf are local to the thread. */ -extern int yescrypt_kdf( +#ifdef YESCRYPT_INTERNAL +static int yescrypt_kdf( yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, const yescrypt_params_t *params, uint8_t *buf, size_t buflen); +#endif /** * yescrypt_r(shared, local, passwd, passwdlen, setting, key, buf, buflen): @@ -211,8 +213,3 @@ extern uint8_t *yescrypt_r( const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, uint8_t *buf, size_t buflen); - -extern const uint8_t *decode64(uint8_t *dst, size_t *dstlen, - const uint8_t *src, size_t srclen); -extern uint8_t *encode64(uint8_t *dst, size_t dstlen, - const uint8_t *src, size_t srclen); From 9b6e03fd094d0947b04e25617d1d322bfadf7466 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 03:03:23 +0200 Subject: [PATCH 098/277] libbb/yescrypt: remove unused yescrypt_region_t tmp Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 93938e69c7..3ee9bfa434 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -806,7 +806,6 @@ static int yescrypt_kdf_body( uint64_t NROM, uint8_t *buf, size_t buflen) { - yescrypt_region_t tmp; const salsa20_blk_t *VROM; size_t B_size, V_size, XY_size, need; uint8_t *B, *S; @@ -884,7 +883,6 @@ static int yescrypt_kdf_body( if (need < S_size) goto out_EINVAL; } - init_region(&tmp); if (local->aligned_size < need) { if (free_region(local)) return -1; @@ -956,11 +954,6 @@ static int yescrypt_kdf_body( explicit_bzero(dk, sizeof(dk)); } - if (free_region(&tmp)) { - explicit_bzero(buf, buflen); /* must preserve errno */ - return -1; - } - /* Success! */ return 0; From 1524540613427b22cc9acf9e8fb1748f5dd425f7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 03:22:46 +0200 Subject: [PATCH 099/277] libbb/yescrypt: remove unused variable Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index c04e074bdc..388bf1a12a 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -182,8 +182,7 @@ uint8_t *yescrypt_r( uint8_t *buf, size_t buflen) { unsigned char saltbin[64], hashbin[32]; - const uint8_t *src, *saltstr, *salt; - const uint8_t *saltend; + const uint8_t *src, *saltstr, *saltend; uint8_t *dst; size_t need, prefixlen, saltstrlen, saltlen; uint32_t flavor, N_log2; @@ -265,19 +264,15 @@ uint8_t *yescrypt_r( if (!saltend || (size_t)(saltend - saltstr) != saltstrlen) goto fail; - salt = saltbin; - need = prefixlen + saltstrlen + 1 + HASH_LEN + 1; if (need > buflen || need < saltstrlen) goto fail; - if (yescrypt_kdf(local, passwd, passwdlen, salt, saltlen, + if (yescrypt_kdf(local, passwd, passwdlen, saltbin, saltlen, ¶ms, hashbin, sizeof(hashbin))) goto fail; - dst = buf; - memcpy(dst, setting, prefixlen + saltstrlen); - dst += prefixlen + saltstrlen; + dst = mempcpy(buf, setting, prefixlen + saltstrlen); *dst++ = '$'; dst = encode64(dst, buflen - (dst - buf), hashbin, sizeof(hashbin)); From 1c977008e0b84f0d277367901b981c0d958dbdce Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 03:40:06 +0200 Subject: [PATCH 100/277] libbb/yescrypt: code shrink in parameter decoding function old new delta decode64_uint32 166 167 +1 yescrypt_r 1342 1311 -31 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 1/-31) Total: -30 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 48 ++++++++++------------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 388bf1a12a..65c92f22f6 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -41,33 +41,36 @@ static NOINLINE uint32_t atoi64(uint8_t src) static NOINLINE const uint8_t *decode64_uint32( uint32_t *dst, - const uint8_t *src, uint32_t min) + const uint8_t *src, uint32_t val) { uint32_t start = 0, end = 47, chars = 1, bits = 0; uint32_t c; + if (!src) /* prevous decode failed already? */ + goto fail; + c = atoi64(*src++); if (c > 63) goto fail; - *dst = min; while (c > end) { - *dst += (end + 1 - start) << bits; + val += (end + 1 - start) << bits; start = end + 1; end = start + (62 - end) / 2; chars++; bits += 6; } - *dst += (c - start) << bits; + val += (c - start) << bits; while (--chars) { c = atoi64(*src++); if (c > 63) goto fail; bits -= 6; - *dst += c << bits; + val += c << bits; } + *dst = val; return src; @@ -192,8 +195,8 @@ uint8_t *yescrypt_r( src = setting + 3; src = decode64_uint32(&flavor, src, 0); - if (!src) - return NULL; + //if (!src) + // return NULL; if (flavor < YESCRYPT_RW) { params.flags = flavor; @@ -204,48 +207,32 @@ uint8_t *yescrypt_r( } src = decode64_uint32(&N_log2, src, 1); - if (!src || N_log2 > 63) + if (/*!src ||*/ N_log2 > 63) return NULL; params.N = (uint64_t)1 << N_log2; src = decode64_uint32(¶ms.r, src, 1); if (!src) return NULL; - if (*src != '$') { uint32_t have; - src = decode64_uint32(&have, src, 1); - if (!src) - return NULL; - - if (have & 1) { + if (have & 1) src = decode64_uint32(¶ms.p, src, 2); - if (!src) - return NULL; - } - - if (have & 2) { + if (have & 2) src = decode64_uint32(¶ms.t, src, 1); - if (!src) - return NULL; - } - - if (have & 4) { + if (have & 4) src = decode64_uint32(¶ms.g, src, 1); - if (!src) - return NULL; - } - if (have & 8) { uint32_t NROM_log2; src = decode64_uint32(&NROM_log2, src, 1); - if (!src || NROM_log2 > 63) + if (/*!src ||*/ NROM_log2 > 63) return NULL; params.NROM = (uint64_t)1 << NROM_log2; } } - + if (!src) + return NULL; if (*src++ != '$') return NULL; @@ -274,7 +261,6 @@ uint8_t *yescrypt_r( dst = mempcpy(buf, setting, prefixlen + saltstrlen); *dst++ = '$'; - dst = encode64(dst, buflen - (dst - buf), hashbin, sizeof(hashbin)); explicit_bzero(hashbin, sizeof(hashbin)); if (!dst || dst >= buf + buflen) From 85d3c48217bcbbe670aa466705fee703b95cbc36 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 03:59:59 +0200 Subject: [PATCH 101/277] libbb/yescrypt: code shrink in salt decoding function old new delta yescrypt_r 1311 1288 -23 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 65c92f22f6..8d75fa051a 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -233,33 +233,29 @@ uint8_t *yescrypt_r( } if (!src) return NULL; - if (*src++ != '$') + if (*src != '$') return NULL; - prefixlen = src - setting; - - saltstr = src; - src = (uint8_t *)strrchr((char *)saltstr, '$'); - if (src) - saltstrlen = src - saltstr; - else - saltstrlen = strlen((char *)saltstr); + saltstr = src + 1; + src = (uint8_t *)strchrnul((char *)saltstr, '$'); + prefixlen = src - setting; /* len("$y$$") */ + saltstrlen = src - saltstr; /* len("") */ + /* src points to end of salt ('$' or NUL byte), won't be used past this point */ saltlen = sizeof(saltbin); saltend = decode64(saltbin, &saltlen, saltstr, saltstrlen); + if (saltend != saltstr + saltstrlen) + goto fail; /* saltbin[] is too small, or bad char during decode */ - if (!saltend || (size_t)(saltend - saltstr) != saltstrlen) - goto fail; - - need = prefixlen + saltstrlen + 1 + HASH_LEN + 1; - if (need > buflen || need < saltstrlen) + need = prefixlen + 1 + HASH_LEN + 1; + if (need > buflen || need < prefixlen) goto fail; if (yescrypt_kdf(local, passwd, passwdlen, saltbin, saltlen, ¶ms, hashbin, sizeof(hashbin))) goto fail; - dst = mempcpy(buf, setting, prefixlen + saltstrlen); + dst = mempcpy(buf, setting, prefixlen); *dst++ = '$'; dst = encode64(dst, buflen - (dst - buf), hashbin, sizeof(hashbin)); explicit_bzero(hashbin, sizeof(hashbin)); From 736589f877be576274503fdcd066aff4b5c12949 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 06:05:08 +0200 Subject: [PATCH 102/277] libbb/yescrypt: reduce the number of function parameters function old new delta yescrypt_kdf32_body - 1077 +1077 static.smix 739 753 +14 yescrypt_init_local 34 - -34 yes_crypt 87 50 -37 yescrypt_free_local 49 - -49 yescrypt_r 1288 1217 -71 static.yescrypt_kdf_body 1166 - -1166 ------------------------------------------------------------------------------ (add/remove: 1/3 grow/shrink: 1/2 up/down: 1091/-1357) Total: -266 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 74 ++++++++---------- libbb/yescrypt/alg-yescrypt-kdf.c | 104 ++++++++++++------------- libbb/yescrypt/alg-yescrypt-platform.c | 37 ++++----- libbb/yescrypt/alg-yescrypt.h | 87 +++++++++------------ 4 files changed, 136 insertions(+), 166 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 8d75fa051a..ebc531b08c 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -179,62 +179,64 @@ static const uint8_t *decode64( } uint8_t *yescrypt_r( - yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, uint8_t *buf, size_t buflen) { - unsigned char saltbin[64], hashbin[32]; + yescrypt_ctx_t yctx[1]; + unsigned char hashbin32[32]; const uint8_t *src, *saltstr, *saltend; uint8_t *dst; - size_t need, prefixlen, saltstrlen, saltlen; + size_t need, prefixlen, saltstrlen; uint32_t flavor, N_log2; - yescrypt_params_t params = { .p = 1 }; + + memset(yctx, 0, sizeof(yctx)); + yctx->param.p = 1; /* we assume setting starts with "$y$" (caller must ensure this) */ src = setting + 3; src = decode64_uint32(&flavor, src, 0); //if (!src) - // return NULL; + // goto fail; if (flavor < YESCRYPT_RW) { - params.flags = flavor; + yctx->param.flags = flavor; } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { - params.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); + yctx->param.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); } else { - return NULL; + goto fail; } src = decode64_uint32(&N_log2, src, 1); if (/*!src ||*/ N_log2 > 63) - return NULL; - params.N = (uint64_t)1 << N_log2; + goto fail; + yctx->param.N = (uint64_t)1 << N_log2; - src = decode64_uint32(¶ms.r, src, 1); + src = decode64_uint32(&yctx->param.r, src, 1); if (!src) - return NULL; + goto fail; if (*src != '$') { uint32_t have; src = decode64_uint32(&have, src, 1); if (have & 1) - src = decode64_uint32(¶ms.p, src, 2); + src = decode64_uint32(&yctx->param.p, src, 2); if (have & 2) - src = decode64_uint32(¶ms.t, src, 1); + src = decode64_uint32(&yctx->param.t, src, 1); if (have & 4) - src = decode64_uint32(¶ms.g, src, 1); + src = decode64_uint32(&yctx->param.g, src, 1); if (have & 8) { uint32_t NROM_log2; src = decode64_uint32(&NROM_log2, src, 1); if (/*!src ||*/ NROM_log2 > 63) - return NULL; - params.NROM = (uint64_t)1 << NROM_log2; + goto fail; + yctx->param.NROM = (uint64_t)1 << NROM_log2; } } if (!src) - return NULL; + goto fail; if (*src != '$') - return NULL; + goto fail; saltstr = src + 1; src = (uint8_t *)strchrnul((char *)saltstr, '$'); @@ -242,8 +244,8 @@ uint8_t *yescrypt_r( saltstrlen = src - saltstr; /* len("") */ /* src points to end of salt ('$' or NUL byte), won't be used past this point */ - saltlen = sizeof(saltbin); - saltend = decode64(saltbin, &saltlen, saltstr, saltstrlen); + yctx->saltlen = sizeof(yctx->salt); + saltend = decode64(yctx->salt, &yctx->saltlen, saltstr, saltstrlen); if (saltend != saltstr + saltstrlen) goto fail; /* saltbin[] is too small, or bad char during decode */ @@ -251,34 +253,22 @@ uint8_t *yescrypt_r( if (need > buflen || need < prefixlen) goto fail; - if (yescrypt_kdf(local, passwd, passwdlen, saltbin, saltlen, - ¶ms, hashbin, sizeof(hashbin))) + if (yescrypt_kdf32(yctx, passwd, passwdlen, hashbin32)) goto fail; dst = mempcpy(buf, setting, prefixlen); *dst++ = '$'; - dst = encode64(dst, buflen - (dst - buf), hashbin, sizeof(hashbin)); - explicit_bzero(hashbin, sizeof(hashbin)); + dst = encode64(dst, buflen - (dst - buf), hashbin32, sizeof(hashbin32)); if (!dst || dst >= buf + buflen) - return NULL; + goto fail; *dst = 0; /* NUL termination */ - + ret: + free_region(yctx->local); + explicit_bzero(yctx, sizeof(yctx)); + explicit_bzero(hashbin32, sizeof(hashbin32)); return buf; - fail: - explicit_bzero(saltbin, sizeof(saltbin)); - explicit_bzero(hashbin, sizeof(hashbin)); - return NULL; -} - -int yescrypt_init_local(yescrypt_local_t *local) -{ - init_region(local); - return 0; -} - -int yescrypt_free_local(yescrypt_local_t *local) -{ - return free_region(local); + buf = NULL; + goto ret; } diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 3ee9bfa434..2b84564b9f 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -798,29 +798,27 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, * This optimized implementation currently limits N to the range from 4 to * 2^31, but other implementations might not. */ -static int yescrypt_kdf_body( - yescrypt_local_t *local, - const uint8_t *passwd, size_t passwdlen, - const uint8_t *salt, size_t saltlen, - yescrypt_flags_t flags, uint64_t N, uint32_t r, uint32_t p, uint32_t t, - uint64_t NROM, - uint8_t *buf, size_t buflen) +static int yescrypt_kdf32_body( + yescrypt_ctx_t *yctx, + const uint8_t *passwd, size_t passwdlen, + yescrypt_flags_t flags, uint64_t N, uint32_t t, + uint8_t *buf32) { const salsa20_blk_t *VROM; size_t B_size, V_size, XY_size, need; uint8_t *B, *S; salsa20_blk_t *V, *XY; uint8_t sha256[32]; - uint8_t dk[sizeof(sha256)], *dkp = buf; + uint8_t dk[sizeof(sha256)], *dkp = buf32; /* Sanity-check parameters */ switch (flags & YESCRYPT_MODE_MASK) { case 0: /* classic scrypt - can't have anything non-standard */ - if (flags || t || NROM) + if (flags || t || yctx->param.NROM) goto out_EINVAL; break; case YESCRYPT_WORM: - if (flags != YESCRYPT_WORM || NROM) + if (flags != YESCRYPT_WORM || yctx->param.NROM) goto out_EINVAL; break; case YESCRYPT_RW: @@ -842,6 +840,9 @@ static int yescrypt_kdf_body( if (buflen > (((uint64_t)1 << 32) - 1) * 32) goto out_EINVAL; #endif + { + const uint32_t r = yctx->param.r; + const uint32_t p = yctx->param.p; if ((uint64_t)r * (uint64_t)p >= 1 << 30) goto out_EINVAL; if (N > UINT32_MAX) @@ -862,7 +863,7 @@ static int yescrypt_kdf_body( } VROM = NULL; - if (NROM) + if (yctx->param.NROM) goto out_EINVAL; /* Allocate memory */ @@ -883,15 +884,14 @@ static int yescrypt_kdf_body( if (need < S_size) goto out_EINVAL; } - if (local->aligned_size < need) { - if (free_region(local)) - return -1; - if (!alloc_region(local, need)) - return -1; + if (yctx->local->aligned_size < need) { + free_region(yctx->local); + alloc_region(yctx->local, need); + dbg("allocated local:%u", need); } if (flags & YESCRYPT_ALLOC_ONLY) return -3; /* expected "failure" */ - B = (uint8_t *)local->aligned; + B = (uint8_t *)yctx->local->aligned; V = (salsa20_blk_t *)((uint8_t *)B + B_size); XY = (salsa20_blk_t *)((uint8_t *)V + V_size); S = NULL; @@ -906,28 +906,28 @@ static int yescrypt_kdf_body( passwdlen = sizeof(sha256); } - PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, 1, B, B_size); + PBKDF2_SHA256(passwd, passwdlen, yctx->salt, yctx->saltlen, 1, B, B_size); if (flags) memcpy(sha256, B, sizeof(sha256)); if (p == 1 || (flags & YESCRYPT_RW)) { - smix(B, r, N, p, t, flags, V, NROM, VROM, XY, S, sha256); + smix(B, r, N, p, t, flags, V, yctx->param.NROM, VROM, XY, S, sha256); } else { uint32_t i; for (i = 0; i < p; i++) { smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, V, - NROM, VROM, XY, NULL, NULL); + yctx->param.NROM, VROM, XY, NULL, NULL); } } - dkp = buf; - if (flags && buflen < sizeof(dk)) { + dkp = buf32; + if (flags && /*buflen:*/32 < sizeof(dk)) { PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, dk, sizeof(dk)); dkp = dk; } - PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, buf, buflen); + PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, buf32, /*buflen:*/32); /* * Except when computing classic scrypt, allow all computation so far @@ -941,11 +941,11 @@ static int yescrypt_kdf_body( HMAC_SHA256_Buf(dkp, sizeof(dk), "Client Key", 10, sha256); /* Compute StoredKey */ { - size_t clen = buflen; + size_t clen = /*buflen:*/32; if (clen > sizeof(dk)) clen = sizeof(dk); SHA256_Buf(sha256, sizeof(sha256), dk); - memcpy(buf, dk, clen); + memcpy(buf32, dk, clen); } } @@ -960,6 +960,7 @@ static int yescrypt_kdf_body( out_EINVAL: errno = EINVAL; return -1; + } } /** @@ -970,21 +971,18 @@ static int yescrypt_kdf_body( * the addition of g, which controls hash upgrades (0 for no upgrades so far). */ static -int yescrypt_kdf( - yescrypt_local_t *local, +int yescrypt_kdf32( + yescrypt_ctx_t *yctx, const uint8_t *passwd, size_t passwdlen, - const uint8_t *salt, size_t saltlen, - const yescrypt_params_t *params, - uint8_t *buf, size_t buflen) + uint8_t *buf32) { - yescrypt_flags_t flags = params->flags; - uint64_t N = params->N; - uint32_t r = params->r; - uint32_t p = params->p; - uint32_t t = params->t; - uint32_t g = params->g; - uint64_t NROM = params->NROM; - uint8_t dk[32]; + yescrypt_flags_t flags = yctx->param.flags; + uint64_t N = yctx->param.N; + uint32_t r = yctx->param.r; + uint32_t p = yctx->param.p; + uint32_t t = yctx->param.t; + uint32_t g = yctx->param.g; + uint8_t dk32[32]; int retval; /* Support for hash upgrades has been temporarily removed */ @@ -998,30 +996,30 @@ int yescrypt_kdf( && N / p >= 0x100 && N / p * r >= 0x20000 ) { - if (yescrypt_kdf_body(local, - passwd, passwdlen, salt, saltlen, - flags | YESCRYPT_ALLOC_ONLY, N, r, p, t, NROM, - buf, buflen) != -3 + if (yescrypt_kdf32_body(yctx, + passwd, passwdlen, + flags | YESCRYPT_ALLOC_ONLY, N, t, + buf32) != -3 ) { errno = EINVAL; return -1; } - retval = yescrypt_kdf_body(local, - passwd, passwdlen, salt, saltlen, - flags | YESCRYPT_PREHASH, N >> 6, r, p, 0, NROM, - dk, sizeof(dk)); + retval = yescrypt_kdf32_body(yctx, + passwd, passwdlen, + flags | YESCRYPT_PREHASH, N >> 6, 0, + dk32); if (retval) return retval; - passwd = dk; - passwdlen = sizeof(dk); + passwd = dk32; + passwdlen = sizeof(dk32); } - retval = yescrypt_kdf_body(local, - passwd, passwdlen, salt, saltlen, - flags, N, r, p, t, NROM, buf, buflen); + retval = yescrypt_kdf32_body(yctx, + passwd, passwdlen, + flags, N, t, buf32); #ifndef SKIP_MEMZERO - if (passwd == dk) - explicit_bzero(dk, sizeof(dk)); + if (passwd == dk32) + explicit_bzero(dk32, sizeof(dk32)); #endif return retval; } diff --git a/libbb/yescrypt/alg-yescrypt-platform.c b/libbb/yescrypt/alg-yescrypt-platform.c index 09809c4b0f..41627df2d4 100644 --- a/libbb/yescrypt/alg-yescrypt-platform.c +++ b/libbb/yescrypt/alg-yescrypt-platform.c @@ -18,7 +18,7 @@ * SUCH DAMAGE. */ -static void *alloc_region(yescrypt_region_t *region, size_t size) +static void alloc_region(yescrypt_region_t *region, size_t size) { size_t base_size = size; uint8_t *base, *aligned; @@ -27,39 +27,32 @@ static void *alloc_region(yescrypt_region_t *region, size_t size) //(if defined(MAP_HUGETLB) && defined(MAP_HUGE_2MB)) using 2MB pages #else /* mmap not available */ base = aligned = NULL; - if (size + 63 < size) { - errno = ENOMEM; - } else { - base = malloc(size + 63); - if (base) { - aligned = base + 63; - aligned -= (uintptr_t)aligned & 63; - } + if (size + 63 < size) + bb_die_memory_exhausted(); + base = malloc(size + 63); + if (base) { + aligned = base + 63; + aligned -= (uintptr_t)aligned & 63; } #endif region->base = base; region->aligned = aligned; region->base_size = base ? base_size : 0; region->aligned_size = base ? size : 0; - return aligned; } -static inline void init_region(yescrypt_region_t *region) +static void free_region(yescrypt_region_t *region) { - region->base = region->aligned = NULL; - region->base_size = region->aligned_size = 0; -} - -static int free_region(yescrypt_region_t *region) -{ - if (region->base) { #if 0 //def MAP_ANON + if (region->base) { if (munmap(region->base, region->base_size)) return -1; + } #else - free(region->base); + free(region->base); #endif - } - init_region(region); - return 0; + region->base = NULL; + region->aligned = NULL; + region->base_size = 0; + region->aligned_size = 0; } diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 7f2b7f471b..cac1959f9c 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -27,21 +27,13 @@ * This file was originally written by Colin Percival as part of the Tarsnap * online backup system. */ - -/** - * Internal type used by the memory allocator. Please do not use it directly. - * Use yescrypt_shared_t and yescrypt_local_t as appropriate instead, since - * they might differ from each other in a future version. - */ -typedef struct { - void *base, *aligned; - size_t base_size, aligned_size; -} yescrypt_region_t; - -/** - * Types for shared (ROM) and thread-local (RAM) data structures. - */ -typedef yescrypt_region_t yescrypt_local_t; +#ifdef YESCRYPT_INTERNAL +# if 1 +# define dbg(...) ((void)0) +# else +# define dbg(...) bb_error_msg(__VA_ARGS__) +# endif +#endif /** * Two 64-bit tags placed 48 bytes to the end of a ROM in host byte endianness @@ -100,6 +92,16 @@ typedef uint32_t yescrypt_flags_t; YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH) #endif +/** + * Internal type used by the memory allocator. Please do not use it directly. + * Use yescrypt_shared_t and yescrypt_local_t as appropriate instead, since + * they might differ from each other in a future version. + */ +typedef struct { + void *base, *aligned; + size_t base_size, aligned_size; +} yescrypt_region_t; + /** * yescrypt parameters combined into one struct. N, r, p are the same as in * classic scrypt, except that the meaning of p changes when YESCRYPT_RW is @@ -112,6 +114,19 @@ typedef struct { uint64_t NROM; } yescrypt_params_t; +typedef struct { + yescrypt_params_t param; + + /* salt in binary form */ + /* stored here to cut down on the amont of function paramaters */ + unsigned char salt[64]; + size_t saltlen; + + /* used by the memory allocator */ + //yescrypt_region_t shared[1]; + yescrypt_region_t local[1]; +} yescrypt_ctx_t; + /* How many chars base-64 encoded bytes require? */ #define BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) /* The /etc/passwd-style hash is "$" */ @@ -125,28 +140,6 @@ typedef struct { #define HASH_SIZE 32 #define HASH_LEN BYTES2CHARS(HASH_SIZE) -/** - * yescrypt_init_local(local): - * Initialize the thread-local (RAM) data structure. Actual memory allocation - * is currently fully postponed until a call to yescrypt_kdf() or yescrypt_r(). - * - * Return 0 on success; or -1 on error. - * - * MT-safe as long as local is local to the thread. - */ -extern int yescrypt_init_local(yescrypt_local_t *local); - -/** - * yescrypt_free_local(local): - * Free memory that may have been allocated for an initialized thread-local - * (RAM) data structure. - * - * Return 0 on success; or -1 on error. - * - * MT-safe as long as local is local to the thread. - */ -extern int yescrypt_free_local(yescrypt_local_t *local); - /** * yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, params, * buf, buflen): @@ -180,17 +173,13 @@ extern int yescrypt_free_local(yescrypt_local_t *local); * and shared->aligned_size fields may optionally be set by the caller directly * (e.g., to a mapped SysV shm segment), without using yescrypt_init_shared(). * - * local must be initialized with yescrypt_init_local(). - * * MT-safe as long as local and buf are local to the thread. */ #ifdef YESCRYPT_INTERNAL -static int yescrypt_kdf( - yescrypt_local_t *local, - const uint8_t *passwd, size_t passwdlen, - const uint8_t *salt, size_t saltlen, - const yescrypt_params_t *params, - uint8_t *buf, size_t buflen); +static int yescrypt_kdf32( + yescrypt_ctx_t *yctx, + const uint8_t *passwd, size_t passwdlen, + uint8_t *buf32); #endif /** @@ -209,7 +198,7 @@ static int yescrypt_kdf( * MT-safe as long as local and buf are local to the thread. */ extern uint8_t *yescrypt_r( - yescrypt_local_t *local, - const uint8_t *passwd, size_t passwdlen, - const uint8_t *setting, - uint8_t *buf, size_t buflen); + const uint8_t *passwd, size_t passwdlen, + const uint8_t *setting, + uint8_t *buf, size_t buflen +); From 8be6dafe94690f24829901680c4768847a41b8f2 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 06:50:18 +0200 Subject: [PATCH 103/277] libbb/yescrypt: use mmap for allocation, it's large - 16Mbytes This automatically gives it alignment sufficient for any future SIMD optimizations. function old new delta yescrypt_r 1217 1221 +4 yescrypt_kdf32_body 1077 1059 -18 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 4/-18) Total: -14 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 22 +++++++++++++ libbb/yescrypt/alg-yescrypt-kdf.c | 3 +- libbb/yescrypt/alg-yescrypt-platform.c | 43 +++++++++----------------- libbb/yescrypt/alg-yescrypt.h | 13 +++----- 4 files changed, 43 insertions(+), 38 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index ebc531b08c..a54e21ce29 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -197,6 +197,9 @@ uint8_t *yescrypt_r( src = setting + 3; src = decode64_uint32(&flavor, src, 0); + dbg("yescrypt flavor=0x%x YESCRYPT_RW:%u", + (unsigned)flavor, !!(flavor & YESCRYPT_RW) + ); //if (!src) // goto fail; @@ -204,6 +207,22 @@ uint8_t *yescrypt_r( yctx->param.flags = flavor; } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { yctx->param.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); + dbg("yctx->param.flags=0x%x", (unsigned)yctx->param.flags); + dbg(" YESCRYPT_RW:%u" , !!(yctx->param.flags & YESCRYPT_RW )); + dbg(" YESCRYPT_ROUNDS_6:%u" , !!(yctx->param.flags & YESCRYPT_ROUNDS_6 )); + dbg(" YESCRYPT_GATHER_2:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_2 )); + dbg(" YESCRYPT_GATHER_4:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_4 )); + dbg(" YESCRYPT_GATHER_8:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_8 )); + dbg(" YESCRYPT_SIMPLE_2:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_2 )); + dbg(" YESCRYPT_SIMPLE_4:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_4 )); + dbg(" YESCRYPT_SIMPLE_8:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_8 )); + dbg(" YESCRYPT_SBOX_12K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_12K )); + dbg(" YESCRYPT_SBOX_24K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_24K )); + dbg(" YESCRYPT_SBOX_48K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_48K )); + dbg(" YESCRYPT_SBOX_96K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_96K )); + dbg(" YESCRYPT_SBOX_192K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_192K)); + dbg(" YESCRYPT_SBOX_384K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_384K)); + dbg(" YESCRYPT_SBOX_768K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_768K)); } else { goto fail; } @@ -212,13 +231,16 @@ uint8_t *yescrypt_r( if (/*!src ||*/ N_log2 > 63) goto fail; yctx->param.N = (uint64_t)1 << N_log2; + dbg("yctx->param.N=%llu (1<<%u)", (unsigned long long)yctx->param.N, (unsigned)N_log2); src = decode64_uint32(&yctx->param.r, src, 1); if (!src) goto fail; + dbg("yctx->param.r=%u", yctx->param.r); if (*src != '$') { uint32_t have; src = decode64_uint32(&have, src, 1); + dbg("yescrypt has extended params:0x%x", (unsigned)have); if (have & 1) src = decode64_uint32(&yctx->param.p, src, 2); if (have & 2) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 2b84564b9f..01a66a6a87 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -887,7 +887,8 @@ static int yescrypt_kdf32_body( if (yctx->local->aligned_size < need) { free_region(yctx->local); alloc_region(yctx->local, need); - dbg("allocated local:%u", need); + dbg("allocated local:%u 0x%x", need, need); + /* standard "j9T" params allocate 16Mbytes here */ } if (flags & YESCRYPT_ALLOC_ONLY) return -3; /* expected "failure" */ diff --git a/libbb/yescrypt/alg-yescrypt-platform.c b/libbb/yescrypt/alg-yescrypt-platform.c index 41627df2d4..8dd5feb550 100644 --- a/libbb/yescrypt/alg-yescrypt-platform.c +++ b/libbb/yescrypt/alg-yescrypt-platform.c @@ -20,39 +20,26 @@ static void alloc_region(yescrypt_region_t *region, size_t size) { - size_t base_size = size; - uint8_t *base, *aligned; - -#if 0 //def MAP_ANON - use mmap, possibly -//(if defined(MAP_HUGETLB) && defined(MAP_HUGE_2MB)) using 2MB pages -#else /* mmap not available */ - base = aligned = NULL; - if (size + 63 < size) + int flags = +# ifdef MAP_NOCORE /* huh? */ + MAP_NOCORE | +# endif + MAP_ANON | MAP_PRIVATE; + uint8_t *base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + if (base == MAP_FAILED) bb_die_memory_exhausted(); - base = malloc(size + 63); - if (base) { - aligned = base + 63; - aligned -= (uintptr_t)aligned & 63; - } -#endif - region->base = base; - region->aligned = aligned; - region->base_size = base ? base_size : 0; - region->aligned_size = base ? size : 0; + //region->base = base; + //region->base_size = size; + region->aligned = base; + region->aligned_size = size; } static void free_region(yescrypt_region_t *region) { -#if 0 //def MAP_ANON - if (region->base) { - if (munmap(region->base, region->base_size)) - return -1; - } -#else - free(region->base); -#endif - region->base = NULL; + if (region->aligned) + munmap(region->aligned, region->aligned_size); + //region->base = NULL; + //region->base_size = 0; region->aligned = NULL; - region->base_size = 0; region->aligned_size = 0; } diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index cac1959f9c..ebd705cf04 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -35,13 +35,6 @@ # endif #endif -/** - * Two 64-bit tags placed 48 bytes to the end of a ROM in host byte endianness - * (and followed by 32 bytes of the ROM digest). - */ -#define YESCRYPT_ROM_TAG1 0x7470797263736579 /* "yescrypt" */ -#define YESCRYPT_ROM_TAG2 0x687361684d4f522d /* "-ROMhash" */ - /** * Type and possible values for the flags argument of yescrypt_kdf(), * yescrypt_encode_params_r(), yescrypt_encode_params(). Most of these may be @@ -98,8 +91,10 @@ typedef uint32_t yescrypt_flags_t; * they might differ from each other in a future version. */ typedef struct { - void *base, *aligned; - size_t base_size, aligned_size; +// void *base; + void *aligned; +// size_t base_size; + size_t aligned_size; } yescrypt_region_t; /** From ffac25d056beda59d8a8f6506325f6b4b33cf5ad Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 07:43:04 +0200 Subject: [PATCH 104/277] libbb/yescrypt: comment tweaks Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 13 ++++++++----- libbb/yescrypt/alg-yescrypt.h | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index a54e21ce29..da7fa5e0f8 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -197,15 +197,15 @@ uint8_t *yescrypt_r( src = setting + 3; src = decode64_uint32(&flavor, src, 0); - dbg("yescrypt flavor=0x%x YESCRYPT_RW:%u", - (unsigned)flavor, !!(flavor & YESCRYPT_RW) - ); + /* "j9T" returns: 0x2f */ + dbg("yescrypt flavor=0x%x YESCRYPT_RW:%u", (unsigned)flavor, !!(flavor & YESCRYPT_RW)); //if (!src) // goto fail; if (flavor < YESCRYPT_RW) { yctx->param.flags = flavor; } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { + /* "j9T" sets flags to 0xb6 */ yctx->param.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); dbg("yctx->param.flags=0x%x", (unsigned)yctx->param.flags); dbg(" YESCRYPT_RW:%u" , !!(yctx->param.flags & YESCRYPT_RW )); @@ -231,12 +231,15 @@ uint8_t *yescrypt_r( if (/*!src ||*/ N_log2 > 63) goto fail; yctx->param.N = (uint64_t)1 << N_log2; + /* "j9T" sets to 4096 (1<<12) */ dbg("yctx->param.N=%llu (1<<%u)", (unsigned long long)yctx->param.N, (unsigned)N_log2); src = decode64_uint32(&yctx->param.r, src, 1); + /* "j9T" sets to 32 */ + dbg("yctx->param.r=%u", yctx->param.r); + if (!src) goto fail; - dbg("yctx->param.r=%u", yctx->param.r); if (*src != '$') { uint32_t have; src = decode64_uint32(&have, src, 1); @@ -269,7 +272,7 @@ uint8_t *yescrypt_r( yctx->saltlen = sizeof(yctx->salt); saltend = decode64(yctx->salt, &yctx->saltlen, saltstr, saltstrlen); if (saltend != saltstr + saltstrlen) - goto fail; /* saltbin[] is too small, or bad char during decode */ + goto fail; /* salt[] is too small, or bad char during decode */ need = prefixlen + 1 + HASH_LEN + 1; if (need > buflen || need < prefixlen) diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index ebd705cf04..edabbc2228 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -113,7 +113,7 @@ typedef struct { yescrypt_params_t param; /* salt in binary form */ - /* stored here to cut down on the amont of function paramaters */ + /* stored here to cut down on the amount of function paramaters */ unsigned char salt[64]; size_t saltlen; From 4e5a6b6dbb77f735c4f10b61dd32173ccc3a842a Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 10:35:22 +0200 Subject: [PATCH 105/277] libbb/yescrypt: remove extra sha256 implementation function old new delta libcperciva_HMAC_SHA256_Init - 159 +159 libcperciva_HMAC_SHA256_Final - 56 +56 SHA256_Buf - 40 +40 static.smix 753 759 +6 yescrypt_kdf32_body 1059 1060 +1 .rodata 105803 105799 -4 initial_state 32 - -32 libcperciva_SHA256_Init 37 - -37 static.cpu_to_be32_vect 51 - -51 _HMAC_SHA256_Final 55 - -55 PAD 64 - -64 libcperciva_HMAC_SHA256_Buf 132 58 -74 libcperciva_SHA256_Buf 86 - -86 SHA256_Pad_Almost 131 - -131 _SHA256_Final 195 - -195 _SHA256_Update 198 - -198 _HMAC_SHA256_Init 213 - -213 Krnd 256 - -256 PBKDF2_SHA256 1003 386 -617 SHA256_Transform 3083 - -3083 ------------------------------------------------------------------------------ (add/remove: 3/12 grow/shrink: 2/3 up/down: 262/-5096) Total: -4834 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-sha256.c | 498 +++--------------------------------- libbb/yescrypt/alg-sha256.h | 62 +---- 2 files changed, 39 insertions(+), 521 deletions(-) diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 0c1b846be1..038ac0ddbe 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -25,281 +25,6 @@ * SUCH DAMAGE. */ -#if defined(__GNUC__) -#define restrict __restrict -#else -#define restrict -#endif - -/* SHA256 round constants. */ -static const uint32_t Krnd[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 -}; - -/* Elementary functions used by SHA256 */ -#define Ch(x, y, z) ((x & (y ^ z)) ^ z) -#if 1 /* Explicit caching/reuse of common subexpression between rounds */ -#define Maj(x, y, z) (y ^ ((x_xor_y = x ^ y) & y_xor_z)) -#else /* Let the compiler cache/reuse or not */ -#define Maj(x, y, z) (y ^ ((x ^ y) & (y ^ z))) -#endif -#define SHR(x, n) (x >> n) -#define ROTR(x, n) ((x >> n) | (x << (32 - n))) -#define S0(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) -#define S1(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) -#define s0(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHR(x, 3)) -#define s1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10)) - -/* SHA256 round function */ -#define RND(a, b, c, d, e, f, g, h, k) \ - h += S1(e) + Ch(e, f, g) + k; \ - d += h; \ - h += S0(a) + Maj(a, b, c); \ - y_xor_z = x_xor_y; - -/* Adjusted round function for rotating state */ -#define RNDr(S, W, i, ii) \ - RND(S[(64 - i) % 8], S[(65 - i) % 8], \ - S[(66 - i) % 8], S[(67 - i) % 8], \ - S[(68 - i) % 8], S[(69 - i) % 8], \ - S[(70 - i) % 8], S[(71 - i) % 8], \ - W[i + ii] + Krnd[i + ii]) - -/* Message schedule computation */ -#define MSCH(W, ii, i) \ - W[i + ii + 16] = s1(W[i + ii + 14]) + W[i + ii + 9] + s0(W[i + ii + 1]) + W[i + ii] - -/* - * SHA256 block compression function. The 256-bit state is transformed via - * the 512-bit input block to produce a new state. - */ -static void -SHA256_Transform(uint32_t state[static restrict 8], - const uint8_t block[static restrict 64], - uint32_t W[static restrict 64], uint32_t S[static restrict 8]) -{ - int i; - - /* 1. Prepare the first part of the message schedule W. */ - be32dec_vect(W, block, 16); - - /* 2. Initialize working variables. */ - memcpy(S, state, 32); - - /* 3. Mix. */ - for (i = 0; i <= 48; i += 16) { - uint32_t x_xor_y, y_xor_z = S[(65 - i) % 8] ^ S[(66 - i) % 8]; - RNDr(S, W, 0, i); - RNDr(S, W, 1, i); - RNDr(S, W, 2, i); - RNDr(S, W, 3, i); - RNDr(S, W, 4, i); - RNDr(S, W, 5, i); - RNDr(S, W, 6, i); - RNDr(S, W, 7, i); - RNDr(S, W, 8, i); - RNDr(S, W, 9, i); - RNDr(S, W, 10, i); - RNDr(S, W, 11, i); - RNDr(S, W, 12, i); - RNDr(S, W, 13, i); - RNDr(S, W, 14, i); - RNDr(S, W, 15, i); - - if (i == 48) - break; - - MSCH(W, 0, i); - MSCH(W, 1, i); - MSCH(W, 2, i); - MSCH(W, 3, i); - MSCH(W, 4, i); - MSCH(W, 5, i); - MSCH(W, 6, i); - MSCH(W, 7, i); - MSCH(W, 8, i); - MSCH(W, 9, i); - MSCH(W, 10, i); - MSCH(W, 11, i); - MSCH(W, 12, i); - MSCH(W, 13, i); - MSCH(W, 14, i); - MSCH(W, 15, i); - } - - /* 4. Mix local working variables into global state. */ - state[0] += S[0]; - state[1] += S[1]; - state[2] += S[2]; - state[3] += S[3]; - state[4] += S[4]; - state[5] += S[5]; - state[6] += S[6]; - state[7] += S[7]; -} - -static const uint8_t PAD[64] = { - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; - -/* Add padding and terminating bit-count. */ -static void -SHA256_Pad(SHA256_CTX * ctx, uint32_t tmp32[static restrict 72]) -{ - size_t r; - - /* Figure out how many bytes we have buffered. */ - r = (ctx->count >> 3) & 0x3f; - - /* Pad to 56 mod 64, transforming if we finish a block en route. */ - if (r < 56) { - /* Pad to 56 mod 64. */ - memcpy(&ctx->buf[r], PAD, 56 - r); - } else { - /* Finish the current block and mix. */ - memcpy(&ctx->buf[r], PAD, 64 - r); - SHA256_Transform(ctx->state, ctx->buf, &tmp32[0], &tmp32[64]); - - /* The start of the final block is all zeroes. */ - memset(&ctx->buf[0], 0, 56); - } - - /* Add the terminating bit-count. */ - be64enc(&ctx->buf[56], ctx->count); - - /* Mix in the final block. */ - SHA256_Transform(ctx->state, ctx->buf, &tmp32[0], &tmp32[64]); -} - -/* Magic initialization constants. */ -static const uint32_t initial_state[8] = { - 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, - 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 -}; - -/** - * SHA256_Init(ctx): - * Initialize the SHA256 context ${ctx}. - */ -void -SHA256_Init(SHA256_CTX * ctx) -{ - - /* Zero bits processed so far. */ - ctx->count = 0; - - /* Initialize state. */ - memcpy(ctx->state, initial_state, sizeof(initial_state)); -} - -/** - * SHA256_Update(ctx, in, len): - * Input ${len} bytes from ${in} into the SHA256 context ${ctx}. - */ -static void -_SHA256_Update(SHA256_CTX * ctx, const void * in, size_t len, - uint32_t tmp32[static restrict 72]) -{ - uint32_t r; - const uint8_t * src = in; - - /* Return immediately if we have nothing to do. */ - if (len == 0) - return; - - /* Number of bytes left in the buffer from previous updates. */ - r = (ctx->count >> 3) & 0x3f; - - /* Update number of bits. */ - ctx->count += (uint64_t)(len) << 3; - - /* Handle the case where we don't need to perform any transforms. */ - if (len < 64 - r) { - memcpy(&ctx->buf[r], src, len); - return; - } - - /* Finish the current block. */ - memcpy(&ctx->buf[r], src, 64 - r); - SHA256_Transform(ctx->state, ctx->buf, &tmp32[0], &tmp32[64]); - src += 64 - r; - len -= 64 - r; - - /* Perform complete blocks. */ - while (len >= 64) { - SHA256_Transform(ctx->state, src, &tmp32[0], &tmp32[64]); - src += 64; - len -= 64; - } - - /* Copy left over data into buffer. */ - memcpy(ctx->buf, src, len); -} - -/* Wrapper function for intermediate-values sanitization. */ -void -SHA256_Update(SHA256_CTX * ctx, const void * in, size_t len) -{ - uint32_t tmp32[72]; - - /* Call the real function. */ - _SHA256_Update(ctx, in, len, tmp32); - - /* Clean the stack. */ - explicit_bzero(tmp32, 288); -} - -/** - * SHA256_Final(digest, ctx): - * Output the SHA256 hash of the data input to the context ${ctx} into the - * buffer ${digest}. - */ -static void -_SHA256_Final(uint8_t digest[32], SHA256_CTX * ctx, - uint32_t tmp32[static restrict 72]) -{ - - /* Add padding. */ - SHA256_Pad(ctx, tmp32); - - /* Write the hash. */ - be32enc_vect(digest, ctx->state, 8); -} - -/* Wrapper function for intermediate-values sanitization. */ -void -SHA256_Final(uint8_t digest[32], SHA256_CTX * ctx) -{ - uint32_t tmp32[72]; - - /* Call the real function. */ - _SHA256_Final(digest, ctx, tmp32); - - /* Clear the context state. */ - explicit_bzero(ctx, sizeof(SHA256_CTX)); - - /* Clean the stack. */ - explicit_bzero(tmp32, 288); -} - /** * SHA256_Buf(in, len, digest): * Compute the SHA256 hash of ${len} bytes from ${in} and write it to ${digest}. @@ -307,16 +32,10 @@ SHA256_Final(uint8_t digest[32], SHA256_CTX * ctx) void SHA256_Buf(const void * in, size_t len, uint8_t digest[32]) { - SHA256_CTX ctx; - uint32_t tmp32[72]; - - SHA256_Init(&ctx); - _SHA256_Update(&ctx, in, len, tmp32); - _SHA256_Final(digest, &ctx, tmp32); - - /* Clean the stack. */ - explicit_bzero(&ctx, sizeof(SHA256_CTX)); - explicit_bzero(tmp32, 288); + sha256_ctx_t ctx; + sha256_begin(&ctx); + sha256_hash(&ctx, in, len); + sha256_end(&ctx, digest); } /** @@ -325,52 +44,36 @@ SHA256_Buf(const void * in, size_t len, uint8_t digest[32]) * ${K}. */ static void -_HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen, - uint32_t tmp32[static restrict 72], uint8_t pad[static restrict 64], - uint8_t khash[static restrict 32]) +HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen) { + uint8_t pad[64]; + uint8_t khash[32]; const uint8_t * K = _K; size_t i; /* If Klen > 64, the key is really SHA256(K). */ if (Klen > 64) { - SHA256_Init(&ctx->ictx); - _SHA256_Update(&ctx->ictx, K, Klen, tmp32); - _SHA256_Final(khash, &ctx->ictx, tmp32); +// SHA256_Init(&ctx->ictx); +// _SHA256_Update(&ctx->ictx, K, Klen, tmp32); +// _SHA256_Final(khash, &ctx->ictx, tmp32); + SHA256_Buf(K, Klen, khash); K = khash; Klen = 32; } /* Inner SHA256 operation is SHA256(K xor [block of 0x36] || data). */ - SHA256_Init(&ctx->ictx); + sha256_begin(&ctx->ictx); memset(pad, 0x36, 64); for (i = 0; i < Klen; i++) pad[i] ^= K[i]; - _SHA256_Update(&ctx->ictx, pad, 64, tmp32); + sha256_hash(&ctx->ictx, pad, 64); /* Outer SHA256 operation is SHA256(K xor [block of 0x5c] || hash). */ - SHA256_Init(&ctx->octx); + sha256_begin(&ctx->octx); memset(pad, 0x5c, 64); for (i = 0; i < Klen; i++) pad[i] ^= K[i]; - _SHA256_Update(&ctx->octx, pad, 64, tmp32); -} - -/* Wrapper function for intermediate-values sanitization. */ -void -HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen) -{ - uint32_t tmp32[72]; - uint8_t pad[64]; - uint8_t khash[32]; - - /* Call the real function. */ - _HMAC_SHA256_Init(ctx, _K, Klen, tmp32, pad, khash); - - /* Clean the stack. */ - explicit_bzero(tmp32, 288); - explicit_bzero(khash, 32); - explicit_bzero(pad, 64); + sha256_hash(&ctx->octx, pad, 64); } /** @@ -378,25 +81,10 @@ HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen) * Input ${len} bytes from ${in} into the HMAC-SHA256 context ${ctx}. */ static void -_HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len, - uint32_t tmp32[static restrict 72]) -{ - - /* Feed data to the inner SHA256 operation. */ - _SHA256_Update(&ctx->ictx, in, len, tmp32); -} - -/* Wrapper function for intermediate-values sanitization. */ -void HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len) { - uint32_t tmp32[72]; - - /* Call the real function. */ - _HMAC_SHA256_Update(ctx, in, len, tmp32); - - /* Clean the stack. */ - explicit_bzero(tmp32, 288); + /* Feed data to the inner SHA256 operation. */ + sha256_hash(&ctx->ictx, in, len); } /** @@ -405,36 +93,16 @@ HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len) * buffer ${digest}. */ static void -_HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx, - uint32_t tmp32[static restrict 72], uint8_t ihash[static restrict 32]) +HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx) { + uint8_t ihash[32]; /* Finish the inner SHA256 operation. */ - _SHA256_Final(ihash, &ctx->ictx, tmp32); - + sha256_end(&ctx->ictx, ihash); /* Feed the inner hash to the outer SHA256 operation. */ - _SHA256_Update(&ctx->octx, ihash, 32, tmp32); - + sha256_hash(&ctx->octx, ihash, 32); /* Finish the outer SHA256 operation. */ - _SHA256_Final(digest, &ctx->octx, tmp32); -} - -/* Wrapper function for intermediate-values sanitization. */ -void -HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx) -{ - uint32_t tmp32[72]; - uint8_t ihash[32]; - - /* Call the real function. */ - _HMAC_SHA256_Final(digest, ctx, tmp32, ihash); - - /* Clear the context state. */ - explicit_bzero(ctx, sizeof(HMAC_SHA256_CTX)); - - /* Clean the stack. */ - explicit_bzero(tmp32, 288); - explicit_bzero(ihash, 32); + sha256_end(&ctx->octx, digest); } /** @@ -442,49 +110,14 @@ HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx) * Compute the HMAC-SHA256 of ${len} bytes from ${in} using the key ${K} of * length ${Klen}, and write the result to ${digest}. */ -void -HMAC_SHA256_Buf(const void * K, size_t Klen, const void * in, size_t len, - uint8_t digest[32]) +static void +HMAC_SHA256_Buf(const void *K, size_t Klen, const void *in, size_t len, + uint8_t digest[32]) { HMAC_SHA256_CTX ctx; - uint32_t tmp32[72]; - uint8_t tmp8[96]; - - _HMAC_SHA256_Init(&ctx, K, Klen, tmp32, &tmp8[0], &tmp8[64]); - _HMAC_SHA256_Update(&ctx, in, len, tmp32); - _HMAC_SHA256_Final(digest, &ctx, tmp32, &tmp8[0]); - - /* Clean the stack. */ - explicit_bzero(&ctx, sizeof(HMAC_SHA256_CTX)); - explicit_bzero(tmp32, 288); - explicit_bzero(tmp8, 96); -} - -/* Add padding and terminating bit-count, but don't invoke Transform yet. */ -static int -SHA256_Pad_Almost(SHA256_CTX * ctx, uint8_t len[static restrict 8], - uint32_t tmp32[static restrict 72]) -{ - uint32_t r; - - r = (ctx->count >> 3) & 0x3f; - if (r >= 56) - return -1; - - /* - * Convert length to a vector of bytes -- we do this now rather - * than later because the length will change after we pad. - */ - be64enc(len, ctx->count); - - /* Add 1--56 bytes so that the resulting length is 56 mod 64. */ - _SHA256_Update(ctx, PAD, 56 - r, tmp32); - - /* Add the terminating bit-count. */ - ctx->buf[63] = len[7]; - _SHA256_Update(ctx, len, 7, tmp32); - - return 0; + HMAC_SHA256_Init(&ctx, K, Klen); + HMAC_SHA256_Update(&ctx, in, len); + HMAC_SHA256_Final(digest, &ctx); } /** @@ -493,15 +126,11 @@ SHA256_Pad_Almost(SHA256_CTX * ctx, uint8_t len[static restrict 8], * write the output to buf. The value dkLen must be at most 32 * (2^32 - 1). */ void -PBKDF2_SHA256(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, - size_t saltlen, uint64_t c, uint8_t * buf, size_t dkLen) +PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, + const uint8_t *salt, size_t saltlen, + uint64_t c, uint8_t *buf, size_t dkLen) { HMAC_SHA256_CTX Phctx, PShctx, hctx; - uint32_t tmp32[72]; - union { - uint8_t tmp8[96]; - uint32_t state[8]; - } u; size_t i; uint8_t ivec[4]; uint8_t U[32]; @@ -513,54 +142,12 @@ PBKDF2_SHA256(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, /* Sanity-check. */ assert(dkLen <= 32 * (size_t)(UINT32_MAX)); - if (c == 1 && (dkLen & 31) == 0 && (saltlen & 63) <= 51) { - uint32_t oldcount; - uint8_t * ivecp; - - /* Compute HMAC state after processing P and S. */ - _HMAC_SHA256_Init(&hctx, passwd, passwdlen, - tmp32, &u.tmp8[0], &u.tmp8[64]); - _HMAC_SHA256_Update(&hctx, salt, saltlen, tmp32); - - /* Prepare ictx padding. */ - oldcount = hctx.ictx.count & (0x3f << 3); - _HMAC_SHA256_Update(&hctx, "\0\0\0", 4, tmp32); - if ((hctx.ictx.count & (0x3f << 3)) < oldcount || - SHA256_Pad_Almost(&hctx.ictx, u.tmp8, tmp32)) - goto generic; /* Can't happen due to saltlen check */ - ivecp = hctx.ictx.buf + (oldcount >> 3); - - /* Prepare octx padding. */ - hctx.octx.count += 32 << 3; - SHA256_Pad_Almost(&hctx.octx, u.tmp8, tmp32); - - /* Iterate through the blocks. */ - for (i = 0; i * 32 < dkLen; i++) { - /* Generate INT(i + 1). */ - be32enc(ivecp, (uint32_t)(i + 1)); - - /* Compute U_1 = PRF(P, S || INT(i)). */ - memcpy(u.state, hctx.ictx.state, sizeof(u.state)); - SHA256_Transform(u.state, hctx.ictx.buf, - &tmp32[0], &tmp32[64]); - be32enc_vect(hctx.octx.buf, u.state, 8); - memcpy(u.state, hctx.octx.state, sizeof(u.state)); - SHA256_Transform(u.state, hctx.octx.buf, - &tmp32[0], &tmp32[64]); - be32enc_vect(&buf[i * 32], u.state, 8); - } - - goto cleanup; - } - -generic: /* Compute HMAC state after processing P. */ - _HMAC_SHA256_Init(&Phctx, passwd, passwdlen, - tmp32, &u.tmp8[0], &u.tmp8[64]); + HMAC_SHA256_Init(&Phctx, passwd, passwdlen); /* Compute HMAC state after processing P and S. */ memcpy(&PShctx, &Phctx, sizeof(HMAC_SHA256_CTX)); - _HMAC_SHA256_Update(&PShctx, salt, saltlen, tmp32); + HMAC_SHA256_Update(&PShctx, salt, saltlen); /* Iterate through the blocks. */ for (i = 0; i * 32 < dkLen; i++) { @@ -569,8 +156,8 @@ PBKDF2_SHA256(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, /* Compute U_1 = PRF(P, S || INT(i)). */ memcpy(&hctx, &PShctx, sizeof(HMAC_SHA256_CTX)); - _HMAC_SHA256_Update(&hctx, ivec, 4, tmp32); - _HMAC_SHA256_Final(T, &hctx, tmp32, u.tmp8); + HMAC_SHA256_Update(&hctx, ivec, 4); + HMAC_SHA256_Final(T, &hctx); if (c > 1) { /* T_i = U_1 ... */ @@ -579,8 +166,8 @@ PBKDF2_SHA256(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, for (j = 2; j <= c; j++) { /* Compute U_j. */ memcpy(&hctx, &Phctx, sizeof(HMAC_SHA256_CTX)); - _HMAC_SHA256_Update(&hctx, U, 32, tmp32); - _HMAC_SHA256_Final(U, &hctx, tmp32, u.tmp8); + HMAC_SHA256_Update(&hctx, U, 32); + HMAC_SHA256_Final(U, &hctx); /* ... xor U_j ... */ for (k = 0; k < 32; k++) @@ -594,15 +181,4 @@ PBKDF2_SHA256(const uint8_t * passwd, size_t passwdlen, const uint8_t * salt, clen = 32; memcpy(&buf[i * 32], T, clen); } - - /* Clean the stack. */ - explicit_bzero(&Phctx, sizeof(HMAC_SHA256_CTX)); - explicit_bzero(&PShctx, sizeof(HMAC_SHA256_CTX)); - explicit_bzero(U, 32); - explicit_bzero(T, 32); - -cleanup: - explicit_bzero(&hctx, sizeof(HMAC_SHA256_CTX)); - explicit_bzero(tmp32, 288); - explicit_bzero(&u, sizeof(u)); } diff --git a/libbb/yescrypt/alg-sha256.h b/libbb/yescrypt/alg-sha256.h index 1e75307d3b..8a49682673 100644 --- a/libbb/yescrypt/alg-sha256.h +++ b/libbb/yescrypt/alg-sha256.h @@ -28,43 +28,12 @@ * Use #defines in order to avoid namespace collisions with anyone else's * SHA256 code (e.g., the code in OpenSSL). */ -#define SHA256_Init libcperciva_SHA256_Init -#define SHA256_Update libcperciva_SHA256_Update -#define SHA256_Final libcperciva_SHA256_Final -#define SHA256_Buf libcperciva_SHA256_Buf -#define SHA256_CTX libcperciva_SHA256_CTX #define HMAC_SHA256_Init libcperciva_HMAC_SHA256_Init #define HMAC_SHA256_Update libcperciva_HMAC_SHA256_Update #define HMAC_SHA256_Final libcperciva_HMAC_SHA256_Final #define HMAC_SHA256_Buf libcperciva_HMAC_SHA256_Buf #define HMAC_SHA256_CTX libcperciva_HMAC_SHA256_CTX -/* Context structure for SHA256 operations. */ -typedef struct { - uint32_t state[8]; - uint64_t count; - uint8_t buf[64]; -} SHA256_CTX; - -/** - * SHA256_Init(ctx): - * Initialize the SHA256 context ${ctx}. - */ -extern void SHA256_Init(SHA256_CTX *); - -/** - * SHA256_Update(ctx, in, len): - * Input ${len} bytes from ${in} into the SHA256 context ${ctx}. - */ -extern void SHA256_Update(SHA256_CTX *, const void *, size_t); - -/** - * SHA256_Final(digest, ctx): - * Output the SHA256 hash of the data input to the context ${ctx} into the - * buffer ${digest}. - */ -extern void SHA256_Final(uint8_t[32], SHA256_CTX *); - /** * SHA256_Buf(in, len, digest): * Compute the SHA256 hash of ${len} bytes from ${in} and write it to ${digest}. @@ -73,37 +42,10 @@ extern void SHA256_Buf(const void *, size_t, uint8_t[32]); /* Context structure for HMAC-SHA256 operations. */ typedef struct { - SHA256_CTX ictx; - SHA256_CTX octx; + sha256_ctx_t ictx; + sha256_ctx_t octx; } HMAC_SHA256_CTX; -/** - * HMAC_SHA256_Init(ctx, K, Klen): - * Initialize the HMAC-SHA256 context ${ctx} with ${Klen} bytes of key from - * ${K}. - */ -extern void HMAC_SHA256_Init(HMAC_SHA256_CTX *, const void *, size_t); - -/** - * HMAC_SHA256_Update(ctx, in, len): - * Input ${len} bytes from ${in} into the HMAC-SHA256 context ${ctx}. - */ -extern void HMAC_SHA256_Update(HMAC_SHA256_CTX *, const void *, size_t); - -/** - * HMAC_SHA256_Final(digest, ctx): - * Output the HMAC-SHA256 of the data input to the context ${ctx} into the - * buffer ${digest}. - */ -extern void HMAC_SHA256_Final(uint8_t[32], HMAC_SHA256_CTX *); - -/** - * HMAC_SHA256_Buf(K, Klen, in, len, digest): - * Compute the HMAC-SHA256 of ${len} bytes from ${in} using the key ${K} of - * length ${Klen}, and write the result to ${digest}. - */ -extern void HMAC_SHA256_Buf(const void *, size_t, const void *, size_t, uint8_t[32]); - /** * PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, c, buf, dkLen): * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and From 62abd47815f0ee2f6c0ea6549fabe6d5c307ef8d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 10:50:46 +0200 Subject: [PATCH 106/277] Move "sha256-hash a memory array and produce the digest" helper to libbb Signed-off-by: Denys Vlasenko --- include/libbb.h | 1 + libbb/yescrypt/alg-sha256.c | 26 +++++--------------------- libbb/yescrypt/alg-sha256.h | 6 ------ libbb/yescrypt/alg-yescrypt-kdf.c | 2 +- 4 files changed, 7 insertions(+), 28 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 9a0a2f9167..270a9d5939 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2233,6 +2233,7 @@ enum { SHA512_OUTSIZE = 64, SHA3_OUTSIZE = 28, }; +void FAST_FUNC sha256_block(const void *in, size_t len, uint8_t hash[32]); extern uint32_t *global_crc32_table; uint32_t *crc32_filltable(uint32_t *tbl256, int endian) FAST_FUNC; diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 038ac0ddbe..315c094a29 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -25,38 +25,22 @@ * SUCH DAMAGE. */ -/** - * SHA256_Buf(in, len, digest): - * Compute the SHA256 hash of ${len} bytes from ${in} and write it to ${digest}. - */ -void -SHA256_Buf(const void * in, size_t len, uint8_t digest[32]) -{ - sha256_ctx_t ctx; - sha256_begin(&ctx); - sha256_hash(&ctx, in, len); - sha256_end(&ctx, digest); -} - /** * HMAC_SHA256_Init(ctx, K, Klen): * Initialize the HMAC-SHA256 context ${ctx} with ${Klen} bytes of key from * ${K}. */ static void -HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen) +HMAC_SHA256_Init(HMAC_SHA256_CTX *ctx, const void *_K, size_t Klen) { uint8_t pad[64]; uint8_t khash[32]; - const uint8_t * K = _K; + const uint8_t *K = _K; size_t i; /* If Klen > 64, the key is really SHA256(K). */ if (Klen > 64) { -// SHA256_Init(&ctx->ictx); -// _SHA256_Update(&ctx->ictx, K, Klen, tmp32); -// _SHA256_Final(khash, &ctx->ictx, tmp32); - SHA256_Buf(K, Klen, khash); + sha256_block(K, Klen, khash); K = khash; Klen = 32; } @@ -81,7 +65,7 @@ HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen) * Input ${len} bytes from ${in} into the HMAC-SHA256 context ${ctx}. */ static void -HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len) +HMAC_SHA256_Update(HMAC_SHA256_CTX *ctx, const void *in, size_t len) { /* Feed data to the inner SHA256 operation. */ sha256_hash(&ctx->ictx, in, len); @@ -93,7 +77,7 @@ HMAC_SHA256_Update(HMAC_SHA256_CTX * ctx, const void * in, size_t len) * buffer ${digest}. */ static void -HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX * ctx) +HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX *ctx) { uint8_t ihash[32]; diff --git a/libbb/yescrypt/alg-sha256.h b/libbb/yescrypt/alg-sha256.h index 8a49682673..6d2cc0a04d 100644 --- a/libbb/yescrypt/alg-sha256.h +++ b/libbb/yescrypt/alg-sha256.h @@ -34,12 +34,6 @@ #define HMAC_SHA256_Buf libcperciva_HMAC_SHA256_Buf #define HMAC_SHA256_CTX libcperciva_HMAC_SHA256_CTX -/** - * SHA256_Buf(in, len, digest): - * Compute the SHA256 hash of ${len} bytes from ${in} and write it to ${digest}. - */ -extern void SHA256_Buf(const void *, size_t, uint8_t[32]); - /* Context structure for HMAC-SHA256 operations. */ typedef struct { sha256_ctx_t ictx; diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 01a66a6a87..5c1f1006a6 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -945,7 +945,7 @@ static int yescrypt_kdf32_body( size_t clen = /*buflen:*/32; if (clen > sizeof(dk)) clen = sizeof(dk); - SHA256_Buf(sha256, sizeof(sha256), dk); + sha256_block(sha256, sizeof(sha256), dk); memcpy(buf32, dk, clen); } } From b21cd481db85a193677bc956ab14c6418711d8e4 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 11:02:03 +0200 Subject: [PATCH 107/277] libbb/yescrypt: code shrink function old new delta libcperciva_HMAC_SHA256_Final 56 53 -3 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-sha256.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 315c094a29..a1d4275e69 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -72,19 +72,17 @@ HMAC_SHA256_Update(HMAC_SHA256_CTX *ctx, const void *in, size_t len) } /** - * HMAC_SHA256_Final(digest, ctx): + * HMAC_SHA256_Final(ctx, digest): * Output the HMAC-SHA256 of the data input to the context ${ctx} into the * buffer ${digest}. */ static void -HMAC_SHA256_Final(uint8_t digest[32], HMAC_SHA256_CTX *ctx) +HMAC_SHA256_Final(HMAC_SHA256_CTX *ctx, uint8_t digest[32]) { - uint8_t ihash[32]; - /* Finish the inner SHA256 operation. */ - sha256_end(&ctx->ictx, ihash); + sha256_end(&ctx->ictx, digest); /* using digest[] as scratch space */ /* Feed the inner hash to the outer SHA256 operation. */ - sha256_hash(&ctx->octx, ihash, 32); + sha256_hash(&ctx->octx, digest, 32); /* using digest[] as scratch space */ /* Finish the outer SHA256 operation. */ sha256_end(&ctx->octx, digest); } @@ -101,7 +99,7 @@ HMAC_SHA256_Buf(const void *K, size_t Klen, const void *in, size_t len, HMAC_SHA256_CTX ctx; HMAC_SHA256_Init(&ctx, K, Klen); HMAC_SHA256_Update(&ctx, in, len); - HMAC_SHA256_Final(digest, &ctx); + HMAC_SHA256_Final(&ctx, digest); } /** @@ -141,18 +139,16 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, /* Compute U_1 = PRF(P, S || INT(i)). */ memcpy(&hctx, &PShctx, sizeof(HMAC_SHA256_CTX)); HMAC_SHA256_Update(&hctx, ivec, 4); - HMAC_SHA256_Final(T, &hctx); + HMAC_SHA256_Final(&hctx, T); if (c > 1) { /* T_i = U_1 ... */ memcpy(U, T, 32); - for (j = 2; j <= c; j++) { /* Compute U_j. */ memcpy(&hctx, &Phctx, sizeof(HMAC_SHA256_CTX)); HMAC_SHA256_Update(&hctx, U, 32); - HMAC_SHA256_Final(U, &hctx); - + HMAC_SHA256_Final(&hctx, U); /* ... xor U_j ... */ for (k = 0; k < 32; k++) T[k] ^= U[k]; From 0893bc3bac8705b22679ad77f39ee56d3ba728c9 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 11:12:13 +0200 Subject: [PATCH 108/277] libbb/yescrypt: code shrink static.PBKDF2_SHA256 - 189 +189 HMAC_SHA256_Init - 159 +159 HMAC_SHA256_Buf - 58 +58 HMAC_SHA256_Final - 53 +53 i2a64 - 42 +42 yescrypt_r 1221 1215 -6 yescrypt_kdf32_body 1064 1046 -18 i64c 42 - -42 libcperciva_HMAC_SHA256_Final 53 - -53 libcperciva_HMAC_SHA256_Buf 58 - -58 ascii64 65 - -65 libcperciva_HMAC_SHA256_Init 159 - -159 PBKDF2_SHA256 386 - -386 ------------------------------------------------------------------------------ (add/remove: 5/6 grow/shrink: 0/2 up/down: 501/-787) Total: -286 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 2 ++ libbb/pw_encrypt.c | 12 +++++------- libbb/pw_encrypt_des.c | 8 ++++---- libbb/yescrypt/alg-sha256.c | 2 +- libbb/yescrypt/alg-sha256.h | 18 ------------------ libbb/yescrypt/alg-yescrypt-common.c | 2 +- libbb/yescrypt/y.c | 6 ------ 7 files changed, 13 insertions(+), 37 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 270a9d5939..e88499a80f 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2189,6 +2189,8 @@ char *decode_base64(char *dst, const char **pp_src) FAST_FUNC; char *decode_base32(char *dst, const char **pp_src) FAST_FUNC; void read_base64(FILE *src_stream, FILE *dst_stream, int flags) FAST_FUNC; +int FAST_FUNC i2a64(int i); + typedef struct md5_ctx_t { uint8_t wbuffer[64]; /* always correctly aligned for uint64_t */ void (*process_block)(struct md5_ctx_t*) FAST_FUNC; diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 1d530974ef..97dee7229e 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -13,11 +13,10 @@ #endif #include "libbb.h" -/* static const uint8_t ascii64[] ALIGN1 = +/* 0..63 -> * "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; */ - -static int i64c(int i) +int FAST_FUNC i2a64(int i) { i &= 0x3f; if (i == 0) @@ -45,8 +44,8 @@ int FAST_FUNC crypt_make_rand64encoded(char *p, int cnt /*, int x */) * It has no problem with visibly alternating lowest bit * but is also weak in cryptographic sense + needs div, * which needs more code (and slower) on many CPUs */ - *p++ = i64c(x >> 16); - *p++ = i64c(x >> 22); + *p++ = i2a64(x >> 16); + *p++ = i2a64(x >> 22); } while (--cnt); *p = '\0'; return x; @@ -120,8 +119,7 @@ static char* to64(char *s, unsigned v, int n) { while (--n >= 0) { - /* *s++ = ascii64[v & 0x3f]; */ - *s++ = i64c(v); + *s++ = i2a64(v); v >>= 6; } return s; diff --git a/libbb/pw_encrypt_des.c b/libbb/pw_encrypt_des.c index fe8237cfef..c836ab6848 100644 --- a/libbb/pw_encrypt_des.c +++ b/libbb/pw_encrypt_des.c @@ -703,10 +703,10 @@ to64_msb_first(char *s, unsigned v) *s++ = ascii64[(v >> 6) & 0x3f]; /* bits 11..6 */ *s = ascii64[v & 0x3f]; /* bits 5..0 */ #endif - *s++ = i64c(v >> 18); /* bits 23..18 */ - *s++ = i64c(v >> 12); /* bits 17..12 */ - *s++ = i64c(v >> 6); /* bits 11..6 */ - *s = i64c(v); /* bits 5..0 */ + *s++ = i2a64(v >> 18); /* bits 23..18 */ + *s++ = i2a64(v >> 12); /* bits 17..12 */ + *s++ = i2a64(v >> 6); /* bits 11..6 */ + *s = i2a64(v); /* bits 5..0 */ } static char * diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index a1d4275e69..a17028b6b3 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -107,7 +107,7 @@ HMAC_SHA256_Buf(const void *K, size_t Klen, const void *in, size_t len, * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and * write the output to buf. The value dkLen must be at most 32 * (2^32 - 1). */ -void +static void PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, uint64_t c, uint8_t *buf, size_t dkLen) diff --git a/libbb/yescrypt/alg-sha256.h b/libbb/yescrypt/alg-sha256.h index 6d2cc0a04d..862f49dbe3 100644 --- a/libbb/yescrypt/alg-sha256.h +++ b/libbb/yescrypt/alg-sha256.h @@ -24,26 +24,8 @@ * SUCH DAMAGE. */ -/* - * Use #defines in order to avoid namespace collisions with anyone else's - * SHA256 code (e.g., the code in OpenSSL). - */ -#define HMAC_SHA256_Init libcperciva_HMAC_SHA256_Init -#define HMAC_SHA256_Update libcperciva_HMAC_SHA256_Update -#define HMAC_SHA256_Final libcperciva_HMAC_SHA256_Final -#define HMAC_SHA256_Buf libcperciva_HMAC_SHA256_Buf -#define HMAC_SHA256_CTX libcperciva_HMAC_SHA256_CTX - /* Context structure for HMAC-SHA256 operations. */ typedef struct { sha256_ctx_t ictx; sha256_ctx_t octx; } HMAC_SHA256_CTX; - -/** - * PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, c, buf, dkLen): - * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and - * write the output to buf. The value dkLen must be at most 32 * (2^32 - 1). - */ -extern void PBKDF2_SHA256(const uint8_t *, size_t, const uint8_t *, size_t, - uint64_t, uint8_t *, size_t); diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index da7fa5e0f8..7a1e92cab3 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -88,7 +88,7 @@ static uint8_t *encode64_uint32_fixed( for (bits = 0; bits < srcbits; bits += 6) { if (dstlen < 2) return NULL; - *dst++ = itoa64[src & 0x3f]; + *dst++ = i2a64(src); dstlen--; src >>= 6; } diff --git a/libbb/yescrypt/y.c b/libbb/yescrypt/y.c index 042c439a0c..2c6afd4f89 100644 --- a/libbb/yescrypt/y.c +++ b/libbb/yescrypt/y.c @@ -124,12 +124,6 @@ VECTOR_TO_CPU(be,32); VECTOR_TO_CPU(be,64); -const unsigned char ascii64[65] = - "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; -/* 0000000000111111111122222222223333333333444444444455555555556666 */ -/* 0123456789012345678901234567890123456789012345678901234567890123 */ -#define itoa64 ascii64 - #define YESCRYPT_INTERNAL #include "alg-sha256.h" #include "alg-yescrypt.h" From 9c4cd75d12894cd9139d549dfe445f18572ee7d1 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 11:16:43 +0200 Subject: [PATCH 109/277] restore mangled comment Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 97dee7229e..71f7731fd1 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -77,9 +77,9 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) // The "j9T$" below is the default "yescrypt parameters" encoded by yescrypt_encode_params_r(): // //shadow-4.17.4/src/passwd.c -// salt = crypt_make_rand64encoded(NULL, NULL); +// salt = crypt_make_salt(NULL, NULL); //shadow-4.17.4/lib/salt.c -//const char *crypt_make_rand64encoded(const char *meth, void *arg) +//const char *crypt_make_salt(const char *meth, void *arg) // if (streq(method, "YESCRYPT")) { // MAGNUM(result, 'y'); // salt_len = YESCRYPT_SALT_SIZE; // 24 From b8f76001662ad4b3073945589a0cb270b627c994 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 18:27:16 +0200 Subject: [PATCH 110/277] libbb/yescrypt: more efficient byteswapping function old new delta static.smix1 631 604 -27 static.smix2 452 420 -32 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-59) Total: -59 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-sha256.c | 10 +-- libbb/yescrypt/alg-yescrypt-kdf.c | 8 +- libbb/yescrypt/y.c | 122 ------------------------------ 3 files changed, 8 insertions(+), 132 deletions(-) diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index a17028b6b3..28d8c52960 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -114,16 +114,12 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, { HMAC_SHA256_CTX Phctx, PShctx, hctx; size_t i; - uint8_t ivec[4]; uint8_t U[32]; uint8_t T[32]; uint64_t j; int k; size_t clen; - /* Sanity-check. */ - assert(dkLen <= 32 * (size_t)(UINT32_MAX)); - /* Compute HMAC state after processing P. */ HMAC_SHA256_Init(&Phctx, passwd, passwdlen); @@ -133,12 +129,14 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, /* Iterate through the blocks. */ for (i = 0; i * 32 < dkLen; i++) { + uint32_t ivec; + /* Generate INT(i + 1). */ - be32enc(ivec, (uint32_t)(i + 1)); + ivec = SWAP_BE32((uint32_t)(i + 1)); /* Compute U_1 = PRF(P, S || INT(i)). */ memcpy(&hctx, &PShctx, sizeof(HMAC_SHA256_CTX)); - HMAC_SHA256_Update(&hctx, ivec, 4); + HMAC_SHA256_Update(&hctx, &ivec, 4); HMAC_SHA256_Final(&hctx, T); if (c > 1) { diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 5c1f1006a6..ee8fb408ef 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -501,7 +501,7 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, yescrypt_flags_t flags, salsa20_blk_t *dst = &X[i]; size_t k; for (k = 0; k < 16; k++) - tmp->w[k] = le32dec((const uint8_t *) &src->w[k]); + tmp->w[k] = SWAP_LE32(src->w[k]); salsa20_simd_shuffle(tmp, dst); } @@ -591,7 +591,7 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, yescrypt_flags_t flags, salsa20_blk_t *dst = (salsa20_blk_t *)&B[i * 64]; size_t k; for (k = 0; k < 16; k++) - le32enc((uint8_t *)&tmp->w[k], src->w[k]); + tmp->w[k] = SWAP_LE32(src->w[k]); salsa20_simd_unshuffle(tmp, dst); } } @@ -621,7 +621,7 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, salsa20_blk_t *dst = &X[i]; size_t k; for (k = 0; k < 16; k++) - tmp->w[k] = le32dec((const uint8_t *)&src->w[k]); + tmp->w[k] = SWAP_LE32(src->w[k]); salsa20_simd_shuffle(tmp, dst); } @@ -676,7 +676,7 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, salsa20_blk_t *dst = (salsa20_blk_t *)&B[i * 64]; size_t k; for (k = 0; k < 16; k++) - le32enc((uint8_t *)&tmp->w[k], src->w[k]); + tmp->w[k] = SWAP_LE32(src->w[k]); salsa20_simd_unshuffle(tmp, dst); } } diff --git a/libbb/yescrypt/y.c b/libbb/yescrypt/y.c index 2c6afd4f89..92c6eb7a87 100644 --- a/libbb/yescrypt/y.c +++ b/libbb/yescrypt/y.c @@ -2,128 +2,6 @@ #include -#include - -static inline void -cpu_to_le32 (unsigned char *buf, uint32_t n) -{ - buf[0] = (unsigned char)((n & 0x000000FFu) >> 0); - buf[1] = (unsigned char)((n & 0x0000FF00u) >> 8); - buf[2] = (unsigned char)((n & 0x00FF0000u) >> 16); - buf[3] = (unsigned char)((n & 0xFF000000u) >> 24); -} -static inline uint32_t -le32_to_cpu (const unsigned char *buf) -{ - return ((((uint32_t)buf[0]) << 0) | - (((uint32_t)buf[1]) << 8) | - (((uint32_t)buf[2]) << 16) | - (((uint32_t)buf[3]) << 24) ); -} - -/* Alternative names used in code derived from Colin Percival's - cryptography libraries. */ -#define le32enc cpu_to_le32 -#define le32dec le32_to_cpu -#define le64enc cpu_to_le64 -#define le64dec le64_to_cpu - -#define be32enc cpu_to_be32 -#define be32dec be32_to_cpu -#define be64enc cpu_to_be64 -#define be64dec be64_to_cpu - -#define be32enc_vect cpu_to_be32_vect -#define be32dec_vect be32_to_cpu_vect -#define be64enc_vect cpu_to_be64_vect -#define be64dec_vect be64_to_cpu_vect - - -//USED ONY BY SHA256 for be32_to_cpu_vect(): -static inline void -cpu_to_be32(unsigned char *buf, uint32_t n) -{ - buf[0] = (unsigned char)((n & 0xFF000000u) >> 24); - buf[1] = (unsigned char)((n & 0x00FF0000u) >> 16); - buf[2] = (unsigned char)((n & 0x0000FF00u) >> 8); - buf[3] = (unsigned char)((n & 0x000000FFu) >> 0); -} -static inline void -cpu_to_be64 (unsigned char *buf, uint64_t n) -{ - buf[0] = (unsigned char)((n & 0xFF00000000000000ull) >> 56); - buf[1] = (unsigned char)((n & 0x00FF000000000000ull) >> 48); - buf[2] = (unsigned char)((n & 0x0000FF0000000000ull) >> 40); - buf[3] = (unsigned char)((n & 0x000000FF00000000ull) >> 32); - buf[4] = (unsigned char)((n & 0x00000000FF000000ull) >> 24); - buf[5] = (unsigned char)((n & 0x0000000000FF0000ull) >> 16); - buf[6] = (unsigned char)((n & 0x000000000000FF00ull) >> 8); - buf[7] = (unsigned char)((n & 0x00000000000000FFull) >> 0); -} -static inline uint32_t -be32_to_cpu (const unsigned char *buf) -{ - return ((((uint32_t)buf[0]) << 24) | - (((uint32_t)buf[1]) << 16) | - (((uint32_t)buf[2]) << 8) | - (((uint32_t)buf[3]) << 0) ); -} -static inline uint64_t -be64_to_cpu (const unsigned char *buf) -{ - return ((((uint64_t)buf[0]) << 56) | - (((uint64_t)buf[1]) << 48) | - (((uint64_t)buf[2]) << 40) | - (((uint64_t)buf[3]) << 32) | - (((uint64_t)buf[4]) << 24) | - (((uint64_t)buf[5]) << 16) | - (((uint64_t)buf[6]) << 8) | - (((uint64_t)buf[7]) << 0) ); -} -/* Template: Define a function named cpu_to__vect that - takes a vector SRC of LEN integers, each of type uint_t, and - writes them to the buffer DST in the endianness defined by END. - Caution: LEN is the number of vector elements, not the total size - of the buffers. */ -#define VECTOR_CPU_TO(end, bits) VECTOR_CPU_TO_(end##bits, uint##bits##_t) -#define VECTOR_CPU_TO_(prim, stype) \ - static inline void \ - cpu_to_##prim##_vect(uint8_t *dst, const stype *src, size_t len) \ - { \ - while (len) \ - { \ - cpu_to_##prim(dst, *src); \ - src += 1; \ - dst += sizeof(stype); \ - len -= 1; \ - } \ - } struct _swallow_semicolon -/* Template: Define a function named _to_cpu_vect that - reads a vector of LEN integers, each of type uint_t, from the - buffer SRC, in the endianness defined by END, and writes them to - the vector DST. Caution: LEN is the number of vector elements, not - the total size of the buffers. */ -#define VECTOR_TO_CPU(end, bits) VECTOR_TO_CPU_(end##bits, uint##bits##_t) -#define VECTOR_TO_CPU_(prim, dtype) \ - static inline void \ - prim##_to_cpu_vect(dtype *dst, const uint8_t *src, size_t len) \ - { \ - while (len) \ - { \ - *dst = prim##_to_cpu(src); \ - src += sizeof(dtype); \ - dst += 1; \ - len -= 1; \ - } \ - } struct _swallow_semicolon -/* These are the vectorized endianness-conversion functions that are - presently used. Add more as necessary. */ -VECTOR_CPU_TO(be,32); -VECTOR_CPU_TO(be,64); -VECTOR_TO_CPU(be,32); -VECTOR_TO_CPU(be,64); - - #define YESCRYPT_INTERNAL #include "alg-sha256.h" #include "alg-yescrypt.h" From f5af8aefeae7fd73167403dd407e9a14aead5948 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 18:35:37 +0200 Subject: [PATCH 111/277] libbb/yescrypt: shrink PBKDF2 function old new delta static.PBKDF2_SHA256 189 176 -13 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-sha256.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 28d8c52960..f56b905ada 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -77,7 +77,7 @@ HMAC_SHA256_Update(HMAC_SHA256_CTX *ctx, const void *in, size_t len) * buffer ${digest}. */ static void -HMAC_SHA256_Final(HMAC_SHA256_CTX *ctx, uint8_t digest[32]) +HMAC_SHA256_Final(HMAC_SHA256_CTX *ctx, void *digest) { /* Finish the inner SHA256 operation. */ sha256_end(&ctx->ictx, digest); /* using digest[] as scratch space */ @@ -93,8 +93,7 @@ HMAC_SHA256_Final(HMAC_SHA256_CTX *ctx, uint8_t digest[32]) * length ${Klen}, and write the result to ${digest}. */ static void -HMAC_SHA256_Buf(const void *K, size_t Klen, const void *in, size_t len, - uint8_t digest[32]) +HMAC_SHA256_Buf(const void *K, size_t Klen, const void *in, size_t len, void *digest) { HMAC_SHA256_CTX ctx; HMAC_SHA256_Init(&ctx, K, Klen); @@ -114,11 +113,6 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, { HMAC_SHA256_CTX Phctx, PShctx, hctx; size_t i; - uint8_t U[32]; - uint8_t T[32]; - uint64_t j; - int k; - size_t clen; /* Compute HMAC state after processing P. */ HMAC_SHA256_Init(&Phctx, passwd, passwdlen); @@ -128,14 +122,19 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, HMAC_SHA256_Update(&PShctx, salt, saltlen); /* Iterate through the blocks. */ - for (i = 0; i * 32 < dkLen; i++) { + for (i = 0; dkLen != 0; i++) { + uint64_t U[32 / 8]; + uint64_t T[32 / 8]; + uint64_t j; uint32_t ivec; + size_t clen; + int k; /* Generate INT(i + 1). */ ivec = SWAP_BE32((uint32_t)(i + 1)); /* Compute U_1 = PRF(P, S || INT(i)). */ - memcpy(&hctx, &PShctx, sizeof(HMAC_SHA256_CTX)); + hctx = PShctx; HMAC_SHA256_Update(&hctx, &ivec, 4); HMAC_SHA256_Final(&hctx, T); @@ -144,19 +143,20 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, memcpy(U, T, 32); for (j = 2; j <= c; j++) { /* Compute U_j. */ - memcpy(&hctx, &Phctx, sizeof(HMAC_SHA256_CTX)); + hctx = Phctx; HMAC_SHA256_Update(&hctx, U, 32); HMAC_SHA256_Final(&hctx, U); /* ... xor U_j ... */ - for (k = 0; k < 32; k++) + for (k = 0; k < 32 / 8; k++) T[k] ^= U[k]; } } /* Copy as many bytes as necessary into buf. */ - clen = dkLen - i * 32; + clen = dkLen; if (clen > 32) clen = 32; - memcpy(&buf[i * 32], T, clen); + buf = mempcpy(buf, T, clen); + dkLen -= clen; } } From 23b5527f5c400a300c56afa36a6a5abaa81adbb7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 19:08:57 +0200 Subject: [PATCH 112/277] libbb/yescrypt: use common ACSII char-to-64 conversion code function old new delta a2i64 - 46 +46 yescrypt_r 1215 1235 +20 decode64_uint32 167 177 +10 atoi64 25 - -25 ascii_to_bin 53 - -53 static.atoi64_partial 77 - -77 ------------------------------------------------------------------------------ (add/remove: 1/3 grow/shrink: 2/0 up/down: 76/-155) Total: -79 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 1 + libbb/pw_encrypt.c | 25 +++++++++++++++++++++++++ libbb/pw_encrypt_des.c | 23 ++--------------------- libbb/yescrypt/alg-yescrypt-common.c | 22 +++------------------- 4 files changed, 31 insertions(+), 40 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index e88499a80f..1c23d2f66c 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2190,6 +2190,7 @@ char *decode_base32(char *dst, const char **pp_src) FAST_FUNC; void read_base64(FILE *src_stream, FILE *dst_stream, int flags) FAST_FUNC; int FAST_FUNC i2a64(int i); +int FAST_FUNC a2i64(char c); typedef struct md5_ctx_t { uint8_t wbuffer[64]; /* always correctly aligned for uint64_t */ diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 71f7731fd1..af84606bf0 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -30,6 +30,31 @@ int FAST_FUNC i2a64(int i) return ('a' - 38 + i); } +/* Returns >=64 for invalid chars */ +int FAST_FUNC a2i64(char c) +{ + unsigned char ch = c; + if (ch >= 'a') + /* "a..z" to 38..63 */ + /* anything after "z": positive int >= 64 */ + return (ch - 'a' + 38); + + if (ch > 'Z') + /* after "Z" but before "a": positive byte >= 64 */ + return ch; + + if (ch >= 'A') + /* "A..Z" to 12..37 */ + return (ch - 'A' + 12); + + if (ch > '9') + return 64; + + /* "./0123456789" to 0,1,2..11 */ + /* anything before "." becomes positive byte >= 64 */ + return (unsigned char)(ch - '.'); +} + int FAST_FUNC crypt_make_rand64encoded(char *p, int cnt /*, int x */) { /* was: x += ... */ diff --git a/libbb/pw_encrypt_des.c b/libbb/pw_encrypt_des.c index c836ab6848..8b5edaaed4 100644 --- a/libbb/pw_encrypt_des.c +++ b/libbb/pw_encrypt_des.c @@ -200,25 +200,6 @@ static const uint32_t bits32[32] ALIGN4 = { static const uint8_t bits8[8] ALIGN1 = { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }; -static int -ascii_to_bin(char ch) -{ - if (ch > 'z') - return 0; - if (ch >= 'a') - return (ch - 'a' + 38); - if (ch > 'Z') - return 0; - if (ch >= 'A') - return (ch - 'A' + 12); - if (ch > '9') - return 0; - if (ch >= '.') - return (ch - '.'); - return 0; -} - - /* Static stuff that stays resident and doesn't change after * being initialized, and therefore doesn't need to be made * reentrant. */ @@ -740,8 +721,8 @@ des_crypt(struct des_ctx *ctx, char output[DES_OUT_BUFSIZE], */ output[0] = salt_str[0]; output[1] = salt_str[1]; - salt = (ascii_to_bin(salt_str[1]) << 6) - | ascii_to_bin(salt_str[0]); + salt = (a2i64(salt_str[1]) << 6) + | a2i64(salt_str[0]); setup_salt(ctx, salt); /* set ctx->saltbits for do_des() */ /* Do it. */ diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 7a1e92cab3..b9a5c51ac0 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -23,22 +23,6 @@ * yescrypt_params_t field, and convert salt ti binary - * both of these are negligible compared to main hashing operation */ -static NOINLINE uint32_t atoi64(uint8_t src) -{ - static const uint8_t atoi64_partial[77] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 64, 64, 64, 64, 64, 64, 64, - 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 64, 64, 64, 64, 64, 64, - 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 - }; - if (src >= '.' && src <= 'z') - return atoi64_partial[src - '.']; - return 64; -} - static NOINLINE const uint8_t *decode64_uint32( uint32_t *dst, const uint8_t *src, uint32_t val) @@ -49,7 +33,7 @@ static NOINLINE const uint8_t *decode64_uint32( if (!src) /* prevous decode failed already? */ goto fail; - c = atoi64(*src++); + c = a2i64(*src++); if (c > 63) goto fail; @@ -64,7 +48,7 @@ static NOINLINE const uint8_t *decode64_uint32( val += (c - start) << bits; while (--chars) { - c = atoi64(*src++); + c = a2i64(*src++); if (c > 63) goto fail; bits -= 6; @@ -138,7 +122,7 @@ static const uint8_t *decode64( while (dstpos <= *dstlen && srclen) { uint32_t value = 0, bits = 0; while (srclen--) { - uint32_t c = atoi64(*src); + uint32_t c = a2i64(*src); if (c > 63) { srclen = 0; break; From 82bbbd2e538c29a585a6cb00be2e999f50bf865c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 19:14:09 +0200 Subject: [PATCH 113/277] libbb: code shrink in DES crypt function old new delta pw_encrypt 941 945 +4 bits32 128 - -128 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 1/0 up/down: 4/-128) Total: -124 bytes Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt_des.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/libbb/pw_encrypt_des.c b/libbb/pw_encrypt_des.c index 8b5edaaed4..38c76a15c5 100644 --- a/libbb/pw_encrypt_des.c +++ b/libbb/pw_encrypt_des.c @@ -186,17 +186,6 @@ static const uint8_t pbox[32] ALIGN1 = { 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25 }; -static const uint32_t bits32[32] ALIGN4 = { - 0x80000000, 0x40000000, 0x20000000, 0x10000000, - 0x08000000, 0x04000000, 0x02000000, 0x01000000, - 0x00800000, 0x00400000, 0x00200000, 0x00100000, - 0x00080000, 0x00040000, 0x00020000, 0x00010000, - 0x00008000, 0x00004000, 0x00002000, 0x00001000, - 0x00000800, 0x00000400, 0x00000200, 0x00000100, - 0x00000080, 0x00000040, 0x00000020, 0x00000010, - 0x00000008, 0x00000004, 0x00000002, 0x00000001 -}; - static const uint8_t bits8[8] ALIGN1 = { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }; @@ -335,11 +324,18 @@ des_init(struct des_ctx *ctx, const struct const_des_ctx *cctx) int i, j, b, k, inbit, obit; uint32_t p; const uint32_t *bits28, *bits24; + uint32_t bits32[32]; if (!ctx) ctx = xmalloc(sizeof(*ctx)); const_ctx = cctx; + p = 0x80000000U; + for (i = 0; p; i++) { + bits32[i] = p; + p >>= 1; + } + #if USE_REPETITIVE_SPEEDUP old_rawkey0 = old_rawkey1 = 0; old_salt = 0; From 447eb6bf71cf9d73a23f15c11b62e9e886e712ff Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 19:30:58 +0200 Subject: [PATCH 114/277] libbb: in DES crypt, error out on invalid salt chars function old new delta des_crypt 1308 1327 +19 Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt_des.c | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/libbb/pw_encrypt_des.c b/libbb/pw_encrypt_des.c index 38c76a15c5..bfa039bb58 100644 --- a/libbb/pw_encrypt_des.c +++ b/libbb/pw_encrypt_des.c @@ -674,12 +674,6 @@ do_des(struct des_ctx *ctx, /*uint32_t l_in, uint32_t r_in,*/ uint32_t *l_out, u static void to64_msb_first(char *s, unsigned v) { -#if 0 - *s++ = ascii64[(v >> 18) & 0x3f]; /* bits 23..18 */ - *s++ = ascii64[(v >> 12) & 0x3f]; /* bits 17..12 */ - *s++ = ascii64[(v >> 6) & 0x3f]; /* bits 11..6 */ - *s = ascii64[v & 0x3f]; /* bits 5..0 */ -#endif *s++ = i2a64(v >> 18); /* bits 23..18 */ *s++ = i2a64(v >> 12); /* bits 17..12 */ *s++ = i2a64(v >> 6); /* bits 11..6 */ @@ -717,34 +711,19 @@ des_crypt(struct des_ctx *ctx, char output[DES_OUT_BUFSIZE], */ output[0] = salt_str[0]; output[1] = salt_str[1]; - salt = (a2i64(salt_str[1]) << 6) - | a2i64(salt_str[0]); + + salt = a2i64(salt_str[0]); + if (salt >= 64) + return NULL; /* bad salt char */ + salt |= (a2i64(salt_str[1]) << 6); + if (salt >= (64 << 6)) + return NULL; /* bad salt char */ setup_salt(ctx, salt); /* set ctx->saltbits for do_des() */ /* Do it. */ do_des(ctx, /*0, 0,*/ &r0, &r1, 25 /* count */); /* Now encode the result. */ -#if 0 -{ - uint32_t l = (r0 >> 8); - q = (uint8_t *)output + 2; - *q++ = ascii64[(l >> 18) & 0x3f]; /* bits 31..26 of r0 */ - *q++ = ascii64[(l >> 12) & 0x3f]; /* bits 25..20 of r0 */ - *q++ = ascii64[(l >> 6) & 0x3f]; /* bits 19..14 of r0 */ - *q++ = ascii64[l & 0x3f]; /* bits 13..8 of r0 */ - l = ((r0 << 16) | (r1 >> 16)); - *q++ = ascii64[(l >> 18) & 0x3f]; /* bits 7..2 of r0 */ - *q++ = ascii64[(l >> 12) & 0x3f]; /* bits 1..2 of r0 and 31..28 of r1 */ - *q++ = ascii64[(l >> 6) & 0x3f]; /* bits 27..22 of r1 */ - *q++ = ascii64[l & 0x3f]; /* bits 21..16 of r1 */ - l = r1 << 2; - *q++ = ascii64[(l >> 12) & 0x3f]; /* bits 15..10 of r1 */ - *q++ = ascii64[(l >> 6) & 0x3f]; /* bits 9..4 of r1 */ - *q++ = ascii64[l & 0x3f]; /* bits 3..0 of r1 + 00 */ - *q = 0; -} -#else /* Each call takes low-order 24 bits and stores 4 chars */ /* bits 31..8 of r0 */ to64_msb_first(output + 2, (r0 >> 8)); @@ -754,7 +733,6 @@ des_crypt(struct des_ctx *ctx, char output[DES_OUT_BUFSIZE], to64_msb_first(output + 10, (r1 << 8)); /* extra zero byte is encoded as '.', fixing it */ output[13] = '\0'; -#endif return output; } From aebe6f71320a168504ca56cf9064d6cad744d770 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 19:47:29 +0200 Subject: [PATCH 115/277] libbb/yescrypt: remove inlines (whcih gcc detects anyway) and dead ifderfs Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 93 ++++++++++++++++--------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index ee8fb408ef..329e30d1fa 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -42,17 +42,14 @@ #define unlikely(exp) (exp) #endif -#if 0 //def __SSE__ -... -#else /* !defined(__SSE2__) */ - typedef union { uint32_t w[16]; uint64_t d[8]; } salsa20_blk_t; -static inline void salsa20_simd_shuffle(const salsa20_blk_t *Bin, - salsa20_blk_t *Bout) +static void salsa20_simd_shuffle( + const salsa20_blk_t *Bin, + salsa20_blk_t *Bout) { #define COMBINE(out, in1, in2) \ Bout->d[out] = Bin->w[in1 * 2] | ((uint64_t)Bin->w[in2 * 2 + 1] << 32); @@ -67,8 +64,9 @@ static inline void salsa20_simd_shuffle(const salsa20_blk_t *Bin, #undef COMBINE } -static inline void salsa20_simd_unshuffle(const salsa20_blk_t *Bin, - salsa20_blk_t *Bout) +static void salsa20_simd_unshuffle( + const salsa20_blk_t *Bin, + salsa20_blk_t *Bout) { #define UNCOMBINE(out, in1, in2) \ Bout->w[out * 2] = Bin->d[in1]; \ @@ -83,6 +81,7 @@ static inline void salsa20_simd_unshuffle(const salsa20_blk_t *Bin, UNCOMBINE(7, 3, 1) #undef UNCOMBINE } + #define DECL_X \ salsa20_blk_t X; #define DECL_Y \ @@ -105,8 +104,9 @@ static inline void salsa20_simd_unshuffle(const salsa20_blk_t *Bin, * salsa20(B): * Apply the Salsa20 core to the provided block. */ -static inline void salsa20(salsa20_blk_t *restrict B, - salsa20_blk_t *restrict Bout, uint32_t doublerounds) +static void salsa20(salsa20_blk_t *restrict B, + salsa20_blk_t *restrict Bout, + uint32_t doublerounds) { salsa20_blk_t X; #define x X.w @@ -193,15 +193,15 @@ static inline void salsa20(salsa20_blk_t *restrict B, #define INTEGERIFY (uint32_t)X.d[0] -#endif /* !defined(__SSE2__) */ - /** * blockmix_salsa8(Bin, Bout, r): * Compute Bout = BlockMix_{salsa20/8, r}(Bin). The input Bin must be 128r * bytes in length; the output Bout must also be the same size. */ -static void blockmix_salsa8(const salsa20_blk_t *restrict Bin, - salsa20_blk_t *restrict Bout, size_t r) +static void blockmix_salsa8( + const salsa20_blk_t *restrict Bin, + salsa20_blk_t *restrict Bout, + size_t r) { size_t i; DECL_X @@ -213,9 +213,11 @@ static void blockmix_salsa8(const salsa20_blk_t *restrict Bin, } } -static uint32_t blockmix_salsa8_xor(const salsa20_blk_t *restrict Bin1, - const salsa20_blk_t *restrict Bin2, salsa20_blk_t *restrict Bout, - size_t r) +static uint32_t blockmix_salsa8_xor( + const salsa20_blk_t *restrict Bin1, + const salsa20_blk_t *restrict Bin2, + salsa20_blk_t *restrict Bout, + size_t r) { size_t i; DECL_X @@ -257,10 +259,6 @@ static uint32_t blockmix_salsa8_xor(const salsa20_blk_t *restrict Bin1, #define FORCE_REGALLOC_3 /* empty */ #define MAYBE_MEMORY_BARRIER /* empty */ -#if 0 //def __SSE2__ -... -#else /* !defined(__SSE2__) */ - #define PWXFORM_SIMD(x0, x1) { \ uint64_t x = x0 & Smask2; \ uint64_t *p0 = (uint64_t *)(S0 + (uint32_t)x); \ @@ -274,7 +272,6 @@ static uint32_t blockmix_salsa8_xor(const salsa20_blk_t *restrict Bin1, PWXFORM_SIMD(X.d[2], X.d[3]) \ PWXFORM_SIMD(X.d[4], X.d[5]) \ PWXFORM_SIMD(X.d[6], X.d[7]) -#endif /* * This offset helps address the 256-byte write block via the single-byte @@ -321,8 +318,11 @@ typedef struct { * Compute Bout = BlockMix_pwxform{salsa20/2, r, S}(Bin). The input Bin must * be 128r bytes in length; the output Bout must also be the same size. */ -static void blockmix(const salsa20_blk_t *restrict Bin, - salsa20_blk_t *restrict Bout, size_t r, pwxform_ctx_t *restrict ctx) +static void blockmix( + const salsa20_blk_t *restrict Bin, + salsa20_blk_t *restrict Bout, + size_t r, + pwxform_ctx_t *restrict ctx) { uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; @@ -353,8 +353,10 @@ static void blockmix(const salsa20_blk_t *restrict Bin, } static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, - const salsa20_blk_t *restrict Bin2, salsa20_blk_t *Bout, - size_t r, int Bin2_in_ROM, pwxform_ctx_t *restrict ctx) + const salsa20_blk_t *restrict Bin2, + salsa20_blk_t *Bout, + size_t r, int Bin2_in_ROM, + pwxform_ctx_t *restrict ctx) { uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; @@ -413,9 +415,11 @@ static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, return INTEGERIFY; } -static uint32_t blockmix_xor_save(salsa20_blk_t *restrict Bin1out, - salsa20_blk_t *restrict Bin2, - size_t r, pwxform_ctx_t *restrict ctx) +static uint32_t blockmix_xor_save( + salsa20_blk_t *restrict Bin1out, + salsa20_blk_t *restrict Bin2, + size_t r, + pwxform_ctx_t *restrict ctx) { uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; @@ -426,13 +430,6 @@ static uint32_t blockmix_xor_save(salsa20_blk_t *restrict Bin1out, /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; -#ifdef PREFETCH - PREFETCH(&Bin2[r], _MM_HINT_T0) - for (i = 0; i < r; i++) { - PREFETCH(&Bin2[i], _MM_HINT_T0) - } -#endif - XOR_X_2(Bin1out[r], Bin2[r]) DECL_SMASK2REG @@ -487,9 +484,12 @@ static inline uint32_t integerify(const salsa20_blk_t *B, size_t r) * The array V must be aligned to a multiple of 64 bytes, and arrays B and XY * to a multiple of at least 16 bytes. */ -static void smix1(uint8_t *B, size_t r, uint32_t N, yescrypt_flags_t flags, - salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, - salsa20_blk_t *XY, pwxform_ctx_t *ctx) +static void smix1(uint8_t *B, size_t r, uint32_t N, + yescrypt_flags_t flags, + salsa20_blk_t *V, + uint32_t NROM, const salsa20_blk_t *VROM, + salsa20_blk_t *XY, + pwxform_ctx_t *ctx) { size_t s = 2 * r; salsa20_blk_t *X = V, *Y = &V[s]; @@ -605,8 +605,11 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, yescrypt_flags_t flags, * 64 bytes, and arrays B and XY to a multiple of at least 16 bytes. */ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, - yescrypt_flags_t flags, salsa20_blk_t *V, uint32_t NROM, - const salsa20_blk_t *VROM, salsa20_blk_t *XY, pwxform_ctx_t *ctx) + yescrypt_flags_t flags, + salsa20_blk_t *V, + uint32_t NROM, const salsa20_blk_t *VROM, + salsa20_blk_t *XY, + pwxform_ctx_t *ctx) { size_t s = 2 * r; salsa20_blk_t *X = XY, *Y = &XY[s]; @@ -705,9 +708,11 @@ static uint64_t p2floor(uint64_t x) * might also result in cache bank conflicts). */ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, - yescrypt_flags_t flags, - salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, - salsa20_blk_t *XY, uint8_t *S, uint8_t *passwd) + yescrypt_flags_t flags, + salsa20_blk_t *V, + uint32_t NROM, const salsa20_blk_t *VROM, + salsa20_blk_t *XY, + uint8_t *S, uint8_t *passwd) { size_t s = 2 * r; uint32_t Nchunk; From 0574928c85630c28a48d08b01078031c356cf1cf Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 20:08:53 +0200 Subject: [PATCH 116/277] libbb/yescrypt: remove unused prefetching code Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 61 +++++++++---------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 329e30d1fa..781e1f0bbe 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -48,8 +48,8 @@ typedef union { } salsa20_blk_t; static void salsa20_simd_shuffle( - const salsa20_blk_t *Bin, - salsa20_blk_t *Bout) + const salsa20_blk_t *Bin, + salsa20_blk_t *Bout) { #define COMBINE(out, in1, in2) \ Bout->d[out] = Bin->w[in1 * 2] | ((uint64_t)Bin->w[in2 * 2 + 1] << 32); @@ -222,15 +222,6 @@ static uint32_t blockmix_salsa8_xor( size_t i; DECL_X -#ifdef PREFETCH - PREFETCH(&Bin2[r * 2 - 1], _MM_HINT_T0) - for (i = 0; i < r - 1; i++) { - PREFETCH(&Bin2[i * 2], _MM_HINT_T0) - PREFETCH(&Bin2[i * 2 + 1], _MM_HINT_T0) - } - PREFETCH(&Bin2[i * 2], _MM_HINT_T0) -#endif - XOR_X_2(Bin1[r * 2 - 1], Bin2[r * 2 - 1]) for (i = 0; i < r; i++) { XOR_X(Bin1[i * 2]) @@ -355,7 +346,7 @@ static void blockmix( static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, const salsa20_blk_t *restrict Bin2, salsa20_blk_t *Bout, - size_t r, int Bin2_in_ROM, + size_t r, pwxform_ctx_t *restrict ctx) { uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; @@ -366,22 +357,6 @@ static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; -#ifdef PREFETCH - if (Bin2_in_ROM) { - PREFETCH(&Bin2[r], _MM_HINT_NTA) - for (i = 0; i < r; i++) { - PREFETCH(&Bin2[i], _MM_HINT_NTA) - } - } else { - PREFETCH(&Bin2[r], _MM_HINT_T0) - for (i = 0; i < r; i++) { - PREFETCH(&Bin2[i], _MM_HINT_T0) - } - } -#else - (void)Bin2_in_ROM; /* unused */ -#endif - XOR_X_2(Bin1[r], Bin2[r]) DECL_SMASK2REG @@ -510,10 +485,10 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, const salsa20_blk_t *V_j; V_j = &VROM[(NROM - 1) * s]; - j = blockmix_xor(X, V_j, Y, r, 1, ctx) & (NROM - 1); + j = blockmix_xor(X, V_j, Y, r, ctx) & (NROM - 1); V_j = &VROM[j * s]; X = Y + s; - j = blockmix_xor(Y, V_j, X, r, 1, ctx); + j = blockmix_xor(Y, V_j, X, r, ctx); for (n = 2; n < N; n <<= 1) { uint32_t m = (n < N / 2) ? n : (N - 1 - n); @@ -522,10 +497,10 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, j += i - 1; V_j = &V[j * s]; Y = X + s; - j = blockmix_xor(X, V_j, Y, r, 0, ctx) & (NROM - 1); + j = blockmix_xor(X, V_j, Y, r, ctx) & (NROM - 1); V_j = &VROM[j * s]; X = Y + s; - j = blockmix_xor(Y, V_j, X, r, 1, ctx); + j = blockmix_xor(Y, V_j, X, r, ctx); } } n >>= 1; @@ -534,9 +509,9 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, j += N - 2 - n; V_j = &V[j * s]; Y = X + s; - j = blockmix_xor(X, V_j, Y, r, 0, ctx) & (NROM - 1); + j = blockmix_xor(X, V_j, Y, r, ctx) & (NROM - 1); V_j = &VROM[j * s]; - blockmix_xor(Y, V_j, XY, r, 1, ctx); + blockmix_xor(Y, V_j, XY, r, ctx); } else if (flags & YESCRYPT_RW) { uint32_t n; salsa20_blk_t *V_j; @@ -553,12 +528,12 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, j &= n - 1; j += i - 1; V_j = &V[j * s]; - j = blockmix_xor(X, V_j, Y, r, 0, ctx); + j = blockmix_xor(X, V_j, Y, r, ctx); j &= n - 1; j += i; V_j = &V[j * s]; X = Y + s; - j = blockmix_xor(Y, V_j, X, r, 0, ctx); + j = blockmix_xor(Y, V_j, X, r, ctx); } } n >>= 1; @@ -567,11 +542,11 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, j += N - 2 - n; V_j = &V[j * s]; Y = X + s; - j = blockmix_xor(X, V_j, Y, r, 0, ctx); + j = blockmix_xor(X, V_j, Y, r, ctx); j &= n - 1; j += N - 1 - n; V_j = &V[j * s]; - blockmix_xor(Y, V_j, XY, r, 0, ctx); + blockmix_xor(Y, V_j, XY, r, ctx); } else { N -= 2; do { @@ -641,14 +616,14 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, const salsa20_blk_t *VROM_j; j = blockmix_xor_save(X, V_j, r, ctx) & (NROM - 1); VROM_j = &VROM[j * s]; - j = blockmix_xor(X, VROM_j, X, r, 1, ctx) & (N - 1); + j = blockmix_xor(X, VROM_j, X, r, ctx) & (N - 1); } while (Nloop -= 2); } else if (VROM) { do { const salsa20_blk_t *V_j = &V[j * s]; - j = blockmix_xor(X, V_j, X, r, 0, ctx) & (NROM - 1); + j = blockmix_xor(X, V_j, X, r, ctx) & (NROM - 1); V_j = &VROM[j * s]; - j = blockmix_xor(X, V_j, X, r, 1, ctx) & (N - 1); + j = blockmix_xor(X, V_j, X, r, ctx) & (N - 1); } while (Nloop -= 2); } else if (flags & YESCRYPT_RW) { do { @@ -660,9 +635,9 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, } else if (ctx) { do { const salsa20_blk_t *V_j = &V[j * s]; - j = blockmix_xor(X, V_j, X, r, 0, ctx) & (N - 1); + j = blockmix_xor(X, V_j, X, r, ctx) & (N - 1); V_j = &V[j * s]; - j = blockmix_xor(X, V_j, X, r, 0, ctx) & (N - 1); + j = blockmix_xor(X, V_j, X, r, ctx) & (N - 1); } while (Nloop -= 2); } else { do { From f3a1b7be72a2f2699ba25ecdb2c93b90e109f725 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 20:59:00 +0200 Subject: [PATCH 117/277] libbb/yescrypt: remove one NOINLINE, add copyright headers, merge two source files function old new delta pw_encrypt 945 974 +29 yes_crypt 50 - -50 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 1/0 up/down: 29/-50) Total: -21 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 2 +- libbb/yescrypt/alg-yescrypt-kdf.c | 27 ++++++++++++++ libbb/yescrypt/alg-yescrypt-platform.c | 45 ----------------------- libbb/yescrypt/alg-yescrypt.h | 51 +++----------------------- libbb/yescrypt/y.c | 10 ++++- 5 files changed, 41 insertions(+), 94 deletions(-) delete mode 100644 libbb/yescrypt/alg-yescrypt-platform.c diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index b9a5c51ac0..435eaeccab 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -258,7 +258,7 @@ uint8_t *yescrypt_r( if (saltend != saltstr + saltstrlen) goto fail; /* salt[] is too small, or bad char during decode */ - need = prefixlen + 1 + HASH_LEN + 1; + need = prefixlen + 1 + YESCRYPT_HASH_LEN + 1; if (need > buflen || need < prefixlen) goto fail; diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 781e1f0bbe..27ef2caa4d 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -759,6 +759,33 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, } } +/* Allocator code */ + +static void alloc_region(yescrypt_region_t *region, size_t size) +{ + int flags = +# ifdef MAP_NOCORE /* huh? */ + MAP_NOCORE | +# endif + MAP_ANON | MAP_PRIVATE; + uint8_t *base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + if (base == MAP_FAILED) + bb_die_memory_exhausted(); + //region->base = base; + //region->base_size = size; + region->aligned = base; + region->aligned_size = size; +} + +static void free_region(yescrypt_region_t *region) +{ + if (region->aligned) + munmap(region->aligned, region->aligned_size); + //region->base = NULL; + //region->base_size = 0; + region->aligned = NULL; + region->aligned_size = 0; +} /** * yescrypt_kdf_body(shared, local, passwd, passwdlen, salt, saltlen, * flags, N, r, p, t, NROM, buf, buflen): diff --git a/libbb/yescrypt/alg-yescrypt-platform.c b/libbb/yescrypt/alg-yescrypt-platform.c deleted file mode 100644 index 8dd5feb550..0000000000 --- a/libbb/yescrypt/alg-yescrypt-platform.c +++ /dev/null @@ -1,45 +0,0 @@ -/*- - * Copyright 2013-2018,2022 Alexander Peslyak - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -static void alloc_region(yescrypt_region_t *region, size_t size) -{ - int flags = -# ifdef MAP_NOCORE /* huh? */ - MAP_NOCORE | -# endif - MAP_ANON | MAP_PRIVATE; - uint8_t *base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); - if (base == MAP_FAILED) - bb_die_memory_exhausted(); - //region->base = base; - //region->base_size = size; - region->aligned = base; - region->aligned_size = size; -} - -static void free_region(yescrypt_region_t *region) -{ - if (region->aligned) - munmap(region->aligned, region->aligned_size); - //region->base = NULL; - //region->base_size = 0; - region->aligned = NULL; - region->aligned_size = 0; -} diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index edabbc2228..5b442c2c94 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -64,6 +64,7 @@ typedef uint32_t yescrypt_flags_t; #define YESCRYPT_SBOX_192K 0x280 #define YESCRYPT_SBOX_384K 0x300 #define YESCRYPT_SBOX_768K 0x380 + #ifdef YESCRYPT_INTERNAL /* Private */ #define YESCRYPT_MODE_MASK 0x003 @@ -123,59 +124,17 @@ typedef struct { } yescrypt_ctx_t; /* How many chars base-64 encoded bytes require? */ -#define BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) +#define YESCRYPT_BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) /* The /etc/passwd-style hash is "$" */ /* * "$y$", up to 8 params of up to 6 chars each, '$', salt * Alternatively, but that's smaller: * "$7$", 3 params encoded as 1+5+5 chars, salt */ -#define PREFIX_LEN (3 + 8 * 6 + 1 + BYTES2CHARS(32)) - -#define HASH_SIZE 32 -#define HASH_LEN BYTES2CHARS(HASH_SIZE) +#define YESCRYPT_PREFIX_LEN (3 + 8 * 6 + 1 + YESCRYPT_BYTES2CHARS(32)) -/** - * yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, params, - * buf, buflen): - * Compute scrypt(passwd[0 .. passwdlen - 1], salt[0 .. saltlen - 1], N, r, - * p, buflen), or a revision of scrypt as requested by flags and shared, and - * write the result into buf. The parameters N, r, p, and buflen must satisfy - * the same conditions as with crypto_scrypt(). t controls computation time - * while not affecting peak memory usage (t = 0 is optimal unless higher N*r - * is not affordable while higher t is). g controls hash upgrades (g = 0 for - * no upgrades so far). shared and flags may request special modes. local is - * the thread-local data structure, allowing to preserve and reuse a memory - * allocation across calls, thereby reducing processing overhead. - * - * Return 0 on success; or -1 on error. - * - * Classic scrypt is available by setting shared = NULL, flags = 0, and t = 0. - * - * Setting YESCRYPT_WORM enables only minimal deviations from classic scrypt: - * support for the t parameter, and pre- and post-hashing. - * - * Setting YESCRYPT_RW fully enables yescrypt. As a side effect of differences - * between the algorithms, it also prevents p > 1 from growing the threads' - * combined processing time and memory allocation (like it did with classic - * scrypt and YESCRYPT_WORM), treating p as a divider rather than a multiplier. - * - * Passing a shared structure, with ROM contents previously computed by - * yescrypt_init_shared(), enables the use of ROM and requires YESCRYPT_RW. - * - * In order to allow for initialization of the ROM to be split into a separate - * program (or separate invocation of the same program), the shared->aligned - * and shared->aligned_size fields may optionally be set by the caller directly - * (e.g., to a mapped SysV shm segment), without using yescrypt_init_shared(). - * - * MT-safe as long as local and buf are local to the thread. - */ -#ifdef YESCRYPT_INTERNAL -static int yescrypt_kdf32( - yescrypt_ctx_t *yctx, - const uint8_t *passwd, size_t passwdlen, - uint8_t *buf32); -#endif +#define YESCRYPT_HASH_SIZE 32 +#define YESCRYPT_HASH_LEN YESCRYPT_BYTES2CHARS(YESCRYPT_HASH_SIZE) /** * yescrypt_r(shared, local, passwd, passwdlen, setting, key, buf, buflen): diff --git a/libbb/yescrypt/y.c b/libbb/yescrypt/y.c index 92c6eb7a87..e7d447531e 100644 --- a/libbb/yescrypt/y.c +++ b/libbb/yescrypt/y.c @@ -1,11 +1,17 @@ +/* + * The compilation unit for yescrypt-related code. + * + * Copyright (C) 2025 by Denys Vlasenko + * + * Licensed under GPLv2, see file LICENSE in this source tree. + */ //kbuild:lib-$(CONFIG_USE_BB_CRYPT_YES) += y.o -#include +#include "libbb.h" #define YESCRYPT_INTERNAL #include "alg-sha256.h" #include "alg-yescrypt.h" #include "alg-sha256.c" -#include "alg-yescrypt-platform.c" #include "alg-yescrypt-kdf.c" #include "alg-yescrypt-common.c" From 5e17e3c6f49cef45a86ed9438941ca2d4f6ae906 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 21:59:03 +0200 Subject: [PATCH 118/277] libbb: shuffle ascii64 code around, shrink i2a64() function old new delta num2str64_4chars_msb_first - 55 +55 num2str64_lsb_first - 33 +33 i2a64 42 25 -17 to64 33 - -33 to64_msb_first 55 - -55 ------------------------------------------------------------------------------ (add/remove: 2/2 grow/shrink: 0/1 up/down: 88/-105) Total: -17 bytes Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt.c | 74 +----------------------------------------- libbb/pw_encrypt_des.c | 15 ++------- libbb/pw_encrypt_md5.c | 4 +-- libbb/pw_encrypt_sha.c | 2 +- 4 files changed, 7 insertions(+), 88 deletions(-) diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index af84606bf0..4acc330395 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -13,68 +13,7 @@ #endif #include "libbb.h" -/* 0..63 -> - * "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - */ -int FAST_FUNC i2a64(int i) -{ - i &= 0x3f; - if (i == 0) - return '.'; - if (i == 1) - return '/'; - if (i < 12) - return ('0' - 2 + i); - if (i < 38) - return ('A' - 12 + i); - return ('a' - 38 + i); -} - -/* Returns >=64 for invalid chars */ -int FAST_FUNC a2i64(char c) -{ - unsigned char ch = c; - if (ch >= 'a') - /* "a..z" to 38..63 */ - /* anything after "z": positive int >= 64 */ - return (ch - 'a' + 38); - - if (ch > 'Z') - /* after "Z" but before "a": positive byte >= 64 */ - return ch; - - if (ch >= 'A') - /* "A..Z" to 12..37 */ - return (ch - 'A' + 12); - - if (ch > '9') - return 64; - - /* "./0123456789" to 0,1,2..11 */ - /* anything before "." becomes positive byte >= 64 */ - return (unsigned char)(ch - '.'); -} - -int FAST_FUNC crypt_make_rand64encoded(char *p, int cnt /*, int x */) -{ - /* was: x += ... */ - unsigned x = getpid() + monotonic_us(); - do { - /* x = (x*1664525 + 1013904223) % 2^32 generator is lame - * (low-order bit is not "random", etc...), - * but for our purposes it is good enough */ - x = x*1664525 + 1013904223; - /* BTW, Park and Miller's "minimal standard generator" is - * x = x*16807 % ((2^31)-1) - * It has no problem with visibly alternating lowest bit - * but is also weak in cryptographic sense + needs div, - * which needs more code (and slower) on many CPUs */ - *p++ = i2a64(x >> 16); - *p++ = i2a64(x >> 22); - } while (--cnt); - *p = '\0'; - return x; -} +#include "pw_ascii64.c" char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) { @@ -139,17 +78,6 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) } #if ENABLE_USE_BB_CRYPT - -static char* -to64(char *s, unsigned v, int n) -{ - while (--n >= 0) { - *s++ = i2a64(v); - v >>= 6; - } - return s; -} - /* * DES and MD5 crypt implementations are taken from uclibc. * They were modified to not use static buffers. diff --git a/libbb/pw_encrypt_des.c b/libbb/pw_encrypt_des.c index bfa039bb58..ca8aa9bccf 100644 --- a/libbb/pw_encrypt_des.c +++ b/libbb/pw_encrypt_des.c @@ -671,15 +671,6 @@ do_des(struct des_ctx *ctx, /*uint32_t l_in, uint32_t r_in,*/ uint32_t *l_out, u #define DES_OUT_BUFSIZE 21 -static void -to64_msb_first(char *s, unsigned v) -{ - *s++ = i2a64(v >> 18); /* bits 23..18 */ - *s++ = i2a64(v >> 12); /* bits 17..12 */ - *s++ = i2a64(v >> 6); /* bits 11..6 */ - *s = i2a64(v); /* bits 5..0 */ -} - static char * NOINLINE des_crypt(struct des_ctx *ctx, char output[DES_OUT_BUFSIZE], @@ -726,11 +717,11 @@ des_crypt(struct des_ctx *ctx, char output[DES_OUT_BUFSIZE], /* Now encode the result. */ /* Each call takes low-order 24 bits and stores 4 chars */ /* bits 31..8 of r0 */ - to64_msb_first(output + 2, (r0 >> 8)); + num2str64_4chars_msb_first(output + 2, (r0 >> 8)); /* bits 7..0 of r0 and 31..16 of r1 */ - to64_msb_first(output + 6, (r0 << 16) | (r1 >> 16)); + num2str64_4chars_msb_first(output + 6, (r0 << 16) | (r1 >> 16)); /* bits 15..0 of r1 and two zero bits (plus extra zero byte) */ - to64_msb_first(output + 10, (r1 << 8)); + num2str64_4chars_msb_first(output + 10, (r1 << 8)); /* extra zero byte is encoded as '.', fixing it */ output[13] = '\0'; diff --git a/libbb/pw_encrypt_md5.c b/libbb/pw_encrypt_md5.c index 1e52ecaea8..92d039f960 100644 --- a/libbb/pw_encrypt_md5.c +++ b/libbb/pw_encrypt_md5.c @@ -149,9 +149,9 @@ md5_crypt(char result[MD5_OUT_BUFSIZE], const unsigned char *pw, const unsigned final[16] = final[5]; for (i = 0; i < 5; i++) { unsigned l = (final[i] << 16) | (final[i+6] << 8) | final[i+12]; - p = to64(p, l, 4); + p = num2str64_lsb_first(p, l, 4); } - p = to64(p, final[11], 2); + p = num2str64_lsb_first(p, final[11], 2); *p = '\0'; /* Don't leave anything around in vm they could use. */ diff --git a/libbb/pw_encrypt_sha.c b/libbb/pw_encrypt_sha.c index 5457d7ab6a..5162939201 100644 --- a/libbb/pw_encrypt_sha.c +++ b/libbb/pw_encrypt_sha.c @@ -198,7 +198,7 @@ sha_crypt(/*const*/ char *key_data, /*const*/ char *salt_data) #define b64_from_24bit(B2, B1, B0, N) \ do { \ unsigned w = ((B2) << 16) | ((B1) << 8) | (B0); \ - resptr = to64(resptr, w, N); \ + resptr = num2str64_lsb_first(resptr, w, N); \ } while (0) if (_32or64 == 32) { /* sha256 */ unsigned i = 0; From 53de6e6150ea5538930e1963eb87ada153093ea0 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 6 Jul 2025 22:43:28 +0200 Subject: [PATCH 119/277] libbb/yescrypt: use common ascii64 encoding routine function old new delta num2str64_lsb_first 33 46 +13 yescrypt_r 1235 1133 -102 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/1 up/down: 13/-102) Total: -89 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 1 + libbb/yescrypt/alg-yescrypt-common.c | 98 ++++++++++------------------ libbb/yescrypt/alg-yescrypt.h | 4 +- 3 files changed, 37 insertions(+), 66 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 1c23d2f66c..b761b10917 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2191,6 +2191,7 @@ void read_base64(FILE *src_stream, FILE *dst_stream, int flags) FAST_FUNC; int FAST_FUNC i2a64(int i); int FAST_FUNC a2i64(char c); +char* FAST_FUNC num2str64_lsb_first(char *s, unsigned v, int n); typedef struct md5_ctx_t { uint8_t wbuffer[64]; /* always correctly aligned for uint64_t */ diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 435eaeccab..5bdf1893ef 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -19,8 +19,8 @@ */ /* Not inlining: - * decode64 fuinctions are only used to read - * yescrypt_params_t field, and convert salt ti binary - + * de/encode64 functions are only used to read + * yescrypt_params_t field, and convert salt to binary - * both of these are negligible compared to main hashing operation */ static NOINLINE const uint8_t *decode64_uint32( @@ -63,56 +63,6 @@ static NOINLINE const uint8_t *decode64_uint32( return NULL; } -static uint8_t *encode64_uint32_fixed( - uint8_t *dst, size_t dstlen, - uint32_t src, uint32_t srcbits) -{ - uint32_t bits; - - for (bits = 0; bits < srcbits; bits += 6) { - if (dstlen < 2) - return NULL; - *dst++ = i2a64(src); - dstlen--; - src >>= 6; - } - - if (src || dstlen < 1) - return NULL; - - *dst = 0; /* NUL terminate just in case */ - - return dst; -} - -static uint8_t *encode64( - uint8_t *dst, size_t dstlen, - const uint8_t *src, size_t srclen) -{ - size_t i; - - for (i = 0; i < srclen; ) { - uint8_t *dnext; - uint32_t value = 0, bits = 0; - do { - value |= (uint32_t)src[i++] << bits; - bits += 8; - } while (bits < 24 && i < srclen); - dnext = encode64_uint32_fixed(dst, dstlen, value, bits); - if (!dnext) - return NULL; - dstlen -= dnext - dst; - dst = dnext; - } - - if (dstlen < 1) - return NULL; - - *dst = 0; /* NUL terminate just in case */ - - return dst; -} - static const uint8_t *decode64( uint8_t *dst, size_t *dstlen, const uint8_t *src, size_t srclen) @@ -156,21 +106,43 @@ static const uint8_t *decode64( *dstlen = dstpos; return src; } - fail: *dstlen = 0; return NULL; } -uint8_t *yescrypt_r( +static char *encode64( + char *dst, size_t dstlen, + const uint8_t *src, size_t srclen) +{ + while (srclen) { + uint32_t value = 0, b = 0; + do { + value |= (uint32_t)(*src++ << b); + b += 8; + srclen--; + } while (srclen && b < 24); + + b >>= 3; /* number of bits to number of bytes */ + b++; /* 1, 2 or 3 bytes will become 2, 3 or 4 ascii64 chars */ + dstlen -= b; + if ((ssize_t)dstlen <= 0) + return NULL; + dst = num2str64_lsb_first(dst, value, b); + } + *dst = '\0'; + return dst; +} + +char *yescrypt_r( const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, - uint8_t *buf, size_t buflen) + char *buf, size_t buflen) { yescrypt_ctx_t yctx[1]; unsigned char hashbin32[32]; + char *dst; const uint8_t *src, *saltstr, *saltend; - uint8_t *dst; size_t need, prefixlen, saltstrlen; uint32_t flavor, N_log2; @@ -241,11 +213,11 @@ uint8_t *yescrypt_r( goto fail; yctx->param.NROM = (uint64_t)1 << NROM_log2; } + if (!src) + goto fail; + if (*src != '$') + goto fail; } - if (!src) - goto fail; - if (*src != '$') - goto fail; saltstr = src + 1; src = (uint8_t *)strchrnul((char *)saltstr, '$'); @@ -268,16 +240,14 @@ uint8_t *yescrypt_r( dst = mempcpy(buf, setting, prefixlen); *dst++ = '$'; dst = encode64(dst, buflen - (dst - buf), hashbin32, sizeof(hashbin32)); - if (!dst || dst >= buf + buflen) + if (!dst) goto fail; - - *dst = 0; /* NUL termination */ ret: free_region(yctx->local); explicit_bzero(yctx, sizeof(yctx)); explicit_bzero(hashbin32, sizeof(hashbin32)); return buf; -fail: + fail: buf = NULL; goto ret; } diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 5b442c2c94..996af333fa 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -151,8 +151,8 @@ typedef struct { * * MT-safe as long as local and buf are local to the thread. */ -extern uint8_t *yescrypt_r( +extern char *yescrypt_r( const uint8_t *passwd, size_t passwdlen, const uint8_t *setting, - uint8_t *buf, size_t buflen + char *buf, size_t buflen ); From a2a5db41a3ace98e7eff53aca231053be7717c66 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 00:04:10 +0200 Subject: [PATCH 120/277] libbb/pw_ascii64.c: add forgotten source file Signed-off-by: Denys Vlasenko --- libbb/pw_ascii64.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 libbb/pw_ascii64.c diff --git a/libbb/pw_ascii64.c b/libbb/pw_ascii64.c new file mode 100644 index 0000000000..3993932cab --- /dev/null +++ b/libbb/pw_ascii64.c @@ -0,0 +1,91 @@ +/* vi: set sw=4 ts=4: */ +/* + * Utility routines. + * + * Copyright (C) 1999-2004 by Erik Andersen + * + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ + +/* Returns >=64 for invalid chars */ +int FAST_FUNC a2i64(char c) +{ + unsigned char ch = c; + if (ch >= 'a') + /* "a..z" to 38..63 */ + /* anything after "z": positive int >= 64 */ + return (ch - 'a' + 38); + + if (ch > 'Z') + /* after "Z" but before "a": positive byte >= 64 */ + return ch; + + if (ch >= 'A') + /* "A..Z" to 12..37 */ + return (ch - 'A' + 12); + + if (ch > '9') + return 64; + + /* "./0123456789" to 0,1,2..11 */ + /* anything before "." becomes positive byte >= 64 */ + return (unsigned char)(ch - '.'); +} + +/* 0..63 -> + * "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + */ +int FAST_FUNC i2a64(int i) +{ + i &= 0x3f; + + i += '.'; + /* the above maps 0..11 to "./0123456789": + * ACSII codes of "./" are ('0'-2) and ('0'-1) */ + + if (i > '9') + i += ('A' - '9' - 1); + if (i > 'Z') + i += ('a' - 'Z' - 1); + return i; +} + +char* FAST_FUNC +num2str64_lsb_first(char *s, unsigned v, int n) +{ + while (--n >= 0) { + *s++ = i2a64(v); + v >>= 6; + } + return s; +} + +static void +num2str64_4chars_msb_first(char *s, unsigned v) +{ + *s++ = i2a64(v >> 18); /* bits 23..18 */ + *s++ = i2a64(v >> 12); /* bits 17..12 */ + *s++ = i2a64(v >> 6); /* bits 11..6 */ + *s = i2a64(v); /* bits 5..0 */ +} + +int FAST_FUNC crypt_make_rand64encoded(char *p, int cnt /*, int x */) +{ + /* was: x += ... */ + unsigned x = getpid() + monotonic_us(); + do { + /* x = (x*1664525 + 1013904223) % 2^32 generator is lame + * (low-order bit is not "random", etc...), + * but for our purposes it is good enough */ + x = x*1664525 + 1013904223; + /* BTW, Park and Miller's "minimal standard generator" is + * x = x*16807 % ((2^31)-1) + * It has no problem with visibly alternating lowest bit + * but is also weak in cryptographic sense + needs div, + * which needs more code (and slower) on many CPUs */ + *p++ = i2a64(x >> 16); + *p++ = i2a64(x >> 22); + } while (--cnt); + *p = '\0'; + return x; +} From e2091c98425e2e1615db7dcad9556c928ea5123d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 04:57:16 +0200 Subject: [PATCH 121/277] libbb: add two more forgotten source files Signed-off-by: Denys Vlasenko --- libbb/hash_sha256_block.c | 19 +++++++++++++++++++ libbb/pw_encrypt_yes.c | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 libbb/hash_sha256_block.c create mode 100644 libbb/pw_encrypt_yes.c diff --git a/libbb/hash_sha256_block.c b/libbb/hash_sha256_block.c new file mode 100644 index 0000000000..3c43663215 --- /dev/null +++ b/libbb/hash_sha256_block.c @@ -0,0 +1,19 @@ +/* vi: set sw=4 ts=4: */ +/* + * Utility routines. + * + * Copyright (C) 2025 Denys Vlasenko + * + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ +//kbuild:lib-y += hash_sha256_block.o +#include "libbb.h" + +void FAST_FUNC +sha256_block(const void *in, size_t len, uint8_t hash[32]) +{ + sha256_ctx_t ctx; + sha256_begin(&ctx); + sha256_hash(&ctx, in, len); + sha256_end(&ctx, hash); +} diff --git a/libbb/pw_encrypt_yes.c b/libbb/pw_encrypt_yes.c new file mode 100644 index 0000000000..50bd064189 --- /dev/null +++ b/libbb/pw_encrypt_yes.c @@ -0,0 +1,24 @@ +/* + * Utility routines. + * + * Copyright (C) 2025 by Denys Vlasenko + * + * Licensed under GPLv2, see file LICENSE in this source tree. + */ +#include "yescrypt/alg-yescrypt.h" + +static char * +yes_crypt(const char *passwd, const char *salt_data) +{ + /* prefix, '$', hash, NUL */ + char buf[YESCRYPT_PREFIX_LEN + 1 + YESCRYPT_HASH_LEN + 1]; + char *retval; + + retval = yescrypt_r( + (const uint8_t *)passwd, strlen(passwd), + (const uint8_t *)salt_data, + buf, sizeof(buf)); + /* The returned value is either buf[], or NULL on error */ + + return xstrdup(retval); +} From 1a0913d57ce8287703cfe666d9240e3a147ea30d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 07:44:01 +0200 Subject: [PATCH 122/277] libbb: factor out HMAC code from TLS function old new delta hmac_block - 88 +88 hmac_peek_hash - 61 +61 hmac_end - 50 +50 hmac_begin 140 177 +37 hmac_hash_v - 30 +30 .rodata 105799 105787 -12 hmac_sha_precomputed 54 - -54 hmac_sha_precomputed_v 69 - -69 hmac 83 - -83 ------------------------------------------------------------------------------ (add/remove: 5/3 grow/shrink: 1/1 up/down: 266/-218) Total: 48 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 40 +++++++++--- libbb/hash_hmac.c | 99 ++++++++++++++++++++++++++++ networking/tls.c | 161 +++++++++------------------------------------- 3 files changed, 161 insertions(+), 139 deletions(-) create mode 100644 libbb/hash_hmac.c diff --git a/include/libbb.h b/include/libbb.h index b761b10917..3f60acaa0b 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2193,6 +2193,16 @@ int FAST_FUNC i2a64(int i); int FAST_FUNC a2i64(char c); char* FAST_FUNC num2str64_lsb_first(char *s, unsigned v, int n); +enum { + /* how many bytes XYZ_end() fills */ + MD5_OUTSIZE = 16, + SHA1_OUTSIZE = 20, + SHA256_OUTSIZE = 32, + SHA512_OUTSIZE = 64, + SHA3_OUTSIZE = 28, + /* size of input block */ + SHA2_INSIZE = 64, +}; typedef struct md5_ctx_t { uint8_t wbuffer[64]; /* always correctly aligned for uint64_t */ void (*process_block)(struct md5_ctx_t*) FAST_FUNC; @@ -2226,18 +2236,32 @@ unsigned sha512_end(sha512_ctx_t *ctx, void *resbuf) FAST_FUNC; void sha3_begin(sha3_ctx_t *ctx) FAST_FUNC; void sha3_hash(sha3_ctx_t *ctx, const void *buffer, size_t len) FAST_FUNC; unsigned sha3_end(sha3_ctx_t *ctx, void *resbuf) FAST_FUNC; +void FAST_FUNC sha256_block(const void *in, size_t len, uint8_t hash[32]); /* TLS benefits from knowing that sha1 and sha256 share these. Give them "agnostic" names too */ typedef struct md5_ctx_t md5sha_ctx_t; #define md5sha_hash md5_hash #define sha_end sha1_end -enum { - MD5_OUTSIZE = 16, - SHA1_OUTSIZE = 20, - SHA256_OUTSIZE = 32, - SHA512_OUTSIZE = 64, - SHA3_OUTSIZE = 28, -}; -void FAST_FUNC sha256_block(const void *in, size_t len, uint8_t hash[32]); + +/* RFC 2104 HMAC (hash-based message authentication code) */ +typedef struct hmac_ctx { + md5sha_ctx_t hashed_key_xor_ipad; + md5sha_ctx_t hashed_key_xor_opad; +} hmac_ctx_t; +#define HMAC_ONLY_SHA256 (!ENABLE_FEATURE_TLS_SHA1) +typedef void md5sha_begin_func(md5sha_ctx_t *ctx) FAST_FUNC; +#if HMAC_ONLY_SHA256 +#define hmac_begin(ctx,key,key_size,begin) \ + hmac_begin(ctx,key,key_size) +#endif +void FAST_FUNC hmac_begin(hmac_ctx_t *ctx, uint8_t *key, unsigned key_size, md5sha_begin_func *begin); +static ALWAYS_INLINE void hmac_hash(hmac_ctx_t *ctx, const void *in, size_t len) +{ + md5sha_hash(&ctx->hashed_key_xor_ipad, in, len); +} +unsigned FAST_FUNC hmac_end(hmac_ctx_t *ctx, uint8_t *out); +/* HMAC helpers for TLS: */ +void FAST_FUNC hmac_hash_v(hmac_ctx_t *ctx, va_list va); +unsigned FAST_FUNC hmac_peek_hash(hmac_ctx_t *ctx, uint8_t *out, ...); extern uint32_t *global_crc32_table; uint32_t *crc32_filltable(uint32_t *tbl256, int endian) FAST_FUNC; diff --git a/libbb/hash_hmac.c b/libbb/hash_hmac.c new file mode 100644 index 0000000000..8cf9369497 --- /dev/null +++ b/libbb/hash_hmac.c @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2025 Denys Vlasenko + * + * Licensed under GPLv2, see file LICENSE in this source tree. + */ +//kbuild:lib-$(CONFIG_TLS) += hash_hmac.o +//kbuild:lib-$(CONFIG_USE_BB_CRYPT_YES) += hash_hmac.o + +#include "libbb.h" + +// RFC 2104: +// HMAC(key, text) based on a hash H (say, sha256) is: +// ipad = [0x36 x INSIZE] +// opad = [0x5c x INSIZE] +// HMAC(key, text) = H((key XOR opad) + H((key XOR ipad) + text)) +// +// H(key XOR opad) and H(key XOR ipad) can be precomputed +// if we often need HMAC hmac with the same key. +// +// text is often given in disjoint pieces. +void FAST_FUNC hmac_begin(hmac_ctx_t *ctx, uint8_t *key, unsigned key_size, md5sha_begin_func *begin) +{ +#if HMAC_ONLY_SHA256 +#define begin sha256_begin +#endif + uint8_t key_xor_ipad[SHA2_INSIZE]; + uint8_t key_xor_opad[SHA2_INSIZE]; + unsigned i; + + // "The authentication key can be of any length up to INSIZE, the + // block length of the hash function. Applications that use keys longer + // than INSIZE bytes will first hash the key using H and then use the + // resultant OUTSIZE byte string as the actual key to HMAC." + if (key_size > SHA2_INSIZE) { + uint8_t tempkey[SHA1_OUTSIZE < SHA256_OUTSIZE ? SHA256_OUTSIZE : SHA1_OUTSIZE]; + /* use ctx->hashed_key_xor_ipad as scratch ctx */ + begin(&ctx->hashed_key_xor_ipad); + md5sha_hash(&ctx->hashed_key_xor_ipad, key, key_size); + key_size = sha_end(&ctx->hashed_key_xor_ipad, tempkey); + key = tempkey; + } + + for (i = 0; i < key_size; i++) { + key_xor_ipad[i] = key[i] ^ 0x36; + key_xor_opad[i] = key[i] ^ 0x5c; + } + for (; i < SHA2_INSIZE; i++) { + key_xor_ipad[i] = 0x36; + key_xor_opad[i] = 0x5c; + } + + begin(&ctx->hashed_key_xor_ipad); + begin(&ctx->hashed_key_xor_opad); + md5sha_hash(&ctx->hashed_key_xor_ipad, key_xor_ipad, SHA2_INSIZE); + md5sha_hash(&ctx->hashed_key_xor_opad, key_xor_opad, SHA2_INSIZE); +} +#undef begin + +unsigned FAST_FUNC hmac_end(hmac_ctx_t *ctx, uint8_t *out) +{ + unsigned len = sha_end(&ctx->hashed_key_xor_ipad, out); + /* out = H((key XOR opad) + out) */ + md5sha_hash(&ctx->hashed_key_xor_opad, out, len); + return sha_end(&ctx->hashed_key_xor_opad, out); +} + +/* TLS helpers */ + +void FAST_FUNC hmac_hash_v( + hmac_ctx_t *ctx, + va_list va) +{ + uint8_t *in; + + /* ctx->hashed_key_xor_ipad contains unclosed "H((key XOR ipad) +" state */ + /* ctx->hashed_key_xor_opad contains unclosed "H((key XOR opad) +" state */ + + /* calculate out = H((key XOR ipad) + text) */ + while ((in = va_arg(va, uint8_t*)) != NULL) { + unsigned size = va_arg(va, unsigned); + md5sha_hash(&ctx->hashed_key_xor_ipad, in, size); + } +} + +/* Using HMAC state, make a copy of it (IOW: without affecting this state!) + * hash in the list of (ptr,size) blocks, and finish the HMAC to out[] buffer. + * This function is useful for TLS PRF. + */ +unsigned FAST_FUNC hmac_peek_hash(hmac_ctx_t *ctx, uint8_t *out, ...) +{ + hmac_ctx_t tmpctx = *ctx; /* struct copy */ + va_list va; + + va_start(va, out); + hmac_hash_v(&tmpctx, va); + va_end(va); + + return hmac_end(&tmpctx, out); +} diff --git a/networking/tls.c b/networking/tls.c index 8d074c058c..b8caf1e4b4 100644 --- a/networking/tls.c +++ b/networking/tls.c @@ -188,8 +188,6 @@ #define TLS_MAX_OUTBUF (1 << 14) enum { - SHA_INSIZE = 64, - AES128_KEYSIZE = 16, AES256_KEYSIZE = 32, @@ -393,128 +391,6 @@ static void hash_handshake(tls_state_t *tls, const char *fmt, const void *buffer # define TLS_MAC_SIZE(tls) (tls)->MAC_size #endif -// RFC 2104: -// HMAC(key, text) based on a hash H (say, sha256) is: -// ipad = [0x36 x INSIZE] -// opad = [0x5c x INSIZE] -// HMAC(key, text) = H((key XOR opad) + H((key XOR ipad) + text)) -// -// H(key XOR opad) and H(key XOR ipad) can be precomputed -// if we often need HMAC hmac with the same key. -// -// text is often given in disjoint pieces. -typedef struct hmac_precomputed { - md5sha_ctx_t hashed_key_xor_ipad; - md5sha_ctx_t hashed_key_xor_opad; -} hmac_precomputed_t; - -typedef void md5sha_begin_func(md5sha_ctx_t *ctx) FAST_FUNC; -#if !ENABLE_FEATURE_TLS_SHA1 -#define hmac_begin(pre,key,key_size,begin) \ - hmac_begin(pre,key,key_size) -#define begin sha256_begin -#endif -static void hmac_begin(hmac_precomputed_t *pre, uint8_t *key, unsigned key_size, md5sha_begin_func *begin) -{ - uint8_t key_xor_ipad[SHA_INSIZE]; - uint8_t key_xor_opad[SHA_INSIZE]; -// uint8_t tempkey[SHA1_OUTSIZE < SHA256_OUTSIZE ? SHA256_OUTSIZE : SHA1_OUTSIZE]; - unsigned i; - - // "The authentication key can be of any length up to INSIZE, the - // block length of the hash function. Applications that use keys longer - // than INSIZE bytes will first hash the key using H and then use the - // resultant OUTSIZE byte string as the actual key to HMAC." - if (key_size > SHA_INSIZE) { - bb_simple_error_msg_and_die("HMAC key>64"); //does not happen (yet?) -// md5sha_ctx_t ctx; -// begin(&ctx); -// md5sha_hash(&ctx, key, key_size); -// key_size = sha_end(&ctx, tempkey); -// //key = tempkey; - right? RIGHT? why does it work without this? -// // because SHA_INSIZE is 64, but hmac() is always called with -// // key_size = tls->MAC_size = SHA1/256_OUTSIZE (20 or 32), -// // and prf_hmac_sha256() -> hmac_sha256() key sizes are: -// // - RSA_PREMASTER_SIZE is 48 -// // - CURVE25519_KEYSIZE is 32 -// // - master_secret[] is 48 - } - - for (i = 0; i < key_size; i++) { - key_xor_ipad[i] = key[i] ^ 0x36; - key_xor_opad[i] = key[i] ^ 0x5c; - } - for (; i < SHA_INSIZE; i++) { - key_xor_ipad[i] = 0x36; - key_xor_opad[i] = 0x5c; - } - - begin(&pre->hashed_key_xor_ipad); - begin(&pre->hashed_key_xor_opad); - md5sha_hash(&pre->hashed_key_xor_ipad, key_xor_ipad, SHA_INSIZE); - md5sha_hash(&pre->hashed_key_xor_opad, key_xor_opad, SHA_INSIZE); -} -#undef begin - -static unsigned hmac_sha_precomputed_v( - hmac_precomputed_t *pre, - uint8_t *out, - va_list va) -{ - uint8_t *text; - unsigned len; - - /* pre->hashed_key_xor_ipad contains unclosed "H((key XOR ipad) +" state */ - /* pre->hashed_key_xor_opad contains unclosed "H((key XOR opad) +" state */ - - /* calculate out = H((key XOR ipad) + text) */ - while ((text = va_arg(va, uint8_t*)) != NULL) { - unsigned text_size = va_arg(va, unsigned); - md5sha_hash(&pre->hashed_key_xor_ipad, text, text_size); - } - len = sha_end(&pre->hashed_key_xor_ipad, out); - - /* out = H((key XOR opad) + out) */ - md5sha_hash(&pre->hashed_key_xor_opad, out, len); - return sha_end(&pre->hashed_key_xor_opad, out); -} - -static unsigned hmac_sha_precomputed(hmac_precomputed_t *pre_init, uint8_t *out, ...) -{ - hmac_precomputed_t pre; - va_list va; - unsigned len; - - va_start(va, out); - pre = *pre_init; /* struct copy */ - len = hmac_sha_precomputed_v(&pre, out, va); - va_end(va); - return len; -} - -#if !ENABLE_FEATURE_TLS_SHA1 -#define hmac(tls,out,key,key_size,...) \ - hmac(out,key,key_size, __VA_ARGS__) -#endif -static unsigned hmac(tls_state_t *tls, uint8_t *out, uint8_t *key, unsigned key_size, ...) -{ - hmac_precomputed_t pre; - va_list va; - unsigned len; - - va_start(va, key_size); - - hmac_begin(&pre, key, key_size, - (ENABLE_FEATURE_TLS_SHA1 && tls->MAC_size == SHA1_OUTSIZE) - ? sha1_begin - : sha256_begin - ); - len = hmac_sha_precomputed_v(&pre, out, va); - - va_end(va); - return len; -} - // RFC 5246: // 5. HMAC and the Pseudorandom Function //... @@ -559,7 +435,7 @@ static void prf_hmac_sha256(/*tls_state_t *tls,*/ const char *label, uint8_t *seed, unsigned seed_size) { - hmac_precomputed_t pre; + hmac_ctx_t ctx; uint8_t a[TLS_MAX_MAC_SIZE]; uint8_t *out_p = outbuf; unsigned label_size = strlen(label); @@ -569,26 +445,26 @@ static void prf_hmac_sha256(/*tls_state_t *tls,*/ #define SEED label, label_size, seed, seed_size #define A a, MAC_size - hmac_begin(&pre, secret, secret_size, sha256_begin); + hmac_begin(&ctx, secret, secret_size, sha256_begin); /* A(1) = HMAC_hash(secret, seed) */ - hmac_sha_precomputed(&pre, a, SEED, NULL); + hmac_peek_hash(&ctx, a, SEED, NULL); for (;;) { /* HMAC_hash(secret, A(1) + seed) */ if (outbuf_size <= MAC_size) { /* Last, possibly incomplete, block */ /* (use a[] as temp buffer) */ - hmac_sha_precomputed(&pre, a, A, SEED, NULL); + hmac_peek_hash(&ctx, a, A, SEED, NULL); memcpy(out_p, a, outbuf_size); return; } /* Not last block. Store directly to result buffer */ - hmac_sha_precomputed(&pre, out_p, A, SEED, NULL); + hmac_peek_hash(&ctx, out_p, A, SEED, NULL); out_p += MAC_size; outbuf_size -= MAC_size; /* A(2) = HMAC_hash(secret, A(1)) */ - hmac_sha_precomputed(&pre, a, A, NULL); + hmac_peek_hash(&ctx, a, A, NULL); } #undef A #undef SECRET @@ -655,6 +531,29 @@ static void *tls_get_zeroed_outbuf(tls_state_t *tls, int len) return record; } +/* Calculate the HMAC over the list of blocks */ +#if !ENABLE_FEATURE_TLS_SHA1 +#define hmac_block(tls,out,key,key_size,...) \ + hmac_block(out,key,key_size, __VA_ARGS__) +#endif +static unsigned hmac_block(tls_state_t *tls, uint8_t *out, uint8_t *key, unsigned key_size, ...) +{ + hmac_ctx_t ctx; + va_list va; + + hmac_begin(&ctx, key, key_size, + (ENABLE_FEATURE_TLS_SHA1 && tls->MAC_size == SHA1_OUTSIZE) + ? sha1_begin + : sha256_begin + ); + + va_start(va, key_size); + hmac_hash_v(&ctx, va); + va_end(va); + + return hmac_end(&ctx, out); +} + static void xwrite_encrypted_and_hmac_signed(tls_state_t *tls, unsigned size, unsigned type) { uint8_t *buf = tls->outbuf + OUTBUF_PFX; @@ -676,7 +575,7 @@ static void xwrite_encrypted_and_hmac_signed(tls_state_t *tls, unsigned size, un xhdr->len16_lo = size & 0xff; /* Calculate MAC signature */ - hmac(tls, buf + size, /* result */ + hmac_block(tls, buf + size, /* result */ tls->client_write_MAC_key, TLS_MAC_SIZE(tls), &tls->write_seq64_be, sizeof(tls->write_seq64_be), xhdr, RECHDR_LEN, From c11730490ad68737120d569b9760e2c35e28977e Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 08:21:44 +0200 Subject: [PATCH 123/277] libbb/yescrypt: remove redundant SHA256 HMAC implementation function old new delta hmac_blocks - 88 +88 static.PBKDF2_SHA256 176 213 +37 yescrypt_kdf32_body 1046 1052 +6 static.smix 759 762 +3 hmac_block 88 64 -24 HMAC_SHA256_Final 53 - -53 HMAC_SHA256_Buf 58 - -58 HMAC_SHA256_Init 159 - -159 ------------------------------------------------------------------------------ (add/remove: 1/3 grow/shrink: 3/1 up/down: 134/-294) Total: -160 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 10 +++- libbb/hash_hmac.c | 10 +++- libbb/yescrypt/alg-sha256.c | 94 +++---------------------------- libbb/yescrypt/alg-yescrypt-kdf.c | 24 ++++++-- networking/tls.c | 8 +-- 5 files changed, 48 insertions(+), 98 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 3f60acaa0b..cbf723f7eb 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -2253,12 +2253,20 @@ typedef void md5sha_begin_func(md5sha_ctx_t *ctx) FAST_FUNC; #define hmac_begin(ctx,key,key_size,begin) \ hmac_begin(ctx,key,key_size) #endif -void FAST_FUNC hmac_begin(hmac_ctx_t *ctx, uint8_t *key, unsigned key_size, md5sha_begin_func *begin); +void FAST_FUNC hmac_begin(hmac_ctx_t *ctx, const uint8_t *key, unsigned key_size, md5sha_begin_func *begin); static ALWAYS_INLINE void hmac_hash(hmac_ctx_t *ctx, const void *in, size_t len) { md5sha_hash(&ctx->hashed_key_xor_ipad, in, len); } unsigned FAST_FUNC hmac_end(hmac_ctx_t *ctx, uint8_t *out); +#if HMAC_ONLY_SHA256 +#define hmac_block(key,key_size,begin,in,sz,out) \ + hmac_block(key,key_size,in,sz,out) +#endif +unsigned FAST_FUNC hmac_block(const uint8_t *key, unsigned key_size, + md5sha_begin_func *begin, + const void *in, unsigned sz, + uint8_t *out); /* HMAC helpers for TLS: */ void FAST_FUNC hmac_hash_v(hmac_ctx_t *ctx, va_list va); unsigned FAST_FUNC hmac_peek_hash(hmac_ctx_t *ctx, uint8_t *out, ...); diff --git a/libbb/hash_hmac.c b/libbb/hash_hmac.c index 8cf9369497..9e48e0f511 100644 --- a/libbb/hash_hmac.c +++ b/libbb/hash_hmac.c @@ -18,7 +18,7 @@ // if we often need HMAC hmac with the same key. // // text is often given in disjoint pieces. -void FAST_FUNC hmac_begin(hmac_ctx_t *ctx, uint8_t *key, unsigned key_size, md5sha_begin_func *begin) +void FAST_FUNC hmac_begin(hmac_ctx_t *ctx, const uint8_t *key, unsigned key_size, md5sha_begin_func *begin) { #if HMAC_ONLY_SHA256 #define begin sha256_begin @@ -64,6 +64,14 @@ unsigned FAST_FUNC hmac_end(hmac_ctx_t *ctx, uint8_t *out) return sha_end(&ctx->hashed_key_xor_opad, out); } +unsigned FAST_FUNC hmac_block(const uint8_t *key, unsigned key_size, md5sha_begin_func *begin, const void *in, unsigned sz, uint8_t *out) +{ + hmac_ctx_t ctx; + hmac_begin(&ctx, key, key_size, begin); + hmac_hash(&ctx, in, sz); + return hmac_end(&ctx, out); +} + /* TLS helpers */ void FAST_FUNC hmac_hash_v( diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index f56b905ada..1ccffa1e55 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -25,82 +25,6 @@ * SUCH DAMAGE. */ -/** - * HMAC_SHA256_Init(ctx, K, Klen): - * Initialize the HMAC-SHA256 context ${ctx} with ${Klen} bytes of key from - * ${K}. - */ -static void -HMAC_SHA256_Init(HMAC_SHA256_CTX *ctx, const void *_K, size_t Klen) -{ - uint8_t pad[64]; - uint8_t khash[32]; - const uint8_t *K = _K; - size_t i; - - /* If Klen > 64, the key is really SHA256(K). */ - if (Klen > 64) { - sha256_block(K, Klen, khash); - K = khash; - Klen = 32; - } - - /* Inner SHA256 operation is SHA256(K xor [block of 0x36] || data). */ - sha256_begin(&ctx->ictx); - memset(pad, 0x36, 64); - for (i = 0; i < Klen; i++) - pad[i] ^= K[i]; - sha256_hash(&ctx->ictx, pad, 64); - - /* Outer SHA256 operation is SHA256(K xor [block of 0x5c] || hash). */ - sha256_begin(&ctx->octx); - memset(pad, 0x5c, 64); - for (i = 0; i < Klen; i++) - pad[i] ^= K[i]; - sha256_hash(&ctx->octx, pad, 64); -} - -/** - * HMAC_SHA256_Update(ctx, in, len): - * Input ${len} bytes from ${in} into the HMAC-SHA256 context ${ctx}. - */ -static void -HMAC_SHA256_Update(HMAC_SHA256_CTX *ctx, const void *in, size_t len) -{ - /* Feed data to the inner SHA256 operation. */ - sha256_hash(&ctx->ictx, in, len); -} - -/** - * HMAC_SHA256_Final(ctx, digest): - * Output the HMAC-SHA256 of the data input to the context ${ctx} into the - * buffer ${digest}. - */ -static void -HMAC_SHA256_Final(HMAC_SHA256_CTX *ctx, void *digest) -{ - /* Finish the inner SHA256 operation. */ - sha256_end(&ctx->ictx, digest); /* using digest[] as scratch space */ - /* Feed the inner hash to the outer SHA256 operation. */ - sha256_hash(&ctx->octx, digest, 32); /* using digest[] as scratch space */ - /* Finish the outer SHA256 operation. */ - sha256_end(&ctx->octx, digest); -} - -/** - * HMAC_SHA256_Buf(K, Klen, in, len, digest): - * Compute the HMAC-SHA256 of ${len} bytes from ${in} using the key ${K} of - * length ${Klen}, and write the result to ${digest}. - */ -static void -HMAC_SHA256_Buf(const void *K, size_t Klen, const void *in, size_t len, void *digest) -{ - HMAC_SHA256_CTX ctx; - HMAC_SHA256_Init(&ctx, K, Klen); - HMAC_SHA256_Update(&ctx, in, len); - HMAC_SHA256_Final(&ctx, digest); -} - /** * PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, c, buf, dkLen): * Compute PBKDF2(passwd, salt, c, dkLen) using HMAC-SHA256 as the PRF, and @@ -111,15 +35,15 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, uint64_t c, uint8_t *buf, size_t dkLen) { - HMAC_SHA256_CTX Phctx, PShctx, hctx; + hmac_ctx_t Phctx, PShctx; size_t i; /* Compute HMAC state after processing P. */ - HMAC_SHA256_Init(&Phctx, passwd, passwdlen); + hmac_begin(&Phctx, passwd, passwdlen, sha256_begin); /* Compute HMAC state after processing P and S. */ - memcpy(&PShctx, &Phctx, sizeof(HMAC_SHA256_CTX)); - HMAC_SHA256_Update(&PShctx, salt, saltlen); + PShctx = Phctx; + hmac_hash(&PShctx, salt, saltlen); /* Iterate through the blocks. */ for (i = 0; dkLen != 0; i++) { @@ -134,18 +58,16 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, ivec = SWAP_BE32((uint32_t)(i + 1)); /* Compute U_1 = PRF(P, S || INT(i)). */ - hctx = PShctx; - HMAC_SHA256_Update(&hctx, &ivec, 4); - HMAC_SHA256_Final(&hctx, T); + hmac_peek_hash(&PShctx, (void*)T, &ivec, 4, NULL); +//TODO: the above is a vararg function, might incur some ABI pain +//does libbb need a non-vararg version with just one (buf,len)? if (c > 1) { /* T_i = U_1 ... */ memcpy(U, T, 32); for (j = 2; j <= c; j++) { /* Compute U_j. */ - hctx = Phctx; - HMAC_SHA256_Update(&hctx, U, 32); - HMAC_SHA256_Final(&hctx, U); + hmac_peek_hash(&Phctx, (void*)U, U, 32, NULL); /* ... xor U_j ... */ for (k = 0; k < 32 / 8; k++) T[k] ^= U[k]; diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 27ef2caa4d..f1f06621e4 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -735,8 +735,12 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, ctx_i->S0 = Si + Sbytes / 3 * 2; ctx_i->w = 0; if (i == 0) - HMAC_SHA256_Buf(Bp + (128 * r - 64), 64, - passwd, 32, passwd); + hmac_block( + /* key,len: */ Bp + (128 * r - 64), 64, + /* hash fn: */ sha256_begin, + /* in,len: */ passwd, 32, + /* outbuf: */ passwd + ); } smix1(Bp, r, Np, flags, Vp, NROM, VROM, XYp, ctx_i); smix2(Bp, r, p2floor(Np), Nloop_rw, flags, Vp, @@ -907,9 +911,12 @@ static int yescrypt_kdf32_body( S = (uint8_t *)XY + XY_size; if (flags) { - HMAC_SHA256_Buf("yescrypt-prehash", - (flags & YESCRYPT_PREHASH) ? 16 : 8, - passwd, passwdlen, sha256); + hmac_block( + /* key,len: */ (const void*)"yescrypt-prehash", (flags & YESCRYPT_PREHASH) ? 16 : 8, + /* hash fn: */ sha256_begin, + /* in,len: */ passwd, passwdlen, + /* outbuf: */ sha256 + ); passwd = sha256; passwdlen = sizeof(sha256); } @@ -946,7 +953,12 @@ static int yescrypt_kdf32_body( */ if (flags && !(flags & YESCRYPT_PREHASH)) { /* Compute ClientKey */ - HMAC_SHA256_Buf(dkp, sizeof(dk), "Client Key", 10, sha256); + hmac_block( + /* key,len: */ dkp, sizeof(dk), + /* hash fn: */ sha256_begin, + /* in,len: */ "Client Key", 10, + /* outbuf: */ sha256 + ); /* Compute StoredKey */ { size_t clen = /*buflen:*/32; diff --git a/networking/tls.c b/networking/tls.c index b8caf1e4b4..098cf7cacc 100644 --- a/networking/tls.c +++ b/networking/tls.c @@ -533,10 +533,10 @@ static void *tls_get_zeroed_outbuf(tls_state_t *tls, int len) /* Calculate the HMAC over the list of blocks */ #if !ENABLE_FEATURE_TLS_SHA1 -#define hmac_block(tls,out,key,key_size,...) \ - hmac_block(out,key,key_size, __VA_ARGS__) +#define hmac_blocks(tls,out,key,key_size,...) \ + hmac_blocks(out,key,key_size, __VA_ARGS__) #endif -static unsigned hmac_block(tls_state_t *tls, uint8_t *out, uint8_t *key, unsigned key_size, ...) +static unsigned hmac_blocks(tls_state_t *tls, uint8_t *out, uint8_t *key, unsigned key_size, ...) { hmac_ctx_t ctx; va_list va; @@ -575,7 +575,7 @@ static void xwrite_encrypted_and_hmac_signed(tls_state_t *tls, unsigned size, un xhdr->len16_lo = size & 0xff; /* Calculate MAC signature */ - hmac_block(tls, buf + size, /* result */ + hmac_blocks(tls, buf + size, /* result */ tls->client_write_MAC_key, TLS_MAC_SIZE(tls), &tls->write_seq64_be, sizeof(tls->write_seq64_be), xhdr, RECHDR_LEN, From f464be22bd63bf0326bc14a755cbac282fad159a Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 10:39:14 +0200 Subject: [PATCH 124/277] libbb/yescrypt: make it possible to set constant parameters, and set YESCRYPT_RW function old new delta yescrypt_kdf32_body 1052 1420 +368 yescrypt_r 1133 1084 -49 static.smix 762 - -762 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 1/1 up/down: 368/-811) Total: -443 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 61 ++++++-------- libbb/yescrypt/alg-yescrypt-kdf.c | 58 ++++++------- libbb/yescrypt/alg-yescrypt.h | 117 ++++++++++++++++++++------- testsuite/cryptpw.tests | 2 +- 4 files changed, 142 insertions(+), 96 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 5bdf1893ef..db6e098c7e 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -144,7 +144,7 @@ char *yescrypt_r( char *dst; const uint8_t *src, *saltstr, *saltend; size_t need, prefixlen, saltstrlen; - uint32_t flavor, N_log2; + uint32_t u32; memset(yctx, 0, sizeof(yctx)); yctx->param.p = 1; @@ -152,43 +152,34 @@ char *yescrypt_r( /* we assume setting starts with "$y$" (caller must ensure this) */ src = setting + 3; - src = decode64_uint32(&flavor, src, 0); + src = decode64_uint32(&yctx->param.flags, src, 0); /* "j9T" returns: 0x2f */ - dbg("yescrypt flavor=0x%x YESCRYPT_RW:%u", (unsigned)flavor, !!(flavor & YESCRYPT_RW)); //if (!src) // goto fail; - if (flavor < YESCRYPT_RW) { - yctx->param.flags = flavor; - } else if (flavor <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { + if (yctx->param.flags < YESCRYPT_RW) { + dbg("yctx->param.flags=0x%x", (unsigned)yctx->param.flags); + goto fail; // bbox: we don't support scrypt - only yescrypt + } else if (yctx->param.flags <= YESCRYPT_RW + (YESCRYPT_RW_FLAVOR_MASK >> 2)) { /* "j9T" sets flags to 0xb6 */ - yctx->param.flags = YESCRYPT_RW + ((flavor - YESCRYPT_RW) << 2); + yctx->param.flags = YESCRYPT_RW + ((yctx->param.flags - YESCRYPT_RW) << 2); dbg("yctx->param.flags=0x%x", (unsigned)yctx->param.flags); - dbg(" YESCRYPT_RW:%u" , !!(yctx->param.flags & YESCRYPT_RW )); - dbg(" YESCRYPT_ROUNDS_6:%u" , !!(yctx->param.flags & YESCRYPT_ROUNDS_6 )); - dbg(" YESCRYPT_GATHER_2:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_2 )); - dbg(" YESCRYPT_GATHER_4:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_4 )); - dbg(" YESCRYPT_GATHER_8:%u" , !!(yctx->param.flags & YESCRYPT_GATHER_8 )); - dbg(" YESCRYPT_SIMPLE_2:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_2 )); - dbg(" YESCRYPT_SIMPLE_4:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_4 )); - dbg(" YESCRYPT_SIMPLE_8:%u" , !!(yctx->param.flags & YESCRYPT_SIMPLE_8 )); - dbg(" YESCRYPT_SBOX_12K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_12K )); - dbg(" YESCRYPT_SBOX_24K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_24K )); - dbg(" YESCRYPT_SBOX_48K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_48K )); - dbg(" YESCRYPT_SBOX_96K:%u" , !!(yctx->param.flags & YESCRYPT_SBOX_96K )); - dbg(" YESCRYPT_SBOX_192K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_192K)); - dbg(" YESCRYPT_SBOX_384K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_384K)); - dbg(" YESCRYPT_SBOX_768K:%u", !!(yctx->param.flags & YESCRYPT_SBOX_768K)); + dbg(" YESCRYPT_RW:%u", !!(yctx->param.flags & YESCRYPT_RW)); + dbg((yctx->param.flags & YESCRYPT_RW_FLAVOR_MASK) == + (YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K) + ? " YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K" + : " flags are not standard" + ); } else { goto fail; } - src = decode64_uint32(&N_log2, src, 1); - if (/*!src ||*/ N_log2 > 63) + src = decode64_uint32(&u32, src, 1); + if (/*!src ||*/ u32 > 63) goto fail; - yctx->param.N = (uint64_t)1 << N_log2; + yctx->param.N = (uint64_t)1 << u32; /* "j9T" sets to 4096 (1<<12) */ - dbg("yctx->param.N=%llu (1<<%u)", (unsigned long long)yctx->param.N, (unsigned)N_log2); + dbg("yctx->param.N=%llu (1<<%u)", (unsigned long long)yctx->param.N, (unsigned)u32); src = decode64_uint32(&yctx->param.r, src, 1); /* "j9T" sets to 32 */ @@ -197,21 +188,19 @@ char *yescrypt_r( if (!src) goto fail; if (*src != '$') { - uint32_t have; - src = decode64_uint32(&have, src, 1); + src = decode64_uint32(&u32, src, 1); dbg("yescrypt has extended params:0x%x", (unsigned)have); - if (have & 1) + if (u32 & 1) src = decode64_uint32(&yctx->param.p, src, 2); - if (have & 2) + if (u32 & 2) src = decode64_uint32(&yctx->param.t, src, 1); - if (have & 4) + if (u32 & 4) src = decode64_uint32(&yctx->param.g, src, 1); - if (have & 8) { - uint32_t NROM_log2; - src = decode64_uint32(&NROM_log2, src, 1); - if (/*!src ||*/ NROM_log2 > 63) + if (u32 & 8) { + src = decode64_uint32(&u32, src, 1); + if (/*!src ||*/ u32 > 63) goto fail; - yctx->param.NROM = (uint64_t)1 << NROM_log2; + yctx->param.NROM = (uint64_t)1 << u32; } if (!src) goto fail; diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index f1f06621e4..13ae62b7cf 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -460,7 +460,7 @@ static inline uint32_t integerify(const salsa20_blk_t *B, size_t r) * to a multiple of at least 16 bytes. */ static void smix1(uint8_t *B, size_t r, uint32_t N, - yescrypt_flags_t flags, + uint32_t flags, salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, salsa20_blk_t *XY, @@ -513,6 +513,7 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, V_j = &VROM[j * s]; blockmix_xor(Y, V_j, XY, r, ctx); } else if (flags & YESCRYPT_RW) { +//can't use flags___YESCRYPT_RW, smix1() may be called with flags = 0 uint32_t n; salsa20_blk_t *V_j; @@ -580,7 +581,7 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, * 64 bytes, and arrays B and XY to a multiple of at least 16 bytes. */ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, - yescrypt_flags_t flags, + uint32_t flags, salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, salsa20_blk_t *XY, @@ -610,6 +611,7 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, * because our SMix resets YESCRYPT_RW for the smix2() calls operating on the * entire V when p > 1. */ +//and this is why bbox can't use flags___YESCRYPT_RW in this function if (VROM && (flags & YESCRYPT_RW)) { do { salsa20_blk_t *V_j = &V[j * s]; @@ -683,7 +685,7 @@ static uint64_t p2floor(uint64_t x) * might also result in cache bank conflicts). */ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, - yescrypt_flags_t flags, + uint32_t flags, salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, salsa20_blk_t *XY, @@ -696,7 +698,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, Nchunk = N / p; Nloop_all = Nchunk; - if (flags & YESCRYPT_RW) { + if (flags___YESCRYPT_RW) { if (t <= 1) { if (t) Nloop_all *= 2; /* 2/3 */ @@ -711,7 +713,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, } Nloop_rw = 0; - if (flags & YESCRYPT_RW) + if (flags___YESCRYPT_RW) Nloop_rw = Nloop_all / p; Nchunk &= ~(uint32_t)1; /* round down to even */ @@ -725,7 +727,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, salsa20_blk_t *Vp = &V[Vchunk * s]; salsa20_blk_t *XYp = XY; pwxform_ctx_t *ctx_i = NULL; - if (flags & YESCRYPT_RW) { + if (flags___YESCRYPT_RW) { uint8_t *Si = S + i * Salloc; smix1(Bp, 1, Sbytes / 128, 0 /* no flags */, (salsa20_blk_t *)Si, 0, NULL, XYp, NULL); @@ -752,12 +754,12 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, uint8_t *Bp = &B[128 * r * i]; salsa20_blk_t *XYp = XY; pwxform_ctx_t *ctx_i = NULL; - if (flags & YESCRYPT_RW) { + if (flags___YESCRYPT_RW) { uint8_t *Si = S + i * Salloc; ctx_i = (pwxform_ctx_t *)(Si + Sbytes); } smix2(Bp, r, N, Nloop_all - Nloop_rw, - flags & (yescrypt_flags_t)~YESCRYPT_RW, + flags & (uint32_t)~YESCRYPT_RW, V, NROM, VROM, XYp, ctx_i); } } @@ -812,7 +814,7 @@ static void free_region(yescrypt_region_t *region) static int yescrypt_kdf32_body( yescrypt_ctx_t *yctx, const uint8_t *passwd, size_t passwdlen, - yescrypt_flags_t flags, uint64_t N, uint32_t t, + uint32_t flags, uint64_t N, uint32_t t, uint8_t *buf32) { const salsa20_blk_t *VROM; @@ -823,13 +825,13 @@ static int yescrypt_kdf32_body( uint8_t dk[sizeof(sha256)], *dkp = buf32; /* Sanity-check parameters */ - switch (flags & YESCRYPT_MODE_MASK) { + switch (flags___YESCRYPT_MODE_MASK) { case 0: /* classic scrypt - can't have anything non-standard */ - if (flags || t || yctx->param.NROM) + if (flags || t || YCTX_param_NROM) goto out_EINVAL; break; case YESCRYPT_WORM: - if (flags != YESCRYPT_WORM || yctx->param.NROM) + if (flags != YESCRYPT_WORM || YCTX_param_NROM) goto out_EINVAL; break; case YESCRYPT_RW: @@ -852,8 +854,8 @@ static int yescrypt_kdf32_body( goto out_EINVAL; #endif { - const uint32_t r = yctx->param.r; - const uint32_t p = yctx->param.p; + const uint32_t r = YCTX_param_r; + const uint32_t p = YCTX_param_p; if ((uint64_t)r * (uint64_t)p >= 1 << 30) goto out_EINVAL; if (N > UINT32_MAX) @@ -863,7 +865,7 @@ static int yescrypt_kdf32_body( if (r > SIZE_MAX / 256 / p || N > SIZE_MAX / 128 / r) goto out_EINVAL; - if (flags & YESCRYPT_RW) { + if (flags___YESCRYPT_RW) { /* p cannot be greater than SIZE_MAX/Salloc on 64-bit systems, but it can on 32-bit systems. */ #pragma GCC diagnostic push @@ -874,7 +876,7 @@ static int yescrypt_kdf32_body( } VROM = NULL; - if (yctx->param.NROM) + if (YCTX_param_NROM) goto out_EINVAL; /* Allocate memory */ @@ -889,7 +891,7 @@ static int yescrypt_kdf32_body( need += XY_size; if (need < XY_size) goto out_EINVAL; - if (flags & YESCRYPT_RW) { + if (flags___YESCRYPT_RW) { size_t S_size = (size_t)Salloc * p; need += S_size; if (need < S_size) @@ -907,7 +909,7 @@ static int yescrypt_kdf32_body( V = (salsa20_blk_t *)((uint8_t *)B + B_size); XY = (salsa20_blk_t *)((uint8_t *)V + V_size); S = NULL; - if (flags & YESCRYPT_RW) + if (flags___YESCRYPT_RW) S = (uint8_t *)XY + XY_size; if (flags) { @@ -926,13 +928,13 @@ static int yescrypt_kdf32_body( if (flags) memcpy(sha256, B, sizeof(sha256)); - if (p == 1 || (flags & YESCRYPT_RW)) { - smix(B, r, N, p, t, flags, V, yctx->param.NROM, VROM, XY, S, sha256); + if (p == 1 || (flags___YESCRYPT_RW)) { + smix(B, r, N, p, t, flags, V, YCTX_param_NROM, VROM, XY, S, sha256); } else { uint32_t i; for (i = 0; i < p; i++) { smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, V, - yctx->param.NROM, VROM, XY, NULL, NULL); + YCTX_param_NROM, VROM, XY, NULL, NULL); } } @@ -996,12 +998,12 @@ int yescrypt_kdf32( const uint8_t *passwd, size_t passwdlen, uint8_t *buf32) { - yescrypt_flags_t flags = yctx->param.flags; - uint64_t N = yctx->param.N; - uint32_t r = yctx->param.r; - uint32_t p = yctx->param.p; - uint32_t t = yctx->param.t; - uint32_t g = yctx->param.g; + uint32_t flags = YCTX_param_flags; + uint64_t N = YCTX_param_N; + uint32_t r = YCTX_param_r; + uint32_t p = YCTX_param_p; + uint32_t t = YCTX_param_t; + uint32_t g = YCTX_param_g; uint8_t dk32[32]; int retval; @@ -1011,7 +1013,7 @@ int yescrypt_kdf32( return -1; } - if ((flags & YESCRYPT_RW) + if ((flags___YESCRYPT_RW) && p >= 1 && N / p >= 0x100 && N / p * r >= 0x20000 diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 996af333fa..97475d89f3 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -42,28 +42,32 @@ * Please refer to the description of yescrypt_kdf() below for the meaning of * these flags. */ -typedef uint32_t yescrypt_flags_t; +/* yescrypt flags: + * bits pos: 7654321076543210 + * ss r w + * sbox gg y + */ /* Public */ #define YESCRYPT_WORM 1 #define YESCRYPT_RW 0x002 -#define YESCRYPT_ROUNDS_3 0x000 -#define YESCRYPT_ROUNDS_6 0x004 -#define YESCRYPT_GATHER_1 0x000 -#define YESCRYPT_GATHER_2 0x008 -#define YESCRYPT_GATHER_4 0x010 -#define YESCRYPT_GATHER_8 0x018 -#define YESCRYPT_SIMPLE_1 0x000 -#define YESCRYPT_SIMPLE_2 0x020 -#define YESCRYPT_SIMPLE_4 0x040 -#define YESCRYPT_SIMPLE_8 0x060 -#define YESCRYPT_SBOX_6K 0x000 -#define YESCRYPT_SBOX_12K 0x080 -#define YESCRYPT_SBOX_24K 0x100 -#define YESCRYPT_SBOX_48K 0x180 -#define YESCRYPT_SBOX_96K 0x200 -#define YESCRYPT_SBOX_192K 0x280 -#define YESCRYPT_SBOX_384K 0x300 -#define YESCRYPT_SBOX_768K 0x380 +#define YESCRYPT_ROUNDS_3 0x000 //r=0 +#define YESCRYPT_ROUNDS_6 0x004 //r=1 +#define YESCRYPT_GATHER_1 0x000 //gg=00 +#define YESCRYPT_GATHER_2 0x008 //gg=01 +#define YESCRYPT_GATHER_4 0x010 //gg=10 +#define YESCRYPT_GATHER_8 0x018 //gg=11 +#define YESCRYPT_SIMPLE_1 0x000 //ss=00 +#define YESCRYPT_SIMPLE_2 0x020 //ss=01 +#define YESCRYPT_SIMPLE_4 0x040 //ss=11 +#define YESCRYPT_SIMPLE_8 0x060 //ss=11 +#define YESCRYPT_SBOX_6K 0x000 //sbox=0000 +#define YESCRYPT_SBOX_12K 0x080 //sbox=0001 +#define YESCRYPT_SBOX_24K 0x100 //sbox=0010 +#define YESCRYPT_SBOX_48K 0x180 //sbox=0011 +#define YESCRYPT_SBOX_96K 0x200 //sbox=0100 +#define YESCRYPT_SBOX_192K 0x280 //sbox=0101 +#define YESCRYPT_SBOX_384K 0x300 //sbox=0110 +#define YESCRYPT_SBOX_768K 0x380 //sbox=0111 #ifdef YESCRYPT_INTERNAL /* Private */ @@ -86,6 +90,19 @@ typedef uint32_t yescrypt_flags_t; YESCRYPT_ALLOC_ONLY | YESCRYPT_PREHASH) #endif +/* How many chars base-64 encoded bytes require? */ +#define YESCRYPT_BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) +/* The /etc/passwd-style hash is "$" */ +/* + * "$y$", up to 8 params of up to 6 chars each, '$', salt + * Alternatively, but that's smaller: + * "$7$", 3 params encoded as 1+5+5 chars, salt + */ +#define YESCRYPT_PREFIX_LEN (3 + 8 * 6 + 1 + YESCRYPT_BYTES2CHARS(32)) + +#define YESCRYPT_HASH_SIZE 32 +#define YESCRYPT_HASH_LEN YESCRYPT_BYTES2CHARS(YESCRYPT_HASH_SIZE) + /** * Internal type used by the memory allocator. Please do not use it directly. * Use yescrypt_shared_t and yescrypt_local_t as appropriate instead, since @@ -104,7 +121,7 @@ typedef struct { * set. flags, t, g, NROM are special to yescrypt. */ typedef struct { - yescrypt_flags_t flags; + uint32_t flags; uint64_t N; uint32_t r, p, t, g; uint64_t NROM; @@ -123,18 +140,56 @@ typedef struct { yescrypt_region_t local[1]; } yescrypt_ctx_t; -/* How many chars base-64 encoded bytes require? */ -#define YESCRYPT_BYTES2CHARS(bytes) ((((bytes) * 8) + 5) / 6) -/* The /etc/passwd-style hash is "$" */ -/* - * "$y$", up to 8 params of up to 6 chars each, '$', salt - * Alternatively, but that's smaller: - * "$7$", 3 params encoded as 1+5+5 chars, salt - */ -#define YESCRYPT_PREFIX_LEN (3 + 8 * 6 + 1 + YESCRYPT_BYTES2CHARS(32)) +// How much can save by forcing "standard" value by commenting the next line: +// 160 bytes +//#define YCTX_param_flags yctx->param.flags +// 260 bytes +//#define flags___YESCRYPT_RW (flags & YESCRYPT_RW) +// 140 bytes +//#define flags___YESCRYPT_MODE_MASK (flags & YESCRYPT_MODE_MASK) +// ^^^^ forcing the above since the code already requires (checks for) this +// 50 bytes +#define YCTX_param_N yctx->param.N +// -100 bytes (negative!!!) +#define YCTX_param_r yctx->param.r +// 400 bytes +#define YCTX_param_p yctx->param.p +// 130 bytes +#define YCTX_param_t yctx->param.t +// 2 bytes +#define YCTX_param_g yctx->param.g +// 1 bytes +// ^^^^ this looks wrong, compiler should be able to constant-propagate the fact that NROM code is dead +#define YCTX_param_NROM yctx->param.NROM -#define YESCRYPT_HASH_SIZE 32 -#define YESCRYPT_HASH_LEN YESCRYPT_BYTES2CHARS(YESCRYPT_HASH_SIZE) +// standard ("j9T") values: +#ifndef YCTX_param_flags +#define YCTX_param_flags (YESCRYPT_RW | YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K) +#endif +#ifndef flags___YESCRYPT_RW +#define flags___YESCRYPT_RW ((void)flags, YESCRYPT_RW) +#endif +#ifndef flags___YESCRYPT_MODE_MASK +#define flags___YESCRYPT_MODE_MASK ((void)flags, YESCRYPT_RW) +#endif +#ifndef YCTX_param_N +#define YCTX_param_N 4096 +#endif +#ifndef YCTX_param_r +#define YCTX_param_r 32 +#endif +#ifndef YCTX_param_p +#define YCTX_param_p 1 +#endif +#ifndef YCTX_param_t +#define YCTX_param_t 0 +#endif +#ifndef YCTX_param_g +#define YCTX_param_g 0 +#endif +#ifndef YCTX_param_NROM +#define YCTX_param_NROM 0 +#endif /** * yescrypt_r(shared, local, passwd, passwdlen, setting, key, buf, buflen): diff --git a/testsuite/cryptpw.tests b/testsuite/cryptpw.tests index a17123218e..739fb4e9f6 100755 --- a/testsuite/cryptpw.tests +++ b/testsuite/cryptpw.tests @@ -60,7 +60,7 @@ testing 'cryptpw yescrypt' \ 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123456789012345678901234' \ '$y$j9T$123456789012345678901234$AKxw5OX/T4jD.v./IW.5tE/j7izNjw06fg3OvH1LsN9\n' \ '' '' -testing 'cryptpw yescrypt with non-standard cost 4 instead of 5 (j8T instead of j9T)' \ +testing 'cryptpw yescrypt with non-standard N=2048 instead of 4096 (j8T instead of j9T)' \ 'cryptpw -m yescrypt qweRTY123@-+ j8T\$123456789012345678901234' \ '$y$j8T$123456789012345678901234$JQUUfopCxlfZNE8f.THJwbOkhy.XtB3GIjo9HUVioWB\n' \ '' '' From 63025e8bca426f1013ac0ae7ef103bf8a7f71622 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 10:50:20 +0200 Subject: [PATCH 125/277] libbb/yescrypt: remove forgotten SHA256 HMAC header file Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-sha256.h | 31 ------------------------------- libbb/yescrypt/y.c | 1 - 2 files changed, 32 deletions(-) delete mode 100644 libbb/yescrypt/alg-sha256.h diff --git a/libbb/yescrypt/alg-sha256.h b/libbb/yescrypt/alg-sha256.h deleted file mode 100644 index 862f49dbe3..0000000000 --- a/libbb/yescrypt/alg-sha256.h +++ /dev/null @@ -1,31 +0,0 @@ -/*- - * Copyright 2005-2016 Colin Percival - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* Context structure for HMAC-SHA256 operations. */ -typedef struct { - sha256_ctx_t ictx; - sha256_ctx_t octx; -} HMAC_SHA256_CTX; diff --git a/libbb/yescrypt/y.c b/libbb/yescrypt/y.c index e7d447531e..d5ab8903fe 100644 --- a/libbb/yescrypt/y.c +++ b/libbb/yescrypt/y.c @@ -10,7 +10,6 @@ #include "libbb.h" #define YESCRYPT_INTERNAL -#include "alg-sha256.h" #include "alg-yescrypt.h" #include "alg-sha256.c" #include "alg-yescrypt-kdf.c" From 75758c73608f3c6be9ea2d338a199a8aa11c51e2 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 17:08:32 +0200 Subject: [PATCH 126/277] libbb/yescrypt: accept longer salts (up to 84 chars) function old new delta cryptpw_main 214 223 +9 chpasswd_main 347 356 +9 passwd_main 931 934 +3 yescrypt_r 1084 1056 -28 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 3/1 up/down: 21/-28) Total: -7 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 5 +- libbb/yescrypt/alg-yescrypt-common.c | 76 +++++++++++++++++++++++++++- libbb/yescrypt/alg-yescrypt.h | 5 ++ testsuite/cryptpw.tests | 4 ++ 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index cbf723f7eb..544ca31556 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1819,9 +1819,10 @@ extern int crypt_make_rand64encoded(char *p, int cnt /*, int rnd*/) FAST_FUNC; * "$6$" * #define MAX_PW_SALT_LEN (3 + 16 + 1) * yescrypt: - * "$y$j9T$" + * "$y$" "$" + * (84 chars are ascii64-encoded 64 binary bytes) */ -#define MAX_PW_SALT_LEN (7 + 24 + 1) +#define MAX_PW_SALT_LEN (3 + 8*6 + 1 + 84 + 1) extern char* crypt_make_pw_salt(char p[MAX_PW_SALT_LEN], const char *algo) FAST_FUNC; /* Returns number of lines changed, or -1 on error */ diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index db6e098c7e..1e896df64b 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -63,6 +63,75 @@ static NOINLINE const uint8_t *decode64_uint32( return NULL; } +#if 1 +static const uint8_t *decode64( + uint8_t *dst, size_t *dstlen, + const uint8_t *src, size_t srclen) +{ + size_t dstpos = 0; + + dbg_dec64("src:'%s' len:%d", src, (int)srclen); + for (;;) { + uint32_t c, value = 0; + int bits = 0; + while (srclen != 0) { + srclen--; + c = a2i64(*src); + if (c > 63) { /* bad ascii64 char, stop decoding at it */ + srclen = 0; + break; + } + src++; + value |= c << bits; + bits += 6; + if (bits == 24) /* got 4 chars */ + goto store; + } + /* we read entire src, or met a non-ascii64 char (such as "$") */ + if (bits == 0) + break; + /* else: we got last, partial bit block - store it */ + store: + dbg_dec64(" storing bits:%d v:%08x", bits, (int)SWAP_BE32(value)); //BE to see lsb first + while (dstpos < *dstlen) { + if (srclen == 0 && value == 0) { + /* Example: mkpasswd PWD '$y$j9T$123': + * the "123" is bits:18 value:03,51,00 + * is considered to be 2 bytes, not 3! + * + * '$y$j9T$zzz' in upstream fails outright (3rd byte isn't zero). + * IOW: for upstream, validity of salt depends on VALUE, + * not just size of salt. Which is a bug. + * The '$y$j9T$zzz.' salt is the same + * (it adds 6 zero msbits) but upstream works with it, + * thus '$y$j9T$zzz' should work too and give the same result. + */ + goto end; + } + dstpos++; + *dst++ = value; + value >>= 8; + bits -= 8; + if (bits <= 0) /* can get negative, if we e.g. had 6 bits */ + goto next; + } + dbg_dec64(" ERR: bits:%d dst[] is too small", bits); + goto fail; + next: + if (srclen == 0) + break; + } + end: + /* here, srclen is 0, no need to check */ + *dstlen = dstpos; + dbg_dec64("dec64: OK, dst[%d]", (int)dstpos); + return src; +fail: + *dstlen = 0; + return NULL; +} +#else +/* Buggy (and larger) original code */ static const uint8_t *decode64( uint8_t *dst, size_t *dstlen, const uint8_t *src, size_t srclen) @@ -87,6 +156,7 @@ static const uint8_t *decode64( break; if (bits < 12) /* must have at least one full byte */ goto fail; + dbg_dec64(" storing bits:%d v:%08x", (int)bits, (int)SWAP_BE32(value)); //BE to see lsb first while (dstpos++ < *dstlen) { *dst++ = value; value >>= 8; @@ -104,12 +174,14 @@ static const uint8_t *decode64( if (!srclen && dstpos <= *dstlen) { *dstlen = dstpos; + dbg_dec64("dec64: OK, dst[%d]", (int)dstpos); return src; } fail: - *dstlen = 0; + /* *dstlen = 0; - not needed, caller detects error by seeing NULL */ return NULL; } +#endif static char *encode64( char *dst, size_t dstlen, @@ -189,7 +261,7 @@ char *yescrypt_r( goto fail; if (*src != '$') { src = decode64_uint32(&u32, src, 1); - dbg("yescrypt has extended params:0x%x", (unsigned)have); + dbg("yescrypt has extended params:0x%x", (unsigned)u32); if (u32 & 1) src = decode64_uint32(&yctx->param.p, src, 2); if (u32 & 2) diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 97475d89f3..4554e3de3b 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -33,6 +33,11 @@ # else # define dbg(...) bb_error_msg(__VA_ARGS__) # endif +# if 1 +# define dbg_dec64(...) ((void)0) +# else +# define dbg_dec64(...) bb_error_msg(__VA_ARGS__) +# endif #endif /** diff --git a/testsuite/cryptpw.tests b/testsuite/cryptpw.tests index 739fb4e9f6..ef04e20d74 100755 --- a/testsuite/cryptpw.tests +++ b/testsuite/cryptpw.tests @@ -73,6 +73,10 @@ testing 'cryptpw yescrypt with 3-char salt' \ 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123' \ '$y$j9T$123$A34DMIGUbUIo3bjx66Wtk2IFoREMIw6d49it25KQh2D\n' \ '' '' +testing 'cryptpw yescrypt with 84-char salt (max size)' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123456789012345678901234567890123456789012345678901234567890123456789012345678901234' \ + '$y$j9T$123456789012345678901234567890123456789012345678901234567890123456789012345678901234$ubrUuPCpI97LIMlVMt/A0Mhs/kBK2UBJYcQSxEZSlz4\n' \ + '' '' testing 'cryptpw yescrypt implicit' \ 'cryptpw qweRTY123@-+ \$y\$j9T\$123456789012345678901234' \ '$y$j9T$123456789012345678901234$AKxw5OX/T4jD.v./IW.5tE/j7izNjw06fg3OvH1LsN9\n' \ From 479d8db99faa005783fc483d0830314d6156e51f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 17:26:47 +0200 Subject: [PATCH 127/277] libbb/yescrypt: fix salts ending in dots (corresponding to binary zeros) Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 2 +- testsuite/cryptpw.tests | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 1e896df64b..262fe82fbc 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -94,7 +94,7 @@ static const uint8_t *decode64( store: dbg_dec64(" storing bits:%d v:%08x", bits, (int)SWAP_BE32(value)); //BE to see lsb first while (dstpos < *dstlen) { - if (srclen == 0 && value == 0) { + if (srclen == 0 && value == 0 && bits < 8) { /* Example: mkpasswd PWD '$y$j9T$123': * the "123" is bits:18 value:03,51,00 * is considered to be 2 bytes, not 3! diff --git a/testsuite/cryptpw.tests b/testsuite/cryptpw.tests index ef04e20d74..ab8f32d8e0 100755 --- a/testsuite/cryptpw.tests +++ b/testsuite/cryptpw.tests @@ -65,6 +65,9 @@ testing 'cryptpw yescrypt with non-standard N=2048 instead of 4096 (j8T instead '$y$j8T$123456789012345678901234$JQUUfopCxlfZNE8f.THJwbOkhy.XtB3GIjo9HUVioWB\n' \ '' '' # mkpasswd-5.6.2 allows short salts for yescrypt +# ...but there is a catch. Not all of them. +# The "partial" (not fitting in whole bytes) ascii64-encoded salt +# is a special case. For example, "$zzz" would not even work in upstream. testing 'cryptpw yescrypt with empty salt' \ 'cryptpw -m yescrypt qweRTY123@-+ j9T\$' \ '$y$j9T$$hpeksL94GXNRwnA00L3c8WFy0khFAUbCpBSak.N3Bp.\n' \ @@ -73,6 +76,23 @@ testing 'cryptpw yescrypt with 3-char salt' \ 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123' \ '$y$j9T$123$A34DMIGUbUIo3bjx66Wtk2IFoREMIw6d49it25KQh2D\n' \ '' '' +# "." is not allowed in mkpasswd-5.6.2 +# .................................... +# ".." is decoded into one zero byte (not two) +testing 'cryptpw yescrypt with 2-char salt ".."' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$..' \ + '$y$j9T$..$yVHeOayxOGg6cHL3.dg10u7T.qSgySfLN3uhSVSLNn/\n' \ + '' '' +# "..." is decoded into two zero bytes (not three, not one) +testing 'cryptpw yescrypt with 3-char salt "..."' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$...' \ + '$y$j9T$...$xHvJ5USZ7hFyXYbOijtEOMfZRS23cWIxu2eIBXRymA5\n' \ + '' '' +# "...." is decoded into three zero bytes (no surprises here) +testing 'cryptpw yescrypt with 4-char salt "...."' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$....' \ + '$y$j9T$....$wOnauYL2/NEtr6YQi9pi8AtV7L57sEbVOAnWJIcP9q2\n' \ + '' '' testing 'cryptpw yescrypt with 84-char salt (max size)' \ 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123456789012345678901234567890123456789012345678901234567890123456789012345678901234' \ '$y$j9T$123456789012345678901234567890123456789012345678901234567890123456789012345678901234$ubrUuPCpI97LIMlVMt/A0Mhs/kBK2UBJYcQSxEZSlz4\n' \ From 5e9b95ff40de7e9b3395e51d456647ae99b2e8b2 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 17:42:35 +0200 Subject: [PATCH 128/277] cryptpw: -m sha512crypt must also be accepted Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt.c | 2 +- testsuite/cryptpw.tests | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 4acc330395..3b2fea00da 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -30,7 +30,7 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) *salt_ptr++ = '$'; #if !ENABLE_USE_BB_CRYPT || ENABLE_USE_BB_CRYPT_SHA if ((algo[0]|0x20) == 's') { /* sha */ - salt[1] = '5' + (strcasecmp(algo, "sha512") == 0); + salt[1] = '5' + (strncasecmp(algo, "sha512", 6) == 0); len = 16 / 2; } #endif diff --git a/testsuite/cryptpw.tests b/testsuite/cryptpw.tests index ab8f32d8e0..beac35efe6 100755 --- a/testsuite/cryptpw.tests +++ b/testsuite/cryptpw.tests @@ -49,6 +49,10 @@ testing 'cryptpw sha512' \ 'cryptpw -m sha512 QWErty 123456789012345678901234567890' \ '$6$1234567890123456$KB7QqxFyqmJSWyQYcCuGeFukgz1bPQoipWZf7.9L7z3k8UNTXa6UikbKcUGDc2ANn7DOGmDaroxDgpK16w/RE0\n' \ '' '' +testing 'cryptpw sha512crypt' \ + 'cryptpw -m sha512crypt QWErty 123456789012345678901234567890' \ + '$6$1234567890123456$KB7QqxFyqmJSWyQYcCuGeFukgz1bPQoipWZf7.9L7z3k8UNTXa6UikbKcUGDc2ANn7DOGmDaroxDgpK16w/RE0\n' \ + '' '' testing 'cryptpw sha512 rounds=99999' \ 'cryptpw -m sha512 QWErty rounds=99999\$123456789012345678901234567890' \ '$6$rounds=99999$1234567890123456$BfF6gD6ZjUmwawH5QaAglYAxtU./yvsz0fcQ464l49aMI2DZW3j5ri28CrxK7riPWNpLuUpfaIdY751SBYKUH.\n' \ From e3484095f9fd48d771df4e49db8b6384b5219c21 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 18:03:10 +0200 Subject: [PATCH 129/277] libbb/yescrypt: code shrink function old new delta yescrypt_r 1063 1048 -15 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 262fe82fbc..99e8b1277f 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -126,8 +126,8 @@ static const uint8_t *decode64( *dstlen = dstpos; dbg_dec64("dec64: OK, dst[%d]", (int)dstpos); return src; -fail: - *dstlen = 0; + fail: + /* *dstlen = 0; - not needed, caller detects error by seeing NULL */ return NULL; } #else @@ -177,7 +177,7 @@ static const uint8_t *decode64( dbg_dec64("dec64: OK, dst[%d]", (int)dstpos); return src; } -fail: + fail: /* *dstlen = 0; - not needed, caller detects error by seeing NULL */ return NULL; } From 67b98c7b1b4e0eeeacdd3086baf24248c8ceeee9 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 18:28:56 +0200 Subject: [PATCH 130/277] libbb/yescrypt: no need to find salt-terminating "$" twice function old new delta yescrypt_r 1048 1029 -19 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 99e8b1277f..dfcf32145c 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -66,7 +66,7 @@ static NOINLINE const uint8_t *decode64_uint32( #if 1 static const uint8_t *decode64( uint8_t *dst, size_t *dstlen, - const uint8_t *src, size_t srclen) + const uint8_t *src) { size_t dstpos = 0; @@ -74,11 +74,9 @@ static const uint8_t *decode64( for (;;) { uint32_t c, value = 0; int bits = 0; - while (srclen != 0) { - srclen--; + while (*src && *src != '$') { c = a2i64(*src); if (c > 63) { /* bad ascii64 char, stop decoding at it */ - srclen = 0; break; } src++; @@ -94,7 +92,7 @@ static const uint8_t *decode64( store: dbg_dec64(" storing bits:%d v:%08x", bits, (int)SWAP_BE32(value)); //BE to see lsb first while (dstpos < *dstlen) { - if (srclen == 0 && value == 0 && bits < 8) { + if (!(*src && *src != '$') && value == 0 && bits < 8) { /* Example: mkpasswd PWD '$y$j9T$123': * the "123" is bits:18 value:03,51,00 * is considered to be 2 bytes, not 3! @@ -118,7 +116,7 @@ static const uint8_t *decode64( dbg_dec64(" ERR: bits:%d dst[] is too small", bits); goto fail; next: - if (srclen == 0) + if (!*src || *src == '$') break; } end: @@ -214,8 +212,8 @@ char *yescrypt_r( yescrypt_ctx_t yctx[1]; unsigned char hashbin32[32]; char *dst; - const uint8_t *src, *saltstr, *saltend; - size_t need, prefixlen, saltstrlen; + const uint8_t *src, *saltend; + size_t need, prefixlen; uint32_t u32; memset(yctx, 0, sizeof(yctx)); @@ -280,17 +278,13 @@ char *yescrypt_r( goto fail; } - saltstr = src + 1; - src = (uint8_t *)strchrnul((char *)saltstr, '$'); - prefixlen = src - setting; /* len("$y$$") */ - saltstrlen = src - saltstr; /* len("") */ - /* src points to end of salt ('$' or NUL byte), won't be used past this point */ - yctx->saltlen = sizeof(yctx->salt); - saltend = decode64(yctx->salt, &yctx->saltlen, saltstr, saltstrlen); - if (saltend != saltstr + saltstrlen) + src++; /* now points to salt */ + saltend = decode64(yctx->salt, &yctx->saltlen, src); + if (!saltend || (*saltend != '\0' && *saltend != '$')) goto fail; /* salt[] is too small, or bad char during decode */ + prefixlen = saltend - setting; need = prefixlen + 1 + YESCRYPT_HASH_LEN + 1; if (need > buflen || need < prefixlen) goto fail; From 7798f651a48926636944d556a158a9b569a56367 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 21:36:31 +0200 Subject: [PATCH 131/277] add libbb/yescrypt/PARAMETERS Signed-off-by: Denys Vlasenko --- libbb/yescrypt/PARAMETERS | 196 +++++++++++++++++++++++++++ libbb/yescrypt/alg-sha256.c | 9 +- libbb/yescrypt/alg-yescrypt-common.c | 3 +- 3 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 libbb/yescrypt/PARAMETERS diff --git a/libbb/yescrypt/PARAMETERS b/libbb/yescrypt/PARAMETERS new file mode 100644 index 0000000000..d9f5d24e6c --- /dev/null +++ b/libbb/yescrypt/PARAMETERS @@ -0,0 +1,196 @@ + Optimal yescrypt configuration. + +yescrypt is very flexible, but configuring it optimally is complicated. +Here are some guidelines to simplify near-optimal configuration. We +start by listing the parameters and their typical values, and then give +currently recommended parameter sets by use case. + + + Parameters and their typical values. + +Set flags (yescrypt flavor) to YESCRYPT_DEFAULTS to use the currently +recommended flavor. (Other flags values exist for compatibility and for +specialized cases where you think you know what you're doing.) + +Set N (block count) based on target memory usage and running time, as +well as on the value of r (block size in 128 byte units). N must be a +power of two. + +Set r (block size) to 8 (so that N is in KiB, which is convenient) or to +another small value (if more optimal or for fine-tuning of the total +size and/or running time). Reasonable values for r are from 8 to 96. + +Set p (parallelism) to 1 meaning no thread-level parallelism within one +computation of yescrypt. (Use of thread-level parallelism within +yescrypt makes sense for ROM initialization and for key derivation at +high memory usage, but usually not for password hashing where +parallelism is available through concurrent authentication attempts. +Don't use p > 1 unnecessarily.) + +Set t (time) to 0 to use the optimal running time for a given memory +usage. This will allow you to maximize the memory usage (the value of +N*r) while staying within your running time constraints. (Non-zero t +makes sense in specialized cases where you can't afford higher memory +usage but can afford more time.) + +Set g (upgrades) to 0 because there have been no hash upgrades yet. + +Set NROM (block count of ROM) to 0 unless you use a ROM (see below). +NROM must be a power of two. + + + Password hashing for user authentication, no ROM. + +Small and fast (memory usage 2 MiB, performance like bcrypt cost 2^5 - +latency 2-3 ms and throughput 10,000+ per second on a 16-core server): + +flags = YESCRYPT_DEFAULTS, N = 2048, r = 8, p = 1, t = 0, g = 0, NROM = 0 + +Large and slow (memory usage 16 MiB, performance like bcrypt cost 2^8 - +latency 10-30 ms and throughput 1000+ per second on a 16-core server): + +flags = YESCRYPT_DEFAULTS, N = 4096, r = 32, p = 1, t = 0, g = 0, NROM = 0 + +Of course, even heavier and slower settings are possible, if affordable. +Simply double the value of N as many times as needed. Since N must be a +power of two, you may use r (in the range of 8 to 32) or/and t (in the +range of 0 to 2) for fine-tuning the running time, but first bring N to +the maximum you can afford. If this feels too complicated, just use one +of the two parameter sets given above (preferably the second) as-is. + + + Password hashing for user authentication, with ROM. + +It's similar to the above, except that you need to adjust r, set NROM, +and initialize the ROM. + +First decide on a ROM size, such as making it a large portion of your +dedicated authentication servers' RAM sizes. Since NROM (block count) +must be a power of two, you might need to choose r (block size) based on +how your desired ROM size corresponds to a power of two. Also tuning +for performance on current hardware, you'll likely end up with r in the +range from slightly below 16 to 32. For example, to use 15/16 of a +server's 256 GiB RAM as ROM (thus, making it 240 GiB), you could use +r=15 or r=30. To use 23/24 of a server's 384 GiB RAM as ROM (thus, +making it 368 GiB), you'd use r=23. Then set NROM to your desired ROM +size in KiB divided by 128*r. Note that these examples might (or might +not) be too extreme, leaving little memory for the rest of the system. +You could as well opt for 7/8 with r=14 or 11/12 with r=11 or r=22. + +Note that higher r may make placing of ROM in e.g. NVMe flash memory +instead of in RAM more reasonable (or less unreasonable) than it would +have been with a lower r. If this is a concern as it relates to +possible attacks and you do not intend to ever do it defensively, you +might want to keep r lower (e.g., prefer r=15 over r=30 in the example +above, even if 30 performs slightly faster). + +Your adjustments to r, if you deviate from powers of two, will also +result in weirder memory usage per hash. Like 1.75 MiB at r=14 instead +of 2 MiB at r=8 that you would have used without a ROM. That's OK. + +For ROM initialization, which you do with yescrypt_init_shared(), use +the same r and NROM that you'd later use for password hashing, choose p +based on your servers' physical and/or logical CPU count (maybe +considering eventual upgrades as you won't be able to change this later, +but without going unnecessarily high - e.g., p=28, p=56, or p=112 make +sense on servers that currently have 28 physical / 56 logical CPUs), and +set the rest of the parameters to: + +flags = YESCRYPT_DEFAULTS, N = 0, t = 0, g = 0 + +N is set to 0 because it isn't relevant during ROM initialization (you +can use different values of N for hashing passwords with the same ROM). + +To keep the ROM in e.g. SysV shared memory and reuse it across your +authentication service restarts, you'd need to allocate the memory and +set the flags to "YESCRYPT_DEFAULTS | YESCRYPT_SHARED_PREALLOCATED". + +For actual password hashing, you'd use your chosen values for N, r, +NROM, and set the rest of the parameters to: + +flags = YESCRYPT_DEFAULTS, p = 1, t = 0, g = 0 + +Note that although you'd use a large p for ROM initialization, you +should use p=1 for actual password hashing like you would without a ROM. + +Do not forget to pass the ROM into the actual password hashing (and keep +r and NROM set accordingly). + +Since N must be a power of two and r is dependent on ROM size, you may +use t (in the range of 0 to 2) for fine-tuning the running time, but +first bring N to the maximum you can afford. + +If this feels too complicated, or even if it doesn't, please consider +engaging Openwall for your yescrypt deployment. We'd be happy to help. + + + Password-based key derivation. + +(Or rather passphrase-based.) + +Use settings similar to those for password hashing without a ROM, but +adjusted for higher memory usage and running time, and optionally with +thread-level parallelism. + +Small and fast (memory usage 128 MiB, running time under 100 ms on a +fast desktop): + +flags = YESCRYPT_DEFAULTS, N = 32768, r = 32, p = 1, t = 0, g = 0, NROM = 0 + +Large and fast (memory usage 1 GiB, running time under 200 ms on a fast +quad-core desktop not including memory allocation overhead, under 250 ms +with the overhead included), but requires build with OpenMP support (or +otherwise will run as slow as yet be weaker than its p=1 alternative): + +flags = YESCRYPT_DEFAULTS, N = 262144, r = 32, p = 4, t = 0, g = 0, NROM = 0 + +Large and slower (memory usage 1 GiB, running time under 300 ms on a +fast quad-core desktop not including memory allocation overhead, under +350 ms with the overhead included), also requires build with OpenMP +support (or otherwise will run slower than the p=1 alternative below): + +flags = YESCRYPT_DEFAULTS, N = 262144, r = 32, p = 4, t = 2, g = 0, NROM = 0 + +Large and slow (memory usage 1 GiB, running time under 600 ms on a fast +desktop not including memory allocation overhead, under 650 ms with the +overhead included): + +flags = YESCRYPT_DEFAULTS, N = 262144, r = 32, p = 1, t = 0, g = 0, NROM = 0 + +Just like with password hashing, even heavier and slower settings are +possible, if affordable, and you achieve them by adjusting N, r, t in +the same way and in the same preferred ranges (please see the section on +password hashing without a ROM, above). Unlike with password hashing, +it makes some sense to go above t=2 if you expect that your users might +not be able to afford more memory but can afford more time. However, +increasing the memory usage provides better protection, and we don't +recommend forcing your users to wait for more than 1 second as they +could as well type more characters in that time. If this feels too +complicated, just use one of the above parameter sets as-is. + + + Amortization of memory allocation overhead. + +It takes a significant fraction of yescrypt's total running time to +allocate memory from the operating system, especially considering that +the kernel zeroizes the memory before handing it over to your program. + +Unless you naturally need to compute yescrypt just once per process, you +may achieve greater efficiency by fully using advanced yescrypt APIs +that let you preserve and reuse the memory allocation across yescrypt +invocations. This is done by reusing the structure pointed to by the +"yescrypt_local_t *local" argument of yescrypt_r() or yescrypt_kdf() +without calling yescrypt_free_local() inbetween the repeated invocations +of yescrypt. + + + YESCRYPT_DEFAULTS macro. + +Please note that the value of the YESCRYPT_DEFAULTS macro might change +later, so if you use the macro like it's recommended here then for +results reproducible across versions you might need to store its value +somewhere along with the hashes or the encrypted data. + +If you use yescrypt's standard hash string encoding, then yescrypt +already encodes and decodes this value for you, so you don't need to +worry about this. diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 1ccffa1e55..25446406b4 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -36,7 +36,7 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, uint64_t c, uint8_t *buf, size_t dkLen) { hmac_ctx_t Phctx, PShctx; - size_t i; + uint32_t i; /* Compute HMAC state after processing P. */ hmac_begin(&Phctx, passwd, passwdlen, sha256_begin); @@ -46,7 +46,7 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, hmac_hash(&PShctx, salt, saltlen); /* Iterate through the blocks. */ - for (i = 0; dkLen != 0; i++) { + for (i = 0; dkLen != 0; ) { uint64_t U[32 / 8]; uint64_t T[32 / 8]; uint64_t j; @@ -54,8 +54,9 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, size_t clen; int k; - /* Generate INT(i + 1). */ - ivec = SWAP_BE32((uint32_t)(i + 1)); + /* Generate INT(i). */ + i++; + ivec = SWAP_BE32(i); /* Compute U_1 = PRF(P, S || INT(i)). */ hmac_peek_hash(&PShctx, (void*)T, &ivec, 4, NULL); diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index dfcf32145c..5d8be587ae 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -92,7 +92,7 @@ static const uint8_t *decode64( store: dbg_dec64(" storing bits:%d v:%08x", bits, (int)SWAP_BE32(value)); //BE to see lsb first while (dstpos < *dstlen) { - if (!(*src && *src != '$') && value == 0 && bits < 8) { + if ((!*src || *src == '$') && value == 0 && bits < 8) { /* Example: mkpasswd PWD '$y$j9T$123': * the "123" is bits:18 value:03,51,00 * is considered to be 2 bytes, not 3! @@ -120,7 +120,6 @@ static const uint8_t *decode64( break; } end: - /* here, srclen is 0, no need to check */ *dstlen = dstpos; dbg_dec64("dec64: OK, dst[%d]", (int)dstpos); return src; From e5d3a87633eac2a8a17d909b98a1e6dd21f80489 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 21:52:39 +0200 Subject: [PATCH 132/277] libbb/yescrypt: 64-bit build fixes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 13ae62b7cf..d24b051507 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -823,6 +823,7 @@ static int yescrypt_kdf32_body( salsa20_blk_t *V, *XY; uint8_t sha256[32]; uint8_t dk[sizeof(sha256)], *dkp = buf32; + uint32_t r, p; /* Sanity-check parameters */ switch (flags___YESCRYPT_MODE_MASK) { @@ -849,13 +850,9 @@ static int yescrypt_kdf32_body( default: goto out_EINVAL; } -#if SIZE_MAX > UINT32_MAX - if (buflen > (((uint64_t)1 << 32) - 1) * 32) - goto out_EINVAL; -#endif - { - const uint32_t r = YCTX_param_r; - const uint32_t p = YCTX_param_p; + + r = YCTX_param_r; + p = YCTX_param_p; if ((uint64_t)r * (uint64_t)p >= 1 << 30) goto out_EINVAL; if (N > UINT32_MAX) @@ -982,7 +979,6 @@ static int yescrypt_kdf32_body( out_EINVAL: errno = EINVAL; return -1; - } } /** From f8e9bd30d73f2acf6818da71a2ba44748151b716 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 22:34:31 +0200 Subject: [PATCH 133/277] libbb/yescrypt: disable unrolling in two places Also, make many define macros safer function old new delta blockmix 2300 814 -1486 blockmix_xor 4606 1543 -3063 blockmix_xor_save 4737 1620 -3117 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/3 up/down: 0/-7666) Total: -7666 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 255 +++++++++++++++++++----------- 1 file changed, 159 insertions(+), 96 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index d24b051507..ab095eae16 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -42,6 +42,15 @@ #define unlikely(exp) (exp) #endif +// Not a size win if 0 +#define UNROLL_COPY 1 + +// -5324 bytes if 0: +#define UNROLL_PWXFORM_ROUND 0 +// -4864 bytes if 0: +#define UNROLL_PWXFORM 0 +// both 0: -7666 bytes + typedef union { uint32_t w[16]; uint64_t d[8]; @@ -52,15 +61,17 @@ static void salsa20_simd_shuffle( salsa20_blk_t *Bout) { #define COMBINE(out, in1, in2) \ - Bout->d[out] = Bin->w[in1 * 2] | ((uint64_t)Bin->w[in2 * 2 + 1] << 32); - COMBINE(0, 0, 2) - COMBINE(1, 5, 7) - COMBINE(2, 2, 4) - COMBINE(3, 7, 1) - COMBINE(4, 4, 6) - COMBINE(5, 1, 3) - COMBINE(6, 6, 0) - COMBINE(7, 3, 5) +do { \ + Bout->d[out] = Bin->w[in1 * 2] | ((uint64_t)Bin->w[in2 * 2 + 1] << 32); \ +} while (0) + COMBINE(0, 0, 2); + COMBINE(1, 5, 7); + COMBINE(2, 2, 4); + COMBINE(3, 7, 1); + COMBINE(4, 4, 6); + COMBINE(5, 1, 3); + COMBINE(6, 6, 0); + COMBINE(7, 3, 5); #undef COMBINE } @@ -69,25 +80,29 @@ static void salsa20_simd_unshuffle( salsa20_blk_t *Bout) { #define UNCOMBINE(out, in1, in2) \ +do { \ Bout->w[out * 2] = Bin->d[in1]; \ - Bout->w[out * 2 + 1] = Bin->d[in2] >> 32; - UNCOMBINE(0, 0, 6) - UNCOMBINE(1, 5, 3) - UNCOMBINE(2, 2, 0) - UNCOMBINE(3, 7, 5) - UNCOMBINE(4, 4, 2) - UNCOMBINE(5, 1, 7) - UNCOMBINE(6, 6, 4) - UNCOMBINE(7, 3, 1) + Bout->w[out * 2 + 1] = Bin->d[in2] >> 32; \ +} while (0) + UNCOMBINE(0, 0, 6); + UNCOMBINE(1, 5, 3); + UNCOMBINE(2, 2, 0); + UNCOMBINE(3, 7, 5); + UNCOMBINE(4, 4, 2); + UNCOMBINE(5, 1, 7); + UNCOMBINE(6, 6, 4); + UNCOMBINE(7, 3, 1); #undef UNCOMBINE } #define DECL_X \ - salsa20_blk_t X; + salsa20_blk_t X #define DECL_Y \ - salsa20_blk_t Y; + salsa20_blk_t Y +#if UNROLL_COPY #define COPY(out, in) \ +do { \ (out).d[0] = (in).d[0]; \ (out).d[1] = (in).d[1]; \ (out).d[2] = (in).d[2]; \ @@ -95,9 +110,17 @@ static void salsa20_simd_unshuffle( (out).d[4] = (in).d[4]; \ (out).d[5] = (in).d[5]; \ (out).d[6] = (in).d[6]; \ - (out).d[7] = (in).d[7]; + (out).d[7] = (in).d[7]; \ +} while (0) +#else +#define COPY(out, in) \ +do { \ + for (int copyi=0; copyi<8; copyi++) \ + (out).d[copyi] = (in).d[copyi]; \ +} while (0) +#endif -#define READ_X(in) COPY(X, in) +#define READ_X(in) COPY(X, in) #define WRITE_X(out) COPY(out, X) /** @@ -154,7 +177,6 @@ static void salsa20(salsa20_blk_t *restrict B, B->w[i + 3] = Bout->w[i + 3] += B->w[i + 3]; } } - #if 0 /* Too expensive */ explicit_bzero(&X, sizeof(X)); @@ -165,9 +187,10 @@ static void salsa20(salsa20_blk_t *restrict B, * Apply the Salsa20/2 core to the block provided in X. */ #define SALSA20_2(out) \ - salsa20(&X, &out, 1); + salsa20(&X, &out, 1) #define XOR(out, in1, in2) \ +do { \ (out).d[0] = (in1).d[0] ^ (in2).d[0]; \ (out).d[1] = (in1).d[1] ^ (in2).d[1]; \ (out).d[2] = (in1).d[2] ^ (in2).d[2]; \ @@ -175,23 +198,28 @@ static void salsa20(salsa20_blk_t *restrict B, (out).d[4] = (in1).d[4] ^ (in2).d[4]; \ (out).d[5] = (in1).d[5] ^ (in2).d[5]; \ (out).d[6] = (in1).d[6] ^ (in2).d[6]; \ - (out).d[7] = (in1).d[7] ^ (in2).d[7]; + (out).d[7] = (in1).d[7] ^ (in2).d[7]; \ +} while (0) -#define XOR_X(in) XOR(X, X, in) +#define XOR_X(in) XOR(X, X, in) #define XOR_X_2(in1, in2) XOR(X, in1, in2) #define XOR_X_WRITE_XOR_Y_2(out, in) \ - XOR(Y, out, in) \ - COPY(out, Y) \ - XOR(X, X, Y) +do { \ + XOR(Y, out, in); \ + COPY(out, Y); \ + XOR(X, X, Y); \ +} while (0) /** * Apply the Salsa20/8 core to the block provided in X ^ in. */ #define SALSA20_8_XOR_MEM(in, out) \ +do { \ XOR_X(in); \ - salsa20(&X, &out, 4); + salsa20(&X, &out, 4); \ +} while (0) -#define INTEGERIFY (uint32_t)X.d[0] +#define INTEGERIFY ((uint32_t)X.d[0]) /** * blockmix_salsa8(Bin, Bout, r): @@ -204,12 +232,12 @@ static void blockmix_salsa8( size_t r) { size_t i; - DECL_X + DECL_X; - READ_X(Bin[r * 2 - 1]) + READ_X(Bin[r * 2 - 1]); for (i = 0; i < r; i++) { - SALSA20_8_XOR_MEM(Bin[i * 2], Bout[i]) - SALSA20_8_XOR_MEM(Bin[i * 2 + 1], Bout[r + i]) + SALSA20_8_XOR_MEM(Bin[i * 2], Bout[i]); + SALSA20_8_XOR_MEM(Bin[i * 2 + 1], Bout[r + i]); } } @@ -220,14 +248,14 @@ static uint32_t blockmix_salsa8_xor( size_t r) { size_t i; - DECL_X + DECL_X; - XOR_X_2(Bin1[r * 2 - 1], Bin2[r * 2 - 1]) + XOR_X_2(Bin1[r * 2 - 1], Bin2[r * 2 - 1]); for (i = 0; i < r; i++) { - XOR_X(Bin1[i * 2]) - SALSA20_8_XOR_MEM(Bin2[i * 2], Bout[i]) - XOR_X(Bin1[i * 2 + 1]) - SALSA20_8_XOR_MEM(Bin2[i * 2 + 1], Bout[r + i]) + XOR_X(Bin1[i * 2]); + SALSA20_8_XOR_MEM(Bin2[i * 2], Bout[i]); + XOR_X(Bin1[i * 2 + 1]); + SALSA20_8_XOR_MEM(Bin2[i * 2 + 1], Bout[r + i]); } return INTEGERIFY; @@ -242,27 +270,38 @@ static uint32_t blockmix_salsa8_xor( /* Derived values. Not tunable except via Swidth above. */ #define PWXbytes (PWXgather * PWXsimple * 8) -#define Sbytes (3 * (1 << Swidth) * PWXsimple * 8) -#define Smask (((1 << Swidth) - 1) * PWXsimple * 8) -#define Smask2 (((uint64_t)Smask << 32) | Smask) +#define Sbytes (3 * (1 << Swidth) * PWXsimple * 8) +#define Smask (((1 << Swidth) - 1) * PWXsimple * 8) +#define Smask2 (((uint64_t)Smask << 32) | Smask) -#define DECL_SMASK2REG /* empty */ -#define FORCE_REGALLOC_3 /* empty */ -#define MAYBE_MEMORY_BARRIER /* empty */ +#define DECL_SMASK2REG do {} while (0) +#define FORCE_REGALLOC_3 do {} while (0) +#define MAYBE_MEMORY_BARRIER do {} while (0) -#define PWXFORM_SIMD(x0, x1) { \ +#define PWXFORM_SIMD(x0, x1) \ +do { \ uint64_t x = x0 & Smask2; \ uint64_t *p0 = (uint64_t *)(S0 + (uint32_t)x); \ uint64_t *p1 = (uint64_t *)(S1 + (x >> 32)); \ x0 = ((x0 >> 32) * (uint32_t)x0 + p0[0]) ^ p1[0]; \ x1 = ((x1 >> 32) * (uint32_t)x1 + p0[1]) ^ p1[1]; \ -} +} while (0) +#if UNROLL_PWXFORM_ROUND +#define PWXFORM_ROUND \ +do { \ + PWXFORM_SIMD(X.d[0], X.d[1]); \ + PWXFORM_SIMD(X.d[2], X.d[3]); \ + PWXFORM_SIMD(X.d[4], X.d[5]); \ + PWXFORM_SIMD(X.d[6], X.d[7]); \ +} while (0) +#else #define PWXFORM_ROUND \ - PWXFORM_SIMD(X.d[0], X.d[1]) \ - PWXFORM_SIMD(X.d[2], X.d[3]) \ - PWXFORM_SIMD(X.d[4], X.d[5]) \ - PWXFORM_SIMD(X.d[6], X.d[7]) +do { \ + for (int pwxi=0; pwxi<8; pwxi+=2) \ + PWXFORM_SIMD(X.d[pwxi], X.d[pwxi + 1]); \ +} while (0) +#endif /* * This offset helps address the 256-byte write block via the single-byte @@ -275,19 +314,23 @@ static uint32_t blockmix_salsa8_xor( #define PWXFORM_WRITE_OFFSET 0x7c #define PWXFORM_WRITE \ - WRITE_X(*(salsa20_blk_t *)(Sw - PWXFORM_WRITE_OFFSET)) \ - Sw += 64; - -#define PWXFORM { \ +do { \ + WRITE_X(*(salsa20_blk_t *)(Sw - PWXFORM_WRITE_OFFSET)); \ + Sw += 64; \ +} while (0) + +#if UNROLL_PWXFORM +#define PWXFORM \ +do { \ uint8_t *Sw = S2 + w + PWXFORM_WRITE_OFFSET; \ - FORCE_REGALLOC_3 \ - MAYBE_MEMORY_BARRIER \ - PWXFORM_ROUND \ - PWXFORM_ROUND PWXFORM_WRITE \ - PWXFORM_ROUND PWXFORM_WRITE \ - PWXFORM_ROUND PWXFORM_WRITE \ - PWXFORM_ROUND PWXFORM_WRITE \ - PWXFORM_ROUND \ + FORCE_REGALLOC_3; \ + MAYBE_MEMORY_BARRIER; \ + PWXFORM_ROUND; \ + PWXFORM_ROUND; PWXFORM_WRITE; \ + PWXFORM_ROUND; PWXFORM_WRITE; \ + PWXFORM_ROUND; PWXFORM_WRITE; \ + PWXFORM_ROUND; PWXFORM_WRITE; \ + PWXFORM_ROUND; \ w = (w + 64 * 4) & Smask2; \ { \ uint8_t *Stmp = S2; \ @@ -295,7 +338,27 @@ static uint32_t blockmix_salsa8_xor( S1 = S0; \ S0 = Stmp; \ } \ -} +} while (0) +#else +#define PWXFORM \ +do { \ + uint8_t *Sw = S2 + w + PWXFORM_WRITE_OFFSET; \ + FORCE_REGALLOC_3; \ + MAYBE_MEMORY_BARRIER; \ + PWXFORM_ROUND; \ + for (int pwxj=0; pwxj<4; pwxj++) {\ + PWXFORM_ROUND; PWXFORM_WRITE; \ + } \ + PWXFORM_ROUND; \ + w = (w + 64 * 4) & Smask2; \ + { \ + uint8_t *Stmp = S2; \ + S2 = S1; \ + S1 = S0; \ + S0 = Stmp; \ + } \ +} while (0) +#endif typedef struct { uint8_t *S0, *S1, *S2; @@ -318,29 +381,29 @@ static void blockmix( uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; size_t i; - DECL_X + DECL_X; /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; - READ_X(Bin[r]) + READ_X(Bin[r]); - DECL_SMASK2REG + DECL_SMASK2REG; i = 0; do { - XOR_X(Bin[i]) - PWXFORM + XOR_X(Bin[i]); + PWXFORM; if (unlikely(i >= r)) break; - WRITE_X(Bout[i]) + WRITE_X(Bout[i]); i++; } while (1); ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; ctx->w = w; - SALSA20_2(Bout[i]) + SALSA20_2(Bout[i]); } static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, @@ -352,31 +415,31 @@ static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; size_t i; - DECL_X + DECL_X; /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; - XOR_X_2(Bin1[r], Bin2[r]) + XOR_X_2(Bin1[r], Bin2[r]); - DECL_SMASK2REG + DECL_SMASK2REG; i = 0; r--; do { - XOR_X(Bin1[i]) - XOR_X(Bin2[i]) - PWXFORM - WRITE_X(Bout[i]) + XOR_X(Bin1[i]); + XOR_X(Bin2[i]); + PWXFORM; + WRITE_X(Bout[i]); - XOR_X(Bin1[i + 1]) - XOR_X(Bin2[i + 1]) - PWXFORM + XOR_X(Bin1[i + 1]); + XOR_X(Bin2[i + 1]); + PWXFORM; if (unlikely(i >= r)) break; - WRITE_X(Bout[i + 1]) + WRITE_X(Bout[i + 1]); i += 2; } while (1); @@ -385,7 +448,7 @@ static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; ctx->w = w; - SALSA20_2(Bout[i]) + SALSA20_2(Bout[i]); return INTEGERIFY; } @@ -399,30 +462,30 @@ static uint32_t blockmix_xor_save( uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; size_t i; - DECL_X - DECL_Y + DECL_X; + DECL_Y; /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; - XOR_X_2(Bin1out[r], Bin2[r]) + XOR_X_2(Bin1out[r], Bin2[r]); - DECL_SMASK2REG + DECL_SMASK2REG; i = 0; r--; do { - XOR_X_WRITE_XOR_Y_2(Bin2[i], Bin1out[i]) - PWXFORM - WRITE_X(Bin1out[i]) + XOR_X_WRITE_XOR_Y_2(Bin2[i], Bin1out[i]); + PWXFORM; + WRITE_X(Bin1out[i]); - XOR_X_WRITE_XOR_Y_2(Bin2[i + 1], Bin1out[i + 1]) - PWXFORM + XOR_X_WRITE_XOR_Y_2(Bin2[i + 1], Bin1out[i + 1]); + PWXFORM; if (unlikely(i >= r)) break; - WRITE_X(Bin1out[i + 1]) + WRITE_X(Bin1out[i + 1]); i += 2; } while (1); @@ -431,7 +494,7 @@ static uint32_t blockmix_xor_save( ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; ctx->w = w; - SALSA20_2(Bin1out[i]) + SALSA20_2(Bin1out[i]); return INTEGERIFY; } From 8466c3e78fa10d1a3e2bf1a75657fd6d1f4aec30 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 7 Jul 2025 23:07:58 +0200 Subject: [PATCH 134/277] libbb/yescrypt: madvise(MADV_HUGEPAGE) our usually very large allocation Nearly ~2 faster run when buffer is gigabytes in size function old new delta yescrypt_kdf32_body 1386 1406 +20 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index ab095eae16..4c2cfe8494 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -832,14 +832,25 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, static void alloc_region(yescrypt_region_t *region, size_t size) { + uint8_t *base; int flags = # ifdef MAP_NOCORE /* huh? */ MAP_NOCORE | # endif MAP_ANON | MAP_PRIVATE; - uint8_t *base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + + base = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); if (base == MAP_FAILED) bb_die_memory_exhausted(); + +#if defined(MADV_HUGEPAGE) + /* Reduces mkpasswd qweRTY123@-+ '$y$jHT$123' + * (which allocates 4 Gbytes) + * run time from 10.543s to 5.635s + * Seen on linux-5.18.0. + */ + madvise(base, size, MADV_HUGEPAGE); +#endif //region->base = base; //region->base_size = size; region->aligned = base; @@ -960,7 +971,7 @@ static int yescrypt_kdf32_body( if (yctx->local->aligned_size < need) { free_region(yctx->local); alloc_region(yctx->local, need); - dbg("allocated local:%u 0x%x", need, need); + dbg("allocated local:%lu 0x%lx", (long)need, (long)need); /* standard "j9T" params allocate 16Mbytes here */ } if (flags & YESCRYPT_ALLOC_ONLY) From d18ac080e4bb7d63e0ec0dea16bacc6ac455f390 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Jul 2025 00:14:24 +0200 Subject: [PATCH 135/277] libbb/yescrypt: code shrink Setting EINVAL in errno is not necessary, just error return works. function old new delta yescrypt_kdf32_body 1434 1423 -11 yescrypt_r 1029 990 -39 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-50) Total: -50 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 4 +- libbb/yescrypt/alg-yescrypt-kdf.c | 64 ++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 5d8be587ae..e5d0450586 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -288,8 +288,10 @@ char *yescrypt_r( if (need > buflen || need < prefixlen) goto fail; - if (yescrypt_kdf32(yctx, passwd, passwdlen, hashbin32)) + if (yescrypt_kdf32(yctx, passwd, passwdlen, hashbin32)) { + dbg("error in yescrypt_kdf32"); goto fail; + } dst = mempcpy(buf, setting, prefixlen); *dst++ = '$'; diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 4c2cfe8494..c998de51d9 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -927,22 +927,42 @@ static int yescrypt_kdf32_body( r = YCTX_param_r; p = YCTX_param_p; - if ((uint64_t)r * (uint64_t)p >= 1 << 30) + if ((uint64_t)r * (uint64_t)p >= 1 << 30) { + dbg("r * n >= 2^30"); goto out_EINVAL; - if (N > UINT32_MAX) + } + if (N > UINT32_MAX) { + dbg("N > 0x%lx", (long)UINT32_MAX); goto out_EINVAL; - if ((N & (N - 1)) != 0 || N <= 3 || r < 1 || p < 1) + } + if ((N & (N - 1)) != 0 +//TODO: ^^^^^^^^^^^^^^^^^^^^^^ do not check this, code guarantees power-of-2 + || N <= 3 + || r < 1 + || p < 1 + ) { + dbg("bad N, r or p"); goto out_EINVAL; - if (r > SIZE_MAX / 256 / p || - N > SIZE_MAX / 128 / r) + } + if (r > SIZE_MAX / 256 / p + || N > SIZE_MAX / 128 / r + ) { + /* 32-bit testcase: mkpasswd qweRTY123@-+ '$y$jHT$123' + * (works on 64-bit, needs buffer > 4Gbytes) + */ + dbg("r > SIZE_MAX / 256 / p? %c", "NY"[r > SIZE_MAX / 256 / p]); + dbg("N > SIZE_MAX / 128 / r? %c", "NY"[N > SIZE_MAX / 128 / r]); goto out_EINVAL; + } if (flags___YESCRYPT_RW) { /* p cannot be greater than SIZE_MAX/Salloc on 64-bit systems, but it can on 32-bit systems. */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wtype-limits" - if (N / p <= 3 || p > SIZE_MAX / Salloc) + if (N / p <= 3 || p > SIZE_MAX / Salloc) { + dbg("bad p:%ld", (long)p); goto out_EINVAL; + } #pragma GCC diagnostic pop } @@ -956,17 +976,23 @@ static int yescrypt_kdf32_body( need = V_size; B_size = (size_t)128 * r * p; need += B_size; - if (need < B_size) + if (need < B_size) { + dbg("integer overflow at += B_size(%lu)", (long)B_size); goto out_EINVAL; + } XY_size = (size_t)256 * r; need += XY_size; - if (need < XY_size) + if (need < XY_size) { + dbg("integer overflow at += XY_size(%lu)", (long)XY_size); goto out_EINVAL; + } if (flags___YESCRYPT_RW) { size_t S_size = (size_t)Salloc * p; need += S_size; - if (need < S_size) + if (need < S_size) { + dbg("integer overflow at += S_size(%lu)", (long)S_size); goto out_EINVAL; + } } if (yctx->local->aligned_size < need) { free_region(yctx->local); @@ -1050,8 +1076,8 @@ static int yescrypt_kdf32_body( /* Success! */ return 0; -out_EINVAL: - errno = EINVAL; + out_EINVAL: + //bbox does not need this: errno = EINVAL; return -1; } @@ -1079,7 +1105,7 @@ int yescrypt_kdf32( /* Support for hash upgrades has been temporarily removed */ if (g) { - errno = EINVAL; + //bbox does not need this: errno = EINVAL; return -1; } @@ -1093,15 +1119,17 @@ int yescrypt_kdf32( flags | YESCRYPT_ALLOC_ONLY, N, t, buf32) != -3 ) { - errno = EINVAL; + dbg("yescrypt_kdf32_body: not -3"); return -1; } retval = yescrypt_kdf32_body(yctx, passwd, passwdlen, flags | YESCRYPT_PREHASH, N >> 6, 0, dk32); - if (retval) + if (retval) { + dbg("yescrypt_kdf32_body(PREHASH):%d", retval); return retval; + } passwd = dk32; passwdlen = sizeof(dk32); } @@ -1109,9 +1137,9 @@ int yescrypt_kdf32( retval = yescrypt_kdf32_body(yctx, passwd, passwdlen, flags, N, t, buf32); -#ifndef SKIP_MEMZERO - if (passwd == dk32) - explicit_bzero(dk32, sizeof(dk32)); -#endif + + explicit_bzero(dk32, sizeof(dk32)); + + dbg("yescrypt_kdf32_body:%d", retval); return retval; } From 78bd8a44b75cf39ea17c8a586ba35776d835551f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Jul 2025 03:36:17 +0200 Subject: [PATCH 136/277] libbb/yescrypt: explain and shrink decode64_uint32() function old new delta decode64_uint32 177 141 -36 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 87 ++++++++++++++++++++++++---- libbb/yescrypt/alg-yescrypt.h | 2 + 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index e5d0450586..c85b2a0b9b 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -27,41 +27,106 @@ static NOINLINE const uint8_t *decode64_uint32( uint32_t *dst, const uint8_t *src, uint32_t val) { - uint32_t start = 0, end = 47, chars = 1, bits = 0; + uint32_t start = 0, end = 47, bits = 0; uint32_t c; - if (!src) /* prevous decode failed already? */ + if (!src) /* previous decode failed already? */ goto fail; c = a2i64(*src++); if (c > 63) goto fail; +// The encoding of number N: +// start = 0 end = 47 +// If N < 48, it is encoded verbatim, else +// N -= 48 +// start = end+1 = 48 +// end += (64-end)/2 = 55 +// If N < (end+1-start)<<6 = 8<<6, it is encoded as 48+(N>>6)|low6bits (that is, 48...55|<6bit>), else +// N -= 8<<6 +// start = end+1 = 56 +// end += (64-end)/2 = 59 +// If N < (end+1-start)<<2*6 = 4<<12, it is encoded as 56+(N>>2*6)|low12bits (that is, 56...59|<6bit>|<6bit>), else +// ...same for 60..61|<6bit>|<6bit>|<6bit> +// .......same for 62|<6bit>|<6bit>|<6bit>|<6bit> +// .......same for 63|<6bit>|<6bit>|<6bit>|<6bit>|<6bit> + dbg_dec64("c:%d val:0x%08x", (int)c, (unsigned)val); while (c > end) { + dbg_dec64("c:%d > end:%d", (int)c, (int)end); val += (end + 1 - start) << bits; + dbg_dec64("val+=0x%08x", (int)((end + 1 - start) << bits)); + dbg_dec64(" val:0x%08x", (unsigned)val); start = end + 1; - end = start + (62 - end) / 2; - chars++; + end += (64 - end) / 2; bits += 6; + dbg_dec64("start=%d", (int)start); + dbg_dec64("end=%d", (int)end); + dbg_dec64("bits=%d", (int)bits); } val += (c - start) << bits; + dbg_dec64("final val+=0x%08x", (int)((c - start) << bits)); + dbg_dec64(" val:0x%08x", (unsigned)val); - while (--chars) { + while (bits != 0) { c = a2i64(*src++); if (c > 63) goto fail; bits -= 6; val += c << bits; + dbg_dec64("low bits val+=0x%08x", (int)(c << bits)); + dbg_dec64(" val:0x%08x", (unsigned)val); } + ret: *dst = val; - return src; + fail: + val = 0; + src = NULL; + goto ret; +} -fail: - *dst = 0; - return NULL; +#if TEST_DECODE64 +static void test_decode64_uint32(void) +{ + const uint8_t *src, *end; + uint32_t u32; + int a = 48; + int b = 8<<6; // 0x0200 + int c = 4<<12; // 0x04000 + int d = 2<<18; // 0x080000 + int e = 1<<24; // 0x1000000 + + src = (void*)"wzzz"; + end = decode64_uint32(&u32, src, 0); + if (u32 != 0x0003ffff+c+b+a) bb_error_msg_and_die("Incorrect decode '%s':0x%08x", src, (unsigned)u32); + if (end != src + 4) bb_error_msg_and_die("Incorrect decode '%s': %p end:%p", src, src, end); + src = (void*)"xzzz"; + end = decode64_uint32(&u32, src, 0); + if (u32 != 0x0007ffff+c+b+a) bb_error_msg_and_die("Incorrect decode '%s':0x%08x", src, (unsigned)u32); + if (end != src + 4) bb_error_msg_and_die("Incorrect decode '%s': %p end:%p", src, src, end); + // Note how the last representable "x---" encoding, 0x7ffff, is exactly d-1! + // And if we now increment it, we get: + src = (void*)"y...."; + end = decode64_uint32(&u32, src, 0); + if (u32 != 0x00000000+d+c+b+a) bb_error_msg_and_die("Incorrect decode '%s':0x%08x", src, (unsigned)u32); + if (end != src + 5) bb_error_msg_and_die("Incorrect decode '%s': %p end:%p", src, src, end); + src = (void*)"yzzzz"; + end = decode64_uint32(&u32, src, 0); + if (u32 != 0x00ffffff+d+c+b+a) bb_error_msg_and_die("Incorrect decode '%s':0x%08x", src, (unsigned)u32); + if (end != src + 5) bb_error_msg_and_die("Incorrect decode '%s': %p end:%p", src, src, end); + + src = (void*)"zzzzzz"; + end = decode64_uint32(&u32, src, 0); + if (u32 != 0x3fffffff+e+d+c+b+a) bb_error_msg_and_die("Incorrect decode '%s':0x%08x", src, (unsigned)u32); + if (end != src + 6) bb_error_msg_and_die("Incorrect decode '%s': %p end:%p", src, src, end); + + bb_error_msg("test_decode64_uint32() OK"); } +#else +# define test_decode64_uint32() ((void)0) +#endif #if 1 static const uint8_t *decode64( @@ -70,7 +135,7 @@ static const uint8_t *decode64( { size_t dstpos = 0; - dbg_dec64("src:'%s' len:%d", src, (int)srclen); + dbg_dec64("src:'%s'", src); for (;;) { uint32_t c, value = 0; int bits = 0; @@ -215,6 +280,8 @@ char *yescrypt_r( size_t need, prefixlen; uint32_t u32; + test_decode64_uint32(); + memset(yctx, 0, sizeof(yctx)); yctx->param.p = 1; diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 4554e3de3b..e558cfdc52 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -28,6 +28,7 @@ * online backup system. */ #ifdef YESCRYPT_INTERNAL + # if 1 # define dbg(...) ((void)0) # else @@ -39,6 +40,7 @@ # define dbg_dec64(...) bb_error_msg(__VA_ARGS__) # endif #endif +#define TEST_DECODE64 0 /** * Type and possible values for the flags argument of yescrypt_kdf(), From d0f0874d573f33f9f14372a4513f22f76c559479 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Jul 2025 05:25:31 +0200 Subject: [PATCH 137/277] libbb/yescrypt: disable code which accepts unusual yescrypt parameters Almost any reasonable yescrypt hashes in /etc/shadow should only ever use "jXY" parameters which set N and r. Fancy multi-byte-encoded wide integers are not needed for that. function old new delta static.yescrypt_kdf32_body - 899 +899 static.PBKDF2_SHA256 213 219 +6 decode64_uint32 141 - -141 yescrypt_r 990 805 -185 yescrypt_kdf32_body 1423 - -1423 ------------------------------------------------------------------------------ (add/remove: 1/2 grow/shrink: 1/1 up/down: 905/-1749) Total: -844 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 25 +++++- libbb/yescrypt/alg-yescrypt.h | 120 +++++++++++++++------------ 2 files changed, 92 insertions(+), 53 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index c85b2a0b9b..e48be65813 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -18,6 +18,23 @@ * SUCH DAMAGE. */ +#if RESTRICTED_PARAMS + +#define decode64_uint32(dst, src, min) \ +({ \ + uint32_t d32 = a2i64(*(src)); \ + if (d32 > 47) \ + goto fail; \ + *(dst) = d32 + (min); \ + ++src; \ +}) +#define test_decode64_uint32() ((void)0) +#define FULL_PARAMS(...) + +#else + +#define FULL_PARAMS(...) __VA_ARGS__ + /* Not inlining: * de/encode64 functions are only used to read * yescrypt_params_t field, and convert salt to binary - @@ -128,6 +145,8 @@ static void test_decode64_uint32(void) # define test_decode64_uint32() ((void)0) #endif +#endif /* !RESTRICTED_PARAMS */ + #if 1 static const uint8_t *decode64( uint8_t *dst, size_t *dstlen, @@ -283,7 +302,7 @@ char *yescrypt_r( test_decode64_uint32(); memset(yctx, 0, sizeof(yctx)); - yctx->param.p = 1; + FULL_PARAMS(yctx->param.p = 1;) /* we assume setting starts with "$y$" (caller must ensure this) */ src = setting + 3; @@ -324,6 +343,9 @@ char *yescrypt_r( if (!src) goto fail; if (*src != '$') { +#if RESTRICTED_PARAMS + goto fail; +#else src = decode64_uint32(&u32, src, 1); dbg("yescrypt has extended params:0x%x", (unsigned)u32); if (u32 & 1) @@ -342,6 +364,7 @@ char *yescrypt_r( goto fail; if (*src != '$') goto fail; +#endif } yctx->saltlen = sizeof(yctx->salt); diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index e558cfdc52..a1d540c085 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -29,6 +29,8 @@ */ #ifdef YESCRYPT_INTERNAL +// busybox debug and size-reduction configuration + # if 1 # define dbg(...) ((void)0) # else @@ -42,6 +44,68 @@ #endif #define TEST_DECODE64 0 +/* Only accept one-char parameters in hash, and only first three? + * Almost any reasonable yescrypt hashes in /etc/shadow should + * only ever use "jXY" parameters which set N and r. + * Fancy multi-byte-encoded wide integers are not needed for that. + */ +#define RESTRICTED_PARAMS 1 +/* Note: if you enable the above, please also enable + * YCTX_param_p, YCTX_param_t, YCTX_param_g, YCTX_param_NROM + * optimizations. + */ + +// How much we save by forcing "standard" value by commenting the next line: +// 160 bytes +//#define YCTX_param_flags yctx->param.flags +// 260 bytes +//#define flags___YESCRYPT_RW (flags & YESCRYPT_RW) +// 140 bytes +//#define flags___YESCRYPT_MODE_MASK (flags & YESCRYPT_MODE_MASK) +// ^^^^ forcing the above since the code already requires (checks for) this +// 50 bytes +#define YCTX_param_N yctx->param.N +// -100 bytes (negative!!!) +#define YCTX_param_r yctx->param.r +// 400 bytes +//#define YCTX_param_p yctx->param.p +// 130 bytes +//#define YCTX_param_t yctx->param.t +// 2 bytes +//#define YCTX_param_g yctx->param.g +// 1 bytes +// ^^^^ this looks wrong, compiler should be able to constant-propagate the fact that NROM code is dead +//#define YCTX_param_NROM yctx->param.NROM + +#ifndef YCTX_param_flags +#define YCTX_param_flags (YESCRYPT_RW | YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K) +#endif +#ifndef flags___YESCRYPT_RW +#define flags___YESCRYPT_RW ((void)flags, YESCRYPT_RW) +#endif +#ifndef flags___YESCRYPT_MODE_MASK +#define flags___YESCRYPT_MODE_MASK ((void)flags, YESCRYPT_RW) +#endif +// standard ("j9T") values: +#ifndef YCTX_param_N +#define YCTX_param_N 4096 +#endif +#ifndef YCTX_param_r +#define YCTX_param_r 32 +#endif +#ifndef YCTX_param_p +#define YCTX_param_p 1 +#endif +#ifndef YCTX_param_t +#define YCTX_param_t 0 +#endif +#ifndef YCTX_param_g +#define YCTX_param_g 0 +#endif +#ifndef YCTX_param_NROM +#define YCTX_param_NROM 0 +#endif + /** * Type and possible values for the flags argument of yescrypt_kdf(), * yescrypt_encode_params_r(), yescrypt_encode_params(). Most of these may be @@ -129,9 +193,12 @@ typedef struct { */ typedef struct { uint32_t flags; + uint32_t r; uint64_t N; - uint32_t r, p, t, g; +#if !RESTRICTED_PARAMS + uint32_t p, t, g; uint64_t NROM; +#endif } yescrypt_params_t; typedef struct { @@ -147,57 +214,6 @@ typedef struct { yescrypt_region_t local[1]; } yescrypt_ctx_t; -// How much can save by forcing "standard" value by commenting the next line: -// 160 bytes -//#define YCTX_param_flags yctx->param.flags -// 260 bytes -//#define flags___YESCRYPT_RW (flags & YESCRYPT_RW) -// 140 bytes -//#define flags___YESCRYPT_MODE_MASK (flags & YESCRYPT_MODE_MASK) -// ^^^^ forcing the above since the code already requires (checks for) this -// 50 bytes -#define YCTX_param_N yctx->param.N -// -100 bytes (negative!!!) -#define YCTX_param_r yctx->param.r -// 400 bytes -#define YCTX_param_p yctx->param.p -// 130 bytes -#define YCTX_param_t yctx->param.t -// 2 bytes -#define YCTX_param_g yctx->param.g -// 1 bytes -// ^^^^ this looks wrong, compiler should be able to constant-propagate the fact that NROM code is dead -#define YCTX_param_NROM yctx->param.NROM - -// standard ("j9T") values: -#ifndef YCTX_param_flags -#define YCTX_param_flags (YESCRYPT_RW | YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K) -#endif -#ifndef flags___YESCRYPT_RW -#define flags___YESCRYPT_RW ((void)flags, YESCRYPT_RW) -#endif -#ifndef flags___YESCRYPT_MODE_MASK -#define flags___YESCRYPT_MODE_MASK ((void)flags, YESCRYPT_RW) -#endif -#ifndef YCTX_param_N -#define YCTX_param_N 4096 -#endif -#ifndef YCTX_param_r -#define YCTX_param_r 32 -#endif -#ifndef YCTX_param_p -#define YCTX_param_p 1 -#endif -#ifndef YCTX_param_t -#define YCTX_param_t 0 -#endif -#ifndef YCTX_param_g -#define YCTX_param_g 0 -#endif -#ifndef YCTX_param_NROM -#define YCTX_param_NROM 0 -#endif - /** * yescrypt_r(shared, local, passwd, passwdlen, setting, key, buf, buflen): * Compute and encode an scrypt or enhanced scrypt hash of passwd given the From 093070879476bad95595ab6352bb23f565b85347 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Jul 2025 06:51:17 +0200 Subject: [PATCH 138/277] libbb/yescrypt: disable NROM code function old new delta smix1 - 595 +595 smix2 - 414 +414 static.yescrypt_kdf32_body 899 847 -52 static.smix2 420 - -420 static.smix1 604 - -604 ------------------------------------------------------------------------------ (add/remove: 2/2 grow/shrink: 0/1 up/down: 1009/-1076) Total: -67 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 43 ++++++++++++++++++++----------- libbb/yescrypt/alg-yescrypt.h | 34 +++++++++++++++--------- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index c998de51d9..f421db111a 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -42,15 +42,6 @@ #define unlikely(exp) (exp) #endif -// Not a size win if 0 -#define UNROLL_COPY 1 - -// -5324 bytes if 0: -#define UNROLL_PWXFORM_ROUND 0 -// -4864 bytes if 0: -#define UNROLL_PWXFORM 0 -// both 0: -7666 bytes - typedef union { uint32_t w[16]; uint64_t d[8]; @@ -100,7 +91,7 @@ do { \ #define DECL_Y \ salsa20_blk_t Y -#if UNROLL_COPY +#if KDF_UNROLL_COPY #define COPY(out, in) \ do { \ (out).d[0] = (in).d[0]; \ @@ -287,7 +278,7 @@ do { \ x1 = ((x1 >> 32) * (uint32_t)x1 + p0[1]) ^ p1[1]; \ } while (0) -#if UNROLL_PWXFORM_ROUND +#if KDF_UNROLL_PWXFORM_ROUND #define PWXFORM_ROUND \ do { \ PWXFORM_SIMD(X.d[0], X.d[1]); \ @@ -319,7 +310,7 @@ do { \ Sw += 64; \ } while (0) -#if UNROLL_PWXFORM +#if KDF_UNROLL_PWXFORM #define PWXFORM \ do { \ uint8_t *Sw = S2 + w + PWXFORM_WRITE_OFFSET; \ @@ -522,6 +513,10 @@ static inline uint32_t integerify(const salsa20_blk_t *B, size_t r) * The array V must be aligned to a multiple of 64 bytes, and arrays B and XY * to a multiple of at least 16 bytes. */ +#if DISABLE_NROM_CODE +#define smix1(B,r,N,flags,V,NROM,VROM,XY,ctx) \ + smix1(B,r,N,flags,V,XY,ctx) +#endif static void smix1(uint8_t *B, size_t r, uint32_t N, uint32_t flags, salsa20_blk_t *V, @@ -529,6 +524,10 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, salsa20_blk_t *XY, pwxform_ctx_t *ctx) { +#if DISABLE_NROM_CODE + uint32_t NROM = 0; + const salsa20_blk_t *VROM = NULL; +#endif size_t s = 2 * r; salsa20_blk_t *X = V, *Y = &V[s]; uint32_t i, j; @@ -643,6 +642,10 @@ static void smix1(uint8_t *B, size_t r, uint32_t N, * least 2. Nloop must be even. The array V must be aligned to a multiple of * 64 bytes, and arrays B and XY to a multiple of at least 16 bytes. */ +#if DISABLE_NROM_CODE +#define smix2(B,r,N,Nloop,flags,V,NROM,VROM,XY,ctx) \ + smix2(B,r,N,Nloop,flags,V,XY,ctx) +#endif static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, uint32_t flags, salsa20_blk_t *V, @@ -650,6 +653,10 @@ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, salsa20_blk_t *XY, pwxform_ctx_t *ctx) { +#if DISABLE_NROM_CODE + uint32_t NROM = 0; + const salsa20_blk_t *VROM = NULL; +#endif size_t s = 2 * r; salsa20_blk_t *X = XY, *Y = &XY[s]; uint32_t i, j; @@ -747,6 +754,10 @@ static uint64_t p2floor(uint64_t x) * and helps avoid false sharing in OpenMP-enabled builds when p > 1, but it * might also result in cache bank conflicts). */ +#if DISABLE_NROM_CODE +#define smix(B,r,N,p,t,flags,V,NROM,VROM,XY,S,passwd) \ + smix(B,r,N,p,t,flags,V,XY,S,passwd) +#endif static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, uint32_t flags, salsa20_blk_t *V, @@ -891,7 +902,9 @@ static int yescrypt_kdf32_body( uint32_t flags, uint64_t N, uint32_t t, uint8_t *buf32) { +#if !DISABLE_NROM_CODE const salsa20_blk_t *VROM; +#endif size_t B_size, V_size, XY_size, need; uint8_t *B, *S; salsa20_blk_t *V, *XY; @@ -935,9 +948,7 @@ static int yescrypt_kdf32_body( dbg("N > 0x%lx", (long)UINT32_MAX); goto out_EINVAL; } - if ((N & (N - 1)) != 0 -//TODO: ^^^^^^^^^^^^^^^^^^^^^^ do not check this, code guarantees power-of-2 - || N <= 3 + if (N <= 3 || r < 1 || p < 1 ) { @@ -966,9 +977,11 @@ static int yescrypt_kdf32_body( #pragma GCC diagnostic pop } +#if !DISABLE_NROM_CODE VROM = NULL; if (YCTX_param_NROM) goto out_EINVAL; +#endif /* Allocate memory */ V = NULL; diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index a1d540c085..2a94348090 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -27,10 +27,10 @@ * This file was originally written by Colin Percival as part of the Tarsnap * online backup system. */ -#ifdef YESCRYPT_INTERNAL // busybox debug and size-reduction configuration +#ifdef YESCRYPT_INTERNAL # if 1 # define dbg(...) ((void)0) # else @@ -41,19 +41,20 @@ # else # define dbg_dec64(...) bb_error_msg(__VA_ARGS__) # endif +# define TEST_DECODE64 0 #endif -#define TEST_DECODE64 0 -/* Only accept one-char parameters in hash, and only first three? - * Almost any reasonable yescrypt hashes in /etc/shadow should - * only ever use "jXY" parameters which set N and r. - * Fancy multi-byte-encoded wide integers are not needed for that. - */ -#define RESTRICTED_PARAMS 1 -/* Note: if you enable the above, please also enable - * YCTX_param_p, YCTX_param_t, YCTX_param_g, YCTX_param_NROM - * optimizations. - */ + +// Only accept one-char parameters in salt, and only first three? +// Almost any reasonable yescrypt hashes in /etc/shadow should +// only ever use "jXY" parameters which set N and r. +// Fancy multi-byte-encoded wide integers are not needed for that. +#define RESTRICTED_PARAMS 1 +// Note: if you enable the above, please also enable +// YCTX_param_p, YCTX_param_t, YCTX_param_g, YCTX_param_NROM +// optimizations, and DISABLE_NROM_CODE. + +#define DISABLE_NROM_CODE 1 // How much we save by forcing "standard" value by commenting the next line: // 160 bytes @@ -106,6 +107,15 @@ #define YCTX_param_NROM 0 #endif +// "Faster, or smaller code" knobs: +// Not a size win if disabled, so keeping it 1: +#define KDF_UNROLL_COPY 1 +// -5324 bytes if 0: +#define KDF_UNROLL_PWXFORM_ROUND 0 +// -4864 bytes if 0: +#define KDF_UNROLL_PWXFORM 0 +// both 0: -7666 bytes + /** * Type and possible values for the flags argument of yescrypt_kdf(), * yescrypt_encode_params_r(), yescrypt_encode_params(). Most of these may be From 7b313c34651038f39d98ff105102b16a6be94e1c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Jul 2025 10:04:47 +0200 Subject: [PATCH 139/277] libbb: code shrink in sha_crypt() Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt_sha.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbb/pw_encrypt_sha.c b/libbb/pw_encrypt_sha.c index 5162939201..695a5c07fa 100644 --- a/libbb/pw_encrypt_sha.c +++ b/libbb/pw_encrypt_sha.c @@ -84,8 +84,7 @@ sha_crypt(/*const*/ char *key_data, /*const*/ char *salt_data) as a scratch space later. */ salt_data = xstrndup(salt_data, salt_len); /* add "salt$" to result */ - strcpy(resptr, salt_data); - resptr += salt_len; + resptr = stpcpy(resptr, salt_data); *resptr++ = '$'; /* key data doesn't need much processing */ key_len = strlen(key_data); From e9c21c5c3a698468be1daceaad931901e1d63aed Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 8 Jul 2025 10:34:38 +0200 Subject: [PATCH 140/277] hush: fix SEGV on "echo << >" and such function old new delta .rodata 105787 105823 +36 fetch_heredocs 461 477 +16 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 52/0) Total: 52 bytes Signed-off-by: Denys Vlasenko --- shell/hush.c | 5 +++++ .../hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.right | 1 + .../hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.tests | 2 ++ .../hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.right | 1 + .../hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.tests | 2 ++ 5 files changed, 11 insertions(+) create mode 100644 shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.right create mode 100755 shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.tests create mode 100644 shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.right create mode 100755 shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.tests diff --git a/shell/hush.c b/shell/hush.c index 0b2bc01f56..d1f687f9d5 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -4625,6 +4625,11 @@ static int fetch_heredocs(o_string *as_string, struct pipe *pi, int heredoc_cnt, redir->rd_type = REDIRECT_HEREDOC2; /* redir->rd_dup is (ab)used to indicate <<- */ + if (!redir->rd_filename) { + /* examples: "echo <<", "echo <<<", "echo <<>" */ + syntax_error("missing here document terminator"); + return -1; + } p = fetch_till_str(as_string, input, redir->rd_filename, redir->rd_dup); if (!p) { diff --git a/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.right b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.right new file mode 100644 index 0000000000..7af73557ae --- /dev/null +++ b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.right @@ -0,0 +1 @@ +hush: syntax error: missing here document terminator diff --git a/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.tests b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.tests new file mode 100755 index 0000000000..33ccde26b2 --- /dev/null +++ b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF1.tests @@ -0,0 +1,2 @@ +echo << +echo Done: diff --git a/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.right b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.right new file mode 100644 index 0000000000..7af73557ae --- /dev/null +++ b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.right @@ -0,0 +1 @@ +hush: syntax error: missing here document terminator diff --git a/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.tests b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.tests new file mode 100755 index 0000000000..fcda110450 --- /dev/null +++ b/shell/hush_test/hush-heredoc/heredoc_syntax_err_no_EOF2.tests @@ -0,0 +1,2 @@ +echo << > +echo Done: From c305c81c94a086fb09444b1ea6f31fb911c25ec0 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 9 Jul 2025 06:51:04 +0200 Subject: [PATCH 141/277] libbb: introduce and use block-XOR functions On x86_64, they can be done in 16-byte blocks 64-bit: function old new delta xorbuf_3 - 84 +84 xorbuf64_3_aligned64 - 58 +58 smix1 687 712 +25 xwrite_encrypted 520 534 +14 xorbuf16_aligned_long - 13 +13 tls_xread_record 733 742 +9 xorbuf 21 13 -8 xorbuf_aligned_AES_BLOCK_SIZE 15 - -15 blockmix 814 762 -52 blockmix_salsa8 317 198 -119 blockmix_xor_save 1620 1499 -121 blockmix_xor 1543 1322 -221 ------------------------------------------------------------------------------ (add/remove: 4/1 grow/shrink: 3/5 up/down: 203/-536) Total: -333 bytes 32-bit: function old new delta xorbuf_3 - 76 +76 xorbuf64_3_aligned64 - 36 +36 xorbuf16_aligned_long - 23 +23 xwrite_encrypted 499 507 +8 tls_xread_record 646 650 +4 xorbuf 22 11 -11 xorbuf_aligned_AES_BLOCK_SIZE 23 - -23 blockmix 1083 938 -145 blockmix_salsa8 415 210 -205 blockmix_salsa8_xor 601 163 -438 blockmix_xor 2103 1533 -570 blockmix_xor_save 2614 1859 -755 ------------------------------------------------------------------------------ (add/remove: 4/1 grow/shrink: 2/6 up/down: 147/-2147) Total: -2000 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 10 +++ libbb/bitops.c | 108 ++++++++++++++++++++++++++++++ libbb/yescrypt/alg-sha256.c | 1 + libbb/yescrypt/alg-yescrypt-kdf.c | 7 ++ networking/tls.c | 39 +++-------- networking/tls.h | 5 +- networking/tls_aesgcm.c | 5 +- 7 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 libbb/bitops.c diff --git a/include/libbb.h b/include/libbb.h index 544ca31556..79427fb315 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1113,6 +1113,16 @@ char *bin2hex(char *dst, const char *src, int count) FAST_FUNC; /* Reverse */ char* hex2bin(char *dst, const char *src, int count) FAST_FUNC; +void FAST_FUNC xorbuf_3(void *dst, const void *src1, const void *src2, unsigned count); +void FAST_FUNC xorbuf(void* buf, const void* mask, unsigned count); +void FAST_FUNC xorbuf16_aligned_long(void* buf, const void* mask); +void FAST_FUNC xorbuf64_3_aligned64(void *dst, const void *src1, const void *src2); +#if BB_UNALIGNED_MEMACCESS_OK +# define xorbuf16(buf,mask) xorbuf16_aligned_long(buf,mask) +#else +void FAST_FUNC xorbuf16(void* buf, const void* mask); +#endif + /* Generate a UUID */ void generate_uuid(uint8_t *buf) FAST_FUNC; diff --git a/libbb/bitops.c b/libbb/bitops.c new file mode 100644 index 0000000000..5f239676c8 --- /dev/null +++ b/libbb/bitops.c @@ -0,0 +1,108 @@ +/* + * Utility routines. + * + * Copyright (C) 2025 by Denys Vlasenko + * + * Licensed under GPLv2, see file LICENSE in this source tree. + */ +//kbuild:lib-y += bitops.o + +#include "libbb.h" + +void FAST_FUNC xorbuf_3(void *dst, const void *src1, const void *src2, unsigned count) +{ + uint8_t *d = dst; + const uint8_t *s1 = src1; + const uint8_t *s2 = src2; +#if BB_UNALIGNED_MEMACCESS_OK + while (count >= sizeof(long)) { + *(long*)d = *(long*)s1 ^ *(long*)s2; + count -= sizeof(long); + d += sizeof(long); + s1 += sizeof(long); + s2 += sizeof(long); + } +#endif + while (count--) + *d++ = *s1++ ^ *s2++; +} + +void FAST_FUNC xorbuf(void *dst, const void *src, unsigned count) +{ + xorbuf_3(dst, dst, src, count); +} + +void FAST_FUNC xorbuf16_aligned_long(void *dst, const void *src) +{ +#if defined(__SSE__) /* any x86_64 has it */ + asm volatile( +"\n movups (%0),%%xmm0" +"\n movups (%1),%%xmm1" // can't just xorps(%1),%%xmm0: +"\n xorps %%xmm1,%%xmm0" // SSE requires 16-byte alignment +"\n movups %%xmm0,(%0)" +"\n" + : "=r" (dst), "=r" (src) + : "0" (dst), "1" (src) + : "xmm0", "xmm1", "memory" + ); +#else + unsigned long *d = dst; + const unsigned long *s = src; + d[0] ^= s[0]; +# if LONG_MAX <= 0x7fffffffffffffff + d[1] ^= s[1]; +# if LONG_MAX == 0x7fffffff + d[2] ^= s[2]; + d[3] ^= s[3]; +# endif +# endif +#endif +} + +void FAST_FUNC xorbuf64_3_aligned64(void *dst, const void *src1, const void *src2) +{ +#if defined(__SSE__) /* any x86_64 has it */ + asm volatile( +"\n movups 0*16(%1),%%xmm0" +"\n movups 0*16(%2),%%xmm1" // can't just xorps(%2),%%xmm0: +"\n xorps %%xmm1,%%xmm0" // SSE requires 16-byte alignment, we have only 8-byte +"\n movups %%xmm0,0*16(%0)" +"\n movups 1*16(%1),%%xmm0" +"\n movups 1*16(%2),%%xmm1" +"\n xorps %%xmm1,%%xmm0" +"\n movups %%xmm0,1*16(%0)" +"\n movups 2*16(%1),%%xmm0" +"\n movups 2*16(%2),%%xmm1" +"\n xorps %%xmm1,%%xmm0" +"\n movups %%xmm0,2*16(%0)" +"\n movups 3*16(%1),%%xmm0" +"\n movups 3*16(%2),%%xmm1" +"\n xorps %%xmm1,%%xmm0" +"\n movups %%xmm0,3*16(%0)" +"\n" + : "=r" (dst), "=r" (src1), "=r" (src2) + : "0" (dst), "1" (src1), "2" (src2) + : "xmm0", "xmm1", "memory" + ); +#else + long *d = dst; + const long *s1 = src1; + const long *s2 = src2; + unsigned count = 64 / sizeof(long); + do { + *d++ = *s1++ ^ *s2++; + } while (--count != 0); +#endif +} + +#if !BB_UNALIGNED_MEMACCESS_OK +void FAST_FUNC xorbuf16(void *dst, const void *src) +{ +#define p_aligned(a) (((uintptr_t)(a) & (sizeof(long)-1)) == 0) + if (p_aligned(src) && p_aligned(dst)) { + xorbuf16_aligned_long(dst, src); + return; + } + xorbuf_3(dst, dst, src, 16); +} +#endif diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 25446406b4..20e8d1ee47 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -72,6 +72,7 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, /* ... xor U_j ... */ for (k = 0; k < 32 / 8; k++) T[k] ^= U[k]; + //TODO: xorbuf32_aligned_long(T, U); } } diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index f421db111a..112862ec9f 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -180,6 +180,7 @@ static void salsa20(salsa20_blk_t *restrict B, #define SALSA20_2(out) \ salsa20(&X, &out, 1) +#if 0 #define XOR(out, in1, in2) \ do { \ (out).d[0] = (in1).d[0] ^ (in2).d[0]; \ @@ -191,6 +192,12 @@ do { \ (out).d[6] = (in1).d[6] ^ (in2).d[6]; \ (out).d[7] = (in1).d[7] ^ (in2).d[7]; \ } while (0) +#else +#define XOR(out, in1, in2) \ +do { \ + xorbuf64_3_aligned64(&(out).d, &(in1).d, &(in2).d); \ +} while (0) +#endif #define XOR_X(in) XOR(X, X, in) #define XOR_X_2(in1, in2) XOR(X, in1, in2) diff --git a/networking/tls.c b/networking/tls.c index 098cf7cacc..ac6f0767f1 100644 --- a/networking/tls.c +++ b/networking/tls.c @@ -333,34 +333,6 @@ void FAST_FUNC tls_get_random(void *buf, unsigned len) xfunc_die(); } -static void xorbuf3(void *dst, const void *src1, const void *src2, unsigned count) -{ - uint8_t *d = dst; - const uint8_t *s1 = src1; - const uint8_t* s2 = src2; - while (count--) - *d++ = *s1++ ^ *s2++; -} - -void FAST_FUNC xorbuf(void *dst, const void *src, unsigned count) -{ - xorbuf3(dst, dst, src, count); -} - -void FAST_FUNC xorbuf_aligned_AES_BLOCK_SIZE(void *dst, const void *src) -{ - unsigned long *d = dst; - const unsigned long *s = src; - d[0] ^= s[0]; -#if ULONG_MAX <= 0xffffffffffffffff - d[1] ^= s[1]; - #if ULONG_MAX == 0xffffffff - d[2] ^= s[2]; - d[3] ^= s[3]; - #endif -#endif -} - #if !TLS_DEBUG_HASH # define hash_handshake(tls, fmt, buffer, len) \ hash_handshake(tls, buffer, len) @@ -764,8 +736,13 @@ static void xwrite_encrypted_aesgcm(tls_state_t *tls, unsigned size, unsigned ty cnt++; COUNTER(nonce) = htonl(cnt); /* yes, first cnt here is 2 (!) */ aes_encrypt_one_block(&tls->aes_encrypt, nonce, scratch); - n = remaining > AES_BLOCK_SIZE ? AES_BLOCK_SIZE : remaining; - xorbuf(buf, scratch, n); + if (remaining >= AES_BLOCK_SIZE) { + n = AES_BLOCK_SIZE; + xorbuf_AES_BLOCK_SIZE(buf, scratch); + } else { + n = remaining; + xorbuf(buf, scratch, n); + } buf += n; remaining -= n; } @@ -923,7 +900,7 @@ static void tls_aesgcm_decrypt(tls_state_t *tls, uint8_t *buf, int size) COUNTER(nonce) = htonl(cnt); /* yes, first cnt here is 2 (!) */ aes_encrypt_one_block(&tls->aes_decrypt, nonce, scratch); n = remaining > AES_BLOCK_SIZE ? AES_BLOCK_SIZE : remaining; - xorbuf3(buf, scratch, buf + 8, n); + xorbuf_3(buf, scratch, buf + 8, n); buf += n; remaining -= n; } diff --git a/networking/tls.h b/networking/tls.h index 0173b87b27..9751d30ff7 100644 --- a/networking/tls.h +++ b/networking/tls.h @@ -82,10 +82,9 @@ typedef int16_t int16; void tls_get_random(void *buf, unsigned len) FAST_FUNC; -void xorbuf(void* buf, const void* mask, unsigned count) FAST_FUNC; - #define ALIGNED_long ALIGNED(sizeof(long)) -void xorbuf_aligned_AES_BLOCK_SIZE(void* buf, const void* mask) FAST_FUNC; +#define xorbuf_aligned_AES_BLOCK_SIZE(dst,src) xorbuf16_aligned_long(dst,src) +#define xorbuf_AES_BLOCK_SIZE(dst,src) xorbuf16(dst,src) #define matrixCryptoGetPrngData(buf, len, userPtr) (tls_get_random(buf, len), PS_SUCCESS) diff --git a/networking/tls_aesgcm.c b/networking/tls_aesgcm.c index 5ddcdd2ad8..9c2381a571 100644 --- a/networking/tls_aesgcm.c +++ b/networking/tls_aesgcm.c @@ -167,10 +167,7 @@ void FAST_FUNC aesgcm_GHASH(byte* h, blocks = cSz / AES_BLOCK_SIZE; partial = cSz % AES_BLOCK_SIZE; while (blocks--) { - if (BB_UNALIGNED_MEMACCESS_OK) // c is not guaranteed to be aligned - xorbuf_aligned_AES_BLOCK_SIZE(x, c); - else - xorbuf(x, c, AES_BLOCK_SIZE); + xorbuf_AES_BLOCK_SIZE(x, c); GMULT(x, h); c += AES_BLOCK_SIZE; } From 11d4c08d7541408e4fbb7daaaf63aba1d07685ea Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 9 Jul 2025 08:21:47 +0200 Subject: [PATCH 142/277] libbb/bitops.c: add inlining comment Signed-off-by: Denys Vlasenko --- libbb/bitops.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/libbb/bitops.c b/libbb/bitops.c index 5f239676c8..467e1a2d93 100644 --- a/libbb/bitops.c +++ b/libbb/bitops.c @@ -58,6 +58,26 @@ void FAST_FUNC xorbuf16_aligned_long(void *dst, const void *src) # endif #endif } +// The above can be inlined in libbb.h, in a way where compiler +// is even free to use better addressing modes than (%reg), and +// to keep the result in a register +// (to not store it to memory after each XOR): +//#if defined(__SSE__) +//#include +//^^^ or just: typedef float __m128_u attribute((__vector_size__(16),__may_alias__,__aligned__(1))); +//static ALWAYS_INLINE void xorbuf16_aligned_long(void *dst, const void *src) +//{ +// __m128_u xmm0, xmm1; +// asm volatile( +//"\n xorps %1,%0" +// : "=x" (xmm0), "=x" (xmm1) +// : "0" (*(__m128_u*)dst), "1" (*(__m128_u*)src) +// ); +// *(__m128_u*)dst = xmm0; // this store may be optimized out! +//} +//#endif +// but I don't trust gcc optimizer enough to not generate some monstrosity. +// See GMULT() function in TLS code as an example. void FAST_FUNC xorbuf64_3_aligned64(void *dst, const void *src1, const void *src2) { From 95f169f3bb07528c9302c604ff2a1f40b41a98d8 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 9 Jul 2025 10:38:11 +0200 Subject: [PATCH 143/277] libbb/yescrypt: code shrink function old new delta static.yescrypt_kdf32_body 847 823 -24 yescrypt_r 805 767 -38 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-62) Total: -62 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 19 ++++++++++++------- libbb/yescrypt/alg-yescrypt-kdf.c | 24 ++++++++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index e48be65813..1c063b895f 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -23,8 +23,8 @@ #define decode64_uint32(dst, src, min) \ ({ \ uint32_t d32 = a2i64(*(src)); \ - if (d32 > 47) \ - goto fail; \ + if (d32 > 47) \ + goto fail; \ *(dst) = d32 + (min); \ ++src; \ }) @@ -292,8 +292,12 @@ char *yescrypt_r( const uint8_t *setting, char *buf, size_t buflen) { - yescrypt_ctx_t yctx[1]; - unsigned char hashbin32[32]; + struct { + yescrypt_ctx_t yctx[1]; + unsigned char hashbin32[32]; + } u; +#define yctx u.yctx +#define hashbin32 u.hashbin32 char *dst; const uint8_t *src, *saltend; size_t need, prefixlen; @@ -375,7 +379,7 @@ char *yescrypt_r( prefixlen = saltend - setting; need = prefixlen + 1 + YESCRYPT_HASH_LEN + 1; - if (need > buflen || need < prefixlen) + if (need > buflen /*overflow is quite unlikely: || need < prefixlen*/) goto fail; if (yescrypt_kdf32(yctx, passwd, passwdlen, hashbin32)) { @@ -390,10 +394,11 @@ char *yescrypt_r( goto fail; ret: free_region(yctx->local); - explicit_bzero(yctx, sizeof(yctx)); - explicit_bzero(hashbin32, sizeof(hashbin32)); + explicit_bzero(&u, sizeof(u)); return buf; fail: buf = NULL; goto ret; +#undef yctx +#undef hashbin32 } diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 112862ec9f..29d9efc076 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -915,8 +915,13 @@ static int yescrypt_kdf32_body( size_t B_size, V_size, XY_size, need; uint8_t *B, *S; salsa20_blk_t *V, *XY; - uint8_t sha256[32]; - uint8_t dk[sizeof(sha256)], *dkp = buf32; + struct { + uint8_t sha256[32]; + uint8_t dk[32]; + } u; +#define sha256 u.sha256 +#define dk u.dk + uint8_t *dkp = buf32; uint32_t r, p; /* Sanity-check parameters */ @@ -1083,15 +1088,16 @@ static int yescrypt_kdf32_body( size_t clen = /*buflen:*/32; if (clen > sizeof(dk)) clen = sizeof(dk); - sha256_block(sha256, sizeof(sha256), dk); - memcpy(buf32, dk, clen); + if (sizeof(dk) != 32) { /* not true, optimize it out */ + sha256_block(sha256, sizeof(sha256), dk); + memcpy(buf32, dk, clen); + } else { + sha256_block(sha256, sizeof(sha256), buf32); + } } } - if (flags) { - explicit_bzero(sha256, sizeof(sha256)); - explicit_bzero(dk, sizeof(dk)); - } + explicit_bzero(&u, sizeof(u)); /* Success! */ return 0; @@ -1099,6 +1105,8 @@ static int yescrypt_kdf32_body( out_EINVAL: //bbox does not need this: errno = EINVAL; return -1; +#undef sha256 +#undef dk } /** From e62bfbcaed051146b83a096e778b5822069c160b Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 11 Jul 2025 08:56:10 +0200 Subject: [PATCH 144/277] libbb/yescrypt: code shrink function old new delta salsa20 684 650 -34 blockmix_salsa8 198 144 -54 blockmix 762 565 -197 blockmix_xor 1322 1028 -294 blockmix_xor_save 1499 1103 -396 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 0/5 up/down: 0/-975) Total: -975 bytes Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 9 +++------ libbb/yescrypt/alg-yescrypt.h | 8 ++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 29d9efc076..1c254e2e28 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -106,8 +106,7 @@ do { \ #else #define COPY(out, in) \ do { \ - for (int copyi=0; copyi<8; copyi++) \ - (out).d[copyi] = (in).d[copyi]; \ + memcpy((out).d, (in).d, sizeof((in).d)); \ } while (0) #endif @@ -161,11 +160,9 @@ static void salsa20(salsa20_blk_t *restrict B, { uint32_t i; salsa20_simd_shuffle(&X, Bout); - for (i = 0; i < 16; i += 4) { + for (i = 0; i < 16; i++) { + // bbox: note: was unrolled x4 B->w[i] = Bout->w[i] += B->w[i]; - B->w[i + 1] = Bout->w[i + 1] += B->w[i + 1]; - B->w[i + 2] = Bout->w[i + 2] += B->w[i + 2]; - B->w[i + 3] = Bout->w[i + 3] += B->w[i + 3]; } } #if 0 diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 2a94348090..0b93945af5 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -107,14 +107,14 @@ #define YCTX_param_NROM 0 #endif -// "Faster, or smaller code" knobs: -// Not a size win if disabled, so keeping it 1: -#define KDF_UNROLL_COPY 1 +// "Faster/smaller code" knobs: +// -941 bytes: +#define KDF_UNROLL_COPY 0 // -5324 bytes if 0: #define KDF_UNROLL_PWXFORM_ROUND 0 // -4864 bytes if 0: #define KDF_UNROLL_PWXFORM 0 -// both 0: -7666 bytes +// if both this ^^^^^^^^^^ and PWXFORM_ROUND set to 0: -7666 bytes /** * Type and possible values for the flags argument of yescrypt_kdf(), From abe0b45cd7098f0f83f727d4d4c4c09dc2b172f7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 15 Jul 2025 05:47:18 +0200 Subject: [PATCH 145/277] libbb/yescrypt: code shrink function old new delta blockmix_xor 1177 702 -475 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 1c254e2e28..f75361d966 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -421,26 +421,19 @@ static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, i = 0; r--; - do { + for (;;) { XOR_X(Bin1[i]); XOR_X(Bin2[i]); PWXFORM; - WRITE_X(Bout[i]); - - XOR_X(Bin1[i + 1]); - XOR_X(Bin2[i + 1]); - PWXFORM; - - if (unlikely(i >= r)) + if (unlikely(i > r)) break; + WRITE_X(Bout[i]); + i++; + } - WRITE_X(Bout[i + 1]); - - i += 2; - } while (1); - i++; - - ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; + ctx->S0 = S0; + ctx->S1 = S1; + ctx->S2 = S2; ctx->w = w; SALSA20_2(Bout[i]); From b823735b7eb6428e827cf463123d3caaa48804ff Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 17 Jul 2025 17:01:40 +0200 Subject: [PATCH 146/277] libbb/yescrypt: actually, largest allowed salt is 86 chars, support that function old new delta yescrypt_r 767 756 -11 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-common.c | 26 +++++++++++++++----------- libbb/yescrypt/alg-yescrypt.h | 2 +- testsuite/cryptpw.tests | 10 +++++++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c index 1c063b895f..c518237874 100644 --- a/libbb/yescrypt/alg-yescrypt-common.c +++ b/libbb/yescrypt/alg-yescrypt-common.c @@ -152,13 +152,13 @@ static const uint8_t *decode64( uint8_t *dst, size_t *dstlen, const uint8_t *src) { - size_t dstpos = 0; + unsigned dstpos = 0; dbg_dec64("src:'%s'", src); for (;;) { uint32_t c, value = 0; int bits = 0; - while (*src && *src != '$') { + while (*src != '\0' && *src != '$') { c = a2i64(*src); if (c > 63) { /* bad ascii64 char, stop decoding at it */ break; @@ -174,9 +174,11 @@ static const uint8_t *decode64( break; /* else: we got last, partial bit block - store it */ store: - dbg_dec64(" storing bits:%d v:%08x", bits, (int)SWAP_BE32(value)); //BE to see lsb first - while (dstpos < *dstlen) { - if ((!*src || *src == '$') && value == 0 && bits < 8) { + dbg_dec64(" storing bits:%d dstpos:%u v:%08x", bits, dstpos, (int)SWAP_BE32(value)); //BE to see lsb first + for (;;) { + if ((*src == '\0' || *src == '$') + && value == 0 && bits < 8 + ) { /* Example: mkpasswd PWD '$y$j9T$123': * the "123" is bits:18 value:03,51,00 * is considered to be 2 bytes, not 3! @@ -190,17 +192,18 @@ static const uint8_t *decode64( */ goto end; } - dstpos++; + if (dstpos >= *dstlen) { + dbg_dec64(" ERR: bits:%d dstpos:%u dst[] is too small", bits, dstpos); + goto fail; + } *dst++ = value; + dstpos++; value >>= 8; bits -= 8; if (bits <= 0) /* can get negative, if we e.g. had 6 bits */ - goto next; + break; } - dbg_dec64(" ERR: bits:%d dst[] is too small", bits); - goto fail; - next: - if (!*src || *src == '$') + if (*src == '\0' || *src == '$') break; } end: @@ -376,6 +379,7 @@ char *yescrypt_r( saltend = decode64(yctx->salt, &yctx->saltlen, src); if (!saltend || (*saltend != '\0' && *saltend != '$')) goto fail; /* salt[] is too small, or bad char during decode */ + dbg_dec64("salt is %d ascii64 chars -> %d bytes (in binary)", (int)(saltend - src), (int)yctx->saltlen); prefixlen = saltend - setting; need = prefixlen + 1 + YESCRYPT_HASH_LEN + 1; diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 0b93945af5..5051efbb45 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -139,7 +139,7 @@ #define YESCRYPT_GATHER_8 0x018 //gg=11 #define YESCRYPT_SIMPLE_1 0x000 //ss=00 #define YESCRYPT_SIMPLE_2 0x020 //ss=01 -#define YESCRYPT_SIMPLE_4 0x040 //ss=11 +#define YESCRYPT_SIMPLE_4 0x040 //ss=10 #define YESCRYPT_SIMPLE_8 0x060 //ss=11 #define YESCRYPT_SBOX_6K 0x000 //sbox=0000 #define YESCRYPT_SBOX_12K 0x080 //sbox=0001 diff --git a/testsuite/cryptpw.tests b/testsuite/cryptpw.tests index beac35efe6..83bfde5212 100755 --- a/testsuite/cryptpw.tests +++ b/testsuite/cryptpw.tests @@ -97,9 +97,13 @@ testing 'cryptpw yescrypt with 4-char salt "...."' \ 'cryptpw -m yescrypt qweRTY123@-+ j9T\$....' \ '$y$j9T$....$wOnauYL2/NEtr6YQi9pi8AtV7L57sEbVOAnWJIcP9q2\n' \ '' '' -testing 'cryptpw yescrypt with 84-char salt (max size)' \ - 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123456789012345678901234567890123456789012345678901234567890123456789012345678901234' \ - '$y$j9T$123456789012345678901234567890123456789012345678901234567890123456789012345678901234$ubrUuPCpI97LIMlVMt/A0Mhs/kBK2UBJYcQSxEZSlz4\n' \ +# 84 chars = 21 4-char blocks which decode into 21*3 = 63 bytes. +# The last byte of the maximum allowed salt size has to come from an incomplete +# char block. E.g. "z/" encodes byte 0x7f. "z1" is 0xff. +# Anything larger (e.g. "z2") is an error (it encodes 0x13f). +testing 'cryptpw yescrypt with 86-char salt (max size)' \ + 'cryptpw -m yescrypt qweRTY123@-+ j9T\$123456789012345678901234567890123456789012345678901234567890123456789012345678901234z/' \ + '$y$j9T$123456789012345678901234567890123456789012345678901234567890123456789012345678901234z/$Exxe8IoPXiddFsqj7iqCanRf8FyquAoB0/uceLmLjG.\n' \ '' '' testing 'cryptpw yescrypt implicit' \ 'cryptpw qweRTY123@-+ \$y\$j9T\$123456789012345678901234' \ From dde90f1d4746a2efef20fc227cd38bbcb58d546c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 17 Jul 2025 17:34:14 +0200 Subject: [PATCH 147/277] libbb.h: increase MAX_PW_SALT_LEN Signed-off-by: Denys Vlasenko --- include/libbb.h | 12 ++++++------ loginutils/cryptpw.c | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index 79427fb315..4f44680aae 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1825,14 +1825,14 @@ extern int obscure(const char *old, const char *newval, const struct passwd *pwd extern int crypt_make_rand64encoded(char *p, int cnt /*, int rnd*/) FAST_FUNC; /* Size of char salt[] to hold randomly-generated salt string * sha256/512: - * "$5$" - * "$6$" - * #define MAX_PW_SALT_LEN (3 + 16 + 1) + * "$5$" ["rounds=999999999$"] "" + * "$6$" ["rounds=999999999$"] "" + * #define MAX_PW_SALT_LEN (3 + sizeof("rounds=999999999$")-1 + 16 + 1) * yescrypt: - * "$y$" "$" - * (84 chars are ascii64-encoded 64 binary bytes) + * "$y$" "$" + * (86 chars are ascii64-encoded 64 binary bytes) */ -#define MAX_PW_SALT_LEN (3 + 8*6 + 1 + 84 + 1) +#define MAX_PW_SALT_LEN (3 + 8*6 + 1 + 86 + 1) extern char* crypt_make_pw_salt(char p[MAX_PW_SALT_LEN], const char *algo) FAST_FUNC; /* Returns number of lines changed, or -1 on error */ diff --git a/loginutils/cryptpw.c b/loginutils/cryptpw.c index 1c338540fc..c0f6280cd3 100644 --- a/loginutils/cryptpw.c +++ b/loginutils/cryptpw.c @@ -84,8 +84,7 @@ to cryptpw. -a option (alias for -m) came from cryptpw. int cryptpw_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int cryptpw_main(int argc UNUSED_PARAM, char **argv) { - /* Supports: cryptpw -m sha256 PASS 'rounds=999999999$SALT' */ - char salt[MAX_PW_SALT_LEN + sizeof("rounds=999999999$")]; + char salt[MAX_PW_SALT_LEN]; char *salt_ptr; char *password; const char *opt_m, *opt_S; From d2ef2c75c59b9c5c9017f7ce39d01c103ffd0b1e Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 18 Jul 2025 07:28:32 +0200 Subject: [PATCH 148/277] password applets: update help text Signed-off-by: Denys Vlasenko --- include/usage.src.h | 6 +++--- loginutils/chpasswd.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/usage.src.h b/include/usage.src.h index 5d20388341..0881337f85 100644 --- a/include/usage.src.h +++ b/include/usage.src.h @@ -17,11 +17,11 @@ #define scripted_trivial_usage NOUSAGE_STR #define scripted_full_usage "" -#if !ENABLE_USE_BB_CRYPT || ENABLE_USE_BB_CRYPT_SHA -# define CRYPT_METHODS_HELP_STR "des,md5,sha256/512" \ +#if !ENABLE_USE_BB_CRYPT +# define CRYPT_METHODS_HELP_STR "des,md5,sha256/512,yescrypt" \ " (default "CONFIG_FEATURE_DEFAULT_PASSWD_ALGO")" #else -# define CRYPT_METHODS_HELP_STR "des,md5" \ +# define CRYPT_METHODS_HELP_STR "des,md5"IF_USE_BB_CRYPT_SHA(",sha256/512")IF_USE_BB_CRYPT_YES(",yescrypt") \ " (default "CONFIG_FEATURE_DEFAULT_PASSWD_ALGO")" #endif diff --git a/loginutils/chpasswd.c b/loginutils/chpasswd.c index 65530b6147..353f19961a 100644 --- a/loginutils/chpasswd.c +++ b/loginutils/chpasswd.c @@ -17,7 +17,7 @@ //config: default "des" //config: depends on PASSWD || CRYPTPW || CHPASSWD //config: help -//config: Possible choices are "d[es]", "m[d5]", "s[ha256]" or "sha512". +//config: Possible choices: "d[es]", "m[d5]", "s[ha256]", "sha512", "yescrypt" //applet:IF_CHPASSWD(APPLET(chpasswd, BB_DIR_USR_SBIN, BB_SUID_DROP)) From 504f44436f2a3903d9a71185c4e5bc62d923a8fb Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 18 Jul 2025 07:29:24 +0200 Subject: [PATCH 149/277] sulogin: set ^C to SIG_DFL before execing the shell function old new delta sulogin_main 331 340 +9 Signed-off-by: Denys Vlasenko --- loginutils/sulogin.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/loginutils/sulogin.c b/loginutils/sulogin.c index 9c927ed79b..984889915e 100644 --- a/loginutils/sulogin.c +++ b/loginutils/sulogin.c @@ -79,7 +79,7 @@ int sulogin_main(int argc UNUSED_PARAM, char **argv) break; } pause_after_failed_login(); - bb_simple_info_msg("Login incorrect"); + bb_simple_error_msg("Login incorrect"); } /* util-linux 2.36.1 compat: no message */ @@ -119,9 +119,12 @@ int sulogin_main(int argc UNUSED_PARAM, char **argv) } /* - * Note: login does this (should we do it too?): + * Note: login does this. util-linux's sulogin does NOT. + * But it's rather unpleasant to have non-functioning ^C in a shell, + * and surprisingly, there is no easy way to remove SIG_IGN from ^C + * in the shell. So, we are doing it: */ - /*signal(SIGINT, SIG_DFL);*/ + signal(SIGINT, SIG_DFL); /* Exec shell with no additional parameters. Never returns. */ exec_shell(shell, /* -p? then shell is login:*/(opts & 1), NULL); From 27e1fc679e963baf929e9245294088f016f4eee3 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 18 Jul 2025 10:34:17 +0200 Subject: [PATCH 150/277] ntpd: replace two bb_info_msg's with bb_error_msg All other messages are printed via bb_error_msg in this applet. Signed-off-by: Denys Vlasenko --- networking/ntpd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/networking/ntpd.c b/networking/ntpd.c index 4c4842cb29..dd0a9c91fb 100644 --- a/networking/ntpd.c +++ b/networking/ntpd.c @@ -1057,7 +1057,7 @@ step_time(double offset) } tval = tvn.tv_sec; strftime_YYYYMMDDHHMMSS(buf, sizeof(buf), &tval); - bb_info_msg("setting time to %s.%06u (offset %+fs)", buf, (unsigned)tvn.tv_usec, offset); + bb_error_msg("setting time to %s.%06u (offset %+fs)", buf, (unsigned)tvn.tv_usec, offset); //maybe? G.FREQHOLD_cnt = 0; /* Correct various fields which contain time-relative values: */ @@ -1976,7 +1976,7 @@ recv_and_process_peer_pkt(peer_t *p) p->reachable_bits |= 1; if ((MAX_VERBOSE && G.verbose) || (option_mask32 & OPT_w)) { - bb_info_msg("reply from %s: offset:%+f delay:%f status:0x%02x strat:%d refid:0x%08x rootdelay:%f reach:0x%02x", + bb_error_msg("reply from %s: offset:%+f delay:%f status:0x%02x strat:%d refid:0x%08x rootdelay:%f reach:0x%02x", p->p_dotted, offset, p->p_raw_delay, From ee2ef17cc72e606c8d7f4e25fcc99ff8085d57d5 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 18 Jul 2025 10:38:18 +0200 Subject: [PATCH 151/277] tftp: use bb_error_msg instead of info_msg for a debug message This is the only use of bb_info_msg in tftp. Signed-off-by: Denys Vlasenko --- networking/tftp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking/tftp.c b/networking/tftp.c index f5b4367ca9..b698a92884 100644 --- a/networking/tftp.c +++ b/networking/tftp.c @@ -250,7 +250,7 @@ static int tftp_blksize_check(const char *blksize_str, int maxsize) return -1; } # if ENABLE_TFTP_DEBUG - bb_info_msg("using blksize %u", blksize); + bb_error_msg("using blksize %u", blksize); # endif return blksize; } From 94591a103038db3a336879c6d444162c1f467342 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 19 Jul 2025 02:26:20 +0200 Subject: [PATCH 152/277] bootchartd: check the result of opendir("/proc") Signed-off-by: Denys Vlasenko --- init/bootchartd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/bootchartd.c b/init/bootchartd.c index 0929890a32..a5447c6ad4 100644 --- a/init/bootchartd.c +++ b/init/bootchartd.c @@ -133,7 +133,7 @@ static void dump_file(FILE *fp, const char *filename) static int dump_procs(FILE *fp, int look_for_login_process) { struct dirent *entry; - DIR *dir = opendir("/proc"); + DIR *dir = xopendir("/proc"); int found_login_process = 0; fputs(G.jiffy_line, fp); From 51b45ce28a6a5bbc035b200f170d520f94b7e59f Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 19 Jul 2025 17:20:35 +0200 Subject: [PATCH 153/277] libbb/yescrypt: code shrink function old new delta blockmix_xor_save 1169 708 -461 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 53 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index f75361d966..01503c6e03 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -368,10 +368,10 @@ typedef struct { * be 128r bytes in length; the output Bout must also be the same size. */ static void blockmix( - const salsa20_blk_t *restrict Bin, - salsa20_blk_t *restrict Bout, - size_t r, - pwxform_ctx_t *restrict ctx) + const salsa20_blk_t *restrict Bin, + salsa20_blk_t *restrict Bout, + size_t r, + pwxform_ctx_t *restrict ctx) { uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; @@ -386,22 +386,25 @@ static void blockmix( DECL_SMASK2REG; i = 0; - do { + for (;;) { XOR_X(Bin[i]); PWXFORM; if (unlikely(i >= r)) break; WRITE_X(Bout[i]); i++; - } while (1); + } - ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; + ctx->S0 = S0; + ctx->S1 = S1; + ctx->S2 = S2; ctx->w = w; SALSA20_2(Bout[i]); } -static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, +static uint32_t blockmix_xor( + const salsa20_blk_t *Bin1, const salsa20_blk_t *restrict Bin2, salsa20_blk_t *Bout, size_t r, @@ -462,24 +465,18 @@ static uint32_t blockmix_xor_save( i = 0; r--; - do { + for (;;) { XOR_X_WRITE_XOR_Y_2(Bin2[i], Bin1out[i]); PWXFORM; - WRITE_X(Bin1out[i]); - - XOR_X_WRITE_XOR_Y_2(Bin2[i + 1], Bin1out[i + 1]); - PWXFORM; - - if (unlikely(i >= r)) + if (unlikely(i > r)) break; + WRITE_X(Bin1out[i]); + i++; + } - WRITE_X(Bin1out[i + 1]); - - i += 2; - } while (1); - i++; - - ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; + ctx->S0 = S0; + ctx->S1 = S1; + ctx->S2 = S2; ctx->w = w; SALSA20_2(Bin1out[i]); @@ -801,7 +798,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, if (flags___YESCRYPT_RW) { uint8_t *Si = S + i * Salloc; smix1(Bp, 1, Sbytes / 128, 0 /* no flags */, - (salsa20_blk_t *)Si, 0, NULL, XYp, NULL); + (salsa20_blk_t *)Si, 0, NULL, XYp, NULL); ctx_i = (pwxform_ctx_t *)(Si + Sbytes); ctx_i->S2 = Si; ctx_i->S1 = Si + Sbytes / 3; @@ -809,7 +806,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, ctx_i->w = 0; if (i == 0) hmac_block( - /* key,len: */ Bp + (128 * r - 64), 64, + /* key,len: */ Bp + (128 * r - 64), 64, /* hash fn: */ sha256_begin, /* in,len: */ passwd, 32, /* outbuf: */ passwd @@ -817,7 +814,7 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, } smix1(Bp, r, Np, flags, Vp, NROM, VROM, XYp, ctx_i); smix2(Bp, r, p2floor(Np), Nloop_rw, flags, Vp, - NROM, VROM, XYp, ctx_i); + NROM, VROM, XYp, ctx_i); } if (Nloop_all > Nloop_rw) { @@ -830,8 +827,8 @@ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, ctx_i = (pwxform_ctx_t *)(Si + Sbytes); } smix2(Bp, r, N, Nloop_all - Nloop_rw, - flags & (uint32_t)~YESCRYPT_RW, - V, NROM, VROM, XYp, ctx_i); + flags & (uint32_t)~YESCRYPT_RW, + V, NROM, VROM, XYp, ctx_i); } } } @@ -1046,7 +1043,7 @@ static int yescrypt_kdf32_body( uint32_t i; for (i = 0; i < p; i++) { smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, V, - YCTX_param_NROM, VROM, XY, NULL, NULL); + YCTX_param_NROM, VROM, XY, NULL, NULL); } } From 6979467a62c4bc58eeede0436d06c0cd57649705 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 19 Jul 2025 18:17:24 +0200 Subject: [PATCH 154/277] cryptpw: fix detection of crypt algo from salt (was broken if default isn't DES) The symptom is: "cryptpw ... implicit" testsuite tests were failing if CONFIG_FEATURE_DEFAULT_PASSWD_ALGO is not "des". function old new delta cryptpw_main 223 283 +60 pw_encrypt 974 975 +1 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 61/0) Total: 61 bytes Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt.c | 11 +++++------ loginutils/cryptpw.c | 34 +++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 3b2fea00da..56191b00e3 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -37,9 +37,8 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) #if !ENABLE_USE_BB_CRYPT || ENABLE_USE_BB_CRYPT_YES if ((algo[0]|0x20) == 'y') { /* yescrypt */ salt[1] = 'y'; - len = 24 / 2; + len = 22 / 2; // The "j9T$" below is the default "yescrypt parameters" encoded by yescrypt_encode_params_r(): -// //shadow-4.17.4/src/passwd.c // salt = crypt_make_salt(NULL, NULL); //shadow-4.17.4/lib/salt.c @@ -105,13 +104,13 @@ static char *my_crypt(const char *key, const char *salt) if (salt[0] == '$' && salt[1] && salt[2] == '$') { if (salt[1] == '1') return md5_crypt(xzalloc(MD5_OUT_BUFSIZE), (unsigned char*)key, (unsigned char*)salt); -#if ENABLE_USE_BB_CRYPT_YES - if (salt[1] == 'y') - return yes_crypt(key, salt); -#endif #if ENABLE_USE_BB_CRYPT_SHA if (salt[1] == '5' || salt[1] == '6') return sha_crypt((char*)key, (char*)salt); +#endif +#if ENABLE_USE_BB_CRYPT_YES + if (salt[1] == 'y') + return yes_crypt(key, salt); #endif } diff --git a/loginutils/cryptpw.c b/loginutils/cryptpw.c index c0f6280cd3..666deff0b2 100644 --- a/loginutils/cryptpw.c +++ b/loginutils/cryptpw.c @@ -99,7 +99,7 @@ int cryptpw_main(int argc UNUSED_PARAM, char **argv) ; #endif fd = STDIN_FILENO; - opt_m = CONFIG_FEATURE_DEFAULT_PASSWD_ALGO; + opt_m = NULL; opt_S = NULL; /* at most two non-option arguments; -P NUM */ getopt32long(argv, "^" "sP:+S:m:a:" "\0" "?2", @@ -113,10 +113,34 @@ int cryptpw_main(int argc UNUSED_PARAM, char **argv) if (argv[0] && !opt_S) opt_S = argv[1]; - salt_ptr = crypt_make_pw_salt(salt, opt_m); - if (opt_S) - /* put user's data after the "$N$" prefix */ - safe_strncpy(salt_ptr, opt_S, sizeof(salt) - (sizeof("$N$")-1)); + if (opt_S && !opt_S[0]) { + /* mkpasswd 5.6.2 compat: SALT of "" + * is treated as not specified + * (both forms: -S "" and argv[1] of "") + */ + opt_S = NULL; + } + + if (opt_m) { + /* "cryptpw -m ALGO PASSWORD [SALT]" */ + /* generate "$x$" algo prefix + random salt */ + salt_ptr = crypt_make_pw_salt(salt, opt_m); + if (opt_S) { + /* "cryptpw -m ALGO PASSWORD SALT" */ + /* put SALT data after the "$x$" prefix */ + safe_strncpy(salt_ptr, opt_S, sizeof(salt) - (sizeof("$N$")-1)); + } + } else { + if (!opt_S) { + /* "cryptpw PASSWORD" */ + /* generate random salt with default algo */ + crypt_make_pw_salt(salt, CONFIG_FEATURE_DEFAULT_PASSWD_ALGO); + } else { + /* "cryptpw PASSWORD '$x$SALT'" */ + /* use given salt; algo will be detected by pw_encrypt() */ + safe_strncpy(salt, opt_S, sizeof(salt)); + } + } xmove_fd(fd, STDIN_FILENO); From c6b9e763731545f8e7f3482a90128859496c2964 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 19 Jul 2025 18:42:32 +0200 Subject: [PATCH 155/277] libbb: crypt_make_pw_salt(): fix yescrypt's random salt last char function old new delta crypt_make_pw_salt 128 146 +18 Signed-off-by: Denys Vlasenko --- libbb/pw_encrypt.c | 14 +++++++++++--- miscutils/crond.c | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/libbb/pw_encrypt.c b/libbb/pw_encrypt.c index 56191b00e3..93653de9ff 100644 --- a/libbb/pw_encrypt.c +++ b/libbb/pw_encrypt.c @@ -17,7 +17,7 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) { - int len = 2/2; + int len = 2 / 2; char *salt_ptr = salt; /* Standard chpasswd uses uppercase algos ("MD5", not "md5"). @@ -36,8 +36,8 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) #endif #if !ENABLE_USE_BB_CRYPT || ENABLE_USE_BB_CRYPT_YES if ((algo[0]|0x20) == 'y') { /* yescrypt */ + int rnd; salt[1] = 'y'; - len = 22 / 2; // The "j9T$" below is the default "yescrypt parameters" encoded by yescrypt_encode_params_r(): //shadow-4.17.4/src/passwd.c // salt = crypt_make_salt(NULL, NULL); @@ -61,8 +61,16 @@ char* FAST_FUNC crypt_make_pw_salt(char salt[MAX_PW_SALT_LEN], const char *algo) // params.r = 32; // N in 4KiB // params.N = 1ULL << (count + 7); // 3 -> 1024, 4 -> 2048, ... 11 -> 262144 // yescrypt_encode_params_r(¶ms, rbytes, nrbytes, outbuf, o_size) // always "$y$j9T$" + len = 22 / 2; salt_ptr = stpcpy(salt_ptr, "j9T$"); - crypt_make_rand64encoded(salt_ptr, len); /* appends 2*len random chars */ + /* append 2*len random chars */ + rnd = crypt_make_rand64encoded(salt_ptr, len); + /* fix up last char: it must be in 0..3 range (encoded as one of "./01"). + * IOW: salt_ptr[20..21] encode 16th random byte, must not be > 0xff. + * Without this, we can generate salts which are rejected + * by implementations with more strict salt length check. + */ + salt_ptr[21] = i2a64(rnd & 3); /* For "mkpasswd -m yescrypt PASS j9T$" use case, * "j9T$" is considered part of salt, * need to return pointer to 'j'. Without -4, diff --git a/miscutils/crond.c b/miscutils/crond.c index b3762d327f..96131cae4a 100644 --- a/miscutils/crond.c +++ b/miscutils/crond.c @@ -177,7 +177,7 @@ static void crondlog(unsigned level, const char *msg, va_list va) { if (level >= G.log_level) { /* - * We are called only for info meesages. + * We are called only for info messages. * Warnings/errors use plain bb_[p]error_msg's, which * need not touch syslog_level * (they are ok with LOG_ERR default). From 77a49a61b2957f532c4a736fd3ddf1154aecc176 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 20 Jul 2025 08:40:31 +0200 Subject: [PATCH 156/277] libbb/yescrypt: de-unroll salsa20() function old new delta salsa20 760 296 -464 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-yescrypt-kdf.c | 96 ++++++++++++++++++++++++------- libbb/yescrypt/alg-yescrypt.h | 3 +- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c index 01503c6e03..a9a1bd5915 100644 --- a/libbb/yescrypt/alg-yescrypt-kdf.c +++ b/libbb/yescrypt/alg-yescrypt-kdf.c @@ -129,30 +129,82 @@ static void salsa20(salsa20_blk_t *restrict B, do { #define R(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) /* Operate on columns */ - x[ 4] ^= R(x[ 0]+x[12], 7); x[ 8] ^= R(x[ 4]+x[ 0], 9); - x[12] ^= R(x[ 8]+x[ 4],13); x[ 0] ^= R(x[12]+x[ 8],18); - - x[ 9] ^= R(x[ 5]+x[ 1], 7); x[13] ^= R(x[ 9]+x[ 5], 9); - x[ 1] ^= R(x[13]+x[ 9],13); x[ 5] ^= R(x[ 1]+x[13],18); - - x[14] ^= R(x[10]+x[ 6], 7); x[ 2] ^= R(x[14]+x[10], 9); - x[ 6] ^= R(x[ 2]+x[14],13); x[10] ^= R(x[ 6]+x[ 2],18); - - x[ 3] ^= R(x[15]+x[11], 7); x[ 7] ^= R(x[ 3]+x[15], 9); - x[11] ^= R(x[ 7]+x[ 3],13); x[15] ^= R(x[11]+x[ 7],18); - +#if KDF_UNROLL_SALSA20 + x[ 4] ^= R(x[ 0]+x[12], 7); // x[j] ^= R(x[k]+x[l], CONST) + x[ 8] ^= R(x[ 4]+x[ 0], 9); + x[12] ^= R(x[ 8]+x[ 4],13); + x[ 0] ^= R(x[12]+x[ 8],18); + + x[ 9] ^= R(x[ 5]+x[ 1], 7); + x[13] ^= R(x[ 9]+x[ 5], 9); + x[ 1] ^= R(x[13]+x[ 9],13); + x[ 5] ^= R(x[ 1]+x[13],18); + + x[14] ^= R(x[10]+x[ 6], 7); + x[ 2] ^= R(x[14]+x[10], 9); + x[ 6] ^= R(x[ 2]+x[14],13); + x[10] ^= R(x[ 6]+x[ 2],18); + + x[ 3] ^= R(x[15]+x[11], 7); + x[ 7] ^= R(x[ 3]+x[15], 9); + x[11] ^= R(x[ 7]+x[ 3],13); + x[15] ^= R(x[11]+x[ 7],18); +#else + { + unsigned j, k, l; + j = 4; k = 0; l = 12; + for (;;) { + uint32_t t; + x[j] ^= ({ t = x[k] + x[l]; R(t, 7); }); l = k; k = j; j = (j+4) & 0xf; + x[j] ^= ({ t = x[k] + x[l]; R(t, 9); }); l = k; k = j; j = (j+4) & 0xf; + x[j] ^= ({ t = x[k] + x[l]; R(t,13); }); l = k; k = j; j = (j+4) & 0xf; + x[j] ^= ({ t = x[k] + x[l]; R(t,18); }); + if (j == 15) break; + l = j + 1; k = j + 5; j = (j+9) & 0xf; + } + } +#endif /* Operate on rows */ - x[ 1] ^= R(x[ 0]+x[ 3], 7); x[ 2] ^= R(x[ 1]+x[ 0], 9); - x[ 3] ^= R(x[ 2]+x[ 1],13); x[ 0] ^= R(x[ 3]+x[ 2],18); - - x[ 6] ^= R(x[ 5]+x[ 4], 7); x[ 7] ^= R(x[ 6]+x[ 5], 9); - x[ 4] ^= R(x[ 7]+x[ 6],13); x[ 5] ^= R(x[ 4]+x[ 7],18); - - x[11] ^= R(x[10]+x[ 9], 7); x[ 8] ^= R(x[11]+x[10], 9); - x[ 9] ^= R(x[ 8]+x[11],13); x[10] ^= R(x[ 9]+x[ 8],18); +#if KDF_UNROLL_SALSA20 +// i=0 n=0 + x[ 1] ^= R(x[ 0]+x[ 3], 7); // [i + (n+1)&3] [i + (n+0)&3] [i + (n+3)&3] + x[ 2] ^= R(x[ 1]+x[ 0], 9); // [i + (n+2)&3] [i + (n+1)&3] [i + (n+0)&3] + x[ 3] ^= R(x[ 2]+x[ 1],13); // [i + (n+3)&3] [i + (n+2)&3] [i + (n+1)&3] + x[ 0] ^= R(x[ 3]+x[ 2],18); // [i + (n+0)&3] [i + (n+3)&3] [i + (n+2)&3] +// i=4 n=1 ^^^j^^^ ^^^k^^^ ^^^l^^^ + x[ 6] ^= R(x[ 5]+x[ 4], 7); // [i + (n+1)&3] [i + (n+0)&3] [i + (n+3)&3] + x[ 7] ^= R(x[ 6]+x[ 5], 9); // [i + (n+2)&3] [i + (n+1)&3] [i + (n+0)&3] + x[ 4] ^= R(x[ 7]+x[ 6],13); // [i + (n+3)&3] [i + (n+2)&3] [i + (n+1)&3] + x[ 5] ^= R(x[ 4]+x[ 7],18); // [i + (n+0)&3] [i + (n+3)&3] [i + (n+2)&3] +// i=8 n=2 + x[11] ^= R(x[10]+x[ 9], 7); // [i + (n+1)&3] [i + (n+0)&3] [i + (n+3)&3] + x[ 8] ^= R(x[11]+x[10], 9); // [i + (n+2)&3] [i + (n+1)&3] [i + (n+0)&3] + x[ 9] ^= R(x[ 8]+x[11],13); // [i + (n+3)&3] [i + (n+2)&3] [i + (n+1)&3] + x[10] ^= R(x[ 9]+x[ 8],18); // [i + (n+0)&3] [i + (n+3)&3] [i + (n+2)&3] +// i=12 n=3 + x[12] ^= R(x[15]+x[14], 7); // [i + (n+1)&3] [i + (n+0)&3] [i + (n+3)&3] + x[13] ^= R(x[12]+x[15], 9); // [i + (n+2)&3] [i + (n+1)&3] [i + (n+0)&3] + x[14] ^= R(x[13]+x[12],13); // [i + (n+3)&3] [i + (n+2)&3] [i + (n+1)&3] + x[15] ^= R(x[14]+x[13],18); // [i + (n+0)&3] [i + (n+3)&3] [i + (n+2)&3] +#else + { + unsigned j, k, l; + uint32_t *xrow; + j = 1; k = 0; l = 3; + xrow = &x[0]; + for (;;) { + uint32_t t; + xrow[j] ^= ({ t = xrow[k] + xrow[l]; R(t, 7); }); l = k; k = j; j = (j+1) & 3; + xrow[j] ^= ({ t = xrow[k] + xrow[l]; R(t, 9); }); l = k; k = j; j = (j+1) & 3; + xrow[j] ^= ({ t = xrow[k] + xrow[l]; R(t,13); }); l = k; k = j; j = (j+1) & 3; + xrow[j] ^= ({ t = xrow[k] + xrow[l]; R(t,18); }); + if (j == 3) break; + l = j; k = j + 1; j = (j+2) & 3; + xrow += 4; + } + } +#endif - x[12] ^= R(x[15]+x[14], 7); x[13] ^= R(x[12]+x[15], 9); - x[14] ^= R(x[13]+x[12],13); x[15] ^= R(x[14]+x[13],18); #undef R } while (--doublerounds); #undef x diff --git a/libbb/yescrypt/alg-yescrypt.h b/libbb/yescrypt/alg-yescrypt.h index 5051efbb45..b69843f5d2 100644 --- a/libbb/yescrypt/alg-yescrypt.h +++ b/libbb/yescrypt/alg-yescrypt.h @@ -44,7 +44,6 @@ # define TEST_DECODE64 0 #endif - // Only accept one-char parameters in salt, and only first three? // Almost any reasonable yescrypt hashes in /etc/shadow should // only ever use "jXY" parameters which set N and r. @@ -115,6 +114,8 @@ // -4864 bytes if 0: #define KDF_UNROLL_PWXFORM 0 // if both this ^^^^^^^^^^ and PWXFORM_ROUND set to 0: -7666 bytes +// -464 bytes: +#define KDF_UNROLL_SALSA20 0 /** * Type and possible values for the flags argument of yescrypt_kdf(), From d23ad559044af2f706d7b7a8ede05107b4c10cf7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Mon, 21 Jul 2025 08:14:08 +0200 Subject: [PATCH 157/277] libbb: change define names in SHA-NI code to match kernel source more closely No code changes Signed-off-by: Denys Vlasenko --- libbb/hash_sha256_hwaccel_x86-32.S | 166 ++++++++++++++--------------- libbb/hash_sha256_hwaccel_x86-64.S | 166 ++++++++++++++--------------- 2 files changed, 166 insertions(+), 166 deletions(-) diff --git a/libbb/hash_sha256_hwaccel_x86-32.S b/libbb/hash_sha256_hwaccel_x86-32.S index a0e4a571a0..332b7513fc 100644 --- a/libbb/hash_sha256_hwaccel_x86-32.S +++ b/libbb/hash_sha256_hwaccel_x86-32.S @@ -34,10 +34,10 @@ #define MSG %xmm0 #define STATE0 %xmm1 #define STATE1 %xmm2 -#define MSGTMP0 %xmm3 -#define MSGTMP1 %xmm4 -#define MSGTMP2 %xmm5 -#define MSGTMP3 %xmm6 +#define MSG0 %xmm3 +#define MSG1 %xmm4 +#define MSG2 %xmm5 +#define MSG3 %xmm6 #define XMMTMP %xmm7 @@ -46,9 +46,9 @@ .balign 8 # allow decoders to fetch at least 2 first insns sha256_process_block64_shaNI: - movu128 76+0*16(%eax), XMMTMP /* ABCD (little-endian dword order) */ + movu128 76+0*16(%eax), XMMTMP /* ABCD (shown least-significant-dword-first) */ movu128 76+1*16(%eax), STATE1 /* EFGH */ -/* shufps takes dwords 0,1 from *2nd* operand, and dwords 2,3 from 1st one */ +/* shufps: dwords 0,1 of the result are selected from *2nd* operand, and dwords 2,3 from 1st operand */ mova128 STATE1, STATE0 /* --- -------------- ABCD -- EFGH */ shufps SHUF(1,0,1,0), XMMTMP, STATE0 /* FEBA */ @@ -61,7 +61,7 @@ sha256_process_block64_shaNI: /* Rounds 0-3 */ movu128 0*16(DATA_PTR), MSG pshufb XMMTMP, MSG - mova128 MSG, MSGTMP0 + mova128 MSG, MSG0 paddd 0*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG @@ -70,170 +70,170 @@ sha256_process_block64_shaNI: /* Rounds 4-7 */ movu128 1*16(DATA_PTR), MSG pshufb XMMTMP, MSG - mova128 MSG, MSGTMP1 + mova128 MSG, MSG1 paddd 1*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP1, MSGTMP0 + sha256msg1 MSG1, MSG0 /* Rounds 8-11 */ movu128 2*16(DATA_PTR), MSG pshufb XMMTMP, MSG - mova128 MSG, MSGTMP2 + mova128 MSG, MSG2 paddd 2*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP2, MSGTMP1 + sha256msg1 MSG2, MSG1 /* Rounds 12-15 */ movu128 3*16(DATA_PTR), MSG pshufb XMMTMP, MSG /* ...to here */ - mova128 MSG, MSGTMP3 + mova128 MSG, MSG3 paddd 3*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP3, XMMTMP - palignr $4, MSGTMP2, XMMTMP - paddd XMMTMP, MSGTMP0 - sha256msg2 MSGTMP3, MSGTMP0 + mova128 MSG3, XMMTMP + palignr $4, MSG2, XMMTMP + paddd XMMTMP, MSG0 + sha256msg2 MSG3, MSG0 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP3, MSGTMP2 + sha256msg1 MSG3, MSG2 /* Rounds 16-19 */ - mova128 MSGTMP0, MSG + mova128 MSG0, MSG paddd 4*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP0, XMMTMP - palignr $4, MSGTMP3, XMMTMP - paddd XMMTMP, MSGTMP1 - sha256msg2 MSGTMP0, MSGTMP1 + mova128 MSG0, XMMTMP + palignr $4, MSG3, XMMTMP + paddd XMMTMP, MSG1 + sha256msg2 MSG0, MSG1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP0, MSGTMP3 + sha256msg1 MSG0, MSG3 /* Rounds 20-23 */ - mova128 MSGTMP1, MSG + mova128 MSG1, MSG paddd 5*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP1, XMMTMP - palignr $4, MSGTMP0, XMMTMP - paddd XMMTMP, MSGTMP2 - sha256msg2 MSGTMP1, MSGTMP2 + mova128 MSG1, XMMTMP + palignr $4, MSG0, XMMTMP + paddd XMMTMP, MSG2 + sha256msg2 MSG1, MSG2 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP1, MSGTMP0 + sha256msg1 MSG1, MSG0 /* Rounds 24-27 */ - mova128 MSGTMP2, MSG + mova128 MSG2, MSG paddd 6*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP2, XMMTMP - palignr $4, MSGTMP1, XMMTMP - paddd XMMTMP, MSGTMP3 - sha256msg2 MSGTMP2, MSGTMP3 + mova128 MSG2, XMMTMP + palignr $4, MSG1, XMMTMP + paddd XMMTMP, MSG3 + sha256msg2 MSG2, MSG3 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP2, MSGTMP1 + sha256msg1 MSG2, MSG1 /* Rounds 28-31 */ - mova128 MSGTMP3, MSG + mova128 MSG3, MSG paddd 7*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP3, XMMTMP - palignr $4, MSGTMP2, XMMTMP - paddd XMMTMP, MSGTMP0 - sha256msg2 MSGTMP3, MSGTMP0 + mova128 MSG3, XMMTMP + palignr $4, MSG2, XMMTMP + paddd XMMTMP, MSG0 + sha256msg2 MSG3, MSG0 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP3, MSGTMP2 + sha256msg1 MSG3, MSG2 /* Rounds 32-35 */ - mova128 MSGTMP0, MSG + mova128 MSG0, MSG paddd 8*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP0, XMMTMP - palignr $4, MSGTMP3, XMMTMP - paddd XMMTMP, MSGTMP1 - sha256msg2 MSGTMP0, MSGTMP1 + mova128 MSG0, XMMTMP + palignr $4, MSG3, XMMTMP + paddd XMMTMP, MSG1 + sha256msg2 MSG0, MSG1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP0, MSGTMP3 + sha256msg1 MSG0, MSG3 /* Rounds 36-39 */ - mova128 MSGTMP1, MSG + mova128 MSG1, MSG paddd 9*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP1, XMMTMP - palignr $4, MSGTMP0, XMMTMP - paddd XMMTMP, MSGTMP2 - sha256msg2 MSGTMP1, MSGTMP2 + mova128 MSG1, XMMTMP + palignr $4, MSG0, XMMTMP + paddd XMMTMP, MSG2 + sha256msg2 MSG1, MSG2 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP1, MSGTMP0 + sha256msg1 MSG1, MSG0 /* Rounds 40-43 */ - mova128 MSGTMP2, MSG + mova128 MSG2, MSG paddd 10*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP2, XMMTMP - palignr $4, MSGTMP1, XMMTMP - paddd XMMTMP, MSGTMP3 - sha256msg2 MSGTMP2, MSGTMP3 + mova128 MSG2, XMMTMP + palignr $4, MSG1, XMMTMP + paddd XMMTMP, MSG3 + sha256msg2 MSG2, MSG3 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP2, MSGTMP1 + sha256msg1 MSG2, MSG1 /* Rounds 44-47 */ - mova128 MSGTMP3, MSG + mova128 MSG3, MSG paddd 11*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP3, XMMTMP - palignr $4, MSGTMP2, XMMTMP - paddd XMMTMP, MSGTMP0 - sha256msg2 MSGTMP3, MSGTMP0 + mova128 MSG3, XMMTMP + palignr $4, MSG2, XMMTMP + paddd XMMTMP, MSG0 + sha256msg2 MSG3, MSG0 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP3, MSGTMP2 + sha256msg1 MSG3, MSG2 /* Rounds 48-51 */ - mova128 MSGTMP0, MSG + mova128 MSG0, MSG paddd 12*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP0, XMMTMP - palignr $4, MSGTMP3, XMMTMP - paddd XMMTMP, MSGTMP1 - sha256msg2 MSGTMP0, MSGTMP1 + mova128 MSG0, XMMTMP + palignr $4, MSG3, XMMTMP + paddd XMMTMP, MSG1 + sha256msg2 MSG0, MSG1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP0, MSGTMP3 + sha256msg1 MSG0, MSG3 /* Rounds 52-55 */ - mova128 MSGTMP1, MSG + mova128 MSG1, MSG paddd 13*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP1, XMMTMP - palignr $4, MSGTMP0, XMMTMP - paddd XMMTMP, MSGTMP2 - sha256msg2 MSGTMP1, MSGTMP2 + mova128 MSG1, XMMTMP + palignr $4, MSG0, XMMTMP + paddd XMMTMP, MSG2 + sha256msg2 MSG1, MSG2 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 /* Rounds 56-59 */ - mova128 MSGTMP2, MSG + mova128 MSG2, MSG paddd 14*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP2, XMMTMP - palignr $4, MSGTMP1, XMMTMP - paddd XMMTMP, MSGTMP3 - sha256msg2 MSGTMP2, MSGTMP3 + mova128 MSG2, XMMTMP + palignr $4, MSG1, XMMTMP + paddd XMMTMP, MSG3 + sha256msg2 MSG2, MSG3 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 /* Rounds 60-63 */ - mova128 MSGTMP3, MSG + mova128 MSG3, MSG paddd 15*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG @@ -241,7 +241,7 @@ sha256_process_block64_shaNI: /* Write hash values back in the correct order */ mova128 STATE0, XMMTMP -/* shufps takes dwords 0,1 from *2nd* operand, and dwords 2,3 from 1st one */ +/* shufps: dwords 0,1 of the result are selected from *2nd* operand, and dwords 2,3 from 1st operand */ /* --- -------------- HGDC -- FEBA */ shufps SHUF(3,2,3,2), STATE1, STATE0 /* ABCD */ shufps SHUF(1,0,1,0), STATE1, XMMTMP /* EFGH */ diff --git a/libbb/hash_sha256_hwaccel_x86-64.S b/libbb/hash_sha256_hwaccel_x86-64.S index 172c2eae26..f8911968be 100644 --- a/libbb/hash_sha256_hwaccel_x86-64.S +++ b/libbb/hash_sha256_hwaccel_x86-64.S @@ -34,10 +34,10 @@ #define MSG %xmm0 #define STATE0 %xmm1 #define STATE1 %xmm2 -#define MSGTMP0 %xmm3 -#define MSGTMP1 %xmm4 -#define MSGTMP2 %xmm5 -#define MSGTMP3 %xmm6 +#define MSG0 %xmm3 +#define MSG1 %xmm4 +#define MSG2 %xmm5 +#define MSG3 %xmm6 #define XMMTMP %xmm7 @@ -49,9 +49,9 @@ .balign 8 # allow decoders to fetch at least 2 first insns sha256_process_block64_shaNI: - movu128 80+0*16(%rdi), XMMTMP /* ABCD (little-endian dword order) */ + movu128 80+0*16(%rdi), XMMTMP /* ABCD (shown least-significant-dword-first) */ movu128 80+1*16(%rdi), STATE1 /* EFGH */ -/* shufps takes dwords 0,1 from *2nd* operand, and dwords 2,3 from 1st one */ +/* shufps: dwords 0,1 of the result are selected from *2nd* operand, and dwords 2,3 from 1st operand */ mova128 STATE1, STATE0 /* --- -------------- ABCD -- EFGH */ shufps SHUF(1,0,1,0), XMMTMP, STATE0 /* FEBA */ @@ -68,7 +68,7 @@ sha256_process_block64_shaNI: /* Rounds 0-3 */ movu128 0*16(DATA_PTR), MSG pshufb XMMTMP, MSG - mova128 MSG, MSGTMP0 + mova128 MSG, MSG0 paddd 0*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG @@ -77,170 +77,170 @@ sha256_process_block64_shaNI: /* Rounds 4-7 */ movu128 1*16(DATA_PTR), MSG pshufb XMMTMP, MSG - mova128 MSG, MSGTMP1 + mova128 MSG, MSG1 paddd 1*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP1, MSGTMP0 + sha256msg1 MSG1, MSG0 /* Rounds 8-11 */ movu128 2*16(DATA_PTR), MSG pshufb XMMTMP, MSG - mova128 MSG, MSGTMP2 + mova128 MSG, MSG2 paddd 2*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP2, MSGTMP1 + sha256msg1 MSG2, MSG1 /* Rounds 12-15 */ movu128 3*16(DATA_PTR), MSG pshufb XMMTMP, MSG /* ...to here */ - mova128 MSG, MSGTMP3 + mova128 MSG, MSG3 paddd 3*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP3, XMMTMP - palignr $4, MSGTMP2, XMMTMP - paddd XMMTMP, MSGTMP0 - sha256msg2 MSGTMP3, MSGTMP0 + mova128 MSG3, XMMTMP + palignr $4, MSG2, XMMTMP + paddd XMMTMP, MSG0 + sha256msg2 MSG3, MSG0 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP3, MSGTMP2 + sha256msg1 MSG3, MSG2 /* Rounds 16-19 */ - mova128 MSGTMP0, MSG + mova128 MSG0, MSG paddd 4*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP0, XMMTMP - palignr $4, MSGTMP3, XMMTMP - paddd XMMTMP, MSGTMP1 - sha256msg2 MSGTMP0, MSGTMP1 + mova128 MSG0, XMMTMP + palignr $4, MSG3, XMMTMP + paddd XMMTMP, MSG1 + sha256msg2 MSG0, MSG1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP0, MSGTMP3 + sha256msg1 MSG0, MSG3 /* Rounds 20-23 */ - mova128 MSGTMP1, MSG + mova128 MSG1, MSG paddd 5*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP1, XMMTMP - palignr $4, MSGTMP0, XMMTMP - paddd XMMTMP, MSGTMP2 - sha256msg2 MSGTMP1, MSGTMP2 + mova128 MSG1, XMMTMP + palignr $4, MSG0, XMMTMP + paddd XMMTMP, MSG2 + sha256msg2 MSG1, MSG2 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP1, MSGTMP0 + sha256msg1 MSG1, MSG0 /* Rounds 24-27 */ - mova128 MSGTMP2, MSG + mova128 MSG2, MSG paddd 6*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP2, XMMTMP - palignr $4, MSGTMP1, XMMTMP - paddd XMMTMP, MSGTMP3 - sha256msg2 MSGTMP2, MSGTMP3 + mova128 MSG2, XMMTMP + palignr $4, MSG1, XMMTMP + paddd XMMTMP, MSG3 + sha256msg2 MSG2, MSG3 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP2, MSGTMP1 + sha256msg1 MSG2, MSG1 /* Rounds 28-31 */ - mova128 MSGTMP3, MSG + mova128 MSG3, MSG paddd 7*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP3, XMMTMP - palignr $4, MSGTMP2, XMMTMP - paddd XMMTMP, MSGTMP0 - sha256msg2 MSGTMP3, MSGTMP0 + mova128 MSG3, XMMTMP + palignr $4, MSG2, XMMTMP + paddd XMMTMP, MSG0 + sha256msg2 MSG3, MSG0 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP3, MSGTMP2 + sha256msg1 MSG3, MSG2 /* Rounds 32-35 */ - mova128 MSGTMP0, MSG + mova128 MSG0, MSG paddd 8*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP0, XMMTMP - palignr $4, MSGTMP3, XMMTMP - paddd XMMTMP, MSGTMP1 - sha256msg2 MSGTMP0, MSGTMP1 + mova128 MSG0, XMMTMP + palignr $4, MSG3, XMMTMP + paddd XMMTMP, MSG1 + sha256msg2 MSG0, MSG1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP0, MSGTMP3 + sha256msg1 MSG0, MSG3 /* Rounds 36-39 */ - mova128 MSGTMP1, MSG + mova128 MSG1, MSG paddd 9*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP1, XMMTMP - palignr $4, MSGTMP0, XMMTMP - paddd XMMTMP, MSGTMP2 - sha256msg2 MSGTMP1, MSGTMP2 + mova128 MSG1, XMMTMP + palignr $4, MSG0, XMMTMP + paddd XMMTMP, MSG2 + sha256msg2 MSG1, MSG2 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP1, MSGTMP0 + sha256msg1 MSG1, MSG0 /* Rounds 40-43 */ - mova128 MSGTMP2, MSG + mova128 MSG2, MSG paddd 10*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP2, XMMTMP - palignr $4, MSGTMP1, XMMTMP - paddd XMMTMP, MSGTMP3 - sha256msg2 MSGTMP2, MSGTMP3 + mova128 MSG2, XMMTMP + palignr $4, MSG1, XMMTMP + paddd XMMTMP, MSG3 + sha256msg2 MSG2, MSG3 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP2, MSGTMP1 + sha256msg1 MSG2, MSG1 /* Rounds 44-47 */ - mova128 MSGTMP3, MSG + mova128 MSG3, MSG paddd 11*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP3, XMMTMP - palignr $4, MSGTMP2, XMMTMP - paddd XMMTMP, MSGTMP0 - sha256msg2 MSGTMP3, MSGTMP0 + mova128 MSG3, XMMTMP + palignr $4, MSG2, XMMTMP + paddd XMMTMP, MSG0 + sha256msg2 MSG3, MSG0 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP3, MSGTMP2 + sha256msg1 MSG3, MSG2 /* Rounds 48-51 */ - mova128 MSGTMP0, MSG + mova128 MSG0, MSG paddd 12*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP0, XMMTMP - palignr $4, MSGTMP3, XMMTMP - paddd XMMTMP, MSGTMP1 - sha256msg2 MSGTMP0, MSGTMP1 + mova128 MSG0, XMMTMP + palignr $4, MSG3, XMMTMP + paddd XMMTMP, MSG1 + sha256msg2 MSG0, MSG1 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 - sha256msg1 MSGTMP0, MSGTMP3 + sha256msg1 MSG0, MSG3 /* Rounds 52-55 */ - mova128 MSGTMP1, MSG + mova128 MSG1, MSG paddd 13*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP1, XMMTMP - palignr $4, MSGTMP0, XMMTMP - paddd XMMTMP, MSGTMP2 - sha256msg2 MSGTMP1, MSGTMP2 + mova128 MSG1, XMMTMP + palignr $4, MSG0, XMMTMP + paddd XMMTMP, MSG2 + sha256msg2 MSG1, MSG2 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 /* Rounds 56-59 */ - mova128 MSGTMP2, MSG + mova128 MSG2, MSG paddd 14*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - mova128 MSGTMP2, XMMTMP - palignr $4, MSGTMP1, XMMTMP - paddd XMMTMP, MSGTMP3 - sha256msg2 MSGTMP2, MSGTMP3 + mova128 MSG2, XMMTMP + palignr $4, MSG1, XMMTMP + paddd XMMTMP, MSG3 + sha256msg2 MSG2, MSG3 shuf128_32 $0x0E, MSG, MSG sha256rnds2 MSG, STATE1, STATE0 /* Rounds 60-63 */ - mova128 MSGTMP3, MSG + mova128 MSG3, MSG paddd 15*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 shuf128_32 $0x0E, MSG, MSG @@ -252,7 +252,7 @@ sha256_process_block64_shaNI: /* Write hash values back in the correct order */ mova128 STATE0, XMMTMP -/* shufps takes dwords 0,1 from *2nd* operand, and dwords 2,3 from 1st one */ +/* shufps: dwords 0,1 of the result are selected from *2nd* operand, and dwords 2,3 from 1st operand */ /* --- -------------- HGDC -- FEBA */ shufps SHUF(3,2,3,2), STATE1, STATE0 /* ABCD */ shufps SHUF(1,0,1,0), STATE1, XMMTMP /* EFGH */ From ed22c5bd4a537f016321a888743a8b12f6bb15a9 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 22 Jul 2025 00:03:02 +0200 Subject: [PATCH 158/277] libbb: SHA-NI code shrink function old new delta sha256_process_block64_shaNI 673 657 -16 Signed-off-by: Denys Vlasenko --- libbb/hash_sha256_hwaccel_x86-32.S | 50 ++++++++++++++++++++---------- libbb/hash_sha256_hwaccel_x86-64.S | 50 ++++++++++++++++++++---------- 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/libbb/hash_sha256_hwaccel_x86-32.S b/libbb/hash_sha256_hwaccel_x86-32.S index 332b7513fc..6362ae3823 100644 --- a/libbb/hash_sha256_hwaccel_x86-32.S +++ b/libbb/hash_sha256_hwaccel_x86-32.S @@ -41,7 +41,7 @@ #define XMMTMP %xmm7 -#define SHUF(a,b,c,d) $(a+(b<<2)+(c<<4)+(d<<6)) +#define SHUF(a,b,c,d) $((a)+((b)<<2)+((c)<<4)+((d)<<6)) .balign 8 # allow decoders to fetch at least 2 first insns sha256_process_block64_shaNI: @@ -58,13 +58,29 @@ sha256_process_block64_shaNI: mova128 PSHUFFLE_BSWAP32_FLIP_MASK, XMMTMP movl $K256+8*16, SHA256CONSTANTS +// sha256rnds2 instruction uses only lower 64 bits of MSG. +// The code below needs to move upper 64 bits to lower 64 bits +// for the second sha256rnds2 invocation +// (what remains in upper bits does not matter). +// There are several ways to do it: +// movhlps MSG, MSG // abcd -> cdcd (3 bytes of code) +// shuf128_32 SHUF(2,3,n,n), MSG, MSG // abcd -> cdXX (4 bytes) +// punpckhqdq MSG, MSG // abcd -> cdcd (4 bytes) +// psrldq $8, MSG // abcd -> cd00 (5 bytes) +// palignr $8, MSG, MSG // abcd -> cdab (6 bytes, SSSE3 insn) +#define MOVE_UPPER64_DOWN(reg) movhlps reg, reg +//#define MOVE_UPPER64_DOWN(reg) shuf128_32 SHUF(2,3,0,0), reg, reg +//#define MOVE_UPPER64_DOWN(reg) punpckhqdq reg, reg +//#define MOVE_UPPER64_DOWN(reg) psrldq $8, reg +//#define MOVE_UPPER64_DOWN(reg) palignr $8, reg, reg + /* Rounds 0-3 */ movu128 0*16(DATA_PTR), MSG pshufb XMMTMP, MSG mova128 MSG, MSG0 paddd 0*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Rounds 4-7 */ @@ -73,7 +89,7 @@ sha256_process_block64_shaNI: mova128 MSG, MSG1 paddd 1*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG1, MSG0 @@ -83,7 +99,7 @@ sha256_process_block64_shaNI: mova128 MSG, MSG2 paddd 2*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG2, MSG1 @@ -98,7 +114,7 @@ sha256_process_block64_shaNI: palignr $4, MSG2, XMMTMP paddd XMMTMP, MSG0 sha256msg2 MSG3, MSG0 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG3, MSG2 @@ -110,7 +126,7 @@ sha256_process_block64_shaNI: palignr $4, MSG3, XMMTMP paddd XMMTMP, MSG1 sha256msg2 MSG0, MSG1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG0, MSG3 @@ -122,7 +138,7 @@ sha256_process_block64_shaNI: palignr $4, MSG0, XMMTMP paddd XMMTMP, MSG2 sha256msg2 MSG1, MSG2 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG1, MSG0 @@ -134,7 +150,7 @@ sha256_process_block64_shaNI: palignr $4, MSG1, XMMTMP paddd XMMTMP, MSG3 sha256msg2 MSG2, MSG3 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG2, MSG1 @@ -146,7 +162,7 @@ sha256_process_block64_shaNI: palignr $4, MSG2, XMMTMP paddd XMMTMP, MSG0 sha256msg2 MSG3, MSG0 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG3, MSG2 @@ -158,7 +174,7 @@ sha256_process_block64_shaNI: palignr $4, MSG3, XMMTMP paddd XMMTMP, MSG1 sha256msg2 MSG0, MSG1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG0, MSG3 @@ -170,7 +186,7 @@ sha256_process_block64_shaNI: palignr $4, MSG0, XMMTMP paddd XMMTMP, MSG2 sha256msg2 MSG1, MSG2 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG1, MSG0 @@ -182,7 +198,7 @@ sha256_process_block64_shaNI: palignr $4, MSG1, XMMTMP paddd XMMTMP, MSG3 sha256msg2 MSG2, MSG3 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG2, MSG1 @@ -194,7 +210,7 @@ sha256_process_block64_shaNI: palignr $4, MSG2, XMMTMP paddd XMMTMP, MSG0 sha256msg2 MSG3, MSG0 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG3, MSG2 @@ -206,7 +222,7 @@ sha256_process_block64_shaNI: palignr $4, MSG3, XMMTMP paddd XMMTMP, MSG1 sha256msg2 MSG0, MSG1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG0, MSG3 @@ -218,7 +234,7 @@ sha256_process_block64_shaNI: palignr $4, MSG0, XMMTMP paddd XMMTMP, MSG2 sha256msg2 MSG1, MSG2 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Rounds 56-59 */ @@ -229,14 +245,14 @@ sha256_process_block64_shaNI: palignr $4, MSG1, XMMTMP paddd XMMTMP, MSG3 sha256msg2 MSG2, MSG3 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Rounds 60-63 */ mova128 MSG3, MSG paddd 15*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Write hash values back in the correct order */ diff --git a/libbb/hash_sha256_hwaccel_x86-64.S b/libbb/hash_sha256_hwaccel_x86-64.S index f8911968be..92f00ebcda 100644 --- a/libbb/hash_sha256_hwaccel_x86-64.S +++ b/libbb/hash_sha256_hwaccel_x86-64.S @@ -44,7 +44,7 @@ #define SAVE0 %xmm8 #define SAVE1 %xmm9 -#define SHUF(a,b,c,d) $(a+(b<<2)+(c<<4)+(d<<6)) +#define SHUF(a,b,c,d) $((a)+((b)<<2)+((c)<<4)+((d)<<6)) .balign 8 # allow decoders to fetch at least 2 first insns sha256_process_block64_shaNI: @@ -65,13 +65,29 @@ sha256_process_block64_shaNI: mova128 STATE0, SAVE0 mova128 STATE1, SAVE1 +// sha256rnds2 instruction uses only lower 64 bits of MSG. +// The code below needs to move upper 64 bits to lower 64 bits +// for the second sha256rnds2 invocation +// (what remains in upper bits does not matter). +// There are several ways to do it: +// movhlps MSG, MSG // abcd -> cdcd (3 bytes of code) +// shuf128_32 SHUF(2,3,n,n), MSG, MSG // abcd -> cdXX (4 bytes) +// punpckhqdq MSG, MSG // abcd -> cdcd (4 bytes) +// psrldq $8, MSG // abcd -> cd00 (5 bytes) +// palignr $8, MSG, MSG // abcd -> cdab (6 bytes, SSSE3 insn) +#define MOVE_UPPER64_DOWN(reg) movhlps reg, reg +//#define MOVE_UPPER64_DOWN(reg) shuf128_32 SHUF(2,3,0,0), reg, reg +//#define MOVE_UPPER64_DOWN(reg) punpckhqdq reg, reg +//#define MOVE_UPPER64_DOWN(reg) psrldq $8, reg +//#define MOVE_UPPER64_DOWN(reg) palignr $8, reg, reg + /* Rounds 0-3 */ movu128 0*16(DATA_PTR), MSG pshufb XMMTMP, MSG mova128 MSG, MSG0 paddd 0*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Rounds 4-7 */ @@ -80,7 +96,7 @@ sha256_process_block64_shaNI: mova128 MSG, MSG1 paddd 1*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG1, MSG0 @@ -90,7 +106,7 @@ sha256_process_block64_shaNI: mova128 MSG, MSG2 paddd 2*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG2, MSG1 @@ -105,7 +121,7 @@ sha256_process_block64_shaNI: palignr $4, MSG2, XMMTMP paddd XMMTMP, MSG0 sha256msg2 MSG3, MSG0 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG3, MSG2 @@ -117,7 +133,7 @@ sha256_process_block64_shaNI: palignr $4, MSG3, XMMTMP paddd XMMTMP, MSG1 sha256msg2 MSG0, MSG1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG0, MSG3 @@ -129,7 +145,7 @@ sha256_process_block64_shaNI: palignr $4, MSG0, XMMTMP paddd XMMTMP, MSG2 sha256msg2 MSG1, MSG2 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG1, MSG0 @@ -141,7 +157,7 @@ sha256_process_block64_shaNI: palignr $4, MSG1, XMMTMP paddd XMMTMP, MSG3 sha256msg2 MSG2, MSG3 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG2, MSG1 @@ -153,7 +169,7 @@ sha256_process_block64_shaNI: palignr $4, MSG2, XMMTMP paddd XMMTMP, MSG0 sha256msg2 MSG3, MSG0 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG3, MSG2 @@ -165,7 +181,7 @@ sha256_process_block64_shaNI: palignr $4, MSG3, XMMTMP paddd XMMTMP, MSG1 sha256msg2 MSG0, MSG1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG0, MSG3 @@ -177,7 +193,7 @@ sha256_process_block64_shaNI: palignr $4, MSG0, XMMTMP paddd XMMTMP, MSG2 sha256msg2 MSG1, MSG2 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG1, MSG0 @@ -189,7 +205,7 @@ sha256_process_block64_shaNI: palignr $4, MSG1, XMMTMP paddd XMMTMP, MSG3 sha256msg2 MSG2, MSG3 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG2, MSG1 @@ -201,7 +217,7 @@ sha256_process_block64_shaNI: palignr $4, MSG2, XMMTMP paddd XMMTMP, MSG0 sha256msg2 MSG3, MSG0 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG3, MSG2 @@ -213,7 +229,7 @@ sha256_process_block64_shaNI: palignr $4, MSG3, XMMTMP paddd XMMTMP, MSG1 sha256msg2 MSG0, MSG1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 sha256msg1 MSG0, MSG3 @@ -225,7 +241,7 @@ sha256_process_block64_shaNI: palignr $4, MSG0, XMMTMP paddd XMMTMP, MSG2 sha256msg2 MSG1, MSG2 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Rounds 56-59 */ @@ -236,14 +252,14 @@ sha256_process_block64_shaNI: palignr $4, MSG1, XMMTMP paddd XMMTMP, MSG3 sha256msg2 MSG2, MSG3 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Rounds 60-63 */ mova128 MSG3, MSG paddd 15*16-8*16(SHA256CONSTANTS), MSG sha256rnds2 MSG, STATE0, STATE1 - shuf128_32 $0x0E, MSG, MSG + MOVE_UPPER64_DOWN(MSG) sha256rnds2 MSG, STATE1, STATE0 /* Add current hash values with previously saved */ From e022ff88d14fe916d74fd1311b657dedfec59425 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Tue, 22 Jul 2025 16:43:22 +0200 Subject: [PATCH 159/277] libbb: SHA-NI - document that unpckhpd also works, no code changes Signed-off-by: Denys Vlasenko --- libbb/hash_sha256_hwaccel_x86-32.S | 2 ++ libbb/hash_sha256_hwaccel_x86-64.S | 2 ++ 2 files changed, 4 insertions(+) diff --git a/libbb/hash_sha256_hwaccel_x86-32.S b/libbb/hash_sha256_hwaccel_x86-32.S index 6362ae3823..8d84055e83 100644 --- a/libbb/hash_sha256_hwaccel_x86-32.S +++ b/libbb/hash_sha256_hwaccel_x86-32.S @@ -66,11 +66,13 @@ sha256_process_block64_shaNI: // movhlps MSG, MSG // abcd -> cdcd (3 bytes of code) // shuf128_32 SHUF(2,3,n,n), MSG, MSG // abcd -> cdXX (4 bytes) // punpckhqdq MSG, MSG // abcd -> cdcd (4 bytes) +// unpckhpd MSG, MSG // abcd -> cdcd (4 bytes) // psrldq $8, MSG // abcd -> cd00 (5 bytes) // palignr $8, MSG, MSG // abcd -> cdab (6 bytes, SSSE3 insn) #define MOVE_UPPER64_DOWN(reg) movhlps reg, reg //#define MOVE_UPPER64_DOWN(reg) shuf128_32 SHUF(2,3,0,0), reg, reg //#define MOVE_UPPER64_DOWN(reg) punpckhqdq reg, reg +//#define MOVE_UPPER64_DOWN(reg) unpckhpd reg, reg //#define MOVE_UPPER64_DOWN(reg) psrldq $8, reg //#define MOVE_UPPER64_DOWN(reg) palignr $8, reg, reg diff --git a/libbb/hash_sha256_hwaccel_x86-64.S b/libbb/hash_sha256_hwaccel_x86-64.S index 92f00ebcda..ee3abbd1fb 100644 --- a/libbb/hash_sha256_hwaccel_x86-64.S +++ b/libbb/hash_sha256_hwaccel_x86-64.S @@ -73,11 +73,13 @@ sha256_process_block64_shaNI: // movhlps MSG, MSG // abcd -> cdcd (3 bytes of code) // shuf128_32 SHUF(2,3,n,n), MSG, MSG // abcd -> cdXX (4 bytes) // punpckhqdq MSG, MSG // abcd -> cdcd (4 bytes) +// unpckhpd MSG, MSG // abcd -> cdcd (4 bytes) // psrldq $8, MSG // abcd -> cd00 (5 bytes) // palignr $8, MSG, MSG // abcd -> cdab (6 bytes, SSSE3 insn) #define MOVE_UPPER64_DOWN(reg) movhlps reg, reg //#define MOVE_UPPER64_DOWN(reg) shuf128_32 SHUF(2,3,0,0), reg, reg //#define MOVE_UPPER64_DOWN(reg) punpckhqdq reg, reg +//#define MOVE_UPPER64_DOWN(reg) unpckhpd reg, reg //#define MOVE_UPPER64_DOWN(reg) psrldq $8, reg //#define MOVE_UPPER64_DOWN(reg) palignr $8, reg, reg From 628a7b2f4708037056a1432d7b6b1792c67e486a Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 27 Jul 2025 14:24:23 +0200 Subject: [PATCH 160/277] telnetd: improve --help function old new delta packed_usage 34996 35021 +25 Signed-off-by: Denys Vlasenko --- networking/telnetd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/networking/telnetd.c b/networking/telnetd.c index bfeea1400b..a5a7830478 100644 --- a/networking/telnetd.c +++ b/networking/telnetd.c @@ -104,8 +104,8 @@ //usage:#define telnetd_full_usage "\n\n" //usage: "Handle incoming telnet connections" //usage: IF_NOT_FEATURE_TELNETD_STANDALONE(" via inetd") "\n" -//usage: "\n -l LOGIN Exec LOGIN on connect" -//usage: "\n -f ISSUE_FILE Display ISSUE_FILE instead of /etc/issue" +//usage: "\n -l LOGIN Exec LOGIN on connect (default /bin/login)" +//usage: "\n -f ISSUE_FILE Display ISSUE_FILE instead of /etc/issue.net" //usage: "\n -K Close connection as soon as login exits" //usage: "\n (normally wait until all programs close slave pty)" //usage: IF_FEATURE_TELNETD_STANDALONE( From dcbb69e0a5cce9058afc5a96b9862982ae17cb39 Mon Sep 17 00:00:00 2001 From: Andy Knowles Date: Wed, 30 Jul 2025 18:29:48 +0200 Subject: [PATCH 161/277] sha384sum: new applet function old new delta sha512384_end - 198 +198 packed_usage 35021 35134 +113 init384 - 80 +80 sha384_begin - 19 +19 sha384_end - 10 +10 applet_names 2823 2833 +10 md5_sha1_sum_main 501 507 +6 sha3_end 54 59 +5 applet_main 1628 1632 +4 show_usage_if_dash_dash_help 79 72 -7 hash_file 358 344 -14 sha512_end 197 10 -187 ------------------------------------------------------------------------------ (add/remove: 4/0 grow/shrink: 5/3 up/down: 445/-208) Total: 237 bytes Signed-off-by: Andy Knowles Signed-off-by: Denys Vlasenko --- NOFORK_NOEXEC.lst | 1 + coreutils/md5_sha1_sum.c | 58 +++++++++++++++++++++++++++------------ include/libbb.h | 7 ++++- libbb/hash_md5_sha.c | 58 ++++++++++++++++++++++++++++++--------- testsuite/md5sum.tests | 1 + testsuite/sha384sum.tests | 3 ++ 6 files changed, 97 insertions(+), 31 deletions(-) create mode 100755 testsuite/sha384sum.tests diff --git a/NOFORK_NOEXEC.lst b/NOFORK_NOEXEC.lst index 055f9fb248..a000de45b8 100644 --- a/NOFORK_NOEXEC.lst +++ b/NOFORK_NOEXEC.lst @@ -336,6 +336,7 @@ setuidgid - noexec. spawner sha1sum - noexec. runner sha256sum - noexec. runner sha3sum - noexec. runner +sha384sum - noexec. runner sha512sum - noexec. runner showkey - interactive, longterm shred - runner diff --git a/coreutils/md5_sha1_sum.c b/coreutils/md5_sha1_sum.c index 978d328f12..4506aeb56f 100644 --- a/coreutils/md5_sha1_sum.c +++ b/coreutils/md5_sha1_sum.c @@ -23,6 +23,12 @@ //config: help //config: Compute and check SHA256 message digest //config: +//config:config SHA384SUM +//config: bool "sha384sum (7.3 kb)" +//config: default y +//config: help +//config: Compute and check SHA384 message digest +//config: //config:config SHA512SUM //config: bool "sha512sum (7.3 kb)" //config: default y @@ -35,13 +41,13 @@ //config: help //config: Compute and check SHA3 message digest //config: -//config:comment "Common options for md5sum, sha1sum, sha256sum, sha512sum, sha3sum" -//config: depends on MD5SUM || SHA1SUM || SHA256SUM || SHA512SUM || SHA3SUM +//config:comment "Common options for md5sum, sha1sum, sha256sum, ..., sha3sum" +//config: depends on MD5SUM || SHA1SUM || SHA256SUM || SHA384SUM || SHA512SUM || SHA3SUM //config: //config:config FEATURE_MD5_SHA1_SUM_CHECK //config: bool "Enable -c, -s and -w options" //config: default y -//config: depends on MD5SUM || SHA1SUM || SHA256SUM || SHA512SUM || SHA3SUM +//config: depends on MD5SUM || SHA1SUM || SHA256SUM || SHA384SUM || SHA512SUM || SHA3SUM //config: help //config: Enabling the -c options allows files to be checked //config: against pre-calculated hash values. @@ -51,11 +57,13 @@ //applet:IF_SHA1SUM(APPLET_NOEXEC(sha1sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha1sum)) //applet:IF_SHA3SUM(APPLET_NOEXEC(sha3sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha3sum)) //applet:IF_SHA256SUM(APPLET_NOEXEC(sha256sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha256sum)) +//applet:IF_SHA384SUM(APPLET_NOEXEC(sha384sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha384sum)) //applet:IF_SHA512SUM(APPLET_NOEXEC(sha512sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha512sum)) //kbuild:lib-$(CONFIG_MD5SUM) += md5_sha1_sum.o //kbuild:lib-$(CONFIG_SHA1SUM) += md5_sha1_sum.o //kbuild:lib-$(CONFIG_SHA256SUM) += md5_sha1_sum.o +//kbuild:lib-$(CONFIG_SHA384SUM) += md5_sha1_sum.o //kbuild:lib-$(CONFIG_SHA512SUM) += md5_sha1_sum.o //kbuild:lib-$(CONFIG_SHA3SUM) += md5_sha1_sum.o @@ -99,6 +107,16 @@ //usage: "\n -w Warn about improperly formatted checksum lines" //usage: ) //usage: +//usage:#define sha384sum_trivial_usage +//usage: IF_FEATURE_MD5_SHA1_SUM_CHECK("[-c[sw]] ")"[FILE]..." +//usage:#define sha384sum_full_usage "\n\n" +//usage: "Print" IF_FEATURE_MD5_SHA1_SUM_CHECK(" or check") " SHA384 checksums" +//usage: IF_FEATURE_MD5_SHA1_SUM_CHECK( "\n" +//usage: "\n -c Check sums against list in FILEs" +//usage: "\n -s Don't output anything, status code shows success" +//usage: "\n -w Warn about improperly formatted checksum lines" +//usage: ) +//usage: //usage:#define sha512sum_trivial_usage //usage: IF_FEATURE_MD5_SHA1_SUM_CHECK("[-c[sw]] ")"[FILE]..." //usage:#define sha512sum_full_usage "\n\n" @@ -130,11 +148,12 @@ enum { /* 4th letter of applet_name is... */ - HASH_MD5 = 's', /* "md5>sstotal64, init384, sizeof(init384)); + /*ctx->total64[0] = ctx->total64[1] = 0; - already done */ +} +#endif + +#if NEED_SHA512 +/* Initialize structure containing state of computation. + (FIPS 180-2:5.3.4) */ void FAST_FUNC sha512_begin(sha512_ctx_t *ctx) { int i; @@ -1332,7 +1351,7 @@ unsigned FAST_FUNC sha1_end(sha1_ctx_t *ctx, void *resbuf) } #if NEED_SHA512 -unsigned FAST_FUNC sha512_end(sha512_ctx_t *ctx, void *resbuf) +static unsigned FAST_FUNC sha512384_end(sha512_ctx_t *ctx, void *resbuf, unsigned outsize) { unsigned bufpos = ctx->total64[0] & 127; @@ -1363,11 +1382,21 @@ unsigned FAST_FUNC sha512_end(sha512_ctx_t *ctx, void *resbuf) for (i = 0; i < ARRAY_SIZE(ctx->hash); ++i) ctx->hash[i] = SWAP_BE64(ctx->hash[i]); } - memcpy(resbuf, ctx->hash, sizeof(ctx->hash)); - return sizeof(ctx->hash); + memcpy(resbuf, ctx->hash, outsize); + return outsize; +} +unsigned FAST_FUNC sha512_end(sha384_ctx_t *ctx, void *resbuf) +{ + return sha512384_end(ctx, resbuf, SHA512_OUTSIZE); } #endif /* NEED_SHA512 */ +#if ENABLE_SHA384SUM +unsigned FAST_FUNC sha384_end(sha384_ctx_t *ctx, void *resbuf) +{ + return sha512384_end(ctx, resbuf, SHA384_OUTSIZE); +} +#endif /* * The Keccak sponge function, designed by Guido Bertoni, Joan Daemen, @@ -1904,6 +1933,8 @@ void FAST_FUNC sha3_hash(sha3_ctx_t *ctx, const void *buffer, size_t len) unsigned FAST_FUNC sha3_end(sha3_ctx_t *ctx, void *resbuf) { + unsigned hash_len; + /* Padding */ uint8_t *buf = (uint8_t*)ctx->state; /* @@ -1926,6 +1957,7 @@ unsigned FAST_FUNC sha3_end(sha3_ctx_t *ctx, void *resbuf) sha3_process_block72(ctx->state); /* Output */ - memcpy(resbuf, ctx->state, 64); - return 64; + hash_len = (1600/8 - ctx->input_block_bytes) / 2; + memcpy(resbuf, ctx->state, hash_len); + return hash_len; } diff --git a/testsuite/md5sum.tests b/testsuite/md5sum.tests index cca26dc642..634dcd4aa4 100755 --- a/testsuite/md5sum.tests +++ b/testsuite/md5sum.tests @@ -9,6 +9,7 @@ # efe30c482e0b687e0cca0612f42ca29b # d41337e834377140ae7f98460d71d908598ef04f # 8e1d3ed57ebc130f0f72508446559eeae06451ae6d61b1e8ce46370cfb8963c3 +# c01420bb6613d4bd00396a82033e59e81486cbb045ae0dd2b4c3332e581b3ce09fb1946d6e283acec685778ff205d485 # fe413e0f177324d1353893ca0772ceba83fd41512ba63895a0eebb703ef9feac2fb4e92b2cb430b3bda41b46b0cb4ea8307190a5cc795157cfb680a9cd635d0f if ! test "$1"; then diff --git a/testsuite/sha384sum.tests b/testsuite/sha384sum.tests new file mode 100755 index 0000000000..f1449d195b --- /dev/null +++ b/testsuite/sha384sum.tests @@ -0,0 +1,3 @@ +#!/bin/sh + +. ./md5sum.tests sha384sum c01420bb6613d4bd00396a82033e59e81486cbb045ae0dd2b4c3332e581b3ce09fb1946d6e283acec685778ff205d485 From 65deb9f7dd23b2108655e6c7cb597defbce8a7ed Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 30 Jul 2025 19:11:26 +0200 Subject: [PATCH 162/277] sha3sum: add -aBITS tests Signed-off-by: Denys Vlasenko --- testsuite/md5sum.tests | 6 +++--- testsuite/sha3sum.tests | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/testsuite/md5sum.tests b/testsuite/md5sum.tests index 634dcd4aa4..a6d2b7ffb4 100755 --- a/testsuite/md5sum.tests +++ b/testsuite/md5sum.tests @@ -32,9 +32,9 @@ text=`yes "$text" | head -c 9999` result=`( n=0 while test $n -le 999; do - echo "$text" | head -c $n | "$sum" + echo "$text" | head -c $n | $sum n=$(($n+1)) -done | "$sum" +done | $sum )` if test x"$result" != x"$expected -"; then echo "FAIL: $sum (r:$result exp:$expected)" @@ -45,7 +45,7 @@ fi # GNU compat: -c EMPTY must fail (exitcode 1)! >EMPTY -if "$sum" -c EMPTY 2>/dev/null; then +if $sum -c EMPTY 2>/dev/null; then echo "FAIL: $sum -c EMPTY" : $((FAILCOUNT++)) else diff --git a/testsuite/sha3sum.tests b/testsuite/sha3sum.tests index 2cd8e3bf21..6311417351 100755 --- a/testsuite/sha3sum.tests +++ b/testsuite/sha3sum.tests @@ -1,3 +1,7 @@ #!/bin/sh -. ./md5sum.tests sha3sum 11659f09370139f8ef384f4a6260947fafa6e4fcd87a1ef3f35410e9 + (. ./md5sum.tests sha3sum 11659f09370139f8ef384f4a6260947fafa6e4fcd87a1ef3f35410e9) \ +&& (. ./md5sum.tests "sha3sum -a224" 11659f09370139f8ef384f4a6260947fafa6e4fcd87a1ef3f35410e9) \ +&& (. ./md5sum.tests "sha3sum -a256" 6f69c8d36a9a579a943d878dab38c179d2a9dde12b244aa8840002c0f3d5bb73) \ +&& (. ./md5sum.tests "sha3sum -a384" 303913449042257996a869e0378323193b4f58d90eea801b12186a3d65640bd3403d3404c63527424ec43dff842c0cd0) \ +&& (. ./md5sum.tests "sha3sum -a512" e14814dccc2fef967af74eb6710885b35dfe660a362c0609b642404987d24a13dac66ad037e6affa5c42631110231655fcf4c972b1457ac49fb83af8113fc51f) From 427f80873274c9e4d3860a193e6f1e82b0f61997 Mon Sep 17 00:00:00 2001 From: Ladislav Michl Date: Wed, 14 Jun 2023 16:35:06 +0200 Subject: [PATCH 163/277] date: return failure exit code on clock_settime error Signed-off-by: Ladislav Michl Signed-off-by: Denys Vlasenko --- coreutils/date.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreutils/date.c b/coreutils/date.c index 3a89b6caf0..ef482af1b2 100644 --- a/coreutils/date.c +++ b/coreutils/date.c @@ -289,7 +289,7 @@ int date_main(int argc UNUSED_PARAM, char **argv) /* if setting time, set it */ if ((opt & OPT_SET) && clock_settime(CLOCK_REALTIME, &ts) < 0) { - bb_simple_perror_msg("can't set date"); + bb_simple_perror_msg_and_die("can't set date"); } } From 41f677ec5daccaeae616a0bbf508d2a2d54bdac7 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Sat, 31 May 2025 09:56:09 +0800 Subject: [PATCH 164/277] crond: reap orphaned grandchildren to prevent zombie buildup If a cron job launches a background task, e.g. `sh -c "sleep 5 &"`, the shell exits immediately and the `sleep` process is re-parented to PID 1. When BusyBox `crond` itself happens to be PID 1 (common in a minimal container), those orphans become direct children of `crond`. Because `crond` only calls waitpid() for the PIDs it explicitly tracks, these processes remain forever in Z state and the container slowly fills with zombies. Add a small `while (waitpid(-1, NULL, WNOHANG) > 0)` sweep at the end of check_completions() so any stray children are reaped. When `crond` is not PID 1 the loop returns -ECHILD immediately, so behaviour and overhead on a normal system are unchanged. Size impact: +12 bytes on x86-64 (gcc 13.3.0 -Os, static) Signed-off-by: Valentin Lab Signed-off-by: Denys Vlasenko --- miscutils/crond.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/miscutils/crond.c b/miscutils/crond.c index 96131cae4a..b29745576f 100644 --- a/miscutils/crond.c +++ b/miscutils/crond.c @@ -1001,6 +1001,10 @@ static int check_completions(void) /* else: r == 0: "process is still running" */ file->cf_has_running = 1; } + + /* Reap any other children we don't actively track */ + while (waitpid(-1, NULL, WNOHANG) > 0); + //FIXME: if !file->cf_has_running && file->deleted: delete it! //otherwise deleted entries will stay forever, right? num_still_running += file->cf_has_running; From a98b95b715359a8b002d1cb8e1f998a4afa2c73e Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Wed, 30 Jul 2025 20:02:23 +0200 Subject: [PATCH 165/277] *: use safe_waitpid() or wait_any_nohang() where approppriate function old new delta crond_main 1227 1237 +10 init_main 804 794 -10 wait_one 263 252 -11 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 1/2 up/down: 10/-21) Total: -11 bytes Signed-off-by: Denys Vlasenko --- e2fsprogs/fsck.c | 4 +--- init/init.c | 2 +- miscutils/crond.c | 10 +++++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/e2fsprogs/fsck.c b/e2fsprogs/fsck.c index fd4ea737c5..f7e93497dc 100644 --- a/e2fsprogs/fsck.c +++ b/e2fsprogs/fsck.c @@ -423,13 +423,11 @@ static int wait_one(int flags) /* if (G.noexecute) { already returned -1; } */ while (1) { - pid = waitpid(-1, &status, flags); + pid = safe_waitpid(-1, &status, flags); kill_all_if_got_signal(); if (pid == 0) /* flags == WNOHANG and no children exited */ return -1; if (pid < 0) { - if (errno == EINTR) - continue; if (errno == ECHILD) { /* paranoia */ bb_simple_error_msg("wait: no more children"); return -1; diff --git a/init/init.c b/init/init.c index 797e0a0ebc..294be99529 100644 --- a/init/init.c +++ b/init/init.c @@ -1201,7 +1201,7 @@ int init_main(int argc UNUSED_PARAM, char **argv) int status; struct init_action *a; - wpid = waitpid(-1, &status, WNOHANG); + wpid = wait_any_nohang(&status); if (wpid <= 0) break; diff --git a/miscutils/crond.c b/miscutils/crond.c index b29745576f..6a384fdfba 100644 --- a/miscutils/crond.c +++ b/miscutils/crond.c @@ -989,7 +989,7 @@ static int check_completions(void) if (line->cl_pid <= 0) continue; - r = waitpid(line->cl_pid, NULL, WNOHANG); + r = safe_waitpid(line->cl_pid, NULL, WNOHANG); if (r < 0 || r == line->cl_pid) { process_finished_job(file->cf_username, line); if (line->cl_pid == 0) { @@ -1002,8 +1002,12 @@ static int check_completions(void) file->cf_has_running = 1; } - /* Reap any other children we don't actively track */ - while (waitpid(-1, NULL, WNOHANG) > 0); + /* Reap any other children we don't actively track. + * Reportedly, some people run crond as init process! + * Thus, we need to reap orphans, like init does. + */ + while (wait_any_nohang(NULL) > 0) + continue; //FIXME: if !file->cf_has_running && file->deleted: delete it! //otherwise deleted entries will stay forever, right? From 551bfdb97f45f0277a408ec2d44ee18967b98304 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 31 Jul 2025 18:35:11 +0200 Subject: [PATCH 166/277] ls: implement -q, fix -w0, reduce startup time function old new delta ls_main 598 660 +62 ls_longopts - 47 +47 G_isatty - 36 +36 print_name 102 134 +32 display_files 358 374 +16 .rodata 105829 105833 +4 vgetopt32 1330 1317 -13 static.ls_longopts 47 - -47 ------------------------------------------------------------------------------ (add/remove: 2/1 grow/shrink: 4/1 up/down: 197/-60) Total: 137 bytes Signed-off-by: Denys Vlasenko --- coreutils/ls.c | 171 ++++++++++++++++++++++++++++++----------------- libbb/getopt32.c | 13 +++- 2 files changed, 122 insertions(+), 62 deletions(-) diff --git a/coreutils/ls.c b/coreutils/ls.c index cc809b7979..eaccd1a172 100644 --- a/coreutils/ls.c +++ b/coreutils/ls.c @@ -126,6 +126,8 @@ //usage: "\n -F Append indicator (one of */=@|) to names" //usage: ) //usage: "\n -l Long format" +////usage: "\n -g Long format without group column" +////TODO: support -G too ("suppress owner column", GNUism) //usage: "\n -i List inode numbers" //usage: "\n -n List numeric UIDs and GIDs instead of names" //usage: "\n -s List allocated blocks" @@ -159,6 +161,8 @@ //usage: IF_FEATURE_LS_WIDTH( //usage: "\n -w N Format N columns wide" //usage: ) +////usage: "\n -Q Double-quote names" +////usage: "\n -q Replace unprintable chars with '?'" //usage: IF_FEATURE_LS_COLOR( //usage: "\n --color[={always,never,auto}]" //usage: ) @@ -196,27 +200,47 @@ SPLIT_SUBDIR = 2, /* -Cadi1l Std options, busybox always supports */ /* -gnsxA Std options, busybox always supports */ -/* -Q GNU option, busybox always supports */ -/* -k Std option, busybox always supports (by ignoring) */ -/* It means "for -s, show sizes in kbytes" */ -/* Seems to only affect "POSIXLY_CORRECT=1 ls -sk" */ -/* since otherwise -s shows kbytes anyway */ +/* -Q GNU option, busybox always supports: */ +/* -Q, --quote-name */ +/* enclose entry names in double quotes */ /* -LHRctur Std options, busybox optionally supports */ /* -Fp Std options, busybox optionally supports */ /* -SXvhTw GNU options, busybox optionally supports */ /* -T WIDTH Ignored (we don't use tabs on output) */ /* -Z SELinux mandated option, busybox optionally supports */ +/* -q Std option, busybox always supports: */ +/* https://pubs.opengroup.org/onlinepubs/9699919799/utilities/ls.html: */ +/* Force each instance of non-printable filename characters and */ +/* characters to be written as the ('?') */ +/* character. Implementations may provide this option by default */ +/* if the output is to a terminal device. */ +/* -k Std option, busybox always supports (by ignoring) */ +/* It means "for -s, show sizes in kbytes" */ +/* Seems to only affect "POSIXLY_CORRECT=1 ls -sk" */ +/* since otherwise -s shows kbytes anyway */ #define ls_options \ - "Cadi1lgnsxAk" /* 12 opts, total 12 */ \ - IF_FEATURE_LS_FILETYPES("Fp") /* 2, 14 */ \ - IF_FEATURE_LS_RECURSIVE("R") /* 1, 15 */ \ - IF_SELINUX("Z") /* 1, 16 */ \ - "Q" /* 1, 17 */ \ - IF_FEATURE_LS_TIMESTAMPS("ctu") /* 3, 20 */ \ - IF_FEATURE_LS_SORTFILES("SXrv") /* 4, 24 */ \ - IF_FEATURE_LS_FOLLOWLINKS("LH") /* 2, 26 */ \ - IF_FEATURE_HUMAN_READABLE("h") /* 1, 27 */ \ - IF_FEATURE_LS_WIDTH("T:w:") /* 2, 29 */ + "Cadi1lgnsxA" /* 11 opts, total 11 */ \ + IF_FEATURE_LS_FILETYPES("Fp") /* 2, 13 */ \ + IF_FEATURE_LS_RECURSIVE("R") /* 1, 14 */ \ + IF_SELINUX("Z") /* 1, 15 */ \ + "Q" /* 1, 16 */ \ + IF_FEATURE_LS_TIMESTAMPS("ctu") /* 3, 19 */ \ + IF_FEATURE_LS_SORTFILES("SXrv") /* 4, 23 */ \ + IF_FEATURE_LS_FOLLOWLINKS("LH") /* 2, 25 */ \ + IF_FEATURE_HUMAN_READABLE("h") /* 1, 26 */ \ + IF_FEATURE_LS_WIDTH("T:w:") /* 2, 28 */ \ + IF_LONG_OPTS("\xff") /* 1, 29 */ \ + IF_LONG_OPTS("\xfe") /* 1, 30 */ \ + IF_LONG_OPTS("\xfd") /* 1, 31 */ \ + "qk" /* 2, 33 */ + +#if ENABLE_LONG_OPTS +static const char ls_longopts[] ALIGN1 = + "full-time\0" No_argument "\xff" + "group-directories-first\0" No_argument "\xfe" + IF_FEATURE_LS_COLOR("color\0" Optional_argument "\xfd") +; +#endif enum { OPT_C = (1 << 0), @@ -230,29 +254,31 @@ enum { OPT_s = (1 << 8), OPT_x = (1 << 9), OPT_A = (1 << 10), - //OPT_k = (1 << 11), - OPTBIT_F = 12, - OPTBIT_p, /* 13 */ + OPTBIT_F = 11, + OPTBIT_p, /* 12 */ OPTBIT_R = OPTBIT_F + 2 * ENABLE_FEATURE_LS_FILETYPES, OPTBIT_Z = OPTBIT_R + 1 * ENABLE_FEATURE_LS_RECURSIVE, OPTBIT_Q = OPTBIT_Z + 1 * ENABLE_SELINUX, - OPTBIT_c, /* 17 */ - OPTBIT_t, /* 18 */ - OPTBIT_u, /* 19 */ + OPTBIT_c, /* 16 */ + OPTBIT_t, /* 17 */ + OPTBIT_u, /* 18 */ OPTBIT_S = OPTBIT_c + 3 * ENABLE_FEATURE_LS_TIMESTAMPS, - OPTBIT_X, /* 21 */ - OPTBIT_r, /* 22 */ - OPTBIT_v, /* 23 */ + OPTBIT_X, /* 20 */ + OPTBIT_r, /* 21 */ + OPTBIT_v, /* 22 */ OPTBIT_L = OPTBIT_S + 4 * ENABLE_FEATURE_LS_SORTFILES, - OPTBIT_H, /* 25 */ + OPTBIT_H, /* 24 */ OPTBIT_h = OPTBIT_L + 2 * ENABLE_FEATURE_LS_FOLLOWLINKS, OPTBIT_T = OPTBIT_h + 1 * ENABLE_FEATURE_HUMAN_READABLE, - OPTBIT_w, /* 28 */ + OPTBIT_w, /* 27 */ OPTBIT_full_time = OPTBIT_T + 2 * ENABLE_FEATURE_LS_WIDTH, OPTBIT_dirs_first, - OPTBIT_color, /* 31 */ - /* with long opts, we use all 32 bits */ + OPTBIT_color, /* 30 */ + OPTBIT_q = OPTBIT_color + 1, /* 31 */ + OPTBIT_k = OPTBIT_q + 1, /* 32 */ + /* with all options enabled, we use all 32 bits and even one extra bit! */ + /* this works because -k is ignored, and getopt32 allows such "ignore" options past 31th bit */ OPT_F = (1 << OPTBIT_F) * ENABLE_FEATURE_LS_FILETYPES, OPT_p = (1 << OPTBIT_p) * ENABLE_FEATURE_LS_FILETYPES, @@ -274,6 +300,8 @@ enum { OPT_full_time = (1 << OPTBIT_full_time ) * ENABLE_LONG_OPTS, OPT_dirs_first = (1 << OPTBIT_dirs_first) * ENABLE_LONG_OPTS, OPT_color = (1 << OPTBIT_color ) * ENABLE_FEATURE_LS_COLOR, + OPT_q = (1 << OPTBIT_q), + //-k is ignored: OPT_k = (1 << OPTBIT_k), }; /* @@ -327,6 +355,7 @@ struct globals { #endif smallint exit_code; smallint show_dirname; + smallint tty_out; #if ENABLE_FEATURE_LS_WIDTH unsigned terminal_width; # define G_terminal_width (G.terminal_width) @@ -343,16 +372,21 @@ struct globals { setup_common_bufsiz(); \ /* we have to zero it out because of NOEXEC */ \ memset(&G, 0, sizeof(G)); \ - IF_FEATURE_LS_WIDTH(G_terminal_width = TERMINAL_WIDTH;) \ + IF_FEATURE_LS_WIDTH(G_terminal_width = ~0U;) \ IF_FEATURE_LS_TIMESTAMPS(time(&G.current_time_t);) \ } while (0) #define ESC "\033" +static int G_isatty(void) +{ + if (!G.tty_out) /* not known yet? */ + G.tty_out = isatty(STDOUT_FILENO) + 1; + return (G.tty_out == 2); +} /*** Output code ***/ - /* FYI type values: 1:fifo 2:char 4:dir 6:blk 8:file 10:link 12:socket * (various wacky OSes: 13:Sun door 14:BSD whiteout 5:XENIX named file * 3/7:multiplexed char/block device) @@ -420,6 +454,9 @@ static unsigned calc_name_len(const char *name) unsigned len; uni_stat_t uni_stat; + if (!(option_mask32 & OPT_q)) + return strlen(name); + // TODO: quote tab as \t, etc, if -Q name = printable_string2(&uni_stat, name); @@ -449,6 +486,11 @@ static unsigned print_name(const char *name) unsigned len; uni_stat_t uni_stat; + if (!(option_mask32 & OPT_q)) { + fputs_stdout(name); + return strlen(name); + } + // TODO: quote tab as \t, etc, if -Q name = printable_string2(&uni_stat, name); @@ -646,7 +688,7 @@ static void display_files(struct dnode **dn, unsigned nfiles) unsigned i, ncols, nrows, row, nc; unsigned column; unsigned nexttab; - unsigned column_width = 0; /* used only by coulmnal output */ + unsigned column_width = 0; /* used only by columnar output */ if (option_mask32 & (OPT_l|OPT_1)) { ncols = 1; @@ -691,6 +733,11 @@ static void display_files(struct dnode **dn, unsigned nfiles) } nexttab = column + column_width; column += display_single(dn[i]); + } else { + /* if -w999999999, ncols can be very large */ + //bb_error_msg(" col:%u ncol:%u i:%i", nc, ncols, i); sleep1(); + /* without "break", we loop millions of times here */ + break; } } putchar('\n'); @@ -1090,25 +1137,11 @@ int ls_main(int argc UNUSED_PARAM, char **argv) /* need to initialize since --color has _an optional_ argument */ const char *color_opt = color_str; /* "always" */ #endif -#if ENABLE_LONG_OPTS - static const char ls_longopts[] ALIGN1 = - "full-time\0" No_argument "\xff" - "group-directories-first\0" No_argument "\xfe" - IF_FEATURE_LS_COLOR("color\0" Optional_argument "\xfd") - ; -#endif INIT_G(); init_unicode(); -#if ENABLE_FEATURE_LS_WIDTH - /* obtain the terminal width */ - G_terminal_width = get_terminal_width(STDIN_FILENO); - /* go one less... */ - G_terminal_width--; -#endif - /* process options */ opt = getopt32long(argv, "^" ls_options @@ -1144,6 +1177,17 @@ int ls_main(int argc UNUSED_PARAM, char **argv) exit(0); #endif +#if ENABLE_FEATURE_LS_WIDTH + if ((int)G_terminal_width < 0) { + /* obtain the terminal width */ + G_terminal_width = get_terminal_width(STDIN_FILENO); + /* go one less... */ + G_terminal_width--; + } + if (G_terminal_width == 0) /* -w0 */ + G_terminal_width = INT_MAX; /* "infinite" */ +#endif + #if ENABLE_SELINUX if (opt & OPT_Z) { if (!is_selinux_enabled()) @@ -1157,7 +1201,7 @@ int ls_main(int argc UNUSED_PARAM, char **argv) char *p = getenv("LS_COLORS"); /* LS_COLORS is unset, or (not empty && not "none") ? */ if (!p || (p[0] && strcmp(p, "none") != 0)) { - if (isatty(STDOUT_FILENO)) { + if (G_isatty()) { /* check isatty() last because it's expensive (syscall) */ G_show_color = 1; } @@ -1166,15 +1210,19 @@ int ls_main(int argc UNUSED_PARAM, char **argv) if (opt & OPT_color) { if (color_opt[0] == 'n') G_show_color = 0; - else switch (index_in_substrings(color_str, color_opt)) { - case 3: - case 4: - case 5: - if (!is_TERM_dumb() && isatty(STDOUT_FILENO)) { - case 0: - case 1: - case 2: - G_show_color = 1; + else if (!G_show_color) { + /* if() is not needed, but avoids extra isatty() if G_show_color is already set */ + /* Check --color=COLOR_OPT and maybe set show_color=1 */ + switch (index_in_substrings(color_str, color_opt)) { + case 3: // auto + case 4: // tty + case 5: // if-tty + if (!is_TERM_dumb() && G_isatty()) { + case 0: // always + case 1: // yes + case 2: // force + G_show_color = 1; + } } } } @@ -1182,7 +1230,7 @@ int ls_main(int argc UNUSED_PARAM, char **argv) /* sort out which command line options take precedence */ if (ENABLE_FEATURE_LS_RECURSIVE && (opt & OPT_d)) - option_mask32 &= ~OPT_R; /* no recurse if listing only dir */ + opt = option_mask32 &= ~OPT_R; /* no recurse if listing only dir */ if (!(opt & OPT_l)) { /* not -l? */ if (ENABLE_FEATURE_LS_TIMESTAMPS && ENABLE_FEATURE_LS_SORTFILES) { /* when to sort by time? -t[cu] sorts by time even with -l */ @@ -1190,18 +1238,21 @@ int ls_main(int argc UNUSED_PARAM, char **argv) /* without -l, bare -c or -u enable sort too */ /* (with -l, bare -c or -u just select which time to show) */ if (opt & (OPT_c|OPT_u)) { - option_mask32 |= OPT_t; + opt = option_mask32 |= OPT_t; } } } /* choose a display format if one was not already specified by an option */ - if (!(option_mask32 & (OPT_l|OPT_1|OPT_x|OPT_C))) - option_mask32 |= (isatty(STDOUT_FILENO) ? OPT_C : OPT_1); + if (!(opt & (OPT_l|OPT_1|OPT_x|OPT_C))) + opt = option_mask32 |= (G_isatty() ? OPT_C : OPT_1); + + if (!(opt & OPT_q) && G_isatty()) + opt = option_mask32 |= OPT_q; if (ENABLE_FTPD && applet_name[0] == 'f') { /* ftpd secret backdoor. dirs first are much nicer */ - option_mask32 |= OPT_dirs_first; + opt = option_mask32 |= OPT_dirs_first; } argv += optind; diff --git a/libbb/getopt32.c b/libbb/getopt32.c index b5efa19ac5..4c05dcb97f 100644 --- a/libbb/getopt32.c +++ b/libbb/getopt32.c @@ -530,6 +530,7 @@ vgetopt32(char **argv, const char *applet_opts, const char *applet_long_options, * "fake" short options, like this one: * wget $'-\203' "Test: test" http://kernel.org/ * (supposed to act as --header, but doesn't) */ + next_opt: #if ENABLE_LONG_OPTS while ((c = getopt_long(argc, argv, applet_opts, long_options, NULL)) != -1) { @@ -544,8 +545,16 @@ vgetopt32(char **argv, const char *applet_opts, const char *applet_long_options, * but we construct long opts so that flag * is always NULL (see above) */ if (on_off->opt_char == '\0' /* && c != '\0' */) { - /* c is probably '?' - "bad option" */ - goto error; + /* We reached the end of complementary[] and did not find -c */ + if (c == '?') /* getopt says: "bad option, or option has no required argument" */ + goto error; + /* if there were options beyond 32 bits (example: ls), + * they got no complementary[] slot, and no result bit. + * IOW: they must be "accept but ignore" options. + * For them, we end up here. + */ + //bb_error_msg("ignored option '%c', skipping", c); + goto next_opt; } } if (flags & on_off->incongruously) From bb18473216253b8602ce081dd944f854aad9e572 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 31 Jul 2025 19:24:08 +0200 Subject: [PATCH 167/277] ls: lack of -q should not suppress -Q function old new delta print_name 134 137 +3 display_files 374 375 +1 ------------------------------------------------------------------------------ (add/remove: 0/0 grow/shrink: 2/0 up/down: 4/0) Total: 4 bytes Signed-off-by: Denys Vlasenko --- coreutils/ls.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/coreutils/ls.c b/coreutils/ls.c index eaccd1a172..c725be92d5 100644 --- a/coreutils/ls.c +++ b/coreutils/ls.c @@ -454,16 +454,17 @@ static unsigned calc_name_len(const char *name) unsigned len; uni_stat_t uni_stat; - if (!(option_mask32 & OPT_q)) + if (!(option_mask32 & (OPT_q|OPT_Q))) return strlen(name); - // TODO: quote tab as \t, etc, if -Q name = printable_string2(&uni_stat, name); if (!(option_mask32 & OPT_Q)) { return uni_stat.unicode_width; } + // TODO: quote chars 7..13 as \a,b,t,n,v,f,r + // other chars <32 or >127 as \ooo octal len = 2 + uni_stat.unicode_width; while (*name) { if (*name == '"' || *name == '\\') { @@ -486,12 +487,11 @@ static unsigned print_name(const char *name) unsigned len; uni_stat_t uni_stat; - if (!(option_mask32 & OPT_q)) { + if (!(option_mask32 & (OPT_q|OPT_Q))) { fputs_stdout(name); return strlen(name); } - // TODO: quote tab as \t, etc, if -Q name = printable_string2(&uni_stat, name); if (!(option_mask32 & OPT_Q)) { @@ -499,6 +499,8 @@ static unsigned print_name(const char *name) return uni_stat.unicode_width; } + // TODO: quote chars 7..13 as \a,b,t,n,v,f,r + // other chars <32 or >127 as \ooo octal len = 2 + uni_stat.unicode_width; putchar('"'); while (*name) { From 4f3a56dc12a2126bdacff73f1cfe586d06e800c0 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 1 Aug 2025 01:02:43 +0200 Subject: [PATCH 168/277] ls: fix -Q to match GNU function old new delta print_name 137 229 +92 display_files 375 402 +27 c_escape_conv_str00 - 24 +24 display 1476 1485 +9 conv_str 33 - -33 ------------------------------------------------------------------------------ (add/remove: 2/1 grow/shrink: 3/0 up/down: 152/-33) Total: 119 bytes Signed-off-by: Denys Vlasenko --- coreutils/ls.c | 50 +++++++++++++++++++++++++++++++---------- include/libbb.h | 3 +++ libbb/c_escape.c | 20 +++++++++++++++++ libbb/dump.c | 39 ++++++++++++++++---------------- testsuite/hexdump.tests | 10 +++++++++ testsuite/ls.tests | 26 +++++++++++++++++++-- 6 files changed, 114 insertions(+), 34 deletions(-) create mode 100644 libbb/c_escape.c diff --git a/coreutils/ls.c b/coreutils/ls.c index c725be92d5..9e4b830322 100644 --- a/coreutils/ls.c +++ b/coreutils/ls.c @@ -457,19 +457,29 @@ static unsigned calc_name_len(const char *name) if (!(option_mask32 & (OPT_q|OPT_Q))) return strlen(name); - name = printable_string2(&uni_stat, name); - if (!(option_mask32 & OPT_Q)) { + printable_string2(&uni_stat, name); return uni_stat.unicode_width; } - // TODO: quote chars 7..13 as \a,b,t,n,v,f,r - // other chars <32 or >127 as \ooo octal - len = 2 + uni_stat.unicode_width; + len = 2 + strlen(name); while (*name) { + unsigned char ch = (unsigned char)*name; + if (ch < ' ' || ch > 0x7e) { + ch -= 7; + if ((signed char)ch >= 0 && ch <= 6) { + // quote chars 7..13 as \a,b,t,n,v,f,r + len++; + goto next; + } + // other chars <32 or >126 as \ooo octal + len += 3; + goto next; + } if (*name == '"' || *name == '\\') { len++; } + next: name++; } return len; @@ -492,23 +502,39 @@ static unsigned print_name(const char *name) return strlen(name); } - name = printable_string2(&uni_stat, name); - if (!(option_mask32 & OPT_Q)) { + name = printable_string2(&uni_stat, name); fputs_stdout(name); return uni_stat.unicode_width; } - // TODO: quote chars 7..13 as \a,b,t,n,v,f,r - // other chars <32 or >127 as \ooo octal - len = 2 + uni_stat.unicode_width; + len = 2 + strlen(name); putchar('"'); while (*name) { - if (*name == '"' || *name == '\\') { + unsigned char ch = (unsigned char)*name; + if (ch < ' ' || ch > 0x7e) { + putchar('\\'); + ch -= 7; + if ((signed char)ch >= 0 && ch <= 6) { + // quote chars 7..13 as \a,b,t,n,v,f,r + ch = c_escape_conv_str07[1 + 3 * ch]; + len++; + goto put_ch; + } + // other chars <32 or >126 as \ooo octal + ch = (unsigned char)*name; + putchar('0' + ((ch>>6) & 7)); + putchar('0' + ((ch>>3) & 7)); + ch = '0' + (ch & 7); + len += 3; + goto put_ch; + } + if (ch == '"' || ch == '\\') { putchar('\\'); len++; } - putchar(*name); + put_ch: + putchar(ch); name++; } putchar('"'); diff --git a/include/libbb.h b/include/libbb.h index 7105bd4797..cdc05049ce 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1113,6 +1113,9 @@ char *bin2hex(char *dst, const char *src, int count) FAST_FUNC; /* Reverse */ char* hex2bin(char *dst, const char *src, int count) FAST_FUNC; +extern const char c_escape_conv_str00[]; +#define c_escape_conv_str07 (c_escape_conv_str00+3) + void FAST_FUNC xorbuf_3(void *dst, const void *src1, const void *src2, unsigned count); void FAST_FUNC xorbuf(void* buf, const void* mask, unsigned count); void FAST_FUNC xorbuf16_aligned_long(void* buf, const void* mask); diff --git a/libbb/c_escape.c b/libbb/c_escape.c new file mode 100644 index 0000000000..6c109f2e01 --- /dev/null +++ b/libbb/c_escape.c @@ -0,0 +1,20 @@ +/* vi: set sw=4 ts=4: */ +/* + * Copyright (C) 2025 by Denys Vlasenko + * + * Licensed under GPLv2, see file LICENSE in this source tree. + */ +//kbuild:lib-y += c_escape.o + +#include "libbb.h" + +const char c_escape_conv_str00[] ALIGN1 = + "\\""0""\0" // [0]:00 + "\\""a""\0" // [1]:07 + "\\""b""\0" // [2]:08 + "\\""t""\0" // [3]:09 + "\\""n""\0" // [4]:0a + "\\""v""\0" // [5]:0b + "\\""f""\0" // [6]:0c + "\\""r" // [7]:0d + ; diff --git a/libbb/dump.c b/libbb/dump.c index ac5d47d9ea..0cc7775d60 100644 --- a/libbb/dump.c +++ b/libbb/dump.c @@ -478,37 +478,36 @@ static void bpad(PR *pr) continue; } -static const char conv_str[] ALIGN1 = - "\0" "\\""0""\0" - "\007""\\""a""\0" - "\b" "\\""b""\0" - "\f" "\\""f""\0" - "\n" "\\""n""\0" - "\r" "\\""r""\0" - "\t" "\\""t""\0" - "\v" "\\""v""\0" - ; - static void conv_c(PR *pr, unsigned char *p) { - const char *str = conv_str; - - do { - if (*p == *str) { - ++str; - goto strpr; /* map e.g. '\n' to "\\n" */ - } - str += 4; - } while (*str); + const char *str; + unsigned char ch; + + ch = *p; + if (ch == 0 || (ch -= 6, (signed char)ch > 0 && ch <= 7)) { + /* map chars 0,7..13 to "\0","\{a,b,t,n,v,f,r}" */ + str = c_escape_conv_str00 + 3 * ch; + goto strpr; + } if (isprint_asciionly(*p)) { *pr->cchar = 'c'; printf(pr->fmt, *p); } else { +#if 1 char buf[4]; /* gcc-8.0.1 needs lots of casts to shut up */ sprintf(buf, "%03o", (unsigned)(uint8_t)*p); str = buf; +#else // use faster version? +20 bytes of code + char buf[4]; + buf[3] = '\0'; + ch = *p; + buf[2] = '0' + (ch & 7); ch >>= 3; + buf[1] = '0' + (ch & 7); ch >>= 3; + buf[0] = '0' + ch; + str = buf; +#endif strpr: *pr->cchar = 's'; printf(pr->fmt, str); diff --git a/testsuite/hexdump.tests b/testsuite/hexdump.tests index b2f6a2201c..d2c0a5dc85 100755 --- a/testsuite/hexdump.tests +++ b/testsuite/hexdump.tests @@ -56,6 +56,16 @@ testing "hexdump -e %3_u" \ " \ "" "$input" +testing "hexdump -e %3_c" \ + "hexdump -e '16/1 \" %3_c\" \"\n\"'" \ +' \\0 001 002 003 004 005 006 \\a \\b \\t \\n \\v \\f \\r 016 017 + 020 021 022 023 024 025 026 027 030 031 032 033 034 035 036 037 + p q r s t u v w x y z { | } ~ 177 + 200 201 202 203 204 205 206 207 210 211 212 213 214 215 216 217 + 360 361 362 363 364 365 366 367 370 371 372 373 374 375 376 377 +' \ + "" "$input" + testing "hexdump -e /1 %d" \ "hexdump -e '16/1 \" %4d\" \"\n\"'" \ "\ diff --git a/testsuite/ls.tests b/testsuite/ls.tests index 9309d366be..a959110344 100755 --- a/testsuite/ls.tests +++ b/testsuite/ls.tests @@ -19,7 +19,7 @@ test x"$CONFIG_UNICODE_SUPPORT" = x"y" \ && test x"$CONFIG_LAST_SUPPORTED_WCHAR" = x"767" \ && test x"$CONFIG_FEATURE_LS_SORTFILES" = x"y" \ && testing "ls unicode test with codepoints limited to 767" \ -"(cd ls.testdir && sh ../ls.mk_uni_tests) && ls -1 ls.testdir" \ +"(cd ls.testdir && sh ../ls.mk_uni_tests) && ls -1q ls.testdir" \ '0001_1__Some_correct_UTF-8_text___________________________________________| 0002_2__Boundary_condition_test_cases_____________________________________| 0003_2.1__First_possible_sequence_of_a_certain_length_____________________| @@ -138,7 +138,7 @@ test x"$CONFIG_UNICODE_SUPPORT" = x"y" \ && test x"$CONFIG_SUBST_WCHAR" = x"63" \ && test x"$CONFIG_LAST_SUPPORTED_WCHAR" = x"0" \ && testing "ls unicode test with unlimited codepoints" \ -"(cd ls.testdir && sh ../ls.mk_uni_tests) && ls -1 ls.testdir" \ +"(cd ls.testdir && sh ../ls.mk_uni_tests) && ls -1q ls.testdir" \ '0001_1__Some_correct_UTF-8_text___________________________________________| 0002_2__Boundary_condition_test_cases_____________________________________| 0003_2.1__First_possible_sequence_of_a_certain_length_____________________| @@ -262,6 +262,28 @@ test x"$CONFIG_FEATURE_LS_SORTFILES" = x"y" \ "A\nB\nA\nB\nA\nB\n" \ "" "" +rm -rf ls.testdir 2>/dev/null +mkdir ls.testdir || exit 1 +touch "`printf "ls.testdir/\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f_\x7f\x80\xfe\xff_\x22_\x27_\x5c"`" + +sq="'" + +# testing "test name" "command" "expected result" "file input" "stdin" +testing "ls -q" \ +'ls -q ls.testdir' \ +'???????????????????????????????_????_"_'$sq'_\\''\n' \ +"" "" + +testing "ls -Q" \ +'ls -Q ls.testdir' \ +'"\\001\\002\\003\\004\\005\\006\\a\\b\\t\\n\\v\\f\\r\\016\\017\\020\\021\\022\\023\\024\\025\\026\\027\\030\\031\\032\\033\\034\\035\\036\\037_\\177\\200\\376\\377_\\"_'$sq'_\\\\"\n' \ +"" "" + +testing "ls -qQ" \ +'ls -qQ ls.testdir' \ +'"\\001\\002\\003\\004\\005\\006\\a\\b\\t\\n\\v\\f\\r\\016\\017\\020\\021\\022\\023\\024\\025\\026\\027\\030\\031\\032\\033\\034\\035\\036\\037_\\177\\200\\376\\377_\\"_'$sq'_\\\\"\n' \ +"" "" + # Clean up rm -rf ls.testdir 2>/dev/null From e95086b2fadcddd77723a8fea77f8319eb5dbf69 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 1 Aug 2025 02:21:38 +0200 Subject: [PATCH 169/277] libbb/yescrypt: code shrink function old new delta static.PBKDF2_SHA256 219 165 -54 Signed-off-by: Denys Vlasenko --- libbb/yescrypt/alg-sha256.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/libbb/yescrypt/alg-sha256.c b/libbb/yescrypt/alg-sha256.c index 20e8d1ee47..dc748c9681 100644 --- a/libbb/yescrypt/alg-sha256.c +++ b/libbb/yescrypt/alg-sha256.c @@ -47,9 +47,12 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, /* Iterate through the blocks. */ for (i = 0; dkLen != 0; ) { - uint64_t U[32 / 8]; - uint64_t T[32 / 8]; - uint64_t j; + long U[32 / sizeof(long)]; + long T[32 / sizeof(long)]; +// Do not make these ^^ uint64_t[]. Keep them long[]. +// Even though the XORing loop below is optimized out, +// gcc is not smart enough to realize that 64-bit alignment of the stack +// is no longer useful, and generates ~50 more bytes of code on i386... uint32_t ivec; size_t clen; int k; @@ -64,13 +67,15 @@ PBKDF2_SHA256(const uint8_t *passwd, size_t passwdlen, //does libbb need a non-vararg version with just one (buf,len)? if (c > 1) { +//in yescrypt, c is always 1, so this if() branch is optimized out + uint64_t j; /* T_i = U_1 ... */ memcpy(U, T, 32); for (j = 2; j <= c; j++) { /* Compute U_j. */ hmac_peek_hash(&Phctx, (void*)U, U, 32, NULL); /* ... xor U_j ... */ - for (k = 0; k < 32 / 8; k++) + for (k = 0; k < 32 / sizeof(long); k++) T[k] ^= U[k]; //TODO: xorbuf32_aligned_long(T, U); } From 0fea54a983dd72f8b725f8aeaf0494c7f86043f7 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 1 Aug 2025 03:01:38 +0200 Subject: [PATCH 170/277] libbb/dump: much faster formatting of %_u on x86 function old new delta display 1485 1481 -4 Signed-off-by: Denys Vlasenko --- include/platform.h | 2 +- libbb/dump.c | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/include/platform.h b/include/platform.h index ea0512f365..e10c5b558a 100644 --- a/include/platform.h +++ b/include/platform.h @@ -187,7 +187,7 @@ #elif defined(BYTE_ORDER) && BYTE_ORDER == LITTLE_ENDIAN # define BB_BIG_ENDIAN 0 # define BB_LITTLE_ENDIAN 1 -#elif defined(__386__) +#elif defined(__i386__) # define BB_BIG_ENDIAN 0 # define BB_LITTLE_ENDIAN 1 #else diff --git a/libbb/dump.c b/libbb/dump.c index 0cc7775d60..d340945760 100644 --- a/libbb/dump.c +++ b/libbb/dump.c @@ -494,12 +494,28 @@ static void conv_c(PR *pr, unsigned char *p) *pr->cchar = 'c'; printf(pr->fmt, *p); } else { -#if 1 +#if defined(__i386__) || defined(__x86_64__) + /* Abuse partial register operations */ + uint32_t buf; + unsigned n = *p; + asm ( //00000000 00000000 00000000 aabbbccc +"\n shll $10,%%eax" //00000000 000000aa bbbccc00 00000000 +"\n shrw $5,%%ax" //00000000 000000aa 00000bbb ccc00000 +"\n shrb $5,%%al" //00000000 000000aa 00000bbb 00000ccc +"\n shll $8,%%eax" //000000aa 00000bbb 00000ccc 00000000 +"\n bswapl %%eax" //00000000 00000ccc 00000bbb 000000aa +"\n addl $0x303030,%%eax" +"\n" : "=a" (n) + : "0" (n) + ); + buf = n; + str = (void*)&buf; +#elif 1 char buf[4]; /* gcc-8.0.1 needs lots of casts to shut up */ sprintf(buf, "%03o", (unsigned)(uint8_t)*p); str = buf; -#else // use faster version? +20 bytes of code +#else // use faster version? +20 bytes of code relative to sprintf() method char buf[4]; buf[3] = '\0'; ch = *p; From 4f43bda9472aeea372f6ab30bbef229b6c5f2c76 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 1 Aug 2025 12:06:02 +0200 Subject: [PATCH 171/277] ls: code shrink Looks like gcc can figure this out by itself, but let's be explicit Signed-off-by: Denys Vlasenko --- coreutils/ls.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/coreutils/ls.c b/coreutils/ls.c index 9e4b830322..200baff031 100644 --- a/coreutils/ls.c +++ b/coreutils/ls.c @@ -449,6 +449,13 @@ static char append_char(mode_t mode) } #endif +/* Return the number of used columns. + * Note that only columnar output uses return value. + * -l and -1 modes don't care. + * coreutils 7.2 also supports: + * ls -b (--escape) = octal escapes (although it doesn't look like working) + * ls -N (--literal) = not escape at all + */ static unsigned calc_name_len(const char *name) { unsigned len; @@ -458,6 +465,7 @@ static unsigned calc_name_len(const char *name) return strlen(name); if (!(option_mask32 & OPT_Q)) { + /* the most likely branch: "ls" to tty (it auto-enables -q behavior) */ printable_string2(&uni_stat, name); return uni_stat.unicode_width; } @@ -467,16 +475,16 @@ static unsigned calc_name_len(const char *name) unsigned char ch = (unsigned char)*name; if (ch < ' ' || ch > 0x7e) { ch -= 7; - if ((signed char)ch >= 0 && ch <= 6) { - // quote chars 7..13 as \a,b,t,n,v,f,r - len++; - goto next; + if (ch <= 6) { + /* quote chars 7..13 as \a,b,t,n,v,f,r */ + goto two; } - // other chars <32 or >126 as \ooo octal + /* other chars <32 or >126 as \ooo octal */ len += 3; goto next; } if (*name == '"' || *name == '\\') { + two: len++; } next: @@ -484,14 +492,6 @@ static unsigned calc_name_len(const char *name) } return len; } - -/* Return the number of used columns. - * Note that only columnar output uses return value. - * -l and -1 modes don't care. - * coreutils 7.2 also supports: - * ls -b (--escape) = octal escapes (although it doesn't look like working) - * ls -N (--literal) = not escape at all - */ static unsigned print_name(const char *name) { unsigned len; @@ -503,6 +503,7 @@ static unsigned print_name(const char *name) } if (!(option_mask32 & OPT_Q)) { + /* the most likely branch: "ls" to tty (it auto-enables -q behavior) */ name = printable_string2(&uni_stat, name); fputs_stdout(name); return uni_stat.unicode_width; @@ -515,15 +516,14 @@ static unsigned print_name(const char *name) if (ch < ' ' || ch > 0x7e) { putchar('\\'); ch -= 7; - if ((signed char)ch >= 0 && ch <= 6) { - // quote chars 7..13 as \a,b,t,n,v,f,r + if (ch <= 6) { + /* quote chars 7..13 as \a,b,t,n,v,f,r */ ch = c_escape_conv_str07[1 + 3 * ch]; - len++; - goto put_ch; + goto two; } - // other chars <32 or >126 as \ooo octal + /* other chars <32 or >126 as \ooo octal */ ch = (unsigned char)*name; - putchar('0' + ((ch>>6) & 7)); + putchar('0' + (ch>>6)); putchar('0' + ((ch>>3) & 7)); ch = '0' + (ch & 7); len += 3; @@ -531,6 +531,7 @@ static unsigned print_name(const char *name) } if (ch == '"' || ch == '\\') { putchar('\\'); + two: len++; } put_ch: From 2d7ff2c909c6dce95a8780d00e2089f0c507dd25 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Fri, 1 Aug 2025 18:16:46 +0200 Subject: [PATCH 172/277] ls: don't do unnecessary tty operations when called from ftpd function old new delta ls_main 660 690 +30 Signed-off-by: Denys Vlasenko --- coreutils/ls.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/coreutils/ls.c b/coreutils/ls.c index 200baff031..c24561576c 100644 --- a/coreutils/ls.c +++ b/coreutils/ls.c @@ -1206,6 +1206,18 @@ int ls_main(int argc UNUSED_PARAM, char **argv) exit(0); #endif + /* ftpd secret backdoor? */ + if (ENABLE_FTPD && applet_name[0] == 'f') { + /* dirs first are much nicer */ + opt = option_mask32 |= OPT_dirs_first; + /* don't show SEcontext */ + IF_SELINUX(opt = option_mask32 &= ~OPT_Z;) + /* do not query stdout about size and tty-ness */ + IF_FEATURE_LS_WIDTH(G_terminal_width = INT_MAX;) + G.tty_out = 1; /* not a tty */ + goto skip_if_ftpd; + } + #if ENABLE_FEATURE_LS_WIDTH if ((int)G_terminal_width < 0) { /* obtain the terminal width */ @@ -1256,6 +1268,7 @@ int ls_main(int argc UNUSED_PARAM, char **argv) } } #endif + skip_if_ftpd: /* sort out which command line options take precedence */ if (ENABLE_FEATURE_LS_RECURSIVE && (opt & OPT_d)) @@ -1279,11 +1292,6 @@ int ls_main(int argc UNUSED_PARAM, char **argv) if (!(opt & OPT_q) && G_isatty()) opt = option_mask32 |= OPT_q; - if (ENABLE_FTPD && applet_name[0] == 'f') { - /* ftpd secret backdoor. dirs first are much nicer */ - opt = option_mask32 |= OPT_dirs_first; - } - argv += optind; if (!argv[0]) *--argv = (char*)"."; From 91d8b4eb5c770dfcc05f74dd0bd7b3fabc236530 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 2 Aug 2025 07:18:56 +0200 Subject: [PATCH 173/277] ftpd: code shrink, move replace_char() to libbb function old new delta modprobe_main 803 804 +1 escape_text 127 122 -5 replace 18 - -18 ------------------------------------------------------------------------------ (add/remove: 0/1 grow/shrink: 1/1 up/down: 1/-23) Total: -22 bytes Signed-off-by: Denys Vlasenko --- include/libbb.h | 13 +++++++++++++ libbb/replace.c | 14 ++++++++++++++ modutils/modprobe-small.c | 21 ++++++--------------- modutils/modprobe.c | 2 +- modutils/modutils.c | 9 --------- modutils/modutils.h | 1 - networking/ftpd.c | 39 ++++++++++++--------------------------- 7 files changed, 46 insertions(+), 53 deletions(-) diff --git a/include/libbb.h b/include/libbb.h index cdc05049ce..8952001922 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -1113,6 +1113,19 @@ char *bin2hex(char *dst, const char *src, int count) FAST_FUNC; /* Reverse */ char* hex2bin(char *dst, const char *src, int count) FAST_FUNC; +/* Returns strlen as a bonus */ +//size_t replace_char(char *s, char what, char with) FAST_FUNC; +static inline size_t replace_char(char *str, char from, char to) +{ + char *p = str; + while (*p) { + if (*p == from) + *p = to; + p++; + } + return p - str; +} + extern const char c_escape_conv_str00[]; #define c_escape_conv_str07 (c_escape_conv_str00+3) diff --git a/libbb/replace.c b/libbb/replace.c index 6183d3e6fa..bc26b04cc9 100644 --- a/libbb/replace.c +++ b/libbb/replace.c @@ -46,3 +46,17 @@ char* FAST_FUNC xmalloc_substitute_string(const char *src, int count, const char //dbg_msg("subst9:'%s'", buf); return buf; } + +#if 0 /* inlined in libbb.h */ +/* Returns strlen as a bonus */ +size_t FAST_FUNC replace_char(char *str, char from, char to) +{ + char *p = str; + while (*p) { + if (*p == from) + *p = to; + p++; + } + return p - str; +} +#endif diff --git a/modutils/modprobe-small.c b/modutils/modprobe-small.c index 77e42e3fb1..31a215a297 100644 --- a/modutils/modprobe-small.c +++ b/modutils/modprobe-small.c @@ -186,15 +186,6 @@ static char* find_keyword(char *ptr, size_t len, const char *word) return NULL; } -static void replace(char *s, char what, char with) -{ - while (*s) { - if (what == *s) - *s = with; - ++s; - } -} - static char *filename2modname(const char *filename, char *modname) { int i; @@ -230,7 +221,7 @@ static char* str_2_list(const char *str) dst[len] = '\0'; memcpy(dst, str, len); //TODO: protect against 2+ spaces: "word word" - replace(dst, ' ', '\0'); + replace_char(dst, ' ', '\0'); return dst; } @@ -369,14 +360,14 @@ static int parse_module(module_info *info, const char *pathname) } bksp(); /* remove last ' ' */ info->aliases = copy_stringbuf(); - replace(info->aliases, '-', '_'); + replace_char(info->aliases, '-', '_'); /* "dependency1 depandency2" */ reset_stringbuf(); ptr = find_keyword(module_image, len, "depends="); if (ptr && *ptr) { - replace(ptr, ',', ' '); - replace(ptr, '-', '_'); + replace_char(ptr, ',', ' '); + replace_char(ptr, '-', '_'); dbg2_error_msg("dep:'%s'", ptr); append(ptr); } @@ -707,7 +698,7 @@ static int process_module(char *name, const char *cmdline_options) dbg1_error_msg("process_module('%s','%s')", name, cmdline_options); - replace(name, '-', '_'); + replace_char(name, '-', '_'); dbg1_error_msg("already_loaded:%d is_remove:%d", already_loaded(name), is_remove); @@ -735,7 +726,7 @@ static int process_module(char *name, const char *cmdline_options) char *opt_filename = xasprintf("/etc/modules/%s", name); options = xmalloc_open_read_close(opt_filename, NULL); if (options) - replace(options, '\n', ' '); + replace_char(options, '\n', ' '); #if ENABLE_FEATURE_CMDLINE_MODULE_OPTIONS if (cmdline_options) { /* NB: cmdline_options always have one leading ' ' diff --git a/modutils/modprobe.c b/modutils/modprobe.c index 543f53e996..f890abe53e 100644 --- a/modutils/modprobe.c +++ b/modutils/modprobe.c @@ -579,7 +579,7 @@ int modprobe_main(int argc UNUSED_PARAM, char **argv) parser_t *p = config_open2(CONFIG_DEFAULT_DEPMOD_FILE, xfopen_for_read); for (i = 0; argv[i]; i++) - replace(argv[i], '-', '_'); + replace_char(argv[i], '-', '_'); while (config_read(p, tokens, 2, 1, "# \t", PARSE_NORMAL)) { colon = last_char_is(tokens[0], ':'); diff --git a/modutils/modutils.c b/modutils/modutils.c index cbff209610..862f71f572 100644 --- a/modutils/modutils.c +++ b/modutils/modutils.c @@ -69,15 +69,6 @@ void FAST_FUNC moddb_free(module_db *db) } } -void FAST_FUNC replace(char *s, char what, char with) -{ - while (*s) { - if (what == *s) - *s = with; - ++s; - } -} - int FAST_FUNC string_to_llist(char *string, llist_t **llist, const char *delim) { char *tok; diff --git a/modutils/modutils.h b/modutils/modutils.h index 4a702e97c0..9b05116d1e 100644 --- a/modutils/modutils.h +++ b/modutils/modutils.h @@ -47,7 +47,6 @@ module_entry *moddb_get(module_db *db, const char *s) FAST_FUNC; module_entry *moddb_get_or_create(module_db *db, const char *s) FAST_FUNC; void moddb_free(module_db *db) FAST_FUNC; -void replace(char *s, char what, char with) FAST_FUNC; int string_to_llist(char *string, llist_t **llist, const char *delim) FAST_FUNC; char *filename2modname(const char *filename, char *modname) FAST_FUNC; #if ENABLE_FEATURE_CMDLINE_MODULE_OPTIONS diff --git a/networking/ftpd.c b/networking/ftpd.c index 0d6a289c7c..c3125410e0 100644 --- a/networking/ftpd.c +++ b/networking/ftpd.c @@ -190,54 +190,39 @@ struct globals { } while (0) +/* escape_text("pfx:", str, (0xff << 8) + '\r') + * Duplicate 0xff, append \r ^^^^^^^^^^^^^^^^^^ + */ static char * escape_text(const char *prepend, const char *str, unsigned escapee) { - unsigned retlen, remainlen, chunklen; - char *ret, *found; + char *ret, *p; char append; append = (char)escapee; escapee >>= 8; - remainlen = strlen(str); - retlen = strlen(prepend); - ret = xmalloc(retlen + remainlen * 2 + 1 + 1); - strcpy(ret, prepend); + ret = xmalloc(strlen(prepend) + strlen(str) * 2 + 1 + 1); + p = stpcpy(ret, prepend); for (;;) { - found = strchrnul(str, escapee); - chunklen = found - str + 1; + char *found = strchrnul(str, escapee); - /* Copy chunk up to and including escapee (or NUL) to ret */ - memcpy(ret + retlen, str, chunklen); - retlen += chunklen; + /* Copy up to and including escapee (or NUL) */ + p = mempcpy(p, str, found - str + 1); if (*found == '\0') { /* It wasn't escapee, it was NUL! */ - ret[retlen - 1] = append; /* replace NUL */ - ret[retlen] = '\0'; /* add NUL */ break; } - ret[retlen++] = escapee; /* duplicate escapee */ str = found + 1; + *p++ = escapee; /* duplicate escapee */ } + p[-1] = append; /* replace NUL */ + *p = '\0'; /* add NUL */ return ret; } -/* Returns strlen as a bonus */ -static unsigned -replace_char(char *str, char from, char to) -{ - char *p = str; - while (*p) { - if (*p == from) - *p = to; - p++; - } - return p - str; -} - static void verbose_log(const char *str) { From 7d782e8813d1c3e7c9a96371ec37aaef1da08da9 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 3 Aug 2025 00:11:11 +0200 Subject: [PATCH 174/277] shell: fix the check of empty HISTFILE The check for "" history file is done on a wrong state pointer. Fixes: shell: empty HISTFILE disables history saving, just as unset one did Signed-off-by: Denys Vlasenko --- libbb/lineedit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbb/lineedit.c b/libbb/lineedit.c index 43d1da35c9..10cc0433bc 100644 --- a/libbb/lineedit.c +++ b/libbb/lineedit.c @@ -1564,7 +1564,7 @@ void FAST_FUNC save_history(line_input_t *st) FILE *fp; /* bash compat: HISTFILE="" disables history saving */ - if (!st || !st->hist_file || !state->hist_file[0]) + if (!st || !st->hist_file || !st->hist_file[0]) return; if (st->cnt_history <= st->cnt_history_in_file) return; /* no new entries were added */ From e591316e020258d07dc987c5f2f4b7a03d786ddd Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 3 Aug 2025 00:44:32 +0200 Subject: [PATCH 175/277] hush: rename hush_exit to save_history_run_exit_trap_and_exit, sigexit to restore_ttypgrp_and_killsig_or__exit Signed-off-by: Denys Vlasenko --- shell/hush.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/shell/hush.c b/shell/hush.c index d1f687f9d5..5707b4b31a 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -1929,7 +1929,7 @@ static void restore_G_args(save_arg_t *sv, char **argv) * "trap - SIGxxx": * if sig is in special_sig_mask, set handler back to: * record_pending_signo, or to IGN if it's a tty stop signal - * if sig is in fatal_sig_mask, set handler back to sigexit. + * if sig is in fatal_sig_mask, set handler back to restore_ttypgrp_and_killsig_or__exit. * else: set handler back to SIG_DFL * "trap 'cmd' SIGxxx": * set handler to record_pending_signo. @@ -2002,7 +2002,7 @@ static sighandler_t install_sighandler(int sig, sighandler_t handler) return old_sa.sa_handler; } -static void hush_exit(int exitcode) NORETURN; +static void save_history_run_exit_trap_and_exit(int exitcode) NORETURN; static void restore_ttypgrp_and__exit(void) NORETURN; static void restore_ttypgrp_and__exit(void) @@ -2010,7 +2010,7 @@ static void restore_ttypgrp_and__exit(void) /* xfunc has failed! die die die */ /* no EXIT traps, this is an escape hatch! */ G.exiting = 1; - hush_exit(xfunc_error_retval); + save_history_run_exit_trap_and_exit(xfunc_error_retval); } #if ENABLE_HUSH_JOB @@ -2046,8 +2046,8 @@ static void fflush_and__exit(void) * or called directly with -EXITCODE. * We also call it if xfunc is exiting. */ -static void sigexit(int sig) NORETURN; -static void sigexit(int sig) +static void restore_ttypgrp_and_killsig_or__exit(int sig) NORETURN; +static void restore_ttypgrp_and_killsig_or__exit(int sig) { /* Careful: we can end up here after [v]fork. Do not restore * tty pgrp then, only top-level shell process does that */ @@ -2081,7 +2081,7 @@ static sighandler_t pick_sighandler(unsigned sig) #if ENABLE_HUSH_JOB /* is sig fatal? */ if (G_fatal_sig_mask & sigmask) - handler = sigexit; + handler = restore_ttypgrp_and_killsig_or__exit; else #endif /* sig has special handling? */ @@ -2102,7 +2102,7 @@ static sighandler_t pick_sighandler(unsigned sig) static const char* FAST_FUNC get_local_var_value(const char *name); /* Restores tty foreground process group, and exits. */ -static void hush_exit(int exitcode) +static void save_history_run_exit_trap_and_exit(int exitcode) { #if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT if (G.line_input_state) { @@ -2155,7 +2155,7 @@ static void hush_exit(int exitcode) fflush_all(); #if ENABLE_HUSH_JOB - sigexit(- (exitcode & 0xff)); + restore_ttypgrp_and_killsig_or__exit(- (exitcode & 0xff)); #else _exit(exitcode); #endif @@ -2240,7 +2240,7 @@ static int check_and_run_traps(void) } } /* this restores tty pgrp, then kills us with SIGHUP */ - sigexit(SIGHUP); + restore_ttypgrp_and_killsig_or__exit(SIGHUP); } #endif #if ENABLE_HUSH_FAST @@ -10144,7 +10144,7 @@ static int run_list(struct pipe *pi) if (rcode != 0 && G.o_opt[OPT_O_ERREXIT]) { debug_printf_exec("ERREXIT:1 errexit_depth:%d\n", G.errexit_depth); if (G.errexit_depth == 0) - hush_exit(rcode); + save_history_run_exit_trap_and_exit(rcode); } G.errexit_depth = sv_errexit_depth; @@ -10905,7 +10905,7 @@ int hush_main(int argc, char **argv) parse_and_run_file(hfopen(NULL)); /* stdin */ final_return: - hush_exit(G.last_exitcode); + save_history_run_exit_trap_and_exit(G.last_exitcode); } /* @@ -11091,19 +11091,19 @@ static int FAST_FUNC builtin_exit(char **argv) * TODO: we can use G.exiting = -1 as indicator "last cmd was exit" */ - /* note: EXIT trap is run by hush_exit */ + /* note: EXIT trap is run by save_history_run_exit_trap_and_exit */ argv = skip_dash_dash(argv); if (argv[0] == NULL) { #if ENABLE_HUSH_TRAP if (G.pre_trap_exitcode >= 0) /* "exit" in trap uses $? from before the trap */ - hush_exit(G.pre_trap_exitcode); + save_history_run_exit_trap_and_exit(G.pre_trap_exitcode); #endif - hush_exit(G.last_exitcode); + save_history_run_exit_trap_and_exit(G.last_exitcode); } /* mimic bash: exit 123abc == exit 255 + error msg */ xfunc_error_retval = 255; /* bash: exit -2 == exit 254, no error msg */ - hush_exit(xatoi(argv[0]) & 0xff); + save_history_run_exit_trap_and_exit(xatoi(argv[0]) & 0xff); } #if ENABLE_HUSH_TYPE From cdbb6957f1bfafbcbe7e9abbb79979c8cd29881c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 3 Aug 2025 01:02:55 +0200 Subject: [PATCH 176/277] hush: don't save history in subshells; on NOMMU, don't be interactive on reexec Signed-off-by: Denys Vlasenko --- shell/hush.c | 126 +++++++++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/shell/hush.c b/shell/hush.c index 5707b4b31a..5f4c0cca59 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -930,6 +930,12 @@ struct globals { # define G_flag_return_in_progress 0 #endif smallint exiting; /* used to prevent EXIT trap recursion */ +#if !BB_MMU + smallint reexeced_on_NOMMU; +# define G_reexeced_on_NOMMU (G.reexeced_on_NOMMU) +#else +# define G_reexeced_on_NOMMU 0 +#endif /* These support $? */ smalluint last_exitcode; smalluint expand_exitcode; @@ -2105,7 +2111,9 @@ static const char* FAST_FUNC get_local_var_value(const char *name); static void save_history_run_exit_trap_and_exit(int exitcode) { #if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT - if (G.line_input_state) { + if (G.line_input_state + && getpid() == G.root_pid /* exits in subshells do not save history */ + ) { const char *hp; # if ENABLE_FEATURE_SH_HISTFILESIZE // in bash: @@ -10218,6 +10226,53 @@ static int run_and_free_list(struct pipe *pi) /* * Initialization and main */ +#if ENABLE_FEATURE_EDITING +static void init_line_editing(void) +{ + G.line_input_state = new_line_input_t(FOR_SHELL); +# if ENABLE_FEATURE_TAB_COMPLETION + G.line_input_state->get_exe_name = hush_command_name; +# endif +# if EDITING_HAS_sh_get_var + G.line_input_state->sh_get_var = get_local_var_value; +# endif +# if ENABLE_HUSH_SAVEHISTORY && MAX_HISTORY > 0 + { + const char *hp = get_local_var_value("HISTFILE"); + if (!hp) { + hp = get_local_var_value("HOME"); + if (hp) { + hp = concat_path_file(hp, ".hush_history"); + /* Make HISTFILE set on exit (else history won't be saved) */ + set_local_var_from_halves("HISTFILE", hp); + } + } else { + hp = xstrdup(hp); + } + if (hp) { + G.line_input_state->hist_file = hp; + } +# if ENABLE_FEATURE_SH_HISTFILESIZE + hp = get_local_var_value("HISTSIZE"); + /* Using HISTFILESIZE above to limit max_history would be WRONG: + * users may set HISTFILESIZE=0 in their profile scripts + * to prevent _saving_ of history files, but still want to have + * non-zero history limit for in-memory list. + */ +// in bash, runtime history size is controlled by HISTSIZE (0=no history), +// HISTFILESIZE controls on-disk history file size (in lines, 0=no history): + G.line_input_state->max_history = size_from_HISTFILESIZE(hp); +// HISTFILESIZE: "The shell sets the default value to the value of HISTSIZE after reading any startup files." +// HISTSIZE: "The shell sets the default value to 500 after reading any startup files." +// (meaning: if the value wasn't set after startup files, the default value is set as described above) +# endif + } +# endif +} +#else +# define init_line_editing() ((void)0) +#endif + static void install_sighandlers(unsigned mask) { sighandler_t old_handler; @@ -10356,7 +10411,6 @@ static int set_mode(int state, char mode, const char *o_opt) int hush_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int hush_main(int argc, char **argv) { - pid_t cached_getpid; enum { OPT_login = (1 << 0), }; @@ -10384,9 +10438,6 @@ int hush_main(int argc, char **argv) _exit(0); } G.argv0_for_re_execing = argv[0]; - if (G.argv0_for_re_execing[0] == '-') - /* reexeced hush should never be a login shell */ - G.argv0_for_re_execing++; #endif #if ENABLE_HUSH_TRAP # if ENABLE_HUSH_FUNCTIONS @@ -10399,9 +10450,8 @@ int hush_main(int argc, char **argv) G.count_SIGCHLD++; /* ensure it is != G.handled_SIGCHLD */ #endif - cached_getpid = getpid(); /* for tcsetpgrp() during init */ - G.root_pid = cached_getpid; /* for $PID (NOMMU can override via -$HEXPID:HEXPPID:...) */ - G.root_ppid = getppid(); /* for $PPID (NOMMU can override) */ + G.root_pid = getpid(); /* for $PID (NOMMU can override via -$HEXPID:HEXPPID:...) */ + G.root_ppid = getppid(); /* for $PPID (NOMMU can override) */ /* Deal with HUSH_VERSION */ debug_printf_env("unsetenv '%s'\n", "HUSH_VERSION"); @@ -10568,6 +10618,7 @@ int hush_main(int argc, char **argv) case '$': { unsigned long long empty_trap_mask; + G.reexeced_on_NOMMU = 1; G.root_pid = bb_strtou(optarg, &optarg, 16); optarg++; G.root_ppid = bb_strtou(optarg, &optarg, 16); @@ -10647,7 +10698,9 @@ int hush_main(int argc, char **argv) G.global_argv[0] = argv[0]; /* If we are login shell... */ - if (flags & OPT_login) { + if (!G_reexeced_on_NOMMU /* reexeced hush should never be a login shell */ + && (flags & OPT_login) + ) { const char *hp = NULL; HFILE *input; @@ -10767,7 +10820,9 @@ int hush_main(int argc, char **argv) * Refer to Posix.2, the description of the 'sh' utility. */ #if ENABLE_HUSH_JOB - if (isatty(STDIN_FILENO) && isatty(STDOUT_FILENO)) { + if (!G_reexeced_on_NOMMU + && isatty(STDIN_FILENO) && isatty(STDOUT_FILENO) + ) { G_saved_tty_pgrp = tcgetpgrp(STDIN_FILENO); debug_printf("saved_tty_pgrp:%d\n", G_saved_tty_pgrp); if (G_saved_tty_pgrp < 0) @@ -10813,57 +10868,18 @@ int hush_main(int argc, char **argv) * (bash, too, does this only if ctty is available) */ bb_setpgrp(); /* is the same as setpgid(our_pid, our_pid); */ /* Grab control of the terminal */ - tcsetpgrp(G_interactive_fd, cached_getpid); + tcsetpgrp(G_interactive_fd, G.root_pid); } enable_restore_tty_pgrp_on_exit(); - -# if ENABLE_FEATURE_EDITING - G.line_input_state = new_line_input_t(FOR_SHELL); -# if ENABLE_FEATURE_TAB_COMPLETION - G.line_input_state->get_exe_name = hush_command_name; -# endif -# if EDITING_HAS_sh_get_var - G.line_input_state->sh_get_var = get_local_var_value; -# endif -# endif -# if ENABLE_HUSH_SAVEHISTORY && MAX_HISTORY > 0 - { - const char *hp = get_local_var_value("HISTFILE"); - if (!hp) { - hp = get_local_var_value("HOME"); - if (hp) { - hp = concat_path_file(hp, ".hush_history"); - /* Make HISTFILE set on exit (else history won't be saved) */ - set_local_var_from_halves("HISTFILE", hp); - } - } else { - hp = xstrdup(hp); - } - if (hp) { - G.line_input_state->hist_file = hp; - } -# if ENABLE_FEATURE_SH_HISTFILESIZE - hp = get_local_var_value("HISTSIZE"); - /* Using HISTFILESIZE above to limit max_history would be WRONG: - * users may set HISTFILESIZE=0 in their profile scripts - * to prevent _saving_ of history files, but still want to have - * non-zero history limit for in-memory list. - */ -// in bash, runtime history size is controlled by HISTSIZE (0=no history), -// HISTFILESIZE controls on-disk history file size (in lines, 0=no history): - G.line_input_state->max_history = size_from_HISTFILESIZE(hp); -// HISTFILESIZE: "The shell sets the default value to the value of HISTSIZE after reading any startup files." -// HISTSIZE: "The shell sets the default value to 500 after reading any startup files." -// (meaning: if the value wasn't set after startup files, the default value is set as described above) -# endif - } -# endif + init_line_editing(); } else { install_special_sighandlers(); } #elif ENABLE_HUSH_INTERACTIVE /* No job control compiled in, only prompt/line editing */ - if (isatty(STDIN_FILENO) && isatty(STDOUT_FILENO)) { + if (!G_reexeced_on_NOMMU + && isatty(STDIN_FILENO) && isatty(STDOUT_FILENO) + ) { G_interactive_fd = dup_CLOEXEC(STDIN_FILENO, 254); if (G_interactive_fd < 0) { /* try to dup to any fd */ @@ -10872,6 +10888,8 @@ int hush_main(int argc, char **argv) /* give up */ G_interactive_fd = 0; } + if (G_interactive_fd) + init_line_editing(); } install_special_sighandlers(); #else From c2ef7c6dadb72e0f1a024440d2d24f6e6525af9c Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 3 Aug 2025 09:32:36 +0200 Subject: [PATCH 177/277] hush: if !JOBS, skip tests which wouldn't work Signed-off-by: Denys Vlasenko --- shell/hush_test/hush-misc/sig_exitcode.tests | 3 +++ shell/hush_test/hush-misc/wait1.tests | 3 +++ shell/hush_test/hush-misc/wait2.tests | 3 +++ shell/hush_test/hush-misc/wait3.tests | 3 +++ shell/hush_test/hush-misc/wait4.tests | 3 +++ shell/hush_test/hush-misc/wait5.tests | 3 +++ shell/hush_test/hush-misc/wait6.tests | 3 +++ shell/hush_test/hush-signals/catch.tests | 3 +++ shell/hush_test/hush-signals/signal1.tests | 3 +++ shell/hush_test/hush-signals/signal8.tests | 3 +++ shell/hush_test/hush-signals/signal_read2.tests | 3 +++ shell/hush_test/hush-signals/subshell.tests | 3 +++ shell/hush_test/run-all | 7 +++++-- 13 files changed, 41 insertions(+), 2 deletions(-) diff --git a/shell/hush_test/hush-misc/sig_exitcode.tests b/shell/hush_test/hush-misc/sig_exitcode.tests index 7879dc8547..67b4500f47 100755 --- a/shell/hush_test/hush-misc/sig_exitcode.tests +++ b/shell/hush_test/hush-misc/sig_exitcode.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + exec 2>&1 $THIS_SH -c 'kill -9 $$' diff --git a/shell/hush_test/hush-misc/wait1.tests b/shell/hush_test/hush-misc/wait1.tests index f9cf6d48c5..54120319be 100755 --- a/shell/hush_test/hush-misc/wait1.tests +++ b/shell/hush_test/hush-misc/wait1.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + sleep 2 & sleep 1 & wait $! echo $? jobs diff --git a/shell/hush_test/hush-misc/wait2.tests b/shell/hush_test/hush-misc/wait2.tests index be20f95a52..60f382c9f4 100755 --- a/shell/hush_test/hush-misc/wait2.tests +++ b/shell/hush_test/hush-misc/wait2.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + sleep 3 & sleep 2 & sleep 1 wait $! echo $? diff --git a/shell/hush_test/hush-misc/wait3.tests b/shell/hush_test/hush-misc/wait3.tests index ac541c3fc6..aceed11266 100755 --- a/shell/hush_test/hush-misc/wait3.tests +++ b/shell/hush_test/hush-misc/wait3.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + sleep 2 & (sleep 1;exit 3) & wait $! echo $? jobs diff --git a/shell/hush_test/hush-misc/wait4.tests b/shell/hush_test/hush-misc/wait4.tests index cc34059ac9..c979a38b6e 100755 --- a/shell/hush_test/hush-misc/wait4.tests +++ b/shell/hush_test/hush-misc/wait4.tests @@ -1,2 +1,5 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + sleep 1 | (sleep 1;exit 3) & wait %1 echo Three:$? diff --git a/shell/hush_test/hush-misc/wait5.tests b/shell/hush_test/hush-misc/wait5.tests index 1b4762d89c..e0ac8c2511 100755 --- a/shell/hush_test/hush-misc/wait5.tests +++ b/shell/hush_test/hush-misc/wait5.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + sleep 0 | (sleep 0;exit 3) & sleep 1 echo Zero:$? diff --git a/shell/hush_test/hush-misc/wait6.tests b/shell/hush_test/hush-misc/wait6.tests index c237131993..c09002ab00 100755 --- a/shell/hush_test/hush-misc/wait6.tests +++ b/shell/hush_test/hush-misc/wait6.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + # In bash, "wait $!" extracts correct exitcode even if bg task has already exited # It prints 0, then 3: (sleep 0; exit 3) & sleep 1 diff --git a/shell/hush_test/hush-signals/catch.tests b/shell/hush_test/hush-signals/catch.tests index d2a21d17e0..4b1a08e8f5 100755 --- a/shell/hush_test/hush-signals/catch.tests +++ b/shell/hush_test/hush-signals/catch.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test ("User defined signal 2" text not emitted) +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + # avoid ugly warnings about signals not being caught trap ":" USR1 USR2 diff --git a/shell/hush_test/hush-signals/signal1.tests b/shell/hush_test/hush-signals/signal1.tests index 61943467af..c83fa1254f 100755 --- a/shell/hush_test/hush-signals/signal1.tests +++ b/shell/hush_test/hush-signals/signal1.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + trap "echo got signal" USR1 for try in 1 2 3 4 5; do diff --git a/shell/hush_test/hush-signals/signal8.tests b/shell/hush_test/hush-signals/signal8.tests index 731af7477c..cd57901649 100755 --- a/shell/hush_test/hush-signals/signal8.tests +++ b/shell/hush_test/hush-signals/signal8.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + "$THIS_SH" -c ' exit_func() { echo "Removing traps" diff --git a/shell/hush_test/hush-signals/signal_read2.tests b/shell/hush_test/hush-signals/signal_read2.tests index eab5b9b5b5..11accd5ab5 100755 --- a/shell/hush_test/hush-signals/signal_read2.tests +++ b/shell/hush_test/hush-signals/signal_read2.tests @@ -1,3 +1,6 @@ +# If job control is disabled, skip the test ("Hangup" not emitted) +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + $THIS_SH -c ' (sleep 1; kill -HUP $$) & while true; do diff --git a/shell/hush_test/hush-signals/subshell.tests b/shell/hush_test/hush-signals/subshell.tests index d877f2b82e..856c922d30 100755 --- a/shell/hush_test/hush-signals/subshell.tests +++ b/shell/hush_test/hush-signals/subshell.tests @@ -1,5 +1,8 @@ # Non-empty traps should be reset in subshell +# If job control is disabled, skip the test ("Terminated" text not emitted) +test "`type jobs`" = "jobs is a shell builtin" || exit 77 + # HUP is special in interactive shells trap '' HUP # QUIT is always special diff --git a/shell/hush_test/run-all b/shell/hush_test/run-all index 7345fee435..ff29ca0b13 100755 --- a/shell/hush_test/run-all +++ b/shell/hush_test/run-all @@ -67,9 +67,12 @@ do_test() # echo Running test: "$x" echo -n "$1/$x:" ( - "$THIS_SH" "./$x" 2>&1 | \ - grep -va "^hush: using fallback suid method$" >"$name.xx" + "$THIS_SH" "./$x" >"$name.xx" 2>&1 r=$? + # filter !FEATURE_SUID_CONFIG_QUIET message + sed -i \ + -e "/^hush: using fallback suid method$/d" \ + "$name.xx" # filter C library differences sed -i \ -e "/: invalid option /s:'::g" \ From 26fdc7c6ef7457f409539d44cbff53b82a48f979 Mon Sep 17 00:00:00 2001 From: Noelle Leigh Date: Fri, 4 Jul 2025 08:22:03 -0400 Subject: [PATCH 178/277] docs: Add links to cmd description from cmd list In the generated `docs/busybox.pod` file, add a link for each command in the big command list that goes to that command's description, like a compact table of contents. This seems to only affect the generated HTML version of the usage page. To add `id` attributes to each usage section title, each command now uses `=head2` instead of `=item` for their section heading. To add links to each command in the list, the list was unindented so that it could use rich text, and each command was marked as code text (C<>) so they are styled in monospace. These changes mean that the HTML and mandoc command list will expand to fit the available width of the viewport, rather than wrapping at a fixed line-length. The plain text version retains its existing wrapping behavior. Because of the new code text formatting, `--quotes=none` was added to the pod2man and pod2text invocations in Makefile.custom to prevent the command list from quoting every single command. Signed-off-by: Noelle Leigh Signed-off-by: Denys Vlasenko --- Makefile.custom | 4 ++-- applets/usage_pod.c | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Makefile.custom b/Makefile.custom index 6f679c4e12..ea5edb1a8a 100644 --- a/Makefile.custom +++ b/Makefile.custom @@ -156,12 +156,12 @@ docs/busybox.pod: $(srctree)/docs/busybox_header.pod \ docs/BusyBox.txt: docs/busybox.pod $(disp_doc) $(Q)-mkdir -p docs - $(Q)-pod2text $< > $@ + $(Q)-pod2text --quotes=none $< > $@ docs/busybox.1: docs/busybox.pod $(disp_doc) $(Q)-mkdir -p docs - $(Q)-pod2man --center=busybox --release="version $(KERNELVERSION)" $< > $@ + $(Q)-pod2man --quotes=none --center=busybox --release="version $(KERNELVERSION)" $< > $@ docs/BusyBox.html: docs/busybox.net/BusyBox.html $(disp_doc) diff --git a/applets/usage_pod.c b/applets/usage_pod.c index 9e6d3f0ee5..2c177be905 100644 --- a/applets/usage_pod.c +++ b/applets/usage_pod.c @@ -67,30 +67,37 @@ int main(void) } if (col == 0) { col = 6; - printf("\t"); } else { printf(", "); } - printf("%s", usage_array[i].aname); + if (usage_array[i].usage[0] != NOUSAGE_STR[0]) { + /* + * If the applet usage string will be included in the final document + * optimistically link to its header (which is just the applet name). + */ + printf("L|/\"%1$s\">", usage_array[i].aname); + } else { + /* Without a usage string, just output the applet name with no link. */ + printf("C<%s>", usage_array[i].aname); + } col += len2; } printf("\n\n"); printf("=head1 COMMAND DESCRIPTIONS\n\n"); - printf("=over 4\n\n"); for (i = 0; i < num_messages; i++) { if (usage_array[i].aname[0] >= 'a' && usage_array[i].aname[0] <= 'z' && usage_array[i].usage[0] != NOUSAGE_STR[0] ) { - printf("=item B<%s>\n\n", usage_array[i].aname); + /* This is the heading that will be linked from the command list. */ + printf("=head2 %s\n\n", usage_array[i].aname); if (usage_array[i].usage[0]) printf("%s %s\n\n", usage_array[i].aname, usage_array[i].usage); else printf("%s\n\n", usage_array[i].aname); } } - printf("=back\n\n"); return 0; } From 4539262b6a0a9eb029c113b27661734e773d0988 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 3 Aug 2025 16:12:00 +0200 Subject: [PATCH 179/277] hush: do not save history on xfunc error, it's unsafe Also, make second calls to install_special_sighandlers() less haphazard. It's not a fix for anything, but looking more organized now. function old new delta fflush_restore_ttypgrp_and__exit - 20 +20 restore_ttypgrp_and__exit 21 - -21 hush_main 1181 1152 -29 ------------------------------------------------------------------------------ (add/remove: 1/1 grow/shrink: 0/1 up/down: 20/-50) Total: -30 bytes Signed-off-by: Denys Vlasenko --- shell/hush.c | 84 +++++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/shell/hush.c b/shell/hush.c index 5f4c0cca59..3967f65e6e 100644 --- a/shell/hush.c +++ b/shell/hush.c @@ -2008,19 +2008,6 @@ static sighandler_t install_sighandler(int sig, sighandler_t handler) return old_sa.sa_handler; } -static void save_history_run_exit_trap_and_exit(int exitcode) NORETURN; - -static void restore_ttypgrp_and__exit(void) NORETURN; -static void restore_ttypgrp_and__exit(void) -{ - /* xfunc has failed! die die die */ - /* no EXIT traps, this is an escape hatch! */ - G.exiting = 1; - save_history_run_exit_trap_and_exit(xfunc_error_retval); -} - -#if ENABLE_HUSH_JOB - /* Needed only on some libc: * It was observed that on exit(), fgetc'ed buffered data * gets "unwound" via lseek(fd, -NUM, SEEK_CUR). @@ -2034,26 +2021,20 @@ static void restore_ttypgrp_and__exit(void) * and in `cmd` handling. * If set as die_func(), this makes xfunc_die() exit via _exit(), not exit(): */ -static void fflush_and__exit(void) NORETURN; -static void fflush_and__exit(void) +static NORETURN void fflush_and__exit(void) { fflush_all(); _exit(xfunc_error_retval); } -/* After [v]fork, in child: do not restore tty pgrp on xfunc death */ -# define disable_restore_tty_pgrp_on_exit() (die_func = fflush_and__exit) -/* After [v]fork, in parent: restore tty pgrp on xfunc death */ -# define enable_restore_tty_pgrp_on_exit() (die_func = restore_ttypgrp_and__exit) - +#if ENABLE_HUSH_JOB /* Restores tty foreground process group, and exits. * May be called as signal handler for fatal signal * (will resend signal to itself, producing correct exit state) * or called directly with -EXITCODE. * We also call it if xfunc is exiting. */ -static void restore_ttypgrp_and_killsig_or__exit(int sig) NORETURN; -static void restore_ttypgrp_and_killsig_or__exit(int sig) +static NORETURN void restore_ttypgrp_and_killsig_or__exit(int sig) { /* Careful: we can end up here after [v]fork. Do not restore * tty pgrp then, only top-level shell process does that */ @@ -2071,6 +2052,19 @@ static void restore_ttypgrp_and_killsig_or__exit(int sig) kill_myself_with_sig(sig); /* does not return */ } + +static NORETURN void fflush_restore_ttypgrp_and__exit(void) +{ + /* xfunc has failed! die die die */ + fflush_all(); + restore_ttypgrp_and_killsig_or__exit(- xfunc_error_retval); +} + +/* After [v]fork, in child: do not restore tty pgrp on xfunc death */ +# define disable_restore_tty_pgrp_on_exit() (die_func = fflush_and__exit) +/* After [v]fork, in parent: restore tty pgrp on xfunc death */ +# define enable_restore_tty_pgrp_on_exit() (die_func = fflush_restore_ttypgrp_and__exit) + #else # define disable_restore_tty_pgrp_on_exit() ((void)0) @@ -2107,8 +2101,10 @@ static sighandler_t pick_sighandler(unsigned sig) static const char* FAST_FUNC get_local_var_value(const char *name); -/* Restores tty foreground process group, and exits. */ -static void save_history_run_exit_trap_and_exit(int exitcode) +/* Self-explanatory. + * Restores tty foreground process group too. + */ +static NORETURN void save_history_run_exit_trap_and_exit(int exitcode) { #if ENABLE_FEATURE_EDITING_SAVE_ON_EXIT if (G.line_input_state @@ -10423,6 +10419,11 @@ int hush_main(int argc, char **argv) struct variable *shell_ver; INIT_G(); +#if ENABLE_HUSH_JOB + die_func = fflush_restore_ttypgrp_and__exit; +#else + die_func = fflush_and__exit; +#endif if (EXIT_SUCCESS != 0) /* if EXIT_SUCCESS == 0, it is already done */ G.last_exitcode = EXIT_SUCCESS; #if !BB_MMU @@ -10534,27 +10535,24 @@ int hush_main(int argc, char **argv) * PS4='+ ' */ + /* Shell is non-interactive at first. We need to call + * install_special_sighandlers() if we are going to execute "sh