[prev in list] [next in list] [prev in thread] [next in thread]
List: busybox
Subject: [PATCH v4] ash: add process substitution in bash-compatibility mode
From: Ron Yorston <rmy () pobox ! com>
Date: 2019-01-29 8:49:52
Message-ID: 5c5013b0.dO+RXJvwM+95HNmf%rmy () pobox ! com
[Download RAW message or body]
Process substitution is a Korn shell feature that's also available
in bash and some other shells. This patch implements process
substitution in ash when ASH_BASH_COMPAT is enabled.
function old new delta
argstr 1313 1477 +164
readtoken1 3265 3366 +101
close_new_fds - 71 +71
strtodest - 56 +56
evaltree 606 647 +41
.rodata 173923 173951 +28
cmdputs 393 418 +25
evalvar 697 714 +17
evalcommand 1724 1740 +16
cmdloop 404 411 +7
static.spclchars 9 11 +2
varvalue 669 626 -43
------------------------------------------------------------------------------
(add/remove: 2/0 grow/shrink: 9/1 up/down: 528/-43) Total: 485 bytes
text data bss dec hex filename
928951 4199 1888 935038 e447e busybox_old
929434 4199 1888 935521 e4661 busybox_unstripped
v2: Replace array of file descriptors with a linked list.
Include tests that were unaccountably omitted from v1.
v3: Update linked list code to the intended version.
v4: Change order of conditional code in cmdputs().
Signed-off-by: Ron Yorston <rmy@pobox.com>
---
include/platform.h | 2 +
shell/ash.c | 255 ++++++++++++++++---
shell/ash_test/ash-psubst/bash_procsub.right | 9 +
shell/ash_test/ash-psubst/bash_procsub.tests | 33 +++
4 files changed, 257 insertions(+), 42 deletions(-)
create mode 100644 shell/ash_test/ash-psubst/bash_procsub.right
create mode 100755 shell/ash_test/ash-psubst/bash_procsub.tests
diff --git a/include/platform.h b/include/platform.h
index 50365a31c..7dd9551e2 100644
--- a/include/platform.h
+++ b/include/platform.h
@@ -417,6 +417,8 @@ typedef unsigned smalluint;
#define HAVE_NET_ETHERNET_H 1
#define HAVE_SYS_STATFS_H 1
#define HAVE_PRINTF_PERCENTM 1
+#define HAVE_DEV_FD 1
+#define DEV_FD_PREFIX "/dev/fd/"
#if defined(__UCLIBC__)
# if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)
diff --git a/shell/ash.c b/shell/ash.c
index a284b084d..82712aa13 100644
--- a/shell/ash.c
+++ b/shell/ash.c
@@ -224,6 +224,14 @@
#define BASH_XTRACEFD ENABLE_ASH_BASH_COMPAT
#define BASH_READ_D ENABLE_ASH_BASH_COMPAT
#define IF_BASH_READ_D IF_ASH_BASH_COMPAT
+/* <(...) and >(...) */
+#if HAVE_DEV_FD
+# define BASH_PROCESS_SUBST ENABLE_ASH_BASH_COMPAT
+# define IF_BASH_PROCESS_SUBST IF_ASH_BASH_COMPAT
+#else
+# define BASH_PROCESS_SUBST 0
+# define IF_BASH_PROCESS_SUBST(...)
+#endif
#if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
/* Bionic at least up to version 24 has no glob() */
@@ -356,6 +364,11 @@ struct jmploc {
jmp_buf loc;
};
+struct psfd {
+ int fd;
+ struct psfd *next;
+};
+
struct globals_misc {
uint8_t exitstatus; /* exit status of last command */
uint8_t back_exitstatus;/* exit status of backquoted command */
@@ -365,7 +378,9 @@ struct globals_misc {
int shlvl;
#define rootshell (!shlvl)
int errlinno;
-
+#if BASH_PROCESS_SUBST
+ struct psfd *psfdlist;
+#endif
char *minusc; /* argument to -c option */
char *curdir; // = nullstr; /* current working directory */
@@ -444,6 +459,7 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc;
#define rootpid (G_misc.rootpid )
#define shlvl (G_misc.shlvl )
#define errlinno (G_misc.errlinno )
+#define psfdlist (G_misc.psfdlist )
#define minusc (G_misc.minusc )
#define curdir (G_misc.curdir )
#define physdir (G_misc.physdir )
@@ -714,6 +730,12 @@ out2str(const char *p)
#define CTLENDARI ((unsigned char)'\207')
#define CTLQUOTEMARK ((unsigned char)'\210')
#define CTL_LAST CTLQUOTEMARK
+#if BASH_PROCESS_SUBST
+# define CTLTOPROC ((unsigned char)'\211')
+# define CTLFROMPROC ((unsigned char)'\212')
+# undef CTL_LAST
+# define CTL_LAST CTLFROMPROC
+#endif
/* variable substitution byte (follows CTLVAR) */
#define VSTYPE 0x0f /* type of variable substitution */
@@ -985,6 +1007,10 @@ trace_puts_quoted(char *s)
case CTLESC: c = 'e'; goto backslash;
case CTLVAR: c = 'v'; goto backslash;
case CTLBACKQ: c = 'q'; goto backslash;
+#if BASH_PROCESS_SUBST
+ case CTLTOPROC: c = 'p'; goto backslash;
+ case CTLFROMPROC: c = 'P'; goto backslash;
+#endif
backslash:
putc('\\', tracefile);
putc(c, tracefile);
@@ -1146,8 +1172,17 @@ sharg(union node *arg, FILE *fp)
case CTLENDVAR:
putc('}', fp);
break;
+#if BASH_PROCESS_SUBST
+ case CTLTOPROC:
+ putc('>', fp);
+ goto backq;
+ case CTLFROMPROC:
+ putc('<', fp);
+ goto backq;
+#endif
case CTLBACKQ:
putc('$', fp);
+ IF_BASH_PROCESS_SUBST(backq:)
putc('(', fp);
shtree(bqlist->n, -1, NULL, fp);
putc(')', fp);
@@ -3153,8 +3188,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
/* 134 CTLARI */ CCTL_CCTL_CCTL_CCTL,
/* 135 CTLENDARI */ CCTL_CCTL_CCTL_CCTL,
/* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
+#if BASH_PROCESS_SUBST
+ /* 137 CTLTOPROC */ CCTL_CCTL_CCTL_CCTL,
+ /* 138 CTLFROMPROC */ CCTL_CCTL_CCTL_CCTL,
+#else
/* 137 */ CWORD_CWORD_CWORD_CWORD,
/* 138 */ CWORD_CWORD_CWORD_CWORD,
+#endif
/* 139 */ CWORD_CWORD_CWORD_CWORD,
/* 140 */ CWORD_CWORD_CWORD_CWORD,
/* 141 */ CWORD_CWORD_CWORD_CWORD,
@@ -4723,9 +4763,24 @@ cmdputs(const char *s)
quoted >>= 1;
subtype = 0;
goto dostr;
+#if BASH_PROCESS_SUBST
+ case CTLBACKQ:
+ c = '$';
+ str = "(...)";
+ break;
+ case CTLTOPROC:
+ c = '>';
+ str = "(...)";
+ break;
+ case CTLFROMPROC:
+ c = '<';
+ str = "(...)";
+ break;
+#else
case CTLBACKQ:
str = "$(...)";
goto dostr;
+#endif
#if ENABLE_FEATURE_SH_MATH
case CTLARI:
str = "$((";
@@ -6370,6 +6425,39 @@ exptilde(char *startp, char *p, int flags)
return startp;
}
+#if BASH_PROCESS_SUBST
+static void
+add_fd_to_list(int fd)
+{
+ struct psfd *p = (struct psfd *)xmalloc(sizeof(struct psfd));
+ p->fd = fd;
+ p->next = psfdlist;
+ psfdlist = p;
+}
+
+static void
+close_new_fds(struct psfd *stop)
+{
+ struct psfd *p;
+
+ INT_OFF;
+ p = psfdlist;
+ while (p && p != stop) {
+ close(p->fd);
+ p = p->next;
+ free(psfdlist);
+ psfdlist = p;
+ }
+ INT_ON;
+}
+
+static void
+close_all_fds(void)
+{
+ close_new_fds(NULL);
+}
+#endif
+
/*
* Execute a command inside back quotes. If it's a builtin command, we
* want to save its output in a block obtained from malloc. Otherwise
@@ -6403,10 +6491,20 @@ evaltreenr(union node *n, int flags)
}
static void FAST_FUNC
-evalbackcmd(union node *n, struct backcmd *result)
+evalbackcmd(union node *n, struct backcmd *result
+ IF_BASH_PROCESS_SUBST(, int ctl))
{
int pip[2];
struct job *jp;
+#if BASH_PROCESS_SUBST
+ /* determine end of pipe used by parent (ip) and child (ic) */
+ const int ip = (ctl == CTLTOPROC);
+ const int ic = !(ctl == CTLTOPROC);
+#else
+ const int ctl = CTLBACKQ;
+ const int ip = 0;
+ const int ic = 1;
+#endif
result->fd = -1;
result->buf = NULL;
@@ -6418,15 +6516,17 @@ evalbackcmd(union node *n, struct backcmd *result)
if (pipe(pip) < 0)
ash_msg_and_raise_perror("can't create pipe");
- jp = makejob(/*n,*/ 1);
- if (forkshell(jp, n, FORK_NOJOB) == 0) {
+ /* process substitution uses NULL job/node, like openhere() */
+ jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
+ if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
/* child */
FORCE_INT_ON;
- close(pip[0]);
- if (pip[1] != 1) {
- /*close(1);*/
- dup2_or_raise(pip[1], 1);
- close(pip[1]);
+ close(pip[ip]);
+ /* ic is index of child end of pipe *and* fd to connect it to */
+ if (pip[ic] != ic) {
+ /*close(ic);*/
+ dup2_or_raise(pip[ic], ic);
+ close(pip[ic]);
}
/* TODO: eflag clearing makes the following not abort:
* ash -c 'set -e; z=$(false;echo foo); echo $z'
@@ -6442,8 +6542,18 @@ evalbackcmd(union node *n, struct backcmd *result)
/* NOTREACHED */
}
/* parent */
- close(pip[1]);
- result->fd = pip[0];
+#if BASH_PROCESS_SUBST
+ if (ctl != CTLBACKQ) {
+ int fd = fcntl(pip[ip], F_DUPFD, 64);
+ if (fd > 0) {
+ close(pip[ip]);
+ pip[ip] = fd;
+ }
+ add_fd_to_list(pip[ip]);
+ }
+#endif
+ close(pip[ic]);
+ result->fd = pip[ip];
result->jp = jp;
out:
@@ -6455,8 +6565,11 @@ evalbackcmd(union node *n, struct backcmd *result)
* Expand stuff in backwards quotes.
*/
static void
-expbackq(union node *cmd, int flag)
+expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
{
+#if !BASH_PROCESS_SUBST
+ const int ctl = CTLBACKQ;
+#endif
struct backcmd in;
int i;
char buf[128];
@@ -6469,29 +6582,34 @@ expbackq(union node *cmd, int flag)
INT_OFF;
startloc = expdest - (char *)stackblock();
pushstackmark(&smark, startloc);
- evalbackcmd(cmd, &in);
+ evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
popstackmark(&smark);
- p = in.buf;
- i = in.nleft;
- if (i == 0)
- goto read;
- for (;;) {
- memtodest(p, i, syntax, flag & QUOTES_ESC);
+ if (ctl == CTLBACKQ) {
+ p = in.buf;
+ i = in.nleft;
+ if (i == 0)
+ goto read;
+ for (;;) {
+ memtodest(p, i, syntax, flag & QUOTES_ESC);
read:
- if (in.fd < 0)
- break;
- i = nonblock_immune_read(in.fd, buf, sizeof(buf));
- TRACE(("expbackq: read returns %d\n", i));
- if (i <= 0)
- break;
- p = buf;
- }
+ if (in.fd < 0)
+ break;
+ i = nonblock_immune_read(in.fd, buf, sizeof(buf));
+ TRACE(("expbackq: read returns %d\n", i));
+ if (i <= 0)
+ break;
+ p = buf;
+ }
- free(in.buf);
- if (in.fd >= 0) {
- close(in.fd);
- back_exitstatus = waitforjob(in.jp);
+ free(in.buf);
+ if (in.fd >= 0) {
+ close(in.fd);
+ back_exitstatus = waitforjob(in.jp);
+ }
+ } else {
+ sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
+ strtodest(buf, BASESYNTAX, 0);
}
INT_ON;
@@ -6587,6 +6705,10 @@ argstr(char *p, int flags)
CTLESC,
CTLVAR,
CTLBACKQ,
+#if BASH_PROCESS_SUBST
+ CTLTOPROC,
+ CTLFROMPROC,
+#endif
#if ENABLE_FEATURE_SH_MATH
CTLENDARI,
#endif
@@ -6689,8 +6811,12 @@ argstr(char *p, int flags)
p = evalvar(p, flags | inquotes);
TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
goto start;
+#if BASH_PROCESS_SUBST
+ case CTLTOPROC:
+ case CTLFROMPROC:
+#endif
case CTLBACKQ:
- expbackq(argbackq->n, flags | inquotes);
+ expbackq(argbackq->n, flags | inquotes IF_BASH_PROCESS_SUBST(, c));
argbackq = argbackq->next;
goto start;
#if ENABLE_FEATURE_SH_MATH
@@ -7433,7 +7559,8 @@ evalvar(char *p, int flag)
unsigned char c = *p++;
if (c == CTLESC)
p++;
- else if (c == CTLBACKQ) {
+ else if (c == CTLBACKQ
+ IF_BASH_PROCESS_SUBST(|| c == CTLTOPROC || c == CTLFROMPROC)) {
if (varlen >= 0)
argbackq = argbackq->next;
} else if (c == CTLVAR) {
@@ -9035,6 +9162,9 @@ evaltree(union node *n, int flags)
int checkexit = 0;
int (*evalfn)(union node *, int);
int status = 0;
+#if BASH_PROCESS_SUBST
+ struct psfd *procsub_stop = psfdlist;
+#endif
if (n == NULL) {
TRACE(("evaltree(NULL) called\n"));
@@ -9139,6 +9269,15 @@ evaltree(union node *n, int flags)
break;
}
out:
+#if BASH_PROCESS_SUBST
+ /* If we're in a shell function we close new fds here. Otherwise
+ * f() { while true; do cat <(echo hi); done }
+ * f
+ * runs out of file descriptors.
+ */
+ if (funcline != 0)
+ close_new_fds(procsub_stop);
+#endif
/* Order of checks below is important:
* signal handlers trigger before exit caused by "set -e".
*/
@@ -10268,6 +10407,15 @@ evalcommand(union node *cmd, int flags)
unwindredir(redir_stop);
unwindfiles(file_stop);
unwindlocalvars(localvar_stop);
+#if BASH_PROCESS_SUBST
+ /* Only close fds if we aren't in a shell function. If we don't
+ * close fds here
+ * while true; do cat <(echo hi); done
+ * runs out of file descriptors.
+ */
+ if (funcline == 0)
+ close_all_fds();
+#endif
if (lastarg) {
/* dsl: I think this is intended to be used to support
* '_' in 'vi' command mode during line editing...
@@ -12005,11 +12153,13 @@ realeofmark(const char *eofmark)
* using goto's to implement the subroutine linkage. The following macros
* will run code that appears at the end of readtoken1.
*/
+enum { OLD, NEW, PSUB };
#define CHECKEND() {goto checkend; checkend_return:;}
#define PARSEREDIR() {goto parseredir; parseredir_return:;}
#define PARSESUB() {goto parsesub; parsesub_return:;}
-#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
-#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
+#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEPROCSUB() {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
#define PARSEARITH() {goto parsearith; parsearith_return:;}
static int
readtoken1(int c, int syntax, char *eofmark, int striptabs)
@@ -12020,7 +12170,7 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
size_t len;
struct nodelist *bqlist;
smallint quotef;
- smallint oldstyle;
+ smallint style;
smallint pssyntax; /* we are expanding a prompt string */
IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
/* syntax stack */
@@ -12203,6 +12353,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
c = 0x100 + '>'; /* flag &> */
pungetc();
}
+#endif
+#if BASH_PROCESS_SUBST
+ if (c == '<' || c == '>') {
+ if (pgetc() == '(') {
+ PARSEPROCSUB();
+ break;
+ }
+ pungetc();
+ }
#endif
goto endword; /* exit outer loop */
}
@@ -12591,7 +12750,7 @@ parsebackq: {
memcpy(str, stackblock(), savelen);
}
- if (oldstyle) {
+ if (style == OLD) {
/* We must read until the closing backquote, giving special
* treatment to some slashes, and then push the string and
* reread it as input, interpreting it normally.
@@ -12649,20 +12808,20 @@ parsebackq: {
*nlpp = stzalloc(sizeof(**nlpp));
/* (*nlpp)->next = NULL; - stzalloc did it */
- if (oldstyle) {
+ if (style == OLD) {
saveprompt = doprompt;
doprompt = 0;
}
n = list(2);
- if (oldstyle)
+ if (style == OLD)
doprompt = saveprompt;
else if (readtoken() != TRP)
raise_error_unexpected_syntax(TRP);
(*nlpp)->n = n;
- if (oldstyle) {
+ if (style == OLD) {
/*
* Start reading from old file again, ignoring any pushed back
* tokens left from the backquote parsing
@@ -12677,9 +12836,18 @@ parsebackq: {
memcpy(out, str, savelen);
STADJUST(savelen, out);
}
- USTPUTC(CTLBACKQ, out);
- if (oldstyle)
+#if BASH_PROCESS_SUBST
+ if (style == PSUB)
+ USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
+ else
+#endif
+ USTPUTC(CTLBACKQ, out);
+ if (style == OLD)
goto parsebackq_oldreturn;
+#if BASH_PROCESS_SUBST
+ else if (style == PSUB)
+ goto parsebackq_psreturn;
+#endif
goto parsebackq_newreturn;
}
@@ -13115,6 +13283,9 @@ cmdloop(int top)
#if JOBS
if (doing_jobctl)
showjobs(SHOW_CHANGED|SHOW_STDERR);
+#endif
+#if BASH_PROCESS_SUBST
+ close_all_fds();
#endif
inter = 0;
if (iflag && top) {
diff --git a/shell/ash_test/ash-psubst/bash_procsub.right b/shell/ash_test/ash-psubst/bash_procsub.right
new file mode 100644
index 000000000..aa16a96be
--- /dev/null
+++ b/shell/ash_test/ash-psubst/bash_procsub.right
@@ -0,0 +1,9 @@
+hello 1
+hello 2
+hello 3
+<(echo "hello 0")
+hello 4
+HI THERE
+hello error
+hello error
+hello stderr
diff --git a/shell/ash_test/ash-psubst/bash_procsub.tests b/shell/ash_test/ash-psubst/bash_procsub.tests
new file mode 100755
index 000000000..63b836782
--- /dev/null
+++ b/shell/ash_test/ash-psubst/bash_procsub.tests
@@ -0,0 +1,33 @@
+# simplest case
+cat <(echo "hello 1")
+
+# can have more than one
+cat <(echo "hello 2") <(echo "hello 3")
+
+# doesn't work in quotes
+echo "<(echo \"hello 0\")"
+
+# process substitution can be nested inside command substitution
+echo $(cat <(echo "hello 4"))
+
+# example from http://wiki.bash-hackers.org/syntax/expansion/proc_subst
+# process substitutions can be passed to a function as parameters or
+# variables
+f() {
+ cat "$1" >"$x"
+}
+x=>(tr '[:lower:]' '[:upper:]') f <(echo 'hi there')
+
+# process substitution can be combined with redirection on exec
+rm -f err
+# save stderr
+exec 4>&2
+# copy stderr to a file
+exec 2> >(tee err)
+echo "hello error" >&2
+sync
+# restore stderr
+exec 2>&4
+cat err
+rm -f err
+echo "hello stderr" >&2
--
2.20.1
_______________________________________________
busybox mailing list
busybox@busybox.net
http://lists.busybox.net/mailman/listinfo/busybox
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic