src/eval.c

   1 /*      $OpenBSD: eval.c,v 1.40 2013/09/14 20:09:30 millert Exp $       */
   2
   3 /*-
   4  * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
   5  *               2011, 2012, 2013, 2014, 2015, 2016, 2017
   6  *      mirabilos <m@mirbsd.org>
   7  *
   8  * Provided that these terms and disclaimer and all copyright notices
   9  * are retained or reproduced in an accompanying document, permission
  10  * is granted to deal in this work without restriction, including un-
  11  * limited rights to use, publicly perform, distribute, sell, modify,
  12  * merge, give away, or sublicence.
  13  *
  14  * This work is provided "AS IS" and WITHOUT WARRANTY of any kind, to
  15  * the utmost extent permitted by applicable law, neither express nor
  16  * implied; without malicious intent or gross negligence. In no event
  17  * may a licensor, author or contributor be held liable for indirect,
  18  * direct, other damage, loss, or other issues arising in any way out
  19  * of dealing in the work, even if advised of the possibility of such
  20  * damage or existence of a defect, except proven that it results out
  21  * of said person's immediate fault when using the work as intended.
  22  */
  23
  24 #include "sh.h"
  25
  26 __RCSID("$MirOS: src/bin/mksh/eval.c,v 1.201 2017/04/06 01:59:54 tg Exp $");
  27
  28 /*
  29  * string expansion
  30  *
  31  * first pass: quoting, IFS separation, ~, ${}, $() and $(()) substitution.
  32  * second pass: alternation ({,}), filename expansion (*?[]).
  33  */
  34
  35 /* expansion generator state */
  36 typedef struct {
  37         /* not including an "int type;" member, see expand() */
  38         /* string */
  39         const char *str;
  40         /* source */
  41         union {
  42                 /* string[] */
  43                 const char **strv;
  44                 /* file */
  45                 struct shf *shf;
  46         } u;
  47         /* variable in ${var...} */
  48         struct tbl *var;
  49         /* split "$@" / call waitlast in $() */
  50         bool split;
  51 } Expand;
  52
  53 #define XBASE           0       /* scanning original */
  54 #define XSUB            1       /* expanding ${} string */
  55 #define XARGSEP         2       /* ifs0 between "$*" */
  56 #define XARG            3       /* expanding $*, $@ */
  57 #define XCOM            4       /* expanding $() */
  58 #define XNULLSUB        5       /* "$@" when $# is 0 (don't generate word) */
  59 #define XSUBMID         6       /* middle of expanding ${} */
  60
  61 /* States used for field splitting */
  62 #define IFS_WORD        0       /* word has chars (or quotes except "$@") */
  63 #define IFS_WS          1       /* have seen IFS white-space */
  64 #define IFS_NWS         2       /* have seen IFS non-white-space */
  65 #define IFS_IWS         3       /* beginning of word, ignore IFS WS */
  66 #define IFS_QUOTE       4       /* beg.w/quote, become IFS_WORD unless "$@" */
  67
  68 static int varsub(Expand *, const char *, const char *, int *, int *);
  69 static int comsub(Expand *, const char *, int);
  70 static char *valsub(struct op *, Area *);
  71 static char *trimsub(char *, char *, int);
  72 static void glob(char *, XPtrV *, bool);
  73 static void globit(XString *, char **, char *, XPtrV *, int);
  74 static const char *maybe_expand_tilde(const char *, XString *, char **, bool);
  75 #ifndef MKSH_NOPWNAM
  76 static char *homedir(char *);
  77 #endif
  78 static void alt_expand(XPtrV *, char *, char *, char *, int);
  79 static int utflen(const char *) MKSH_A_PURE;
  80 static void utfincptr(const char *, mksh_ari_t *);
  81
  82 /* UTFMODE functions */
  83 static int
  84 utflen(const char *s)
  85 {
  86         size_t n;
  87
  88         if (UTFMODE) {
  89                 n = 0;
  90                 while (*s) {
  91                         s += utf_ptradj(s);
  92                         ++n;
  93                 }
  94         } else
  95                 n = strlen(s);
  96
  97         if (n > 2147483647)
  98                 n = 2147483647;
  99         return ((int)n);
 100 }
 101
 102 static void
 103 utfincptr(const char *s, mksh_ari_t *lp)
 104 {
 105         const char *cp = s;
 106
 107         while ((*lp)--)
 108                 cp += utf_ptradj(cp);
 109         *lp = cp - s;
 110 }
 111
 112 /* compile and expand word */
 113 char *
 114 substitute(const char *cp, int f)
 115 {
 116         struct source *s, *sold;
 117
 118         sold = source;
 119         s = pushs(SWSTR, ATEMP);
 120         s->start = s->str = cp;
 121         source = s;
 122         if (yylex(ONEWORD) != LWORD)
 123                 internal_errorf(Tbadsubst);
 124         source = sold;
 125         afree(s, ATEMP);
 126         return (evalstr(yylval.cp, f));
 127 }
 128
 129 /*
 130  * expand arg-list
 131  */
 132 char **
 133 eval(const char **ap, int f)
 134 {
 135         XPtrV w;
 136
 137         if (*ap == NULL) {
 138                 union mksh_ccphack vap;
 139
 140                 vap.ro = ap;
 141                 return (vap.rw);
 142         }
 143         XPinit(w, 32);
 144         /* space for shell name */
 145         XPput(w, NULL);
 146         while (*ap != NULL)
 147                 expand(*ap++, &w, f);
 148         XPput(w, NULL);
 149         return ((char **)XPclose(w) + 1);
 150 }
 151
 152 /*
 153  * expand string
 154  */
 155 char *
 156 evalstr(const char *cp, int f)
 157 {
 158         XPtrV w;
 159         char *dp = null;
 160
 161         XPinit(w, 1);
 162         expand(cp, &w, f);
 163         if (XPsize(w))
 164                 dp = *XPptrv(w);
 165         XPfree(w);
 166         return (dp);
 167 }
 168
 169 /*
 170  * expand string - return only one component
 171  * used from iosetup to expand redirection files
 172  */
 173 char *
 174 evalonestr(const char *cp, int f)
 175 {
 176         XPtrV w;
 177         char *rv;
 178
 179         XPinit(w, 1);
 180         expand(cp, &w, f);
 181         switch (XPsize(w)) {
 182         case 0:
 183                 rv = null;
 184                 break;
 185         case 1:
 186                 rv = (char *) *XPptrv(w);
 187                 break;
 188         default:
 189                 rv = evalstr(cp, f & ~DOGLOB);
 190                 break;
 191         }
 192         XPfree(w);
 193         return (rv);
 194 }
 195
 196 /* for nested substitution: ${var:=$var2} */
 197 typedef struct SubType {
 198         struct tbl *var;        /* variable for ${var..} */
 199         struct SubType *prev;   /* old type */
 200         struct SubType *next;   /* poped type (to avoid re-allocating) */
 201         size_t  base;           /* start position of expanded word */
 202         short   stype;          /* [=+-?%#] action after expanded word */
 203         short   f;              /* saved value of f (DOPAT, etc) */
 204         uint8_t quotep;         /* saved value of quote (for ${..[%#]..}) */
 205         uint8_t quotew;         /* saved value of quote (for ${..[+-=]..}) */
 206 } SubType;
 207
 208 void
 209 expand(
 210     /* input word */
 211     const char *ccp,
 212     /* output words */
 213     XPtrV *wp,
 214     /* DO* flags */
 215     int f)
 216 {
 217         int c = 0;
 218         /* expansion type */
 219         int type;
 220         /* quoted */
 221         int quote = 0;
 222         /* destination string and live pointer */
 223         XString ds;
 224         char *dp;
 225         /* source */
 226         const char *sp;
 227         /* second pass flags */
 228         int fdo;
 229         /* have word */
 230         int word;
 231         /* field splitting of parameter/command substitution */
 232         int doblank;
 233         /* expansion variables */
 234         Expand x = {
 235                 NULL, { NULL }, NULL, 0
 236         };
 237         SubType st_head, *st;
 238         /* record number of trailing newlines in COMSUB */
 239         int newlines = 0;
 240         bool saw_eq, make_magic;
 241         unsigned int tilde_ok;
 242         size_t len;
 243         char *cp;
 244
 245         if (ccp == NULL)
 246                 internal_errorf("expand(NULL)");
 247         /* for alias, readonly, set, typeset commands */
 248         if ((f & DOVACHECK) && is_wdvarassign(ccp)) {
 249                 f &= ~(DOVACHECK | DOBLANK | DOGLOB | DOTILDE);
 250                 f |= DOASNTILDE | DOSCALAR;
 251         }
 252         if (Flag(FNOGLOB))
 253                 f &= ~DOGLOB;
 254         if (Flag(FMARKDIRS))
 255                 f |= DOMARKDIRS;
 256         if (Flag(FBRACEEXPAND) && (f & DOGLOB))
 257                 f |= DOBRACE;
 258
 259         /* init destination string */
 260         Xinit(ds, dp, 128, ATEMP);
 261         type = XBASE;
 262         sp = ccp;
 263         fdo = 0;
 264         saw_eq = false;
 265         /* must be 1/0 */
 266         tilde_ok = (f & (DOTILDE | DOASNTILDE)) ? 1 : 0;
 267         doblank = 0;
 268         make_magic = false;
 269         word = (f&DOBLANK) ? IFS_WS : IFS_WORD;
 270         /* clang doesn't know OSUBST comes before CSUBST */
 271         memset(&st_head, 0, sizeof(st_head));
 272         st = &st_head;
 273
 274         while (/* CONSTCOND */ 1) {
 275                 Xcheck(ds, dp);
 276
 277                 switch (type) {
 278                 case XBASE:
 279                         /* original prefixed string */
 280                         c = *sp++;
 281                         switch (c) {
 282                         case EOS:
 283                                 c = 0;
 284                                 break;
 285                         case CHAR:
 286                                 c = *sp++;
 287                                 break;
 288                         case QCHAR:
 289                                 /* temporary quote */
 290                                 quote |= 2;
 291                                 c = *sp++;
 292                                 break;
 293                         case OQUOTE:
 294                                 if (word != IFS_WORD)
 295                                         word = IFS_QUOTE;
 296                                 tilde_ok = 0;
 297                                 quote = 1;
 298                                 continue;
 299                         case CQUOTE:
 300                                 if (word == IFS_QUOTE)
 301                                         word = IFS_WORD;
 302                                 quote = st->quotew;
 303                                 continue;
 304                         case COMASUB:
 305                         case COMSUB:
 306                         case FUNASUB:
 307                         case FUNSUB:
 308                         case VALSUB:
 309                                 tilde_ok = 0;
 310                                 if (f & DONTRUNCOMMAND) {
 311                                         word = IFS_WORD;
 312                                         *dp++ = '$';
 313                                         switch (c) {
 314                                         case COMASUB:
 315                                         case COMSUB:
 316                                                 *dp++ = '(';
 317                                                 c = ')';
 318                                                 break;
 319                                         case FUNASUB:
 320                                         case FUNSUB:
 321                                         case VALSUB:
 322                                                 *dp++ = '{';
 323                                                 *dp++ = c == VALSUB ? '|' : ' ';
 324                                                 c = '}';
 325                                                 break;
 326                                         }
 327                                         while (*sp != '\0') {
 328                                                 Xcheck(ds, dp);
 329                                                 *dp++ = *sp++;
 330                                         }
 331                                         if (c == '}')
 332                                                 *dp++ = ';';
 333                                         *dp++ = c;
 334                                 } else {
 335                                         type = comsub(&x, sp, c);
 336                                         if (type != XBASE && (f & DOBLANK))
 337                                                 doblank++;
 338                                         sp = strnul(sp) + 1;
 339                                         newlines = 0;
 340                                 }
 341                                 continue;
 342                         case EXPRSUB:
 343                                 tilde_ok = 0;
 344                                 if (f & DONTRUNCOMMAND) {
 345                                         word = IFS_WORD;
 346                                         *dp++ = '$'; *dp++ = '('; *dp++ = '(';
 347                                         while (*sp != '\0') {
 348                                                 Xcheck(ds, dp);
 349                                                 *dp++ = *sp++;
 350                                         }
 351                                         *dp++ = ')'; *dp++ = ')';
 352                                 } else {
 353                                         struct tbl v;
 354
 355                                         v.flag = DEFINED|ISSET|INTEGER;
 356                                         /* not default */
 357                                         v.type = 10;
 358                                         v.name[0] = '\0';
 359                                         v_evaluate(&v, substitute(sp, 0),
 360                                             KSH_UNWIND_ERROR, true);
 361                                         sp = strnul(sp) + 1;
 362                                         x.str = str_val(&v);
 363                                         type = XSUB;
 364                                         if (f & DOBLANK)
 365                                                 doblank++;
 366                                 }
 367                                 continue;
 368                         case OSUBST: {
 369                                 /* ${{#}var{:}[=+-?#%]word} */
 370                         /*-
 371                          * format is:
 372                          *      OSUBST [{x] plain-variable-part \0
 373                          *          compiled-word-part CSUBST [}x]
 374                          * This is where all syntax checking gets done...
 375                          */
 376                                 /* skip the { or x (}) */
 377                                 const char *varname = ++sp;
 378                                 int stype;
 379                                 int slen = 0;
 380
 381                                 /* skip variable */
 382                                 sp = cstrchr(sp, '\0') + 1;
 383                                 type = varsub(&x, varname, sp, &stype, &slen);
 384                                 if (type < 0) {
 385                                         char *beg, *end, *str;
 386  unwind_substsyn:
 387                                         /* restore sp */
 388                                         sp = varname - 2;
 389                                         beg = wdcopy(sp, ATEMP);
 390                                         end = (wdscan(cstrchr(sp, '\0') + 1,
 391                                             CSUBST) - sp) + beg;
 392                                         /* ({) the } or x is already skipped */
 393                                         if (end < wdscan(beg, EOS))
 394                                                 *end = EOS;
 395                                         str = snptreef(NULL, 64, Tf_S, beg);
 396                                         afree(beg, ATEMP);
 397                                         errorf(Tf_sD_s, str, Tbadsubst);
 398                                 }
 399                                 if (f & DOBLANK)
 400                                         doblank++;
 401                                 tilde_ok = 0;
 402                                 if (word == IFS_QUOTE && type != XNULLSUB)
 403                                         word = IFS_WORD;
 404                                 if (type == XBASE) {
 405                                         /* expand? */
 406                                         if (!st->next) {
 407                                                 SubType *newst;
 408
 409                                                 newst = alloc(sizeof(SubType), ATEMP);
 410                                                 newst->next = NULL;
 411                                                 newst->prev = st;
 412                                                 st->next = newst;
 413                                         }
 414                                         st = st->next;
 415                                         st->stype = stype;
 416                                         st->base = Xsavepos(ds, dp);
 417                                         st->f = f;
 418                                         if (x.var == vtemp) {
 419                                                 st->var = tempvar(vtemp->name);
 420                                                 st->var->flag &= ~INTEGER;
 421                                                 /* can't fail here */
 422                                                 setstr(st->var,
 423                                                     str_val(x.var),
 424                                                     KSH_RETURN_ERROR | 0x4);
 425                                         } else
 426                                                 st->var = x.var;
 427
 428                                         st->quotew = st->quotep = quote;
 429                                         /* skip qualifier(s) */
 430                                         if (stype)
 431                                                 sp += slen;
 432                                         switch (stype & 0x17F) {
 433                                         case 0x100 | '#':
 434                                                 x.str = shf_smprintf("%08X",
 435                                                     (unsigned int)hash(str_val(st->var)));
 436                                                 break;
 437                                         case 0x100 | 'Q': {
 438                                                 struct shf shf;
 439
 440                                                 shf_sopen(NULL, 0, SHF_WR|SHF_DYNAMIC, &shf);
 441                                                 print_value_quoted(&shf, str_val(st->var));
 442                                                 x.str = shf_sclose(&shf);
 443                                                 break;
 444                                             }
 445                                         case '0': {
 446                                                 char *beg, *mid, *end, *stg;
 447                                                 mksh_ari_t from = 0, num = -1, flen, finc = 0;
 448
 449                                                 beg = wdcopy(sp, ATEMP);
 450                                                 mid = beg + (wdscan(sp, ADELIM) - sp);
 451                                                 stg = beg + (wdscan(sp, CSUBST) - sp);
 452                                                 mid[-2] = EOS;
 453                                                 if (mid[-1] == /*{*/'}') {
 454                                                         sp += mid - beg - 1;
 455                                                         end = NULL;
 456                                                 } else {
 457                                                         end = mid +
 458                                                             (wdscan(mid, ADELIM) - mid);
 459                                                         if (end[-1] != /*{*/ '}')
 460                                                                 /* more than max delimiters */
 461                                                                 goto unwind_substsyn;
 462                                                         end[-2] = EOS;
 463                                                         sp += end - beg - 1;
 464                                                 }
 465                                                 evaluate(substitute(stg = wdstrip(beg, 0), 0),
 466                                                     &from, KSH_UNWIND_ERROR, true);
 467                                                 afree(stg, ATEMP);
 468                                                 if (end) {
 469                                                         evaluate(substitute(stg = wdstrip(mid, 0), 0),
 470                                                             &num, KSH_UNWIND_ERROR, true);
 471                                                         afree(stg, ATEMP);
 472                                                 }
 473                                                 afree(beg, ATEMP);
 474                                                 beg = str_val(st->var);
 475                                                 flen = utflen(beg);
 476                                                 if (from < 0) {
 477                                                         if (-from < flen)
 478                                                                 finc = flen + from;
 479                                                 } else
 480                                                         finc = from < flen ? from : flen;
 481                                                 if (UTFMODE)
 482                                                         utfincptr(beg, &finc);
 483                                                 beg += finc;
 484                                                 flen = utflen(beg);
 485                                                 if (num < 0 || num > flen)
 486                                                         num = flen;
 487                                                 if (UTFMODE)
 488                                                         utfincptr(beg, &num);
 489                                                 strndupx(x.str, beg, num, ATEMP);
 490                                                 goto do_CSUBST;
 491                                             }
 492                                         case 0x100 | '/':
 493                                         case '/': {
 494                                                 char *s, *p, *d, *sbeg, *end;
 495                                                 char *pat = NULL, *rrep = null;
 496                                                 char fpat = 0, *tpat1, *tpat2;
 497                                                 char *ws, *wpat, *wrep;
 498
 499                                                 s = ws = wdcopy(sp, ATEMP);
 500                                                 p = s + (wdscan(sp, ADELIM) - sp);
 501                                                 d = s + (wdscan(sp, CSUBST) - sp);
 502                                                 p[-2] = EOS;
 503                                                 if (p[-1] == /*{*/'}')
 504                                                         d = NULL;
 505                                                 else
 506                                                         d[-2] = EOS;
 507                                                 sp += (d ? d : p) - s - 1;
 508                                                 if (!(stype & 0x180) &&
 509                                                     s[0] == CHAR &&
 510                                                     (s[1] == '#' || s[1] == '%'))
 511                                                         fpat = s[1];
 512                                                 wpat = s + (fpat ? 2 : 0);
 513                                                 wrep = d ? p : NULL;
 514                                                 if (!(stype & 0x100)) {
 515                                                         rrep = wrep ? evalstr(wrep,
 516                                                             DOTILDE | DOSCALAR) :
 517                                                             null;
 518                                                 }
 519
 520                                                 /* prepare string on which to work */
 521                                                 strdupx(s, str_val(st->var), ATEMP);
 522                                                 sbeg = s;
 523  again_search:
 524                                                 pat = evalstr(wpat,
 525                                                     DOTILDE | DOSCALAR | DOPAT);
 526                                                 /* check for special cases */
 527                                                 if (!*pat && !fpat) {
 528                                                         /*
 529                                                          * empty unanchored
 530                                                          * pattern => reject
 531                                                          */
 532                                                         goto no_repl;
 533                                                 }
 534                                                 if ((stype & 0x180) &&
 535                                                     gmatchx(null, pat, false)) {
 536                                                         /*
 537                                                          * pattern matches empty
 538                                                          * string => don't loop
 539                                                          */
 540                                                         stype &= ~0x180;
 541                                                 }
 542
 543                                                 /* first see if we have any match at all */
 544                                                 if (fpat == '#') {
 545                                                         /* anchor at the beginning */
 546                                                         tpat1 = shf_smprintf("%s%c*", pat, MAGIC);
 547                                                         tpat2 = tpat1;
 548                                                 } else if (fpat == '%') {
 549                                                         /* anchor at the end */
 550                                                         tpat1 = shf_smprintf("%c*%s", MAGIC, pat);
 551                                                         tpat2 = pat;
 552                                                 } else {
 553                                                         /* float */
 554                                                         tpat1 = shf_smprintf("%c*%s%c*", MAGIC, pat, MAGIC);
 555                                                         tpat2 = tpat1 + 2;
 556                                                 }
 557  again_repl:
 558                                                 /*
 559                                                  * this would not be necessary if gmatchx would return
 560                                                  * the start and end values of a match found, like re*
 561                                                  */
 562                                                 if (!gmatchx(sbeg, tpat1, false))
 563                                                         goto end_repl;
 564                                                 end = strnul(s);
 565                                                 /* now anchor the beginning of the match */
 566                                                 if (fpat != '#')
 567                                                         while (sbeg <= end) {
 568                                                                 if (gmatchx(sbeg, tpat2, false))
 569                                                                         break;
 570                                                                 else
 571                                                                         sbeg++;
 572                                                         }
 573                                                 /* now anchor the end of the match */
 574                                                 p = end;
 575                                                 if (fpat != '%')
 576                                                         while (p >= sbeg) {
 577                                                                 bool gotmatch;
 578
 579                                                                 c = *p;
 580                                                                 *p = '\0';
 581                                                                 gotmatch = tobool(gmatchx(sbeg, pat, false));
 582                                                                 *p = c;
 583                                                                 if (gotmatch)
 584                                                                         break;
 585                                                                 p--;
 586                                                         }
 587                                                 strndupx(end, sbeg, p - sbeg, ATEMP);
 588                                                 record_match(end);
 589                                                 afree(end, ATEMP);
 590                                                 if (stype & 0x100) {
 591                                                         if (rrep != null)
 592                                                                 afree(rrep, ATEMP);
 593                                                         rrep = wrep ? evalstr(wrep,
 594                                                             DOTILDE | DOSCALAR) :
 595                                                             null;
 596                                                 }
 597                                                 strndupx(end, s, sbeg - s, ATEMP);
 598                                                 d = shf_smprintf(Tf_sss, end, rrep, p);
 599                                                 afree(end, ATEMP);
 600                                                 sbeg = d + (sbeg - s) + strlen(rrep);
 601                                                 afree(s, ATEMP);
 602                                                 s = d;
 603                                                 if (stype & 0x100) {
 604                                                         afree(tpat1, ATEMP);
 605                                                         afree(pat, ATEMP);
 606                                                         goto again_search;
 607                                                 } else if (stype & 0x80)
 608                                                         goto again_repl;
 609  end_repl:
 610                                                 afree(tpat1, ATEMP);
 611                                                 x.str = s;
 612  no_repl:
 613                                                 afree(pat, ATEMP);
 614                                                 if (rrep != null)
 615                                                         afree(rrep, ATEMP);
 616                                                 afree(ws, ATEMP);
 617                                                 goto do_CSUBST;
 618                                             }
 619                                         case '#':
 620                                         case '%':
 621                                                 /* ! DOBLANK,DOBRACE */
 622                                                 f = (f & DONTRUNCOMMAND) |
 623                                                     DOPAT | DOTILDE |
 624                                                     DOTEMP | DOSCALAR;
 625                                                 tilde_ok = 1;
 626                                                 st->quotew = quote = 0;
 627                                                 /*
 628                                                  * Prepend open pattern (so |
 629                                                  * in a trim will work as
 630                                                  * expected)
 631                                                  */
 632                                                 if (!Flag(FSH)) {
 633                                                         *dp++ = MAGIC;
 634                                                         *dp++ = 0x80 | '@';
 635                                                 }
 636                                                 break;
 637                                         case '=':
 638                                                 /*
 639                                                  * Tilde expansion for string
 640                                                  * variables in POSIX mode is
 641                                                  * governed by Austinbug 351.
 642                                                  * In non-POSIX mode historic
 643                                                  * ksh behaviour (enable it!)
 644                                                  * us followed.
 645                                                  * Not doing tilde expansion
 646                                                  * for integer variables is a
 647                                                  * non-POSIX thing - makes
 648                                                  * sense though, since ~ is
 649                                                  * a arithmetic operator.
 650                                                  */
 651                                                 if (!(x.var->flag & INTEGER))
 652                                                         f |= DOASNTILDE | DOTILDE;
 653                                                 f |= DOTEMP | DOSCALAR;
 654                                                 /*
 655                                                  * These will be done after the
 656                                                  * value has been assigned.
 657                                                  */
 658                                                 f &= ~(DOBLANK|DOGLOB|DOBRACE);
 659                                                 tilde_ok = 1;
 660                                                 break;
 661                                         case '?':
 662                                                 if (*sp == CSUBST)
 663                                                         errorf("%s: parameter null or not set",
 664                                                             st->var->name);
 665                                                 f &= ~DOBLANK;
 666                                                 f |= DOTEMP;
 667                                                 /* FALLTHROUGH */
 668                                         default:
 669                                                 /* '-' '+' '?' */
 670                                                 if (quote)
 671                                                         word = IFS_WORD;
 672                                                 else if (dp == Xstring(ds, dp))
 673                                                         word = IFS_IWS;
 674                                                 /* Enable tilde expansion */
 675                                                 tilde_ok = 1;
 676                                                 f |= DOTILDE;
 677                                         }
 678                                 } else
 679                                         /* skip word */
 680                                         sp += wdscan(sp, CSUBST) - sp;
 681                                 continue;
 682                             }
 683                         case CSUBST:
 684                                 /* only get here if expanding word */
 685  do_CSUBST:
 686                                 /* ({) skip the } or x */
 687                                 sp++;
 688                                 /* in case of ${unset:-} */
 689                                 tilde_ok = 0;
 690                                 *dp = '\0';
 691                                 quote = st->quotep;
 692                                 f = st->f;
 693                                 if (f & DOBLANK)
 694                                         doblank--;
 695                                 switch (st->stype & 0x17F) {
 696                                 case '#':
 697                                 case '%':
 698                                         if (!Flag(FSH)) {
 699                                                 /* Append end-pattern */
 700                                                 *dp++ = MAGIC;
 701                                                 *dp++ = ')';
 702                                         }
 703                                         *dp = '\0';
 704                                         dp = Xrestpos(ds, dp, st->base);
 705                                         /*
 706                                          * Must use st->var since calling
 707                                          * global would break things
 708                                          * like x[i+=1].
 709                                          */
 710                                         x.str = trimsub(str_val(st->var),
 711                                                 dp, st->stype);
 712                                         if (x.str[0] != '\0') {
 713                                                 word = IFS_IWS;
 714                                                 type = XSUB;
 715                                         } else if (quote) {
 716                                                 word = IFS_WORD;
 717                                                 type = XSUB;
 718                                         } else {
 719                                                 if (dp == Xstring(ds, dp))
 720                                                         word = IFS_IWS;
 721                                                 type = XNULLSUB;
 722                                         }
 723                                         if (f & DOBLANK)
 724                                                 doblank++;
 725                                         st = st->prev;
 726                                         continue;
 727                                 case '=':
 728                                         /*
 729                                          * Restore our position and substitute
 730                                          * the value of st->var (may not be
 731                                          * the assigned value in the presence
 732                                          * of integer/right-adj/etc attributes).
 733                                          */
 734                                         dp = Xrestpos(ds, dp, st->base);
 735                                         /*
 736                                          * Must use st->var since calling
 737                                          * global would cause with things
 738                                          * like x[i+=1] to be evaluated twice.
 739                                          */
 740                                         /*
 741                                          * Note: not exported by FEXPORT
 742                                          * in AT&T ksh.
 743                                          */
 744                                         /*
 745                                          * XXX POSIX says readonly is only
 746                                          * fatal for special builtins (setstr
 747                                          * does readonly check).
 748                                          */
 749                                         len = strlen(dp) + 1;
 750                                         setstr(st->var,
 751                                             debunk(alloc(len, ATEMP),
 752                                             dp, len), KSH_UNWIND_ERROR);
 753                                         x.str = str_val(st->var);
 754                                         type = XSUB;
 755                                         if (f & DOBLANK)
 756                                                 doblank++;
 757                                         st = st->prev;
 758                                         word = quote || (!*x.str && (f & DOSCALAR)) ? IFS_WORD : IFS_IWS;
 759                                         continue;
 760                                 case '?':
 761                                         dp = Xrestpos(ds, dp, st->base);
 762
 763                                         errorf(Tf_sD_s, st->var->name,
 764                                             debunk(dp, dp, strlen(dp) + 1));
 765                                         break;
 766                                 case '0':
 767                                 case 0x100 | '/':
 768                                 case '/':
 769                                 case 0x100 | '#':
 770                                 case 0x100 | 'Q':
 771                                         dp = Xrestpos(ds, dp, st->base);
 772                                         type = XSUB;
 773                                         word = quote || (!*x.str && (f & DOSCALAR)) ? IFS_WORD : IFS_IWS;
 774                                         if (f & DOBLANK)
 775                                                 doblank++;
 776                                         st = st->prev;
 777                                         continue;
 778                                 /* default: '-' '+' */
 779                                 }
 780                                 st = st->prev;
 781                                 type = XBASE;
 782                                 continue;
 783
 784                         case OPAT:
 785                                 /* open pattern: *(foo|bar) */
 786                                 /* Next char is the type of pattern */
 787                                 make_magic = true;
 788                                 c = *sp++ | 0x80;
 789                                 break;
 790
 791                         case SPAT:
 792                                 /* pattern separator (|) */
 793                                 make_magic = true;
 794                                 c = '|';
 795                                 break;
 796
 797                         case CPAT:
 798                                 /* close pattern */
 799                                 make_magic = true;
 800                                 c = /*(*/ ')';
 801                                 break;
 802                         }
 803                         break;
 804
 805                 case XNULLSUB:
 806                         /*
 807                          * Special case for "$@" (and "${foo[@]}") - no
 808                          * word is generated if $# is 0 (unless there is
 809                          * other stuff inside the quotes).
 810                          */
 811                         type = XBASE;
 812                         if (f & DOBLANK) {
 813                                 doblank--;
 814                                 if (dp == Xstring(ds, dp) && word != IFS_WORD)
 815                                         word = IFS_IWS;
 816                         }
 817                         continue;
 818
 819                 case XSUB:
 820                 case XSUBMID:
 821                         if ((c = *x.str++) == 0) {
 822                                 type = XBASE;
 823                                 if (f & DOBLANK)
 824                                         doblank--;
 825                                 continue;
 826                         }
 827                         break;
 828
 829                 case XARGSEP:
 830                         type = XARG;
 831                         quote = 1;
 832                         /* FALLTHROUGH */
 833                 case XARG:
 834                         if ((c = *x.str++) == '\0') {
 835                                 /*
 836                                  * force null words to be created so
 837                                  * set -- "" 2 ""; echo "$@" will do
 838                                  * the right thing
 839                                  */
 840                                 if (quote && x.split)
 841                                         word = IFS_WORD;
 842                                 if ((x.str = *x.u.strv++) == NULL) {
 843                                         type = XBASE;
 844                                         if (f & DOBLANK)
 845                                                 doblank--;
 846                                         continue;
 847                                 }
 848                                 c = ifs0;
 849                                 if ((f & DOHEREDOC)) {
 850                                         /* pseudo-field-split reliably */
 851                                         if (c == 0)
 852                                                 c = ' ';
 853                                         break;
 854                                 }
 855                                 if ((f & DOSCALAR)) {
 856                                         /* do not field-split */
 857                                         if (x.split) {
 858                                                 c = ' ';
 859                                                 break;
 860                                         }
 861                                         if (c == 0)
 862                                                 continue;
 863                                 }
 864                                 if (c == 0) {
 865                                         if (quote && !x.split)
 866                                                 continue;
 867                                         if (!quote && word == IFS_WS)
 868                                                 continue;
 869                                         /* this is so we don't terminate */
 870                                         c = ' ';
 871                                         /* now force-emit a word */
 872                                         goto emit_word;
 873                                 }
 874                                 if (quote && x.split) {
 875                                         /* terminate word for "$@" */
 876                                         type = XARGSEP;
 877                                         quote = 0;
 878                                 }
 879                         }
 880                         break;
 881
 882                 case XCOM:
 883                         if (x.u.shf == NULL) {
 884                                 /* $(<...) failed */
 885                                 subst_exstat = 1;
 886                                 /* fake EOF */
 887                                 c = -1;
 888                         } else if (newlines) {
 889                                 /* spit out saved NLs */
 890                                 c = '\n';
 891                                 --newlines;
 892                         } else {
 893                                 while ((c = shf_getc(x.u.shf)) == 0 ||
 894 #ifdef MKSH_WITH_TEXTMODE
 895                                        c == '\r' ||
 896 #endif
 897                                        c == '\n') {
 898 #ifdef MKSH_WITH_TEXTMODE
 899                                         if (c == '\r') {
 900                                                 c = shf_getc(x.u.shf);
 901                                                 switch (c) {
 902                                                 case '\n':
 903                                                         break;
 904                                                 default:
 905                                                         shf_ungetc(c, x.u.shf);
 906                                                         /* FALLTHROUGH */
 907                                                 case -1:
 908                                                         c = '\r';
 909                                                         break;
 910                                                 }
 911                                         }
 912 #endif
 913                                         if (c == '\n')
 914                                                 /* save newlines */
 915                                                 newlines++;
 916                                 }
 917                                 if (newlines && c != -1) {
 918                                         shf_ungetc(c, x.u.shf);
 919                                         c = '\n';
 920                                         --newlines;
 921                                 }
 922                         }
 923                         if (c == -1) {
 924                                 newlines = 0;
 925                                 if (x.u.shf)
 926                                         shf_close(x.u.shf);
 927                                 if (x.split)
 928                                         subst_exstat = waitlast();
 929                                 type = XBASE;
 930                                 if (f & DOBLANK)
 931                                         doblank--;
 932                                 continue;
 933                         }
 934                         break;
 935                 }
 936
 937                 /* check for end of word or IFS separation */
 938                 if (c == 0 || (!quote && (f & DOBLANK) && doblank &&
 939                     !make_magic && ctype(c, C_IFS))) {
 940                         /*-
 941                          * How words are broken up:
 942                          *                      |       value of c
 943                          *      word            |       ws      nws     0
 944                          *      -----------------------------------
 945                          *      IFS_WORD                w/WS    w/NWS   w
 946                          *      IFS_WS                  -/WS    -/NWS   -
 947                          *      IFS_NWS                 -/NWS   w/NWS   -
 948                          *      IFS_IWS                 -/WS    w/NWS   -
 949                          * (w means generate a word)
 950                          */
 951                         if ((word == IFS_WORD) || (word == IFS_QUOTE) || (c &&
 952                             (word == IFS_IWS || word == IFS_NWS) &&
 953                             !ctype(c, C_IFSWS))) {
 954  emit_word:
 955                                 if (f & DOHERESTR)
 956                                         *dp++ = '\n';
 957                                 *dp++ = '\0';
 958                                 cp = Xclose(ds, dp);
 959                                 if (fdo & DOBRACE)
 960                                         /* also does globbing */
 961                                         alt_expand(wp, cp, cp,
 962                                             cp + Xlength(ds, (dp - 1)),
 963                                             fdo | (f & DOMARKDIRS));
 964                                 else if (fdo & DOGLOB)
 965                                         glob(cp, wp, tobool(f & DOMARKDIRS));
 966                                 else if ((f & DOPAT) || !(fdo & DOMAGIC))
 967                                         XPput(*wp, cp);
 968                                 else
 969                                         XPput(*wp, debunk(cp, cp,
 970                                             strlen(cp) + 1));
 971                                 fdo = 0;
 972                                 saw_eq = false;
 973                                 /* must be 1/0 */
 974                                 tilde_ok = (f & (DOTILDE | DOASNTILDE)) ? 1 : 0;
 975                                 if (c == 0)
 976                                         return;
 977                                 Xinit(ds, dp, 128, ATEMP);
 978                         } else if (c == 0) {
 979                                 return;
 980                         } else if (type == XSUB && ctype(c, C_IFS) &&
 981                             !ctype(c, C_IFSWS) && Xlength(ds, dp) == 0) {
 982                                 *(cp = alloc(1, ATEMP)) = '\0';
 983                                 XPput(*wp, cp);
 984                                 type = XSUBMID;
 985                         }
 986                         if (word != IFS_NWS)
 987                                 word = ctype(c, C_IFSWS) ? IFS_WS : IFS_NWS;
 988                 } else {
 989                         if (type == XSUB) {
 990                                 if (word == IFS_NWS &&
 991                                     Xlength(ds, dp) == 0) {
 992                                         *(cp = alloc(1, ATEMP)) = '\0';
 993                                         XPput(*wp, cp);
 994                                 }
 995                                 type = XSUBMID;
 996                         }
 997
 998                         /* age tilde_ok info - ~ code tests second bit */
 999                         tilde_ok <<= 1;
1000                         /* mark any special second pass chars */
1001                         if (!quote)
1002                                 switch (c) {
1003                                 case '[':
1004                                 case '!':
1005                                 case '-':
1006                                 case ']':
1007                                         /*
1008                                          * For character classes - doesn't hurt
1009                                          * to have magic !,-,]s outside of
1010                                          * [...] expressions.
1011                                          */
1012                                         if (f & (DOPAT | DOGLOB)) {
1013                                                 fdo |= DOMAGIC;
1014                                                 if (c == '[')
1015                                                         fdo |= f & DOGLOB;
1016                                                 *dp++ = MAGIC;
1017                                         }
1018                                         break;
1019                                 case '*':
1020                                 case '?':
1021                                         if (f & (DOPAT | DOGLOB)) {
1022                                                 fdo |= DOMAGIC | (f & DOGLOB);
1023                                                 *dp++ = MAGIC;
1024                                         }
1025                                         break;
1026                                 case '{':
1027                                 case '}':
1028                                 case ',':
1029                                         if ((f & DOBRACE) && (c == '{' /*}*/ ||
1030                                             (fdo & DOBRACE))) {
1031                                                 fdo |= DOBRACE|DOMAGIC;
1032                                                 *dp++ = MAGIC;
1033                                         }
1034                                         break;
1035                                 case '=':
1036                                         /* Note first unquoted = for ~ */
1037                                         if (!(f & DOTEMP) && (!Flag(FPOSIX) ||
1038                                             (f & DOASNTILDE)) && !saw_eq) {
1039                                                 saw_eq = true;
1040                                                 tilde_ok = 1;
1041                                         }
1042                                         break;
1043                                 case ':':
1044                                         /* : */
1045                                         /* Note unquoted : for ~ */
1046                                         if (!(f & DOTEMP) && (f & DOASNTILDE))
1047                                                 tilde_ok = 1;
1048                                         break;
1049                                 case '~':
1050                                         /*
1051                                          * tilde_ok is reset whenever
1052                                          * any of ' " $( $(( ${ } are seen.
1053                                          * Note that tilde_ok must be preserved
1054                                          * through the sequence ${A=a=}~
1055                                          */
1056                                         if (type == XBASE &&
1057                                             (f & (DOTILDE | DOASNTILDE)) &&
1058                                             (tilde_ok & 2)) {
1059                                                 const char *tcp;
1060                                                 char *tdp = dp;
1061
1062                                                 tcp = maybe_expand_tilde(sp,
1063                                                     &ds, &tdp,
1064                                                     tobool(f & DOASNTILDE));
1065                                                 if (tcp) {
1066                                                         if (dp != tdp)
1067                                                                 word = IFS_WORD;
1068                                                         dp = tdp;
1069                                                         sp = tcp;
1070                                                         continue;
1071                                                 }
1072                                         }
1073                                         break;
1074                                 }
1075                         else
1076                                 /* undo temporary */
1077                                 quote &= ~2;
1078
1079                         if (make_magic) {
1080                                 make_magic = false;
1081                                 fdo |= DOMAGIC | (f & DOGLOB);
1082                                 *dp++ = MAGIC;
1083                         } else if (ISMAGIC(c)) {
1084                                 fdo |= DOMAGIC;
1085                                 *dp++ = MAGIC;
1086                         }
1087                         /* save output char */
1088                         *dp++ = c;
1089                         word = IFS_WORD;
1090                 }
1091         }
1092 }
1093
1094 static bool
1095 hasnonempty(const char **strv)
1096 {
1097         size_t i = 0;
1098
1099         while (strv[i])
1100                 if (*strv[i++])
1101                         return (true);
1102         return (false);
1103 }
1104
1105 /*
1106  * Prepare to generate the string returned by ${} substitution.
1107  */
1108 static int
1109 varsub(Expand *xp, const char *sp, const char *word,
1110     int *stypep,        /* becomes qualifier type */
1111     int *slenp)         /* " " len (=, :=, etc.) valid iff *stypep != 0 */
1112 {
1113         int c;
1114         int state;      /* next state: XBASE, XARG, XSUB, XNULLSUB */
1115         int stype;      /* substitution type */
1116         int slen = 0;
1117         const char *p;
1118         struct tbl *vp;
1119         bool zero_ok = false;
1120
1121         if ((stype = sp[0]) == '\0')
1122                 /* Bad variable name */
1123                 return (-1);
1124
1125         xp->var = NULL;
1126
1127         /*-
1128          * ${#var}, string length (-U: characters, +U: octets) or array size
1129          * ${%var}, string width (-U: screen columns, +U: octets)
1130          */
1131         c = sp[1];
1132         if (stype == '%' && c == '\0')
1133                 return (-1);
1134         if ((stype == '#' || stype == '%') && c != '\0') {
1135                 /* Can't have any modifiers for ${#...} or ${%...} */
1136                 if (*word != CSUBST)
1137                         return (-1);
1138                 sp++;
1139                 /* Check for size of array */
1140                 if ((p = cstrchr(sp, '[')) && (p[1] == '*' || p[1] == '@') &&
1141                     p[2] == ']') {
1142                         int n = 0;
1143
1144                         if (stype != '#')
1145                                 return (-1);
1146                         vp = global(arrayname(sp));
1147                         if (vp->flag & (ISSET|ARRAY))
1148                                 zero_ok = true;
1149                         for (; vp; vp = vp->u.array)
1150                                 if (vp->flag & ISSET)
1151                                         n++;
1152                         c = n;
1153                 } else if (c == '*' || c == '@') {
1154                         if (stype != '#')
1155                                 return (-1);
1156                         c = e->loc->argc;
1157                 } else {
1158                         p = str_val(global(sp));
1159                         zero_ok = p != null;
1160                         if (stype == '#')
1161                                 c = utflen(p);
1162                         else {
1163                                 /* partial utf_mbswidth reimplementation */
1164                                 const char *s = p;
1165                                 unsigned int wc;
1166                                 size_t len;
1167                                 int cw;
1168
1169                                 c = 0;
1170                                 while (*s) {
1171                                         if (!UTFMODE || (len = utf_mbtowc(&wc,
1172                                             s)) == (size_t)-1)
1173                                                 /* not UTFMODE or not UTF-8 */
1174                                                 wc = (unsigned char)(*s++);
1175                                         else
1176                                                 /* UTFMODE and UTF-8 */
1177                                                 s += len;
1178                                         /* wc == char or wchar at s++ */
1179                                         if ((cw = utf_wcwidth(wc)) == -1) {
1180                                                 /* 646, 8859-1, 10646 C0/C1 */
1181                                                 c = -1;
1182                                                 break;
1183                                         }
1184                                         c += cw;
1185                                 }
1186                         }
1187                 }
1188                 if (Flag(FNOUNSET) && c == 0 && !zero_ok)
1189                         errorf(Tf_parm, sp);
1190                 /* unqualified variable/string substitution */
1191                 *stypep = 0;
1192                 xp->str = shf_smprintf(Tf_d, c);
1193                 return (XSUB);
1194         }
1195         if (stype == '!' && c != '\0' && *word == CSUBST) {
1196                 sp++;
1197                 if ((p = cstrchr(sp, '[')) && (p[1] == '*' || p[1] == '@') &&
1198                     p[2] == ']') {
1199                         c = '!';
1200                         stype = 0;
1201                         goto arraynames;
1202                 }
1203                 xp->var = global(sp);
1204                 xp->str = p ? shf_smprintf("%s[%lu]",
1205                     xp->var->name, arrayindex(xp->var)) : xp->var->name;
1206                 *stypep = 0;
1207                 return (XSUB);
1208         }
1209
1210         /* Check for qualifiers in word part */
1211         stype = 0;
1212         c = word[slen + 0] == CHAR ? word[slen + 1] : 0;
1213         if (c == ':') {
1214                 slen += 2;
1215                 stype = 0x80;
1216                 c = word[slen + 0] == CHAR ? word[slen + 1] : 0;
1217         }
1218         if (!stype && c == '/') {
1219                 slen += 2;
1220                 stype = c;
1221                 if (word[slen] == ADELIM && word[slen + 1] == c) {
1222                         slen += 2;
1223                         stype |= 0x80;
1224                 }
1225         } else if (stype == 0x80 && (c == ' ' || c == '0')) {
1226                 stype |= '0';
1227         } else if (ctype(c, C_SUBOP1)) {
1228                 slen += 2;
1229                 stype |= c;
1230         } else if (ksh_issubop2(c)) {
1231                 /* Note: ksh88 allows :%, :%%, etc */
1232                 slen += 2;
1233                 stype = c;
1234                 if (word[slen + 0] == CHAR && c == word[slen + 1]) {
1235                         stype |= 0x80;
1236                         slen += 2;
1237                 }
1238         } else if (c == '@') {
1239                 /* @x where x is command char */
1240                 switch (c = word[slen + 2] == CHAR ? word[slen + 3] : 0) {
1241                 case '#':
1242                 case '/':
1243                 case 'Q':
1244                         break;
1245                 default:
1246                         return (-1);
1247                 }
1248                 stype |= 0x100 | c;
1249                 slen += 4;
1250         } else if (stype)
1251                 /* : is not ok */
1252                 return (-1);
1253         if (!stype && *word != CSUBST)
1254                 return (-1);
1255
1256         c = sp[0];
1257         if (c == '*' || c == '@') {
1258                 switch (stype & 0x17F) {
1259                 /* can't assign to a vector */
1260                 case '=':
1261                 /* can't trim a vector (yet) */
1262                 case '%':
1263                 case '#':
1264                 case '?':
1265                 case '0':
1266                 case 0x100 | '/':
1267                 case '/':
1268                 case 0x100 | '#':
1269                 case 0x100 | 'Q':
1270                         return (-1);
1271                 }
1272                 if (e->loc->argc == 0) {
1273                         xp->str = null;
1274                         xp->var = global(sp);
1275                         state = c == '@' ? XNULLSUB : XSUB;
1276                 } else {
1277                         xp->u.strv = (const char **)e->loc->argv + 1;
1278                         xp->str = *xp->u.strv++;
1279                         /* $@ */
1280                         xp->split = tobool(c == '@');
1281                         state = XARG;
1282                 }
1283                 /* POSIX 2009? */
1284                 zero_ok = true;
1285         } else if ((p = cstrchr(sp, '[')) && (p[1] == '*' || p[1] == '@') &&
1286             p[2] == ']') {
1287                 XPtrV wv;
1288
1289                 switch (stype & 0x17F) {
1290                 /* can't assign to a vector */
1291                 case '=':
1292                 /* can't trim a vector (yet) */
1293                 case '%':
1294                 case '#':
1295                 case '?':
1296                 case '0':
1297                 case 0x100 | '/':
1298                 case '/':
1299                 case 0x100 | '#':
1300                 case 0x100 | 'Q':
1301                         return (-1);
1302                 }
1303                 c = 0;
1304  arraynames:
1305                 XPinit(wv, 32);
1306                 vp = global(arrayname(sp));
1307                 for (; vp; vp = vp->u.array) {
1308                         if (!(vp->flag&ISSET))
1309                                 continue;
1310                         XPput(wv, c == '!' ? shf_smprintf(Tf_lu,
1311                             arrayindex(vp)) :
1312                             str_val(vp));
1313                 }
1314                 if (XPsize(wv) == 0) {
1315                         xp->str = null;
1316                         state = p[1] == '@' ? XNULLSUB : XSUB;
1317                         XPfree(wv);
1318                 } else {
1319                         XPput(wv, 0);
1320                         xp->u.strv = (const char **)XPptrv(wv);
1321                         xp->str = *xp->u.strv++;
1322                         /* ${foo[@]} */
1323                         xp->split = tobool(p[1] == '@');
1324                         state = XARG;
1325                 }
1326         } else {
1327                 xp->var = global(sp);
1328                 xp->str = str_val(xp->var);
1329                 /* can't assign things like $! or $1 */
1330                 if ((stype & 0x17F) == '=' && !*xp->str &&
1331                     ctype(*sp, C_VAR1 | C_DIGIT))
1332                         return (-1);
1333                 state = XSUB;
1334         }
1335
1336         c = stype & 0x7F;
1337         /* test the compiler's code generator */
1338         if (((stype < 0x100) && (ksh_issubop2(c) ||
1339             (((stype & 0x80) ? *xp->str == '\0' : xp->str == null) &&
1340             (state != XARG || (ifs0 || xp->split ?
1341             (xp->u.strv[0] == NULL) : !hasnonempty(xp->u.strv))) ?
1342             c == '=' || c == '-' || c == '?' : c == '+'))) ||
1343             stype == (0x80 | '0') || stype == (0x100 | '#') ||
1344             stype == (0x100 | 'Q') || (stype & 0x7F) == '/')
1345                 /* expand word instead of variable value */
1346                 state = XBASE;
1347         if (Flag(FNOUNSET) && xp->str == null && !zero_ok &&
1348             (ksh_issubop2(c) || (state != XBASE && c != '+')))
1349                 errorf(Tf_parm, sp);
1350         *stypep = stype;
1351         *slenp = slen;
1352         return (state);
1353 }
1354
1355 /*
1356  * Run the command in $(...) and read its output.
1357  */
1358 static int
1359 comsub(Expand *xp, const char *cp, int fn)
1360 {
1361         Source *s, *sold;
1362         struct op *t;
1363         struct shf *shf;
1364         bool doalias = false;
1365         uint8_t old_utfmode = UTFMODE;
1366
1367         switch (fn) {
1368         case COMASUB:
1369                 fn = COMSUB;
1370                 if (0)
1371                         /* FALLTHROUGH */
1372         case FUNASUB:
1373                   fn = FUNSUB;
1374                 doalias = true;
1375         }
1376
1377         s = pushs(SSTRING, ATEMP);
1378         s->start = s->str = cp;
1379         sold = source;
1380         t = compile(s, true, doalias);
1381         afree(s, ATEMP);
1382         source = sold;
1383
1384         UTFMODE = old_utfmode;
1385
1386         if (t == NULL)
1387                 return (XBASE);
1388
1389         /* no waitlast() unless specifically enabled later */
1390         xp->split = false;
1391
1392         if (t->type == TCOM &&
1393             *t->args == NULL && *t->vars == NULL && t->ioact != NULL) {
1394                 /* $(<file) */
1395                 struct ioword *io = *t->ioact;
1396                 char *name;
1397
1398                 switch (io->ioflag & IOTYPE) {
1399                 case IOREAD:
1400                         shf = shf_open(name = evalstr(io->ioname, DOTILDE),
1401                                 O_RDONLY, 0, SHF_MAPHI | SHF_CLEXEC);
1402                         if (shf == NULL)
1403                                 warningf(!Flag(FTALKING), Tf_sD_s_sD_s,
1404                                     name, Tcant_open, "$(<...) input",
1405                                     cstrerror(errno));
1406                         break;
1407                 case IOHERE:
1408                         if (!herein(io, &name)) {
1409                                 xp->str = name;
1410                                 /* as $(…) requires, trim trailing newlines */
1411                                 name += strlen(name);
1412                                 while (name > xp->str && name[-1] == '\n')
1413                                         --name;
1414                                 *name = '\0';
1415                                 return (XSUB);
1416                         }
1417                         shf = NULL;
1418                         break;
1419                 default:
1420                         errorf(Tf_sD_s, T_funny_command,
1421                             snptreef(NULL, 32, Tft_R, io));
1422                 }
1423         } else if (fn == FUNSUB) {
1424                 int ofd1;
1425                 struct temp *tf = NULL;
1426
1427                 /*
1428                  * create a temporary file, open for reading and writing,
1429                  * with an shf open for reading (buffered) but yet unused
1430                  */
1431                 maketemp(ATEMP, TT_FUNSUB, &tf);
1432                 if (!tf->shf) {
1433                         errorf(Tf_temp,
1434                             Tcreate, tf->tffn, cstrerror(errno));
1435                 }
1436                 /* extract shf from temporary file, unlink and free it */
1437                 shf = tf->shf;
1438                 unlink(tf->tffn);
1439                 afree(tf, ATEMP);
1440                 /* save stdout and let it point to the tempfile */
1441                 ofd1 = savefd(1);
1442                 ksh_dup2(shf_fileno(shf), 1, false);
1443                 /*
1444                  * run tree, with output thrown into the tempfile,
1445                  * in a new function block
1446                  */
1447                 valsub(t, NULL);
1448                 subst_exstat = exstat & 0xFF;
1449                 /* rewind the tempfile and restore regular stdout */
1450                 lseek(shf_fileno(shf), (off_t)0, SEEK_SET);
1451                 restfd(1, ofd1);
1452         } else if (fn == VALSUB) {
1453                 xp->str = valsub(t, ATEMP);
1454                 subst_exstat = exstat & 0xFF;
1455                 return (XSUB);
1456         } else {
1457                 int ofd1, pv[2];
1458
1459                 openpipe(pv);
1460                 shf = shf_fdopen(pv[0], SHF_RD, NULL);
1461                 ofd1 = savefd(1);
1462                 if (pv[1] != 1) {
1463                         ksh_dup2(pv[1], 1, false);
1464                         close(pv[1]);
1465                 }
1466                 execute(t, XXCOM | XPIPEO | XFORK, NULL);
1467                 restfd(1, ofd1);
1468                 startlast();
1469                 /* waitlast() */
1470                 xp->split = true;
1471         }
1472
1473         xp->u.shf = shf;
1474         return (XCOM);
1475 }
1476
1477 /*
1478  * perform #pattern and %pattern substitution in ${}
1479  */
1480 static char *
1481 trimsub(char *str, char *pat, int how)
1482 {
1483         char *end = strnul(str);
1484         char *p, c;
1485
1486         switch (how & 0xFF) {
1487         case '#':
1488                 /* shortest match at beginning */
1489                 for (p = str; p <= end; p += utf_ptradj(p)) {
1490                         c = *p; *p = '\0';
1491                         if (gmatchx(str, pat, false)) {
1492                                 record_match(str);
1493                                 *p = c;
1494                                 return (p);
1495                         }
1496                         *p = c;
1497                 }
1498                 break;
1499         case '#'|0x80:
1500                 /* longest match at beginning */
1501                 for (p = end; p >= str; p--) {
1502                         c = *p; *p = '\0';
1503                         if (gmatchx(str, pat, false)) {
1504                                 record_match(str);
1505                                 *p = c;
1506                                 return (p);
1507                         }
1508                         *p = c;
1509                 }
1510                 break;
1511         case '%':
1512                 /* shortest match at end */
1513                 p = end;
1514                 while (p >= str) {
1515                         if (gmatchx(p, pat, false))
1516                                 goto trimsub_match;
1517                         if (UTFMODE) {
1518                                 char *op = p;
1519                                 while ((p-- > str) && ((*p & 0xC0) == 0x80))
1520                                         ;
1521                                 if ((p < str) || (p + utf_ptradj(p) != op))
1522                                         p = op - 1;
1523                         } else
1524                                 --p;
1525                 }
1526                 break;
1527         case '%'|0x80:
1528                 /* longest match at end */
1529                 for (p = str; p <= end; p++)
1530                         if (gmatchx(p, pat, false)) {
1531  trimsub_match:
1532                                 record_match(p);
1533                                 strndupx(end, str, p - str, ATEMP);
1534                                 return (end);
1535                         }
1536                 break;
1537         }
1538
1539         /* no match, return string */
1540         return (str);
1541 }
1542
1543 /*
1544  * glob
1545  * Name derived from V6's /etc/glob, the program that expanded filenames.
1546  */
1547
1548 /* XXX cp not const 'cause slashes are temporarily replaced with NULs... */
1549 static void
1550 glob(char *cp, XPtrV *wp, bool markdirs)
1551 {
1552         int oldsize = XPsize(*wp);
1553
1554         if (glob_str(cp, wp, markdirs) == 0)
1555                 XPput(*wp, debunk(cp, cp, strlen(cp) + 1));
1556         else
1557                 qsort(XPptrv(*wp) + oldsize, XPsize(*wp) - oldsize,
1558                     sizeof(void *), xstrcmp);
1559 }
1560
1561 #define GF_NONE         0
1562 #define GF_EXCHECK      BIT(0)          /* do existence check on file */
1563 #define GF_GLOBBED      BIT(1)          /* some globbing has been done */
1564 #define GF_MARKDIR      BIT(2)          /* add trailing / to directories */
1565
1566 /*
1567  * Apply file globbing to cp and store the matching files in wp. Returns
1568  * the number of matches found.
1569  */
1570 int
1571 glob_str(char *cp, XPtrV *wp, bool markdirs)
1572 {
1573         int oldsize = XPsize(*wp);
1574         XString xs;
1575         char *xp;
1576
1577         Xinit(xs, xp, 256, ATEMP);
1578         globit(&xs, &xp, cp, wp, markdirs ? GF_MARKDIR : GF_NONE);
1579         Xfree(xs, xp);
1580
1581         return (XPsize(*wp) - oldsize);
1582 }
1583
1584 static void
1585 globit(XString *xs,     /* dest string */
1586     char **xpp,         /* ptr to dest end */
1587     char *sp,           /* source path */
1588     XPtrV *wp,          /* output list */
1589     int check)          /* GF_* flags */
1590 {
1591         char *np;               /* next source component */
1592         char *xp = *xpp;
1593         char *se;
1594         char odirsep;
1595
1596         /* This to allow long expansions to be interrupted */
1597         intrcheck();
1598
1599         if (sp == NULL) {
1600                 /* end of source path */
1601                 /*
1602                  * We only need to check if the file exists if a pattern
1603                  * is followed by a non-pattern (eg, foo*x/bar; no check
1604                  * is needed for foo* since the match must exist) or if
1605                  * any patterns were expanded and the markdirs option is set.
1606                  * Symlinks make things a bit tricky...
1607                  */
1608                 if ((check & GF_EXCHECK) ||
1609                     ((check & GF_MARKDIR) && (check & GF_GLOBBED))) {
1610 #define stat_check()    (stat_done ? stat_done : (stat_done = \
1611                             stat(Xstring(*xs, xp), &statb) < 0 ? -1 : 1))
1612                         struct stat lstatb, statb;
1613                         /* -1: failed, 1 ok, 0 not yet done */
1614                         int stat_done = 0;
1615
1616                         if (mksh_lstat(Xstring(*xs, xp), &lstatb) < 0)
1617                                 return;
1618                         /*
1619                          * special case for systems which strip trailing
1620                          * slashes from regular files (eg, /etc/passwd/).
1621                          * SunOS 4.1.3 does this...
1622                          */
1623                         if ((check & GF_EXCHECK) && xp > Xstring(*xs, xp) &&
1624                             mksh_cdirsep(xp[-1]) && !S_ISDIR(lstatb.st_mode) &&
1625                             (!S_ISLNK(lstatb.st_mode) ||
1626                             stat_check() < 0 || !S_ISDIR(statb.st_mode)))
1627                                 return;
1628                         /*
1629                          * Possibly tack on a trailing / if there isn't already
1630                          * one and if the file is a directory or a symlink to a
1631                          * directory
1632                          */
1633                         if (((check & GF_MARKDIR) && (check & GF_GLOBBED)) &&
1634                             xp > Xstring(*xs, xp) && !mksh_cdirsep(xp[-1]) &&
1635                             (S_ISDIR(lstatb.st_mode) ||
1636                             (S_ISLNK(lstatb.st_mode) && stat_check() > 0 &&
1637                             S_ISDIR(statb.st_mode)))) {
1638                                 *xp++ = '/';
1639                                 *xp = '\0';
1640                         }
1641                 }
1642                 strndupx(np, Xstring(*xs, xp), Xlength(*xs, xp), ATEMP);
1643                 XPput(*wp, np);
1644                 return;
1645         }
1646
1647         if (xp > Xstring(*xs, xp))
1648                 *xp++ = '/';
1649         while (mksh_cdirsep(*sp)) {
1650                 Xcheck(*xs, xp);
1651                 *xp++ = *sp++;
1652         }
1653         np = mksh_sdirsep(sp);
1654         if (np != NULL) {
1655                 se = np;
1656                 /* don't assume '/', can be multiple kinds */
1657                 odirsep = *np;
1658                 *np++ = '\0';
1659         } else {
1660                 odirsep = '\0'; /* keep gcc quiet */
1661                 se = sp + strlen(sp);
1662         }
1663
1664
1665         /*
1666          * Check if sp needs globbing - done to avoid pattern checks for strings
1667          * containing MAGIC characters, open [s without the matching close ],
1668          * etc. (otherwise opendir() will be called which may fail because the
1669          * directory isn't readable - if no globbing is needed, only execute
1670          * permission should be required (as per POSIX)).
1671          */
1672         if (!has_globbing(sp, se)) {
1673                 XcheckN(*xs, xp, se - sp + 1);
1674                 debunk(xp, sp, Xnleft(*xs, xp));
1675                 xp += strlen(xp);
1676                 *xpp = xp;
1677                 globit(xs, xpp, np, wp, check);
1678         } else {
1679                 DIR *dirp;
1680                 struct dirent *d;
1681                 char *name;
1682                 size_t len, prefix_len;
1683
1684                 /* xp = *xpp;   copy_non_glob() may have re-alloc'd xs */
1685                 *xp = '\0';
1686                 prefix_len = Xlength(*xs, xp);
1687                 dirp = opendir(prefix_len ? Xstring(*xs, xp) : Tdot);
1688                 if (dirp == NULL)
1689                         goto Nodir;
1690                 while ((d = readdir(dirp)) != NULL) {
1691                         name = d->d_name;
1692                         if (name[0] == '.' &&
1693                             (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
1694                                 /* always ignore . and .. */
1695                                 continue;
1696                         if ((*name == '.' && *sp != '.') ||
1697                             !gmatchx(name, sp, true))
1698                                 continue;
1699
1700                         len = strlen(d->d_name) + 1;
1701                         XcheckN(*xs, xp, len);
1702                         memcpy(xp, name, len);
1703                         *xpp = xp + len - 1;
1704                         globit(xs, xpp, np, wp,
1705                                 (check & GF_MARKDIR) | GF_GLOBBED
1706                                 | (np ? GF_EXCHECK : GF_NONE));
1707                         xp = Xstring(*xs, xp) + prefix_len;
1708                 }
1709                 closedir(dirp);
1710  Nodir:
1711                 ;
1712         }
1713
1714         if (np != NULL)
1715                 *--np = odirsep;
1716 }
1717
1718 /* remove MAGIC from string */
1719 char *
1720 debunk(char *dp, const char *sp, size_t dlen)
1721 {
1722         char *d;
1723         const char *s;
1724
1725         if ((s = cstrchr(sp, MAGIC))) {
1726                 if (s - sp >= (ssize_t)dlen)
1727                         return (dp);
1728                 memmove(dp, sp, s - sp);
1729                 for (d = dp + (s - sp); *s && (d - dp < (ssize_t)dlen); s++)
1730                         if (!ISMAGIC(*s) || !(*++s & 0x80) ||
1731                             !vstrchr("*+?@! ", *s & 0x7f))
1732                                 *d++ = *s;
1733                         else {
1734                                 /* extended pattern operators: *+?@! */
1735                                 if ((*s & 0x7f) != ' ')
1736                                         *d++ = *s & 0x7f;
1737                                 if (d - dp < (ssize_t)dlen)
1738                                         *d++ = '(';
1739                         }
1740                 *d = '\0';
1741         } else if (dp != sp)
1742                 strlcpy(dp, sp, dlen);
1743         return (dp);
1744 }
1745
1746 /*
1747  * Check if p is an unquoted name, possibly followed by a / or :. If so
1748  * puts the expanded version in *dcp,dp and returns a pointer in p just
1749  * past the name, otherwise returns 0.
1750  */
1751 static const char *
1752 maybe_expand_tilde(const char *p, XString *dsp, char **dpp, bool isassign)
1753 {
1754         XString ts;
1755         char *dp = *dpp;
1756         char *tp;
1757         const char *r;
1758
1759         Xinit(ts, tp, 16, ATEMP);
1760         /* : only for DOASNTILDE form */
1761         while (p[0] == CHAR && /* not cdirsep */ p[1] != '/' &&
1762             (!isassign || p[1] != ':')) {
1763                 Xcheck(ts, tp);
1764                 *tp++ = p[1];
1765                 p += 2;
1766         }
1767         *tp = '\0';
1768         r = (p[0] == EOS || p[0] == CHAR || p[0] == CSUBST) ?
1769             do_tilde(Xstring(ts, tp)) : NULL;
1770         Xfree(ts, tp);
1771         if (r) {
1772                 while (*r) {
1773                         Xcheck(*dsp, dp);
1774                         if (ISMAGIC(*r))
1775                                 *dp++ = MAGIC;
1776                         *dp++ = *r++;
1777                 }
1778                 *dpp = dp;
1779                 r = p;
1780         }
1781         return (r);
1782 }
1783
1784 /*
1785  * tilde expansion
1786  *
1787  * based on a version by Arnold Robbins
1788  */
1789 char *
1790 do_tilde(char *cp)
1791 {
1792         char *dp = null;
1793 #ifndef MKSH_NOPWNAM
1794         bool do_simplify = true;
1795 #endif
1796
1797         if (cp[0] == '\0')
1798                 dp = str_val(global("HOME"));
1799         else if (cp[0] == '+' && cp[1] == '\0')
1800                 dp = str_val(global(TPWD));
1801         else if (ksh_isdash(cp))
1802                 dp = str_val(global(TOLDPWD));
1803 #ifndef MKSH_NOPWNAM
1804         else {
1805                 dp = homedir(cp);
1806                 do_simplify = false;
1807         }
1808 #endif
1809
1810         /* if parameters aren't set, don't expand ~ */
1811         if (dp == NULL || dp == null)
1812                 return (NULL);
1813
1814         /* simplify parameters as if cwd upon entry */
1815 #ifndef MKSH_NOPWNAM
1816         if (do_simplify)
1817 #endif
1818           {
1819                 strdupx(dp, dp, ATEMP);
1820                 simplify_path(dp);
1821         }
1822         return (dp);
1823 }
1824
1825 #ifndef MKSH_NOPWNAM
1826 /*
1827  * map userid to user's home directory.
1828  * note that 4.3's getpw adds more than 6K to the shell,
1829  * and the YP version probably adds much more.
1830  * we might consider our own version of getpwnam() to keep the size down.
1831  */
1832 static char *
1833 homedir(char *name)
1834 {
1835         struct tbl *ap;
1836
1837         ap = ktenter(&homedirs, name, hash(name));
1838         if (!(ap->flag & ISSET)) {
1839                 struct passwd *pw;
1840
1841                 pw = getpwnam(name);
1842                 if (pw == NULL)
1843                         return (NULL);
1844                 strdupx(ap->val.s, pw->pw_dir, APERM);
1845                 ap->flag |= DEFINED|ISSET|ALLOC;
1846         }
1847         return (ap->val.s);
1848 }
1849 #endif
1850
1851 static void
1852 alt_expand(XPtrV *wp, char *start, char *exp_start, char *end, int fdo)
1853 {
1854         unsigned int count = 0;
1855         char *brace_start, *brace_end, *comma = NULL;
1856         char *field_start;
1857         char *p = exp_start;
1858
1859         /* search for open brace */
1860         while ((p = strchr(p, MAGIC)) && p[1] != '{' /*}*/)
1861                 p += 2;
1862         brace_start = p;
1863
1864         /* find matching close brace, if any */
1865         if (p) {
1866                 comma = NULL;
1867                 count = 1;
1868                 p += 2;
1869                 while (*p && count) {
1870                         if (ISMAGIC(*p++)) {
1871                                 if (*p == '{' /*}*/)
1872                                         ++count;
1873                                 else if (*p == /*{*/ '}')
1874                                         --count;
1875                                 else if (*p == ',' && count == 1)
1876                                         comma = p;
1877                                 ++p;
1878                         }
1879                 }
1880         }
1881         /* no valid expansions... */
1882         if (!p || count != 0) {
1883                 /*
1884                  * Note that given a{{b,c} we do not expand anything (this is
1885                  * what AT&T ksh does. This may be changed to do the {b,c}
1886                  * expansion. }
1887                  */
1888                 if (fdo & DOGLOB)
1889                         glob(start, wp, tobool(fdo & DOMARKDIRS));
1890                 else
1891                         XPput(*wp, debunk(start, start, end - start));
1892                 return;
1893         }
1894         brace_end = p;
1895         if (!comma) {
1896                 alt_expand(wp, start, brace_end, end, fdo);
1897                 return;
1898         }
1899
1900         /* expand expression */
1901         field_start = brace_start + 2;
1902         count = 1;
1903         for (p = brace_start + 2; p != brace_end; p++) {
1904                 if (ISMAGIC(*p)) {
1905                         if (*++p == '{' /*}*/)
1906                                 ++count;
1907                         else if ((*p == /*{*/ '}' && --count == 0) ||
1908                             (*p == ',' && count == 1)) {
1909                                 char *news;
1910                                 int l1, l2, l3;
1911
1912                                 /*
1913                                  * addition safe since these operate on
1914                                  * one string (separate substrings)
1915                                  */
1916                                 l1 = brace_start - start;
1917                                 l2 = (p - 1) - field_start;
1918                                 l3 = end - brace_end;
1919                                 news = alloc(l1 + l2 + l3 + 1, ATEMP);
1920                                 memcpy(news, start, l1);
1921                                 memcpy(news + l1, field_start, l2);
1922                                 memcpy(news + l1 + l2, brace_end, l3);
1923                                 news[l1 + l2 + l3] = '\0';
1924                                 alt_expand(wp, news, news + l1,
1925                                     news + l1 + l2 + l3, fdo);
1926                                 field_start = p + 1;
1927                         }
1928                 }
1929         }
1930         return;
1931 }
1932
1933 /* helper function due to setjmp/longjmp woes */
1934 static char *
1935 valsub(struct op *t, Area *ap)
1936 {
1937         char * volatile cp = NULL;
1938         struct tbl * volatile vp = NULL;
1939
1940         newenv(E_FUNC);
1941         newblock();
1942         if (ap)
1943                 vp = local("REPLY", false);
1944         if (!kshsetjmp(e->jbuf))
1945                 execute(t, XXCOM | XERROK, NULL);
1946         if (vp)
1947                 strdupx(cp, str_val(vp), ap);
1948         quitenv(NULL);
1949
1950         return (cp);
1951 }