kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),                    \
  73         C(NEED_NOHC_VAL,        "Non-hitcount value is required for 'nohitcount'"),
  74
  75 #undef C
  76 #define C(a, b)         HIST_ERR_##a
  77
  78 enum { ERRORS };
  79
  80 #undef C
  81 #define C(a, b)         b
  82
  83 static const char *err_text[] = { ERRORS };
  84
  85 struct hist_field;
  86
  87 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88                                 struct tracing_map_elt *elt,
  89                                 struct trace_buffer *buffer,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96 #define HIST_CONST_DIGITS_MAX   21
  97 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  98
  99 enum field_op_id {
 100         FIELD_OP_NONE,
 101         FIELD_OP_PLUS,
 102         FIELD_OP_MINUS,
 103         FIELD_OP_UNARY_MINUS,
 104         FIELD_OP_DIV,
 105         FIELD_OP_MULT,
 106 };
 107
 108 enum hist_field_fn {
 109         HIST_FIELD_FN_NOP,
 110         HIST_FIELD_FN_VAR_REF,
 111         HIST_FIELD_FN_COUNTER,
 112         HIST_FIELD_FN_CONST,
 113         HIST_FIELD_FN_LOG2,
 114         HIST_FIELD_FN_BUCKET,
 115         HIST_FIELD_FN_TIMESTAMP,
 116         HIST_FIELD_FN_CPU,
 117         HIST_FIELD_FN_STRING,
 118         HIST_FIELD_FN_DYNSTRING,
 119         HIST_FIELD_FN_RELDYNSTRING,
 120         HIST_FIELD_FN_PSTRING,
 121         HIST_FIELD_FN_S64,
 122         HIST_FIELD_FN_U64,
 123         HIST_FIELD_FN_S32,
 124         HIST_FIELD_FN_U32,
 125         HIST_FIELD_FN_S16,
 126         HIST_FIELD_FN_U16,
 127         HIST_FIELD_FN_S8,
 128         HIST_FIELD_FN_U8,
 129         HIST_FIELD_FN_UMINUS,
 130         HIST_FIELD_FN_MINUS,
 131         HIST_FIELD_FN_PLUS,
 132         HIST_FIELD_FN_DIV,
 133         HIST_FIELD_FN_MULT,
 134         HIST_FIELD_FN_DIV_POWER2,
 135         HIST_FIELD_FN_DIV_NOT_POWER2,
 136         HIST_FIELD_FN_DIV_MULT_SHIFT,
 137         HIST_FIELD_FN_EXECNAME,
 138         HIST_FIELD_FN_STACK,
 139 };
 140
 141 /*
 142  * A hist_var (histogram variable) contains variable information for
 143  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 144  * flag set.  A hist_var has a variable name e.g. ts0, and is
 145  * associated with a given histogram trigger, as specified by
 146  * hist_data.  The hist_var idx is the unique index assigned to the
 147  * variable by the hist trigger's tracing_map.  The idx is what is
 148  * used to set a variable's value and, by a variable reference, to
 149  * retrieve it.
 150  */
 151 struct hist_var {
 152         char                            *name;
 153         struct hist_trigger_data        *hist_data;
 154         unsigned int                    idx;
 155 };
 156
 157 struct hist_field {
 158         struct ftrace_event_field       *field;
 159         unsigned long                   flags;
 160         unsigned long                   buckets;
 161         const char                      *type;
 162         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 163         struct hist_trigger_data        *hist_data;
 164         enum hist_field_fn              fn_num;
 165         unsigned int                    ref;
 166         unsigned int                    size;
 167         unsigned int                    offset;
 168         unsigned int                    is_signed;
 169
 170         /*
 171          * Variable fields contain variable-specific info in var.
 172          */
 173         struct hist_var                 var;
 174         enum field_op_id                operator;
 175         char                            *system;
 176         char                            *event_name;
 177
 178         /*
 179          * The name field is used for EXPR and VAR_REF fields.  VAR
 180          * fields contain the variable name in var.name.
 181          */
 182         char                            *name;
 183
 184         /*
 185          * When a histogram trigger is hit, if it has any references
 186          * to variables, the values of those variables are collected
 187          * into a var_ref_vals array by resolve_var_refs().  The
 188          * current value of each variable is read from the tracing_map
 189          * using the hist field's hist_var.idx and entered into the
 190          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 191          */
 192         unsigned int                    var_ref_idx;
 193         bool                            read_once;
 194
 195         unsigned int                    var_str_idx;
 196
 197         /* Numeric literals are represented as u64 */
 198         u64                             constant;
 199         /* Used to optimize division by constants */
 200         u64                             div_multiplier;
 201 };
 202
 203 static u64 hist_fn_call(struct hist_field *hist_field,
 204                         struct tracing_map_elt *elt,
 205                         struct trace_buffer *buffer,
 206                         struct ring_buffer_event *rbe,
 207                         void *event);
 208
 209 static u64 hist_field_const(struct hist_field *field,
 210                            struct tracing_map_elt *elt,
 211                            struct trace_buffer *buffer,
 212                            struct ring_buffer_event *rbe,
 213                            void *event)
 214 {
 215         return field->constant;
 216 }
 217
 218 static u64 hist_field_counter(struct hist_field *field,
 219                               struct tracing_map_elt *elt,
 220                               struct trace_buffer *buffer,
 221                               struct ring_buffer_event *rbe,
 222                               void *event)
 223 {
 224         return 1;
 225 }
 226
 227 static u64 hist_field_string(struct hist_field *hist_field,
 228                              struct tracing_map_elt *elt,
 229                              struct trace_buffer *buffer,
 230                              struct ring_buffer_event *rbe,
 231                              void *event)
 232 {
 233         char *addr = (char *)(event + hist_field->field->offset);
 234
 235         return (u64)(unsigned long)addr;
 236 }
 237
 238 static u64 hist_field_dynstring(struct hist_field *hist_field,
 239                                 struct tracing_map_elt *elt,
 240                                 struct trace_buffer *buffer,
 241                                 struct ring_buffer_event *rbe,
 242                                 void *event)
 243 {
 244         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 245         int str_loc = str_item & 0xffff;
 246         char *addr = (char *)(event + str_loc);
 247
 248         return (u64)(unsigned long)addr;
 249 }
 250
 251 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 252                                    struct tracing_map_elt *elt,
 253                                    struct trace_buffer *buffer,
 254                                    struct ring_buffer_event *rbe,
 255                                    void *event)
 256 {
 257         u32 *item = event + hist_field->field->offset;
 258         u32 str_item = *item;
 259         int str_loc = str_item & 0xffff;
 260         char *addr = (char *)&item[1] + str_loc;
 261
 262         return (u64)(unsigned long)addr;
 263 }
 264
 265 static u64 hist_field_pstring(struct hist_field *hist_field,
 266                               struct tracing_map_elt *elt,
 267                               struct trace_buffer *buffer,
 268                               struct ring_buffer_event *rbe,
 269                               void *event)
 270 {
 271         char **addr = (char **)(event + hist_field->field->offset);
 272
 273         return (u64)(unsigned long)*addr;
 274 }
 275
 276 static u64 hist_field_log2(struct hist_field *hist_field,
 277                            struct tracing_map_elt *elt,
 278                            struct trace_buffer *buffer,
 279                            struct ring_buffer_event *rbe,
 280                            void *event)
 281 {
 282         struct hist_field *operand = hist_field->operands[0];
 283
 284         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 285
 286         return (u64) ilog2(roundup_pow_of_two(val));
 287 }
 288
 289 static u64 hist_field_bucket(struct hist_field *hist_field,
 290                              struct tracing_map_elt *elt,
 291                              struct trace_buffer *buffer,
 292                              struct ring_buffer_event *rbe,
 293                              void *event)
 294 {
 295         struct hist_field *operand = hist_field->operands[0];
 296         unsigned long buckets = hist_field->buckets;
 297
 298         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 299
 300         if (WARN_ON_ONCE(!buckets))
 301                 return val;
 302
 303         if (val >= LONG_MAX)
 304                 val = div64_ul(val, buckets);
 305         else
 306                 val = (u64)((unsigned long)val / buckets);
 307         return val * buckets;
 308 }
 309
 310 static u64 hist_field_plus(struct hist_field *hist_field,
 311                            struct tracing_map_elt *elt,
 312                            struct trace_buffer *buffer,
 313                            struct ring_buffer_event *rbe,
 314                            void *event)
 315 {
 316         struct hist_field *operand1 = hist_field->operands[0];
 317         struct hist_field *operand2 = hist_field->operands[1];
 318
 319         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 320         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 321
 322         return val1 + val2;
 323 }
 324
 325 static u64 hist_field_minus(struct hist_field *hist_field,
 326                             struct tracing_map_elt *elt,
 327                             struct trace_buffer *buffer,
 328                             struct ring_buffer_event *rbe,
 329                             void *event)
 330 {
 331         struct hist_field *operand1 = hist_field->operands[0];
 332         struct hist_field *operand2 = hist_field->operands[1];
 333
 334         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 335         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 336
 337         return val1 - val2;
 338 }
 339
 340 static u64 hist_field_div(struct hist_field *hist_field,
 341                            struct tracing_map_elt *elt,
 342                            struct trace_buffer *buffer,
 343                            struct ring_buffer_event *rbe,
 344                            void *event)
 345 {
 346         struct hist_field *operand1 = hist_field->operands[0];
 347         struct hist_field *operand2 = hist_field->operands[1];
 348
 349         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 350         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 351
 352         /* Return -1 for the undefined case */
 353         if (!val2)
 354                 return -1;
 355
 356         /* Use shift if the divisor is a power of 2 */
 357         if (!(val2 & (val2 - 1)))
 358                 return val1 >> __ffs64(val2);
 359
 360         return div64_u64(val1, val2);
 361 }
 362
 363 static u64 div_by_power_of_two(struct hist_field *hist_field,
 364                                 struct tracing_map_elt *elt,
 365                                 struct trace_buffer *buffer,
 366                                 struct ring_buffer_event *rbe,
 367                                 void *event)
 368 {
 369         struct hist_field *operand1 = hist_field->operands[0];
 370         struct hist_field *operand2 = hist_field->operands[1];
 371
 372         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 373
 374         return val1 >> __ffs64(operand2->constant);
 375 }
 376
 377 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 378                                 struct tracing_map_elt *elt,
 379                                 struct trace_buffer *buffer,
 380                                 struct ring_buffer_event *rbe,
 381                                 void *event)
 382 {
 383         struct hist_field *operand1 = hist_field->operands[0];
 384         struct hist_field *operand2 = hist_field->operands[1];
 385
 386         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 387
 388         return div64_u64(val1, operand2->constant);
 389 }
 390
 391 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 392                                 struct tracing_map_elt *elt,
 393                                 struct trace_buffer *buffer,
 394                                 struct ring_buffer_event *rbe,
 395                                 void *event)
 396 {
 397         struct hist_field *operand1 = hist_field->operands[0];
 398         struct hist_field *operand2 = hist_field->operands[1];
 399
 400         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 401
 402         /*
 403          * If the divisor is a constant, do a multiplication and shift instead.
 404          *
 405          * Choose Z = some power of 2. If Y <= Z, then:
 406          *     X / Y = (X * (Z / Y)) / Z
 407          *
 408          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 409          *     X / Y = (X * mult) / Z
 410          *
 411          * The division by Z can be replaced by a shift since Z is a power of 2:
 412          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 413          *
 414          * As long, as X < Z the results will not be off by more than 1.
 415          */
 416         if (val1 < (1 << HIST_DIV_SHIFT)) {
 417                 u64 mult = operand2->div_multiplier;
 418
 419                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 420         }
 421
 422         return div64_u64(val1, operand2->constant);
 423 }
 424
 425 static u64 hist_field_mult(struct hist_field *hist_field,
 426                            struct tracing_map_elt *elt,
 427                            struct trace_buffer *buffer,
 428                            struct ring_buffer_event *rbe,
 429                            void *event)
 430 {
 431         struct hist_field *operand1 = hist_field->operands[0];
 432         struct hist_field *operand2 = hist_field->operands[1];
 433
 434         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 435         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 436
 437         return val1 * val2;
 438 }
 439
 440 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 441                                   struct tracing_map_elt *elt,
 442                                   struct trace_buffer *buffer,
 443                                   struct ring_buffer_event *rbe,
 444                                   void *event)
 445 {
 446         struct hist_field *operand = hist_field->operands[0];
 447
 448         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 449         u64 val = (u64)-sval;
 450
 451         return val;
 452 }
 453
 454 #define DEFINE_HIST_FIELD_FN(type)                                      \
 455         static u64 hist_field_##type(struct hist_field *hist_field,     \
 456                                      struct tracing_map_elt *elt,       \
 457                                      struct trace_buffer *buffer,       \
 458                                      struct ring_buffer_event *rbe,     \
 459                                      void *event)                       \
 460 {                                                                       \
 461         type *addr = (type *)(event + hist_field->field->offset);       \
 462                                                                         \
 463         return (u64)(unsigned long)*addr;                               \
 464 }
 465
 466 DEFINE_HIST_FIELD_FN(s64);
 467 DEFINE_HIST_FIELD_FN(u64);
 468 DEFINE_HIST_FIELD_FN(s32);
 469 DEFINE_HIST_FIELD_FN(u32);
 470 DEFINE_HIST_FIELD_FN(s16);
 471 DEFINE_HIST_FIELD_FN(u16);
 472 DEFINE_HIST_FIELD_FN(s8);
 473 DEFINE_HIST_FIELD_FN(u8);
 474
 475 #define for_each_hist_field(i, hist_data)       \
 476         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 477
 478 #define for_each_hist_val_field(i, hist_data)   \
 479         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 480
 481 #define for_each_hist_key_field(i, hist_data)   \
 482         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 483
 484 #define HITCOUNT_IDX            0
 485 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 486
 487 enum hist_field_flags {
 488         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 489         HIST_FIELD_FL_KEY               = 1 << 1,
 490         HIST_FIELD_FL_STRING            = 1 << 2,
 491         HIST_FIELD_FL_HEX               = 1 << 3,
 492         HIST_FIELD_FL_SYM               = 1 << 4,
 493         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 494         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 495         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 496         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 497         HIST_FIELD_FL_LOG2              = 1 << 9,
 498         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 499         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 500         HIST_FIELD_FL_VAR               = 1 << 12,
 501         HIST_FIELD_FL_EXPR              = 1 << 13,
 502         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 503         HIST_FIELD_FL_CPU               = 1 << 15,
 504         HIST_FIELD_FL_ALIAS             = 1 << 16,
 505         HIST_FIELD_FL_BUCKET            = 1 << 17,
 506         HIST_FIELD_FL_CONST             = 1 << 18,
 507         HIST_FIELD_FL_PERCENT           = 1 << 19,
 508         HIST_FIELD_FL_GRAPH             = 1 << 20,
 509 };
 510
 511 struct var_defs {
 512         unsigned int    n_vars;
 513         char            *name[TRACING_MAP_VARS_MAX];
 514         char            *expr[TRACING_MAP_VARS_MAX];
 515 };
 516
 517 struct hist_trigger_attrs {
 518         char            *keys_str;
 519         char            *vals_str;
 520         char            *sort_key_str;
 521         char            *name;
 522         char            *clock;
 523         bool            pause;
 524         bool            cont;
 525         bool            clear;
 526         bool            ts_in_usecs;
 527         bool            no_hitcount;
 528         unsigned int    map_bits;
 529
 530         char            *assignment_str[TRACING_MAP_VARS_MAX];
 531         unsigned int    n_assignments;
 532
 533         char            *action_str[HIST_ACTIONS_MAX];
 534         unsigned int    n_actions;
 535
 536         struct var_defs var_defs;
 537 };
 538
 539 struct field_var {
 540         struct hist_field       *var;
 541         struct hist_field       *val;
 542 };
 543
 544 struct field_var_hist {
 545         struct hist_trigger_data        *hist_data;
 546         char                            *cmd;
 547 };
 548
 549 struct hist_trigger_data {
 550         struct hist_field               *fields[HIST_FIELDS_MAX];
 551         unsigned int                    n_vals;
 552         unsigned int                    n_keys;
 553         unsigned int                    n_fields;
 554         unsigned int                    n_vars;
 555         unsigned int                    n_var_str;
 556         unsigned int                    key_size;
 557         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 558         unsigned int                    n_sort_keys;
 559         struct trace_event_file         *event_file;
 560         struct hist_trigger_attrs       *attrs;
 561         struct tracing_map              *map;
 562         bool                            enable_timestamps;
 563         bool                            remove;
 564         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 565         unsigned int                    n_var_refs;
 566
 567         struct action_data              *actions[HIST_ACTIONS_MAX];
 568         unsigned int                    n_actions;
 569
 570         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 571         unsigned int                    n_field_vars;
 572         unsigned int                    n_field_var_str;
 573         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 574         unsigned int                    n_field_var_hists;
 575
 576         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 577         unsigned int                    n_save_vars;
 578         unsigned int                    n_save_var_str;
 579 };
 580
 581 struct action_data;
 582
 583 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 584                              struct tracing_map_elt *elt,
 585                              struct trace_buffer *buffer, void *rec,
 586                              struct ring_buffer_event *rbe, void *key,
 587                              struct action_data *data, u64 *var_ref_vals);
 588
 589 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 590
 591 enum handler_id {
 592         HANDLER_ONMATCH = 1,
 593         HANDLER_ONMAX,
 594         HANDLER_ONCHANGE,
 595 };
 596
 597 enum action_id {
 598         ACTION_SAVE = 1,
 599         ACTION_TRACE,
 600         ACTION_SNAPSHOT,
 601 };
 602
 603 struct action_data {
 604         enum handler_id         handler;
 605         enum action_id          action;
 606         char                    *action_name;
 607         action_fn_t             fn;
 608
 609         unsigned int            n_params;
 610         char                    *params[SYNTH_FIELDS_MAX];
 611
 612         /*
 613          * When a histogram trigger is hit, the values of any
 614          * references to variables, including variables being passed
 615          * as parameters to synthetic events, are collected into a
 616          * var_ref_vals array.  This var_ref_idx array is an array of
 617          * indices into the var_ref_vals array, one for each synthetic
 618          * event param, and is passed to the synthetic event
 619          * invocation.
 620          */
 621         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 622         struct synth_event      *synth_event;
 623         bool                    use_trace_keyword;
 624         char                    *synth_event_name;
 625
 626         union {
 627                 struct {
 628                         char                    *event;
 629                         char                    *event_system;
 630                 } match_data;
 631
 632                 struct {
 633                         /*
 634                          * var_str contains the $-unstripped variable
 635                          * name referenced by var_ref, and used when
 636                          * printing the action.  Because var_ref
 637                          * creation is deferred to create_actions(),
 638                          * we need a per-action way to save it until
 639                          * then, thus var_str.
 640                          */
 641                         char                    *var_str;
 642
 643                         /*
 644                          * var_ref refers to the variable being
 645                          * tracked e.g onmax($var).
 646                          */
 647                         struct hist_field       *var_ref;
 648
 649                         /*
 650                          * track_var contains the 'invisible' tracking
 651                          * variable created to keep the current
 652                          * e.g. max value.
 653                          */
 654                         struct hist_field       *track_var;
 655
 656                         check_track_val_fn_t    check_val;
 657                         action_fn_t             save_data;
 658                 } track_data;
 659         };
 660 };
 661
 662 struct track_data {
 663         u64                             track_val;
 664         bool                            updated;
 665
 666         unsigned int                    key_len;
 667         void                            *key;
 668         struct tracing_map_elt          elt;
 669
 670         struct action_data              *action_data;
 671         struct hist_trigger_data        *hist_data;
 672 };
 673
 674 struct hist_elt_data {
 675         char *comm;
 676         u64 *var_ref_vals;
 677         char **field_var_str;
 678         int n_field_var_str;
 679 };
 680
 681 struct snapshot_context {
 682         struct tracing_map_elt  *elt;
 683         void                    *key;
 684 };
 685
 686 /*
 687  * Returns the specific division function to use if the divisor
 688  * is constant. This avoids extra branches when the trigger is hit.
 689  */
 690 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 691 {
 692         u64 div = divisor->constant;
 693
 694         if (!(div & (div - 1)))
 695                 return HIST_FIELD_FN_DIV_POWER2;
 696
 697         /* If the divisor is too large, do a regular division */
 698         if (div > (1 << HIST_DIV_SHIFT))
 699                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 700
 701         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 702         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 703 }
 704
 705 static void track_data_free(struct track_data *track_data)
 706 {
 707         struct hist_elt_data *elt_data;
 708
 709         if (!track_data)
 710                 return;
 711
 712         kfree(track_data->key);
 713
 714         elt_data = track_data->elt.private_data;
 715         if (elt_data) {
 716                 kfree(elt_data->comm);
 717                 kfree(elt_data);
 718         }
 719
 720         kfree(track_data);
 721 }
 722
 723 static struct track_data *track_data_alloc(unsigned int key_len,
 724                                            struct action_data *action_data,
 725                                            struct hist_trigger_data *hist_data)
 726 {
 727         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 728         struct hist_elt_data *elt_data;
 729
 730         if (!data)
 731                 return ERR_PTR(-ENOMEM);
 732
 733         data->key = kzalloc(key_len, GFP_KERNEL);
 734         if (!data->key) {
 735                 track_data_free(data);
 736                 return ERR_PTR(-ENOMEM);
 737         }
 738
 739         data->key_len = key_len;
 740         data->action_data = action_data;
 741         data->hist_data = hist_data;
 742
 743         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 744         if (!elt_data) {
 745                 track_data_free(data);
 746                 return ERR_PTR(-ENOMEM);
 747         }
 748
 749         data->elt.private_data = elt_data;
 750
 751         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 752         if (!elt_data->comm) {
 753                 track_data_free(data);
 754                 return ERR_PTR(-ENOMEM);
 755         }
 756
 757         return data;
 758 }
 759
 760 #define HIST_PREFIX "hist:"
 761
 762 static char *last_cmd;
 763 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 764
 765 static int errpos(char *str)
 766 {
 767         if (!str || !last_cmd)
 768                 return 0;
 769
 770         return err_pos(last_cmd, str);
 771 }
 772
 773 static void last_cmd_set(struct trace_event_file *file, char *str)
 774 {
 775         const char *system = NULL, *name = NULL;
 776         struct trace_event_call *call;
 777         int len;
 778
 779         if (!str)
 780                 return;
 781
 782         /* sizeof() contains the nul byte */
 783         len = sizeof(HIST_PREFIX) + strlen(str);
 784         kfree(last_cmd);
 785         last_cmd = kzalloc(len, GFP_KERNEL);
 786         if (!last_cmd)
 787                 return;
 788
 789         strcpy(last_cmd, HIST_PREFIX);
 790         /* Again, sizeof() contains the nul byte */
 791         len -= sizeof(HIST_PREFIX);
 792         strncat(last_cmd, str, len);
 793
 794         if (file) {
 795                 call = file->event_call;
 796                 system = call->class->system;
 797                 if (system) {
 798                         name = trace_event_name(call);
 799                         if (!name)
 800                                 system = NULL;
 801                 }
 802         }
 803
 804         if (system)
 805                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 806 }
 807
 808 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 809 {
 810         if (!last_cmd)
 811                 return;
 812
 813         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 814                         err_type, err_pos);
 815 }
 816
 817 static void hist_err_clear(void)
 818 {
 819         if (last_cmd)
 820                 last_cmd[0] = '\0';
 821         last_cmd_loc[0] = '\0';
 822 }
 823
 824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 825                                     unsigned int *var_ref_idx);
 826
 827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 828                                unsigned int *var_ref_idx)
 829 {
 830         struct tracepoint *tp = event->tp;
 831
 832         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 833                 struct tracepoint_func *probe_func_ptr;
 834                 synth_probe_func_t probe_func;
 835                 void *__data;
 836
 837                 if (!(cpu_online(raw_smp_processor_id())))
 838                         return;
 839
 840                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 841                 if (probe_func_ptr) {
 842                         do {
 843                                 probe_func = probe_func_ptr->func;
 844                                 __data = probe_func_ptr->data;
 845                                 probe_func(__data, var_ref_vals, var_ref_idx);
 846                         } while ((++probe_func_ptr)->func);
 847                 }
 848         }
 849 }
 850
 851 static void action_trace(struct hist_trigger_data *hist_data,
 852                          struct tracing_map_elt *elt,
 853                          struct trace_buffer *buffer, void *rec,
 854                          struct ring_buffer_event *rbe, void *key,
 855                          struct action_data *data, u64 *var_ref_vals)
 856 {
 857         struct synth_event *event = data->synth_event;
 858
 859         trace_synth(event, var_ref_vals, data->var_ref_idx);
 860 }
 861
 862 struct hist_var_data {
 863         struct list_head list;
 864         struct hist_trigger_data *hist_data;
 865 };
 866
 867 static u64 hist_field_timestamp(struct hist_field *hist_field,
 868                                 struct tracing_map_elt *elt,
 869                                 struct trace_buffer *buffer,
 870                                 struct ring_buffer_event *rbe,
 871                                 void *event)
 872 {
 873         struct hist_trigger_data *hist_data = hist_field->hist_data;
 874         struct trace_array *tr = hist_data->event_file->tr;
 875
 876         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 877
 878         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 879                 ts = ns2usecs(ts);
 880
 881         return ts;
 882 }
 883
 884 static u64 hist_field_cpu(struct hist_field *hist_field,
 885                           struct tracing_map_elt *elt,
 886                           struct trace_buffer *buffer,
 887                           struct ring_buffer_event *rbe,
 888                           void *event)
 889 {
 890         int cpu = smp_processor_id();
 891
 892         return cpu;
 893 }
 894
 895 /**
 896  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 897  * @hist_field: The VAR_REF field to check
 898  * @var_data: The hist trigger that owns the variable
 899  * @var_idx: The trigger variable identifier
 900  *
 901  * Check the given VAR_REF field to see whether or not it references
 902  * the given variable associated with the given trigger.
 903  *
 904  * Return: The VAR_REF field if it does reference the variable, NULL if not
 905  */
 906 static struct hist_field *
 907 check_field_for_var_ref(struct hist_field *hist_field,
 908                         struct hist_trigger_data *var_data,
 909                         unsigned int var_idx)
 910 {
 911         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 912
 913         if (hist_field && hist_field->var.idx == var_idx &&
 914             hist_field->var.hist_data == var_data)
 915                 return hist_field;
 916
 917         return NULL;
 918 }
 919
 920 /**
 921  * find_var_ref - Check if a trigger has a reference to a trigger variable
 922  * @hist_data: The hist trigger that might have a reference to the variable
 923  * @var_data: The hist trigger that owns the variable
 924  * @var_idx: The trigger variable identifier
 925  *
 926  * Check the list of var_refs[] on the first hist trigger to see
 927  * whether any of them are references to the variable on the second
 928  * trigger.
 929  *
 930  * Return: The VAR_REF field referencing the variable if so, NULL if not
 931  */
 932 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 933                                        struct hist_trigger_data *var_data,
 934                                        unsigned int var_idx)
 935 {
 936         struct hist_field *hist_field;
 937         unsigned int i;
 938
 939         for (i = 0; i < hist_data->n_var_refs; i++) {
 940                 hist_field = hist_data->var_refs[i];
 941                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 942                         return hist_field;
 943         }
 944
 945         return NULL;
 946 }
 947
 948 /**
 949  * find_any_var_ref - Check if there is a reference to a given trigger variable
 950  * @hist_data: The hist trigger
 951  * @var_idx: The trigger variable identifier
 952  *
 953  * Check to see whether the given variable is currently referenced by
 954  * any other trigger.
 955  *
 956  * The trigger the variable is defined on is explicitly excluded - the
 957  * assumption being that a self-reference doesn't prevent a trigger
 958  * from being removed.
 959  *
 960  * Return: The VAR_REF field referencing the variable if so, NULL if not
 961  */
 962 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 963                                            unsigned int var_idx)
 964 {
 965         struct trace_array *tr = hist_data->event_file->tr;
 966         struct hist_field *found = NULL;
 967         struct hist_var_data *var_data;
 968
 969         list_for_each_entry(var_data, &tr->hist_vars, list) {
 970                 if (var_data->hist_data == hist_data)
 971                         continue;
 972                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 973                 if (found)
 974                         break;
 975         }
 976
 977         return found;
 978 }
 979
 980 /**
 981  * check_var_refs - Check if there is a reference to any of trigger's variables
 982  * @hist_data: The hist trigger
 983  *
 984  * A trigger can define one or more variables.  If any one of them is
 985  * currently referenced by any other trigger, this function will
 986  * determine that.
 987  *
 988  * Typically used to determine whether or not a trigger can be removed
 989  * - if there are any references to a trigger's variables, it cannot.
 990  *
 991  * Return: True if there is a reference to any of trigger's variables
 992  */
 993 static bool check_var_refs(struct hist_trigger_data *hist_data)
 994 {
 995         struct hist_field *field;
 996         bool found = false;
 997         int i;
 998
 999         for_each_hist_field(i, hist_data) {
1000                 field = hist_data->fields[i];
1001                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1002                         if (find_any_var_ref(hist_data, field->var.idx)) {
1003                                 found = true;
1004                                 break;
1005                         }
1006                 }
1007         }
1008
1009         return found;
1010 }
1011
1012 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1013 {
1014         struct trace_array *tr = hist_data->event_file->tr;
1015         struct hist_var_data *var_data, *found = NULL;
1016
1017         list_for_each_entry(var_data, &tr->hist_vars, list) {
1018                 if (var_data->hist_data == hist_data) {
1019                         found = var_data;
1020                         break;
1021                 }
1022         }
1023
1024         return found;
1025 }
1026
1027 static bool field_has_hist_vars(struct hist_field *hist_field,
1028                                 unsigned int level)
1029 {
1030         int i;
1031
1032         if (level > 3)
1033                 return false;
1034
1035         if (!hist_field)
1036                 return false;
1037
1038         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1039             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1040                 return true;
1041
1042         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1043                 struct hist_field *operand;
1044
1045                 operand = hist_field->operands[i];
1046                 if (field_has_hist_vars(operand, level + 1))
1047                         return true;
1048         }
1049
1050         return false;
1051 }
1052
1053 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1054 {
1055         struct hist_field *hist_field;
1056         int i;
1057
1058         for_each_hist_field(i, hist_data) {
1059                 hist_field = hist_data->fields[i];
1060                 if (field_has_hist_vars(hist_field, 0))
1061                         return true;
1062         }
1063
1064         return false;
1065 }
1066
1067 static int save_hist_vars(struct hist_trigger_data *hist_data)
1068 {
1069         struct trace_array *tr = hist_data->event_file->tr;
1070         struct hist_var_data *var_data;
1071
1072         var_data = find_hist_vars(hist_data);
1073         if (var_data)
1074                 return 0;
1075
1076         if (tracing_check_open_get_tr(tr))
1077                 return -ENODEV;
1078
1079         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1080         if (!var_data) {
1081                 trace_array_put(tr);
1082                 return -ENOMEM;
1083         }
1084
1085         var_data->hist_data = hist_data;
1086         list_add(&var_data->list, &tr->hist_vars);
1087
1088         return 0;
1089 }
1090
1091 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1092 {
1093         struct trace_array *tr = hist_data->event_file->tr;
1094         struct hist_var_data *var_data;
1095
1096         var_data = find_hist_vars(hist_data);
1097         if (!var_data)
1098                 return;
1099
1100         if (WARN_ON(check_var_refs(hist_data)))
1101                 return;
1102
1103         list_del(&var_data->list);
1104
1105         kfree(var_data);
1106
1107         trace_array_put(tr);
1108 }
1109
1110 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1111                                          const char *var_name)
1112 {
1113         struct hist_field *hist_field, *found = NULL;
1114         int i;
1115
1116         for_each_hist_field(i, hist_data) {
1117                 hist_field = hist_data->fields[i];
1118                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1119                     strcmp(hist_field->var.name, var_name) == 0) {
1120                         found = hist_field;
1121                         break;
1122                 }
1123         }
1124
1125         return found;
1126 }
1127
1128 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1129                                    struct trace_event_file *file,
1130                                    const char *var_name)
1131 {
1132         struct hist_trigger_data *test_data;
1133         struct event_trigger_data *test;
1134         struct hist_field *hist_field;
1135
1136         lockdep_assert_held(&event_mutex);
1137
1138         hist_field = find_var_field(hist_data, var_name);
1139         if (hist_field)
1140                 return hist_field;
1141
1142         list_for_each_entry(test, &file->triggers, list) {
1143                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1144                         test_data = test->private_data;
1145                         hist_field = find_var_field(test_data, var_name);
1146                         if (hist_field)
1147                                 return hist_field;
1148                 }
1149         }
1150
1151         return NULL;
1152 }
1153
1154 static struct trace_event_file *find_var_file(struct trace_array *tr,
1155                                               char *system,
1156                                               char *event_name,
1157                                               char *var_name)
1158 {
1159         struct hist_trigger_data *var_hist_data;
1160         struct hist_var_data *var_data;
1161         struct trace_event_file *file, *found = NULL;
1162
1163         if (system)
1164                 return find_event_file(tr, system, event_name);
1165
1166         list_for_each_entry(var_data, &tr->hist_vars, list) {
1167                 var_hist_data = var_data->hist_data;
1168                 file = var_hist_data->event_file;
1169                 if (file == found)
1170                         continue;
1171
1172                 if (find_var_field(var_hist_data, var_name)) {
1173                         if (found) {
1174                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1175                                 return NULL;
1176                         }
1177
1178                         found = file;
1179                 }
1180         }
1181
1182         return found;
1183 }
1184
1185 static struct hist_field *find_file_var(struct trace_event_file *file,
1186                                         const char *var_name)
1187 {
1188         struct hist_trigger_data *test_data;
1189         struct event_trigger_data *test;
1190         struct hist_field *hist_field;
1191
1192         lockdep_assert_held(&event_mutex);
1193
1194         list_for_each_entry(test, &file->triggers, list) {
1195                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1196                         test_data = test->private_data;
1197                         hist_field = find_var_field(test_data, var_name);
1198                         if (hist_field)
1199                                 return hist_field;
1200                 }
1201         }
1202
1203         return NULL;
1204 }
1205
1206 static struct hist_field *
1207 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1208 {
1209         struct trace_array *tr = hist_data->event_file->tr;
1210         struct hist_field *hist_field, *found = NULL;
1211         struct trace_event_file *file;
1212         unsigned int i;
1213
1214         for (i = 0; i < hist_data->n_actions; i++) {
1215                 struct action_data *data = hist_data->actions[i];
1216
1217                 if (data->handler == HANDLER_ONMATCH) {
1218                         char *system = data->match_data.event_system;
1219                         char *event_name = data->match_data.event;
1220
1221                         file = find_var_file(tr, system, event_name, var_name);
1222                         if (!file)
1223                                 continue;
1224                         hist_field = find_file_var(file, var_name);
1225                         if (hist_field) {
1226                                 if (found) {
1227                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1228                                                  errpos(var_name));
1229                                         return ERR_PTR(-EINVAL);
1230                                 }
1231
1232                                 found = hist_field;
1233                         }
1234                 }
1235         }
1236         return found;
1237 }
1238
1239 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1240                                          char *system,
1241                                          char *event_name,
1242                                          char *var_name)
1243 {
1244         struct trace_array *tr = hist_data->event_file->tr;
1245         struct hist_field *hist_field = NULL;
1246         struct trace_event_file *file;
1247
1248         if (!system || !event_name) {
1249                 hist_field = find_match_var(hist_data, var_name);
1250                 if (IS_ERR(hist_field))
1251                         return NULL;
1252                 if (hist_field)
1253                         return hist_field;
1254         }
1255
1256         file = find_var_file(tr, system, event_name, var_name);
1257         if (!file)
1258                 return NULL;
1259
1260         hist_field = find_file_var(file, var_name);
1261
1262         return hist_field;
1263 }
1264
1265 static u64 hist_field_var_ref(struct hist_field *hist_field,
1266                               struct tracing_map_elt *elt,
1267                               struct trace_buffer *buffer,
1268                               struct ring_buffer_event *rbe,
1269                               void *event)
1270 {
1271         struct hist_elt_data *elt_data;
1272         u64 var_val = 0;
1273
1274         if (WARN_ON_ONCE(!elt))
1275                 return var_val;
1276
1277         elt_data = elt->private_data;
1278         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1279
1280         return var_val;
1281 }
1282
1283 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1284                              u64 *var_ref_vals, bool self)
1285 {
1286         struct hist_trigger_data *var_data;
1287         struct tracing_map_elt *var_elt;
1288         struct hist_field *hist_field;
1289         unsigned int i, var_idx;
1290         bool resolved = true;
1291         u64 var_val = 0;
1292
1293         for (i = 0; i < hist_data->n_var_refs; i++) {
1294                 hist_field = hist_data->var_refs[i];
1295                 var_idx = hist_field->var.idx;
1296                 var_data = hist_field->var.hist_data;
1297
1298                 if (var_data == NULL) {
1299                         resolved = false;
1300                         break;
1301                 }
1302
1303                 if ((self && var_data != hist_data) ||
1304                     (!self && var_data == hist_data))
1305                         continue;
1306
1307                 var_elt = tracing_map_lookup(var_data->map, key);
1308                 if (!var_elt) {
1309                         resolved = false;
1310                         break;
1311                 }
1312
1313                 if (!tracing_map_var_set(var_elt, var_idx)) {
1314                         resolved = false;
1315                         break;
1316                 }
1317
1318                 if (self || !hist_field->read_once)
1319                         var_val = tracing_map_read_var(var_elt, var_idx);
1320                 else
1321                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1322
1323                 var_ref_vals[i] = var_val;
1324         }
1325
1326         return resolved;
1327 }
1328
1329 static const char *hist_field_name(struct hist_field *field,
1330                                    unsigned int level)
1331 {
1332         const char *field_name = "";
1333
1334         if (level > 1)
1335                 return field_name;
1336
1337         if (field->field)
1338                 field_name = field->field->name;
1339         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1340                  field->flags & HIST_FIELD_FL_ALIAS ||
1341                  field->flags & HIST_FIELD_FL_BUCKET)
1342                 field_name = hist_field_name(field->operands[0], ++level);
1343         else if (field->flags & HIST_FIELD_FL_CPU)
1344                 field_name = "common_cpu";
1345         else if (field->flags & HIST_FIELD_FL_EXPR ||
1346                  field->flags & HIST_FIELD_FL_VAR_REF) {
1347                 if (field->system) {
1348                         static char full_name[MAX_FILTER_STR_VAL];
1349
1350                         strcat(full_name, field->system);
1351                         strcat(full_name, ".");
1352                         strcat(full_name, field->event_name);
1353                         strcat(full_name, ".");
1354                         strcat(full_name, field->name);
1355                         field_name = full_name;
1356                 } else
1357                         field_name = field->name;
1358         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1359                 field_name = "common_timestamp";
1360         else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1361                 if (field->field)
1362                         field_name = field->field->name;
1363                 else
1364                         field_name = "stacktrace";
1365         } else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1366                 field_name = "hitcount";
1367
1368         if (field_name == NULL)
1369                 field_name = "";
1370
1371         return field_name;
1372 }
1373
1374 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1375 {
1376         switch (field_size) {
1377         case 8:
1378                 if (field_is_signed)
1379                         return HIST_FIELD_FN_S64;
1380                 else
1381                         return HIST_FIELD_FN_U64;
1382         case 4:
1383                 if (field_is_signed)
1384                         return HIST_FIELD_FN_S32;
1385                 else
1386                         return HIST_FIELD_FN_U32;
1387         case 2:
1388                 if (field_is_signed)
1389                         return HIST_FIELD_FN_S16;
1390                 else
1391                         return HIST_FIELD_FN_U16;
1392         case 1:
1393                 if (field_is_signed)
1394                         return HIST_FIELD_FN_S8;
1395                 else
1396                         return HIST_FIELD_FN_U8;
1397         }
1398
1399         return HIST_FIELD_FN_NOP;
1400 }
1401
1402 static int parse_map_size(char *str)
1403 {
1404         unsigned long size, map_bits;
1405         int ret;
1406
1407         ret = kstrtoul(str, 0, &size);
1408         if (ret)
1409                 goto out;
1410
1411         map_bits = ilog2(roundup_pow_of_two(size));
1412         if (map_bits < TRACING_MAP_BITS_MIN ||
1413             map_bits > TRACING_MAP_BITS_MAX)
1414                 ret = -EINVAL;
1415         else
1416                 ret = map_bits;
1417  out:
1418         return ret;
1419 }
1420
1421 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1422 {
1423         unsigned int i;
1424
1425         if (!attrs)
1426                 return;
1427
1428         for (i = 0; i < attrs->n_assignments; i++)
1429                 kfree(attrs->assignment_str[i]);
1430
1431         for (i = 0; i < attrs->n_actions; i++)
1432                 kfree(attrs->action_str[i]);
1433
1434         kfree(attrs->name);
1435         kfree(attrs->sort_key_str);
1436         kfree(attrs->keys_str);
1437         kfree(attrs->vals_str);
1438         kfree(attrs->clock);
1439         kfree(attrs);
1440 }
1441
1442 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1443 {
1444         int ret = -EINVAL;
1445
1446         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1447                 return ret;
1448
1449         if ((str_has_prefix(str, "onmatch(")) ||
1450             (str_has_prefix(str, "onmax(")) ||
1451             (str_has_prefix(str, "onchange("))) {
1452                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1453                 if (!attrs->action_str[attrs->n_actions]) {
1454                         ret = -ENOMEM;
1455                         return ret;
1456                 }
1457                 attrs->n_actions++;
1458                 ret = 0;
1459         }
1460         return ret;
1461 }
1462
1463 static int parse_assignment(struct trace_array *tr,
1464                             char *str, struct hist_trigger_attrs *attrs)
1465 {
1466         int len, ret = 0;
1467
1468         if ((len = str_has_prefix(str, "key=")) ||
1469             (len = str_has_prefix(str, "keys="))) {
1470                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1471                 if (!attrs->keys_str) {
1472                         ret = -ENOMEM;
1473                         goto out;
1474                 }
1475         } else if ((len = str_has_prefix(str, "val=")) ||
1476                    (len = str_has_prefix(str, "vals=")) ||
1477                    (len = str_has_prefix(str, "values="))) {
1478                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1479                 if (!attrs->vals_str) {
1480                         ret = -ENOMEM;
1481                         goto out;
1482                 }
1483         } else if ((len = str_has_prefix(str, "sort="))) {
1484                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1485                 if (!attrs->sort_key_str) {
1486                         ret = -ENOMEM;
1487                         goto out;
1488                 }
1489         } else if (str_has_prefix(str, "name=")) {
1490                 attrs->name = kstrdup(str, GFP_KERNEL);
1491                 if (!attrs->name) {
1492                         ret = -ENOMEM;
1493                         goto out;
1494                 }
1495         } else if ((len = str_has_prefix(str, "clock="))) {
1496                 str += len;
1497
1498                 str = strstrip(str);
1499                 attrs->clock = kstrdup(str, GFP_KERNEL);
1500                 if (!attrs->clock) {
1501                         ret = -ENOMEM;
1502                         goto out;
1503                 }
1504         } else if ((len = str_has_prefix(str, "size="))) {
1505                 int map_bits = parse_map_size(str + len);
1506
1507                 if (map_bits < 0) {
1508                         ret = map_bits;
1509                         goto out;
1510                 }
1511                 attrs->map_bits = map_bits;
1512         } else {
1513                 char *assignment;
1514
1515                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1516                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1517                         ret = -EINVAL;
1518                         goto out;
1519                 }
1520
1521                 assignment = kstrdup(str, GFP_KERNEL);
1522                 if (!assignment) {
1523                         ret = -ENOMEM;
1524                         goto out;
1525                 }
1526
1527                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1528         }
1529  out:
1530         return ret;
1531 }
1532
1533 static struct hist_trigger_attrs *
1534 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1535 {
1536         struct hist_trigger_attrs *attrs;
1537         int ret = 0;
1538
1539         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1540         if (!attrs)
1541                 return ERR_PTR(-ENOMEM);
1542
1543         while (trigger_str) {
1544                 char *str = strsep(&trigger_str, ":");
1545                 char *rhs;
1546
1547                 rhs = strchr(str, '=');
1548                 if (rhs) {
1549                         if (!strlen(++rhs)) {
1550                                 ret = -EINVAL;
1551                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1552                                 goto free;
1553                         }
1554                         ret = parse_assignment(tr, str, attrs);
1555                         if (ret)
1556                                 goto free;
1557                 } else if (strcmp(str, "nohitcount") == 0 ||
1558                            strcmp(str, "NOHC") == 0)
1559                         attrs->no_hitcount = true;
1560                 else if (strcmp(str, "pause") == 0)
1561                         attrs->pause = true;
1562                 else if ((strcmp(str, "cont") == 0) ||
1563                          (strcmp(str, "continue") == 0))
1564                         attrs->cont = true;
1565                 else if (strcmp(str, "clear") == 0)
1566                         attrs->clear = true;
1567                 else {
1568                         ret = parse_action(str, attrs);
1569                         if (ret)
1570                                 goto free;
1571                 }
1572         }
1573
1574         if (!attrs->keys_str) {
1575                 ret = -EINVAL;
1576                 goto free;
1577         }
1578
1579         if (!attrs->clock) {
1580                 attrs->clock = kstrdup("global", GFP_KERNEL);
1581                 if (!attrs->clock) {
1582                         ret = -ENOMEM;
1583                         goto free;
1584                 }
1585         }
1586
1587         return attrs;
1588  free:
1589         destroy_hist_trigger_attrs(attrs);
1590
1591         return ERR_PTR(ret);
1592 }
1593
1594 static inline void save_comm(char *comm, struct task_struct *task)
1595 {
1596         if (!task->pid) {
1597                 strcpy(comm, "<idle>");
1598                 return;
1599         }
1600
1601         if (WARN_ON_ONCE(task->pid < 0)) {
1602                 strcpy(comm, "<XXX>");
1603                 return;
1604         }
1605
1606         strncpy(comm, task->comm, TASK_COMM_LEN);
1607 }
1608
1609 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1610 {
1611         unsigned int i;
1612
1613         for (i = 0; i < elt_data->n_field_var_str; i++)
1614                 kfree(elt_data->field_var_str[i]);
1615
1616         kfree(elt_data->field_var_str);
1617
1618         kfree(elt_data->comm);
1619         kfree(elt_data);
1620 }
1621
1622 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1623 {
1624         struct hist_elt_data *elt_data = elt->private_data;
1625
1626         hist_elt_data_free(elt_data);
1627 }
1628
1629 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1630 {
1631         struct hist_trigger_data *hist_data = elt->map->private_data;
1632         unsigned int size = TASK_COMM_LEN;
1633         struct hist_elt_data *elt_data;
1634         struct hist_field *hist_field;
1635         unsigned int i, n_str;
1636
1637         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1638         if (!elt_data)
1639                 return -ENOMEM;
1640
1641         for_each_hist_field(i, hist_data) {
1642                 hist_field = hist_data->fields[i];
1643
1644                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1645                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1646                         if (!elt_data->comm) {
1647                                 kfree(elt_data);
1648                                 return -ENOMEM;
1649                         }
1650                         break;
1651                 }
1652         }
1653
1654         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1655                 hist_data->n_var_str;
1656         if (n_str > SYNTH_FIELDS_MAX) {
1657                 hist_elt_data_free(elt_data);
1658                 return -EINVAL;
1659         }
1660
1661         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1662
1663         size = STR_VAR_LEN_MAX;
1664
1665         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1666         if (!elt_data->field_var_str) {
1667                 hist_elt_data_free(elt_data);
1668                 return -EINVAL;
1669         }
1670         elt_data->n_field_var_str = n_str;
1671
1672         for (i = 0; i < n_str; i++) {
1673                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1674                 if (!elt_data->field_var_str[i]) {
1675                         hist_elt_data_free(elt_data);
1676                         return -ENOMEM;
1677                 }
1678         }
1679
1680         elt->private_data = elt_data;
1681
1682         return 0;
1683 }
1684
1685 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1686 {
1687         struct hist_elt_data *elt_data = elt->private_data;
1688
1689         if (elt_data->comm)
1690                 save_comm(elt_data->comm, current);
1691 }
1692
1693 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1694         .elt_alloc      = hist_trigger_elt_data_alloc,
1695         .elt_free       = hist_trigger_elt_data_free,
1696         .elt_init       = hist_trigger_elt_data_init,
1697 };
1698
1699 static const char *get_hist_field_flags(struct hist_field *hist_field)
1700 {
1701         const char *flags_str = NULL;
1702
1703         if (hist_field->flags & HIST_FIELD_FL_HEX)
1704                 flags_str = "hex";
1705         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1706                 flags_str = "sym";
1707         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1708                 flags_str = "sym-offset";
1709         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1710                 flags_str = "execname";
1711         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1712                 flags_str = "syscall";
1713         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1714                 flags_str = "log2";
1715         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1716                 flags_str = "buckets";
1717         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1718                 flags_str = "usecs";
1719         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1720                 flags_str = "percent";
1721         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1722                 flags_str = "graph";
1723         else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1724                 flags_str = "stacktrace";
1725
1726         return flags_str;
1727 }
1728
1729 static void expr_field_str(struct hist_field *field, char *expr)
1730 {
1731         if (field->flags & HIST_FIELD_FL_VAR_REF)
1732                 strcat(expr, "$");
1733         else if (field->flags & HIST_FIELD_FL_CONST) {
1734                 char str[HIST_CONST_DIGITS_MAX];
1735
1736                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1737                 strcat(expr, str);
1738         }
1739
1740         strcat(expr, hist_field_name(field, 0));
1741
1742         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1743                 const char *flags_str = get_hist_field_flags(field);
1744
1745                 if (flags_str) {
1746                         strcat(expr, ".");
1747                         strcat(expr, flags_str);
1748                 }
1749         }
1750 }
1751
1752 static char *expr_str(struct hist_field *field, unsigned int level)
1753 {
1754         char *expr;
1755
1756         if (level > 1)
1757                 return NULL;
1758
1759         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1760         if (!expr)
1761                 return NULL;
1762
1763         if (!field->operands[0]) {
1764                 expr_field_str(field, expr);
1765                 return expr;
1766         }
1767
1768         if (field->operator == FIELD_OP_UNARY_MINUS) {
1769                 char *subexpr;
1770
1771                 strcat(expr, "-(");
1772                 subexpr = expr_str(field->operands[0], ++level);
1773                 if (!subexpr) {
1774                         kfree(expr);
1775                         return NULL;
1776                 }
1777                 strcat(expr, subexpr);
1778                 strcat(expr, ")");
1779
1780                 kfree(subexpr);
1781
1782                 return expr;
1783         }
1784
1785         expr_field_str(field->operands[0], expr);
1786
1787         switch (field->operator) {
1788         case FIELD_OP_MINUS:
1789                 strcat(expr, "-");
1790                 break;
1791         case FIELD_OP_PLUS:
1792                 strcat(expr, "+");
1793                 break;
1794         case FIELD_OP_DIV:
1795                 strcat(expr, "/");
1796                 break;
1797         case FIELD_OP_MULT:
1798                 strcat(expr, "*");
1799                 break;
1800         default:
1801                 kfree(expr);
1802                 return NULL;
1803         }
1804
1805         expr_field_str(field->operands[1], expr);
1806
1807         return expr;
1808 }
1809
1810 /*
1811  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1812  * of the expression tree to be evaluated.
1813  */
1814 static int contains_operator(char *str, char **sep)
1815 {
1816         enum field_op_id field_op = FIELD_OP_NONE;
1817         char *minus_op, *plus_op, *div_op, *mult_op;
1818
1819
1820         /*
1821          * Report the last occurrence of the operators first, so that the
1822          * expression is evaluated left to right. This is important since
1823          * subtraction and division are not associative.
1824          *
1825          *      e.g
1826          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1827          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1828          */
1829
1830         /*
1831          * First, find lower precedence addition and subtraction
1832          * since the expression will be evaluated recursively.
1833          */
1834         minus_op = strrchr(str, '-');
1835         if (minus_op) {
1836                 /*
1837                  * Unary minus is not supported in sub-expressions. If
1838                  * present, it is always the next root operator.
1839                  */
1840                 if (minus_op == str) {
1841                         field_op = FIELD_OP_UNARY_MINUS;
1842                         goto out;
1843                 }
1844
1845                 field_op = FIELD_OP_MINUS;
1846         }
1847
1848         plus_op = strrchr(str, '+');
1849         if (plus_op || minus_op) {
1850                 /*
1851                  * For operators of the same precedence use to rightmost as the
1852                  * root, so that the expression is evaluated left to right.
1853                  */
1854                 if (plus_op > minus_op)
1855                         field_op = FIELD_OP_PLUS;
1856                 goto out;
1857         }
1858
1859         /*
1860          * Multiplication and division have higher precedence than addition and
1861          * subtraction.
1862          */
1863         div_op = strrchr(str, '/');
1864         if (div_op)
1865                 field_op = FIELD_OP_DIV;
1866
1867         mult_op = strrchr(str, '*');
1868         /*
1869          * For operators of the same precedence use to rightmost as the
1870          * root, so that the expression is evaluated left to right.
1871          */
1872         if (mult_op > div_op)
1873                 field_op = FIELD_OP_MULT;
1874
1875 out:
1876         if (sep) {
1877                 switch (field_op) {
1878                 case FIELD_OP_UNARY_MINUS:
1879                 case FIELD_OP_MINUS:
1880                         *sep = minus_op;
1881                         break;
1882                 case FIELD_OP_PLUS:
1883                         *sep = plus_op;
1884                         break;
1885                 case FIELD_OP_DIV:
1886                         *sep = div_op;
1887                         break;
1888                 case FIELD_OP_MULT:
1889                         *sep = mult_op;
1890                         break;
1891                 case FIELD_OP_NONE:
1892                 default:
1893                         *sep = NULL;
1894                         break;
1895                 }
1896         }
1897
1898         return field_op;
1899 }
1900
1901 static void get_hist_field(struct hist_field *hist_field)
1902 {
1903         hist_field->ref++;
1904 }
1905
1906 static void __destroy_hist_field(struct hist_field *hist_field)
1907 {
1908         if (--hist_field->ref > 1)
1909                 return;
1910
1911         kfree(hist_field->var.name);
1912         kfree(hist_field->name);
1913
1914         /* Can likely be a const */
1915         kfree_const(hist_field->type);
1916
1917         kfree(hist_field->system);
1918         kfree(hist_field->event_name);
1919
1920         kfree(hist_field);
1921 }
1922
1923 static void destroy_hist_field(struct hist_field *hist_field,
1924                                unsigned int level)
1925 {
1926         unsigned int i;
1927
1928         if (level > 3)
1929                 return;
1930
1931         if (!hist_field)
1932                 return;
1933
1934         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1935                 return; /* var refs will be destroyed separately */
1936
1937         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1938                 destroy_hist_field(hist_field->operands[i], level + 1);
1939
1940         __destroy_hist_field(hist_field);
1941 }
1942
1943 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1944                                             struct ftrace_event_field *field,
1945                                             unsigned long flags,
1946                                             char *var_name)
1947 {
1948         struct hist_field *hist_field;
1949
1950         if (field && is_function_field(field))
1951                 return NULL;
1952
1953         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1954         if (!hist_field)
1955                 return NULL;
1956
1957         hist_field->ref = 1;
1958
1959         hist_field->hist_data = hist_data;
1960
1961         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1962                 goto out; /* caller will populate */
1963
1964         if (flags & HIST_FIELD_FL_VAR_REF) {
1965                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1966                 goto out;
1967         }
1968
1969         if (flags & HIST_FIELD_FL_HITCOUNT) {
1970                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1971                 hist_field->size = sizeof(u64);
1972                 hist_field->type = "u64";
1973                 goto out;
1974         }
1975
1976         if (flags & HIST_FIELD_FL_CONST) {
1977                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1978                 hist_field->size = sizeof(u64);
1979                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1980                 if (!hist_field->type)
1981                         goto free;
1982                 goto out;
1983         }
1984
1985         if (flags & HIST_FIELD_FL_STACKTRACE) {
1986                 if (field)
1987                         hist_field->fn_num = HIST_FIELD_FN_STACK;
1988                 else
1989                         hist_field->fn_num = HIST_FIELD_FN_NOP;
1990                 hist_field->size = HIST_STACKTRACE_SIZE;
1991                 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1992                 if (!hist_field->type)
1993                         goto free;
1994                 goto out;
1995         }
1996
1997         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1998                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1999                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
2000                         HIST_FIELD_FN_BUCKET;
2001                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2002                 if (!hist_field->operands[0])
2003                         goto free;
2004                 hist_field->size = hist_field->operands[0]->size;
2005                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2006                 if (!hist_field->type)
2007                         goto free;
2008                 goto out;
2009         }
2010
2011         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2012                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2013                 hist_field->size = sizeof(u64);
2014                 hist_field->type = "u64";
2015                 goto out;
2016         }
2017
2018         if (flags & HIST_FIELD_FL_CPU) {
2019                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2020                 hist_field->size = sizeof(int);
2021                 hist_field->type = "unsigned int";
2022                 goto out;
2023         }
2024
2025         if (WARN_ON_ONCE(!field))
2026                 goto out;
2027
2028         /* Pointers to strings are just pointers and dangerous to dereference */
2029         if (is_string_field(field) &&
2030             (field->filter_type != FILTER_PTR_STRING)) {
2031                 flags |= HIST_FIELD_FL_STRING;
2032
2033                 hist_field->size = MAX_FILTER_STR_VAL;
2034                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2035                 if (!hist_field->type)
2036                         goto free;
2037
2038                 if (field->filter_type == FILTER_STATIC_STRING) {
2039                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2040                         hist_field->size = field->size;
2041                 } else if (field->filter_type == FILTER_DYN_STRING) {
2042                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2043                 } else if (field->filter_type == FILTER_RDYN_STRING)
2044                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2045                 else
2046                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2047         } else {
2048                 hist_field->size = field->size;
2049                 hist_field->is_signed = field->is_signed;
2050                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2051                 if (!hist_field->type)
2052                         goto free;
2053
2054                 hist_field->fn_num = select_value_fn(field->size,
2055                                                      field->is_signed);
2056                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2057                         destroy_hist_field(hist_field, 0);
2058                         return NULL;
2059                 }
2060         }
2061  out:
2062         hist_field->field = field;
2063         hist_field->flags = flags;
2064
2065         if (var_name) {
2066                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2067                 if (!hist_field->var.name)
2068                         goto free;
2069         }
2070
2071         return hist_field;
2072  free:
2073         destroy_hist_field(hist_field, 0);
2074         return NULL;
2075 }
2076
2077 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2078 {
2079         unsigned int i;
2080
2081         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2082                 if (hist_data->fields[i]) {
2083                         destroy_hist_field(hist_data->fields[i], 0);
2084                         hist_data->fields[i] = NULL;
2085                 }
2086         }
2087
2088         for (i = 0; i < hist_data->n_var_refs; i++) {
2089                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2090                 __destroy_hist_field(hist_data->var_refs[i]);
2091                 hist_data->var_refs[i] = NULL;
2092         }
2093 }
2094
2095 static int init_var_ref(struct hist_field *ref_field,
2096                         struct hist_field *var_field,
2097                         char *system, char *event_name)
2098 {
2099         int err = 0;
2100
2101         ref_field->var.idx = var_field->var.idx;
2102         ref_field->var.hist_data = var_field->hist_data;
2103         ref_field->size = var_field->size;
2104         ref_field->is_signed = var_field->is_signed;
2105         ref_field->flags |= var_field->flags &
2106                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2107
2108         if (system) {
2109                 ref_field->system = kstrdup(system, GFP_KERNEL);
2110                 if (!ref_field->system)
2111                         return -ENOMEM;
2112         }
2113
2114         if (event_name) {
2115                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2116                 if (!ref_field->event_name) {
2117                         err = -ENOMEM;
2118                         goto free;
2119                 }
2120         }
2121
2122         if (var_field->var.name) {
2123                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2124                 if (!ref_field->name) {
2125                         err = -ENOMEM;
2126                         goto free;
2127                 }
2128         } else if (var_field->name) {
2129                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2130                 if (!ref_field->name) {
2131                         err = -ENOMEM;
2132                         goto free;
2133                 }
2134         }
2135
2136         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2137         if (!ref_field->type) {
2138                 err = -ENOMEM;
2139                 goto free;
2140         }
2141  out:
2142         return err;
2143  free:
2144         kfree(ref_field->system);
2145         ref_field->system = NULL;
2146         kfree(ref_field->event_name);
2147         ref_field->event_name = NULL;
2148         kfree(ref_field->name);
2149         ref_field->name = NULL;
2150
2151         goto out;
2152 }
2153
2154 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2155                             struct hist_field *var_field)
2156 {
2157         struct hist_field *ref_field;
2158         int i;
2159
2160         for (i = 0; i < hist_data->n_var_refs; i++) {
2161                 ref_field = hist_data->var_refs[i];
2162                 if (ref_field->var.idx == var_field->var.idx &&
2163                     ref_field->var.hist_data == var_field->hist_data)
2164                         return i;
2165         }
2166
2167         return -ENOENT;
2168 }
2169
2170 /**
2171  * create_var_ref - Create a variable reference and attach it to trigger
2172  * @hist_data: The trigger that will be referencing the variable
2173  * @var_field: The VAR field to create a reference to
2174  * @system: The optional system string
2175  * @event_name: The optional event_name string
2176  *
2177  * Given a variable hist_field, create a VAR_REF hist_field that
2178  * represents a reference to it.
2179  *
2180  * This function also adds the reference to the trigger that
2181  * now references the variable.
2182  *
2183  * Return: The VAR_REF field if successful, NULL if not
2184  */
2185 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2186                                          struct hist_field *var_field,
2187                                          char *system, char *event_name)
2188 {
2189         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2190         struct hist_field *ref_field;
2191         int i;
2192
2193         /* Check if the variable already exists */
2194         for (i = 0; i < hist_data->n_var_refs; i++) {
2195                 ref_field = hist_data->var_refs[i];
2196                 if (ref_field->var.idx == var_field->var.idx &&
2197                     ref_field->var.hist_data == var_field->hist_data) {
2198                         get_hist_field(ref_field);
2199                         return ref_field;
2200                 }
2201         }
2202         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2203         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2204                 return NULL;
2205         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2206         if (ref_field) {
2207                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2208                         destroy_hist_field(ref_field, 0);
2209                         return NULL;
2210                 }
2211
2212                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2213                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2214         }
2215
2216         return ref_field;
2217 }
2218
2219 static bool is_var_ref(char *var_name)
2220 {
2221         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2222                 return false;
2223
2224         return true;
2225 }
2226
2227 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2228                                  char *var_name)
2229 {
2230         char *name, *field;
2231         unsigned int i;
2232
2233         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2234                 name = hist_data->attrs->var_defs.name[i];
2235
2236                 if (strcmp(var_name, name) == 0) {
2237                         field = hist_data->attrs->var_defs.expr[i];
2238                         if (contains_operator(field, NULL) || is_var_ref(field))
2239                                 continue;
2240                         return field;
2241                 }
2242         }
2243
2244         return NULL;
2245 }
2246
2247 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2248                                  char *system, char *event_name,
2249                                  char *var_name)
2250 {
2251         struct trace_event_call *call;
2252
2253         if (system && event_name) {
2254                 call = hist_data->event_file->event_call;
2255
2256                 if (strcmp(system, call->class->system) != 0)
2257                         return NULL;
2258
2259                 if (strcmp(event_name, trace_event_name(call)) != 0)
2260                         return NULL;
2261         }
2262
2263         if (!!system != !!event_name)
2264                 return NULL;
2265
2266         if (!is_var_ref(var_name))
2267                 return NULL;
2268
2269         var_name++;
2270
2271         return field_name_from_var(hist_data, var_name);
2272 }
2273
2274 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2275                                         char *system, char *event_name,
2276                                         char *var_name)
2277 {
2278         struct hist_field *var_field = NULL, *ref_field = NULL;
2279         struct trace_array *tr = hist_data->event_file->tr;
2280
2281         if (!is_var_ref(var_name))
2282                 return NULL;
2283
2284         var_name++;
2285
2286         var_field = find_event_var(hist_data, system, event_name, var_name);
2287         if (var_field)
2288                 ref_field = create_var_ref(hist_data, var_field,
2289                                            system, event_name);
2290
2291         if (!ref_field)
2292                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2293
2294         return ref_field;
2295 }
2296
2297 static struct ftrace_event_field *
2298 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2299             char *field_str, unsigned long *flags, unsigned long *buckets)
2300 {
2301         struct ftrace_event_field *field = NULL;
2302         char *field_name, *modifier, *str;
2303         struct trace_array *tr = file->tr;
2304
2305         modifier = str = kstrdup(field_str, GFP_KERNEL);
2306         if (!modifier)
2307                 return ERR_PTR(-ENOMEM);
2308
2309         field_name = strsep(&modifier, ".");
2310         if (modifier) {
2311                 if (strcmp(modifier, "hex") == 0)
2312                         *flags |= HIST_FIELD_FL_HEX;
2313                 else if (strcmp(modifier, "sym") == 0)
2314                         *flags |= HIST_FIELD_FL_SYM;
2315                 /*
2316                  * 'sym-offset' occurrences in the trigger string are modified
2317                  * to 'symXoffset' to simplify arithmetic expression parsing.
2318                  */
2319                 else if (strcmp(modifier, "symXoffset") == 0)
2320                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2321                 else if ((strcmp(modifier, "execname") == 0) &&
2322                          (strcmp(field_name, "common_pid") == 0))
2323                         *flags |= HIST_FIELD_FL_EXECNAME;
2324                 else if (strcmp(modifier, "syscall") == 0)
2325                         *flags |= HIST_FIELD_FL_SYSCALL;
2326                 else if (strcmp(modifier, "stacktrace") == 0)
2327                         *flags |= HIST_FIELD_FL_STACKTRACE;
2328                 else if (strcmp(modifier, "log2") == 0)
2329                         *flags |= HIST_FIELD_FL_LOG2;
2330                 else if (strcmp(modifier, "usecs") == 0)
2331                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2332                 else if (strncmp(modifier, "bucket", 6) == 0) {
2333                         int ret;
2334
2335                         modifier += 6;
2336
2337                         if (*modifier == 's')
2338                                 modifier++;
2339                         if (*modifier != '=')
2340                                 goto error;
2341                         modifier++;
2342                         ret = kstrtoul(modifier, 0, buckets);
2343                         if (ret || !(*buckets))
2344                                 goto error;
2345                         *flags |= HIST_FIELD_FL_BUCKET;
2346                 } else if (strncmp(modifier, "percent", 7) == 0) {
2347                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2348                                 goto error;
2349                         *flags |= HIST_FIELD_FL_PERCENT;
2350                 } else if (strncmp(modifier, "graph", 5) == 0) {
2351                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2352                                 goto error;
2353                         *flags |= HIST_FIELD_FL_GRAPH;
2354                 } else {
2355  error:
2356                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2357                         field = ERR_PTR(-EINVAL);
2358                         goto out;
2359                 }
2360         }
2361
2362         if (strcmp(field_name, "common_timestamp") == 0) {
2363                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2364                 hist_data->enable_timestamps = true;
2365                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2366                         hist_data->attrs->ts_in_usecs = true;
2367         } else if (strcmp(field_name, "stacktrace") == 0) {
2368                 *flags |= HIST_FIELD_FL_STACKTRACE;
2369         } else if (strcmp(field_name, "common_cpu") == 0)
2370                 *flags |= HIST_FIELD_FL_CPU;
2371         else if (strcmp(field_name, "hitcount") == 0)
2372                 *flags |= HIST_FIELD_FL_HITCOUNT;
2373         else {
2374                 field = trace_find_event_field(file->event_call, field_name);
2375                 if (!field || !field->size) {
2376                         /*
2377                          * For backward compatibility, if field_name
2378                          * was "cpu", then we treat this the same as
2379                          * common_cpu. This also works for "CPU".
2380                          */
2381                         if (field && field->filter_type == FILTER_CPU) {
2382                                 *flags |= HIST_FIELD_FL_CPU;
2383                         } else {
2384                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2385                                          errpos(field_name));
2386                                 field = ERR_PTR(-EINVAL);
2387                                 goto out;
2388                         }
2389                 }
2390         }
2391  out:
2392         kfree(str);
2393
2394         return field;
2395 }
2396
2397 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2398                                        struct hist_field *var_ref,
2399                                        char *var_name)
2400 {
2401         struct hist_field *alias = NULL;
2402         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2403
2404         alias = create_hist_field(hist_data, NULL, flags, var_name);
2405         if (!alias)
2406                 return NULL;
2407
2408         alias->fn_num = var_ref->fn_num;
2409         alias->operands[0] = var_ref;
2410
2411         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2412                 destroy_hist_field(alias, 0);
2413                 return NULL;
2414         }
2415
2416         alias->var_ref_idx = var_ref->var_ref_idx;
2417
2418         return alias;
2419 }
2420
2421 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2422                                       char *str, char *var_name,
2423                                       unsigned long *flags)
2424 {
2425         struct trace_array *tr = hist_data->event_file->tr;
2426         struct hist_field *field = NULL;
2427         u64 constant;
2428
2429         if (kstrtoull(str, 0, &constant)) {
2430                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2431                 return NULL;
2432         }
2433
2434         *flags |= HIST_FIELD_FL_CONST;
2435         field = create_hist_field(hist_data, NULL, *flags, var_name);
2436         if (!field)
2437                 return NULL;
2438
2439         field->constant = constant;
2440
2441         return field;
2442 }
2443
2444 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2445                                      struct trace_event_file *file, char *str,
2446                                      unsigned long *flags, char *var_name)
2447 {
2448         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2449         struct ftrace_event_field *field = NULL;
2450         struct hist_field *hist_field = NULL;
2451         unsigned long buckets = 0;
2452         int ret = 0;
2453
2454         if (isdigit(str[0])) {
2455                 hist_field = parse_const(hist_data, str, var_name, flags);
2456                 if (!hist_field) {
2457                         ret = -EINVAL;
2458                         goto out;
2459                 }
2460                 return hist_field;
2461         }
2462
2463         s = strchr(str, '.');
2464         if (s) {
2465                 s = strchr(++s, '.');
2466                 if (s) {
2467                         ref_system = strsep(&str, ".");
2468                         if (!str) {
2469                                 ret = -EINVAL;
2470                                 goto out;
2471                         }
2472                         ref_event = strsep(&str, ".");
2473                         if (!str) {
2474                                 ret = -EINVAL;
2475                                 goto out;
2476                         }
2477                         ref_var = str;
2478                 }
2479         }
2480
2481         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2482         if (!s) {
2483                 hist_field = parse_var_ref(hist_data, ref_system,
2484                                            ref_event, ref_var);
2485                 if (hist_field) {
2486                         if (var_name) {
2487                                 hist_field = create_alias(hist_data, hist_field, var_name);
2488                                 if (!hist_field) {
2489                                         ret = -ENOMEM;
2490                                         goto out;
2491                                 }
2492                         }
2493                         return hist_field;
2494                 }
2495         } else
2496                 str = s;
2497
2498         field = parse_field(hist_data, file, str, flags, &buckets);
2499         if (IS_ERR(field)) {
2500                 ret = PTR_ERR(field);
2501                 goto out;
2502         }
2503
2504         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2505         if (!hist_field) {
2506                 ret = -ENOMEM;
2507                 goto out;
2508         }
2509         hist_field->buckets = buckets;
2510
2511         return hist_field;
2512  out:
2513         return ERR_PTR(ret);
2514 }
2515
2516 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2517                                      struct trace_event_file *file,
2518                                      char *str, unsigned long flags,
2519                                      char *var_name, unsigned int *n_subexprs);
2520
2521 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2522                                       struct trace_event_file *file,
2523                                       char *str, unsigned long flags,
2524                                       char *var_name, unsigned int *n_subexprs)
2525 {
2526         struct hist_field *operand1, *expr = NULL;
2527         unsigned long operand_flags;
2528         int ret = 0;
2529         char *s;
2530
2531         /* Unary minus operator, increment n_subexprs */
2532         ++*n_subexprs;
2533
2534         /* we support only -(xxx) i.e. explicit parens required */
2535
2536         if (*n_subexprs > 3) {
2537                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2538                 ret = -EINVAL;
2539                 goto free;
2540         }
2541
2542         str++; /* skip leading '-' */
2543
2544         s = strchr(str, '(');
2545         if (s)
2546                 str++;
2547         else {
2548                 ret = -EINVAL;
2549                 goto free;
2550         }
2551
2552         s = strrchr(str, ')');
2553         if (s) {
2554                  /* unary minus not supported in sub-expressions */
2555                 if (*(s+1) != '\0') {
2556                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2557                                  errpos(str));
2558                         ret = -EINVAL;
2559                         goto free;
2560                 }
2561                 *s = '\0';
2562         }
2563         else {
2564                 ret = -EINVAL; /* no closing ')' */
2565                 goto free;
2566         }
2567
2568         flags |= HIST_FIELD_FL_EXPR;
2569         expr = create_hist_field(hist_data, NULL, flags, var_name);
2570         if (!expr) {
2571                 ret = -ENOMEM;
2572                 goto free;
2573         }
2574
2575         operand_flags = 0;
2576         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2577         if (IS_ERR(operand1)) {
2578                 ret = PTR_ERR(operand1);
2579                 goto free;
2580         }
2581         if (operand1->flags & HIST_FIELD_FL_STRING) {
2582                 /* String type can not be the operand of unary operator. */
2583                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2584                 destroy_hist_field(operand1, 0);
2585                 ret = -EINVAL;
2586                 goto free;
2587         }
2588
2589         expr->flags |= operand1->flags &
2590                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2591         expr->fn_num = HIST_FIELD_FN_UMINUS;
2592         expr->operands[0] = operand1;
2593         expr->size = operand1->size;
2594         expr->is_signed = operand1->is_signed;
2595         expr->operator = FIELD_OP_UNARY_MINUS;
2596         expr->name = expr_str(expr, 0);
2597         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2598         if (!expr->type) {
2599                 ret = -ENOMEM;
2600                 goto free;
2601         }
2602
2603         return expr;
2604  free:
2605         destroy_hist_field(expr, 0);
2606         return ERR_PTR(ret);
2607 }
2608
2609 /*
2610  * If the operands are var refs, return pointers the
2611  * variable(s) referenced in var1 and var2, else NULL.
2612  */
2613 static int check_expr_operands(struct trace_array *tr,
2614                                struct hist_field *operand1,
2615                                struct hist_field *operand2,
2616                                struct hist_field **var1,
2617                                struct hist_field **var2)
2618 {
2619         unsigned long operand1_flags = operand1->flags;
2620         unsigned long operand2_flags = operand2->flags;
2621
2622         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2623             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2624                 struct hist_field *var;
2625
2626                 var = find_var_field(operand1->var.hist_data, operand1->name);
2627                 if (!var)
2628                         return -EINVAL;
2629                 operand1_flags = var->flags;
2630                 *var1 = var;
2631         }
2632
2633         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2634             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2635                 struct hist_field *var;
2636
2637                 var = find_var_field(operand2->var.hist_data, operand2->name);
2638                 if (!var)
2639                         return -EINVAL;
2640                 operand2_flags = var->flags;
2641                 *var2 = var;
2642         }
2643
2644         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2645             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2646                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2647                 return -EINVAL;
2648         }
2649
2650         return 0;
2651 }
2652
2653 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2654                                      struct trace_event_file *file,
2655                                      char *str, unsigned long flags,
2656                                      char *var_name, unsigned int *n_subexprs)
2657 {
2658         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2659         struct hist_field *var1 = NULL, *var2 = NULL;
2660         unsigned long operand_flags, operand2_flags;
2661         int field_op, ret = -EINVAL;
2662         char *sep, *operand1_str;
2663         enum hist_field_fn op_fn;
2664         bool combine_consts;
2665
2666         if (*n_subexprs > 3) {
2667                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2668                 return ERR_PTR(-EINVAL);
2669         }
2670
2671         field_op = contains_operator(str, &sep);
2672
2673         if (field_op == FIELD_OP_NONE)
2674                 return parse_atom(hist_data, file, str, &flags, var_name);
2675
2676         if (field_op == FIELD_OP_UNARY_MINUS)
2677                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2678
2679         /* Binary operator found, increment n_subexprs */
2680         ++*n_subexprs;
2681
2682         /* Split the expression string at the root operator */
2683         if (!sep)
2684                 return ERR_PTR(-EINVAL);
2685
2686         *sep = '\0';
2687         operand1_str = str;
2688         str = sep+1;
2689
2690         /* Binary operator requires both operands */
2691         if (*operand1_str == '\0' || *str == '\0')
2692                 return ERR_PTR(-EINVAL);
2693
2694         operand_flags = 0;
2695
2696         /* LHS of string is an expression e.g. a+b in a+b+c */
2697         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2698         if (IS_ERR(operand1))
2699                 return ERR_CAST(operand1);
2700
2701         if (operand1->flags & HIST_FIELD_FL_STRING) {
2702                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2703                 ret = -EINVAL;
2704                 goto free_op1;
2705         }
2706
2707         /* RHS of string is another expression e.g. c in a+b+c */
2708         operand_flags = 0;
2709         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2710         if (IS_ERR(operand2)) {
2711                 ret = PTR_ERR(operand2);
2712                 goto free_op1;
2713         }
2714         if (operand2->flags & HIST_FIELD_FL_STRING) {
2715                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2716                 ret = -EINVAL;
2717                 goto free_operands;
2718         }
2719
2720         switch (field_op) {
2721         case FIELD_OP_MINUS:
2722                 op_fn = HIST_FIELD_FN_MINUS;
2723                 break;
2724         case FIELD_OP_PLUS:
2725                 op_fn = HIST_FIELD_FN_PLUS;
2726                 break;
2727         case FIELD_OP_DIV:
2728                 op_fn = HIST_FIELD_FN_DIV;
2729                 break;
2730         case FIELD_OP_MULT:
2731                 op_fn = HIST_FIELD_FN_MULT;
2732                 break;
2733         default:
2734                 ret = -EINVAL;
2735                 goto free_operands;
2736         }
2737
2738         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2739         if (ret)
2740                 goto free_operands;
2741
2742         operand_flags = var1 ? var1->flags : operand1->flags;
2743         operand2_flags = var2 ? var2->flags : operand2->flags;
2744
2745         /*
2746          * If both operands are constant, the expression can be
2747          * collapsed to a single constant.
2748          */
2749         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2750
2751         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2752
2753         flags |= operand1->flags &
2754                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2755
2756         expr = create_hist_field(hist_data, NULL, flags, var_name);
2757         if (!expr) {
2758                 ret = -ENOMEM;
2759                 goto free_operands;
2760         }
2761
2762         operand1->read_once = true;
2763         operand2->read_once = true;
2764
2765         /* The operands are now owned and free'd by 'expr' */
2766         expr->operands[0] = operand1;
2767         expr->operands[1] = operand2;
2768
2769         if (field_op == FIELD_OP_DIV &&
2770                         operand2_flags & HIST_FIELD_FL_CONST) {
2771                 u64 divisor = var2 ? var2->constant : operand2->constant;
2772
2773                 if (!divisor) {
2774                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2775                         ret = -EDOM;
2776                         goto free_expr;
2777                 }
2778
2779                 /*
2780                  * Copy the divisor here so we don't have to look it up
2781                  * later if this is a var ref
2782                  */
2783                 operand2->constant = divisor;
2784                 op_fn = hist_field_get_div_fn(operand2);
2785         }
2786
2787         expr->fn_num = op_fn;
2788
2789         if (combine_consts) {
2790                 if (var1)
2791                         expr->operands[0] = var1;
2792                 if (var2)
2793                         expr->operands[1] = var2;
2794
2795                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2796                 expr->fn_num = HIST_FIELD_FN_CONST;
2797
2798                 expr->operands[0] = NULL;
2799                 expr->operands[1] = NULL;
2800
2801                 /*
2802                  * var refs won't be destroyed immediately
2803                  * See: destroy_hist_field()
2804                  */
2805                 destroy_hist_field(operand2, 0);
2806                 destroy_hist_field(operand1, 0);
2807
2808                 expr->name = expr_str(expr, 0);
2809         } else {
2810                 /* The operand sizes should be the same, so just pick one */
2811                 expr->size = operand1->size;
2812                 expr->is_signed = operand1->is_signed;
2813
2814                 expr->operator = field_op;
2815                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2816                 if (!expr->type) {
2817                         ret = -ENOMEM;
2818                         goto free_expr;
2819                 }
2820
2821                 expr->name = expr_str(expr, 0);
2822         }
2823
2824         return expr;
2825
2826 free_operands:
2827         destroy_hist_field(operand2, 0);
2828 free_op1:
2829         destroy_hist_field(operand1, 0);
2830         return ERR_PTR(ret);
2831
2832 free_expr:
2833         destroy_hist_field(expr, 0);
2834         return ERR_PTR(ret);
2835 }
2836
2837 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2838                                  struct trace_event_file *file)
2839 {
2840         struct event_trigger_data *test;
2841
2842         lockdep_assert_held(&event_mutex);
2843
2844         list_for_each_entry(test, &file->triggers, list) {
2845                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2846                         if (test->private_data == hist_data)
2847                                 return test->filter_str;
2848                 }
2849         }
2850
2851         return NULL;
2852 }
2853
2854 static struct event_command trigger_hist_cmd;
2855 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2856                                     struct trace_event_file *file,
2857                                     char *glob, char *cmd,
2858                                     char *param_and_filter);
2859
2860 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2861                             struct hist_trigger_data *hist_data,
2862                             unsigned int n_keys)
2863 {
2864         struct hist_field *target_hist_field, *hist_field;
2865         unsigned int n, i, j;
2866
2867         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2868                 return false;
2869
2870         i = hist_data->n_vals;
2871         j = target_hist_data->n_vals;
2872
2873         for (n = 0; n < n_keys; n++) {
2874                 hist_field = hist_data->fields[i + n];
2875                 target_hist_field = target_hist_data->fields[j + n];
2876
2877                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2878                         return false;
2879                 if (hist_field->size != target_hist_field->size)
2880                         return false;
2881                 if (hist_field->is_signed != target_hist_field->is_signed)
2882                         return false;
2883         }
2884
2885         return true;
2886 }
2887
2888 static struct hist_trigger_data *
2889 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2890                      struct trace_event_file *file)
2891 {
2892         struct hist_trigger_data *hist_data;
2893         struct event_trigger_data *test;
2894         unsigned int n_keys;
2895
2896         lockdep_assert_held(&event_mutex);
2897
2898         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2899
2900         list_for_each_entry(test, &file->triggers, list) {
2901                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2902                         hist_data = test->private_data;
2903
2904                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2905                                 return hist_data;
2906                 }
2907         }
2908
2909         return NULL;
2910 }
2911
2912 static struct trace_event_file *event_file(struct trace_array *tr,
2913                                            char *system, char *event_name)
2914 {
2915         struct trace_event_file *file;
2916
2917         file = __find_event_file(tr, system, event_name);
2918         if (!file)
2919                 return ERR_PTR(-EINVAL);
2920
2921         return file;
2922 }
2923
2924 static struct hist_field *
2925 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2926                          char *system, char *event_name, char *field_name)
2927 {
2928         struct hist_field *event_var;
2929         char *synthetic_name;
2930
2931         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2932         if (!synthetic_name)
2933                 return ERR_PTR(-ENOMEM);
2934
2935         strcpy(synthetic_name, "synthetic_");
2936         strcat(synthetic_name, field_name);
2937
2938         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2939
2940         kfree(synthetic_name);
2941
2942         return event_var;
2943 }
2944
2945 /**
2946  * create_field_var_hist - Automatically create a histogram and var for a field
2947  * @target_hist_data: The target hist trigger
2948  * @subsys_name: Optional subsystem name
2949  * @event_name: Optional event name
2950  * @field_name: The name of the field (and the resulting variable)
2951  *
2952  * Hist trigger actions fetch data from variables, not directly from
2953  * events.  However, for convenience, users are allowed to directly
2954  * specify an event field in an action, which will be automatically
2955  * converted into a variable on their behalf.
2956  *
2957  * If a user specifies a field on an event that isn't the event the
2958  * histogram currently being defined (the target event histogram), the
2959  * only way that can be accomplished is if a new hist trigger is
2960  * created and the field variable defined on that.
2961  *
2962  * This function creates a new histogram compatible with the target
2963  * event (meaning a histogram with the same key as the target
2964  * histogram), and creates a variable for the specified field, but
2965  * with 'synthetic_' prepended to the variable name in order to avoid
2966  * collision with normal field variables.
2967  *
2968  * Return: The variable created for the field.
2969  */
2970 static struct hist_field *
2971 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2972                       char *subsys_name, char *event_name, char *field_name)
2973 {
2974         struct trace_array *tr = target_hist_data->event_file->tr;
2975         struct hist_trigger_data *hist_data;
2976         unsigned int i, n, first = true;
2977         struct field_var_hist *var_hist;
2978         struct trace_event_file *file;
2979         struct hist_field *key_field;
2980         struct hist_field *event_var;
2981         char *saved_filter;
2982         char *cmd;
2983         int ret;
2984
2985         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2986                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2987                 return ERR_PTR(-EINVAL);
2988         }
2989
2990         file = event_file(tr, subsys_name, event_name);
2991
2992         if (IS_ERR(file)) {
2993                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2994                 ret = PTR_ERR(file);
2995                 return ERR_PTR(ret);
2996         }
2997
2998         /*
2999          * Look for a histogram compatible with target.  We'll use the
3000          * found histogram specification to create a new matching
3001          * histogram with our variable on it.  target_hist_data is not
3002          * yet a registered histogram so we can't use that.
3003          */
3004         hist_data = find_compatible_hist(target_hist_data, file);
3005         if (!hist_data) {
3006                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3007                 return ERR_PTR(-EINVAL);
3008         }
3009
3010         /* See if a synthetic field variable has already been created */
3011         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3012                                              event_name, field_name);
3013         if (!IS_ERR_OR_NULL(event_var))
3014                 return event_var;
3015
3016         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3017         if (!var_hist)
3018                 return ERR_PTR(-ENOMEM);
3019
3020         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3021         if (!cmd) {
3022                 kfree(var_hist);
3023                 return ERR_PTR(-ENOMEM);
3024         }
3025
3026         /* Use the same keys as the compatible histogram */
3027         strcat(cmd, "keys=");
3028
3029         for_each_hist_key_field(i, hist_data) {
3030                 key_field = hist_data->fields[i];
3031                 if (!first)
3032                         strcat(cmd, ",");
3033                 strcat(cmd, key_field->field->name);
3034                 first = false;
3035         }
3036
3037         /* Create the synthetic field variable specification */
3038         strcat(cmd, ":synthetic_");
3039         strcat(cmd, field_name);
3040         strcat(cmd, "=");
3041         strcat(cmd, field_name);
3042
3043         /* Use the same filter as the compatible histogram */
3044         saved_filter = find_trigger_filter(hist_data, file);
3045         if (saved_filter) {
3046                 strcat(cmd, " if ");
3047                 strcat(cmd, saved_filter);
3048         }
3049
3050         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3051         if (!var_hist->cmd) {
3052                 kfree(cmd);
3053                 kfree(var_hist);
3054                 return ERR_PTR(-ENOMEM);
3055         }
3056
3057         /* Save the compatible histogram information */
3058         var_hist->hist_data = hist_data;
3059
3060         /* Create the new histogram with our variable */
3061         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3062                                        "", "hist", cmd);
3063         if (ret) {
3064                 kfree(cmd);
3065                 kfree(var_hist->cmd);
3066                 kfree(var_hist);
3067                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3068                 return ERR_PTR(ret);
3069         }
3070
3071         kfree(cmd);
3072
3073         /* If we can't find the variable, something went wrong */
3074         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3075                                              event_name, field_name);
3076         if (IS_ERR_OR_NULL(event_var)) {
3077                 kfree(var_hist->cmd);
3078                 kfree(var_hist);
3079                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3080                 return ERR_PTR(-EINVAL);
3081         }
3082
3083         n = target_hist_data->n_field_var_hists;
3084         target_hist_data->field_var_hists[n] = var_hist;
3085         target_hist_data->n_field_var_hists++;
3086
3087         return event_var;
3088 }
3089
3090 static struct hist_field *
3091 find_target_event_var(struct hist_trigger_data *hist_data,
3092                       char *subsys_name, char *event_name, char *var_name)
3093 {
3094         struct trace_event_file *file = hist_data->event_file;
3095         struct hist_field *hist_field = NULL;
3096
3097         if (subsys_name) {
3098                 struct trace_event_call *call;
3099
3100                 if (!event_name)
3101                         return NULL;
3102
3103                 call = file->event_call;
3104
3105                 if (strcmp(subsys_name, call->class->system) != 0)
3106                         return NULL;
3107
3108                 if (strcmp(event_name, trace_event_name(call)) != 0)
3109                         return NULL;
3110         }
3111
3112         hist_field = find_var_field(hist_data, var_name);
3113
3114         return hist_field;
3115 }
3116
3117 static inline void __update_field_vars(struct tracing_map_elt *elt,
3118                                        struct trace_buffer *buffer,
3119                                        struct ring_buffer_event *rbe,
3120                                        void *rec,
3121                                        struct field_var **field_vars,
3122                                        unsigned int n_field_vars,
3123                                        unsigned int field_var_str_start)
3124 {
3125         struct hist_elt_data *elt_data = elt->private_data;
3126         unsigned int i, j, var_idx;
3127         u64 var_val;
3128
3129         /* Make sure stacktrace can fit in the string variable length */
3130         BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3131
3132         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3133                 struct field_var *field_var = field_vars[i];
3134                 struct hist_field *var = field_var->var;
3135                 struct hist_field *val = field_var->val;
3136
3137                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3138                 var_idx = var->var.idx;
3139
3140                 if (val->flags & (HIST_FIELD_FL_STRING |
3141                                   HIST_FIELD_FL_STACKTRACE)) {
3142                         char *str = elt_data->field_var_str[j++];
3143                         char *val_str = (char *)(uintptr_t)var_val;
3144                         unsigned int size;
3145
3146                         if (val->flags & HIST_FIELD_FL_STRING) {
3147                                 size = min(val->size, STR_VAR_LEN_MAX);
3148                                 strscpy(str, val_str, size);
3149                         } else {
3150                                 char *stack_start = str + sizeof(unsigned long);
3151                                 int e;
3152
3153                                 e = stack_trace_save((void *)stack_start,
3154                                                      HIST_STACKTRACE_DEPTH,
3155                                                      HIST_STACKTRACE_SKIP);
3156                                 if (e < HIST_STACKTRACE_DEPTH - 1)
3157                                         ((unsigned long *)stack_start)[e] = 0;
3158                                 *((unsigned long *)str) = e;
3159                         }
3160                         var_val = (u64)(uintptr_t)str;
3161                 }
3162                 tracing_map_set_var(elt, var_idx, var_val);
3163         }
3164 }
3165
3166 static void update_field_vars(struct hist_trigger_data *hist_data,
3167                               struct tracing_map_elt *elt,
3168                               struct trace_buffer *buffer,
3169                               struct ring_buffer_event *rbe,
3170                               void *rec)
3171 {
3172         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3173                             hist_data->n_field_vars, 0);
3174 }
3175
3176 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3177                                  struct tracing_map_elt *elt,
3178                                  struct trace_buffer *buffer,  void *rec,
3179                                  struct ring_buffer_event *rbe, void *key,
3180                                  struct action_data *data, u64 *var_ref_vals)
3181 {
3182         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3183                             hist_data->n_save_vars, hist_data->n_field_var_str);
3184 }
3185
3186 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3187                                      struct trace_event_file *file,
3188                                      char *name, int size, const char *type)
3189 {
3190         struct hist_field *var;
3191         int idx;
3192
3193         if (find_var(hist_data, file, name) && !hist_data->remove) {
3194                 var = ERR_PTR(-EINVAL);
3195                 goto out;
3196         }
3197
3198         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3199         if (!var) {
3200                 var = ERR_PTR(-ENOMEM);
3201                 goto out;
3202         }
3203
3204         idx = tracing_map_add_var(hist_data->map);
3205         if (idx < 0) {
3206                 kfree(var);
3207                 var = ERR_PTR(-EINVAL);
3208                 goto out;
3209         }
3210
3211         var->ref = 1;
3212         var->flags = HIST_FIELD_FL_VAR;
3213         var->var.idx = idx;
3214         var->var.hist_data = var->hist_data = hist_data;
3215         var->size = size;
3216         var->var.name = kstrdup(name, GFP_KERNEL);
3217         var->type = kstrdup_const(type, GFP_KERNEL);
3218         if (!var->var.name || !var->type) {
3219                 kfree_const(var->type);
3220                 kfree(var->var.name);
3221                 kfree(var);
3222                 var = ERR_PTR(-ENOMEM);
3223         }
3224  out:
3225         return var;
3226 }
3227
3228 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3229                                           struct trace_event_file *file,
3230                                           char *field_name)
3231 {
3232         struct hist_field *val = NULL, *var = NULL;
3233         unsigned long flags = HIST_FIELD_FL_VAR;
3234         struct trace_array *tr = file->tr;
3235         struct field_var *field_var;
3236         int ret = 0;
3237
3238         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3239                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3240                 ret = -EINVAL;
3241                 goto err;
3242         }
3243
3244         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3245         if (IS_ERR(val)) {
3246                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3247                 ret = PTR_ERR(val);
3248                 goto err;
3249         }
3250
3251         var = create_var(hist_data, file, field_name, val->size, val->type);
3252         if (IS_ERR(var)) {
3253                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3254                 kfree(val);
3255                 ret = PTR_ERR(var);
3256                 goto err;
3257         }
3258
3259         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3260         if (!field_var) {
3261                 kfree(val);
3262                 kfree(var);
3263                 ret =  -ENOMEM;
3264                 goto err;
3265         }
3266
3267         field_var->var = var;
3268         field_var->val = val;
3269  out:
3270         return field_var;
3271  err:
3272         field_var = ERR_PTR(ret);
3273         goto out;
3274 }
3275
3276 /**
3277  * create_target_field_var - Automatically create a variable for a field
3278  * @target_hist_data: The target hist trigger
3279  * @subsys_name: Optional subsystem name
3280  * @event_name: Optional event name
3281  * @var_name: The name of the field (and the resulting variable)
3282  *
3283  * Hist trigger actions fetch data from variables, not directly from
3284  * events.  However, for convenience, users are allowed to directly
3285  * specify an event field in an action, which will be automatically
3286  * converted into a variable on their behalf.
3287  *
3288  * This function creates a field variable with the name var_name on
3289  * the hist trigger currently being defined on the target event.  If
3290  * subsys_name and event_name are specified, this function simply
3291  * verifies that they do in fact match the target event subsystem and
3292  * event name.
3293  *
3294  * Return: The variable created for the field.
3295  */
3296 static struct field_var *
3297 create_target_field_var(struct hist_trigger_data *target_hist_data,
3298                         char *subsys_name, char *event_name, char *var_name)
3299 {
3300         struct trace_event_file *file = target_hist_data->event_file;
3301
3302         if (subsys_name) {
3303                 struct trace_event_call *call;
3304
3305                 if (!event_name)
3306                         return NULL;
3307
3308                 call = file->event_call;
3309
3310                 if (strcmp(subsys_name, call->class->system) != 0)
3311                         return NULL;
3312
3313                 if (strcmp(event_name, trace_event_name(call)) != 0)
3314                         return NULL;
3315         }
3316
3317         return create_field_var(target_hist_data, file, var_name);
3318 }
3319
3320 static bool check_track_val_max(u64 track_val, u64 var_val)
3321 {
3322         if (var_val <= track_val)
3323                 return false;
3324
3325         return true;
3326 }
3327
3328 static bool check_track_val_changed(u64 track_val, u64 var_val)
3329 {
3330         if (var_val == track_val)
3331                 return false;
3332
3333         return true;
3334 }
3335
3336 static u64 get_track_val(struct hist_trigger_data *hist_data,
3337                          struct tracing_map_elt *elt,
3338                          struct action_data *data)
3339 {
3340         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3341         u64 track_val;
3342
3343         track_val = tracing_map_read_var(elt, track_var_idx);
3344
3345         return track_val;
3346 }
3347
3348 static void save_track_val(struct hist_trigger_data *hist_data,
3349                            struct tracing_map_elt *elt,
3350                            struct action_data *data, u64 var_val)
3351 {
3352         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3353
3354         tracing_map_set_var(elt, track_var_idx, var_val);
3355 }
3356
3357 static void save_track_data(struct hist_trigger_data *hist_data,
3358                             struct tracing_map_elt *elt,
3359                             struct trace_buffer *buffer, void *rec,
3360                             struct ring_buffer_event *rbe, void *key,
3361                             struct action_data *data, u64 *var_ref_vals)
3362 {
3363         if (data->track_data.save_data)
3364                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3365                                            key, data, var_ref_vals);
3366 }
3367
3368 static bool check_track_val(struct tracing_map_elt *elt,
3369                             struct action_data *data,
3370                             u64 var_val)
3371 {
3372         struct hist_trigger_data *hist_data;
3373         u64 track_val;
3374
3375         hist_data = data->track_data.track_var->hist_data;
3376         track_val = get_track_val(hist_data, elt, data);
3377
3378         return data->track_data.check_val(track_val, var_val);
3379 }
3380
3381 #ifdef CONFIG_TRACER_SNAPSHOT
3382 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3383 {
3384         /* called with tr->max_lock held */
3385         struct track_data *track_data = tr->cond_snapshot->cond_data;
3386         struct hist_elt_data *elt_data, *track_elt_data;
3387         struct snapshot_context *context = cond_data;
3388         struct action_data *action;
3389         u64 track_val;
3390
3391         if (!track_data)
3392                 return false;
3393
3394         action = track_data->action_data;
3395
3396         track_val = get_track_val(track_data->hist_data, context->elt,
3397                                   track_data->action_data);
3398
3399         if (!action->track_data.check_val(track_data->track_val, track_val))
3400                 return false;
3401
3402         track_data->track_val = track_val;
3403         memcpy(track_data->key, context->key, track_data->key_len);
3404
3405         elt_data = context->elt->private_data;
3406         track_elt_data = track_data->elt.private_data;
3407         if (elt_data->comm)
3408                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3409
3410         track_data->updated = true;
3411
3412         return true;
3413 }
3414
3415 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3416                                      struct tracing_map_elt *elt,
3417                                      struct trace_buffer *buffer, void *rec,
3418                                      struct ring_buffer_event *rbe, void *key,
3419                                      struct action_data *data,
3420                                      u64 *var_ref_vals)
3421 {
3422         struct trace_event_file *file = hist_data->event_file;
3423         struct snapshot_context context;
3424
3425         context.elt = elt;
3426         context.key = key;
3427
3428         tracing_snapshot_cond(file->tr, &context);
3429 }
3430
3431 static void hist_trigger_print_key(struct seq_file *m,
3432                                    struct hist_trigger_data *hist_data,
3433                                    void *key,
3434                                    struct tracing_map_elt *elt);
3435
3436 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3437 {
3438         unsigned int i;
3439
3440         if (!hist_data->n_actions)
3441                 return NULL;
3442
3443         for (i = 0; i < hist_data->n_actions; i++) {
3444                 struct action_data *data = hist_data->actions[i];
3445
3446                 if (data->action == ACTION_SNAPSHOT)
3447                         return data;
3448         }
3449
3450         return NULL;
3451 }
3452
3453 static void track_data_snapshot_print(struct seq_file *m,
3454                                       struct hist_trigger_data *hist_data)
3455 {
3456         struct trace_event_file *file = hist_data->event_file;
3457         struct track_data *track_data;
3458         struct action_data *action;
3459
3460         track_data = tracing_cond_snapshot_data(file->tr);
3461         if (!track_data)
3462                 return;
3463
3464         if (!track_data->updated)
3465                 return;
3466
3467         action = snapshot_action(hist_data);
3468         if (!action)
3469                 return;
3470
3471         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3472         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3473                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3474                    action->track_data.var_str, track_data->track_val);
3475
3476         seq_puts(m, "\ttriggered by event with key: ");
3477         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3478         seq_putc(m, '\n');
3479 }
3480 #else
3481 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3482 {
3483         return false;
3484 }
3485 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3486                                      struct tracing_map_elt *elt,
3487                                      struct trace_buffer *buffer, void *rec,
3488                                      struct ring_buffer_event *rbe, void *key,
3489                                      struct action_data *data,
3490                                      u64 *var_ref_vals) {}
3491 static void track_data_snapshot_print(struct seq_file *m,
3492                                       struct hist_trigger_data *hist_data) {}
3493 #endif /* CONFIG_TRACER_SNAPSHOT */
3494
3495 static void track_data_print(struct seq_file *m,
3496                              struct hist_trigger_data *hist_data,
3497                              struct tracing_map_elt *elt,
3498                              struct action_data *data)
3499 {
3500         u64 track_val = get_track_val(hist_data, elt, data);
3501         unsigned int i, save_var_idx;
3502
3503         if (data->handler == HANDLER_ONMAX)
3504                 seq_printf(m, "\n\tmax: %10llu", track_val);
3505         else if (data->handler == HANDLER_ONCHANGE)
3506                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3507
3508         if (data->action == ACTION_SNAPSHOT)
3509                 return;
3510
3511         for (i = 0; i < hist_data->n_save_vars; i++) {
3512                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3513                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3514                 u64 val;
3515
3516                 save_var_idx = save_var->var.idx;
3517
3518                 val = tracing_map_read_var(elt, save_var_idx);
3519
3520                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3521                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3522                                    (char *)(uintptr_t)(val));
3523                 } else
3524                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3525         }
3526 }
3527
3528 static void ontrack_action(struct hist_trigger_data *hist_data,
3529                            struct tracing_map_elt *elt,
3530                            struct trace_buffer *buffer, void *rec,
3531                            struct ring_buffer_event *rbe, void *key,
3532                            struct action_data *data, u64 *var_ref_vals)
3533 {
3534         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3535
3536         if (check_track_val(elt, data, var_val)) {
3537                 save_track_val(hist_data, elt, data, var_val);
3538                 save_track_data(hist_data, elt, buffer, rec, rbe,
3539                                 key, data, var_ref_vals);
3540         }
3541 }
3542
3543 static void action_data_destroy(struct action_data *data)
3544 {
3545         unsigned int i;
3546
3547         lockdep_assert_held(&event_mutex);
3548
3549         kfree(data->action_name);
3550
3551         for (i = 0; i < data->n_params; i++)
3552                 kfree(data->params[i]);
3553
3554         if (data->synth_event)
3555                 data->synth_event->ref--;
3556
3557         kfree(data->synth_event_name);
3558
3559         kfree(data);
3560 }
3561
3562 static void track_data_destroy(struct hist_trigger_data *hist_data,
3563                                struct action_data *data)
3564 {
3565         struct trace_event_file *file = hist_data->event_file;
3566
3567         destroy_hist_field(data->track_data.track_var, 0);
3568
3569         if (data->action == ACTION_SNAPSHOT) {
3570                 struct track_data *track_data;
3571
3572                 track_data = tracing_cond_snapshot_data(file->tr);
3573                 if (track_data && track_data->hist_data == hist_data) {
3574                         tracing_snapshot_cond_disable(file->tr);
3575                         track_data_free(track_data);
3576                 }
3577         }
3578
3579         kfree(data->track_data.var_str);
3580
3581         action_data_destroy(data);
3582 }
3583
3584 static int action_create(struct hist_trigger_data *hist_data,
3585                          struct action_data *data);
3586
3587 static int track_data_create(struct hist_trigger_data *hist_data,
3588                              struct action_data *data)
3589 {
3590         struct hist_field *var_field, *ref_field, *track_var = NULL;
3591         struct trace_event_file *file = hist_data->event_file;
3592         struct trace_array *tr = file->tr;
3593         char *track_data_var_str;
3594         int ret = 0;
3595
3596         track_data_var_str = data->track_data.var_str;
3597         if (track_data_var_str[0] != '$') {
3598                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3599                 return -EINVAL;
3600         }
3601         track_data_var_str++;
3602
3603         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3604         if (!var_field) {
3605                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3606                 return -EINVAL;
3607         }
3608
3609         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3610         if (!ref_field)
3611                 return -ENOMEM;
3612
3613         data->track_data.var_ref = ref_field;
3614
3615         if (data->handler == HANDLER_ONMAX)
3616                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3617         if (IS_ERR(track_var)) {
3618                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3619                 ret = PTR_ERR(track_var);
3620                 goto out;
3621         }
3622
3623         if (data->handler == HANDLER_ONCHANGE)
3624                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3625         if (IS_ERR(track_var)) {
3626                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3627                 ret = PTR_ERR(track_var);
3628                 goto out;
3629         }
3630         data->track_data.track_var = track_var;
3631
3632         ret = action_create(hist_data, data);
3633  out:
3634         return ret;
3635 }
3636
3637 static int parse_action_params(struct trace_array *tr, char *params,
3638                                struct action_data *data)
3639 {
3640         char *param, *saved_param;
3641         bool first_param = true;
3642         int ret = 0;
3643
3644         while (params) {
3645                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3646                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3647                         ret = -EINVAL;
3648                         goto out;
3649                 }
3650
3651                 param = strsep(&params, ",");
3652                 if (!param) {
3653                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3654                         ret = -EINVAL;
3655                         goto out;
3656                 }
3657
3658                 param = strstrip(param);
3659                 if (strlen(param) < 2) {
3660                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3661                         ret = -EINVAL;
3662                         goto out;
3663                 }
3664
3665                 saved_param = kstrdup(param, GFP_KERNEL);
3666                 if (!saved_param) {
3667                         ret = -ENOMEM;
3668                         goto out;
3669                 }
3670
3671                 if (first_param && data->use_trace_keyword) {
3672                         data->synth_event_name = saved_param;
3673                         first_param = false;
3674                         continue;
3675                 }
3676                 first_param = false;
3677
3678                 data->params[data->n_params++] = saved_param;
3679         }
3680  out:
3681         return ret;
3682 }
3683
3684 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3685                         enum handler_id handler)
3686 {
3687         char *action_name;
3688         int ret = 0;
3689
3690         strsep(&str, ".");
3691         if (!str) {
3692                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3693                 ret = -EINVAL;
3694                 goto out;
3695         }
3696
3697         action_name = strsep(&str, "(");
3698         if (!action_name || !str) {
3699                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3700                 ret = -EINVAL;
3701                 goto out;
3702         }
3703
3704         if (str_has_prefix(action_name, "save")) {
3705                 char *params = strsep(&str, ")");
3706
3707                 if (!params) {
3708                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3709                         ret = -EINVAL;
3710                         goto out;
3711                 }
3712
3713                 ret = parse_action_params(tr, params, data);
3714                 if (ret)
3715                         goto out;
3716
3717                 if (handler == HANDLER_ONMAX)
3718                         data->track_data.check_val = check_track_val_max;
3719                 else if (handler == HANDLER_ONCHANGE)
3720                         data->track_data.check_val = check_track_val_changed;
3721                 else {
3722                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3723                         ret = -EINVAL;
3724                         goto out;
3725                 }
3726
3727                 data->track_data.save_data = save_track_data_vars;
3728                 data->fn = ontrack_action;
3729                 data->action = ACTION_SAVE;
3730         } else if (str_has_prefix(action_name, "snapshot")) {
3731                 char *params = strsep(&str, ")");
3732
3733                 if (!str) {
3734                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3735                         ret = -EINVAL;
3736                         goto out;
3737                 }
3738
3739                 if (handler == HANDLER_ONMAX)
3740                         data->track_data.check_val = check_track_val_max;
3741                 else if (handler == HANDLER_ONCHANGE)
3742                         data->track_data.check_val = check_track_val_changed;
3743                 else {
3744                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3745                         ret = -EINVAL;
3746                         goto out;
3747                 }
3748
3749                 data->track_data.save_data = save_track_data_snapshot;
3750                 data->fn = ontrack_action;
3751                 data->action = ACTION_SNAPSHOT;
3752         } else {
3753                 char *params = strsep(&str, ")");
3754
3755                 if (str_has_prefix(action_name, "trace"))
3756                         data->use_trace_keyword = true;
3757
3758                 if (params) {
3759                         ret = parse_action_params(tr, params, data);
3760                         if (ret)
3761                                 goto out;
3762                 }
3763
3764                 if (handler == HANDLER_ONMAX)
3765                         data->track_data.check_val = check_track_val_max;
3766                 else if (handler == HANDLER_ONCHANGE)
3767                         data->track_data.check_val = check_track_val_changed;
3768
3769                 if (handler != HANDLER_ONMATCH) {
3770                         data->track_data.save_data = action_trace;
3771                         data->fn = ontrack_action;
3772                 } else
3773                         data->fn = action_trace;
3774
3775                 data->action = ACTION_TRACE;
3776         }
3777
3778         data->action_name = kstrdup(action_name, GFP_KERNEL);
3779         if (!data->action_name) {
3780                 ret = -ENOMEM;
3781                 goto out;
3782         }
3783
3784         data->handler = handler;
3785  out:
3786         return ret;
3787 }
3788
3789 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3790                                             char *str, enum handler_id handler)
3791 {
3792         struct action_data *data;
3793         int ret = -EINVAL;
3794         char *var_str;
3795
3796         data = kzalloc(sizeof(*data), GFP_KERNEL);
3797         if (!data)
3798                 return ERR_PTR(-ENOMEM);
3799
3800         var_str = strsep(&str, ")");
3801         if (!var_str || !str) {
3802                 ret = -EINVAL;
3803                 goto free;
3804         }
3805
3806         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3807         if (!data->track_data.var_str) {
3808                 ret = -ENOMEM;
3809                 goto free;
3810         }
3811
3812         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3813         if (ret)
3814                 goto free;
3815  out:
3816         return data;
3817  free:
3818         track_data_destroy(hist_data, data);
3819         data = ERR_PTR(ret);
3820         goto out;
3821 }
3822
3823 static void onmatch_destroy(struct action_data *data)
3824 {
3825         kfree(data->match_data.event);
3826         kfree(data->match_data.event_system);
3827
3828         action_data_destroy(data);
3829 }
3830
3831 static void destroy_field_var(struct field_var *field_var)
3832 {
3833         if (!field_var)
3834                 return;
3835
3836         destroy_hist_field(field_var->var, 0);
3837         destroy_hist_field(field_var->val, 0);
3838
3839         kfree(field_var);
3840 }
3841
3842 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3843 {
3844         unsigned int i;
3845
3846         for (i = 0; i < hist_data->n_field_vars; i++)
3847                 destroy_field_var(hist_data->field_vars[i]);
3848
3849         for (i = 0; i < hist_data->n_save_vars; i++)
3850                 destroy_field_var(hist_data->save_vars[i]);
3851 }
3852
3853 static void save_field_var(struct hist_trigger_data *hist_data,
3854                            struct field_var *field_var)
3855 {
3856         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3857
3858         /* Stack traces are saved in the string storage too */
3859         if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3860                 hist_data->n_field_var_str++;
3861 }
3862
3863
3864 static int check_synth_field(struct synth_event *event,
3865                              struct hist_field *hist_field,
3866                              unsigned int field_pos)
3867 {
3868         struct synth_field *field;
3869
3870         if (field_pos >= event->n_fields)
3871                 return -EINVAL;
3872
3873         field = event->fields[field_pos];
3874
3875         /*
3876          * A dynamic string synth field can accept static or
3877          * dynamic. A static string synth field can only accept a
3878          * same-sized static string, which is checked for later.
3879          */
3880         if (strstr(hist_field->type, "char[") && field->is_string
3881             && field->is_dynamic)
3882                 return 0;
3883
3884         if (strstr(hist_field->type, "long[") && field->is_stack)
3885                 return 0;
3886
3887         if (strcmp(field->type, hist_field->type) != 0) {
3888                 if (field->size != hist_field->size ||
3889                     (!field->is_string && field->is_signed != hist_field->is_signed))
3890                         return -EINVAL;
3891         }
3892
3893         return 0;
3894 }
3895
3896 static struct hist_field *
3897 trace_action_find_var(struct hist_trigger_data *hist_data,
3898                       struct action_data *data,
3899                       char *system, char *event, char *var)
3900 {
3901         struct trace_array *tr = hist_data->event_file->tr;
3902         struct hist_field *hist_field;
3903
3904         var++; /* skip '$' */
3905
3906         hist_field = find_target_event_var(hist_data, system, event, var);
3907         if (!hist_field) {
3908                 if (!system && data->handler == HANDLER_ONMATCH) {
3909                         system = data->match_data.event_system;
3910                         event = data->match_data.event;
3911                 }
3912
3913                 hist_field = find_event_var(hist_data, system, event, var);
3914         }
3915
3916         if (!hist_field)
3917                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3918
3919         return hist_field;
3920 }
3921
3922 static struct hist_field *
3923 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3924                               struct action_data *data, char *system,
3925                               char *event, char *var)
3926 {
3927         struct hist_field *hist_field = NULL;
3928         struct field_var *field_var;
3929
3930         /*
3931          * First try to create a field var on the target event (the
3932          * currently being defined).  This will create a variable for
3933          * unqualified fields on the target event, or if qualified,
3934          * target fields that have qualified names matching the target.
3935          */
3936         field_var = create_target_field_var(hist_data, system, event, var);
3937
3938         if (field_var && !IS_ERR(field_var)) {
3939                 save_field_var(hist_data, field_var);
3940                 hist_field = field_var->var;
3941         } else {
3942                 field_var = NULL;
3943                 /*
3944                  * If no explicit system.event is specified, default to
3945                  * looking for fields on the onmatch(system.event.xxx)
3946                  * event.
3947                  */
3948                 if (!system && data->handler == HANDLER_ONMATCH) {
3949                         system = data->match_data.event_system;
3950                         event = data->match_data.event;
3951                 }
3952
3953                 if (!event)
3954                         goto free;
3955                 /*
3956                  * At this point, we're looking at a field on another
3957                  * event.  Because we can't modify a hist trigger on
3958                  * another event to add a variable for a field, we need
3959                  * to create a new trigger on that event and create the
3960                  * variable at the same time.
3961                  */
3962                 hist_field = create_field_var_hist(hist_data, system, event, var);
3963                 if (IS_ERR(hist_field))
3964                         goto free;
3965         }
3966  out:
3967         return hist_field;
3968  free:
3969         destroy_field_var(field_var);
3970         hist_field = NULL;
3971         goto out;
3972 }
3973
3974 static int trace_action_create(struct hist_trigger_data *hist_data,
3975                                struct action_data *data)
3976 {
3977         struct trace_array *tr = hist_data->event_file->tr;
3978         char *event_name, *param, *system = NULL;
3979         struct hist_field *hist_field, *var_ref;
3980         unsigned int i;
3981         unsigned int field_pos = 0;
3982         struct synth_event *event;
3983         char *synth_event_name;
3984         int var_ref_idx, ret = 0;
3985
3986         lockdep_assert_held(&event_mutex);
3987
3988         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3989         if (data->n_params > SYNTH_FIELDS_MAX)
3990                 return -EINVAL;
3991
3992         if (data->use_trace_keyword)
3993                 synth_event_name = data->synth_event_name;
3994         else
3995                 synth_event_name = data->action_name;
3996
3997         event = find_synth_event(synth_event_name);
3998         if (!event) {
3999                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4000                 return -EINVAL;
4001         }
4002
4003         event->ref++;
4004
4005         for (i = 0; i < data->n_params; i++) {
4006                 char *p;
4007
4008                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4009                 if (!param) {
4010                         ret = -ENOMEM;
4011                         goto err;
4012                 }
4013
4014                 system = strsep(&param, ".");
4015                 if (!param) {
4016                         param = (char *)system;
4017                         system = event_name = NULL;
4018                 } else {
4019                         event_name = strsep(&param, ".");
4020                         if (!param) {
4021                                 kfree(p);
4022                                 ret = -EINVAL;
4023                                 goto err;
4024                         }
4025                 }
4026
4027                 if (param[0] == '$')
4028                         hist_field = trace_action_find_var(hist_data, data,
4029                                                            system, event_name,
4030                                                            param);
4031                 else
4032                         hist_field = trace_action_create_field_var(hist_data,
4033                                                                    data,
4034                                                                    system,
4035                                                                    event_name,
4036                                                                    param);
4037
4038                 if (!hist_field) {
4039                         kfree(p);
4040                         ret = -EINVAL;
4041                         goto err;
4042                 }
4043
4044                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4045                         var_ref = create_var_ref(hist_data, hist_field,
4046                                                  system, event_name);
4047                         if (!var_ref) {
4048                                 kfree(p);
4049                                 ret = -ENOMEM;
4050                                 goto err;
4051                         }
4052
4053                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4054                         if (WARN_ON(var_ref_idx < 0)) {
4055                                 kfree(p);
4056                                 ret = var_ref_idx;
4057                                 goto err;
4058                         }
4059
4060                         data->var_ref_idx[i] = var_ref_idx;
4061
4062                         field_pos++;
4063                         kfree(p);
4064                         continue;
4065                 }
4066
4067                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4068                 kfree(p);
4069                 ret = -EINVAL;
4070                 goto err;
4071         }
4072
4073         if (field_pos != event->n_fields) {
4074                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4075                 ret = -EINVAL;
4076                 goto err;
4077         }
4078
4079         data->synth_event = event;
4080  out:
4081         return ret;
4082  err:
4083         event->ref--;
4084
4085         goto out;
4086 }
4087
4088 static int action_create(struct hist_trigger_data *hist_data,
4089                          struct action_data *data)
4090 {
4091         struct trace_event_file *file = hist_data->event_file;
4092         struct trace_array *tr = file->tr;
4093         struct track_data *track_data;
4094         struct field_var *field_var;
4095         unsigned int i;
4096         char *param;
4097         int ret = 0;
4098
4099         if (data->action == ACTION_TRACE)
4100                 return trace_action_create(hist_data, data);
4101
4102         if (data->action == ACTION_SNAPSHOT) {
4103                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4104                 if (IS_ERR(track_data)) {
4105                         ret = PTR_ERR(track_data);
4106                         goto out;
4107                 }
4108
4109                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4110                                                    cond_snapshot_update);
4111                 if (ret)
4112                         track_data_free(track_data);
4113
4114                 goto out;
4115         }
4116
4117         if (data->action == ACTION_SAVE) {
4118                 if (hist_data->n_save_vars) {
4119                         ret = -EEXIST;
4120                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4121                         goto out;
4122                 }
4123
4124                 for (i = 0; i < data->n_params; i++) {
4125                         param = kstrdup(data->params[i], GFP_KERNEL);
4126                         if (!param) {
4127                                 ret = -ENOMEM;
4128                                 goto out;
4129                         }
4130
4131                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4132                         if (IS_ERR(field_var)) {
4133                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4134                                          errpos(param));
4135                                 ret = PTR_ERR(field_var);
4136                                 kfree(param);
4137                                 goto out;
4138                         }
4139
4140                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4141                         if (field_var->val->flags &
4142                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4143                                 hist_data->n_save_var_str++;
4144                         kfree(param);
4145                 }
4146         }
4147  out:
4148         return ret;
4149 }
4150
4151 static int onmatch_create(struct hist_trigger_data *hist_data,
4152                           struct action_data *data)
4153 {
4154         return action_create(hist_data, data);
4155 }
4156
4157 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4158 {
4159         char *match_event, *match_event_system;
4160         struct action_data *data;
4161         int ret = -EINVAL;
4162
4163         data = kzalloc(sizeof(*data), GFP_KERNEL);
4164         if (!data)
4165                 return ERR_PTR(-ENOMEM);
4166
4167         match_event = strsep(&str, ")");
4168         if (!match_event || !str) {
4169                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4170                 goto free;
4171         }
4172
4173         match_event_system = strsep(&match_event, ".");
4174         if (!match_event) {
4175                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4176                 goto free;
4177         }
4178
4179         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4180                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4181                 goto free;
4182         }
4183
4184         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4185         if (!data->match_data.event) {
4186                 ret = -ENOMEM;
4187                 goto free;
4188         }
4189
4190         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4191         if (!data->match_data.event_system) {
4192                 ret = -ENOMEM;
4193                 goto free;
4194         }
4195
4196         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4197         if (ret)
4198                 goto free;
4199  out:
4200         return data;
4201  free:
4202         onmatch_destroy(data);
4203         data = ERR_PTR(ret);
4204         goto out;
4205 }
4206
4207 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4208 {
4209         hist_data->fields[HITCOUNT_IDX] =
4210                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4211         if (!hist_data->fields[HITCOUNT_IDX])
4212                 return -ENOMEM;
4213
4214         hist_data->n_vals++;
4215         hist_data->n_fields++;
4216
4217         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4218                 return -EINVAL;
4219
4220         return 0;
4221 }
4222
4223 static int __create_val_field(struct hist_trigger_data *hist_data,
4224                               unsigned int val_idx,
4225                               struct trace_event_file *file,
4226                               char *var_name, char *field_str,
4227                               unsigned long flags)
4228 {
4229         struct hist_field *hist_field;
4230         int ret = 0, n_subexprs = 0;
4231
4232         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4233         if (IS_ERR(hist_field)) {
4234                 ret = PTR_ERR(hist_field);
4235                 goto out;
4236         }
4237
4238         /* Some types cannot be a value */
4239         if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4240                                  HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4241                                  HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4242                                  HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) {
4243                 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4244                 ret = -EINVAL;
4245         }
4246
4247         hist_data->fields[val_idx] = hist_field;
4248
4249         ++hist_data->n_vals;
4250         ++hist_data->n_fields;
4251
4252         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4253                 ret = -EINVAL;
4254  out:
4255         return ret;
4256 }
4257
4258 static int create_val_field(struct hist_trigger_data *hist_data,
4259                             unsigned int val_idx,
4260                             struct trace_event_file *file,
4261                             char *field_str)
4262 {
4263         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4264                 return -EINVAL;
4265
4266         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4267 }
4268
4269 static const char no_comm[] = "(no comm)";
4270
4271 static u64 hist_field_execname(struct hist_field *hist_field,
4272                                struct tracing_map_elt *elt,
4273                                struct trace_buffer *buffer,
4274                                struct ring_buffer_event *rbe,
4275                                void *event)
4276 {
4277         struct hist_elt_data *elt_data;
4278
4279         if (WARN_ON_ONCE(!elt))
4280                 return (u64)(unsigned long)no_comm;
4281
4282         elt_data = elt->private_data;
4283
4284         if (WARN_ON_ONCE(!elt_data->comm))
4285                 return (u64)(unsigned long)no_comm;
4286
4287         return (u64)(unsigned long)(elt_data->comm);
4288 }
4289
4290 static u64 hist_field_stack(struct hist_field *hist_field,
4291                             struct tracing_map_elt *elt,
4292                             struct trace_buffer *buffer,
4293                             struct ring_buffer_event *rbe,
4294                             void *event)
4295 {
4296         u32 str_item = *(u32 *)(event + hist_field->field->offset);
4297         int str_loc = str_item & 0xffff;
4298         char *addr = (char *)(event + str_loc);
4299
4300         return (u64)(unsigned long)addr;
4301 }
4302
4303 static u64 hist_fn_call(struct hist_field *hist_field,
4304                         struct tracing_map_elt *elt,
4305                         struct trace_buffer *buffer,
4306                         struct ring_buffer_event *rbe,
4307                         void *event)
4308 {
4309         switch (hist_field->fn_num) {
4310         case HIST_FIELD_FN_VAR_REF:
4311                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4312         case HIST_FIELD_FN_COUNTER:
4313                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4314         case HIST_FIELD_FN_CONST:
4315                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4316         case HIST_FIELD_FN_LOG2:
4317                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4318         case HIST_FIELD_FN_BUCKET:
4319                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4320         case HIST_FIELD_FN_TIMESTAMP:
4321                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4322         case HIST_FIELD_FN_CPU:
4323                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4324         case HIST_FIELD_FN_STRING:
4325                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4326         case HIST_FIELD_FN_DYNSTRING:
4327                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4328         case HIST_FIELD_FN_RELDYNSTRING:
4329                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4330         case HIST_FIELD_FN_PSTRING:
4331                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4332         case HIST_FIELD_FN_S64:
4333                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4334         case HIST_FIELD_FN_U64:
4335                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4336         case HIST_FIELD_FN_S32:
4337                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4338         case HIST_FIELD_FN_U32:
4339                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4340         case HIST_FIELD_FN_S16:
4341                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4342         case HIST_FIELD_FN_U16:
4343                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4344         case HIST_FIELD_FN_S8:
4345                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4346         case HIST_FIELD_FN_U8:
4347                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4348         case HIST_FIELD_FN_UMINUS:
4349                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4350         case HIST_FIELD_FN_MINUS:
4351                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4352         case HIST_FIELD_FN_PLUS:
4353                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4354         case HIST_FIELD_FN_DIV:
4355                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4356         case HIST_FIELD_FN_MULT:
4357                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4358         case HIST_FIELD_FN_DIV_POWER2:
4359                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4360         case HIST_FIELD_FN_DIV_NOT_POWER2:
4361                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4362         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4363                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4364         case HIST_FIELD_FN_EXECNAME:
4365                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4366         case HIST_FIELD_FN_STACK:
4367                 return hist_field_stack(hist_field, elt, buffer, rbe, event);
4368         default:
4369                 return 0;
4370         }
4371 }
4372
4373 /* Convert a var that points to common_pid.execname to a string */
4374 static void update_var_execname(struct hist_field *hist_field)
4375 {
4376         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4377                 HIST_FIELD_FL_EXECNAME;
4378         hist_field->size = MAX_FILTER_STR_VAL;
4379         hist_field->is_signed = 0;
4380
4381         kfree_const(hist_field->type);
4382         hist_field->type = "char[]";
4383
4384         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4385 }
4386
4387 static int create_var_field(struct hist_trigger_data *hist_data,
4388                             unsigned int val_idx,
4389                             struct trace_event_file *file,
4390                             char *var_name, char *expr_str)
4391 {
4392         struct trace_array *tr = hist_data->event_file->tr;
4393         unsigned long flags = 0;
4394         int ret;
4395
4396         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4397                 return -EINVAL;
4398
4399         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4400                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4401                 return -EINVAL;
4402         }
4403
4404         flags |= HIST_FIELD_FL_VAR;
4405         hist_data->n_vars++;
4406         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4407                 return -EINVAL;
4408
4409         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4410
4411         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4412                 update_var_execname(hist_data->fields[val_idx]);
4413
4414         if (!ret && hist_data->fields[val_idx]->flags &
4415             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4416                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4417
4418         return ret;
4419 }
4420
4421 static int create_val_fields(struct hist_trigger_data *hist_data,
4422                              struct trace_event_file *file)
4423 {
4424         unsigned int i, j = 1, n_hitcount = 0;
4425         char *fields_str, *field_str;
4426         int ret;
4427
4428         ret = create_hitcount_val(hist_data);
4429         if (ret)
4430                 goto out;
4431
4432         fields_str = hist_data->attrs->vals_str;
4433         if (!fields_str)
4434                 goto out;
4435
4436         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4437                      j < TRACING_MAP_VALS_MAX; i++) {
4438                 field_str = strsep(&fields_str, ",");
4439                 if (!field_str)
4440                         break;
4441
4442                 if (strcmp(field_str, "hitcount") == 0) {
4443                         if (!n_hitcount++)
4444                                 continue;
4445                 }
4446
4447                 ret = create_val_field(hist_data, j++, file, field_str);
4448                 if (ret)
4449                         goto out;
4450         }
4451
4452         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4453                 ret = -EINVAL;
4454  out:
4455         /* There is only raw hitcount but nohitcount suppresses it. */
4456         if (j == 1 && hist_data->attrs->no_hitcount) {
4457                 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4458                 ret = -ENOENT;
4459         }
4460
4461         return ret;
4462 }
4463
4464 static int create_key_field(struct hist_trigger_data *hist_data,
4465                             unsigned int key_idx,
4466                             unsigned int key_offset,
4467                             struct trace_event_file *file,
4468                             char *field_str)
4469 {
4470         struct trace_array *tr = hist_data->event_file->tr;
4471         struct hist_field *hist_field = NULL;
4472         unsigned long flags = 0;
4473         unsigned int key_size;
4474         int ret = 0, n_subexprs = 0;
4475
4476         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4477                 return -EINVAL;
4478
4479         flags |= HIST_FIELD_FL_KEY;
4480
4481         if (strcmp(field_str, "stacktrace") == 0) {
4482                 flags |= HIST_FIELD_FL_STACKTRACE;
4483                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4484                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4485         } else {
4486                 hist_field = parse_expr(hist_data, file, field_str, flags,
4487                                         NULL, &n_subexprs);
4488                 if (IS_ERR(hist_field)) {
4489                         ret = PTR_ERR(hist_field);
4490                         goto out;
4491                 }
4492
4493                 if (field_has_hist_vars(hist_field, 0)) {
4494                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4495                         destroy_hist_field(hist_field, 0);
4496                         ret = -EINVAL;
4497                         goto out;
4498                 }
4499
4500                 key_size = hist_field->size;
4501         }
4502
4503         hist_data->fields[key_idx] = hist_field;
4504
4505         key_size = ALIGN(key_size, sizeof(u64));
4506         hist_data->fields[key_idx]->size = key_size;
4507         hist_data->fields[key_idx]->offset = key_offset;
4508
4509         hist_data->key_size += key_size;
4510
4511         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4512                 ret = -EINVAL;
4513                 goto out;
4514         }
4515
4516         hist_data->n_keys++;
4517         hist_data->n_fields++;
4518
4519         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4520                 return -EINVAL;
4521
4522         ret = key_size;
4523  out:
4524         return ret;
4525 }
4526
4527 static int create_key_fields(struct hist_trigger_data *hist_data,
4528                              struct trace_event_file *file)
4529 {
4530         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4531         char *fields_str, *field_str;
4532         int ret = -EINVAL;
4533
4534         fields_str = hist_data->attrs->keys_str;
4535         if (!fields_str)
4536                 goto out;
4537
4538         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4539                 field_str = strsep(&fields_str, ",");
4540                 if (!field_str)
4541                         break;
4542                 ret = create_key_field(hist_data, i, key_offset,
4543                                        file, field_str);
4544                 if (ret < 0)
4545                         goto out;
4546                 key_offset += ret;
4547         }
4548         if (fields_str) {
4549                 ret = -EINVAL;
4550                 goto out;
4551         }
4552         ret = 0;
4553  out:
4554         return ret;
4555 }
4556
4557 static int create_var_fields(struct hist_trigger_data *hist_data,
4558                              struct trace_event_file *file)
4559 {
4560         unsigned int i, j = hist_data->n_vals;
4561         int ret = 0;
4562
4563         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4564
4565         for (i = 0; i < n_vars; i++) {
4566                 char *var_name = hist_data->attrs->var_defs.name[i];
4567                 char *expr = hist_data->attrs->var_defs.expr[i];
4568
4569                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4570                 if (ret)
4571                         goto out;
4572         }
4573  out:
4574         return ret;
4575 }
4576
4577 static void free_var_defs(struct hist_trigger_data *hist_data)
4578 {
4579         unsigned int i;
4580
4581         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4582                 kfree(hist_data->attrs->var_defs.name[i]);
4583                 kfree(hist_data->attrs->var_defs.expr[i]);
4584         }
4585
4586         hist_data->attrs->var_defs.n_vars = 0;
4587 }
4588
4589 static int parse_var_defs(struct hist_trigger_data *hist_data)
4590 {
4591         struct trace_array *tr = hist_data->event_file->tr;
4592         char *s, *str, *var_name, *field_str;
4593         unsigned int i, j, n_vars = 0;
4594         int ret = 0;
4595
4596         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4597                 str = hist_data->attrs->assignment_str[i];
4598                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4599                         field_str = strsep(&str, ",");
4600                         if (!field_str)
4601                                 break;
4602
4603                         var_name = strsep(&field_str, "=");
4604                         if (!var_name || !field_str) {
4605                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4606                                          errpos(var_name));
4607                                 ret = -EINVAL;
4608                                 goto free;
4609                         }
4610
4611                         if (n_vars == TRACING_MAP_VARS_MAX) {
4612                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4613                                 ret = -EINVAL;
4614                                 goto free;
4615                         }
4616
4617                         s = kstrdup(var_name, GFP_KERNEL);
4618                         if (!s) {
4619                                 ret = -ENOMEM;
4620                                 goto free;
4621                         }
4622                         hist_data->attrs->var_defs.name[n_vars] = s;
4623
4624                         s = kstrdup(field_str, GFP_KERNEL);
4625                         if (!s) {
4626                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4627                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4628                                 ret = -ENOMEM;
4629                                 goto free;
4630                         }
4631                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4632
4633                         hist_data->attrs->var_defs.n_vars = n_vars;
4634                 }
4635         }
4636
4637         return ret;
4638  free:
4639         free_var_defs(hist_data);
4640
4641         return ret;
4642 }
4643
4644 static int create_hist_fields(struct hist_trigger_data *hist_data,
4645                               struct trace_event_file *file)
4646 {
4647         int ret;
4648
4649         ret = parse_var_defs(hist_data);
4650         if (ret)
4651                 return ret;
4652
4653         ret = create_val_fields(hist_data, file);
4654         if (ret)
4655                 goto out;
4656
4657         ret = create_var_fields(hist_data, file);
4658         if (ret)
4659                 goto out;
4660
4661         ret = create_key_fields(hist_data, file);
4662
4663  out:
4664         free_var_defs(hist_data);
4665
4666         return ret;
4667 }
4668
4669 static int is_descending(struct trace_array *tr, const char *str)
4670 {
4671         if (!str)
4672                 return 0;
4673
4674         if (strcmp(str, "descending") == 0)
4675                 return 1;
4676
4677         if (strcmp(str, "ascending") == 0)
4678                 return 0;
4679
4680         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4681
4682         return -EINVAL;
4683 }
4684
4685 static int create_sort_keys(struct hist_trigger_data *hist_data)
4686 {
4687         struct trace_array *tr = hist_data->event_file->tr;
4688         char *fields_str = hist_data->attrs->sort_key_str;
4689         struct tracing_map_sort_key *sort_key;
4690         int descending, ret = 0;
4691         unsigned int i, j, k;
4692
4693         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4694
4695         if (!fields_str)
4696                 goto out;
4697
4698         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4699                 struct hist_field *hist_field;
4700                 char *field_str, *field_name;
4701                 const char *test_name;
4702
4703                 sort_key = &hist_data->sort_keys[i];
4704
4705                 field_str = strsep(&fields_str, ",");
4706                 if (!field_str)
4707                         break;
4708
4709                 if (!*field_str) {
4710                         ret = -EINVAL;
4711                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4712                         break;
4713                 }
4714
4715                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4716                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4717                         ret = -EINVAL;
4718                         break;
4719                 }
4720
4721                 field_name = strsep(&field_str, ".");
4722                 if (!field_name || !*field_name) {
4723                         ret = -EINVAL;
4724                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4725                         break;
4726                 }
4727
4728                 if (strcmp(field_name, "hitcount") == 0) {
4729                         descending = is_descending(tr, field_str);
4730                         if (descending < 0) {
4731                                 ret = descending;
4732                                 break;
4733                         }
4734                         sort_key->descending = descending;
4735                         continue;
4736                 }
4737
4738                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4739                         unsigned int idx;
4740
4741                         hist_field = hist_data->fields[j];
4742                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4743                                 continue;
4744
4745                         idx = k++;
4746
4747                         test_name = hist_field_name(hist_field, 0);
4748
4749                         if (strcmp(field_name, test_name) == 0) {
4750                                 sort_key->field_idx = idx;
4751                                 descending = is_descending(tr, field_str);
4752                                 if (descending < 0) {
4753                                         ret = descending;
4754                                         goto out;
4755                                 }
4756                                 sort_key->descending = descending;
4757                                 break;
4758                         }
4759                 }
4760                 if (j == hist_data->n_fields) {
4761                         ret = -EINVAL;
4762                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4763                         break;
4764                 }
4765         }
4766
4767         hist_data->n_sort_keys = i;
4768  out:
4769         return ret;
4770 }
4771
4772 static void destroy_actions(struct hist_trigger_data *hist_data)
4773 {
4774         unsigned int i;
4775
4776         for (i = 0; i < hist_data->n_actions; i++) {
4777                 struct action_data *data = hist_data->actions[i];
4778
4779                 if (data->handler == HANDLER_ONMATCH)
4780                         onmatch_destroy(data);
4781                 else if (data->handler == HANDLER_ONMAX ||
4782                          data->handler == HANDLER_ONCHANGE)
4783                         track_data_destroy(hist_data, data);
4784                 else
4785                         kfree(data);
4786         }
4787 }
4788
4789 static int parse_actions(struct hist_trigger_data *hist_data)
4790 {
4791         struct trace_array *tr = hist_data->event_file->tr;
4792         struct action_data *data;
4793         unsigned int i;
4794         int ret = 0;
4795         char *str;
4796         int len;
4797
4798         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4799                 str = hist_data->attrs->action_str[i];
4800
4801                 if ((len = str_has_prefix(str, "onmatch("))) {
4802                         char *action_str = str + len;
4803
4804                         data = onmatch_parse(tr, action_str);
4805                         if (IS_ERR(data)) {
4806                                 ret = PTR_ERR(data);
4807                                 break;
4808                         }
4809                 } else if ((len = str_has_prefix(str, "onmax("))) {
4810                         char *action_str = str + len;
4811
4812                         data = track_data_parse(hist_data, action_str,
4813                                                 HANDLER_ONMAX);
4814                         if (IS_ERR(data)) {
4815                                 ret = PTR_ERR(data);
4816                                 break;
4817                         }
4818                 } else if ((len = str_has_prefix(str, "onchange("))) {
4819                         char *action_str = str + len;
4820
4821                         data = track_data_parse(hist_data, action_str,
4822                                                 HANDLER_ONCHANGE);
4823                         if (IS_ERR(data)) {
4824                                 ret = PTR_ERR(data);
4825                                 break;
4826                         }
4827                 } else {
4828                         ret = -EINVAL;
4829                         break;
4830                 }
4831
4832                 hist_data->actions[hist_data->n_actions++] = data;
4833         }
4834
4835         return ret;
4836 }
4837
4838 static int create_actions(struct hist_trigger_data *hist_data)
4839 {
4840         struct action_data *data;
4841         unsigned int i;
4842         int ret = 0;
4843
4844         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4845                 data = hist_data->actions[i];
4846
4847                 if (data->handler == HANDLER_ONMATCH) {
4848                         ret = onmatch_create(hist_data, data);
4849                         if (ret)
4850                                 break;
4851                 } else if (data->handler == HANDLER_ONMAX ||
4852                            data->handler == HANDLER_ONCHANGE) {
4853                         ret = track_data_create(hist_data, data);
4854                         if (ret)
4855                                 break;
4856                 } else {
4857                         ret = -EINVAL;
4858                         break;
4859                 }
4860         }
4861
4862         return ret;
4863 }
4864
4865 static void print_actions(struct seq_file *m,
4866                           struct hist_trigger_data *hist_data,
4867                           struct tracing_map_elt *elt)
4868 {
4869         unsigned int i;
4870
4871         for (i = 0; i < hist_data->n_actions; i++) {
4872                 struct action_data *data = hist_data->actions[i];
4873
4874                 if (data->action == ACTION_SNAPSHOT)
4875                         continue;
4876
4877                 if (data->handler == HANDLER_ONMAX ||
4878                     data->handler == HANDLER_ONCHANGE)
4879                         track_data_print(m, hist_data, elt, data);
4880         }
4881 }
4882
4883 static void print_action_spec(struct seq_file *m,
4884                               struct hist_trigger_data *hist_data,
4885                               struct action_data *data)
4886 {
4887         unsigned int i;
4888
4889         if (data->action == ACTION_SAVE) {
4890                 for (i = 0; i < hist_data->n_save_vars; i++) {
4891                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4892                         if (i < hist_data->n_save_vars - 1)
4893                                 seq_puts(m, ",");
4894                 }
4895         } else if (data->action == ACTION_TRACE) {
4896                 if (data->use_trace_keyword)
4897                         seq_printf(m, "%s", data->synth_event_name);
4898                 for (i = 0; i < data->n_params; i++) {
4899                         if (i || data->use_trace_keyword)
4900                                 seq_puts(m, ",");
4901                         seq_printf(m, "%s", data->params[i]);
4902                 }
4903         }
4904 }
4905
4906 static void print_track_data_spec(struct seq_file *m,
4907                                   struct hist_trigger_data *hist_data,
4908                                   struct action_data *data)
4909 {
4910         if (data->handler == HANDLER_ONMAX)
4911                 seq_puts(m, ":onmax(");
4912         else if (data->handler == HANDLER_ONCHANGE)
4913                 seq_puts(m, ":onchange(");
4914         seq_printf(m, "%s", data->track_data.var_str);
4915         seq_printf(m, ").%s(", data->action_name);
4916
4917         print_action_spec(m, hist_data, data);
4918
4919         seq_puts(m, ")");
4920 }
4921
4922 static void print_onmatch_spec(struct seq_file *m,
4923                                struct hist_trigger_data *hist_data,
4924                                struct action_data *data)
4925 {
4926         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4927                    data->match_data.event);
4928
4929         seq_printf(m, "%s(", data->action_name);
4930
4931         print_action_spec(m, hist_data, data);
4932
4933         seq_puts(m, ")");
4934 }
4935
4936 static bool actions_match(struct hist_trigger_data *hist_data,
4937                           struct hist_trigger_data *hist_data_test)
4938 {
4939         unsigned int i, j;
4940
4941         if (hist_data->n_actions != hist_data_test->n_actions)
4942                 return false;
4943
4944         for (i = 0; i < hist_data->n_actions; i++) {
4945                 struct action_data *data = hist_data->actions[i];
4946                 struct action_data *data_test = hist_data_test->actions[i];
4947                 char *action_name, *action_name_test;
4948
4949                 if (data->handler != data_test->handler)
4950                         return false;
4951                 if (data->action != data_test->action)
4952                         return false;
4953
4954                 if (data->n_params != data_test->n_params)
4955                         return false;
4956
4957                 for (j = 0; j < data->n_params; j++) {
4958                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4959                                 return false;
4960                 }
4961
4962                 if (data->use_trace_keyword)
4963                         action_name = data->synth_event_name;
4964                 else
4965                         action_name = data->action_name;
4966
4967                 if (data_test->use_trace_keyword)
4968                         action_name_test = data_test->synth_event_name;
4969                 else
4970                         action_name_test = data_test->action_name;
4971
4972                 if (strcmp(action_name, action_name_test) != 0)
4973                         return false;
4974
4975                 if (data->handler == HANDLER_ONMATCH) {
4976                         if (strcmp(data->match_data.event_system,
4977                                    data_test->match_data.event_system) != 0)
4978                                 return false;
4979                         if (strcmp(data->match_data.event,
4980                                    data_test->match_data.event) != 0)
4981                                 return false;
4982                 } else if (data->handler == HANDLER_ONMAX ||
4983                            data->handler == HANDLER_ONCHANGE) {
4984                         if (strcmp(data->track_data.var_str,
4985                                    data_test->track_data.var_str) != 0)
4986                                 return false;
4987                 }
4988         }
4989
4990         return true;
4991 }
4992
4993
4994 static void print_actions_spec(struct seq_file *m,
4995                                struct hist_trigger_data *hist_data)
4996 {
4997         unsigned int i;
4998
4999         for (i = 0; i < hist_data->n_actions; i++) {
5000                 struct action_data *data = hist_data->actions[i];
5001
5002                 if (data->handler == HANDLER_ONMATCH)
5003                         print_onmatch_spec(m, hist_data, data);
5004                 else if (data->handler == HANDLER_ONMAX ||
5005                          data->handler == HANDLER_ONCHANGE)
5006                         print_track_data_spec(m, hist_data, data);
5007         }
5008 }
5009
5010 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5011 {
5012         unsigned int i;
5013
5014         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5015                 kfree(hist_data->field_var_hists[i]->cmd);
5016                 kfree(hist_data->field_var_hists[i]);
5017         }
5018 }
5019
5020 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5021 {
5022         if (!hist_data)
5023                 return;
5024
5025         destroy_hist_trigger_attrs(hist_data->attrs);
5026         destroy_hist_fields(hist_data);
5027         tracing_map_destroy(hist_data->map);
5028
5029         destroy_actions(hist_data);
5030         destroy_field_vars(hist_data);
5031         destroy_field_var_hists(hist_data);
5032
5033         kfree(hist_data);
5034 }
5035
5036 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5037 {
5038         struct tracing_map *map = hist_data->map;
5039         struct ftrace_event_field *field;
5040         struct hist_field *hist_field;
5041         int i, idx = 0;
5042
5043         for_each_hist_field(i, hist_data) {
5044                 hist_field = hist_data->fields[i];
5045                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5046                         tracing_map_cmp_fn_t cmp_fn;
5047
5048                         field = hist_field->field;
5049
5050                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5051                                 cmp_fn = tracing_map_cmp_none;
5052                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5053                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5054                                                              hist_field->is_signed);
5055                         else if (is_string_field(field))
5056                                 cmp_fn = tracing_map_cmp_string;
5057                         else
5058                                 cmp_fn = tracing_map_cmp_num(field->size,
5059                                                              field->is_signed);
5060                         idx = tracing_map_add_key_field(map,
5061                                                         hist_field->offset,
5062                                                         cmp_fn);
5063                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5064                         idx = tracing_map_add_sum_field(map);
5065
5066                 if (idx < 0)
5067                         return idx;
5068
5069                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5070                         idx = tracing_map_add_var(map);
5071                         if (idx < 0)
5072                                 return idx;
5073                         hist_field->var.idx = idx;
5074                         hist_field->var.hist_data = hist_data;
5075                 }
5076         }
5077
5078         return 0;
5079 }
5080
5081 static struct hist_trigger_data *
5082 create_hist_data(unsigned int map_bits,
5083                  struct hist_trigger_attrs *attrs,
5084                  struct trace_event_file *file,
5085                  bool remove)
5086 {
5087         const struct tracing_map_ops *map_ops = NULL;
5088         struct hist_trigger_data *hist_data;
5089         int ret = 0;
5090
5091         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5092         if (!hist_data)
5093                 return ERR_PTR(-ENOMEM);
5094
5095         hist_data->attrs = attrs;
5096         hist_data->remove = remove;
5097         hist_data->event_file = file;
5098
5099         ret = parse_actions(hist_data);
5100         if (ret)
5101                 goto free;
5102
5103         ret = create_hist_fields(hist_data, file);
5104         if (ret)
5105                 goto free;
5106
5107         ret = create_sort_keys(hist_data);
5108         if (ret)
5109                 goto free;
5110
5111         map_ops = &hist_trigger_elt_data_ops;
5112
5113         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5114                                             map_ops, hist_data);
5115         if (IS_ERR(hist_data->map)) {
5116                 ret = PTR_ERR(hist_data->map);
5117                 hist_data->map = NULL;
5118                 goto free;
5119         }
5120
5121         ret = create_tracing_map_fields(hist_data);
5122         if (ret)
5123                 goto free;
5124  out:
5125         return hist_data;
5126  free:
5127         hist_data->attrs = NULL;
5128
5129         destroy_hist_data(hist_data);
5130
5131         hist_data = ERR_PTR(ret);
5132
5133         goto out;
5134 }
5135
5136 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5137                                     struct tracing_map_elt *elt,
5138                                     struct trace_buffer *buffer, void *rec,
5139                                     struct ring_buffer_event *rbe,
5140                                     u64 *var_ref_vals)
5141 {
5142         struct hist_elt_data *elt_data;
5143         struct hist_field *hist_field;
5144         unsigned int i, var_idx;
5145         u64 hist_val;
5146
5147         elt_data = elt->private_data;
5148         elt_data->var_ref_vals = var_ref_vals;
5149
5150         for_each_hist_val_field(i, hist_data) {
5151                 hist_field = hist_data->fields[i];
5152                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5153                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5154                         var_idx = hist_field->var.idx;
5155
5156                         if (hist_field->flags &
5157                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5158                                 unsigned int str_start, var_str_idx, idx;
5159                                 char *str, *val_str;
5160                                 unsigned int size;
5161
5162                                 str_start = hist_data->n_field_var_str +
5163                                         hist_data->n_save_var_str;
5164                                 var_str_idx = hist_field->var_str_idx;
5165                                 idx = str_start + var_str_idx;
5166
5167                                 str = elt_data->field_var_str[idx];
5168                                 val_str = (char *)(uintptr_t)hist_val;
5169
5170                                 if (hist_field->flags & HIST_FIELD_FL_STRING) {
5171                                         size = min(hist_field->size, STR_VAR_LEN_MAX);
5172                                         strscpy(str, val_str, size);
5173                                 } else {
5174                                         char *stack_start = str + sizeof(unsigned long);
5175                                         int e;
5176
5177                                         e = stack_trace_save((void *)stack_start,
5178                                                              HIST_STACKTRACE_DEPTH,
5179                                                              HIST_STACKTRACE_SKIP);
5180                                         if (e < HIST_STACKTRACE_DEPTH - 1)
5181                                                 ((unsigned long *)stack_start)[e] = 0;
5182                                         *((unsigned long *)str) = e;
5183                                 }
5184                                 hist_val = (u64)(uintptr_t)str;
5185                         }
5186                         tracing_map_set_var(elt, var_idx, hist_val);
5187                         continue;
5188                 }
5189                 tracing_map_update_sum(elt, i, hist_val);
5190         }
5191
5192         for_each_hist_key_field(i, hist_data) {
5193                 hist_field = hist_data->fields[i];
5194                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5195                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5196                         var_idx = hist_field->var.idx;
5197                         tracing_map_set_var(elt, var_idx, hist_val);
5198                 }
5199         }
5200
5201         update_field_vars(hist_data, elt, buffer, rbe, rec);
5202 }
5203
5204 static inline void add_to_key(char *compound_key, void *key,
5205                               struct hist_field *key_field, void *rec)
5206 {
5207         size_t size = key_field->size;
5208
5209         if (key_field->flags & HIST_FIELD_FL_STRING) {
5210                 struct ftrace_event_field *field;
5211
5212                 field = key_field->field;
5213                 if (field->filter_type == FILTER_DYN_STRING ||
5214                     field->filter_type == FILTER_RDYN_STRING)
5215                         size = *(u32 *)(rec + field->offset) >> 16;
5216                 else if (field->filter_type == FILTER_STATIC_STRING)
5217                         size = field->size;
5218
5219                 /* ensure NULL-termination */
5220                 if (size > key_field->size - 1)
5221                         size = key_field->size - 1;
5222
5223                 strncpy(compound_key + key_field->offset, (char *)key, size);
5224         } else
5225                 memcpy(compound_key + key_field->offset, key, size);
5226 }
5227
5228 static void
5229 hist_trigger_actions(struct hist_trigger_data *hist_data,
5230                      struct tracing_map_elt *elt,
5231                      struct trace_buffer *buffer, void *rec,
5232                      struct ring_buffer_event *rbe, void *key,
5233                      u64 *var_ref_vals)
5234 {
5235         struct action_data *data;
5236         unsigned int i;
5237
5238         for (i = 0; i < hist_data->n_actions; i++) {
5239                 data = hist_data->actions[i];
5240                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5241         }
5242 }
5243
5244 static void event_hist_trigger(struct event_trigger_data *data,
5245                                struct trace_buffer *buffer, void *rec,
5246                                struct ring_buffer_event *rbe)
5247 {
5248         struct hist_trigger_data *hist_data = data->private_data;
5249         bool use_compound_key = (hist_data->n_keys > 1);
5250         unsigned long entries[HIST_STACKTRACE_DEPTH];
5251         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5252         char compound_key[HIST_KEY_SIZE_MAX];
5253         struct tracing_map_elt *elt = NULL;
5254         struct hist_field *key_field;
5255         u64 field_contents;
5256         void *key = NULL;
5257         unsigned int i;
5258
5259         if (unlikely(!rbe))
5260                 return;
5261
5262         memset(compound_key, 0, hist_data->key_size);
5263
5264         for_each_hist_key_field(i, hist_data) {
5265                 key_field = hist_data->fields[i];
5266
5267                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5268                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5269                         if (key_field->field) {
5270                                 unsigned long *stack, n_entries;
5271
5272                                 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5273                                 stack = (unsigned long *)(long)field_contents;
5274                                 n_entries = *stack;
5275                                 memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5276                         } else {
5277                                 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5278                                                  HIST_STACKTRACE_SKIP);
5279                         }
5280                         key = entries;
5281                 } else {
5282                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5283                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5284                                 key = (void *)(unsigned long)field_contents;
5285                                 use_compound_key = true;
5286                         } else
5287                                 key = (void *)&field_contents;
5288                 }
5289
5290                 if (use_compound_key)
5291                         add_to_key(compound_key, key, key_field, rec);
5292         }
5293
5294         if (use_compound_key)
5295                 key = compound_key;
5296
5297         if (hist_data->n_var_refs &&
5298             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5299                 return;
5300
5301         elt = tracing_map_insert(hist_data->map, key);
5302         if (!elt)
5303                 return;
5304
5305         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5306
5307         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5308                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5309 }
5310
5311 static void hist_trigger_stacktrace_print(struct seq_file *m,
5312                                           unsigned long *stacktrace_entries,
5313                                           unsigned int max_entries)
5314 {
5315         unsigned int spaces = 8;
5316         unsigned int i;
5317
5318         for (i = 0; i < max_entries; i++) {
5319                 if (!stacktrace_entries[i])
5320                         return;
5321
5322                 seq_printf(m, "%*c", 1 + spaces, ' ');
5323                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5324         }
5325 }
5326
5327 static void hist_trigger_print_key(struct seq_file *m,
5328                                    struct hist_trigger_data *hist_data,
5329                                    void *key,
5330                                    struct tracing_map_elt *elt)
5331 {
5332         struct hist_field *key_field;
5333         bool multiline = false;
5334         const char *field_name;
5335         unsigned int i;
5336         u64 uval;
5337
5338         seq_puts(m, "{ ");
5339
5340         for_each_hist_key_field(i, hist_data) {
5341                 key_field = hist_data->fields[i];
5342
5343                 if (i > hist_data->n_vals)
5344                         seq_puts(m, ", ");
5345
5346                 field_name = hist_field_name(key_field, 0);
5347
5348                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5349                         uval = *(u64 *)(key + key_field->offset);
5350                         seq_printf(m, "%s: %llx", field_name, uval);
5351                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5352                         uval = *(u64 *)(key + key_field->offset);
5353                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5354                                    uval, (void *)(uintptr_t)uval);
5355                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5356                         uval = *(u64 *)(key + key_field->offset);
5357                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5358                                    uval, (void *)(uintptr_t)uval);
5359                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5360                         struct hist_elt_data *elt_data = elt->private_data;
5361                         char *comm;
5362
5363                         if (WARN_ON_ONCE(!elt_data))
5364                                 return;
5365
5366                         comm = elt_data->comm;
5367
5368                         uval = *(u64 *)(key + key_field->offset);
5369                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5370                                    comm, uval);
5371                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5372                         const char *syscall_name;
5373
5374                         uval = *(u64 *)(key + key_field->offset);
5375                         syscall_name = get_syscall_name(uval);
5376                         if (!syscall_name)
5377                                 syscall_name = "unknown_syscall";
5378
5379                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5380                                    syscall_name, uval);
5381                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5382                         if (key_field->field)
5383                                 seq_printf(m, "%s.stacktrace", key_field->field->name);
5384                         else
5385                                 seq_puts(m, "stacktrace:\n");
5386                         hist_trigger_stacktrace_print(m,
5387                                                       key + key_field->offset,
5388                                                       HIST_STACKTRACE_DEPTH);
5389                         multiline = true;
5390                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5391                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5392                                    *(u64 *)(key + key_field->offset));
5393                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5394                         unsigned long buckets = key_field->buckets;
5395                         uval = *(u64 *)(key + key_field->offset);
5396                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5397                                    uval, uval + buckets -1);
5398                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5399                         seq_printf(m, "%s: %-50s", field_name,
5400                                    (char *)(key + key_field->offset));
5401                 } else {
5402                         uval = *(u64 *)(key + key_field->offset);
5403                         seq_printf(m, "%s: %10llu", field_name, uval);
5404                 }
5405         }
5406
5407         if (!multiline)
5408                 seq_puts(m, " ");
5409
5410         seq_puts(m, "}");
5411 }
5412
5413 /* Get the 100 times of the percentage of @val in @total */
5414 static inline unsigned int __get_percentage(u64 val, u64 total)
5415 {
5416         if (!total)
5417                 goto div0;
5418
5419         if (val < (U64_MAX / 10000))
5420                 return (unsigned int)div64_ul(val * 10000, total);
5421
5422         total = div64_u64(total, 10000);
5423         if (!total)
5424                 goto div0;
5425
5426         return (unsigned int)div64_ul(val, total);
5427 div0:
5428         return val ? UINT_MAX : 0;
5429 }
5430
5431 #define BAR_CHAR '#'
5432
5433 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5434 {
5435         unsigned int len = __get_percentage(val, max);
5436         int i;
5437
5438         if (len == UINT_MAX) {
5439                 snprintf(buf, size, "[ERROR]");
5440                 return buf;
5441         }
5442
5443         len = len * size / 10000;
5444         for (i = 0; i < len && i < size; i++)
5445                 buf[i] = BAR_CHAR;
5446         while (i < size)
5447                 buf[i++] = ' ';
5448         buf[size] = '\0';
5449
5450         return buf;
5451 }
5452
5453 struct hist_val_stat {
5454         u64 max;
5455         u64 total;
5456 };
5457
5458 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5459                                    const char *field_name, unsigned long flags,
5460                                    struct hist_val_stat *stats,
5461                                    struct tracing_map_elt *elt)
5462 {
5463         u64 val = tracing_map_read_sum(elt, idx);
5464         unsigned int pc;
5465         char bar[21];
5466
5467         if (flags & HIST_FIELD_FL_PERCENT) {
5468                 pc = __get_percentage(val, stats[idx].total);
5469                 if (pc == UINT_MAX)
5470                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5471                 else
5472                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5473                                         pc / 100, pc % 100);
5474         } else if (flags & HIST_FIELD_FL_GRAPH) {
5475                 seq_printf(m, " %s: %20s", field_name,
5476                            __fill_bar_str(bar, 20, val, stats[idx].max));
5477         } else if (flags & HIST_FIELD_FL_HEX) {
5478                 seq_printf(m, " %s: %10llx", field_name, val);
5479         } else {
5480                 seq_printf(m, " %s: %10llu", field_name, val);
5481         }
5482 }
5483
5484 static void hist_trigger_entry_print(struct seq_file *m,
5485                                      struct hist_trigger_data *hist_data,
5486                                      struct hist_val_stat *stats,
5487                                      void *key,
5488                                      struct tracing_map_elt *elt)
5489 {
5490         const char *field_name;
5491         unsigned int i = HITCOUNT_IDX;
5492         unsigned long flags;
5493
5494         hist_trigger_print_key(m, hist_data, key, elt);
5495
5496         /* At first, show the raw hitcount if !nohitcount */
5497         if (!hist_data->attrs->no_hitcount)
5498                 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5499
5500         for (i = 1; i < hist_data->n_vals; i++) {
5501                 field_name = hist_field_name(hist_data->fields[i], 0);
5502                 flags = hist_data->fields[i]->flags;
5503                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5504                         continue;
5505
5506                 seq_puts(m, " ");
5507                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5508         }
5509
5510         print_actions(m, hist_data, elt);
5511
5512         seq_puts(m, "\n");
5513 }
5514
5515 static int print_entries(struct seq_file *m,
5516                          struct hist_trigger_data *hist_data)
5517 {
5518         struct tracing_map_sort_entry **sort_entries = NULL;
5519         struct tracing_map *map = hist_data->map;
5520         int i, j, n_entries;
5521         struct hist_val_stat *stats = NULL;
5522         u64 val;
5523
5524         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5525                                              hist_data->n_sort_keys,
5526                                              &sort_entries);
5527         if (n_entries < 0)
5528                 return n_entries;
5529
5530         /* Calculate the max and the total for each field if needed. */
5531         for (j = 0; j < hist_data->n_vals; j++) {
5532                 if (!(hist_data->fields[j]->flags &
5533                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5534                         continue;
5535                 if (!stats) {
5536                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5537                                        GFP_KERNEL);
5538                         if (!stats) {
5539                                 n_entries = -ENOMEM;
5540                                 goto out;
5541                         }
5542                 }
5543                 for (i = 0; i < n_entries; i++) {
5544                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5545                         stats[j].total += val;
5546                         if (stats[j].max < val)
5547                                 stats[j].max = val;
5548                 }
5549         }
5550
5551         for (i = 0; i < n_entries; i++)
5552                 hist_trigger_entry_print(m, hist_data, stats,
5553                                          sort_entries[i]->key,
5554                                          sort_entries[i]->elt);
5555
5556         kfree(stats);
5557 out:
5558         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5559
5560         return n_entries;
5561 }
5562
5563 static void hist_trigger_show(struct seq_file *m,
5564                               struct event_trigger_data *data, int n)
5565 {
5566         struct hist_trigger_data *hist_data;
5567         int n_entries;
5568
5569         if (n > 0)
5570                 seq_puts(m, "\n\n");
5571
5572         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5573         data->ops->print(m, data);
5574         seq_puts(m, "#\n\n");
5575
5576         hist_data = data->private_data;
5577         n_entries = print_entries(m, hist_data);
5578         if (n_entries < 0)
5579                 n_entries = 0;
5580
5581         track_data_snapshot_print(m, hist_data);
5582
5583         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5584                    (u64)atomic64_read(&hist_data->map->hits),
5585                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5586 }
5587
5588 static int hist_show(struct seq_file *m, void *v)
5589 {
5590         struct event_trigger_data *data;
5591         struct trace_event_file *event_file;
5592         int n = 0, ret = 0;
5593
5594         mutex_lock(&event_mutex);
5595
5596         event_file = event_file_data(m->private);
5597         if (unlikely(!event_file)) {
5598                 ret = -ENODEV;
5599                 goto out_unlock;
5600         }
5601
5602         list_for_each_entry(data, &event_file->triggers, list) {
5603                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5604                         hist_trigger_show(m, data, n++);
5605         }
5606
5607  out_unlock:
5608         mutex_unlock(&event_mutex);
5609
5610         return ret;
5611 }
5612
5613 static int event_hist_open(struct inode *inode, struct file *file)
5614 {
5615         int ret;
5616
5617         ret = security_locked_down(LOCKDOWN_TRACEFS);
5618         if (ret)
5619                 return ret;
5620
5621         return single_open(file, hist_show, file);
5622 }
5623
5624 const struct file_operations event_hist_fops = {
5625         .open = event_hist_open,
5626         .read = seq_read,
5627         .llseek = seq_lseek,
5628         .release = single_release,
5629 };
5630
5631 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5632 static void hist_field_debug_show_flags(struct seq_file *m,
5633                                         unsigned long flags)
5634 {
5635         seq_puts(m, "      flags:\n");
5636
5637         if (flags & HIST_FIELD_FL_KEY)
5638                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5639         else if (flags & HIST_FIELD_FL_HITCOUNT)
5640                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5641         else if (flags & HIST_FIELD_FL_VAR)
5642                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5643         else if (flags & HIST_FIELD_FL_VAR_REF)
5644                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5645         else
5646                 seq_puts(m, "        VAL: normal u64 value\n");
5647
5648         if (flags & HIST_FIELD_FL_ALIAS)
5649                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5650         else if (flags & HIST_FIELD_FL_CONST)
5651                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5652 }
5653
5654 static int hist_field_debug_show(struct seq_file *m,
5655                                  struct hist_field *field, unsigned long flags)
5656 {
5657         if ((field->flags & flags) != flags) {
5658                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5659                 return -EINVAL;
5660         }
5661
5662         hist_field_debug_show_flags(m, field->flags);
5663         if (field->field)
5664                 seq_printf(m, "      ftrace_event_field name: %s\n",
5665                            field->field->name);
5666
5667         if (field->flags & HIST_FIELD_FL_VAR) {
5668                 seq_printf(m, "      var.name: %s\n", field->var.name);
5669                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5670                            field->var.idx);
5671         }
5672
5673         if (field->flags & HIST_FIELD_FL_CONST)
5674                 seq_printf(m, "      constant: %llu\n", field->constant);
5675
5676         if (field->flags & HIST_FIELD_FL_ALIAS)
5677                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5678                            field->var_ref_idx);
5679
5680         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5681                 seq_printf(m, "      name: %s\n", field->name);
5682                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5683                            field->var.idx);
5684                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5685                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5686                            field->var_ref_idx);
5687                 if (field->system)
5688                         seq_printf(m, "      system: %s\n", field->system);
5689                 if (field->event_name)
5690                         seq_printf(m, "      event_name: %s\n", field->event_name);
5691         }
5692
5693         seq_printf(m, "      type: %s\n", field->type);
5694         seq_printf(m, "      size: %u\n", field->size);
5695         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5696
5697         return 0;
5698 }
5699
5700 static int field_var_debug_show(struct seq_file *m,
5701                                 struct field_var *field_var, unsigned int i,
5702                                 bool save_vars)
5703 {
5704         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5705         struct hist_field *field;
5706         int ret = 0;
5707
5708         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5709
5710         field = field_var->var;
5711
5712         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5713
5714         hist_field_debug_show_flags(m, field->flags);
5715         seq_printf(m, "      var.name: %s\n", field->var.name);
5716         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5717                    field->var.idx);
5718
5719         field = field_var->val;
5720
5721         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5722         if (field->field)
5723                 seq_printf(m, "      ftrace_event_field name: %s\n",
5724                            field->field->name);
5725         else {
5726                 ret = -EINVAL;
5727                 goto out;
5728         }
5729
5730         seq_printf(m, "      type: %s\n", field->type);
5731         seq_printf(m, "      size: %u\n", field->size);
5732         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5733 out:
5734         return ret;
5735 }
5736
5737 static int hist_action_debug_show(struct seq_file *m,
5738                                   struct action_data *data, int i)
5739 {
5740         int ret = 0;
5741
5742         if (data->handler == HANDLER_ONMAX ||
5743             data->handler == HANDLER_ONCHANGE) {
5744                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5745                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5746                                             HIST_FIELD_FL_VAR_REF);
5747                 if (ret)
5748                         goto out;
5749
5750                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5751                 ret = hist_field_debug_show(m, data->track_data.track_var,
5752                                             HIST_FIELD_FL_VAR);
5753                 if (ret)
5754                         goto out;
5755         }
5756
5757         if (data->handler == HANDLER_ONMATCH) {
5758                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5759                            i, data->match_data.event_system);
5760                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5761                            i, data->match_data.event);
5762         }
5763 out:
5764         return ret;
5765 }
5766
5767 static int hist_actions_debug_show(struct seq_file *m,
5768                                    struct hist_trigger_data *hist_data)
5769 {
5770         int i, ret = 0;
5771
5772         if (hist_data->n_actions)
5773                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5774
5775         for (i = 0; i < hist_data->n_actions; i++) {
5776                 struct action_data *action = hist_data->actions[i];
5777
5778                 ret = hist_action_debug_show(m, action, i);
5779                 if (ret)
5780                         goto out;
5781         }
5782
5783         if (hist_data->n_save_vars)
5784                 seq_puts(m, "\n  save action variables (save() params):\n");
5785
5786         for (i = 0; i < hist_data->n_save_vars; i++) {
5787                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5788                 if (ret)
5789                         goto out;
5790         }
5791 out:
5792         return ret;
5793 }
5794
5795 static void hist_trigger_debug_show(struct seq_file *m,
5796                                     struct event_trigger_data *data, int n)
5797 {
5798         struct hist_trigger_data *hist_data;
5799         int i, ret;
5800
5801         if (n > 0)
5802                 seq_puts(m, "\n\n");
5803
5804         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5805         data->ops->print(m, data);
5806         seq_puts(m, "#\n\n");
5807
5808         hist_data = data->private_data;
5809
5810         seq_printf(m, "hist_data: %p\n\n", hist_data);
5811         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5812         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5813         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5814
5815         seq_puts(m, "\n  val fields:\n\n");
5816
5817         seq_puts(m, "    hist_data->fields[0]:\n");
5818         ret = hist_field_debug_show(m, hist_data->fields[0],
5819                                     HIST_FIELD_FL_HITCOUNT);
5820         if (ret)
5821                 return;
5822
5823         for (i = 1; i < hist_data->n_vals; i++) {
5824                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5825                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5826                 if (ret)
5827                         return;
5828         }
5829
5830         seq_puts(m, "\n  key fields:\n");
5831
5832         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5833                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5834                 ret = hist_field_debug_show(m, hist_data->fields[i],
5835                                             HIST_FIELD_FL_KEY);
5836                 if (ret)
5837                         return;
5838         }
5839
5840         if (hist_data->n_var_refs)
5841                 seq_puts(m, "\n  variable reference fields:\n");
5842
5843         for (i = 0; i < hist_data->n_var_refs; i++) {
5844                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5845                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5846                                             HIST_FIELD_FL_VAR_REF);
5847                 if (ret)
5848                         return;
5849         }
5850
5851         if (hist_data->n_field_vars)
5852                 seq_puts(m, "\n  field variables:\n");
5853
5854         for (i = 0; i < hist_data->n_field_vars; i++) {
5855                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5856                 if (ret)
5857                         return;
5858         }
5859
5860         ret = hist_actions_debug_show(m, hist_data);
5861         if (ret)
5862                 return;
5863 }
5864
5865 static int hist_debug_show(struct seq_file *m, void *v)
5866 {
5867         struct event_trigger_data *data;
5868         struct trace_event_file *event_file;
5869         int n = 0, ret = 0;
5870
5871         mutex_lock(&event_mutex);
5872
5873         event_file = event_file_data(m->private);
5874         if (unlikely(!event_file)) {
5875                 ret = -ENODEV;
5876                 goto out_unlock;
5877         }
5878
5879         list_for_each_entry(data, &event_file->triggers, list) {
5880                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5881                         hist_trigger_debug_show(m, data, n++);
5882         }
5883
5884  out_unlock:
5885         mutex_unlock(&event_mutex);
5886
5887         return ret;
5888 }
5889
5890 static int event_hist_debug_open(struct inode *inode, struct file *file)
5891 {
5892         int ret;
5893
5894         ret = security_locked_down(LOCKDOWN_TRACEFS);
5895         if (ret)
5896                 return ret;
5897
5898         return single_open(file, hist_debug_show, file);
5899 }
5900
5901 const struct file_operations event_hist_debug_fops = {
5902         .open = event_hist_debug_open,
5903         .read = seq_read,
5904         .llseek = seq_lseek,
5905         .release = single_release,
5906 };
5907 #endif
5908
5909 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5910 {
5911         const char *field_name = hist_field_name(hist_field, 0);
5912
5913         if (hist_field->var.name)
5914                 seq_printf(m, "%s=", hist_field->var.name);
5915
5916         if (hist_field->flags & HIST_FIELD_FL_CPU)
5917                 seq_puts(m, "common_cpu");
5918         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5919                 seq_printf(m, "%llu", hist_field->constant);
5920         else if (field_name) {
5921                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5922                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5923                         seq_putc(m, '$');
5924                 seq_printf(m, "%s", field_name);
5925         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5926                 seq_puts(m, "common_timestamp");
5927
5928         if (hist_field->flags) {
5929                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5930                     !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
5931                     !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
5932                         const char *flags = get_hist_field_flags(hist_field);
5933
5934                         if (flags)
5935                                 seq_printf(m, ".%s", flags);
5936                 }
5937         }
5938         if (hist_field->buckets)
5939                 seq_printf(m, "=%ld", hist_field->buckets);
5940 }
5941
5942 static int event_hist_trigger_print(struct seq_file *m,
5943                                     struct event_trigger_data *data)
5944 {
5945         struct hist_trigger_data *hist_data = data->private_data;
5946         struct hist_field *field;
5947         bool have_var = false;
5948         bool show_val = false;
5949         unsigned int i;
5950
5951         seq_puts(m, HIST_PREFIX);
5952
5953         if (data->name)
5954                 seq_printf(m, "%s:", data->name);
5955
5956         seq_puts(m, "keys=");
5957
5958         for_each_hist_key_field(i, hist_data) {
5959                 field = hist_data->fields[i];
5960
5961                 if (i > hist_data->n_vals)
5962                         seq_puts(m, ",");
5963
5964                 if (field->flags & HIST_FIELD_FL_STACKTRACE) {
5965                         if (field->field)
5966                                 seq_printf(m, "%s.stacktrace", field->field->name);
5967                         else
5968                                 seq_puts(m, "stacktrace");
5969                 } else
5970                         hist_field_print(m, field);
5971         }
5972
5973         seq_puts(m, ":vals=");
5974
5975         for_each_hist_val_field(i, hist_data) {
5976                 field = hist_data->fields[i];
5977                 if (field->flags & HIST_FIELD_FL_VAR) {
5978                         have_var = true;
5979                         continue;
5980                 }
5981
5982                 if (i == HITCOUNT_IDX) {
5983                         if (hist_data->attrs->no_hitcount)
5984                                 continue;
5985                         seq_puts(m, "hitcount");
5986                 } else {
5987                         if (show_val)
5988                                 seq_puts(m, ",");
5989                         hist_field_print(m, field);
5990                 }
5991                 show_val = true;
5992         }
5993
5994         if (have_var) {
5995                 unsigned int n = 0;
5996
5997                 seq_puts(m, ":");
5998
5999                 for_each_hist_val_field(i, hist_data) {
6000                         field = hist_data->fields[i];
6001
6002                         if (field->flags & HIST_FIELD_FL_VAR) {
6003                                 if (n++)
6004                                         seq_puts(m, ",");
6005                                 hist_field_print(m, field);
6006                         }
6007                 }
6008         }
6009
6010         seq_puts(m, ":sort=");
6011
6012         for (i = 0; i < hist_data->n_sort_keys; i++) {
6013                 struct tracing_map_sort_key *sort_key;
6014                 unsigned int idx, first_key_idx;
6015
6016                 /* skip VAR vals */
6017                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6018
6019                 sort_key = &hist_data->sort_keys[i];
6020                 idx = sort_key->field_idx;
6021
6022                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6023                         return -EINVAL;
6024
6025                 if (i > 0)
6026                         seq_puts(m, ",");
6027
6028                 if (idx == HITCOUNT_IDX)
6029                         seq_puts(m, "hitcount");
6030                 else {
6031                         if (idx >= first_key_idx)
6032                                 idx += hist_data->n_vars;
6033                         hist_field_print(m, hist_data->fields[idx]);
6034                 }
6035
6036                 if (sort_key->descending)
6037                         seq_puts(m, ".descending");
6038         }
6039         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6040         if (hist_data->enable_timestamps)
6041                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6042         if (hist_data->attrs->no_hitcount)
6043                 seq_puts(m, ":nohitcount");
6044
6045         print_actions_spec(m, hist_data);
6046
6047         if (data->filter_str)
6048                 seq_printf(m, " if %s", data->filter_str);
6049
6050         if (data->paused)
6051                 seq_puts(m, " [paused]");
6052         else
6053                 seq_puts(m, " [active]");
6054
6055         seq_putc(m, '\n');
6056
6057         return 0;
6058 }
6059
6060 static int event_hist_trigger_init(struct event_trigger_data *data)
6061 {
6062         struct hist_trigger_data *hist_data = data->private_data;
6063
6064         if (!data->ref && hist_data->attrs->name)
6065                 save_named_trigger(hist_data->attrs->name, data);
6066
6067         data->ref++;
6068
6069         return 0;
6070 }
6071
6072 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6073 {
6074         struct trace_event_file *file;
6075         unsigned int i;
6076         char *cmd;
6077         int ret;
6078
6079         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6080                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6081                 cmd = hist_data->field_var_hists[i]->cmd;
6082                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6083                                                "!hist", "hist", cmd);
6084                 WARN_ON_ONCE(ret < 0);
6085         }
6086 }
6087
6088 static void event_hist_trigger_free(struct event_trigger_data *data)
6089 {
6090         struct hist_trigger_data *hist_data = data->private_data;
6091
6092         if (WARN_ON_ONCE(data->ref <= 0))
6093                 return;
6094
6095         data->ref--;
6096         if (!data->ref) {
6097                 if (data->name)
6098                         del_named_trigger(data);
6099
6100                 trigger_data_free(data);
6101
6102                 remove_hist_vars(hist_data);
6103
6104                 unregister_field_var_hists(hist_data);
6105
6106                 destroy_hist_data(hist_data);
6107         }
6108 }
6109
6110 static struct event_trigger_ops event_hist_trigger_ops = {
6111         .trigger                = event_hist_trigger,
6112         .print                  = event_hist_trigger_print,
6113         .init                   = event_hist_trigger_init,
6114         .free                   = event_hist_trigger_free,
6115 };
6116
6117 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6118 {
6119         data->ref++;
6120
6121         save_named_trigger(data->named_data->name, data);
6122
6123         event_hist_trigger_init(data->named_data);
6124
6125         return 0;
6126 }
6127
6128 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6129 {
6130         if (WARN_ON_ONCE(data->ref <= 0))
6131                 return;
6132
6133         event_hist_trigger_free(data->named_data);
6134
6135         data->ref--;
6136         if (!data->ref) {
6137                 del_named_trigger(data);
6138                 trigger_data_free(data);
6139         }
6140 }
6141
6142 static struct event_trigger_ops event_hist_trigger_named_ops = {
6143         .trigger                = event_hist_trigger,
6144         .print                  = event_hist_trigger_print,
6145         .init                   = event_hist_trigger_named_init,
6146         .free                   = event_hist_trigger_named_free,
6147 };
6148
6149 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6150                                                             char *param)
6151 {
6152         return &event_hist_trigger_ops;
6153 }
6154
6155 static void hist_clear(struct event_trigger_data *data)
6156 {
6157         struct hist_trigger_data *hist_data = data->private_data;
6158
6159         if (data->name)
6160                 pause_named_trigger(data);
6161
6162         tracepoint_synchronize_unregister();
6163
6164         tracing_map_clear(hist_data->map);
6165
6166         if (data->name)
6167                 unpause_named_trigger(data);
6168 }
6169
6170 static bool compatible_field(struct ftrace_event_field *field,
6171                              struct ftrace_event_field *test_field)
6172 {
6173         if (field == test_field)
6174                 return true;
6175         if (field == NULL || test_field == NULL)
6176                 return false;
6177         if (strcmp(field->name, test_field->name) != 0)
6178                 return false;
6179         if (strcmp(field->type, test_field->type) != 0)
6180                 return false;
6181         if (field->size != test_field->size)
6182                 return false;
6183         if (field->is_signed != test_field->is_signed)
6184                 return false;
6185
6186         return true;
6187 }
6188
6189 static bool hist_trigger_match(struct event_trigger_data *data,
6190                                struct event_trigger_data *data_test,
6191                                struct event_trigger_data *named_data,
6192                                bool ignore_filter)
6193 {
6194         struct tracing_map_sort_key *sort_key, *sort_key_test;
6195         struct hist_trigger_data *hist_data, *hist_data_test;
6196         struct hist_field *key_field, *key_field_test;
6197         unsigned int i;
6198
6199         if (named_data && (named_data != data_test) &&
6200             (named_data != data_test->named_data))
6201                 return false;
6202
6203         if (!named_data && is_named_trigger(data_test))
6204                 return false;
6205
6206         hist_data = data->private_data;
6207         hist_data_test = data_test->private_data;
6208
6209         if (hist_data->n_vals != hist_data_test->n_vals ||
6210             hist_data->n_fields != hist_data_test->n_fields ||
6211             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6212                 return false;
6213
6214         if (!ignore_filter) {
6215                 if ((data->filter_str && !data_test->filter_str) ||
6216                    (!data->filter_str && data_test->filter_str))
6217                         return false;
6218         }
6219
6220         for_each_hist_field(i, hist_data) {
6221                 key_field = hist_data->fields[i];
6222                 key_field_test = hist_data_test->fields[i];
6223
6224                 if (key_field->flags != key_field_test->flags)
6225                         return false;
6226                 if (!compatible_field(key_field->field, key_field_test->field))
6227                         return false;
6228                 if (key_field->offset != key_field_test->offset)
6229                         return false;
6230                 if (key_field->size != key_field_test->size)
6231                         return false;
6232                 if (key_field->is_signed != key_field_test->is_signed)
6233                         return false;
6234                 if (!!key_field->var.name != !!key_field_test->var.name)
6235                         return false;
6236                 if (key_field->var.name &&
6237                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6238                         return false;
6239         }
6240
6241         for (i = 0; i < hist_data->n_sort_keys; i++) {
6242                 sort_key = &hist_data->sort_keys[i];
6243                 sort_key_test = &hist_data_test->sort_keys[i];
6244
6245                 if (sort_key->field_idx != sort_key_test->field_idx ||
6246                     sort_key->descending != sort_key_test->descending)
6247                         return false;
6248         }
6249
6250         if (!ignore_filter && data->filter_str &&
6251             (strcmp(data->filter_str, data_test->filter_str) != 0))
6252                 return false;
6253
6254         if (!actions_match(hist_data, hist_data_test))
6255                 return false;
6256
6257         return true;
6258 }
6259
6260 static bool existing_hist_update_only(char *glob,
6261                                       struct event_trigger_data *data,
6262                                       struct trace_event_file *file)
6263 {
6264         struct hist_trigger_data *hist_data = data->private_data;
6265         struct event_trigger_data *test, *named_data = NULL;
6266         bool updated = false;
6267
6268         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6269             !hist_data->attrs->clear)
6270                 goto out;
6271
6272         if (hist_data->attrs->name) {
6273                 named_data = find_named_trigger(hist_data->attrs->name);
6274                 if (named_data) {
6275                         if (!hist_trigger_match(data, named_data, named_data,
6276                                                 true))
6277                                 goto out;
6278                 }
6279         }
6280
6281         if (hist_data->attrs->name && !named_data)
6282                 goto out;
6283
6284         list_for_each_entry(test, &file->triggers, list) {
6285                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6286                         if (!hist_trigger_match(data, test, named_data, false))
6287                                 continue;
6288                         if (hist_data->attrs->pause)
6289                                 test->paused = true;
6290                         else if (hist_data->attrs->cont)
6291                                 test->paused = false;
6292                         else if (hist_data->attrs->clear)
6293                                 hist_clear(test);
6294                         updated = true;
6295                         goto out;
6296                 }
6297         }
6298  out:
6299         return updated;
6300 }
6301
6302 static int hist_register_trigger(char *glob,
6303                                  struct event_trigger_data *data,
6304                                  struct trace_event_file *file)
6305 {
6306         struct hist_trigger_data *hist_data = data->private_data;
6307         struct event_trigger_data *test, *named_data = NULL;
6308         struct trace_array *tr = file->tr;
6309         int ret = 0;
6310
6311         if (hist_data->attrs->name) {
6312                 named_data = find_named_trigger(hist_data->attrs->name);
6313                 if (named_data) {
6314                         if (!hist_trigger_match(data, named_data, named_data,
6315                                                 true)) {
6316                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6317                                 ret = -EINVAL;
6318                                 goto out;
6319                         }
6320                 }
6321         }
6322
6323         if (hist_data->attrs->name && !named_data)
6324                 goto new;
6325
6326         lockdep_assert_held(&event_mutex);
6327
6328         list_for_each_entry(test, &file->triggers, list) {
6329                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6330                         if (hist_trigger_match(data, test, named_data, false)) {
6331                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6332                                 ret = -EEXIST;
6333                                 goto out;
6334                         }
6335                 }
6336         }
6337  new:
6338         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6339                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6340                 ret = -ENOENT;
6341                 goto out;
6342         }
6343
6344         if (hist_data->attrs->pause)
6345                 data->paused = true;
6346
6347         if (named_data) {
6348                 data->private_data = named_data->private_data;
6349                 set_named_trigger_data(data, named_data);
6350                 data->ops = &event_hist_trigger_named_ops;
6351         }
6352
6353         if (data->ops->init) {
6354                 ret = data->ops->init(data);
6355                 if (ret < 0)
6356                         goto out;
6357         }
6358
6359         if (hist_data->enable_timestamps) {
6360                 char *clock = hist_data->attrs->clock;
6361
6362                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6363                 if (ret) {
6364                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6365                         goto out;
6366                 }
6367
6368                 tracing_set_filter_buffering(file->tr, true);
6369         }
6370
6371         if (named_data)
6372                 destroy_hist_data(hist_data);
6373  out:
6374         return ret;
6375 }
6376
6377 static int hist_trigger_enable(struct event_trigger_data *data,
6378                                struct trace_event_file *file)
6379 {
6380         int ret = 0;
6381
6382         list_add_tail_rcu(&data->list, &file->triggers);
6383
6384         update_cond_flag(file);
6385
6386         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6387                 list_del_rcu(&data->list);
6388                 update_cond_flag(file);
6389                 ret--;
6390         }
6391
6392         return ret;
6393 }
6394
6395 static bool have_hist_trigger_match(struct event_trigger_data *data,
6396                                     struct trace_event_file *file)
6397 {
6398         struct hist_trigger_data *hist_data = data->private_data;
6399         struct event_trigger_data *test, *named_data = NULL;
6400         bool match = false;
6401
6402         lockdep_assert_held(&event_mutex);
6403
6404         if (hist_data->attrs->name)
6405                 named_data = find_named_trigger(hist_data->attrs->name);
6406
6407         list_for_each_entry(test, &file->triggers, list) {
6408                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6409                         if (hist_trigger_match(data, test, named_data, false)) {
6410                                 match = true;
6411                                 break;
6412                         }
6413                 }
6414         }
6415
6416         return match;
6417 }
6418
6419 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6420                                     struct trace_event_file *file)
6421 {
6422         struct hist_trigger_data *hist_data = data->private_data;
6423         struct event_trigger_data *test, *named_data = NULL;
6424
6425         lockdep_assert_held(&event_mutex);
6426
6427         if (hist_data->attrs->name)
6428                 named_data = find_named_trigger(hist_data->attrs->name);
6429
6430         list_for_each_entry(test, &file->triggers, list) {
6431                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6432                         if (!hist_trigger_match(data, test, named_data, false))
6433                                 continue;
6434                         hist_data = test->private_data;
6435                         if (check_var_refs(hist_data))
6436                                 return true;
6437                         break;
6438                 }
6439         }
6440
6441         return false;
6442 }
6443
6444 static void hist_unregister_trigger(char *glob,
6445                                     struct event_trigger_data *data,
6446                                     struct trace_event_file *file)
6447 {
6448         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6449         struct hist_trigger_data *hist_data = data->private_data;
6450
6451         lockdep_assert_held(&event_mutex);
6452
6453         if (hist_data->attrs->name)
6454                 named_data = find_named_trigger(hist_data->attrs->name);
6455
6456         list_for_each_entry(iter, &file->triggers, list) {
6457                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6458                         if (!hist_trigger_match(data, iter, named_data, false))
6459                                 continue;
6460                         test = iter;
6461                         list_del_rcu(&test->list);
6462                         trace_event_trigger_enable_disable(file, 0);
6463                         update_cond_flag(file);
6464                         break;
6465                 }
6466         }
6467
6468         if (test && test->ops->free)
6469                 test->ops->free(test);
6470
6471         if (hist_data->enable_timestamps) {
6472                 if (!hist_data->remove || test)
6473                         tracing_set_filter_buffering(file->tr, false);
6474         }
6475 }
6476
6477 static bool hist_file_check_refs(struct trace_event_file *file)
6478 {
6479         struct hist_trigger_data *hist_data;
6480         struct event_trigger_data *test;
6481
6482         lockdep_assert_held(&event_mutex);
6483
6484         list_for_each_entry(test, &file->triggers, list) {
6485                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6486                         hist_data = test->private_data;
6487                         if (check_var_refs(hist_data))
6488                                 return true;
6489                 }
6490         }
6491
6492         return false;
6493 }
6494
6495 static void hist_unreg_all(struct trace_event_file *file)
6496 {
6497         struct event_trigger_data *test, *n;
6498         struct hist_trigger_data *hist_data;
6499         struct synth_event *se;
6500         const char *se_name;
6501
6502         lockdep_assert_held(&event_mutex);
6503
6504         if (hist_file_check_refs(file))
6505                 return;
6506
6507         list_for_each_entry_safe(test, n, &file->triggers, list) {
6508                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6509                         hist_data = test->private_data;
6510                         list_del_rcu(&test->list);
6511                         trace_event_trigger_enable_disable(file, 0);
6512
6513                         se_name = trace_event_name(file->event_call);
6514                         se = find_synth_event(se_name);
6515                         if (se)
6516                                 se->ref--;
6517
6518                         update_cond_flag(file);
6519                         if (hist_data->enable_timestamps)
6520                                 tracing_set_filter_buffering(file->tr, false);
6521                         if (test->ops->free)
6522                                 test->ops->free(test);
6523                 }
6524         }
6525 }
6526
6527 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6528                                     struct trace_event_file *file,
6529                                     char *glob, char *cmd,
6530                                     char *param_and_filter)
6531 {
6532         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6533         struct event_trigger_data *trigger_data;
6534         struct hist_trigger_attrs *attrs;
6535         struct hist_trigger_data *hist_data;
6536         char *param, *filter, *p, *start;
6537         struct synth_event *se;
6538         const char *se_name;
6539         bool remove;
6540         int ret = 0;
6541
6542         lockdep_assert_held(&event_mutex);
6543
6544         if (WARN_ON(!glob))
6545                 return -EINVAL;
6546
6547         if (glob[0]) {
6548                 hist_err_clear();
6549                 last_cmd_set(file, param_and_filter);
6550         }
6551
6552         remove = event_trigger_check_remove(glob);
6553
6554         if (event_trigger_empty_param(param_and_filter))
6555                 return -EINVAL;
6556
6557         /*
6558          * separate the trigger from the filter (k:v [if filter])
6559          * allowing for whitespace in the trigger
6560          */
6561         p = param = param_and_filter;
6562         do {
6563                 p = strstr(p, "if");
6564                 if (!p)
6565                         break;
6566                 if (p == param_and_filter)
6567                         return -EINVAL;
6568                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6569                         p++;
6570                         continue;
6571                 }
6572                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6573                         return -EINVAL;
6574                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6575                         p++;
6576                         continue;
6577                 }
6578                 break;
6579         } while (1);
6580
6581         if (!p)
6582                 filter = NULL;
6583         else {
6584                 *(p - 1) = '\0';
6585                 filter = strstrip(p);
6586                 param = strstrip(param);
6587         }
6588
6589         /*
6590          * To simplify arithmetic expression parsing, replace occurrences of
6591          * '.sym-offset' modifier with '.symXoffset'
6592          */
6593         start = strstr(param, ".sym-offset");
6594         while (start) {
6595                 *(start + 4) = 'X';
6596                 start = strstr(start + 11, ".sym-offset");
6597         }
6598
6599         attrs = parse_hist_trigger_attrs(file->tr, param);
6600         if (IS_ERR(attrs))
6601                 return PTR_ERR(attrs);
6602
6603         if (attrs->map_bits)
6604                 hist_trigger_bits = attrs->map_bits;
6605
6606         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6607         if (IS_ERR(hist_data)) {
6608                 destroy_hist_trigger_attrs(attrs);
6609                 return PTR_ERR(hist_data);
6610         }
6611
6612         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6613         if (!trigger_data) {
6614                 ret = -ENOMEM;
6615                 goto out_free;
6616         }
6617
6618         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6619         if (ret < 0)
6620                 goto out_free;
6621
6622         if (remove) {
6623                 if (!have_hist_trigger_match(trigger_data, file))
6624                         goto out_free;
6625
6626                 if (hist_trigger_check_refs(trigger_data, file)) {
6627                         ret = -EBUSY;
6628                         goto out_free;
6629                 }
6630
6631                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6632                 se_name = trace_event_name(file->event_call);
6633                 se = find_synth_event(se_name);
6634                 if (se)
6635                         se->ref--;
6636                 ret = 0;
6637                 goto out_free;
6638         }
6639
6640         if (existing_hist_update_only(glob, trigger_data, file))
6641                 goto out_free;
6642
6643         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6644         if (ret < 0)
6645                 goto out_free;
6646
6647         if (get_named_trigger_data(trigger_data))
6648                 goto enable;
6649
6650         if (has_hist_vars(hist_data))
6651                 save_hist_vars(hist_data);
6652
6653         ret = create_actions(hist_data);
6654         if (ret)
6655                 goto out_unreg;
6656
6657         ret = tracing_map_init(hist_data->map);
6658         if (ret)
6659                 goto out_unreg;
6660 enable:
6661         ret = hist_trigger_enable(trigger_data, file);
6662         if (ret)
6663                 goto out_unreg;
6664
6665         se_name = trace_event_name(file->event_call);
6666         se = find_synth_event(se_name);
6667         if (se)
6668                 se->ref++;
6669  out:
6670         if (ret == 0 && glob[0])
6671                 hist_err_clear();
6672
6673         return ret;
6674  out_unreg:
6675         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6676  out_free:
6677         event_trigger_reset_filter(cmd_ops, trigger_data);
6678
6679         remove_hist_vars(hist_data);
6680
6681         kfree(trigger_data);
6682
6683         destroy_hist_data(hist_data);
6684         goto out;
6685 }
6686
6687 static struct event_command trigger_hist_cmd = {
6688         .name                   = "hist",
6689         .trigger_type           = ETT_EVENT_HIST,
6690         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6691         .parse                  = event_hist_trigger_parse,
6692         .reg                    = hist_register_trigger,
6693         .unreg                  = hist_unregister_trigger,
6694         .unreg_all              = hist_unreg_all,
6695         .get_trigger_ops        = event_hist_get_trigger_ops,
6696         .set_filter             = set_trigger_filter,
6697 };
6698
6699 __init int register_trigger_hist_cmd(void)
6700 {
6701         int ret;
6702
6703         ret = register_event_command(&trigger_hist_cmd);
6704         WARN_ON(ret < 0);
6705
6706         return ret;
6707 }
6708
6709 static void
6710 hist_enable_trigger(struct event_trigger_data *data,
6711                     struct trace_buffer *buffer,  void *rec,
6712                     struct ring_buffer_event *event)
6713 {
6714         struct enable_trigger_data *enable_data = data->private_data;
6715         struct event_trigger_data *test;
6716
6717         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6718                                 lockdep_is_held(&event_mutex)) {
6719                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6720                         if (enable_data->enable)
6721                                 test->paused = false;
6722                         else
6723                                 test->paused = true;
6724                 }
6725         }
6726 }
6727
6728 static void
6729 hist_enable_count_trigger(struct event_trigger_data *data,
6730                           struct trace_buffer *buffer,  void *rec,
6731                           struct ring_buffer_event *event)
6732 {
6733         if (!data->count)
6734                 return;
6735
6736         if (data->count != -1)
6737                 (data->count)--;
6738
6739         hist_enable_trigger(data, buffer, rec, event);
6740 }
6741
6742 static struct event_trigger_ops hist_enable_trigger_ops = {
6743         .trigger                = hist_enable_trigger,
6744         .print                  = event_enable_trigger_print,
6745         .init                   = event_trigger_init,
6746         .free                   = event_enable_trigger_free,
6747 };
6748
6749 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6750         .trigger                = hist_enable_count_trigger,
6751         .print                  = event_enable_trigger_print,
6752         .init                   = event_trigger_init,
6753         .free                   = event_enable_trigger_free,
6754 };
6755
6756 static struct event_trigger_ops hist_disable_trigger_ops = {
6757         .trigger                = hist_enable_trigger,
6758         .print                  = event_enable_trigger_print,
6759         .init                   = event_trigger_init,
6760         .free                   = event_enable_trigger_free,
6761 };
6762
6763 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6764         .trigger                = hist_enable_count_trigger,
6765         .print                  = event_enable_trigger_print,
6766         .init                   = event_trigger_init,
6767         .free                   = event_enable_trigger_free,
6768 };
6769
6770 static struct event_trigger_ops *
6771 hist_enable_get_trigger_ops(char *cmd, char *param)
6772 {
6773         struct event_trigger_ops *ops;
6774         bool enable;
6775
6776         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6777
6778         if (enable)
6779                 ops = param ? &hist_enable_count_trigger_ops :
6780                         &hist_enable_trigger_ops;
6781         else
6782                 ops = param ? &hist_disable_count_trigger_ops :
6783                         &hist_disable_trigger_ops;
6784
6785         return ops;
6786 }
6787
6788 static void hist_enable_unreg_all(struct trace_event_file *file)
6789 {
6790         struct event_trigger_data *test, *n;
6791
6792         list_for_each_entry_safe(test, n, &file->triggers, list) {
6793                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6794                         list_del_rcu(&test->list);
6795                         update_cond_flag(file);
6796                         trace_event_trigger_enable_disable(file, 0);
6797                         if (test->ops->free)
6798                                 test->ops->free(test);
6799                 }
6800         }
6801 }
6802
6803 static struct event_command trigger_hist_enable_cmd = {
6804         .name                   = ENABLE_HIST_STR,
6805         .trigger_type           = ETT_HIST_ENABLE,
6806         .parse                  = event_enable_trigger_parse,
6807         .reg                    = event_enable_register_trigger,
6808         .unreg                  = event_enable_unregister_trigger,
6809         .unreg_all              = hist_enable_unreg_all,
6810         .get_trigger_ops        = hist_enable_get_trigger_ops,
6811         .set_filter             = set_trigger_filter,
6812 };
6813
6814 static struct event_command trigger_hist_disable_cmd = {
6815         .name                   = DISABLE_HIST_STR,
6816         .trigger_type           = ETT_HIST_ENABLE,
6817         .parse                  = event_enable_trigger_parse,
6818         .reg                    = event_enable_register_trigger,
6819         .unreg                  = event_enable_unregister_trigger,
6820         .unreg_all              = hist_enable_unreg_all,
6821         .get_trigger_ops        = hist_enable_get_trigger_ops,
6822         .set_filter             = set_trigger_filter,
6823 };
6824
6825 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6826 {
6827         unregister_event_command(&trigger_hist_enable_cmd);
6828         unregister_event_command(&trigger_hist_disable_cmd);
6829 }
6830
6831 __init int register_trigger_hist_enable_disable_cmds(void)
6832 {
6833         int ret;
6834
6835         ret = register_event_command(&trigger_hist_enable_cmd);
6836         if (WARN_ON(ret < 0))
6837                 return ret;
6838         ret = register_event_command(&trigger_hist_disable_cmd);
6839         if (WARN_ON(ret < 0))
6840                 unregister_trigger_hist_enable_disable_cmds();
6841
6842         return ret;
6843 }