net/haproxy/patches/0003-CONTRIB-halog-sort-URLs-by-avg-bytes_read-or-t-1.4.22.diff

   1 From 69eeb17615cd032ec7b3a9496ddb811095e92cb4 Mon Sep 17 00:00:00 2001
   2 From: Baptiste <bedis9@gmail.com>
   3 Date: Sat, 8 Sep 2012 23:10:03 +0200
   4 Subject: CONTRIB: halog: sort URLs by avg bytes_read or total bytes_read
   5
   6 The patch attached to this mail brings ability to sort URLs by
   7 averaged bytes read and total bytes read in HALog tool.
   8 In most cases, bytes read is also the object size.
   9 The purpose of this patch is to know which URL consume the most
  10 bandwith, in average or in total.
  11 It may be interesting as well to know the standard deviation (ecart
  12 type in french) for some counters (like bytes_read).
  13
  14 The results:
  15 - Sorting by average bytes read per URL:
  16 ./halog -uba <~/tmp/haproxy.log | column -t | head
  17 2246 lines in, 302 lines out, 194 parsing errors
  18 18    0    5101     283    5101   283    126573  2278327  /lib/exe/js.php
  19 1     0    1        1      1      1      106734  106734   /wp-admin/images/screenshots/theme-customizer.png
  20 2     0    2        1      2      1      106511  213022   /wp-admin/css/wp-admin.css
  21 1     0    1        1      1      1      96698   96698    /wp-admin/images/screenshots/captions-1.png
  22 1     0    1        1      1      1      73165   73165    /wp-admin/images/screenshots/flex-header-1.png
  23 4     0    0        0      0      0      64832   259328   /cuisine/wp-content/plugins/stats/open-flash-chart.swf
  24 1     0    0        0      0      0      48647   48647    /wp-admin/images/screenshots/flex-header-3.png
  25 1     0    0        0      0      0      44046   44046    /wp-admin/images/screenshots/captions-2.png
  26 1     0    1        1      1      1      38830   38830    /wp-admin/images/screenshots/flex-header-2.png
  27
  28 - Sorting by total bytes read per URL:
  29 ./halog -ubt <~/tmp/haproxy.log | column -t | head
  30 2246 lines in, 302 lines out, 194 parsing errors
  31 18    0    5101     283    5101   283    126573  2278327  /lib/exe/js.php
  32 60    0    14387    239    14387  239    10081   604865   /lib/exe/css.php
  33 64    2    8820     137    8819   142    7742    495524   /doku.php
  34 14    0    250      17     250    17     24045   336632   /wp-admin/load-scripts.php
  35 71    0    6422     90     6422   90     4048    287419   /wp-admin/
  36 4     0    0        0      0      0      64832   259328   /cuisine/wp-content/plugins/stats/open-flash-chart.swf
  37 2     0    2        1      2      1      106511  213022   /wp-admin/css/wp-admin.css
  38 31    3    5423     174    5040   180    6804    210931   /index
  39 10    0    429      42     429    42     18009   180093   /cuisine/files/2011/10/tarte_figue_amande-e1318281546905-225x300.jpg
  40 (cherry picked from commit 61aaad06e85ae7f46cf5589bce8bc7f9331e7962)
  41 ---
  42  contrib/halog/halog.c |   39 ++++++++++++++++++++++++++++++++-------
  43  1 files changed, 32 insertions(+), 7 deletions(-)
  44
  45 diff --git a/contrib/halog/halog.c b/contrib/halog/halog.c
  46 index f19be0b..274b7f3 100644
  47 --- a/contrib/halog/halog.c
  48 +++ b/contrib/halog/halog.c
  49 @@ -29,6 +29,7 @@
  50  #define SERVER_FIELD 8
  51  #define TIME_FIELD 9
  52  #define STATUS_FIELD 10
  53 +#define BYTES_SENT_FIELD 11
  54  #define TERM_CODES_FIELD 14
  55  #define CONN_FIELD 15
  56  #define QUEUE_LEN_FIELD 16
  57 @@ -67,6 +68,7 @@ struct url_stat {
  58         char *url;
  59         unsigned long long total_time;    /* sum(all reqs' times) */
  60         unsigned long long total_time_ok; /* sum(all OK reqs' times) */
  61 +       unsigned long long total_bytes_sent; /* sum(all bytes sent) */
  62         unsigned int nb_err, nb_req;
  63  };
  64
  65 @@ -94,8 +96,6 @@ struct url_stat {
  66  #define FILT_COUNT_URL_TAVG  0x040000
  67  #define FILT_COUNT_URL_TTOTO 0x080000
  68  #define FILT_COUNT_URL_TAVGO 0x100000
  69 -#define FILT_COUNT_URL_ANY   (FILT_COUNT_URL_ONLY|FILT_COUNT_URL_COUNT|FILT_COUNT_URL_ERR| \
  70 -                             FILT_COUNT_URL_TTOT|FILT_COUNT_URL_TAVG|FILT_COUNT_URL_TTOTO|FILT_COUNT_URL_TAVGO)
  71
  72  #define FILT_HTTP_ONLY       0x200000
  73  #define FILT_TERM_CODE_NAME  0x400000
  74 @@ -106,6 +106,13 @@ struct url_stat {
  75  #define FILT_QUEUE_ONLY            0x4000000
  76  #define FILT_QUEUE_SRV_ONLY        0x8000000
  77
  78 +#define FILT_COUNT_URL_BAVG       0x10000000
  79 +#define FILT_COUNT_URL_BTOT       0x20000000
  80 +
  81 +#define FILT_COUNT_URL_ANY   (FILT_COUNT_URL_ONLY|FILT_COUNT_URL_COUNT|FILT_COUNT_URL_ERR| \
  82 +                             FILT_COUNT_URL_TTOT|FILT_COUNT_URL_TAVG|FILT_COUNT_URL_TTOTO|FILT_COUNT_URL_TAVGO| \
  83 +                             FILT_COUNT_URL_BAVG|FILT_COUNT_URL_BTOT)
  84 +
  85  unsigned int filter = 0;
  86  unsigned int filter_invert = 0;
  87  const char *line;
  88 @@ -128,9 +135,10 @@ void usage(FILE *output, const char *msg)
  89         fprintf(output,
  90                 "%s"
  91                 "Usage: halog [-h|--help] for long help\n"
  92 -               "       halog [-q] [-c] [-v] {-gt|-pct|-st|-tc|-srv|-u|-uc|-ue|-ua|-ut|-uao|-uto}\n"
  93 +               "       halog [-q] [-c]\n"
  94 +               "       {-gt|-pct|-st|-tc|-srv|-u|-uc|-ue|-ua|-ut|-uao|-uto|-uba|-ubt}\n"
  95                 "       [-s <skip>] [-e|-E] [-H] [-rt|-RT <time>] [-ad <delay>] [-ac <count>]\n"
  96 -               "       [-Q|-QS] [-tcn|-TCN <termcode>] [ -hs|-HS [min][:[max]] ] < log\n"
  97 +               "       [-v] [-Q|-QS] [-tcn|-TCN <termcode>] [ -hs|-HS [min][:[max]] ] < log\n"
  98                 "\n",
  99                 msg ? msg : ""
 100                 );
 101 @@ -171,6 +179,7 @@ void help()
 102                "       -u : by URL, -uc : request count, -ue : error count\n"
 103                "       -ua : average response time, -uto : average total time\n"
 104                "       -uao, -uto: average times computed on valid ('OK') requests\n"
 105 +              "       -uba, -ubt: average bytes returned, total bytes returned\n"
 106                );
 107         exit(0);
 108  }
 109 @@ -632,6 +641,10 @@ int main(int argc, char **argv)
 110                         filter |= FILT_COUNT_URL_TAVGO;
 111                 else if (strcmp(argv[0], "-uto") == 0)
 112                         filter |= FILT_COUNT_URL_TTOTO;
 113 +               else if (strcmp(argv[0], "-uba") == 0)
 114 +                       filter |= FILT_COUNT_URL_BAVG;
 115 +               else if (strcmp(argv[0], "-ubt") == 0)
 116 +                       filter |= FILT_COUNT_URL_BTOT;
 117                 else if (strcmp(argv[0], "-o") == 0) {
 118                         if (output_file)
 119                                 die("Fatal: output file name already specified.\n");
 120 @@ -1029,6 +1042,10 @@ int main(int argc, char **argv)
 121                                         ustat->node.val.key = ustat->total_time_ok;
 122                                 else if (filter & FILT_COUNT_URL_TAVGO)
 123                                         ustat->node.val.key = (ustat->nb_req - ustat->nb_err) ? ustat->total_time_ok / (ustat->nb_req - ustat->nb_err) : 0;
 124 +                               else if (filter & FILT_COUNT_URL_BAVG)
 125 +                                       ustat->node.val.key = ustat->nb_req ? ustat->total_bytes_sent / ustat->nb_req : 0;
 126 +                               else if (filter & FILT_COUNT_URL_BTOT)
 127 +                                       ustat->node.val.key = ustat->total_bytes_sent;
 128                                 else
 129                                         ustat->node.val.key = 0;
 130
 131 @@ -1038,19 +1055,21 @@ int main(int argc, char **argv)
 132                         timers[0] = timers[1];
 133                 }
 134
 135 -               printf("#req err ttot tavg oktot okavg url\n");
 136 +               printf("#req err ttot tavg oktot okavg bavg btot url\n");
 137
 138                 /* scan the tree in its reverse sorting order */
 139                 node = eb_last(&timers[0]);
 140                 while (node) {
 141                         ustat = container_of(node, struct url_stat, node.url.node);
 142 -                       printf("%d %d %Ld %Ld %Ld %Ld %s\n",
 143 +                       printf("%d %d %Ld %Ld %Ld %Ld %Ld %Ld %s\n",
 144                                ustat->nb_req,
 145                                ustat->nb_err,
 146                                ustat->total_time,
 147                                ustat->nb_req ? ustat->total_time / ustat->nb_req : 0,
 148                                ustat->total_time_ok,
 149                                (ustat->nb_req - ustat->nb_err) ? ustat->total_time_ok / (ustat->nb_req - ustat->nb_err) : 0,
 150 +                              ustat->nb_req ? ustat->total_bytes_sent / ustat->nb_req : 0,
 151 +                              ustat->total_bytes_sent,
 152                                ustat->url);
 153
 154                         node = eb_prev(node);
 155 @@ -1233,6 +1252,7 @@ void filter_count_url(const char *accept_field, const char *time_field, struct t
 156         struct ebpt_node *ebpt_old;
 157         const char *b, *e;
 158         int f, err, array[5];
 159 +       int val;
 160
 161         /* let's collect the response time */
 162         if (!time_field) {
 163 @@ -1276,12 +1296,16 @@ void filter_count_url(const char *accept_field, const char *time_field, struct t
 164         ustat->total_time = (array[3] >= 0) ? array[3] : array[4];
 165         ustat->total_time_ok = (array[3] >= 0) ? array[3] : 0;
 166
 167 +       e = field_start(e, BYTES_SENT_FIELD - TIME_FIELD + 1);
 168 +       val = str2ic(e);
 169 +       ustat->total_bytes_sent = val;
 170 +
 171         /* the line may be truncated because of a bad request or anything like this,
 172          * without a method. Also, if it does not begin with an quote, let's skip to
 173          * the next field because it's a capture. Let's fall back to the "method" itself
 174          * if there's nothing else.
 175          */
 176 -       e = field_start(e, METH_FIELD - TIME_FIELD + 1); // avg 100 ns per line
 177 +       e = field_start(e, METH_FIELD - BYTES_SENT_FIELD + 1);
 178         while (*e != '"' && *e) {
 179                 /* Note: some syslog servers escape quotes ! */
 180                 if (*e == '\\' && e[1] == '"')
 181 @@ -1324,6 +1348,7 @@ void filter_count_url(const char *accept_field, const char *time_field, struct t
 182                 ustat_old->nb_err += ustat->nb_err;
 183                 ustat_old->total_time += ustat->total_time;
 184                 ustat_old->total_time_ok += ustat->total_time_ok;
 185 +               ustat_old->total_bytes_sent += ustat->total_bytes_sent;
 186         } else {
 187                 ustat->url = ustat->node.url.key = strdup(ustat->node.url.key);
 188                 ustat = NULL; /* node was used */
 189 --
 190 1.7.1
 191