count-http-requests (Source) 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
#!/usr/bin/awk -f
##---------------------------------------------------------------------------##
#   Program: count-http-requests
#   Author:  Brian <genius@groupbcl.ca> :)
#   Date:    August 2019
#
#   Reads an input file created from a Firefox Developer Tools Console
#   trace, counts URLs and their success or failure, and displays a
#   list of URLs, counts, and totals.
#
#   As of August 2019, do the following to create an input file and run
#   this program:
#   * Start Firefox
#   * Press F12 to open the Developer Tools
#   * In the Developer Tools pane or window, click "Console"
#   * Leave only "Requests" turned on; turn off Errors, Warnings, Logs,
#     Info, Debug, CSS, and XHR.
#   * Navigate to the URL of interest
#   * When the page finishes loading, right-click on the main body of the
#     developer tools window and select "Export visible messages to
#     clipboard"
#   * Edit a file (for example, "/tmp/http-requests.A.text" and paste the
#     clipboard contents into it.
#   * Run this program as follows:
#       count-http-requests FILENAME | sort | cut -f2 | less
##---------------------------------------------------------------------------##
# BUUS: This script is part of Brian's Useful Utilities Set
BEGIN { h_count=0; t_count_succeed=0; t_count_fail=0; max_h_len=0 }

# On a GET request, get the FQDN and increment its request count
match($0, /^(GET|POST) *https?:\/\/([^\/]+)/, a) {
    t_count_all++

    # Reverse the host name (e.g. host.domain.tld --> tld.domain.host) to
    # group all hosts in a given domain together
    count = split(a[2], b, /\./)
    host = ""
    for (i=count; i>0; i--) { host = host (i==count ? "" : ".") b[i] }

    # Initialise counters if we haven't seen this host before
    if (host in h_name) { } else {
        h_count++
        h_name[host] = a[2]
        h_count_all[host] = 0
        h_count_succeed[host] = 0
        h_count_fail[host] = 0
    }
    h_count_all[host]++
    if (length(d) > max_h_len) max_h_len = length(d)
}

# On an HTTP response, count success or fail
match($0, /\[HTTP\/... ([0-9][0-9][0-9])/, a) {
    if (a[1] < 400) {
        h_count_succeed[host]++
        t_count_succeed++
    } else {
        h_count_fail[host]++
        t_count_fail++
    }
}

# Display results
END {
    for (host in h_name) {
        i = h_count_all[host] - (h_count_succeed[host] + h_count_fail[host])
        x = ""
        if (h_count_succeed[host])  x = h_count_succeed[host] " succeeded"
        if (h_count_fail[host]) x = x (x ? ", " : "" ) h_count_fail[host] " failed"
        if (i)                  x = x (x ? ", " : "" ) i " never completed"
        printf("%s\t%-" max_h_len+1 "s total %i; %s\n", host, h_name[host] ":",
            h_count_all[host], x)
    }

    print "z\t  Total " t_count_all " requests to " h_count " individual hosts: " \
        t_count_succeed " succeeded, " \
        t_count_fail " failed, " \
        t_count_all - (t_count_succeed + t_count_fail) " never completed"
}

# vim: tabstop=4