Running statistics on multiple lines in bash - linux

Question

I have multiple HTTP headers in one giant file, separated with one empty line.
Host
Connection
Accept
From
User-Agent
Accept-Encoding
Host
Connection
Accept
From
User-Agent
Accept-Encoding
X-Forwarded-For
cookie
Cache-Control
referer
x-fb-sim-hni
Host
Accept
user-agent
x-fb-net-sid
x-fb-net-hni
X-Purpose
accept-encoding
x-fb-http-engine
Connection
User-Agent
Host
Connection
Accept-Encoding
I have approximately 10,000,000 of headers separated with an empty line.
If I want to discover trends, like header order, I want to do aggregate headers to a one-liner (how I can aggregate lines ending with an empty line and do that separately for all headers?):
Host,Connection,Accept,From,User-Agent,Accept-Encoding
and follow with: uniq -c|sort -nk1,
so I could receive:
197897 Host,Connection,Accept,From,User-Agent,Accept-Encoding
8732233 User-Agent,Host,Connection,Accept-Encoding
What would be the best approach and most effective one to parse that massive file and get that data?
Thanks for hints.

Answer 1

Using GNU awk for sorted_in, all you need is:
$ cat tst.awk
BEGIN { RS=""; FS="\n"; OFS="," }
{ $1=$1; cnt[$0]++ }
END {
PROCINFO["sorted_in"] = "#val_num_desc"
for (rec in cnt) {
print cnt[rec] " " rec
}
}
After running dos2unix on the sample you posted (1.5milGETs.txt):
$ time awk -f tst.awk 1.5milGETs.txt > ou.awk
real 0m4.898s
user 0m4.758s
sys 0m0.108s
$ head -10 ou.awk
71639 Host,Accept,User-Agent,Pragma,Connection
70975 Host,ros-SecurityFlags,ros-SessionTicket,ros-Challenge,ros-HeadersHmac,Scs-Ticket,If-Modified-Since,User-Agent
40781 Host,Accept,User-Agent,Pragma,nnCoection,Connection,X-Forwarded-For
35485 Accept,ros-SecurityFlags,ros-SessionTicket,ros-Challenge,ros-HeadersHmac,Scs-Ticket,If-Modified-Since,User-Agent,Accept-Language,UA-CPU,Accept-Encoding,Host,Connection
34005 User-Agent,Host,Connection,Accept-Encoding
30668 Host,User-Agent,Accept-Encoding,Connection
25547 Host,Accept,Accept-Language,Connection,Accept-Encoding,User-Agent
22581 Host,User-Agent,Accept,Accept-Encoding
19311 Host,Connection,Accept,From,User-Agent,Accept-Encoding
14694 Host,Connection,User-Agent,Accept,Referer,Accept-Encoding,Accept-Language,Cookie

Answer 2

Here's an answer written in (POSIX) C, which AFAICT does what OP wants. The C solution seems to be faster than an AWK based solution. That may or may not be useful, it all depends on how frequent the program is run and the input data.
The main takeaway:
The program memory maps the input file and alters the mapped copy.
It replaces newline characters with commas where appropriate, and
newlines with nul characters to separate each entry in the input
file. IOW, foo\nbar\n\nbaz\n becomes foo,bar\0baz\0.
The program also builds a table of entries, which is just an array of
char-pointers into the memory mapped file.
The program sorts the entries using standard string functions, but only moves the pointers values, not the actual data
Then the program creates a new array of unique entries and counts how many instances there are for each string. (This part can probably be made a bit faster)
The array of unique entries is then sorted in descending order
Finally, the program prints the contents of the unique array
Anyway, here's the code. (disclaimer: It's written to be postable here on SO)
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <fcntl.h>
struct uniq {
char *val;
size_t count;
};
struct entry {
char *val;
};
// Some globals
size_t g_filesize;
char* g_baseaddr;
struct entry *g_entries;
size_t g_entrysize, g_entrycapacity;
struct uniq *g_unique;
size_t g_uniquesize, g_uniquecapacity;
static inline void mapfile(const char *filename)
{
int fd;
struct stat st;
if ((fd = open(filename, O_RDWR)) == -1 || fstat(fd, &st)) {
perror(filename);
exit(__LINE__);
}
g_baseaddr = mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
if (g_baseaddr == (void *)MAP_FAILED) {
perror(filename);
close(fd);
exit(__LINE__);
}
close(fd);
g_filesize = st.st_size;
}
// Guestimate how many entries we have. We do this only to avoid early
// reallocs, so this isn't that important. Let's say 100 bytes per entry.
static inline void setup_entry_table(void)
{
g_entrycapacity = g_filesize / 100;
g_entrysize = 0;
size_t cb = sizeof *g_entries * g_entrycapacity;
if ((g_entries = malloc(cb)) == NULL)
exit(__LINE__);
memset(g_entries, 0, cb);
}
static inline void realloc_if_needed(void)
{
if (g_entrysize == g_entrycapacity) {
size_t newcap = g_entrycapacity * 2;
size_t cb = newcap * sizeof *g_entries;
struct entry *tmp = realloc(g_entries, cb);
if (tmp == NULL)
exit(__LINE__);
g_entries = tmp;
g_entrycapacity = newcap;
}
}
static inline void add_entry(char *p)
{
realloc_if_needed();
g_entries[g_entrysize].val = p;
g_entrysize++;
}
// Convert input data to proper entries by replacing \n with either
// ',' or \0. We add \0 to separate the entries.
static inline void convert_to_entries(void)
{
char *endaddr = g_baseaddr + g_filesize;
char *prev, *s = g_baseaddr;
// First entry
prev = s;
while(s < endaddr) {
char *nl = strchr(s, '\n');
if (nl == s) {
if (nl - prev > 0) // Skip empty strings
add_entry(prev);
*nl = '\0'; // Terminate entry
s = nl + 1; // Skip to first byte after \0
prev = s; // This is the start of the 'previous' record
}
else {
*nl = ','; // Replace \n with comma
s = nl + 1; // Move pointer forward (optimization).
if (*s == '\n')
*(s - 1) = '\0';// Don't add trailing comma
}
}
if (prev < s)
add_entry(prev); // Don't forget last entry
}
static int entrycmp(const void *v1, const void *v2)
{
const struct entry *p1 = v1, *p2 = v2;
return strcmp(p1->val, p2->val);
}
// Sort the entries so the pointers point to a sorted list of strings.
static inline void sort_entries(void)
{
qsort(g_entries, g_entrysize, sizeof *g_entries, entrycmp);
}
// We keep things really simple and allocate one unique entry for each
// entry. That's the worst case anyway and then we don't have to test
// for reallocation.
static inline void setup_unique_table(void)
{
size_t cb = sizeof *g_unique * g_entrysize;
if ((g_unique = malloc(cb)) == NULL)
exit(__LINE__);
g_uniquesize = 0;
g_uniquecapacity = g_entrysize;
}
static inline void add_unique(char *s)
{
g_unique[g_uniquesize].val = s;
g_unique[g_uniquesize].count = 1;
g_uniquesize++;
}
// Now count and skip duplicate entries.
// How? Just iterate over the entries table and find duplicates.
// For each duplicate, increment count. For each non-dup,
// add a new entry.
static inline void find_unique_entries(void)
{
char *last = g_entries[0].val;
add_unique(last);
for (size_t i = 1; i < g_entrysize; i++) {
if (strcmp(g_entries[i].val, last) == 0) {
g_unique[g_uniquesize - 1].count++; // Inc last added\'s count
}
else {
last = g_entries[i].val;
add_unique(last);
}
}
}
static inline void print_unique_entries(void)
{
for (size_t i = 0; i < g_uniquesize; i++)
printf("%zu %s\n", g_unique[i].count, g_unique[i].val);
}
static inline void print_entries(void)
{
for (size_t i = 0; i < g_entrysize; i++)
printf("%s\n", g_entries[i].val);
}
static int uniquecmp(const void *v1, const void *v2)
{
const struct uniq *p1 = v1, *p2 = v2;
return (int)p2->count - (int)p1->count;
}
static inline void sort_unique_entries(void)
{
qsort(g_unique, g_uniquesize, sizeof *g_unique, uniquecmp);
}
int main(int argc, char *argv[])
{
if (argc != 2) {
fprintf(stderr, "USAGE: %s filename\n", argv[0]);
exit(__LINE__);
}
mapfile(argv[1]);
setup_entry_table();
convert_to_entries();
if (g_entrysize == 0) // no entries in file.
exit(0);
sort_entries();
setup_unique_table();
find_unique_entries();
sort_unique_entries();
if (0) print_entries();
if (1) print_unique_entries();
// cleanup
free(g_entries);
free(g_unique);
munmap(g_baseaddr, g_filesize);
exit(0);
}

Answer 3

Using your your 1.5milGETs.txt file (and converting the triple \n\n\n to \n\n to separate blocks) you can use ruby in paragraph mode:
$ ruby -F'\n' -lane 'BEGIN{h=Hash.new(0); $/=""
def commafy(n)
n.to_s.reverse.gsub(/...(?=.)/,"\\&,").reverse
end
}
h[$F.join(",")]+=1
# p $_
END{ printf "Total blocks: %s\n", commafy(h.values.sum)
h2=h.sort_by {|k,v| -v}
h2[0..10].map {|k,v| printf "%10s %s\n", commafy(v), k}
}' 1.5milGETs.txt
That prints the total number of blocks, sorts them large->small, prints the top 10.
Prints:
Total blocks: 1,262,522
71,639 Host,Accept,User-Agent,Pragma,Connection
70,975 Host,ros-SecurityFlags,ros-SessionTicket,ros-Challenge,ros-HeadersHmac,Scs-Ticket,If-Modified-Since,User-Agent
40,781 Host,Accept,User-Agent,Pragma,nnCoection,Connection,X-Forwarded-For
35,485 Accept,ros-SecurityFlags,ros-SessionTicket,ros-Challenge,ros-HeadersHmac,Scs-Ticket,If-Modified-Since,User-Agent,Accept-Language,UA-CPU,Accept-Encoding,Host,Connection
34,005 User-Agent,Host,Connection,Accept-Encoding
30,668 Host,User-Agent,Accept-Encoding,Connection
25,547 Host,Accept,Accept-Language,Connection,Accept-Encoding,User-Agent
22,581 Host,User-Agent,Accept,Accept-Encoding
19,311 Host,Connection,Accept,From,User-Agent,Accept-Encoding
14,694 Host,Connection,User-Agent,Accept,Referer,Accept-Encoding,Accept-Language,Cookie
12,290 Host,User-Agent,Accept-Encoding
That takes about 8 seconds on a 6 year old Mac.
Awk will be 3x faster and entirely appropriate for this job.
Ruby will give you more output options and easier analysis of the data. You can create an interactive HTML documents; output JSON, quoted csv, xml trivially; interact with a database; invert keys and values in a statement; filter; etc.

Answer 4

Personally, I'd use a C program, other alternatives exist as well. Here's an awk snippet which folds lines. Not perfect, but should get you started :)
$cat foo.awk
// {
if (NF == 0)
printf("\n");
else
printf("%s ", $0);
}
$ awk -f foo.awk < lots_of_data | sort | uniq -c | sort -nk1
The last statement will take "forever", which is why a C program may be a good alternative. It depends mostly on how often you need to run the commands.

Answer 5

If you have enough memory (10M records, in your sample about 80 chars per record, 800MB, and if you're counting them, I assume a lot of duplicates) you could hash the records to memory and count while hashing:
$ awk 'BEGIN{ RS=""; OFS=","}
{
b="" # reset buffer b
for(i=1;i<=NF;i++) # for every header element in record
b=b (b==""?"":OFS) $i # buffer them and comma separate
a[b]++ # hash to a, counting
}
END { # in the end
for(i in a) # go thru the a hash
print a[i] " " i} # print counts and records
' file
1 Host,Connection,Accept,From,User-Agent,Accept-Encoding
1 cookie,Cache-Control,referer,x-fb-sim-hni,Host,Accept,user-agent,x-fb-net-sid,x-fb-net-hni,X-Purpose,accept-encoding,x-fb-http-engine,Connection
1 User-Agent,Host,Connection,Accept-Encoding
1 Host,Connection,Accept,From,User-Agent,Accept-Encoding,X-Forwarded-For
Output order is random due to the nature of i in a so order the output afterwards as pleases.
Edit:
As #dawg kindly pointed out in the comments, $1=$1 is enough to rebuild the record to comma-separated form:
$ awk 'BEGIN{ RS=""; OFS=","}
{
$1=$1 # rebuild the record
a[$0]++ # hash $0 to a, counting
}
END { # in the end
for(i in a) # go thru the a hash
print a[i] " " i} # print counts and records
' file