Browse Source

Break analyzer out of iptraffic & tests

Added tests for testing address wild card matching and a test jig to put tests
in.

I moved the core of the log analyzer out of iptraffic and into data.o for use
in multiple tools.
master
Jon Foster 2 years ago
parent
commit
8a33f5fdd8
9 changed files with 303 additions and 59 deletions
  1. +3
    -2
      config.h
  2. +68
    -0
      data.cpp
  3. +20
    -0
      data.h
  4. +19
    -57
      iptraffic.cpp
  5. +1
    -0
      tests/.gitignore
  6. +8
    -0
      tests/Makefile
  7. +81
    -0
      tests/data.cpp
  8. +52
    -0
      tests/testit.cpp
  9. +51
    -0
      tests/testit.h

+ 3
- 2
config.h View File

@@ -14,12 +14,13 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "data.h" #include "data.h"
#include "strutil.h"






struct Config { struct Config {
std::vector<std::string> us;
ConnList ignores;
StringList us;
ConnList ignores;


void load(const std::string &fname); void load(const std::string &fname);
}; };


+ 68
- 0
data.cpp View File

@@ -179,3 +179,71 @@ int ConnList::find(Conn &needle) {
for(r=0; r<size(); r++) if((*this)[r]==needle) return r; for(r=0; r<size(); r++) if((*this)[r]==needle) return r;
return -1; return -1;
} }



//////////////////////////////////////////////////////////////////////
// LogAnalyzer
//////////////////////////////////////////////////////////////////////

LogAnalyzer::LogAnalyzer():
us(0)
{ // I'd rather this initialization be static...
dns_ignore.push_back("v=spf1");
dns_ignore.push_back("https:");
dns_del.push_back("NODATA-");
dns_del.push_back("NXDOMAIN-");
}



bool LogAnalyzer::line(const std::string &in) {
int ict=0;
NameVal::iterator nvp;
std::string name, address, s;

/// setup ///

if(!us)
throw std::runtime_error("LogAnalyzer::line: us list is not assigned");
ln=in;

/// DNS query result ///

// TODO: need to get more specific on tying us + them + time to DNS
// TODO: doesn't seem that CNAMEs are getting attached to requests properly.
// the logs are cryptic on this front.
if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) {
if(ln[5]=="reply" || ln[5]=="cached") {
name = ln[6];
address = ln[8];
// Hmm... is this reply an address?
if(pre_match(dns_ignore, address)) return 0; // nope
if(pre_match(dns_del, address)) return 0; // does not exist reply
if((nvp=rdns.find(address))!=rdns.end()) {
if(nvp->second==name) return 0;
//dlog("WARN: DNS address overlap "+address+": "+nvp->second+" : "+name);
}
rdns[address] = name;
//dlog("Added "+address+" = "+name);
return 0;
}
}

/// process connections ///

if((ln.count>5 // old Linux style
&& ln[4]=="kernel:"
&& ln[5]=="ACCEPT"
) || (ln.count>6 // new Linux style
&& ln[4]=="vmunix:"
&& ln[6]=="ACCEPT")
) {
conn = ln;
conn.compact();
if(!pre_match(*us, conn.us)) conn.swap();
if((nvp=rdns.find(conn.them))!=rdns.end()) conn.name = nvp->second;
return 1;
}
return 0;
}

+ 20
- 0
data.h View File

@@ -13,6 +13,7 @@
#ifndef __JFP_IPTRAFFIC_DATA_H__ #ifndef __JFP_IPTRAFFIC_DATA_H__
#define __JFP_IPTRAFFIC_DATA_H__ #define __JFP_IPTRAFFIC_DATA_H__
#include <string> #include <string>
#include <istream>
#include <ostream> #include <ostream>
#include <vector> #include <vector>
#include "strutil.h" #include "strutil.h"
@@ -89,4 +90,23 @@ struct ConnList: public std::vector<Conn> {






//////////////////////////////////////////////////////////////////////
// Log Analyzer
//////////////////////////////////////////////////////////////////////

struct LogAnalyzer {
StringList *us;
StringList dns_ignore, // DNS response prefixes to ignore
dns_del; // DNS response prefixes to /delete/ (ignore)
NameVal rdns; // Reverse DNS lookup cache
Conn conn; // Last connection worked on
Splits ln; // Work buffer for line processing

LogAnalyzer();
// Process a log line. Returns "true" if it were a netfilter entry.
bool line(const std::string &in);
};



#endif #endif

+ 19
- 57
iptraffic.cpp View File

@@ -23,10 +23,12 @@
// - Getting input and output filenams from CLI args // - Getting input and output filenams from CLI args
// - Reading and writing from STDIN & STDOUT // - Reading and writing from STDIN & STDOUT
// - Send all non-data output to stderr // - Send all non-data output to stderr
//
// 2021-08-11 <ChipMaster@YeOlPiShack.net>
// Move main data colation routine into its own class to be shared
// with multiple tools.
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////


// TODO: map names according to time and requesting host. time is probably automatic

#include <string.h> #include <string.h>
#include <string> #include <string>
#include <iostream> #include <iostream>
@@ -49,8 +51,7 @@ using namespace std;


struct IPtraffic: public cBaseApp { struct IPtraffic: public cBaseApp {
Config config; Config config;
StringList dns_ignore, dns_del;
NameVal rdns; // Reverse DNS lookup cache
LogAnalyzer analyze;
istream *log; istream *log;
ostream *out; ostream *out;
LiveBug bug; LiveBug bug;
@@ -60,10 +61,7 @@ struct IPtraffic: public cBaseApp {


IPtraffic(): out(&cout), log(0) IPtraffic(): out(&cout), log(0)
{ // I'd rather this initialization be static... { // I'd rather this initialization be static...
dns_ignore.push_back("v=spf1");
dns_ignore.push_back("https:");
dns_del.push_back("NODATA-");
dns_del.push_back("NXDOMAIN-");
analyze.us = &(config.us);
} }




@@ -127,69 +125,33 @@ struct IPtraffic: public cBaseApp {
// NOTE: the return values isn't really used yet but the channel is here if // NOTE: the return values isn't really used yet but the channel is here if
// it can be of use. // it can be of use.
int do_log() { int do_log() {
if(!config.us.size()) throw CLIerror(
"A configuration file must be specified before input files."
);

Splits ln;
int ict=0;
NameVal::iterator nvp;
string name, address, s;
Conn conn;
bool match;
int ict=0; // ignored netfilter lines
std::string l;


/// parse log file /// /// parse log file ///


if(!config.us.size()) throw CLIerror(
"A configuration file must be specified before input files."
);
line_no=0; line_no=0;
while((*log >> ln)) {
while(std::getline(*log, l)) {
line_no++; line_no++;
cerr << bug << ' ' << line_no << '\r' << flush; cerr << bug << ' ' << line_no << '\r' << flush;


/// DNS query result ///

// TODO: need to get more specific on tying us + them + time to DNS
// TODO: doesn't seem that CNAMEs are getting attached to requests properly.
// the logs are cryptic on this front.
if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) {
if(ln[5]=="reply" || ln[5]=="cached") {
name = ln[6];
address = ln[8];
// Hmm... is this reply an address?
if(pre_match(dns_ignore, address)) continue; // nope
if(pre_match(dns_del, address)) continue; // does not exist reply
if((nvp=rdns.find(address))!=rdns.end()) {
if(nvp->second==name) continue;
dlog("WARN: DNS address overlap "+address+": "+nvp->second+" : "+name);
}
rdns[address] = name;
dlog("Added "+address+" = "+name);
continue;
}
}

/// process connections /// /// process connections ///


if((ln.count>5 // old style
&& ln[4]=="kernel:"
&& ln[5]=="ACCEPT"
) || (ln.count>6 // new style
&& ln[4]=="vmunix:"
&& ln[6]=="ACCEPT")
) {
conn = ln;
conn.compact();
if(!pre_match(config.us, conn.us)) conn.swap();
if((nvp=rdns.find(conn.them))!=rdns.end())
conn.name = nvp->second;
if(config.ignores.find(conn)<0)
*out << ln[0] << " " << ln[1] << " " << ln[2] << " " << conn << "\n";
if(analyze.line(l)) {
if(config.ignores.find(analyze.conn)<0)
*out << analyze.ln[0] << " " << analyze.ln[1] << " " << analyze.ln[2]
<< " " << analyze.conn << "\n";
else else
ict++; ict++;
} }
} }
*out << flush; // make sure all data gets written. *out << flush; // make sure all data gets written.
cerr << "\nIgnored: " << ict << endl;
cerr << "Total rDNS: " << rdns.size() << "\n";
cerr << "\nLines: " << line_no
<< "\nIgnored: " << ict
<< "\nTotal rDNS: " << analyze.rdns.size() << endl;
return 0; return 0;
} }




+ 1
- 0
tests/.gitignore View File

@@ -0,0 +1 @@
/data

+ 8
- 0
tests/Makefile View File

@@ -0,0 +1,8 @@
data: data.cpp ../data.o testit.o
g++ -o $@ $@.cpp ../data.o testit.o

../data.o: ../data.cpp ../data.h
cd .. && make data.o

testit.o: testit.cpp testit.h
g++ -c testit.cpp

+ 81
- 0
tests/data.cpp View File

@@ -0,0 +1,81 @@
//////////////////////////////////////////////////////////////////////
// Test "data" module
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net>
// Started June 28th, 2021
//
// NOTE: This is really incomplete. More tests to come.
//////////////////////////////////////////////////////////////////////
#include <string>
#include <iostream>
#include <stdio.h>
#include "testit.h"
#include "../data.h"



//////////////////////////////////////////////////////////////////////
// TestIt Jig
//////////////////////////////////////////////////////////////////////

TestIt test;



//////////////////////////////////////////////////////////////////////
// Test 1 - Wild Card Address Comparisons
//////////////////////////////////////////////////////////////////////

/// Table ///

struct WildAddrTest {
std::string addr1, addr2;
int result;
};

const int wild_addr_tests_ct = 13;
WildAddrTest wild_addr_tests[wild_addr_tests_ct] = {
// IPv4
{"192.168.255." , "192.168.255.7", 0},
{"192.168.255.7", "192.168.255." , 0},
{"192.168.255.7", "192.168.255.7", 0},
{"192.168.255.7", "192.168.255.8", -1},
{"192.168.254.7", "192.168.255." , -1},
{"192.168.256.7", "192.168.255." , 1},
{"*" , "192.168.255." , 0},
{"192.168.256.7", "*" , 0},
//IPv6
{"2001:0470:000a:0169:" , "2001:0470:000a:0169:0000:0000:0000:0001", 0},
{"2001:0470:000a:0169:0000:0000:0000:0002", "2001:0470:000a:0169:" , 0},
{"2001:0470:000a:0170:" , "2001:0470:000a:0169:0000:0000:0000:0003", 1},
{"2001:0470:000a:0168:0000:0000:0000:0004", "2001:0470:000a:0169:" , -1},
{"2001:0470:000a:0169:0000:0000:0000:0001", "2001:0470:000a:0169:0000:0000:0000:0001", 0},
};

bool wild_addr_test() {
int i;
bool ok = true;
char s[256]; s[255]=0;
test.module("Wild Card Address Match");
for(i=0; i<wild_addr_tests_ct; i++) {
snprintf(s, 255, "Test match %3d", i);
if(!test.test(s,
addr_wild_comp(wild_addr_tests[i].addr1, wild_addr_tests[i].addr2),
wild_addr_tests[i].result
))
ok = false;
}
return ok;
}



//////////////////////////////////////////////////////////////////////
// Lets do it
//////////////////////////////////////////////////////////////////////

int main() {
wild_addr_test();
return !test.report();
}

+ 52
- 0
tests/testit.cpp View File

@@ -0,0 +1,52 @@
//////////////////////////////////////////////////////////////////////
// Test Jig
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net>
// Started June 28th, 2021
// Copyright 2021 JF Possibilities, Inc. All Rights Reserved
//////////////////////////////////////////////////////////////////////
#include <iostream>
#include "testit.h"
using namespace std;



//////////////////////////////////////////////////////////////////////
// TestIt
//////////////////////////////////////////////////////////////////////

const string bools[2] = {
vt100::RED+"FAIL"+vt100::RST, vt100::GRN+"ok"+vt100::RST
};



void TestIt::module(const string &title) {
cout << "\n"
<< vt100::CYN << title << ":\n"
<< string(title.size()+1, '=') << vt100::RST << endl;
}



bool TestIt::test(const string &title, int result, int match) {
bool r = result==match;
cout << vt100::YLW << title << '\t' << bools[r] << endl;
count++;
if(r) passes++;
else fails.push_back(title);
return r;
}



int TestIt::report() {
int r = 0;
if(fails.size()==count) r = 3; // Oh! That's miserable
else if(fails.size()) r = 2; // something worked.
cout << "\n"
<< vt100::CYN << "RESULTS: Tests " << (r ? vt100::YLW : vt100::GRN) << count
<< vt100::CYN << " Passes " << vt100::GRN << passes
<< vt100::CYN << " Fails " << (r ? vt100::RED : vt100::GRN) << fails.size()
<< vt100::RST << "\n";
return r;
}

+ 51
- 0
tests/testit.h View File

@@ -0,0 +1,51 @@
//////////////////////////////////////////////////////////////////////
// Test Jig
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net>
// Started June 28th, 2021
// Copyright 2021 JF Possibilities, Inc. All Rights Reserved
// Included by permission from JF Possibilities, Inc.
//
// Simple framework for running tests and reporting. Mostly this is an
// output formatter but also tracks stats on the tests reported to it.
//////////////////////////////////////////////////////////////////////
#include <string>
#include <vector>



//////////////////////////////////////////////////////////////////////
// Pretty basic mechanism of formatting test output and performing
// pass / fail accounting.
//////////////////////////////////////////////////////////////////////

struct TestIt {
int count; // total count of calls to test()
int passes; // total test() calls that matched
std::vector<std::string> fails; // list of test titles that failed
TestIt(): count(0), passes(0) {};
void module(const std::string &title);// Start a new test module
bool test(const std::string &title, int result, int match); // Record test result
int report(); // Print summary report and return: 0 passed, 2 some failed, 3 compete fail, suitable for return from main()
};



//////////////////////////////////////////////////////////////////////
// VT100 terminal control code constants
//
// Probably should learn to use termcap / curses.
//
// COLORS!
//////////////////////////////////////////////////////////////////////
namespace vt100 {

const std::string
RED = "\e[31;1m",
YLW = "\e[33;1m",
GRN = "\e[32m",
GRY = "\e[2m",
CYN = "\e[36m",
RST = "\e[0m";

}

Loading…
Cancel
Save