Added tests for testing address wild card matching and a test jig to put tests in. I moved the core of the log analyzer out of iptraffic and into data.o for use in multiple tools.master
@@ -14,12 +14,13 @@ | |||
#include <string> | |||
#include <vector> | |||
#include "data.h" | |||
#include "strutil.h" | |||
struct Config { | |||
std::vector<std::string> us; | |||
ConnList ignores; | |||
StringList us; | |||
ConnList ignores; | |||
void load(const std::string &fname); | |||
}; | |||
@@ -179,3 +179,71 @@ int ConnList::find(Conn &needle) { | |||
for(r=0; r<size(); r++) if((*this)[r]==needle) return r; | |||
return -1; | |||
} | |||
////////////////////////////////////////////////////////////////////// | |||
// LogAnalyzer | |||
////////////////////////////////////////////////////////////////////// | |||
LogAnalyzer::LogAnalyzer(): | |||
us(0) | |||
{ // I'd rather this initialization be static... | |||
dns_ignore.push_back("v=spf1"); | |||
dns_ignore.push_back("https:"); | |||
dns_del.push_back("NODATA-"); | |||
dns_del.push_back("NXDOMAIN-"); | |||
} | |||
bool LogAnalyzer::line(const std::string &in) { | |||
int ict=0; | |||
NameVal::iterator nvp; | |||
std::string name, address, s; | |||
/// setup /// | |||
if(!us) | |||
throw std::runtime_error("LogAnalyzer::line: us list is not assigned"); | |||
ln=in; | |||
/// DNS query result /// | |||
// TODO: need to get more specific on tying us + them + time to DNS | |||
// TODO: doesn't seem that CNAMEs are getting attached to requests properly. | |||
// the logs are cryptic on this front. | |||
if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) { | |||
if(ln[5]=="reply" || ln[5]=="cached") { | |||
name = ln[6]; | |||
address = ln[8]; | |||
// Hmm... is this reply an address? | |||
if(pre_match(dns_ignore, address)) return 0; // nope | |||
if(pre_match(dns_del, address)) return 0; // does not exist reply | |||
if((nvp=rdns.find(address))!=rdns.end()) { | |||
if(nvp->second==name) return 0; | |||
//dlog("WARN: DNS address overlap "+address+": "+nvp->second+" : "+name); | |||
} | |||
rdns[address] = name; | |||
//dlog("Added "+address+" = "+name); | |||
return 0; | |||
} | |||
} | |||
/// process connections /// | |||
if((ln.count>5 // old Linux style | |||
&& ln[4]=="kernel:" | |||
&& ln[5]=="ACCEPT" | |||
) || (ln.count>6 // new Linux style | |||
&& ln[4]=="vmunix:" | |||
&& ln[6]=="ACCEPT") | |||
) { | |||
conn = ln; | |||
conn.compact(); | |||
if(!pre_match(*us, conn.us)) conn.swap(); | |||
if((nvp=rdns.find(conn.them))!=rdns.end()) conn.name = nvp->second; | |||
return 1; | |||
} | |||
return 0; | |||
} |
@@ -13,6 +13,7 @@ | |||
#ifndef __JFP_IPTRAFFIC_DATA_H__ | |||
#define __JFP_IPTRAFFIC_DATA_H__ | |||
#include <string> | |||
#include <istream> | |||
#include <ostream> | |||
#include <vector> | |||
#include "strutil.h" | |||
@@ -89,4 +90,23 @@ struct ConnList: public std::vector<Conn> { | |||
////////////////////////////////////////////////////////////////////// | |||
// Log Analyzer | |||
////////////////////////////////////////////////////////////////////// | |||
struct LogAnalyzer { | |||
StringList *us; | |||
StringList dns_ignore, // DNS response prefixes to ignore | |||
dns_del; // DNS response prefixes to /delete/ (ignore) | |||
NameVal rdns; // Reverse DNS lookup cache | |||
Conn conn; // Last connection worked on | |||
Splits ln; // Work buffer for line processing | |||
LogAnalyzer(); | |||
// Process a log line. Returns "true" if it were a netfilter entry. | |||
bool line(const std::string &in); | |||
}; | |||
#endif |
@@ -23,10 +23,12 @@ | |||
// - Getting input and output filenams from CLI args | |||
// - Reading and writing from STDIN & STDOUT | |||
// - Send all non-data output to stderr | |||
// | |||
// 2021-08-11 <ChipMaster@YeOlPiShack.net> | |||
// Move main data colation routine into its own class to be shared | |||
// with multiple tools. | |||
////////////////////////////////////////////////////////////////////// | |||
// TODO: map names according to time and requesting host. time is probably automatic | |||
#include <string.h> | |||
#include <string> | |||
#include <iostream> | |||
@@ -49,8 +51,7 @@ using namespace std; | |||
struct IPtraffic: public cBaseApp { | |||
Config config; | |||
StringList dns_ignore, dns_del; | |||
NameVal rdns; // Reverse DNS lookup cache | |||
LogAnalyzer analyze; | |||
istream *log; | |||
ostream *out; | |||
LiveBug bug; | |||
@@ -60,10 +61,7 @@ struct IPtraffic: public cBaseApp { | |||
IPtraffic(): out(&cout), log(0) | |||
{ // I'd rather this initialization be static... | |||
dns_ignore.push_back("v=spf1"); | |||
dns_ignore.push_back("https:"); | |||
dns_del.push_back("NODATA-"); | |||
dns_del.push_back("NXDOMAIN-"); | |||
analyze.us = &(config.us); | |||
} | |||
@@ -127,69 +125,33 @@ struct IPtraffic: public cBaseApp { | |||
// NOTE: the return values isn't really used yet but the channel is here if | |||
// it can be of use. | |||
int do_log() { | |||
if(!config.us.size()) throw CLIerror( | |||
"A configuration file must be specified before input files." | |||
); | |||
Splits ln; | |||
int ict=0; | |||
NameVal::iterator nvp; | |||
string name, address, s; | |||
Conn conn; | |||
bool match; | |||
int ict=0; // ignored netfilter lines | |||
std::string l; | |||
/// parse log file /// | |||
if(!config.us.size()) throw CLIerror( | |||
"A configuration file must be specified before input files." | |||
); | |||
line_no=0; | |||
while((*log >> ln)) { | |||
while(std::getline(*log, l)) { | |||
line_no++; | |||
cerr << bug << ' ' << line_no << '\r' << flush; | |||
/// DNS query result /// | |||
// TODO: need to get more specific on tying us + them + time to DNS | |||
// TODO: doesn't seem that CNAMEs are getting attached to requests properly. | |||
// the logs are cryptic on this front. | |||
if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) { | |||
if(ln[5]=="reply" || ln[5]=="cached") { | |||
name = ln[6]; | |||
address = ln[8]; | |||
// Hmm... is this reply an address? | |||
if(pre_match(dns_ignore, address)) continue; // nope | |||
if(pre_match(dns_del, address)) continue; // does not exist reply | |||
if((nvp=rdns.find(address))!=rdns.end()) { | |||
if(nvp->second==name) continue; | |||
dlog("WARN: DNS address overlap "+address+": "+nvp->second+" : "+name); | |||
} | |||
rdns[address] = name; | |||
dlog("Added "+address+" = "+name); | |||
continue; | |||
} | |||
} | |||
/// process connections /// | |||
if((ln.count>5 // old style | |||
&& ln[4]=="kernel:" | |||
&& ln[5]=="ACCEPT" | |||
) || (ln.count>6 // new style | |||
&& ln[4]=="vmunix:" | |||
&& ln[6]=="ACCEPT") | |||
) { | |||
conn = ln; | |||
conn.compact(); | |||
if(!pre_match(config.us, conn.us)) conn.swap(); | |||
if((nvp=rdns.find(conn.them))!=rdns.end()) | |||
conn.name = nvp->second; | |||
if(config.ignores.find(conn)<0) | |||
*out << ln[0] << " " << ln[1] << " " << ln[2] << " " << conn << "\n"; | |||
if(analyze.line(l)) { | |||
if(config.ignores.find(analyze.conn)<0) | |||
*out << analyze.ln[0] << " " << analyze.ln[1] << " " << analyze.ln[2] | |||
<< " " << analyze.conn << "\n"; | |||
else | |||
ict++; | |||
} | |||
} | |||
*out << flush; // make sure all data gets written. | |||
cerr << "\nIgnored: " << ict << endl; | |||
cerr << "Total rDNS: " << rdns.size() << "\n"; | |||
cerr << "\nLines: " << line_no | |||
<< "\nIgnored: " << ict | |||
<< "\nTotal rDNS: " << analyze.rdns.size() << endl; | |||
return 0; | |||
} | |||
@@ -0,0 +1 @@ | |||
/data |
@@ -0,0 +1,8 @@ | |||
data: data.cpp ../data.o testit.o | |||
g++ -o $@ $@.cpp ../data.o testit.o | |||
../data.o: ../data.cpp ../data.h | |||
cd .. && make data.o | |||
testit.o: testit.cpp testit.h | |||
g++ -c testit.cpp |
@@ -0,0 +1,81 @@ | |||
////////////////////////////////////////////////////////////////////// | |||
// Test "data" module | |||
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net> | |||
// Started June 28th, 2021 | |||
// | |||
// NOTE: This is really incomplete. More tests to come. | |||
////////////////////////////////////////////////////////////////////// | |||
#include <string> | |||
#include <iostream> | |||
#include <stdio.h> | |||
#include "testit.h" | |||
#include "../data.h" | |||
////////////////////////////////////////////////////////////////////// | |||
// TestIt Jig | |||
////////////////////////////////////////////////////////////////////// | |||
TestIt test; | |||
////////////////////////////////////////////////////////////////////// | |||
// Test 1 - Wild Card Address Comparisons | |||
////////////////////////////////////////////////////////////////////// | |||
/// Table /// | |||
struct WildAddrTest { | |||
std::string addr1, addr2; | |||
int result; | |||
}; | |||
const int wild_addr_tests_ct = 13; | |||
WildAddrTest wild_addr_tests[wild_addr_tests_ct] = { | |||
// IPv4 | |||
{"192.168.255." , "192.168.255.7", 0}, | |||
{"192.168.255.7", "192.168.255." , 0}, | |||
{"192.168.255.7", "192.168.255.7", 0}, | |||
{"192.168.255.7", "192.168.255.8", -1}, | |||
{"192.168.254.7", "192.168.255." , -1}, | |||
{"192.168.256.7", "192.168.255." , 1}, | |||
{"*" , "192.168.255." , 0}, | |||
{"192.168.256.7", "*" , 0}, | |||
//IPv6 | |||
{"2001:0470:000a:0169:" , "2001:0470:000a:0169:0000:0000:0000:0001", 0}, | |||
{"2001:0470:000a:0169:0000:0000:0000:0002", "2001:0470:000a:0169:" , 0}, | |||
{"2001:0470:000a:0170:" , "2001:0470:000a:0169:0000:0000:0000:0003", 1}, | |||
{"2001:0470:000a:0168:0000:0000:0000:0004", "2001:0470:000a:0169:" , -1}, | |||
{"2001:0470:000a:0169:0000:0000:0000:0001", "2001:0470:000a:0169:0000:0000:0000:0001", 0}, | |||
}; | |||
bool wild_addr_test() { | |||
int i; | |||
bool ok = true; | |||
char s[256]; s[255]=0; | |||
test.module("Wild Card Address Match"); | |||
for(i=0; i<wild_addr_tests_ct; i++) { | |||
snprintf(s, 255, "Test match %3d", i); | |||
if(!test.test(s, | |||
addr_wild_comp(wild_addr_tests[i].addr1, wild_addr_tests[i].addr2), | |||
wild_addr_tests[i].result | |||
)) | |||
ok = false; | |||
} | |||
return ok; | |||
} | |||
////////////////////////////////////////////////////////////////////// | |||
// Lets do it | |||
////////////////////////////////////////////////////////////////////// | |||
int main() { | |||
wild_addr_test(); | |||
return !test.report(); | |||
} |
@@ -0,0 +1,52 @@ | |||
////////////////////////////////////////////////////////////////////// | |||
// Test Jig | |||
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net> | |||
// Started June 28th, 2021 | |||
// Copyright 2021 JF Possibilities, Inc. All Rights Reserved | |||
////////////////////////////////////////////////////////////////////// | |||
#include <iostream> | |||
#include "testit.h" | |||
using namespace std; | |||
////////////////////////////////////////////////////////////////////// | |||
// TestIt | |||
////////////////////////////////////////////////////////////////////// | |||
const string bools[2] = { | |||
vt100::RED+"FAIL"+vt100::RST, vt100::GRN+"ok"+vt100::RST | |||
}; | |||
void TestIt::module(const string &title) { | |||
cout << "\n" | |||
<< vt100::CYN << title << ":\n" | |||
<< string(title.size()+1, '=') << vt100::RST << endl; | |||
} | |||
bool TestIt::test(const string &title, int result, int match) { | |||
bool r = result==match; | |||
cout << vt100::YLW << title << '\t' << bools[r] << endl; | |||
count++; | |||
if(r) passes++; | |||
else fails.push_back(title); | |||
return r; | |||
} | |||
int TestIt::report() { | |||
int r = 0; | |||
if(fails.size()==count) r = 3; // Oh! That's miserable | |||
else if(fails.size()) r = 2; // something worked. | |||
cout << "\n" | |||
<< vt100::CYN << "RESULTS: Tests " << (r ? vt100::YLW : vt100::GRN) << count | |||
<< vt100::CYN << " Passes " << vt100::GRN << passes | |||
<< vt100::CYN << " Fails " << (r ? vt100::RED : vt100::GRN) << fails.size() | |||
<< vt100::RST << "\n"; | |||
return r; | |||
} |
@@ -0,0 +1,51 @@ | |||
////////////////////////////////////////////////////////////////////// | |||
// Test Jig | |||
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net> | |||
// Started June 28th, 2021 | |||
// Copyright 2021 JF Possibilities, Inc. All Rights Reserved | |||
// Included by permission from JF Possibilities, Inc. | |||
// | |||
// Simple framework for running tests and reporting. Mostly this is an | |||
// output formatter but also tracks stats on the tests reported to it. | |||
////////////////////////////////////////////////////////////////////// | |||
#include <string> | |||
#include <vector> | |||
////////////////////////////////////////////////////////////////////// | |||
// Pretty basic mechanism of formatting test output and performing | |||
// pass / fail accounting. | |||
////////////////////////////////////////////////////////////////////// | |||
struct TestIt { | |||
int count; // total count of calls to test() | |||
int passes; // total test() calls that matched | |||
std::vector<std::string> fails; // list of test titles that failed | |||
TestIt(): count(0), passes(0) {}; | |||
void module(const std::string &title);// Start a new test module | |||
bool test(const std::string &title, int result, int match); // Record test result | |||
int report(); // Print summary report and return: 0 passed, 2 some failed, 3 compete fail, suitable for return from main() | |||
}; | |||
////////////////////////////////////////////////////////////////////// | |||
// VT100 terminal control code constants | |||
// | |||
// Probably should learn to use termcap / curses. | |||
// | |||
// COLORS! | |||
////////////////////////////////////////////////////////////////////// | |||
namespace vt100 { | |||
const std::string | |||
RED = "\e[31;1m", | |||
YLW = "\e[33;1m", | |||
GRN = "\e[32m", | |||
GRY = "\e[2m", | |||
CYN = "\e[36m", | |||
RST = "\e[0m"; | |||
} |