Added tests for testing address wild card matching and a test jig to put tests in. I moved the core of the log analyzer out of iptraffic and into data.o for use in multiple tools.master
@@ -14,12 +14,13 @@ | |||||
#include <string> | #include <string> | ||||
#include <vector> | #include <vector> | ||||
#include "data.h" | #include "data.h" | ||||
#include "strutil.h" | |||||
struct Config { | struct Config { | ||||
std::vector<std::string> us; | |||||
ConnList ignores; | |||||
StringList us; | |||||
ConnList ignores; | |||||
void load(const std::string &fname); | void load(const std::string &fname); | ||||
}; | }; | ||||
@@ -179,3 +179,71 @@ int ConnList::find(Conn &needle) { | |||||
for(r=0; r<size(); r++) if((*this)[r]==needle) return r; | for(r=0; r<size(); r++) if((*this)[r]==needle) return r; | ||||
return -1; | return -1; | ||||
} | } | ||||
////////////////////////////////////////////////////////////////////// | |||||
// LogAnalyzer | |||||
////////////////////////////////////////////////////////////////////// | |||||
LogAnalyzer::LogAnalyzer(): | |||||
us(0) | |||||
{ // I'd rather this initialization be static... | |||||
dns_ignore.push_back("v=spf1"); | |||||
dns_ignore.push_back("https:"); | |||||
dns_del.push_back("NODATA-"); | |||||
dns_del.push_back("NXDOMAIN-"); | |||||
} | |||||
bool LogAnalyzer::line(const std::string &in) { | |||||
int ict=0; | |||||
NameVal::iterator nvp; | |||||
std::string name, address, s; | |||||
/// setup /// | |||||
if(!us) | |||||
throw std::runtime_error("LogAnalyzer::line: us list is not assigned"); | |||||
ln=in; | |||||
/// DNS query result /// | |||||
// TODO: need to get more specific on tying us + them + time to DNS | |||||
// TODO: doesn't seem that CNAMEs are getting attached to requests properly. | |||||
// the logs are cryptic on this front. | |||||
if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) { | |||||
if(ln[5]=="reply" || ln[5]=="cached") { | |||||
name = ln[6]; | |||||
address = ln[8]; | |||||
// Hmm... is this reply an address? | |||||
if(pre_match(dns_ignore, address)) return 0; // nope | |||||
if(pre_match(dns_del, address)) return 0; // does not exist reply | |||||
if((nvp=rdns.find(address))!=rdns.end()) { | |||||
if(nvp->second==name) return 0; | |||||
//dlog("WARN: DNS address overlap "+address+": "+nvp->second+" : "+name); | |||||
} | |||||
rdns[address] = name; | |||||
//dlog("Added "+address+" = "+name); | |||||
return 0; | |||||
} | |||||
} | |||||
/// process connections /// | |||||
if((ln.count>5 // old Linux style | |||||
&& ln[4]=="kernel:" | |||||
&& ln[5]=="ACCEPT" | |||||
) || (ln.count>6 // new Linux style | |||||
&& ln[4]=="vmunix:" | |||||
&& ln[6]=="ACCEPT") | |||||
) { | |||||
conn = ln; | |||||
conn.compact(); | |||||
if(!pre_match(*us, conn.us)) conn.swap(); | |||||
if((nvp=rdns.find(conn.them))!=rdns.end()) conn.name = nvp->second; | |||||
return 1; | |||||
} | |||||
return 0; | |||||
} |
@@ -13,6 +13,7 @@ | |||||
#ifndef __JFP_IPTRAFFIC_DATA_H__ | #ifndef __JFP_IPTRAFFIC_DATA_H__ | ||||
#define __JFP_IPTRAFFIC_DATA_H__ | #define __JFP_IPTRAFFIC_DATA_H__ | ||||
#include <string> | #include <string> | ||||
#include <istream> | |||||
#include <ostream> | #include <ostream> | ||||
#include <vector> | #include <vector> | ||||
#include "strutil.h" | #include "strutil.h" | ||||
@@ -89,4 +90,23 @@ struct ConnList: public std::vector<Conn> { | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Log Analyzer | |||||
////////////////////////////////////////////////////////////////////// | |||||
struct LogAnalyzer { | |||||
StringList *us; | |||||
StringList dns_ignore, // DNS response prefixes to ignore | |||||
dns_del; // DNS response prefixes to /delete/ (ignore) | |||||
NameVal rdns; // Reverse DNS lookup cache | |||||
Conn conn; // Last connection worked on | |||||
Splits ln; // Work buffer for line processing | |||||
LogAnalyzer(); | |||||
// Process a log line. Returns "true" if it were a netfilter entry. | |||||
bool line(const std::string &in); | |||||
}; | |||||
#endif | #endif |
@@ -23,10 +23,12 @@ | |||||
// - Getting input and output filenams from CLI args | // - Getting input and output filenams from CLI args | ||||
// - Reading and writing from STDIN & STDOUT | // - Reading and writing from STDIN & STDOUT | ||||
// - Send all non-data output to stderr | // - Send all non-data output to stderr | ||||
// | |||||
// 2021-08-11 <ChipMaster@YeOlPiShack.net> | |||||
// Move main data colation routine into its own class to be shared | |||||
// with multiple tools. | |||||
////////////////////////////////////////////////////////////////////// | ////////////////////////////////////////////////////////////////////// | ||||
// TODO: map names according to time and requesting host. time is probably automatic | |||||
#include <string.h> | #include <string.h> | ||||
#include <string> | #include <string> | ||||
#include <iostream> | #include <iostream> | ||||
@@ -49,8 +51,7 @@ using namespace std; | |||||
struct IPtraffic: public cBaseApp { | struct IPtraffic: public cBaseApp { | ||||
Config config; | Config config; | ||||
StringList dns_ignore, dns_del; | |||||
NameVal rdns; // Reverse DNS lookup cache | |||||
LogAnalyzer analyze; | |||||
istream *log; | istream *log; | ||||
ostream *out; | ostream *out; | ||||
LiveBug bug; | LiveBug bug; | ||||
@@ -60,10 +61,7 @@ struct IPtraffic: public cBaseApp { | |||||
IPtraffic(): out(&cout), log(0) | IPtraffic(): out(&cout), log(0) | ||||
{ // I'd rather this initialization be static... | { // I'd rather this initialization be static... | ||||
dns_ignore.push_back("v=spf1"); | |||||
dns_ignore.push_back("https:"); | |||||
dns_del.push_back("NODATA-"); | |||||
dns_del.push_back("NXDOMAIN-"); | |||||
analyze.us = &(config.us); | |||||
} | } | ||||
@@ -127,69 +125,33 @@ struct IPtraffic: public cBaseApp { | |||||
// NOTE: the return values isn't really used yet but the channel is here if | // NOTE: the return values isn't really used yet but the channel is here if | ||||
// it can be of use. | // it can be of use. | ||||
int do_log() { | int do_log() { | ||||
if(!config.us.size()) throw CLIerror( | |||||
"A configuration file must be specified before input files." | |||||
); | |||||
Splits ln; | |||||
int ict=0; | |||||
NameVal::iterator nvp; | |||||
string name, address, s; | |||||
Conn conn; | |||||
bool match; | |||||
int ict=0; // ignored netfilter lines | |||||
std::string l; | |||||
/// parse log file /// | /// parse log file /// | ||||
if(!config.us.size()) throw CLIerror( | |||||
"A configuration file must be specified before input files." | |||||
); | |||||
line_no=0; | line_no=0; | ||||
while((*log >> ln)) { | |||||
while(std::getline(*log, l)) { | |||||
line_no++; | line_no++; | ||||
cerr << bug << ' ' << line_no << '\r' << flush; | cerr << bug << ' ' << line_no << '\r' << flush; | ||||
/// DNS query result /// | |||||
// TODO: need to get more specific on tying us + them + time to DNS | |||||
// TODO: doesn't seem that CNAMEs are getting attached to requests properly. | |||||
// the logs are cryptic on this front. | |||||
if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) { | |||||
if(ln[5]=="reply" || ln[5]=="cached") { | |||||
name = ln[6]; | |||||
address = ln[8]; | |||||
// Hmm... is this reply an address? | |||||
if(pre_match(dns_ignore, address)) continue; // nope | |||||
if(pre_match(dns_del, address)) continue; // does not exist reply | |||||
if((nvp=rdns.find(address))!=rdns.end()) { | |||||
if(nvp->second==name) continue; | |||||
dlog("WARN: DNS address overlap "+address+": "+nvp->second+" : "+name); | |||||
} | |||||
rdns[address] = name; | |||||
dlog("Added "+address+" = "+name); | |||||
continue; | |||||
} | |||||
} | |||||
/// process connections /// | /// process connections /// | ||||
if((ln.count>5 // old style | |||||
&& ln[4]=="kernel:" | |||||
&& ln[5]=="ACCEPT" | |||||
) || (ln.count>6 // new style | |||||
&& ln[4]=="vmunix:" | |||||
&& ln[6]=="ACCEPT") | |||||
) { | |||||
conn = ln; | |||||
conn.compact(); | |||||
if(!pre_match(config.us, conn.us)) conn.swap(); | |||||
if((nvp=rdns.find(conn.them))!=rdns.end()) | |||||
conn.name = nvp->second; | |||||
if(config.ignores.find(conn)<0) | |||||
*out << ln[0] << " " << ln[1] << " " << ln[2] << " " << conn << "\n"; | |||||
if(analyze.line(l)) { | |||||
if(config.ignores.find(analyze.conn)<0) | |||||
*out << analyze.ln[0] << " " << analyze.ln[1] << " " << analyze.ln[2] | |||||
<< " " << analyze.conn << "\n"; | |||||
else | else | ||||
ict++; | ict++; | ||||
} | } | ||||
} | } | ||||
*out << flush; // make sure all data gets written. | *out << flush; // make sure all data gets written. | ||||
cerr << "\nIgnored: " << ict << endl; | |||||
cerr << "Total rDNS: " << rdns.size() << "\n"; | |||||
cerr << "\nLines: " << line_no | |||||
<< "\nIgnored: " << ict | |||||
<< "\nTotal rDNS: " << analyze.rdns.size() << endl; | |||||
return 0; | return 0; | ||||
} | } | ||||
@@ -0,0 +1 @@ | |||||
/data |
@@ -0,0 +1,8 @@ | |||||
data: data.cpp ../data.o testit.o | |||||
g++ -o $@ $@.cpp ../data.o testit.o | |||||
../data.o: ../data.cpp ../data.h | |||||
cd .. && make data.o | |||||
testit.o: testit.cpp testit.h | |||||
g++ -c testit.cpp |
@@ -0,0 +1,81 @@ | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Test "data" module | |||||
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net> | |||||
// Started June 28th, 2021 | |||||
// | |||||
// NOTE: This is really incomplete. More tests to come. | |||||
////////////////////////////////////////////////////////////////////// | |||||
#include <string> | |||||
#include <iostream> | |||||
#include <stdio.h> | |||||
#include "testit.h" | |||||
#include "../data.h" | |||||
////////////////////////////////////////////////////////////////////// | |||||
// TestIt Jig | |||||
////////////////////////////////////////////////////////////////////// | |||||
TestIt test; | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Test 1 - Wild Card Address Comparisons | |||||
////////////////////////////////////////////////////////////////////// | |||||
/// Table /// | |||||
struct WildAddrTest { | |||||
std::string addr1, addr2; | |||||
int result; | |||||
}; | |||||
const int wild_addr_tests_ct = 13; | |||||
WildAddrTest wild_addr_tests[wild_addr_tests_ct] = { | |||||
// IPv4 | |||||
{"192.168.255." , "192.168.255.7", 0}, | |||||
{"192.168.255.7", "192.168.255." , 0}, | |||||
{"192.168.255.7", "192.168.255.7", 0}, | |||||
{"192.168.255.7", "192.168.255.8", -1}, | |||||
{"192.168.254.7", "192.168.255." , -1}, | |||||
{"192.168.256.7", "192.168.255." , 1}, | |||||
{"*" , "192.168.255." , 0}, | |||||
{"192.168.256.7", "*" , 0}, | |||||
//IPv6 | |||||
{"2001:0470:000a:0169:" , "2001:0470:000a:0169:0000:0000:0000:0001", 0}, | |||||
{"2001:0470:000a:0169:0000:0000:0000:0002", "2001:0470:000a:0169:" , 0}, | |||||
{"2001:0470:000a:0170:" , "2001:0470:000a:0169:0000:0000:0000:0003", 1}, | |||||
{"2001:0470:000a:0168:0000:0000:0000:0004", "2001:0470:000a:0169:" , -1}, | |||||
{"2001:0470:000a:0169:0000:0000:0000:0001", "2001:0470:000a:0169:0000:0000:0000:0001", 0}, | |||||
}; | |||||
bool wild_addr_test() { | |||||
int i; | |||||
bool ok = true; | |||||
char s[256]; s[255]=0; | |||||
test.module("Wild Card Address Match"); | |||||
for(i=0; i<wild_addr_tests_ct; i++) { | |||||
snprintf(s, 255, "Test match %3d", i); | |||||
if(!test.test(s, | |||||
addr_wild_comp(wild_addr_tests[i].addr1, wild_addr_tests[i].addr2), | |||||
wild_addr_tests[i].result | |||||
)) | |||||
ok = false; | |||||
} | |||||
return ok; | |||||
} | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Lets do it | |||||
////////////////////////////////////////////////////////////////////// | |||||
int main() { | |||||
wild_addr_test(); | |||||
return !test.report(); | |||||
} |
@@ -0,0 +1,52 @@ | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Test Jig | |||||
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net> | |||||
// Started June 28th, 2021 | |||||
// Copyright 2021 JF Possibilities, Inc. All Rights Reserved | |||||
////////////////////////////////////////////////////////////////////// | |||||
#include <iostream> | |||||
#include "testit.h" | |||||
using namespace std; | |||||
////////////////////////////////////////////////////////////////////// | |||||
// TestIt | |||||
////////////////////////////////////////////////////////////////////// | |||||
const string bools[2] = { | |||||
vt100::RED+"FAIL"+vt100::RST, vt100::GRN+"ok"+vt100::RST | |||||
}; | |||||
void TestIt::module(const string &title) { | |||||
cout << "\n" | |||||
<< vt100::CYN << title << ":\n" | |||||
<< string(title.size()+1, '=') << vt100::RST << endl; | |||||
} | |||||
bool TestIt::test(const string &title, int result, int match) { | |||||
bool r = result==match; | |||||
cout << vt100::YLW << title << '\t' << bools[r] << endl; | |||||
count++; | |||||
if(r) passes++; | |||||
else fails.push_back(title); | |||||
return r; | |||||
} | |||||
int TestIt::report() { | |||||
int r = 0; | |||||
if(fails.size()==count) r = 3; // Oh! That's miserable | |||||
else if(fails.size()) r = 2; // something worked. | |||||
cout << "\n" | |||||
<< vt100::CYN << "RESULTS: Tests " << (r ? vt100::YLW : vt100::GRN) << count | |||||
<< vt100::CYN << " Passes " << vt100::GRN << passes | |||||
<< vt100::CYN << " Fails " << (r ? vt100::RED : vt100::GRN) << fails.size() | |||||
<< vt100::RST << "\n"; | |||||
return r; | |||||
} |
@@ -0,0 +1,51 @@ | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Test Jig | |||||
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net> | |||||
// Started June 28th, 2021 | |||||
// Copyright 2021 JF Possibilities, Inc. All Rights Reserved | |||||
// Included by permission from JF Possibilities, Inc. | |||||
// | |||||
// Simple framework for running tests and reporting. Mostly this is an | |||||
// output formatter but also tracks stats on the tests reported to it. | |||||
////////////////////////////////////////////////////////////////////// | |||||
#include <string> | |||||
#include <vector> | |||||
////////////////////////////////////////////////////////////////////// | |||||
// Pretty basic mechanism of formatting test output and performing | |||||
// pass / fail accounting. | |||||
////////////////////////////////////////////////////////////////////// | |||||
struct TestIt { | |||||
int count; // total count of calls to test() | |||||
int passes; // total test() calls that matched | |||||
std::vector<std::string> fails; // list of test titles that failed | |||||
TestIt(): count(0), passes(0) {}; | |||||
void module(const std::string &title);// Start a new test module | |||||
bool test(const std::string &title, int result, int match); // Record test result | |||||
int report(); // Print summary report and return: 0 passed, 2 some failed, 3 compete fail, suitable for return from main() | |||||
}; | |||||
////////////////////////////////////////////////////////////////////// | |||||
// VT100 terminal control code constants | |||||
// | |||||
// Probably should learn to use termcap / curses. | |||||
// | |||||
// COLORS! | |||||
////////////////////////////////////////////////////////////////////// | |||||
namespace vt100 { | |||||
const std::string | |||||
RED = "\e[31;1m", | |||||
YLW = "\e[33;1m", | |||||
GRN = "\e[32m", | |||||
GRY = "\e[2m", | |||||
CYN = "\e[36m", | |||||
RST = "\e[0m"; | |||||
} |