Browse Source

Begin realtime tracking

* Program to run as daemon and listen to SysLogD on a pipe.
 * C++CMS web app to show accumulated domain names and set their status
 * Sample configs to be used to set things up
master
Jon Foster 1 year ago
parent
commit
cd7d62ca05
20 changed files with 1385 additions and 8 deletions
  1. +11
    -0
      .gitignore
  2. +17
    -6
      Makefile
  3. +1
    -1
      config.cpp
  4. +1
    -1
      config.h
  5. +21
    -0
      controlpanel/Makefile
  6. +36
    -0
      controlpanel/data.h
  7. +9
    -0
      controlpanel/default
  8. +108
    -0
      controlpanel/init
  9. +104
    -0
      controlpanel/mainskin.tmpl
  10. +65
    -0
      controlpanel/sample.js
  11. +171
    -0
      controlpanel/trafficctrl.cpp
  12. +87
    -0
      poorman-ids.dpak
  13. +32
    -0
      sample.conf
  14. +12
    -0
      strutil.cpp
  15. +2
    -0
      strutil.h
  16. +250
    -0
      trafficmon/badtrafficrpt.cpp
  17. +9
    -0
      trafficmon/default
  18. +113
    -0
      trafficmon/init
  19. +1
    -0
      trafficmon/syslog
  20. +335
    -0
      trafficmon/trafficmon.cpp

+ 11
- 0
.gitignore View File

@@ -3,3 +3,14 @@
/iptraffic
/log
/README.html
/trafficmon/badtrafficrpt
/trafficmon/trafficmon
/controlpanel/trafficctrl

# I use these as output of C++ generators. Don't store them:
*.cxx

# Debian (DPAK) packaging artifacts
*.deb
/tmp-dpak-poorman-ids/


+ 17
- 6
Makefile View File

@@ -1,8 +1,21 @@
O=-s


### Program Targets ###

iptraffic: iptraffic.cpp strutil.o data.o config.o cli.o miniini.o
g++ $O -o $@ $@.cpp strutil.o data.o config.o cli.o miniini.o

trafficmon/trafficmon: trafficmon/trafficmon.cpp strutil.o data.o config.o cli.o miniini.o
g++ $O -o $@ $@.cpp strutil.o data.o config.o cli.o miniini.o -lcppdb

trafficmon/badtrafficrpt: trafficmon/badtrafficrpt.cpp cli.o miniini.o strutil.o
g++ $O -o $@ $@.cpp strutil.o cli.o miniini.o -lcppdb



### Libs ###

cli.o: cli.cpp cli.h
g++ $O -c -o $@ cli.cpp

@@ -20,14 +33,12 @@ strutil.o: strutil.cpp strutil.h



.PHONY: run
run: iptraffic
./iptraffic


### Source Maintenance ###

.PHONY: clean distclean
clean:
rm *.o || true
distclean: clean
rm iptraffic || true
rm iptraffic trafficmon/trafficmon trafficmon/badtrafficrpt || true
rm *.deb || true
cd controlpanel && make distclean

+ 1
- 1
config.cpp View File

@@ -32,7 +32,7 @@ void INIconnList::add(const std::string &in) {

if(in=="" || in[0]=='#') return; // remarks
// TODO: we don't want to keep create+destroy-ing these?
TSV tsv(in);
TSV tsv(in);
if(tsv.count!=7) throw
// TODO: really need a line number!
std::runtime_error("INIconnList::add: Incorrect column count in config file line");


+ 1
- 1
config.h View File

@@ -52,7 +52,7 @@ struct INIconnList: public MiniINIgroup {
struct Config: public MiniINI {
INIusList us;
INIconnList ignores;
Config();
Config();
};




+ 21
- 0
controlpanel/Makefile View File

@@ -0,0 +1,21 @@
# Optional compiler flags
#O=-std=c++11

trafficctrl: trafficctrl.cpp data.h ../strutil.o mainskin.o
g++ $O -o $@ $@.cpp mainskin.o ../strutil.o -lcppcms -lcppdb -lbooster

../strutil.o: ../strutil.cpp ../strutil.h
cd .. && make strutil.o

mainskin.cxx: mainskin.tmpl
cppcms_tmpl_cc -o $@ mainskin.tmpl
mainskin.o: mainskin.cxx data.h
g++ $O -c mainskin.cxx



.PHONY: clean distclean
clean:
rm *.o *.cxx || true
distclean: clean
rm trafficctrl || true

+ 36
- 0
controlpanel/data.h View File

@@ -0,0 +1,36 @@
//////////////////////////////////////////////////////////////////////
// Control Panel and domain control data
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net>
// Started August 13th, 2021
// Copyright JF Possibilities, Inc. All rights reserved.
//
// Data for dealing with white / black lists.
//////////////////////////////////////////////////////////////////////
#ifndef __CONTROL_DATA_H__
#define __CONTROL_DATA_H__
#include <string>
#include <vector>
#include <cppcms/base_content.h>
//#include <booster/function.h>
//#include <cppcms/serialization.h>



struct Domain {
enum STATUS {undecided, accepted, blocked};
std::string name; // domain name
std::string decided; // since C++ doesn't have a date/time
int status; // how we should handle it
};



struct DomainList :public cppcms::base_content {
std::vector<Domain> list;
std::string filter; // Which filter was used to show list
int page, pages, page_size, count;
};



#endif

+ 9
- 0
controlpanel/default View File

@@ -0,0 +1,9 @@
# Configuration for init.d/trafficctrl. All entries remarked out below are the
# defaults.

# Configuration file for the TrafficCtrl server
#CONF=/etc/poorman-ids/sample.js
# Where "run" files are placed. This is the Debian+ default:
#RUN=/run
# This needs to match the "daemon.lock" entry in the $CONF file.
#PID=$RUN/poorman-ids/trafficctrl.pid

+ 108
- 0
controlpanel/init View File

@@ -0,0 +1,108 @@
#!/bin/sh
### BEGIN INIT INFO
# Provides: trafficctrl
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Control categories of network traffic
# Description: This service provides an HTTP or FastCGI service to
# provide a control panel for classifying web traffic.
# It does not take any action by itself.
#
# NOTE: you need to use trafficmon to setup the DB first.
### END INIT INFO

NAME="trafficctrl"
DAEMON="/usr/sbin/$NAME"
RUN=/run
CONF=/etc/poorman-ids/sample.js
PID=""

# Pull in config
if [ -r "/etc/default/$NAME" ]; then
. /etc/default/$NAME
fi



### Setup control variables ###

# This is where we put PID files and the pipe
RUN="$RUN/poorman-ids"
# NOTE: this needs to match what $CONF says
[ -n "$PID" ] || PID="$RUN/$NAME.pid"
mkdir -p "$RUN"



### ACTIONS ###

# The main service command
CTRL() {
start-stop-daemon --pidfile "$PID" --exec "$DAEMON" "$@"
}



do_start() {
echo -n "Starting Traffic Control: "
if CTRL --start --oknodo -- -c "$CONF"; then
echo "OK"
else
echo "FAIL"
exit 1
fi
}



do_stop() {
echo -n "Stoping Traffic Control: "
if CTRL --stop --remove-pidfile; then
echo "OK"
else
echo "FAIL"
exit 1
fi
}



do_status() {
echo -n "Traffic Control is: "
if CTRL --status; then
echo "Up"
else
echo "Down"
exit 1
fi
}



### Main()

case "$1" in
start)
do_start
;;
stop)
do_stop
;;
restart)
do_status && do_stop
do_start
;;
status)
do_status
;;
*)
echo "$0 {start | stop | restart | status}"
;;
esac

+ 104
- 0
controlpanel/mainskin.tmpl View File

@@ -0,0 +1,104 @@
<% c++ // Why isn't this done automatically?!?! %>
<% c++ #include <cppcms/view.h> %>
<% c++ #include "data.h" %>
<% skin mainskin %>
<% view domain_list uses ::DomainList %>



<% template title() %><%= filter %> Domain List<% end %>



<% template menu() %>
<b>Filters: </b>
<a href="/">To Be Decided</a> |
<a href="/accepted">Accepted</a> |
<a href="/blocked">Blocks</a>
<% end template %>



<% template pager(int no, int ct) %>
<% if (ct>1) %><div class="pager">
<% if (no>1) %>
<a href="?pg=1">|<</a>
<a href="?pg=<% c++ out() << no-1; %>"><<</a>
<% end if %>
<%= no %> of <%= ct %>
<% if (no<ct) %>
<a href="?pg=<% c++ out() << no+1; %>">>></a>
<a href="?pg=<%= ct %>">>|</a>
<% end if %>
</div><% end if %>
<% end template %>



<% template render() %>
<html><head>
<title><% include title() %></title>
<style>
h1 { text-align: center; }
table {
margin-top: 1em;
border-bottom: 2px solid black;
}
table th {
border-top: 2px solid black;
border-bottom: 2px solid black;
}
div.menu {
border-top: 2px solid black;
border-bottom: 2px solid black;
padding: 0.25em;
}
div.pager {
font-weight: bold;
}
</style>
</head><body>
<div class="menu"><% include menu() %></div>
<h1><% include title() %></h1>
<div id="content">
<% foreach domain rowid r from 1 in list %>
<% include pager(page, pages) %>
<form method="POST">
Whole Domain: <input name="domain" size=50>
<table>
<tr><th><select name="op" value="0">
<option value="0">Undecided</option>
<option value="1">Accept</option>
<option value="2">Block</option>
</select>
<input type="submit" value=">">
</th>
<th>Domain</th>
<th>When Decided</th>
</tr>
<% item %>
<tr>
<td>
<span style="float: left"><% c++ out() << r+content.page_size*(content.page-1); %>.</span>
<center><input type=checkbox name="id" value="<%= domain.name %>"></center>
</td>
<td><%= domain.name %></td>
<td><%= domain.decided %></td>
</tr>
<% end item %>
<tr><td colspan="3">of <%= count %></td></tr>
</table></form>
<% include pager(page, pages) %>
<% empty %>
<h2>Nothing available</h2>
<% end %>
</div>
</body></html>
<% end template %>



<% end view %>
<% end skin %>

+ 65
- 0
controlpanel/sample.js View File

@@ -0,0 +1,65 @@
// This configuration file is used by C++CMS and trafficctrl. Most of the
// format is described on the C++CMS website at:
// http://cppcms.com/wikipp/en/page/cppcms_1x_config
// Its syntax is JSON with the addition of C++ single line remarks, as seen
// here.

// NOTE: At this time no form of security is provided in this application. Its
// expected it will be run through a proxy that will provide httpS & an
// authentication mechanism.



{
// The database to use. See the C++DB connection string reference at:
// http://cppcms.com/sql/cppdb/connstr.html
// This is a MySQL example replace the {...} parts with your site's settings.
"trafficctrl": {
"db": "mysql:user={username};password={password};database={db_name};@pool_size=10;@opt_reconnect=1"
},



// These are C++CMS engine settings. See here for details:
// http://cppcms.com/wikipp/en/page/cppcms_1x_config

// This is a stand alone HTTP server setup:
// "service": {
// "api" : "http", // fastcgi
// "ip": "0.0.0.0",
// "port" : 8080
// // or "socket": "path..."
// },
// "http" : { "script": "" },
// This is a FastCGI example, which could be used to provide HTTPS & passwords:
"service": {
"api" : "fastcgi",
// To serve on a TCP socket: "ip": "0.0.0.0", "port": 8080
// or a socket. preferred if lesser privlieged users have access
// to the server:
"socket": "/run/poorman-ids/traffcctl.fcgi
},

// In context of restricted access environment this is probably OK.
"security" : {
"display_error_message": true,
// This requires JFP patches applied to C++CMS
//"email_error_message": "SysOp@domain.net",
},

// This tells us to run in background, as a daemon and to send errors to
// SysLog. You can run this at the console by remarking out everything below.
// Well.. other than the final '}'
"daemon": {
"enable": true,
"lock": "/run/poorman-ids/trafficctrl.pid",
"user": "www-data"
},
"logging": {
"syslog": {
"enable": true,
"id": "trafficctrl"
}
}
}

+ 171
- 0
controlpanel/trafficctrl.cpp View File

@@ -0,0 +1,171 @@
//////////////////////////////////////////////////////////////////////
// Traffic Montor Control HTTP server
// Written by Jonathan A. Foster <jon@jfpossibilities.com>
// Started August 13th, 2021
// Copyright JF Possibilities, Inc. All rights reserved.
//
// Provide a control panel to manage which domains we want to watch,
// ignore anc block.
//////////////////////////////////////////////////////////////////////
#include <stdlib.h>
#include <string>
#include <stdexcept>
#include <iostream>

/// C++CMS/DB ///

#include <cppcms/service.h>
#include <cppcms/applications_pool.h>
#include <cppcms/mount_point.h>
#include <cppcms/url_dispatcher.h>
#include <cppcms/url_mapper.h>
#include <cppcms/http_request.h>
#include <cppcms/http_response.h>
#include <cppdb/frontend.h>

/// Our app ///

#include "../strutil.h"
#include "data.h"



//////////////////////////////////////////////////////////////////////
// C++CMS App to manage the state of all known domain names.
//////////////////////////////////////////////////////////////////////

const std::string filter_titles[] = { "Undecided", "Accepted", "Blocked" };
const std::string actions[] = { "Reset", "Accept", "Block" };

struct app: public cppcms::application {
std::auto_ptr<cppdb::session> sql;
int items_per_page;


std::string numlist(const std::string &field) {
std::string r;
const cppcms::http::request::form_type &POST = request().post();
for(
cppcms::http::request::form_type::const_iterator it = POST.find(field);
it!=POST.end() && it->first==field;
it++
) {
if(it->second!="") {
if(r!="") r+=",";
r+="'"+sql->escape(it->second)+"'";
}
}
return r;
}
void list(Domain::STATUS sid) {
int i;
std::string s;
StringList list;
DomainList c;
// TODO: put this someplace else?
/// Form processing ///
if(request().request_method()=="POST") {
std::string op = request().post("op");
// nothing to do without a valid "op"
if(op=="0" || op=="1" || op=="2") {
// eliminate NOP busywork
if((s=numlist("id"))!="" && sid!=atoi(op.c_str())) {
*sql << "UPDATE dns SET status="+op+" WHERE name IN ("+s+")"
<< cppdb::exec;
}
if((s=request().post("domain"))!="") {
*sql << "UPDATE dns SET status=? WHERE name=? OR name LIKE ?"
<< op << s << ("%."+s)
<< cppdb::exec;
}
}
}
/// Auto add unknown domains to DNS list ///

cppdb::result r = *sql <<
"SELECT c.them_name "
"FROM connections c LEFT OUTER JOIN dns ON c.them_name=dns.name "
"WHERE c.them_name<>'' AND dns.name IS NULL "
"GROUP BY c.them_name";
while(r.next()) {
r >> s;
list.push_back(s);
}
if(!list.empty()) {
cppdb::statement q = *sql << "INSERT INTO dns (name) VALUES (?)";
for(StringList::iterator i=list.begin(); i!=list.end(); i++) {
q.reset();
q << *i << cppdb::exec;
}
}
/// Produce list of unknowns ///
c.filter = filter_titles[sid];
s = request().get("pg");
if(s=="")
c.page = 1;
else
c.page = atoi(s.c_str());
if(c.page < 1 || c.page > 999999) c.page = 1;
r = *sql << "SELECT name, decided, status FROM dns WHERE status=? "
"LIMIT "+str((c.page-1)*items_per_page)+","+str(items_per_page)
<< sid;
for(i = c.list.size(); r.next(); i++) {
c.list.resize(i+1);
r >> c.list[i].name >> c.list[i].decided >> c.list[i].status;
}
r = *sql << "SELECT count(*) FROM dns WHERE status=?" << sid << cppdb::row;
r >> c.count;
c.page_size = items_per_page;
c.pages = (c.count+items_per_page-1)/items_per_page;
render("mainskin", "domain_list", c);
}
void undecided() { list(Domain::undecided); }
void accepted() { list(Domain::accepted ); }
void blocked() { list(Domain::blocked ); }
app(cppcms::service &s): cppcms::application(s), items_per_page(50) {
sql.reset(new cppdb::session());
sql->open(settings().get<std::string>("trafficctrl.db"));
dispatcher().assign("/blocked/?", &app::blocked, this);
dispatcher().assign("/accepted/?", &app::accepted, this);
dispatcher().assign("/?", &app::undecided, this);
}
};



//////////////////////////////////////////////////////////////////////
// main() - Launch C++CMS with my test app on "/"
//////////////////////////////////////////////////////////////////////

int main(int argc, char **args) {
cppcms::service srv(argc, args);
srv.applications_pool().mount(
cppcms::create_pool<app>(),
cppcms::mount_point("")
);
srv.run();

return 0;
}


+ 87
- 0
poorman-ids.dpak View File

@@ -0,0 +1,87 @@
# This is DPAK Debian packaging source
# DPAK is an tool of JF Possibilities, Inc. Written by ChipMaster.

Source: poorman-ids
Priority: extra
Section: unknown
Maintainer: Jon Foster <jon@jfpossibilities.com>
Homepage: https://yeolpishack.net/repos/ChipMaster/Poor-Mans-IDS/wiki
Description: Poor Man's IDS
A simple tool to alert you to unknown traffic on your network.
Copyright: .
(c) 2021 JF Possibilities, Inc. All rights reserved.
Origin: JFP
Packaged-For: JF Possibilities, Inc.
changelog:
(0.4-2j) unstable; urgency=low
.
** This is an alpha release **
.
* Expanded sample trafficctrl configuration.
.
-- Jon Foster <jon@jfpossibilities.com> Mon, 13 Sep 2021 11:52:58 -0700
.
(0.4-1j) unstable; urgency=low
.
** This is an alpha release **
.
* Added standard setup stuff like "init.d" scripts, syslog conf, and
improved sample configuration files.
.
-- Jon Foster <jon@jfpossibilities.com> Thu, 08 Sep 2021 13:58:40 -0700
.
(0.3-1j) unstable; urgency=low
.
** This is an alpha release **
.
* Initial Debianization and release of tools:
- Log catching daemon
- fCGI / HTTP prioritization tool
- report tool
.
-- Jon Foster <jon@jfpossibilities.com> Thu, 02 Sep 2021 10:58:43 -0700
.
Build: sh
make trafficmon/trafficmon trafficmon/badtrafficrpt
cd controlpanel
make
Clean: sh
make distclean

Package: poorman-ids
Architecture: any
# I think libssl is required by cppcms. libmysqlclient18 is probably cppdb
Depends: libc6, libstdc++6, cppdb (>= 0.3.1-4), cppcms, libssl1.0.0,
libmysqlclient18
Recommends: libmysqlclient18
Description: .
Install: sh
dpak install -sbin trafficmon/trafficmon trafficmon/badtrafficrpt
dpak install -sbin controlpanel/trafficctrl
dpak strip
dpak install -conf -subdir poorman-ids sample.conf controlpanel/sample.js
mkdir -p "$DPAK_ROOT/etc/default"
cp trafficmon/default "$DPAK_ROOT/etc/default/trafficmon"
cp controlpanel/default "$DPAK_ROOT/etc/default/trafficctrl"
mkdir -p "$DPAK_ROOT/etc/init.d"
cp trafficmon/init "$DPAK_ROOT/etc/init.d/trafficmon"
cp controlpanel/init "$DPAK_ROOT/etc/init.d/trafficctrl"
mkdir -p "$DPAK_ROOT/etc/syslog.d"
cp trafficmon/syslog "$DPAK_ROOT/etc/syslog.d/trafficmon"
Finalize: sh
# Clean up permissions in the packaged files & folders.
chmod -R g-s "$DPAK_ROOT"
chmod 700 "$DPAK_ROOT/etc/poorman-ids"
chmod 600 "$DPAK_ROOT/etc/poorman-ids/"*
chmod 644 "$DPAK_ROOT/etc/default/"*
chmod 755 "$DPAK_ROOT/etc/init.d/"*
PostInst: sh
update-rc.d trafficmon defaults
update-rc.d trafficctrl defaults
PreRm: sh
# Shut off services so they are RAM resident after install
service trafficmon stop || true
service trafficctrl stop || true
PostRm: sh
update-rc.d trafficmon remove
update-rc.d trafficctrl remove

+ 32
- 0
sample.conf View File

@@ -0,0 +1,32 @@
# This is a sample config file. A single file can be used as repository of
# information for all of the tools in this package, other than "trafficctrl",
# since its a C++CMS app and C++CMS needs JSON to setup its features.

# List of address prefixes that represent our networks (us)
[us]
192.168.1.



# Traffic monitor (trafficmon) settings
[Traffic Mon]
db user =
db password =
db name =
# db host =



# Sample List of connections to ignore. So far this is only used by iptraffic,
# the CLI log processor. Trafficmon and the badtrafficrpt use data in the DB.
[ignores]

# muttering to self
127.0.0.1 0 127.0.0.1 0 * ICMP 0
127.0.0.1 0 127.0.0.1 0 * UDP 0
127.0.0.1 0 127.0.0.1 0 * TCP 0
127.0.0.1 0 127.0.0.1 0 * ICMP 1
127.0.0.1 0 127.0.0.1 0 * UDP 1
127.0.0.1 0 127.0.0.1 0 * TCP 1
::1 53 ::1 0 * UDP 1
::1 53 ::1 0 * TCP 1

+ 12
- 0
strutil.cpp View File

@@ -37,6 +37,18 @@ std::string str(long long n) {



// TODO: more optimal way to handle?
std::string qesc(const std::string &s) {
std::string r="'";
int i;
for(i=0; i<s.size(); i++) {
if(s[i]=='\'') r+= "''"; else r+=s[i];
}
r+='\'';
return r;
}


//////////////////////////////////////////////////////////////////////
// Splits
//////////////////////////////////////////////////////////////////////


+ 2
- 0
strutil.h View File

@@ -32,6 +32,8 @@ typedef std::map<std::string,std::string> NameVal;

std::string trim(const std::string &s);
std::string str(long long n);
// "query" escape: ' -> ''
std::string qesc(const std::string &s);





+ 250
- 0
trafficmon/badtrafficrpt.cpp View File

@@ -0,0 +1,250 @@
//////////////////////////////////////////////////////////////////////
// Basic Access Log report
// Written by Jonathan A. Foster <jon@jfpossibilities.com>
// Started August 20th, 2021
// Copyright JF Possibilities, Inc. All rights reserved.
//
// To start with this gather data for a specific access period and
// create a report of domain names and addresses that accessed to and
// from the net. All "accepted" and "blocked" accesses will be
// ignored. For the moment we're expecting blocked traffic is actually
// blocked.
//
// The report will contain three columns:
// 1. domain name or address if a domain is not known.
// 2. list of ports that were connected to.
// 3. count of total connections
//////////////////////////////////////////////////////////////////////
#include <string>
#include <map>
#include <iostream>
#include <stdio.h>
#include <libgen.h>
#include <cppdb/frontend.h>

#include "../strutil.h"
#include "../cli.h"
#include "../miniini.h"
using namespace std;



//////////////////////////////////////////////////////////////////////
// A "line" of a report - a single domain/address
//////////////////////////////////////////////////////////////////////

struct ReportLine {
int count;
map<string,int> ports;

// initialize
ReportLine(): count(0) {}
// add an item
void add(const string &port, int _count) {
ports[port]+=_count;
count+=_count;
}
// ports -> string
string port_list() const {
string r;
map<string,int>::const_iterator it = ports.begin();
if(it==ports.end()) return r;
r = it->first;
for(it++; it!=ports.end(); it++) r+=","+it->first;
return r;
}
};



//////////////////////////////////////////////////////////////////////
// A full reports worth of data - domain names / addresses and their
// associated stats.
//////////////////////////////////////////////////////////////////////

struct ReportData: map<string,ReportLine> {

// Add an item to the report
void add(const string &address, const string &port, int count) {
(*this)[address].add(port, count);
}

// Render report in an ASCII table
string ascii() const {
int widths[3] = {0,0,0}; // max column widths: 0: DNS, 1: ports, 3: counts
int x;
char l[256]; l[255]=0;
string s, r, bk;
ReportData::const_iterator it;

for(it = begin(); it!=end(); it++) {
if(it->first.size()>widths[0]) widths[0] = it->first.size();
s = it->second.port_list();
if(s.size()>widths[1]) widths[1]=s.size();
if(it->second.count>widths[2]) widths[2] = it->second.count;
}
// Now conver count max to cols
for(x=0; widths[2]; x++) widths[2] /= 10;
widths[2] = x ? x : 1;
// min col widths for titles
if(widths[0]<6) widths[0]=6;
if(widths[1]<7) widths[1]=7;
if(widths[2]<2) widths[2]=2;

// render report
bk = "+"+string(widths[0]+2, '-')+
"+"+string(widths[1]+2, '-')+
"+"+string(widths[2]+2, '-')+
"+\n";
s = "| %-"+str(widths[0])+"s | %-"+str(widths[1])+"s | %"+str(widths[2])+"d |\n";
r = bk;
snprintf(l, sizeof(l)-1,
("| %-"+str(widths[0])+"s | %-"+str(widths[1])+"s | %-"+str(widths[2])+"s |\n").c_str(),
(const char *)"Remote",
(const char *)"Port(s)",
(const char *)"Ct"
);
r+= l;
r+= bk;
for(it = begin(); it!=end(); it++) {
snprintf(l, sizeof(l)-1, s.c_str(),
it->first.c_str(),
it->second.port_list().c_str(),
it->second.count
);
r+=l;
}
r+=bk;
return r;
}
};
inline ostream &operator<<(ostream &out, const ReportData &r){
return out << r.ascii();
}
namespace cppdb {
result &operator>>(result &qry, ::ReportData &rpt) {
string name, addr, port;
int ct;
qry >> name >> addr >> port >> ct;
if(name=="") name=addr;
rpt.add(name, port, ct);
}
}


//////////////////////////////////////////////////////////////////////
// Our config file.
//
// This is designed so that all parts can use the same config. Tools
// ignore the parts they aren't interested in.
//////////////////////////////////////////////////////////////////////

/* TODO: refactor this as a base class... */
struct ReportConf: public MiniINI {
MiniINIvars traffic_mon; // This app's config variables

ReportConf() { groups["Traffic Mon"] = &traffic_mon; }
};



//////////////////////////////////////////////////////////////////////
// Connection Report Generator Application Class
//////////////////////////////////////////////////////////////////////

struct appConnectionReport: cBaseApp {
ReportConf config;
ReportData rpt;
cppdb::session db;
string start_stamp;
string end_stamp;
int cli_mode; // which non-switch are we processing


appConnectionReport(): cli_mode(0) {}



int help() {
cerr << " FORMAT: " << basename(command_args[0]) << " -c {config} {start} {end}\n"
<< '\n'
<< "The config file must have a [Traffic Mon] section with the database\n"
<< "credentials in it. 'start' and 'stop' are the SQL timestamps for\n"
<< "report time span." << endl;
}
virtual unsigned do_switch(const char *arg) {
if(*arg=='c' && !arg[1]) return 1;
throw CLIerror("Invalid switch "+string(arg));
}



virtual void do_switch_arg(const char *sw, const std::string &val) {
// The only way we get here is with -c
config.load(val);
// TODO: config validity checks
}



virtual void do_arg(const char *arg) {
switch(cli_mode++) {
case 0: start_stamp = arg; return;
case 1: end_stamp = arg; return;
default: throw CLIerror("Invalid arguments");
}
}

int main() {
cppdb::result qry;

/// SETUP & VALIDATE CLI ///

try {
cBaseApp::main(); // Parse CLI args
if(!config.traffic_mon.vals.size()) throw CLIerror(
"You need to load a config file with a [Traffic Mon] section"
);
if(cli_mode!=2) throw CLIerror("Invlaid arguments");
} catch(const CLIerror &e) {
cerr << e.what() << "\n\n";
return help();
}
db.open("mysql:user="+qesc(config.traffic_mon.get("db user"))+
";password="+qesc(config.traffic_mon.get("db password"))+
";host="+qesc(config.traffic_mon.get("db host"))+
";database="+qesc(config.traffic_mon.get("db name"))+
";@opt_reconnect=1");

/// Query & load data ///

qry = db <<
"SELECT c.them_name, c.them, c.them_port, count(*) "
"FROM connections c LEFT OUTER JOIN dns d ON c.them_name=d.name "
"WHERE c.inbound=0 AND (d.status IS NULL or d.status=0) "
"AND tstamp>=? AND tstamp<=? "
"GROUP BY c.them_name, c.them, c.them_port"
<< start_stamp << (end_stamp + " 23:59:59"); // include to the end of the day
while(qry.next()) qry >> rpt;

/// spit out the report ///

cout << "Web access report for " << start_stamp << " - " << end_stamp << "\n\n"
<< rpt;
return 0;
}

};



//////////////////////////////////////////////////////////////////////
// Lets run the report and dump it out
//////////////////////////////////////////////////////////////////////

MAIN(appConnectionReport)

+ 9
- 0
trafficmon/default View File

@@ -0,0 +1,9 @@
# Configuration for init.d/trafficmon. All entries remarked out below are the
# defaults.

# Configuration file for the TrafficMon server
#CONF=/etc/poorman-ids/sample.conf
# Where "run" files are placed. This is the Debian+ default:
#RUN=/run
# This needs to match the pipe speicified in the syslog.d/trafficmon file.
#SOCK=$RUN/poorman-ids/trafficmon.sock

+ 113
- 0
trafficmon/init View File

@@ -0,0 +1,113 @@
#!/bin/sh
### BEGIN INIT INFO
# Provides: trafficmon
# Required-Start: $remote_fs
# Required-Stop: $remote_fs
# X-Start-Before: $syslog
# X-Stop-After: $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Monitor and record net traffic
# Description: This service listens to a pipe for "iptables LOG" and
# dnsmasq DNS query messages. It then records information
# about internet access into a MySQL DB. Typically these
# messages are provided by syslogd.
#
# NOTE: if this is started after syslogd the pipe might not
# be available and syslogd will simply not try to write to
# until a restart.
### END INIT INFO

NAME="trafficmon"
DAEMON="/usr/sbin/$NAME"
RUN=/run
CONF=/etc/poorman-ids/sample.conf
SOCK=""

# Pull in config
if [ -r "/etc/default/$NAME" ]; then
. /etc/default/$NAME
fi



### Setup control variables ###

# This is where we put PID files and the pipe
RUN="$RUN/poorman-ids"
PID="$RUN/$NAME.pid"
mkdir -p "$RUN"
[ -n "$SOCK" ] || SOCK="$RUN/$NAME.sock"



### ACTIONS ###

# The main service command
CTRL() {
start-stop-daemon --pidfile "$PID" --exec "$DAEMON" "$@"
}



do_start() {
echo -n "Starting Traffic Monitor: "
if CTRL --start --oknodo -- -c "$CONF" -b -i "$PID" -b -p "$SOCK"; then
echo "OK"
else
echo "FAIL"
exit 1
fi
}



do_stop() {
echo -n "Stoping Traffic Monitor: "
if CTRL --stop --remove-pidfile; then
echo "OK"
else
echo "FAIL"
exit 1
fi
}



do_status() {
echo -n "Traffic Monitor is: "
if CTRL --status; then
echo "Up"
else
echo "Down"
exit 1
fi
}



### Main()

case "$1" in
start)
do_start
;;
stop)
do_stop
;;
restart)
do_status && do_stop
do_start
;;
status)
do_status
;;
*)
echo "$0 {start | stop | restart | status}"
;;
esac

+ 1
- 0
trafficmon/syslog View File

@@ -0,0 +1 @@
kern,daemon.* |/run/poorman-ids/trafficmon.sock

+ 335
- 0
trafficmon/trafficmon.cpp View File

@@ -0,0 +1,335 @@
//////////////////////////////////////////////////////////////////////
// IP traffic monitor
// Written by Jonathan A. Foster <ChipMaster@YeOlPiShack.net>
// Started August 11th, 2021
//
// The idea is to analyze iptables LOG entries in combination with
// DNSmasq's query log entries and combine them to list the hosts
// that were accessed. This will be read in real time from SysLogD
// using the "pipe" feature. Access records will be logged to MySQL
// for reporting and record keeping.
//////////////////////////////////////////////////////////////////////
// TODO: catch signals and attempt removal of PID file during shutdown.
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <syslog.h>
#include <unistd.h>
#include <string>
#include <iostream>
#include <fstream>
#include <stdexcept>
#include <vector>
#include <map>
#include <cppdb/frontend.h>

#include "../cli.h"
#include "../data.h"
#include "../miniini.h"
#include "../config.h"
using namespace std;



//////////////////////////////////////////////////////////////////////
// Monitor Config
//////////////////////////////////////////////////////////////////////

struct MonitorConf: public MiniINI {
MiniINIvars traffic_mon; // This app's config variables
INIusList us;

MonitorConf() { groups["Traffic Mon"] = &traffic_mon; groups["us"] = &us; }
};



//////////////////////////////////////////////////////////////////////
// Application class to store data passed in through a pipe or
// file(s).
//////////////////////////////////////////////////////////////////////
//#define DEBUG

struct TrafficMon: public cBaseApp {
MonitorConf config;
LogAnalyzer analyze;
istream *log;
cppdb::session db;
LiveBug bug;
bool setup_db;
bool piping; // are we sucking on a pipe?
bool background_me; // daemonize?
string pid_file; // Name of file to write PID in
int inp_ct; // How many inputs were on the CLI
bool running; // Whether or not its time to run logs
long long line_no; // current line # in the active input



TrafficMon():
log( 0),
setup_db( false),
piping( false),
background_me(false),
inp_ct( 0),
running( false),
line_no( 0)
{
analyze.us = &(config.us.vals);
}



void help() {
cerr <<
"\n"
"trafficmon -c {config file} [-i] [-d] [-b] [-p {pipe name} | [{log name} ...]]\n"
"\n"
"Arguments:\n"
" -c Load configuration from file (required)\n"
" -i PID file to write (optional, use with -b)\n"
" -d Create data tables (optional, use once)\n"
" -b Run in background - daemonize (optional, use -i too)\n"
" -p read from pipe creating it if needed (optional, good with -b)\n"
"\n"
"Other arguments are log files to be processed. This can be used to bulk\n"
"load data prior to going live with an always on daemon or for catching\n"
"up if the daemon stopped for some reason.";
ExitCode = 1;
}



unsigned do_switch(const char *sw) {
if(sw[1]==0) {
switch(*sw) {
case 'b': background_me = true; return 0;
case 'c': return 1;
case 'd': setup_db = true; return 0;
case 'i': return 1;
case 'p': piping = true; return !running; // second pass treats it as a file
}
}
throw CLIerror("Unrecognized Switch");
}



void do_switch_arg(const char *sw, const string &val) {
// If "running" is set then we've already done this. So skip it!
if(running) return;

switch(*sw) {

/// Load config file ///

case 'c':
// This is only called with "c". See above
config.load(val);
if(!config.us.vals.size()) throw CLIerror(
"The configuration files MUST contain an [us] section with "
"appropriate values"
);
if(!config.traffic_mon.vals.size()) throw CLIerror(
"The configuration files MUST contain an [Traffic Mon] section with "
"appropriate values"
);

/// configure resources from config file ///

// TODO: pre-validate some of these?
db.open("mysql:user="+qesc(config.traffic_mon.get("db user"))+
";password="+qesc(config.traffic_mon.get("db password"))+
";host="+qesc(config.traffic_mon.get("db host"))+
";database="+qesc(config.traffic_mon.get("db name"))+
";@opt_reconnect=1");
return;
/// PID file to write ID in if we're daemonizing ///
case 'i':
pid_file = val;
return;

/// Make pipe if requested ///
case 'p':
// This can't wait for "running" since we want it to fail in foreground
// Make read/write by process owner (root) only
if(mkfifo(val.c_str(), 0600)<0 && errno!=17 /*already exists*/) {
ExitCode = 2;
throw runtime_error("Making pipe raised error "+str(errno));
}
inp_ct++;
}
}



void do_arg(const char *fname) {
// if not "running" we don't want to do this yet. Just count inputs.
if(!running) {
inp_ct++;
return;
}

// the only thing we expect on the CLI is a log/pipe name
if(log==&cin) log = 0; // won't happen right now
if(log)
((ifstream*)log)->close();
else
log = new ifstream;
if(!background_me) cerr << fname << ":\n";
restart:
((ifstream*)log)->open(fname);
ExitCode = run();
if(piping) {
// If a process closes the write end of the pipe, like during log
// rotation, we receive an EOF, end up here, and have to re-open the pipe
// to be able to receive more on it.
((ifstream*)log)->close();
sleep(1); // This is just to make sure we don't hog CPU in a loop
goto restart;
}
}



// This is the actual data process
// probably should daemonize around here.
int run() {
string l;
cppdb::statement db_add = db << // prepare insert statement
"INSERT INTO connections "
"(us, us_port, them, them_port, them_name, protocol, inbound) "
"VALUES (?,?,?,?,?,?,?)";

/// parse log file ///

line_no=0;
while(getline(*log, l)) {
line_no++;
if(!background_me)
cerr << bug << ' ' << line_no << '\r' << flush;

/// process connections ///

if(analyze.line(l)) {
// TODO: time stamp handling?
// insert record
db_add.reset();
db_add << analyze.conn.us << analyze.conn.us_port << analyze.conn.them
<< analyze.conn.them_port << analyze.conn.name
<< analyze.conn.protocol << analyze.conn.in
<< cppdb::exec;
}
}
// In a real pipe situation this should never get reached.
if(!background_me) {
cerr << "\nLines: " << line_no
<< "\nTotal rDNS: " << analyze.rdns.size() << '\n' << endl;
}
return 0;
}



void create_tables() {

/// Connection recrods ///

db << "CREATE TABLE connections ("
"id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,"
"tstamp TIMESTAMP NOT NULL," // default is NOW()
"us CHAR(80) NOT NULL DEFAULT '',"
"us_port INT NOT NULL DEFAULT 0,"
"them CHAR(80) NOT NULL DEFAULT '',"
"them_port INT NOT NULL DEFAULT 0,"
"them_name CHAR(128) NOT NULL DEFAULT '',"
"protocol CHAR(12) NOT NULL DEFAULT '',"
"inbound TINYINT(1) NOT NULL DEFAULT 0,"
"INDEX tstamp(tstamp, them_name)"
") Engine=MyISAM"
<< cppdb::exec;
// NOTE: MyISAM is thousands of times faster than InnoDB for this sort
// of work load. Mostly continuous writes with occasional reads.
// But will it work for all I'm going to do? AriaDB is slower but
// significantly faster than InnoDB.

/// Status of domain names ///

db << "CREATE TABLE dns ("
"name CHAR(128) NOT NULL PRIMARY KEY,"
"decided TIMESTAMP NOT NULL,"
"status TINYINT(1) NOT NULL DEFAULT 0"
") Engine=MyISAM"
<< cppdb::exec;
}



int main() {
try {
cBaseApp::main();
if(!config.us.vals.size())
// Since -c requires something in [us] we'll only get here if a config
// wasn't loaded.
throw CLIerror("You must specify a config file to load");
if(piping && inp_ct!=1) throw CLIerror(
"Pipe requires one and only one file name to read from."
);
if(setup_db) create_tables();

// if requested attempt daemonizing.
if(background_me) {
if(inp_ct==0) throw CLIerror(
"Backgrounding requires the specification of an input source. "
"STDIN is unavailable when we move to background."
);

if(daemon(0,0)<0) throw CLIerror("Failed to switch to background");
// If we get here we're in the background. No STDIN, STDOUT, STDERR
if(pid_file!="") { // pid_file requested
ofstream pf(pid_file.c_str());
pf << getpid();
}
}
} catch(const CLIerror &e) {
cerr << "ERROR: " << e.what() << "\n";
help();
return ExitCode ? ExitCode : 1; // JIC someone sets a different exit code
}

try {
// CLI processed now lets analyze data.
running = true;
cBaseApp::main(); // re-run CLI args for inputs
if(!log) {
// no inputs spec'd on CLI assume stdin (we're not a daemon)
log = &cin;
ExitCode = run();
}
} catch(const exception &e) {
if(ExitCode==0) ExitCode = 2;
if(background_me) {
openlog("trafficmon", LOG_CONS | LOG_PID, LOG_DAEMON);
syslog(LOG_ALERT, "%s", e.what());
closelog();
} else
cerr << "ERROR: " << e.what() << "\n";
}
return ExitCode;
}



};



//////////////////////////////////////////////////////////////////////
// Run it
//////////////////////////////////////////////////////////////////////

MAIN(TrafficMon)

Loading…
Cancel
Save