From cd7d62ca0534c3d3d5bee3224970c43b60a73f96 Mon Sep 17 00:00:00 2001 From: Jon Foster Date: Mon, 13 Sep 2021 11:35:04 -0700 Subject: [PATCH] Begin realtime tracking * Program to run as daemon and listen to SysLogD on a pipe. * C++CMS web app to show accumulated domain names and set their status * Sample configs to be used to set things up --- .gitignore | 11 ++ Makefile | 23 ++- config.cpp | 2 +- config.h | 2 +- controlpanel/Makefile | 21 +++ controlpanel/data.h | 36 +++++ controlpanel/default | 9 ++ controlpanel/init | 108 ++++++++++++++ controlpanel/mainskin.tmpl | 104 ++++++++++++++ controlpanel/sample.js | 65 +++++++++ controlpanel/trafficctrl.cpp | 171 ++++++++++++++++++++++ poorman-ids.dpak | 87 +++++++++++ sample.conf | 32 +++++ strutil.cpp | 12 ++ strutil.h | 2 + trafficmon/badtrafficrpt.cpp | 250 ++++++++++++++++++++++++++++++++ trafficmon/default | 9 ++ trafficmon/init | 113 +++++++++++++++ trafficmon/syslog | 1 + trafficmon/trafficmon.cpp | 335 +++++++++++++++++++++++++++++++++++++++++++ 20 files changed, 1385 insertions(+), 8 deletions(-) create mode 100644 controlpanel/Makefile create mode 100644 controlpanel/data.h create mode 100644 controlpanel/default create mode 100755 controlpanel/init create mode 100644 controlpanel/mainskin.tmpl create mode 100644 controlpanel/sample.js create mode 100644 controlpanel/trafficctrl.cpp create mode 100644 poorman-ids.dpak create mode 100644 sample.conf create mode 100644 trafficmon/badtrafficrpt.cpp create mode 100644 trafficmon/default create mode 100755 trafficmon/init create mode 100644 trafficmon/syslog create mode 100644 trafficmon/trafficmon.cpp diff --git a/.gitignore b/.gitignore index 911aeb4..c6d83d7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,14 @@ /iptraffic /log /README.html +/trafficmon/badtrafficrpt +/trafficmon/trafficmon +/controlpanel/trafficctrl + +# I use these as output of C++ generators. Don't store them: +*.cxx + +# Debian (DPAK) packaging artifacts +*.deb +/tmp-dpak-poorman-ids/ + diff --git a/Makefile b/Makefile index 3f2f28d..bb1d6af 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,21 @@ O=-s + +### Program Targets ### + iptraffic: iptraffic.cpp strutil.o data.o config.o cli.o miniini.o g++ $O -o $@ $@.cpp strutil.o data.o config.o cli.o miniini.o +trafficmon/trafficmon: trafficmon/trafficmon.cpp strutil.o data.o config.o cli.o miniini.o + g++ $O -o $@ $@.cpp strutil.o data.o config.o cli.o miniini.o -lcppdb + +trafficmon/badtrafficrpt: trafficmon/badtrafficrpt.cpp cli.o miniini.o strutil.o + g++ $O -o $@ $@.cpp strutil.o cli.o miniini.o -lcppdb + + + +### Libs ### + cli.o: cli.cpp cli.h g++ $O -c -o $@ cli.cpp @@ -20,14 +33,12 @@ strutil.o: strutil.cpp strutil.h -.PHONY: run -run: iptraffic - ./iptraffic - - +### Source Maintenance ### .PHONY: clean distclean clean: rm *.o || true distclean: clean - rm iptraffic || true + rm iptraffic trafficmon/trafficmon trafficmon/badtrafficrpt || true + rm *.deb || true + cd controlpanel && make distclean diff --git a/config.cpp b/config.cpp index 1e169f0..801381e 100644 --- a/config.cpp +++ b/config.cpp @@ -32,7 +32,7 @@ void INIconnList::add(const std::string &in) { if(in=="" || in[0]=='#') return; // remarks // TODO: we don't want to keep create+destroy-ing these? - TSV tsv(in); + TSV tsv(in); if(tsv.count!=7) throw // TODO: really need a line number! std::runtime_error("INIconnList::add: Incorrect column count in config file line"); diff --git a/config.h b/config.h index b36553a..d918464 100644 --- a/config.h +++ b/config.h @@ -52,7 +52,7 @@ struct INIconnList: public MiniINIgroup { struct Config: public MiniINI { INIusList us; INIconnList ignores; - Config(); + Config(); }; diff --git a/controlpanel/Makefile b/controlpanel/Makefile new file mode 100644 index 0000000..eefbb9e --- /dev/null +++ b/controlpanel/Makefile @@ -0,0 +1,21 @@ +# Optional compiler flags +#O=-std=c++11 + +trafficctrl: trafficctrl.cpp data.h ../strutil.o mainskin.o + g++ $O -o $@ $@.cpp mainskin.o ../strutil.o -lcppcms -lcppdb -lbooster + +../strutil.o: ../strutil.cpp ../strutil.h + cd .. && make strutil.o + +mainskin.cxx: mainskin.tmpl + cppcms_tmpl_cc -o $@ mainskin.tmpl +mainskin.o: mainskin.cxx data.h + g++ $O -c mainskin.cxx + + + +.PHONY: clean distclean +clean: + rm *.o *.cxx || true +distclean: clean + rm trafficctrl || true diff --git a/controlpanel/data.h b/controlpanel/data.h new file mode 100644 index 0000000..83b62fd --- /dev/null +++ b/controlpanel/data.h @@ -0,0 +1,36 @@ +////////////////////////////////////////////////////////////////////// +// Control Panel and domain control data +// Written by Jonathan A. Foster +// Started August 13th, 2021 +// Copyright JF Possibilities, Inc. All rights reserved. +// +// Data for dealing with white / black lists. +////////////////////////////////////////////////////////////////////// +#ifndef __CONTROL_DATA_H__ +#define __CONTROL_DATA_H__ +#include +#include +#include +//#include +//#include + + + +struct Domain { + enum STATUS {undecided, accepted, blocked}; + std::string name; // domain name + std::string decided; // since C++ doesn't have a date/time + int status; // how we should handle it +}; + + + +struct DomainList :public cppcms::base_content { + std::vector list; + std::string filter; // Which filter was used to show list + int page, pages, page_size, count; +}; + + + +#endif \ No newline at end of file diff --git a/controlpanel/default b/controlpanel/default new file mode 100644 index 0000000..b7338f5 --- /dev/null +++ b/controlpanel/default @@ -0,0 +1,9 @@ +# Configuration for init.d/trafficctrl. All entries remarked out below are the +# defaults. + +# Configuration file for the TrafficCtrl server +#CONF=/etc/poorman-ids/sample.js +# Where "run" files are placed. This is the Debian+ default: +#RUN=/run +# This needs to match the "daemon.lock" entry in the $CONF file. +#PID=$RUN/poorman-ids/trafficctrl.pid diff --git a/controlpanel/init b/controlpanel/init new file mode 100755 index 0000000..89d5c46 --- /dev/null +++ b/controlpanel/init @@ -0,0 +1,108 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: trafficctrl +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Control categories of network traffic +# Description: This service provides an HTTP or FastCGI service to +# provide a control panel for classifying web traffic. +# It does not take any action by itself. +# +# NOTE: you need to use trafficmon to setup the DB first. +### END INIT INFO + +NAME="trafficctrl" +DAEMON="/usr/sbin/$NAME" +RUN=/run +CONF=/etc/poorman-ids/sample.js +PID="" + +# Pull in config +if [ -r "/etc/default/$NAME" ]; then + . /etc/default/$NAME +fi + + + +### Setup control variables ### + +# This is where we put PID files and the pipe +RUN="$RUN/poorman-ids" +# NOTE: this needs to match what $CONF says +[ -n "$PID" ] || PID="$RUN/$NAME.pid" +mkdir -p "$RUN" + + + +### ACTIONS ### + +# The main service command +CTRL() { + start-stop-daemon --pidfile "$PID" --exec "$DAEMON" "$@" +} + + + +do_start() { + echo -n "Starting Traffic Control: " + if CTRL --start --oknodo -- -c "$CONF"; then + echo "OK" + else + echo "FAIL" + exit 1 + fi +} + + + +do_stop() { + echo -n "Stoping Traffic Control: " + if CTRL --stop --remove-pidfile; then + echo "OK" + else + echo "FAIL" + exit 1 + fi +} + + + +do_status() { + echo -n "Traffic Control is: " + if CTRL --status; then + echo "Up" + else + echo "Down" + exit 1 + fi +} + + + +### Main() + +case "$1" in + start) + do_start + ;; + + stop) + do_stop + ;; + + restart) + do_status && do_stop + do_start + ;; + + status) + do_status + ;; + + *) + echo "$0 {start | stop | restart | status}" + ;; + +esac \ No newline at end of file diff --git a/controlpanel/mainskin.tmpl b/controlpanel/mainskin.tmpl new file mode 100644 index 0000000..6d83e75 --- /dev/null +++ b/controlpanel/mainskin.tmpl @@ -0,0 +1,104 @@ +<% c++ // Why isn't this done automatically?!?! %> +<% c++ #include %> +<% c++ #include "data.h" %> +<% skin mainskin %> +<% view domain_list uses ::DomainList %> + + + +<% template title() %><%= filter %> Domain List<% end %> + + + +<% template menu() %> + Filters: + To Be Decided | + Accepted | + Blocks +<% end template %> + + + +<% template pager(int no, int ct) %> + <% if (ct>1) %>
+ <% if (no>1) %> + |< + << + <% end if %> + <%= no %> of <%= ct %> + <% if (no + >> + >| + <% end if %> +
<% end if %> +<% end template %> + + + +<% template render() %> + + <% include title() %> + + +
<% include menu() %>
+ +

<% include title() %>

+ +
+ <% foreach domain rowid r from 1 in list %> + <% include pager(page, pages) %> +
+ Whole Domain: + + + + + + <% item %> + + + + + + <% end item %> + +
+ + DomainWhen Decided
+ <% c++ out() << r+content.page_size*(content.page-1); %>. +
+ 		<%= domain.name %><%= domain.decided %>
of <%= count %>
+ <% include pager(page, pages) %> + <% empty %> +

Nothing available

+ <% end %> +
+ +<% end template %> + + + +<% end view %> +<% end skin %> diff --git a/controlpanel/sample.js b/controlpanel/sample.js new file mode 100644 index 0000000..a3f5f05 --- /dev/null +++ b/controlpanel/sample.js @@ -0,0 +1,65 @@ +// This configuration file is used by C++CMS and trafficctrl. Most of the +// format is described on the C++CMS website at: +// http://cppcms.com/wikipp/en/page/cppcms_1x_config +// Its syntax is JSON with the addition of C++ single line remarks, as seen +// here. + +// NOTE: At this time no form of security is provided in this application. Its +// expected it will be run through a proxy that will provide httpS & an +// authentication mechanism. + + + +{ +// The database to use. See the C++DB connection string reference at: +// http://cppcms.com/sql/cppdb/connstr.html +// This is a MySQL example replace the {...} parts with your site's settings. + "trafficctrl": { + "db": "mysql:user={username};password={password};database={db_name};@pool_size=10;@opt_reconnect=1" + }, + + + +// These are C++CMS engine settings. See here for details: +// http://cppcms.com/wikipp/en/page/cppcms_1x_config + +// This is a stand alone HTTP server setup: +// "service": { +// "api" : "http", // fastcgi +// "ip": "0.0.0.0", +// "port" : 8080 +// // or "socket": "path..." +// }, +// "http" : { "script": "" }, + +// This is a FastCGI example, which could be used to provide HTTPS & passwords: + "service": { + "api" : "fastcgi", + // To serve on a TCP socket: "ip": "0.0.0.0", "port": 8080 + // or a socket. preferred if lesser privlieged users have access + // to the server: + "socket": "/run/poorman-ids/traffcctl.fcgi + }, + + // In context of restricted access environment this is probably OK. + "security" : { + "display_error_message": true, + // This requires JFP patches applied to C++CMS + //"email_error_message": "SysOp@domain.net", + }, + +// This tells us to run in background, as a daemon and to send errors to +// SysLog. You can run this at the console by remarking out everything below. +// Well.. other than the final '}' + "daemon": { + "enable": true, + "lock": "/run/poorman-ids/trafficctrl.pid", + "user": "www-data" + }, + "logging": { + "syslog": { + "enable": true, + "id": "trafficctrl" + } + } +} diff --git a/controlpanel/trafficctrl.cpp b/controlpanel/trafficctrl.cpp new file mode 100644 index 0000000..ac726a5 --- /dev/null +++ b/controlpanel/trafficctrl.cpp @@ -0,0 +1,171 @@ +////////////////////////////////////////////////////////////////////// +// Traffic Montor Control HTTP server +// Written by Jonathan A. Foster +// Started August 13th, 2021 +// Copyright JF Possibilities, Inc. All rights reserved. +// +// Provide a control panel to manage which domains we want to watch, +// ignore anc block. +////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +/// C++CMS/DB /// + +#include +#include +#include +#include +#include +#include +#include +#include + +/// Our app /// + +#include "../strutil.h" +#include "data.h" + + + +////////////////////////////////////////////////////////////////////// +// C++CMS App to manage the state of all known domain names. +////////////////////////////////////////////////////////////////////// + +const std::string filter_titles[] = { "Undecided", "Accepted", "Blocked" }; +const std::string actions[] = { "Reset", "Accept", "Block" }; + +struct app: public cppcms::application { + std::auto_ptr sql; + int items_per_page; + + + + std::string numlist(const std::string &field) { + std::string r; + const cppcms::http::request::form_type &POST = request().post(); + for( + cppcms::http::request::form_type::const_iterator it = POST.find(field); + it!=POST.end() && it->first==field; + it++ + ) { + if(it->second!="") { + if(r!="") r+=","; + r+="'"+sql->escape(it->second)+"'"; + } + } + return r; + } + + + + void list(Domain::STATUS sid) { + int i; + std::string s; + StringList list; + DomainList c; + + // TODO: put this someplace else? + + /// Form processing /// + + if(request().request_method()=="POST") { + std::string op = request().post("op"); + // nothing to do without a valid "op" + if(op=="0" || op=="1" || op=="2") { + // eliminate NOP busywork + if((s=numlist("id"))!="" && sid!=atoi(op.c_str())) { + *sql << "UPDATE dns SET status="+op+" WHERE name IN ("+s+")" + << cppdb::exec; + } + if((s=request().post("domain"))!="") { + *sql << "UPDATE dns SET status=? WHERE name=? OR name LIKE ?" + << op << s << ("%."+s) + << cppdb::exec; + } + } + } + + /// Auto add unknown domains to DNS list /// + + cppdb::result r = *sql << + "SELECT c.them_name " + "FROM connections c LEFT OUTER JOIN dns ON c.them_name=dns.name " + "WHERE c.them_name<>'' AND dns.name IS NULL " + "GROUP BY c.them_name"; + while(r.next()) { + r >> s; + list.push_back(s); + } + + if(!list.empty()) { + cppdb::statement q = *sql << "INSERT INTO dns (name) VALUES (?)"; + for(StringList::iterator i=list.begin(); i!=list.end(); i++) { + q.reset(); + q << *i << cppdb::exec; + } + } + + /// Produce list of unknowns /// + + c.filter = filter_titles[sid]; + s = request().get("pg"); + if(s=="") + c.page = 1; + else + c.page = atoi(s.c_str()); + if(c.page < 1 || c.page > 999999) c.page = 1; + r = *sql << "SELECT name, decided, status FROM dns WHERE status=? " + "LIMIT "+str((c.page-1)*items_per_page)+","+str(items_per_page) + << sid; + for(i = c.list.size(); r.next(); i++) { + c.list.resize(i+1); + r >> c.list[i].name >> c.list[i].decided >> c.list[i].status; + } + r = *sql << "SELECT count(*) FROM dns WHERE status=?" << sid << cppdb::row; + r >> c.count; + c.page_size = items_per_page; + c.pages = (c.count+items_per_page-1)/items_per_page; + render("mainskin", "domain_list", c); + } + + + + void undecided() { list(Domain::undecided); } + void accepted() { list(Domain::accepted ); } + void blocked() { list(Domain::blocked ); } + + + + app(cppcms::service &s): cppcms::application(s), items_per_page(50) { + sql.reset(new cppdb::session()); + sql->open(settings().get("trafficctrl.db")); + + dispatcher().assign("/blocked/?", &app::blocked, this); + dispatcher().assign("/accepted/?", &app::accepted, this); + dispatcher().assign("/?", &app::undecided, this); + } + + + +}; + + + +////////////////////////////////////////////////////////////////////// +// main() - Launch C++CMS with my test app on "/" +////////////////////////////////////////////////////////////////////// + +int main(int argc, char **args) { + cppcms::service srv(argc, args); + srv.applications_pool().mount( + cppcms::create_pool(), + cppcms::mount_point("") + ); + srv.run(); + + return 0; +} + diff --git a/poorman-ids.dpak b/poorman-ids.dpak new file mode 100644 index 0000000..2038d2d --- /dev/null +++ b/poorman-ids.dpak @@ -0,0 +1,87 @@ +# This is DPAK Debian packaging source +# DPAK is an tool of JF Possibilities, Inc. Written by ChipMaster. + +Source: poorman-ids +Priority: extra +Section: unknown +Maintainer: Jon Foster +Homepage: https://yeolpishack.net/repos/ChipMaster/Poor-Mans-IDS/wiki +Description: Poor Man's IDS + A simple tool to alert you to unknown traffic on your network. +Copyright: . + (c) 2021 JF Possibilities, Inc. All rights reserved. +Origin: JFP +Packaged-For: JF Possibilities, Inc. +changelog: + (0.4-2j) unstable; urgency=low + . + ** This is an alpha release ** + . + * Expanded sample trafficctrl configuration. + . + -- Jon Foster Mon, 13 Sep 2021 11:52:58 -0700 + . + (0.4-1j) unstable; urgency=low + . + ** This is an alpha release ** + . + * Added standard setup stuff like "init.d" scripts, syslog conf, and + improved sample configuration files. + . + -- Jon Foster Thu, 08 Sep 2021 13:58:40 -0700 + . + (0.3-1j) unstable; urgency=low + . + ** This is an alpha release ** + . + * Initial Debianization and release of tools: + - Log catching daemon + - fCGI / HTTP prioritization tool + - report tool + . + -- Jon Foster Thu, 02 Sep 2021 10:58:43 -0700 + . +Build: sh + make trafficmon/trafficmon trafficmon/badtrafficrpt + cd controlpanel + make +Clean: sh + make distclean + +Package: poorman-ids +Architecture: any +# I think libssl is required by cppcms. libmysqlclient18 is probably cppdb +Depends: libc6, libstdc++6, cppdb (>= 0.3.1-4), cppcms, libssl1.0.0, + libmysqlclient18 +Recommends: libmysqlclient18 +Description: . +Install: sh + dpak install -sbin trafficmon/trafficmon trafficmon/badtrafficrpt + dpak install -sbin controlpanel/trafficctrl + dpak strip + dpak install -conf -subdir poorman-ids sample.conf controlpanel/sample.js + mkdir -p "$DPAK_ROOT/etc/default" + cp trafficmon/default "$DPAK_ROOT/etc/default/trafficmon" + cp controlpanel/default "$DPAK_ROOT/etc/default/trafficctrl" + mkdir -p "$DPAK_ROOT/etc/init.d" + cp trafficmon/init "$DPAK_ROOT/etc/init.d/trafficmon" + cp controlpanel/init "$DPAK_ROOT/etc/init.d/trafficctrl" + mkdir -p "$DPAK_ROOT/etc/syslog.d" + cp trafficmon/syslog "$DPAK_ROOT/etc/syslog.d/trafficmon" +Finalize: sh + # Clean up permissions in the packaged files & folders. + chmod -R g-s "$DPAK_ROOT" + chmod 700 "$DPAK_ROOT/etc/poorman-ids" + chmod 600 "$DPAK_ROOT/etc/poorman-ids/"* + chmod 644 "$DPAK_ROOT/etc/default/"* + chmod 755 "$DPAK_ROOT/etc/init.d/"* +PostInst: sh + update-rc.d trafficmon defaults + update-rc.d trafficctrl defaults +PreRm: sh + # Shut off services so they are RAM resident after install + service trafficmon stop || true + service trafficctrl stop || true +PostRm: sh + update-rc.d trafficmon remove + update-rc.d trafficctrl remove diff --git a/sample.conf b/sample.conf new file mode 100644 index 0000000..0dbb7c9 --- /dev/null +++ b/sample.conf @@ -0,0 +1,32 @@ +# This is a sample config file. A single file can be used as repository of +# information for all of the tools in this package, other than "trafficctrl", +# since its a C++CMS app and C++CMS needs JSON to setup its features. + +# List of address prefixes that represent our networks (us) +[us] +192.168.1. + + + +# Traffic monitor (trafficmon) settings +[Traffic Mon] +db user = +db password = +db name = +# db host = + + + +# Sample List of connections to ignore. So far this is only used by iptraffic, +# the CLI log processor. Trafficmon and the badtrafficrpt use data in the DB. +[ignores] + +# muttering to self +127.0.0.1 0 127.0.0.1 0 * ICMP 0 +127.0.0.1 0 127.0.0.1 0 * UDP 0 +127.0.0.1 0 127.0.0.1 0 * TCP 0 +127.0.0.1 0 127.0.0.1 0 * ICMP 1 +127.0.0.1 0 127.0.0.1 0 * UDP 1 +127.0.0.1 0 127.0.0.1 0 * TCP 1 +::1 53 ::1 0 * UDP 1 +::1 53 ::1 0 * TCP 1 diff --git a/strutil.cpp b/strutil.cpp index 051cf6a..d7f5f50 100644 --- a/strutil.cpp +++ b/strutil.cpp @@ -37,6 +37,18 @@ std::string str(long long n) { +// TODO: more optimal way to handle? +std::string qesc(const std::string &s) { + std::string r="'"; + int i; + for(i=0; i NameVal; std::string trim(const std::string &s); std::string str(long long n); +// "query" escape: ' -> '' +std::string qesc(const std::string &s); diff --git a/trafficmon/badtrafficrpt.cpp b/trafficmon/badtrafficrpt.cpp new file mode 100644 index 0000000..1b469dc --- /dev/null +++ b/trafficmon/badtrafficrpt.cpp @@ -0,0 +1,250 @@ +////////////////////////////////////////////////////////////////////// +// Basic Access Log report +// Written by Jonathan A. Foster +// Started August 20th, 2021 +// Copyright JF Possibilities, Inc. All rights reserved. +// +// To start with this gather data for a specific access period and +// create a report of domain names and addresses that accessed to and +// from the net. All "accepted" and "blocked" accesses will be +// ignored. For the moment we're expecting blocked traffic is actually +// blocked. +// +// The report will contain three columns: +// 1. domain name or address if a domain is not known. +// 2. list of ports that were connected to. +// 3. count of total connections +////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include + +#include "../strutil.h" +#include "../cli.h" +#include "../miniini.h" +using namespace std; + + + +////////////////////////////////////////////////////////////////////// +// A "line" of a report - a single domain/address +////////////////////////////////////////////////////////////////////// + +struct ReportLine { + int count; + map ports; + + // initialize + ReportLine(): count(0) {} + // add an item + void add(const string &port, int _count) { + ports[port]+=_count; + count+=_count; + } + // ports -> string + string port_list() const { + string r; + map::const_iterator it = ports.begin(); + if(it==ports.end()) return r; + r = it->first; + for(it++; it!=ports.end(); it++) r+=","+it->first; + return r; + } +}; + + + +////////////////////////////////////////////////////////////////////// +// A full reports worth of data - domain names / addresses and their +// associated stats. +////////////////////////////////////////////////////////////////////// + +struct ReportData: map { + + // Add an item to the report + void add(const string &address, const string &port, int count) { + (*this)[address].add(port, count); + } + + // Render report in an ASCII table + string ascii() const { + int widths[3] = {0,0,0}; // max column widths: 0: DNS, 1: ports, 3: counts + int x; + char l[256]; l[255]=0; + string s, r, bk; + ReportData::const_iterator it; + + for(it = begin(); it!=end(); it++) { + if(it->first.size()>widths[0]) widths[0] = it->first.size(); + s = it->second.port_list(); + if(s.size()>widths[1]) widths[1]=s.size(); + if(it->second.count>widths[2]) widths[2] = it->second.count; + } + // Now conver count max to cols + for(x=0; widths[2]; x++) widths[2] /= 10; + widths[2] = x ? x : 1; + // min col widths for titles + if(widths[0]<6) widths[0]=6; + if(widths[1]<7) widths[1]=7; + if(widths[2]<2) widths[2]=2; + + // render report + bk = "+"+string(widths[0]+2, '-')+ + "+"+string(widths[1]+2, '-')+ + "+"+string(widths[2]+2, '-')+ + "+\n"; + s = "| %-"+str(widths[0])+"s | %-"+str(widths[1])+"s | %"+str(widths[2])+"d |\n"; + r = bk; + snprintf(l, sizeof(l)-1, + ("| %-"+str(widths[0])+"s | %-"+str(widths[1])+"s | %-"+str(widths[2])+"s |\n").c_str(), + (const char *)"Remote", + (const char *)"Port(s)", + (const char *)"Ct" + ); + r+= l; + r+= bk; + for(it = begin(); it!=end(); it++) { + snprintf(l, sizeof(l)-1, s.c_str(), + it->first.c_str(), + it->second.port_list().c_str(), + it->second.count + ); + r+=l; + } + r+=bk; + return r; + } +}; +inline ostream &operator<<(ostream &out, const ReportData &r){ + return out << r.ascii(); +} +namespace cppdb { + result &operator>>(result &qry, ::ReportData &rpt) { + string name, addr, port; + int ct; + qry >> name >> addr >> port >> ct; + if(name=="") name=addr; + rpt.add(name, port, ct); + } +} + + +////////////////////////////////////////////////////////////////////// +// Our config file. +// +// This is designed so that all parts can use the same config. Tools +// ignore the parts they aren't interested in. +////////////////////////////////////////////////////////////////////// + +/* TODO: refactor this as a base class... */ +struct ReportConf: public MiniINI { + MiniINIvars traffic_mon; // This app's config variables + + ReportConf() { groups["Traffic Mon"] = &traffic_mon; } +}; + + + +////////////////////////////////////////////////////////////////////// +// Connection Report Generator Application Class +////////////////////////////////////////////////////////////////////// + +struct appConnectionReport: cBaseApp { + ReportConf config; + ReportData rpt; + cppdb::session db; + string start_stamp; + string end_stamp; + int cli_mode; // which non-switch are we processing + + + appConnectionReport(): cli_mode(0) {} + + + + int help() { + cerr << " FORMAT: " << basename(command_args[0]) << " -c {config} {start} {end}\n" + << '\n' + << "The config file must have a [Traffic Mon] section with the database\n" + << "credentials in it. 'start' and 'stop' are the SQL timestamps for\n" + << "report time span." << endl; + } + + + + virtual unsigned do_switch(const char *arg) { + if(*arg=='c' && !arg[1]) return 1; + throw CLIerror("Invalid switch "+string(arg)); + } + + + + virtual void do_switch_arg(const char *sw, const std::string &val) { + // The only way we get here is with -c + config.load(val); + // TODO: config validity checks + } + + + + virtual void do_arg(const char *arg) { + switch(cli_mode++) { + case 0: start_stamp = arg; return; + case 1: end_stamp = arg; return; + default: throw CLIerror("Invalid arguments"); + } + } + + + + int main() { + cppdb::result qry; + + /// SETUP & VALIDATE CLI /// + + try { + cBaseApp::main(); // Parse CLI args + if(!config.traffic_mon.vals.size()) throw CLIerror( + "You need to load a config file with a [Traffic Mon] section" + ); + if(cli_mode!=2) throw CLIerror("Invlaid arguments"); + } catch(const CLIerror &e) { + cerr << e.what() << "\n\n"; + return help(); + } + db.open("mysql:user="+qesc(config.traffic_mon.get("db user"))+ + ";password="+qesc(config.traffic_mon.get("db password"))+ + ";host="+qesc(config.traffic_mon.get("db host"))+ + ";database="+qesc(config.traffic_mon.get("db name"))+ + ";@opt_reconnect=1"); + + /// Query & load data /// + + qry = db << + "SELECT c.them_name, c.them, c.them_port, count(*) " + "FROM connections c LEFT OUTER JOIN dns d ON c.them_name=d.name " + "WHERE c.inbound=0 AND (d.status IS NULL or d.status=0) " + "AND tstamp>=? AND tstamp<=? " + "GROUP BY c.them_name, c.them, c.them_port" + << start_stamp << (end_stamp + " 23:59:59"); // include to the end of the day + while(qry.next()) qry >> rpt; + + /// spit out the report /// + + cout << "Web access report for " << start_stamp << " - " << end_stamp << "\n\n" + << rpt; + return 0; + } + +}; + + + +////////////////////////////////////////////////////////////////////// +// Lets run the report and dump it out +////////////////////////////////////////////////////////////////////// + +MAIN(appConnectionReport) diff --git a/trafficmon/default b/trafficmon/default new file mode 100644 index 0000000..bbdd9e8 --- /dev/null +++ b/trafficmon/default @@ -0,0 +1,9 @@ +# Configuration for init.d/trafficmon. All entries remarked out below are the +# defaults. + +# Configuration file for the TrafficMon server +#CONF=/etc/poorman-ids/sample.conf +# Where "run" files are placed. This is the Debian+ default: +#RUN=/run +# This needs to match the pipe speicified in the syslog.d/trafficmon file. +#SOCK=$RUN/poorman-ids/trafficmon.sock diff --git a/trafficmon/init b/trafficmon/init new file mode 100755 index 0000000..979e16f --- /dev/null +++ b/trafficmon/init @@ -0,0 +1,113 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: trafficmon +# Required-Start: $remote_fs +# Required-Stop: $remote_fs +# X-Start-Before: $syslog +# X-Stop-After: $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Monitor and record net traffic +# Description: This service listens to a pipe for "iptables LOG" and +# dnsmasq DNS query messages. It then records information +# about internet access into a MySQL DB. Typically these +# messages are provided by syslogd. +# +# NOTE: if this is started after syslogd the pipe might not +# be available and syslogd will simply not try to write to +# until a restart. +### END INIT INFO + +NAME="trafficmon" +DAEMON="/usr/sbin/$NAME" +RUN=/run +CONF=/etc/poorman-ids/sample.conf +SOCK="" + +# Pull in config +if [ -r "/etc/default/$NAME" ]; then + . /etc/default/$NAME +fi + + + +### Setup control variables ### + +# This is where we put PID files and the pipe +RUN="$RUN/poorman-ids" +PID="$RUN/$NAME.pid" +mkdir -p "$RUN" +[ -n "$SOCK" ] || SOCK="$RUN/$NAME.sock" + + + +### ACTIONS ### + +# The main service command +CTRL() { + start-stop-daemon --pidfile "$PID" --exec "$DAEMON" "$@" +} + + + +do_start() { + echo -n "Starting Traffic Monitor: " + if CTRL --start --oknodo -- -c "$CONF" -b -i "$PID" -b -p "$SOCK"; then + echo "OK" + else + echo "FAIL" + exit 1 + fi +} + + + +do_stop() { + echo -n "Stoping Traffic Monitor: " + if CTRL --stop --remove-pidfile; then + echo "OK" + else + echo "FAIL" + exit 1 + fi +} + + + +do_status() { + echo -n "Traffic Monitor is: " + if CTRL --status; then + echo "Up" + else + echo "Down" + exit 1 + fi +} + + + +### Main() + +case "$1" in + start) + do_start + ;; + + stop) + do_stop + ;; + + restart) + do_status && do_stop + do_start + ;; + + status) + do_status + ;; + + *) + echo "$0 {start | stop | restart | status}" + ;; + +esac \ No newline at end of file diff --git a/trafficmon/syslog b/trafficmon/syslog new file mode 100644 index 0000000..088902c --- /dev/null +++ b/trafficmon/syslog @@ -0,0 +1 @@ +kern,daemon.* |/run/poorman-ids/trafficmon.sock diff --git a/trafficmon/trafficmon.cpp b/trafficmon/trafficmon.cpp new file mode 100644 index 0000000..57958a3 --- /dev/null +++ b/trafficmon/trafficmon.cpp @@ -0,0 +1,335 @@ +////////////////////////////////////////////////////////////////////// +// IP traffic monitor +// Written by Jonathan A. Foster +// Started August 11th, 2021 +// +// The idea is to analyze iptables LOG entries in combination with +// DNSmasq's query log entries and combine them to list the hosts +// that were accessed. This will be read in real time from SysLogD +// using the "pipe" feature. Access records will be logged to MySQL +// for reporting and record keeping. +////////////////////////////////////////////////////////////////////// +// TODO: catch signals and attempt removal of PID file during shutdown. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cli.h" +#include "../data.h" +#include "../miniini.h" +#include "../config.h" +using namespace std; + + + +////////////////////////////////////////////////////////////////////// +// Monitor Config +////////////////////////////////////////////////////////////////////// + +struct MonitorConf: public MiniINI { + MiniINIvars traffic_mon; // This app's config variables + INIusList us; + + MonitorConf() { groups["Traffic Mon"] = &traffic_mon; groups["us"] = &us; } +}; + + + +////////////////////////////////////////////////////////////////////// +// Application class to store data passed in through a pipe or +// file(s). +////////////////////////////////////////////////////////////////////// +//#define DEBUG + +struct TrafficMon: public cBaseApp { + MonitorConf config; + LogAnalyzer analyze; + istream *log; + cppdb::session db; + LiveBug bug; + bool setup_db; + bool piping; // are we sucking on a pipe? + bool background_me; // daemonize? + string pid_file; // Name of file to write PID in + int inp_ct; // How many inputs were on the CLI + bool running; // Whether or not its time to run logs + long long line_no; // current line # in the active input + + + + TrafficMon(): + log( 0), + setup_db( false), + piping( false), + background_me(false), + inp_ct( 0), + running( false), + line_no( 0) + { + analyze.us = &(config.us.vals); + } + + + + void help() { + cerr << + "\n" + "trafficmon -c {config file} [-i] [-d] [-b] [-p {pipe name} | [{log name} ...]]\n" + "\n" + "Arguments:\n" + " -c Load configuration from file (required)\n" + " -i PID file to write (optional, use with -b)\n" + " -d Create data tables (optional, use once)\n" + " -b Run in background - daemonize (optional, use -i too)\n" + " -p read from pipe creating it if needed (optional, good with -b)\n" + "\n" + "Other arguments are log files to be processed. This can be used to bulk\n" + "load data prior to going live with an always on daemon or for catching\n" + "up if the daemon stopped for some reason."; + ExitCode = 1; + } + + + + unsigned do_switch(const char *sw) { + if(sw[1]==0) { + switch(*sw) { + case 'b': background_me = true; return 0; + case 'c': return 1; + case 'd': setup_db = true; return 0; + case 'i': return 1; + case 'p': piping = true; return !running; // second pass treats it as a file + } + } + throw CLIerror("Unrecognized Switch"); + } + + + + void do_switch_arg(const char *sw, const string &val) { + // If "running" is set then we've already done this. So skip it! + if(running) return; + + switch(*sw) { + + /// Load config file /// + + case 'c': + // This is only called with "c". See above + config.load(val); + if(!config.us.vals.size()) throw CLIerror( + "The configuration files MUST contain an [us] section with " + "appropriate values" + ); + if(!config.traffic_mon.vals.size()) throw CLIerror( + "The configuration files MUST contain an [Traffic Mon] section with " + "appropriate values" + ); + + /// configure resources from config file /// + + // TODO: pre-validate some of these? + db.open("mysql:user="+qesc(config.traffic_mon.get("db user"))+ + ";password="+qesc(config.traffic_mon.get("db password"))+ + ";host="+qesc(config.traffic_mon.get("db host"))+ + ";database="+qesc(config.traffic_mon.get("db name"))+ + ";@opt_reconnect=1"); + return; + + /// PID file to write ID in if we're daemonizing /// + + case 'i': + pid_file = val; + return; + + /// Make pipe if requested /// + + case 'p': + // This can't wait for "running" since we want it to fail in foreground + // Make read/write by process owner (root) only + if(mkfifo(val.c_str(), 0600)<0 && errno!=17 /*already exists*/) { + ExitCode = 2; + throw runtime_error("Making pipe raised error "+str(errno)); + } + inp_ct++; + } + } + + + + void do_arg(const char *fname) { + // if not "running" we don't want to do this yet. Just count inputs. + if(!running) { + inp_ct++; + return; + } + + // the only thing we expect on the CLI is a log/pipe name + if(log==&cin) log = 0; // won't happen right now + if(log) + ((ifstream*)log)->close(); + else + log = new ifstream; + if(!background_me) cerr << fname << ":\n"; +restart: + ((ifstream*)log)->open(fname); + ExitCode = run(); + if(piping) { + // If a process closes the write end of the pipe, like during log + // rotation, we receive an EOF, end up here, and have to re-open the pipe + // to be able to receive more on it. + ((ifstream*)log)->close(); + sleep(1); // This is just to make sure we don't hog CPU in a loop + goto restart; + } + } + + + + // This is the actual data process + // probably should daemonize around here. + int run() { + string l; + cppdb::statement db_add = db << // prepare insert statement + "INSERT INTO connections " + "(us, us_port, them, them_port, them_name, protocol, inbound) " + "VALUES (?,?,?,?,?,?,?)"; + + /// parse log file /// + + line_no=0; + while(getline(*log, l)) { + line_no++; + if(!background_me) + cerr << bug << ' ' << line_no << '\r' << flush; + + /// process connections /// + + if(analyze.line(l)) { + // TODO: time stamp handling? + // insert record + db_add.reset(); + db_add << analyze.conn.us << analyze.conn.us_port << analyze.conn.them + << analyze.conn.them_port << analyze.conn.name + << analyze.conn.protocol << analyze.conn.in + << cppdb::exec; + } + } + // In a real pipe situation this should never get reached. + if(!background_me) { + cerr << "\nLines: " << line_no + << "\nTotal rDNS: " << analyze.rdns.size() << '\n' << endl; + } + return 0; + } + + + + void create_tables() { + + /// Connection recrods /// + + db << "CREATE TABLE connections (" + "id INT NOT NULL AUTO_INCREMENT PRIMARY KEY," + "tstamp TIMESTAMP NOT NULL," // default is NOW() + "us CHAR(80) NOT NULL DEFAULT ''," + "us_port INT NOT NULL DEFAULT 0," + "them CHAR(80) NOT NULL DEFAULT ''," + "them_port INT NOT NULL DEFAULT 0," + "them_name CHAR(128) NOT NULL DEFAULT ''," + "protocol CHAR(12) NOT NULL DEFAULT ''," + "inbound TINYINT(1) NOT NULL DEFAULT 0," + "INDEX tstamp(tstamp, them_name)" + ") Engine=MyISAM" + << cppdb::exec; + // NOTE: MyISAM is thousands of times faster than InnoDB for this sort + // of work load. Mostly continuous writes with occasional reads. + // But will it work for all I'm going to do? AriaDB is slower but + // significantly faster than InnoDB. + + /// Status of domain names /// + + db << "CREATE TABLE dns (" + "name CHAR(128) NOT NULL PRIMARY KEY," + "decided TIMESTAMP NOT NULL," + "status TINYINT(1) NOT NULL DEFAULT 0" + ") Engine=MyISAM" + << cppdb::exec; + } + + + + int main() { + try { + cBaseApp::main(); + if(!config.us.vals.size()) + // Since -c requires something in [us] we'll only get here if a config + // wasn't loaded. + throw CLIerror("You must specify a config file to load"); + if(piping && inp_ct!=1) throw CLIerror( + "Pipe requires one and only one file name to read from." + ); + if(setup_db) create_tables(); + + // if requested attempt daemonizing. + if(background_me) { + if(inp_ct==0) throw CLIerror( + "Backgrounding requires the specification of an input source. " + "STDIN is unavailable when we move to background." + ); + + if(daemon(0,0)<0) throw CLIerror("Failed to switch to background"); + // If we get here we're in the background. No STDIN, STDOUT, STDERR + if(pid_file!="") { // pid_file requested + ofstream pf(pid_file.c_str()); + pf << getpid(); + } + } + } catch(const CLIerror &e) { + cerr << "ERROR: " << e.what() << "\n"; + help(); + return ExitCode ? ExitCode : 1; // JIC someone sets a different exit code + } + + try { + // CLI processed now lets analyze data. + running = true; + cBaseApp::main(); // re-run CLI args for inputs + if(!log) { + // no inputs spec'd on CLI assume stdin (we're not a daemon) + log = &cin; + ExitCode = run(); + } + } catch(const exception &e) { + if(ExitCode==0) ExitCode = 2; + if(background_me) { + openlog("trafficmon", LOG_CONS | LOG_PID, LOG_DAEMON); + syslog(LOG_ALERT, "%s", e.what()); + closelog(); + } else + cerr << "ERROR: " << e.what() << "\n"; + } + return ExitCode; + } + + + +}; + + + +////////////////////////////////////////////////////////////////////// +// Run it +////////////////////////////////////////////////////////////////////// + +MAIN(TrafficMon)