Improved DNS matching

* Report incorrectly formatted ignores in config file.
 * Compact IPv6 addresses in iptables log output to match dnsmasq
   reporting.
 * Handle dnsmasq "cached" log entries.
 * Update README.md

iptraffic output is much improved.

README.md

@@ -158,7 +158,7 @@ stretches to the next header and will either contain name / value
 pairs or specifically formatted text data.
 
 Remarks are lines that start with a pound sign (#). I am not using
-the MS style semicolon (;). Blank lines are also ignored.
+the MS style semicolon (;). Blank lines are ignored.
 
 Currently there are two sections allowed in the file:
 
@@ -167,13 +167,11 @@ Currently there are two sections allowed in the file:
    outside your network (them). This is used to determine direction
    of connection (in/out).
 
-    Each line is a single entry. The entries can be in IPv4 & 6
-    notation and should be in a format that matches your iptables log
-    output. For IPv4 that means no leading zeros. For IPv6 **all**
-    hex digits are usually logged, even the zeros. Address prefixes
-    can be used to define networks and they should end with either
-    "." (IPv4) or ":" (IPv6). With IPv6 it could be useful to end on
-    a boundary other than those denoted by colon.
+    Each line is a single entry. The entries can be in IPv4 or 6
+    notation and should be in a minimalistic format. Basically this
+    means no leading zeros. For IPv6 make sure to use :: as
+    appropriate. Address prefixes can be used to define networks and
+    they should end with either "." (IPv4) or ":" (IPv6).
 
     Example: "192.168.1." matches the 192.168.1.0/24 network.
 
@@ -183,9 +181,11 @@ Currently there are two sections allowed in the file:
 
     TSV records contain the following fields separated by tabs:
 
-    1. Our / internal address or "*" for match all addresses.
+    1. Our / internal address. Follows same wild card specification as
+       described in "us" above.
     2. Our port number. 0 matches all.
-    3. Outside / their IP address. "*" matches all.
+    3. Outside / their IP address. Follows same wild card
+       specification as described in "us" above.
     4. Their port number. Zero matches all.
     5. Domain name associated with the destination. "*" matches all.
     6. IP protocol used (must be all caps):
@@ -217,6 +217,9 @@ Currently there are two sections allowed in the file:
     So this means things like g00gle analytics, CDNs, external
     JavaScript and CSS servers, ... will still be reported.
 
+    Obviously in normal use the list of OK (ignored) content will get
+    quite long. Future tools will handle this better.
+
 
 
 iptraffic Command Line

config.cpp

@@ -5,6 +5,7 @@
 // Copyright JF Possibilities, Inc. All rights reserved.
 //////////////////////////////////////////////////////////////////////
 #include <fstream>
+#include <iostream>
 #include "config.h"
 #include "strutil.h"
 
@@ -15,9 +16,11 @@ void Config::load(const std::string &fname) {
   std::ifstream f(fname.c_str());
   TSV tsv;
   Conn conn;
+  int ln=0;
 
 
   while(std::getline(f, l)) {
+    ln++;
     l = strip(l);
     if(l=="" || l[0]=='#') continue;
     if(l.size()>2 && l[0]=='[' && l.end()[-1]==']') {
@@ -28,6 +31,7 @@ heading:
         /// Read in "us" list ///
 
         while(std::getline(f, l)) {
+          ln++;
           l=strip(l);
           if(l=="" || l[0]=='#') continue;
           if(l.size()>2 && l[0]=='[' && l.end()[-1]==']') goto heading;
@@ -39,9 +43,14 @@ heading:
         /// Read in ignore list ///
 
         while(std::getline(f, l)) {
+          ln++;
           if(l=="" || l[0]=='#') continue;
           if(l.size()>2 && l[0]=='[' && l.end()[-1]==']') goto heading;
           tsv = l;
+          if(tsv.count!=7) {
+            std::cerr << "Incorrrect column count in config file line " << ln << std::endl;
+            continue;
+          }
           if(tsv.count>6) {
             tsv >> conn;
             ignores.push_back(conn);

data.cpp

@@ -4,14 +4,48 @@
 // Started April 23rd, 2021
 // Copyright JF Possibilities, Inc. All rights reserved.
 //////////////////////////////////////////////////////////////////////
+#include <arpa/inet.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdexcept>
+#include <iostream>
 #include "data.h"
 
 
 
 //////////////////////////////////////////////////////////////////////
+// Utils
+//////////////////////////////////////////////////////////////////////
+
+std::string ipv6opt(const std::string &addr) {
+  in6_addr buf;
+  char s[256];
+
+  if(inet_pton(AF_INET6, addr.c_str(), &buf)<1) throw
+    std::runtime_error("ipv6opt: inet_pton() says '"+addr+"' is not a valid IPv6 address");
+  if(!inet_ntop(AF_INET6, &buf, s, 255)) throw // should never happen
+    std::runtime_error("ipv6opt: inet_ntop() refused to convert the address back");
+  return std::string(s);
+}
+
+
+
+int addr_wild_comp(const std::string &str1, const std::string &str2) {
+  int spre1=0, spre2=0;
+
+  if(str1=="*" || str2=="*") return 0;
+  if(str1!="" && (str1.end()[-1]=='.' || str1.end()[-1]==':')) spre1=str1.size();
+  if(str2!="" && (str2.end()[-1]=='.' || str2.end()[-1]==':')) spre2=str2.size();
+  if(spre2>spre1) spre1=spre2;
+  if(spre1)          return strncmp(str1.c_str(), str2.c_str(), spre1);
+  else if(str1<str2) return -1;
+  else if(str1>str2) return  1;
+                     return  0;
+}
+
+
+
+//////////////////////////////////////////////////////////////////////
 // Conn
 //////////////////////////////////////////////////////////////////////
 
@@ -23,6 +57,13 @@ void Conn::clear() {
 
 
 
+void Conn::compact() {
+  if(us.find(':')!=us.npos)   us=ipv6opt(us);
+  if(them.find(':')!=us.npos) them=ipv6opt(them);
+}
+
+
+
 void Conn::swap() {
   std::string s;
   int x;
@@ -74,19 +115,15 @@ Conn &Conn::operator=(const Splits &sp) {
 
 // TODO: does < > have any actual meaning in this context?
 int Conn::cmp(const Conn &gtr) const {
-  if(us!="*" && gtr.us!="*") {
-    if(us<gtr.us) return -1;
-    if(us>gtr.us) return  1;
-  }
+  int r;
+
+  if(r = addr_wild_comp(us, gtr.us)) return r;
   // TODO: auto-wildcard port based on in?
   if(us_port && gtr.us_port) { // 0 = no comparison wildcard
     if(us_port<gtr.us_port) return -1;
     if(us_port>gtr.us_port) return  1;
   }
-  if(them!="*" && gtr.them!="*") {
-    if(them<gtr.them) return -1;
-    if(them>gtr.them) return  1;
-  }
+  if(r = addr_wild_comp(them, gtr.them)) return r;
   if(them_port && gtr.them_port) { // 0 = no comparison wildcard
     if(them_port<gtr.them_port) return -1;
     if(them_port>gtr.them_port) return  1;

data.h

@@ -20,6 +20,28 @@
 
 
 //////////////////////////////////////////////////////////////////////
+// Reduce IPv6 address string to optimal format.
+//
+// The resultant string is produced by standard routines in libC. Its
+// as optimal as it makes it. *DON'T* run filter strings through this
+// or you'll lose partial match capability.
+//////////////////////////////////////////////////////////////////////
+
+std::string ipv6opt(const std::string &addr);
+
+
+
+//////////////////////////////////////////////////////////////////////
+// Wild compare of two address strings
+//
+// returns: 0=match, -1 = <, 1 = >
+//////////////////////////////////////////////////////////////////////
+
+int addr_wild_comp(const std::string &str1, const std::string &str2);
+
+
+
+//////////////////////////////////////////////////////////////////////
 // Network connection between "us" and "them"
 //////////////////////////////////////////////////////////////////////
 typedef unsigned short word;
@@ -35,6 +57,8 @@ struct Conn {
   Conn(): us_port(0), them_port(0), in(false) {}
   // clear data
   void clear();
+  // Compact IPv6 address strings
+  void compact();
   // swap polarity of record
   void swap();
   // scan & copy data from log record in

iptraffic.cpp

@@ -148,8 +148,10 @@ struct IPtraffic: public cBaseApp {
       ///  DNS query result ///
 
       // TODO: need to get more specific on tying us + them + time to DNS
+      // TODO: doesn't seem that CNAMEs are getting attached to requests properly.
+      //       the logs are cryptic on this front.
       if(ln.count>8 && strncmp(ln.fields[4], "dnsmasq[", 8)==0) {
-        if(ln[5]=="reply") {
+        if(ln[5]=="reply" || ln[5]=="cached") {
           name = ln[6];
           address = ln[8];
           // Hmm... is this reply an address?
@@ -175,6 +177,7 @@ struct IPtraffic: public cBaseApp {
       && ln[6]=="ACCEPT")
       ) {
         conn = ln;
+        conn.compact();
         if(!pre_match(config.us, conn.us)) conn.swap();
         if((nvp=rdns.find(conn.them))!=rdns.end())
           conn.name = nvp->second;