Added explanations.

tilfin · tilfin · commit 78ed9e293f49 · 2013-02-10T21:56:56.000+09:00
diff --git a/README.md b/README.md
@@ -1,2 +1,47 @@
 detect-http-attack
-==================
+==================
+
+It is a detecting attack tool for HTTP server such as Apache and Nginx.
+Analyzing access logs, output formated text as results.
+
+To use shell pipelines easily, all I/O targets are STDIN, STDOUT and STDERR.
+
+### Analyze access log:
+
+    ./detect-http-attack.rb < /var/log/nginx/access.log
+
+Targets eight or more consecutive senquential access:
+
+    ./detect-http-attack.rb -s 8 < /var/log/apache/access_log
+
+Regarded as senquential access within 3 seconds:
+
+    ./detect-http-attack.rb -i 3 < /var/log/apache/access_log
+
+### Notify attack while tailing access log:
+
+Notifying attacks whenever detecting them to STDERR, all results are output to a file.
+
+    tail -f /var/log/nginx/access.log | ./detect-http-attack.rb -n > attack.log
+
+### LTSV Format adapted:
+
+Uses Labeled Tab-separated Values (LTSV) format (http://ltsv.org/)
+
+    ./detect-http-attack.rb -ltsv < /var/log/apache/access_ltsv_log
+
+### Settings and Customize:
+
+edits detect-http-attack.conf
+
+### Usage:
+
+    ./detect-http-attack.rb --help
+
+    Usage: detect_http_attack [options]
+        -ltsv                            Log type is LTSV
+        -n                               notify when detecting attack
+        -s COUNT                         Specify minimum sequential count
+        -i SECONDS                       Specify maximum interval seconds
+        -f CONFFILE                      Specify configuration file
+
diff --git a/detect_http_attack.conf b/detect_http_attack.conf
@@ -1,11 +1,15 @@
 #
-# Detection HTTP Attack
+# Detect HTTP Attack
 #
 # Configuration file
 #
 #=========================
 
 
+#=========================
+# Exclueded access targets 
+#==========================
+#
 # Excluded hosts (comma-separated)
 exc_hosts=127.0.0.1
 
@@ -20,39 +24,91 @@ exc_path_match=\.(js|css|jpg|gif|png|ico)$
 # Output Template
 #==================
 #
-# Terminal View Format for "tail -f access_log | "
+# Template consists of three lines 'head', 'body' and 'foot'.
+# Attack log treats sequential accesses from each host as BLOCK.
+# 'head' is a first line of BLOCK.
+# 'body' is a line for each access.
+# 'foot' is a last line of BLOCK.
+#
+# If '-n' flag is specified,
+# attack access is printed as 'serr' line to STDERR
+# whenever they are detected.
+#
+# Each line is specified escape sequences and fields.
+# Field name begins with '$'.
+#
+#
+# == Field List ==
+#
+#  = Apache, Nginx Combined Log Format =
 #
-# head is host, count, ua.
-#   host that is IP address or Hostname is cyan bold.
-#   count that is the number of sequential access is magenta.
-#   ua that is User-Agent is green.
+#  > $host $ident $user [$time] "$req" $status $size $referer "$ua"
+#  > 66.249.XX.XX - - [10/Feb/2013:10:27:45 +0900] "GET / HTTP/1.1" 200 1500 "-" "Searchbot/2.1"
+#  
+#   host    = 66.249.XX.XX
+#   ident   = -
+#   user    = -
+#   time    = 10/Feb/2013:10:27:45 +0900
+#   req     = GET / HTTP/1.1
+#   status  = 200
+#   size    = 1500
+#   referer = -
+#   ua      = Searchbot/2.1
+# 
+#  And futhermore, req is broken down into method, path and version.
+# 
+#   method  = GET
+#   path    = /
+#   version = HTTP/1.1
 #
-# body is date, status, path, size.
-#   date is access date time by Ruby standard output.
-#   status is HTTP status code.
-#   path is the request path of URL.
-#   referer is HTTP Referer.
-#   
-#   Other fields
-#     $size   response size without headers.
-#     
 #
-# foot is empty line.
+#  = LTSV Format =
 #
+#  Field name is applied to each label.
+#  And futhermore, req is broken down into method, path and version.
+#
+
+#
+# == Formats ==
+#
+#  = Terminal View Format for "tail -f access_log | " =
+# 
+#  head is host, count, ua.
+#    host that is IP address or Hostname is cyan bold.
+#    count that is the number of sequential access is magenta.
+#    ua that is User-Agent is green.
+# 
+#  body is date, status, path, size.
+#    date is access date time by Ruby standard output.
+#    status is HTTP status code.
+#    path is the request path of URL.
+#    referer is HTTP Referer.
+#      
+#  foot is empty line.
+# 
+#  serr is red host, count, ua, date, status, path and referer at two lines.
+# 
 head=\e[36m\e[1m$host\e[0m\t\e[35m$count\e[0m\t\e[32m$ua\e[0m\n
 body=$date\t$status\t$path\t$referer\n
 foot=\n
+serr=\e[31m\e[1m$host\e[0m\t\e[33m\e[1m$count\e[0m\t$ua\n$date\t$status\t$path\t$referer\n\n
+
+#
+# = Terminal View Format without color sequences =
+#
 #head=$host\t$count\t$ua\n
 #body=$date\t$status\t$path\t$referer\n
 #foot=\n
+#serr=$host\t$count\t$ua\n$date\t$status\t$path\t$referer\n\n
 
 #
-# LTSV format
+# = LTSV format =
 #
-# head and foot isn't output.
-# body is time, host and req.
+#  head and foot aren't output.
+#  body is time, host and req.
 #
 ##head=
 #body=time:[$time]\thost:$host\treq:$req\n
 ##foot=
+#serr=time:[$time]\thost:$host\treq:$req\n
 
diff --git a/detect_http_attack.rb b/detect_http_attack.rb
@@ -99,9 +99,10 @@ def parse(line)
       values[name.to_sym] = value
     }
 
-    method, path = values[:req].split(' ')
+    method, path, version = values[:req].split(' ')
     values[:method] = method
     values[:path] = path
+    values[:version] = version
     values[:date] = parse_datetime(values[:time])
 
     values
@@ -144,16 +145,18 @@ def get(field)
 
 class Template
 
-  def initialize(head, body, foot)
+  def initialize(head, body, foot, serr)
     @re_field = Regexp.new("\\$[a-z]+")
 
     h = head || "$host\\t$count\\t$ua"
     b = body || "$date\\t$path\\t$referer"
     f = foot || ""
+    s = serr || "$host\\t$count\\t$ua\\n$date\\t$path\\t$referer"
 
     @head = parse_value(h)
     @body = parse_value(b)
     @foot = parse_value(f)
+    @serr = parse_value(s)
   end
 
   def  parse_value(value)
@@ -182,22 +185,26 @@ def  parse_value(value)
     vals
   end
 
-  def print_head(ops, count, row)
+  def print_head(count, row)
     if @head
-      print_row(@head, ops, count, row)
+      print_row(@head, STDOUT, count, row)
     end
   end
 
-  def print_body(ops, row)
-    print_row(@body, ops, "", row)
+  def print_body(row)
+    print_row(@body, STDOUT, "", row)
   end
 
-  def print_foot(ops, count, row)
+  def print_foot(count, row)
     if @foot
-      print_row(@foot, ops, count, row)
+      print_row(@foot, STDOUT, count, row)
     end
   end
 
+  def print_serr(count, row)
+    print_row(@serr, STDERR, count, row)
+  end
+
   def print_row(templ, ops, count, row)
     templ.each do |val|
       if val.instance_of?(String)
@@ -221,6 +228,7 @@ class DetectionProcessor
 
   attr :interval_threshold, true
   attr :sequence_threshold, true
+  attr :notify,  true
   attr :exc_hosts, true
   attr :exc_ua, true
   attr :exc_path,  true
@@ -235,6 +243,8 @@ def initialize(template)
     @exc_ua = nil
     @exc_path = nil
 
+    @realtime_notify = false
+
     @pre_access_map = Hash.new
     @ops = STDOUT
   end
@@ -254,6 +264,10 @@ def proc(row)
       if (date_ts - ts) <= @interval_threshold
         al.push(row)
         @pre_access_map.store(host, [date_ts, al])
+
+        if @notify and al.count >= @sequence_threshold
+          @template.print_serr(al.count, row)
+        end
       else
         if al.count >= @sequence_threshold
           print_access(host, al)
@@ -277,13 +291,13 @@ def finalize
   def print_access(host, al)
     fl = al[0]
 
-    @template.print_head(@ops, al.count, fl)
+    @template.print_head(al.count, fl)
 
     al.each do |row|
-      @template.print_body(@ops, row)
+      @template.print_body(row)
     end
 
-    @template.print_foot(@ops, al.count, fl)
+    @template.print_foot(al.count, fl)
   end
 end
 
@@ -292,12 +306,14 @@ def print_access(host, al)
 def get_opts
   # Settting from Arguments 
 
-  opts = { :parser => 'combined', :max_interval => 3, :min_seq => 8 }
+  opts = { :parser => 'combined', :max_interval => 3, :min_seq => 8, :notify => false }
 
   opt = OptionParser.new
   
   opt.on('-ltsv', 'Log type is LTSV') { |v| opts[:parser] = "ltsv" }
   
+  opt.on('-n', 'notify when detecting attack') { |v| opts[:notify] = true }
+
   opt.on('-s COUNT', 'Specify minimum sequential count') do |count|
     opts[:min_seq] = count.to_i
   end
@@ -327,11 +343,12 @@ def main
 
   conf = Configuration.new(opts[:conf_file])
 
-  template = Template.new(conf.get(:head), conf.get(:body), conf.get(:foot))
+  template = Template.new(conf.get(:head), conf.get(:body), conf.get(:foot), conf.get(:serr))
 
   processor = DetectionProcessor.new(template)
   processor.interval_threshold = opts[:max_interval]
   processor.sequence_threshold = opts[:min_seq]
+  processor.notify = opts[:notify]
 
   exc_hosts = conf.get(:exc_hosts)
   if exc_hosts
@@ -340,7 +357,7 @@ def main
   
   exc_ua = conf.get(:exc_ua_match)
   if exc_ua
-    processor.exc_ua = Regexp.new(exc_ua, "i")
+    processor.exc_ua = Regexp.new(exc_ua, Regexp::IGNORECASE)
   end
 
   exc_path = conf.get(:exc_path_match)
