Merge branch 'master' of https://github.com/mcortizo/ngxtop into mcortizo-master

Allow ngxtop to parse logs directly from stdin. Pre-defined common log format for Apache access log.

3 files changed, 55 insertions, 4 deletions

diff --git a/README.rst b/README.rst
index 099d0bf..2680597 100644
--- a/README.rst
+++ b/README.rst
@@ -1,10 +1,16 @@
-===================================================
-``ngxtop`` - **real-time** metrics for nginx server
-===================================================
+================================================================
+``ngxtop`` - **real-time** metrics for nginx server (and others)
+================================================================
 
 **ngxtop** parses your nginx access log and outputs useful, ``top``-like, metrics of your nginx server.
 So you can tell what is happening with your server in real-time.
 
+``ngxtop`` tries to determine the correct location and format of nginx access log file by default, so you can just run
+``ngxtop`` and having a close look at all requests coming to your nginx server. But it does not limit you to nginx
+and the default top view. ``ngxtop`` is flexible enough for you to configure and change most of its behaviours.
+You can query for different things, specify your log and format, even parse remote Apache common access log with ease.
+See sample usages below for some ideas about what you can do with it.
+
 Installation
 ------------
 
@@ -44,6 +50,11 @@ Usage
         -h, --help  print this help message.
         --version  print version information.
 
+        Advanced / experimental options:
+        -c <file>, --config <file>  allow ngxtop to parse nginx config file for log format and location.
+        -i <filter-expression>, --filter <filter-expression>  filter in, records satisfied given expression are processed.
+        -p <filter-expression>, --pre-filter <filter-expression> in-filter expression to check in pre-parsing phase.
+
 Samples
 -------
 
@@ -109,3 +120,30 @@ List 4xx or 5xx responses together with HTTP referer
     |-----------+----------+----------------|
     | -         |      400 | -              |
 
+Parse apache log from remote server with `common` format
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    $ ssh user@remote_server tail -f /var/log/apache2/access.log | ngxtop -f common
+    running for 20 seconds, 1068 records processed: 53.01 req/sec
+
+    Summary:
+    |   count |   avg_bytes_sent |   2xx |   3xx |   4xx |   5xx |
+    |---------+------------------+-------+-------+-------+-------|
+    |    1068 |        28026.763 |  1029 |    20 |    19 |     0 |
+
+    Detailed:
+    | request_path                             |   count |   avg_bytes_sent |   2xx |   3xx |   4xx |   5xx |
+    |------------------------------------------+---------+------------------+-------+-------+-------+-------|
+    | /xxxxxxxxxx                              |     199 |        55150.402 |   199 |     0 |     0 |     0 |
+    | /xxxxxxxx/xxxxx                          |     167 |        47591.826 |   167 |     0 |     0 |     0 |
+    | /xxxxxxxxxxxxx/xxxxxx                    |      25 |         7432.200 |    25 |     0 |     0 |     0 |
+    | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxxx      |      22 |          698.727 |    22 |     0 |     0 |     0 |
+    | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxx       |      19 |         7431.632 |    19 |     0 |     0 |     0 |
+    | /xxxxx/xxxxx/                            |      18 |         7840.889 |    18 |     0 |     0 |     0 |
+    | /xxxxxxxx/xxxxxxxxxxxxxxxxx              |      15 |         7356.000 |    15 |     0 |     0 |     0 |
+    | /xxxxxxxxxxx/xxxxxxxx                    |      15 |         9978.800 |    15 |     0 |     0 |     0 |
+    | /xxxxx/                                  |      14 |            0.000 |     0 |    14 |     0 |     0 |
+    | /xxxxxxxxxx/xxxxxxxx/xxxxx               |      13 |        20530.154 |    13 |     0 |     0 |     0 |
+
diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py
index 82a2d2a..b8e4804 100644
--- a/ngxtop/config_parser.py
+++ b/ngxtop/config_parser.py
@@ -16,6 +16,9 @@ REGEX_LOG_FORMAT_VARIABLE = r'\$([a-z0-9\_]+)'
 LOG_FORMAT_COMBINED = '$remote_addr - $remote_user [$time_local] ' \
                       '"$request" $status $body_bytes_sent ' \
                       '"$http_referer" "$http_user_agent"'
+LOG_FORMAT_COMMON   = '$remote_addr - $remote_user [$time_local] ' \
+                      '"$request" $status $body_bytes_sent ' \
+                      '"$http_x_forwarded_for"'
 
 # common parser element
 semicolon = Literal(';').suppress()
@@ -127,6 +130,8 @@ def build_pattern(log_format):
     """
     if log_format == 'combined':
         log_format = LOG_FORMAT_COMBINED
+    elif log_format == 'common':
+        log_format = LOG_FORMAT_COMMON
     pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format)
     pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern)
     return re.compile(pattern)
diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py
index f1487a5..6e1bdfe 100755
--- a/ngxtop/ngxtop.py
+++ b/ngxtop/ngxtop.py
@@ -51,6 +51,9 @@ Examples:
 
     Average body bytes sent of 200 responses of requested path begin with 'foo':
     $ ngxtop avg bytes_sent --filter 'status == 200 and request_path.startswith("foo")'
+
+    Analyze apache access log from remote machine using 'common' log format
+    $ ssh remote tail -f /var/log/apache2/access.log | ngxtop -f common
 """
 from __future__ import print_function
 import atexit
@@ -303,7 +306,9 @@ def build_processor(arguments):
 
 def build_source(access_log, arguments):
     # constructing log source
-    if arguments['--no-follow']:
+    if access_log == 'stdin':
+        lines = sys.stdin
+    elif arguments['--no-follow']:
         lines = open(access_log)
     else:
         lines = follow(access_log)
@@ -334,6 +339,9 @@ def setup_reporter(processor, arguments):
 def process(arguments):
     access_log = arguments['--access-log']
     log_format = arguments['--log-format']
+    if access_log is None and not sys.stdin.isatty():
+        # assume logs can be fetched directly from stdin when piped
+        access_log = 'stdin'
     if access_log is None:
         access_log, log_format = detect_log_config(arguments)