From 4137c7da57e74e624a865e357479e7ec0f699b23 Mon Sep 17 00:00:00 2001 From: "Miguel A. Cortizo" Date: Fri, 28 Mar 2014 15:05:17 +0100 Subject: Enable input from stdin, such as pipe output --- README.rst | 16 +++++++++++++--- ngxtop/ngxtop.py | 14 ++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 099d0bf..d1750b4 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,13 @@ -=================================================== -``ngxtop`` - **real-time** metrics for nginx server -=================================================== +================================================================ +``ngxtop`` - **real-time** metrics for nginx server (and others) +================================================================ **ngxtop** parses your nginx access log and outputs useful, ``top``-like, metrics of your nginx server. So you can tell what is happening with your server in real-time. +Can be used also with Apache log files (experimental). In this case, if not log format is specified, 'combined' will be +used. If the script doesn't detect redirections properly you can force it by using the '-s' option. + Installation ------------ @@ -24,6 +27,7 @@ Usage ngxtop [options] ngxtop [options] (print|top|avg|sum) ngxtop info + tail -f /var/log/apache2/access.log | ngxtop [-s] Options: -l , --access-log access log file to parse. @@ -44,6 +48,12 @@ Usage -h, --help print this help message. --version print version information. + Advanced / experimental options: + -c , --config allow ngxtop to parse nginx config file for log format and location. + -i , --filter filter in, records satisfied given expression are processed. + -p , --pre-filter in-filter expression to check in pre-parsing phase. + -s, --from-stdin read lines from stdin. + Samples ------- diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 17af878..12a84c3 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -29,6 +29,7 @@ Options: -c , --config allow ngxtop to parse nginx config file for log format and location. -i , --filter filter in, records satisfied given expression are processed. -p , --pre-filter in-filter expression to check in pre-parsing phase. + -s, --from-stdin read lines from stdin. Examples: All examples read nginx config file for access log location and format. @@ -377,6 +378,8 @@ def build_source(access_log, arguments): # constructing log source if arguments['--no-follow']: lines = open(access_log) + elif (arguments['--from-stdin'] or not sys.stdin.isatty()): + lines = sys.stdin else: lines = follow(access_log) return lines @@ -403,10 +406,13 @@ def process(arguments): access_log = arguments['--access-log'] log_format = arguments['--log-format'] if access_log is None or log_format is None: - config = arguments['--config'] - if config is None: - config = get_nginx_conf_path() - access_log, log_format = extract_nginx_conf(config, access_log) + if not (arguments['--from-stdin'] or sys.stdin.isatty()): + config = arguments['--config'] + if config is None: + config = get_nginx_conf_path() + access_log, log_format = extract_nginx_conf(config, access_log) + else: + log_format = 'combined' else: config = None logging.info('access_log: %s', access_log) -- cgit v1.2.3 From 13cc9df657c01ce66b30e21fcbb5e1de9ea73d75 Mon Sep 17 00:00:00 2001 From: "Miguel A. Cortizo" Date: Fri, 28 Mar 2014 15:15:22 +0100 Subject: Added option for database dump --- README.rst | 1 + ngxtop/ngxtop.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/README.rst b/README.rst index d1750b4..199c055 100644 --- a/README.rst +++ b/README.rst @@ -53,6 +53,7 @@ Usage -i , --filter filter in, records satisfied given expression are processed. -p , --pre-filter in-filter expression to check in pre-parsing phase. -s, --from-stdin read lines from stdin. + -b, --db-dump dump database to disk when finished Samples ------- diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 12a84c3..679f44c 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -30,6 +30,7 @@ Options: -i , --filter filter in, records satisfied given expression are processed. -p , --pre-filter in-filter expression to check in pre-parsing phase. -s, --from-stdin read lines from stdin. + -b, --db-dump dump database to disk Examples: All examples read nginx config file for access log location and format. @@ -62,6 +63,7 @@ import sqlite3 import subprocess import threading import time +from datetime import date import sys try: @@ -111,6 +113,12 @@ DEFAULT_QUERIES = [ DEFAULT_FIELDS = set(['status_type', 'bytes_sent']) +# ============================= +# Global variable for dbdump +# ============================= +processor = None + + # ==================== # Nginx utilities # ==================== @@ -403,6 +411,7 @@ def build_reporter(processor, arguments): def process(arguments): + global processor access_log = arguments['--access-log'] log_format = arguments['--log-format'] if access_log is None or log_format is None: @@ -437,6 +446,21 @@ def process(arguments): logging.info('Processed %d lines in %.3f seconds, %.2f lines/sec.', total, duration, total / duration) +# ================ +# Database dump +# ================ +def dbdump(): + """ + *experimental* if requested, database is dumped to a file when script is interrupted from keyboard + Filename is composed from current date and process id + """ + dbfile = "{}_{}.sql".format(date.today().strftime("%Y%m%d"), os.getpid()) + logging.info("Database dump: %s", dbfile) + with open(dbfile, 'w') as f: + for line in processor.conn.iterdump(): + f.write('%s\n' % line) + + def main(): args = docopt(__doc__, version='xstat 0.1') @@ -451,6 +475,8 @@ def main(): try: process(args) except KeyboardInterrupt: + if args['--db-dump']: + dbdump() sys.exit(0) -- cgit v1.2.3 From 247a8674fc0cb4dfd38ad4f076d05210d50f110b Mon Sep 17 00:00:00 2001 From: "Miguel A. Cortizo" Date: Sun, 30 Mar 2014 13:08:48 +0200 Subject: Improve detection of stdin input and add example, add 'common' format constant, add available variables for filters to doc --- ngxtop/ngxtop.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 679f44c..e8a2e69 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -53,6 +53,14 @@ Examples: Average body bytes sent of 200 responses of requested path begin with 'foo': $ ngxtop avg bytes_sent --filter 'status == 200 and request_path.startswith("foo")' + + Analyze output from remote machine using 'common' log format + $ ssh remote_machine tail -f /var/log/apache2/access.log | ngxtop -f common + +Available variables for filters: + remote_addr, remote_user, time_local, request, status, body_bytes_sent, http_referer, http_user_agent + (if you use 'common' log format, maybe you have http_x_forwarded_for instead of http_user_agent) + """ from __future__ import print_function from contextlib import closing @@ -80,6 +88,9 @@ REGEX_LOG_FORMAT_VARIABLE = r'\$([a-z0-9\_]+)' LOG_FORMAT_COMBINED = '$remote_addr - $remote_user [$time_local] ' \ '"$request" $status $body_bytes_sent ' \ '"$http_referer" "$http_user_agent"' +LOG_FORMAT_COMMON = '$remote_addr - $remote_user [$time_local] ' \ + '"$request" $status $body_bytes_sent ' \ + '"$http_x_forwarded_for"' DEFAULT_QUERIES = [ ('Summary:', @@ -177,6 +188,8 @@ def build_pattern(log_format): """ if log_format == 'combined': return build_pattern(LOG_FORMAT_COMBINED) + elif log_format == 'common': + return build_pattern(LOG_FORMAT_COMMON) pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format) pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) return re.compile(pattern) @@ -185,6 +198,8 @@ def build_pattern(log_format): def extract_variables(log_format): if log_format == 'combined': log_format = LOG_FORMAT_COMBINED + elif log_format == 'common': + log_format = LOG_FORMAT_COMMON for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format): yield match @@ -384,10 +399,10 @@ def build_processor(arguments): def build_source(access_log, arguments): # constructing log source - if arguments['--no-follow']: - lines = open(access_log) - elif (arguments['--from-stdin'] or not sys.stdin.isatty()): + if (access_log == 'stdin'): lines = sys.stdin + elif arguments['--no-follow']: + lines = open(access_log) else: lines = follow(access_log) return lines @@ -414,16 +429,21 @@ def process(arguments): global processor access_log = arguments['--access-log'] log_format = arguments['--log-format'] - if access_log is None or log_format is None: - if not (arguments['--from-stdin'] or sys.stdin.isatty()): + if not access_log and (arguments['--from-stdin'] or not sys.stdin.isatty()): + access_log = 'stdin' + else: + if access_log is None or log_format is None: config = arguments['--config'] if config is None: config = get_nginx_conf_path() access_log, log_format = extract_nginx_conf(config, access_log) else: - log_format = 'combined' - else: - config = None + config = None + + # Maybe nginx is not installed, so we'll fix a default log format if not defined here + if log_format is None: + log_format = 'combined' + logging.info('access_log: %s', access_log) logging.info('log_format: %s', log_format) -- cgit v1.2.3 From c62e0e00435579141b9f8738f6db76b4accab928 Mon Sep 17 00:00:00 2001 From: "Miguel A. Cortizo" Date: Mon, 31 Mar 2014 20:31:47 +0200 Subject: Remove -s option, as it is not really needed --- ngxtop/ngxtop.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index e8a2e69..3355891 100755 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -29,7 +29,6 @@ Options: -c , --config allow ngxtop to parse nginx config file for log format and location. -i , --filter filter in, records satisfied given expression are processed. -p , --pre-filter in-filter expression to check in pre-parsing phase. - -s, --from-stdin read lines from stdin. -b, --db-dump dump database to disk Examples: @@ -429,7 +428,7 @@ def process(arguments): global processor access_log = arguments['--access-log'] log_format = arguments['--log-format'] - if not access_log and (arguments['--from-stdin'] or not sys.stdin.isatty()): + if not access_log and not sys.stdin.isatty(): access_log = 'stdin' else: if access_log is None or log_format is None: -- cgit v1.2.3 From 1c200d51fbae7824a30159714669146d6b214210 Mon Sep 17 00:00:00 2001 From: "Miguel A. Cortizo" Date: Mon, 31 Mar 2014 22:17:03 +0200 Subject: Update README --- README.rst | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 199c055..ecb4904 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ **ngxtop** parses your nginx access log and outputs useful, ``top``-like, metrics of your nginx server. So you can tell what is happening with your server in real-time. -Can be used also with Apache log files (experimental). In this case, if not log format is specified, 'combined' will be -used. If the script doesn't detect redirections properly you can force it by using the '-s' option. +Can read from stdin (experimental), useful for remote log files. In this case, if not log +format is specified, 'combined' will be used. Installation ------------ @@ -27,7 +27,6 @@ Usage ngxtop [options] ngxtop [options] (print|top|avg|sum) ngxtop info - tail -f /var/log/apache2/access.log | ngxtop [-s] Options: -l , --access-log access log file to parse. @@ -52,7 +51,6 @@ Usage -c , --config allow ngxtop to parse nginx config file for log format and location. -i , --filter filter in, records satisfied given expression are processed. -p , --pre-filter in-filter expression to check in pre-parsing phase. - -s, --from-stdin read lines from stdin. -b, --db-dump dump database to disk when finished Samples @@ -120,3 +118,30 @@ List 4xx or 5xx responses together with HTTP referer |-----------+----------+----------------| | - | 400 | - | +Output from remote server using log-format option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + $ ssh user@remote_server tail -f /var/log/apache2/access.log | ngxtop -f common + running for 20 seconds, 1068 records processed: 53.01 req/sec + + Summary: + | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |---------+------------------+-------+-------+-------+-------| + | 1068 | 28026.763 | 1029 | 20 | 19 | 0 | + + Detailed: + | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |------------------------------------------+---------+------------------+-------+-------+-------+-------| + | /xxxxxxxxxx | 199 | 55150.402 | 199 | 0 | 0 | 0 | + | /xxxxxxxx/xxxxx | 167 | 47591.826 | 167 | 0 | 0 | 0 | + | /xxxxxxxxxxxxx/xxxxxx | 25 | 7432.200 | 25 | 0 | 0 | 0 | + | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxxx | 22 | 698.727 | 22 | 0 | 0 | 0 | + | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxx | 19 | 7431.632 | 19 | 0 | 0 | 0 | + | /xxxxx/xxxxx/ | 18 | 7840.889 | 18 | 0 | 0 | 0 | + | /xxxxxxxx/xxxxxxxxxxxxxxxxx | 15 | 7356.000 | 15 | 0 | 0 | 0 | + | /xxxxxxxxxxx/xxxxxxxx | 15 | 9978.800 | 15 | 0 | 0 | 0 | + | /xxxxx/ | 14 | 0.000 | 0 | 14 | 0 | 0 | + | /xxxxxxxxxx/xxxxxxxx/xxxxx | 13 | 20530.154 | 13 | 0 | 0 | 0 | + -- cgit v1.2.3