path: root/peekaboo/toolbox/cuckoo.py

###############################################################################
#                                                                             #
# Peekaboo Extended Email Attachment Behavior Observation Owl                 #
#                                                                             #
# toolbox/                                                                    #
#         cuckoo.py                                                           #
###############################################################################
#                                                                             #
# Copyright (C) 2016-2019  science + computing ag                             #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or (at       #
# your option) any later version.                                             #
#                                                                             #
# This program is distributed in the hope that it will be useful, but         #
# WITHOUT ANY WARRANTY; without even the implied warranty of                  #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# General Public License for more details.                                    #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################


from future.builtins import super  # pylint: disable=wrong-import-order

import re
import os
import locale
import logging
import json
import subprocess
import random
import requests
import urllib3.util.retry

from threading import RLock, Event
from time import sleep
from twisted.internet import protocol, reactor, process

from peekaboo.exceptions import CuckooSubmitFailedException


logger = logging.getLogger(__name__)


class Cuckoo(object):
    """ Parent class, defines interface to Cuckoo. """
    def __init__(self, job_queue):
        self.job_queue = job_queue
        self.shutdown_requested = Event()
        self.shutdown_requested.clear()
        self.running_jobs = {}
        self.running_jobs_lock = RLock()

    def register_running_job(self, job_id, sample):
        """ Register a job as running. Detect if another sample has already
        been registered with the same job ID which obviously must never happen
        because it corrupts our internal housekeeping. Guarded by a lock
        because multiple worker threads will call this routine and check for
        collision and update of job log might otherwise race each other.

        @param job_id: ID of the job to register as running.
        @type job_id: int
        @param sample: Sample object to associate with this job ID
        @type sample: Sample

        @returns: None
        @raises: CuckooSubmitFailedException on job id collision """
        with self.running_jobs_lock:
            if (job_id in self.running_jobs and
                    self.running_jobs[job_id] is not sample):
                raise CuckooSubmitFailedException(
                    'A job with ID %d is already registered as running '
                    'for sample %s' % (job_id, self.running_jobs[job_id]))

            self.running_jobs[job_id] = sample

    def resubmit_with_report(self, job_id):
        """ Resubmit a sample to the job queue after the report became
        available. Retrieves the report from Cuckoo.

        @param job_id: ID of job which has finished.
        @type job_id: int

        @returns: None """
        logger.debug("Analysis done for task #%d" % job_id)

        with self.running_jobs_lock:
            sample = self.running_jobs.pop(job_id, None)

        if sample is None:
            logger.debug('No sample found for job ID %d', job_id)
            return None

        logger.debug('Requesting Cuckoo report for sample %s', sample)
        report = self.get_report(job_id)

        # do not register the report with the sample if we were unable to
        # get it because e.g. it was corrupted or the API connection
        # failed. This will cause the sample to be resubmitted to Cuckoo
        # upon the next try to access the report.
        # TODO: This can cause an endless loop.
        if report is not None:
            reportobj = CuckooReport(report)
            sample.register_cuckoo_report(reportobj)

        self.job_queue.submit(sample, self.__class__)

    def shut_down(self):
        """ Request the module to shut down. """
        self.shutdown_requested.set()

    def reap_children(self):
        pass

    def get_report(self, job_id):
        """ Extract the report of a finished analysis from Cuckoo. To be
        overridden by derived classes for actual implementation. """
        raise NotImplementedError

class CuckooEmbed(Cuckoo):
    """ Runs and interfaces with Cuckoo in IPC. """
    def __init__(self, job_queue, cuckoo_exec, cuckoo_submit,
                 cuckoo_storage, interpreter=None):
        super().__init__(job_queue)
        self.interpreter = interpreter
        self.cuckoo_exec = cuckoo_exec
        self.cuckoo_submit = cuckoo_submit
        self.cuckoo_storage = cuckoo_storage
        self.exit_code = 0

        # process output to get job ID
        patterns = (
            # Example: Success: File "/var/lib/peekaboo/.bashrc" added as task with ID #4
            "Success.*: File .* added as task with ID #([0-9]*)",
            "added as task with ID ([0-9]*)",
        )
        self.job_id_patterns = [re.compile(pattern) for pattern in patterns]
    
    def submit(self, sample):
        """
        Submit a file or directory to Cuckoo for behavioural analysis.
            
        @param sample: Sample object to analyse.
        @return: The job ID used by Cuckoo to identify this analysis task.
        """
        try:
            # cuckoo_submit is a list, make a copy as to not modify the
            # original value
            proc = self.cuckoo_submit.split(' ') + [sample.submit_path]

            # universal_newlines opens channels to child in text mode and
            # returns strings instead of bytes in return which we do to avoid
            # the need to handle decoding ourselves
            p = subprocess.Popen(proc,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 universal_newlines=True)
            p.wait()
        except Exception as error:
            raise CuckooSubmitFailedException(error)
        
        if not p.returncode == 0:
            raise CuckooSubmitFailedException(
                'cuckoo submit returned a non-zero return code.')

        out, err = p.communicate()
        logger.debug("cuckoo submit STDOUT: %s", out)
        logger.debug("cuckoo submit STDERR: %s", err)

        match = None
        pattern_no = 0
        for pattern in self.job_id_patterns:
            match = re.search(pattern, out)
            if match is not None:
                logger.debug('Pattern %d matched.', pattern_no)
                break

            pattern_no += 1

        if match is not None:
            job_id = int(match.group(1))
            self.register_running_job(job_id, sample)
            return job_id

        raise CuckooSubmitFailedException(
            'Unable to extract job ID from given string %s' % out)

    def get_report(self, job_id):
        path = os.path.join(self.cuckoo_storage,
                'analyses/%d/reports/report.json' % job_id)

        if not os.path.isfile(path):
            raise OSError('Cuckoo report not found at %s.' % path)

        logger.debug('Accessing Cuckoo report for task %d at %s ' %
                (job_id, path))

        report = None
        with open(path) as data:
            try:
                report = json.load(data)
            except ValueError as e:
                logger.exception(e)

        return report

    def do(self):
        """ Run Cuckoo sandbox, parse log output, and report back of Peekaboo. """
        command = self.cuckoo_exec.split(' ')

        # allow for injecting a custom interpreter which we use to run cuckoo
        # with python -u for unbuffered standard output
        if self.interpreter:
            command = self.interpreter.split(' ') + command

        reactor.spawnProcess(CuckooServer(self), command[0], command)

        # do not install twisted's signal handlers because it will screw with
        # our logic (install a handler for SIGTERM and SIGCHLD but not for
        # SIGINT). Instead do what their SIGCHLD handler would do and call the
        # global process reaper.
        reactor.run(installSignalHandlers = False)
        process.reapAllProcesses()
        return self.exit_code

    def shut_down(self, exit_code = 0):
        """ Signal handler callback but in this instance also used as callback
        for protocol to ask us to shut down if anything adverse happens to the
        child """
        # the reactor doesn't like it to be stopped more than once and catching
        # the resulting ReactorNotRunning exception is foiled by the fact that
        # sigTerm defers the call through a queue into another thread which
        # insists on logging it
        if not self.shutdown_requested.is_set():
            reactor.sigTerm(0)

        self.shutdown_requested.set()
        self.exit_code = exit_code

    def reap_children(self):
        """ Since we only have one child, SIGCHLD will cause us to shut down
        and we reap all child processes on shutdown. This method is therefore
        (currently) intentionally a no-op. """
        pass


class WhitelistRetry(urllib3.util.retry.Retry):
    """ A Retry class which has a status code whitelist, allowing to retry all
    requests not whitelisted in a hard-core, catch-all manner. """
    def __init__(self, status_whitelist=None, **kwargs):
        super().__init__(**kwargs)
        self.status_whitelist = status_whitelist or set()

    def is_retry(self, method, status_code, has_retry_after=False):
        """ Override Retry's is_retry to introduce our status whitelist logic.
        """
        # we retry all methods so no check if method is retryable here

        if self.status_whitelist and status_code not in self.status_whitelist:
            return True

        return super().is_retry(method, status_code, has_retry_after)


class CuckooApi(Cuckoo):
    """ Interfaces with a Cuckoo installation via its REST API. """
    def __init__(self, job_queue, url="http://localhost:8090", api_token="", poll_interval=5,
                 retries=5, backoff=0.5):
        super().__init__(job_queue)
        self.url = url
        self.api_token = api_token
        self.poll_interval = poll_interval

        # urrlib3 backoff formula:
        # <backoff factor> * (2 ^ (<retry count so far> - 1))
        # with second try intentionally having no sleep,
        # e.g. with retry count==5 and backoff factor==0.5:
        # try 1: fail, sleep(0.5*2^(1-1)==0.5*2^0==0.5*1==0.5->intentionally
        #   overridden to 0)
        # try 2: fail, sleep(0.5*2^(2-1)==0.5*2^1==1)
        # try 3: fail, sleep(0.5*2^(3-1)==0.5*2^2==2)
        # try 4: fail, sleep(0.5*2^(4-1)==0.5*2^3==4)
        # try 5: fail, abort, sleep would've been 8 before try 6
        #
        # Also, use method_whitelist=False to enable POST and other methods for
        # retry which aren't by default because they're not considered
        # idempotent. We assume that with the REST API a request either
        # succeeds or fails without residual effects, making them atomic and
        # idempotent.
        #
        # And finally we retry everything but a 200 response, which admittedly
        # is a bit hard-core but serves our purposes for now.
        retry_config = WhitelistRetry(total=retries,
                                      backoff_factor=backoff,
                                      method_whitelist=False,
                                      status_whitelist=set([200]))
        retry_adapter = requests.adapters