path: root/peekaboo/toolbox/cuckoo.py

###############################################################################
#                                                                             #
# Peekaboo Extended Email Attachment Behavior Observation Owl                 #
#                                                                             #
# toolbox/                                                                    #
#         cuckoo.py                                                           #
###############################################################################
#                                                                             #
# Copyright (C) 2016-2019  science + computing ag                             #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or (at       #
# your option) any later version.                                             #
#                                                                             #
# This program is distributed in the hope that it will be useful, but         #
# WITHOUT ANY WARRANTY; without even the implied warranty of                  #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
# General Public License for more details.                                    #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################


import re
import os
import locale
import logging
import json
import subprocess
import requests
import random
from twisted.internet import protocol, reactor, process
from time import sleep
from peekaboo.exceptions import CuckooAnalysisFailedException


logger = logging.getLogger(__name__)


class Cuckoo:
    """ Parent class, defines interface to Cuckoo. """
    def __init__(self, job_queue):
        self.job_queue = job_queue
        self.shutdown_requested = False
        self.running_jobs = {}

    def resubmit_with_report(self, job_id):
        logger.debug("Analysis done for task #%d" % job_id)

        # thread-safe, no locking required, revisit if splitting into
        # multiple operations
        sample = self.running_jobs.pop(job_id, None)
        if sample is None:
            logger.debug('No sample found for job ID %d', job_id)
            return None

        logger.debug('Requesting Cuckoo report for sample %s', sample)
        report = self.get_report(job_id)

        # do not register the report with the sample if we were unable to
        # get it because e.g. it was corrupted or the API connection
        # failed. This will cause the sample to be resubmitted to Cuckoo
        # upon the next try to access the report.
        # TODO: This can cause an endless loop.
        if report is not None:
            reportobj = CuckooReport(report)
            sample.register_cuckoo_report(reportobj)

        self.job_queue.submit(sample, self.__class__)

    def shut_down(self):
        self.shutdown_requested = True

    def reap_children(self):
        pass

    def get_report(self, job_id):
        """ Extract the report of a finished analysis from Cuckoo. To be
        overridden by derived classes for actual implementation. """
        raise NotImplementedError

class CuckooEmbed(Cuckoo):
    """ Runs and interfaces with Cuckoo in IPC. """
    def __init__(self, job_queue, cuckoo_exec, cuckoo_submit,
                 cuckoo_storage, interpreter=None):
        Cuckoo.__init__(self, job_queue)
        self.interpreter = interpreter
        self.cuckoo_exec = cuckoo_exec
        self.cuckoo_submit = cuckoo_submit
        self.cuckoo_storage = cuckoo_storage
        self.exit_code = 0

        # process output to get job ID
        patterns = (
            # Example: Success: File "/var/lib/peekaboo/.bashrc" added as task with ID #4
            "Success.*: File .* added as task with ID #([0-9]*)",
            "added as task with ID ([0-9]*)",
        )
        self.job_id_patterns = [re.compile(pattern) for pattern in patterns]
    
    def submit(self, sample):
        """
        Submit a file or directory to Cuckoo for behavioural analysis.
            
        @param sample: Sample object to analyse.
        @return: The job ID used by Cuckoo to identify this analysis task.
        """
        try:
            # cuckoo_submit is a list, make a copy as to not modify the
            # original value
            proc = self.cuckoo_submit.split(' ') + [sample.submit_path]

            # universal_newlines opens channels to child in text mode and
            # returns strings instead of bytes in return which we do to avoid
            # the need to handle decoding ourselves
            p = subprocess.Popen(proc,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 universal_newlines=True)
            p.wait()
        except Exception as e:
            raise CuckooAnalysisFailedException(e)
        
        if not p.returncode == 0:
            raise CuckooAnalysisFailedException('cuckoo submit returned a non-zero return code.')
        else:
            out, err = p.communicate()
            logger.debug("cuckoo submit STDOUT: %s", out)
            logger.debug("cuckoo submit STDERR: %s", err)

            match = None
            pattern_no = 0
            for pattern in self.job_id_patterns:
                match = re.search(pattern, out)
                if match is not None:
                    logger.debug('Pattern %d matched.' % pattern_no)
                    break

                pattern_no += 1
            
            if match is not None:
                job_id = int(match.group(1))
                # thread-safe, no locking required, revisit if splitting into
                # multiple operations
                self.running_jobs[job_id] = sample
                return job_id

            raise CuckooAnalysisFailedException(
                'Unable to extract job ID from given string %s' % out)

    def get_report(self, job_id):
        path = os.path.join(self.cuckoo_storage,
                'analyses/%d/reports/report.json' % job_id)

        if not os.path.isfile(path):
            raise OSError('Cuckoo report not found at %s.' % path)

        logger.debug('Accessing Cuckoo report for task %d at %s ' %
                (job_id, path))

        report = None
        with open(path) as data:
            try:
                report = json.load(data)
            except ValueError as e:
                logger.exception(e)

        return report

    def do(self):
        """ Run Cuckoo sandbox, parse log output, and report back of Peekaboo. """
        command = self.cuckoo_exec.split(' ')

        # allow for injecting a custom interpreter which we use to run cuckoo
        # with python -u for unbuffered standard output
        if self.interpreter:
            command = self.interpreter.split(' ') + command

        reactor.spawnProcess(CuckooServer(self), command[0], command)

        # do not install twisted's signal handlers because it will screw with
        # our logic (install a handler for SIGTERM and SIGCHLD but not for
        # SIGINT). Instead do what their SIGCHLD handler would do and call the
        # global process reaper.
        reactor.run(installSignalHandlers = False)
        process.reapAllProcesses()
        return self.exit_code

    def shut_down(self, exit_code = 0):
        """ Signal handler callback but in this instance also used as callback
        for protocol to ask us to shut down if anything adverse happens to the
        child """
        # the reactor doesn't like it to be stopped more than once and catching
        # the resulting ReactorNotRunning exception is foiled by the fact that
        # sigTerm defers the call through a queue into another thread which
        # insists on logging it
        if not self.shutdown_requested:
            reactor.sigTerm(0)

        self.shutdown_requested = True
        self.exit_code = exit_code

    def reap_children(self):
        """ Since we only have one child, SIGCHLD will cause us to shut down
        and we reap all child processes on shutdown. This method is therefore
        (currently) intentionally a no-op. """
        pass

class CuckooApi(Cuckoo):
    """ Interfaces with a Cuckoo installation via its REST API. """
    def __init__(self, job_queue, url="http://localhost:8090", poll_interval=5):
        Cuckoo.__init__(self, job_queue)
        self.url = url
        self.poll_interval = poll_interval
        self.reported = self.__status()["tasks"]["reported"]
        logger.info("Connection to Cuckoo seems to work, %i reported tasks seen", self.reported)
    
    def __get(self, url, method="get", files=""):
        r = ""
        logger.debug("Requesting %s, method %s" % (url, method))
        
        # try 3 times to get a successfull response
        for retry in range(0, 3):
            try:
                if method == "get":
                    r = requests.get("%s/%s" % (self.url, url))
                elif method == "post":
                    r = requests.post("%s/%s" % (self.url, url), files=files)
                else:
                    break
                if r.status_code != 200:
                    continue
                else:
                    return r.json()
            except requests.exceptions.Timeout as e:
                # Maybe set up for a retry, or continue in a retry loop
                print(e)
                if e and retry >= 2:
                    raise e
            except requests.exceptions.TooManyRedirects as e:
                # Tell the user their URL was bad and try a different one
                print(e)
                if e and retry >= 2:
                    raise e
            except requests.exceptions.RequestException as e:
                # catastrophic error. bail.
                print(e)
                if e and retry >= 2:
                    raise e
        return None
    
    def __status(self):
        return self.__get("cuckoo/status")
    
    def submit(self, sample):
        path = sample.submit_path
        filename = os.path.basename(path)
        files = {"file": (filename, open(path, 'rb'))}
        response = self.__get("tasks/create/file", method="post", files=files)
        
        task_id = response["task_id"]
        if task_id > 0:
            # thread-safe, no locking required, revisit if splitting into
            # multiple operations
            self.running_jobs[task_id] = sample
            return task_id
        raise CuckooAnalysisFailedException(
            'Unable to extract job ID from given string %s' % response)

    def get_report(self, job_id):
        logger.debug("Report from Cuckoo API requested, job_id = %d" % job_id)
        return self.__get("tasks/report/%d" % job_id)
    
    def do(self):
        # do the polling for finished jobs
        # record analysis count and call status over and over again
        # logger ......
        
        limit = 1000000
        offset = self.__status()["tasks"]["total"]
        
        while not self.shutdown_requested:
            cuckoo_tasks_list = None
            try:
                cuckoo_tasks_list = self.__get("tasks/list/%i/%i" % (limit, offset))
            except Exception as e:
                logger.warn('Unable to communicate with Cuckoo API: %s' % e)
                pass

            #maxJobID = cuckoo_tasks_list[-1]["id"]
            
            first = True
            if cuckoo_tasks_list:
                for j in cuckoo_tasks_list["tasks"]:
                    if j["status"] == "reported":
                        job_id = j["id"]
                        self.resubmit_with_report(job_id)
            #self.reported = reported
            sleep(float(self.poll_interval))

        return 0

class CuckooServer(protocol.ProcessProtocol):