trackDirFileChanges.py

#! /usr/bin/python

import json, subprocess
from argparse import ArgumentParser
from os import walk
from os.path import join, getsize
from datetime import datetime

parser = ArgumentParser(description="Tracks any changes in a specified directory. Additions, deletions,\n \
                                     changes of files and subdirs are tracked and recorded in a log file.\n \
                                     If user-defined thresholds are exceeded an alert is also created.\n")
parser.add_argument("-d","--dir", dest="dir", required=True, help="The directory monitored (required parameter)")
parser.add_argument("-s","--sizeabs", dest="size_abs", default=30, type=float, help="Number of MB of change in size to trigger an alert (default=30)")
parser.add_argument("-r","--sizerel", dest="size_rel", default=0.05, type=float, help="Fraction of change in size to trigger an alert (default=0.05)")
parser.add_argument("-n","--numabs", dest="num_abs", default=50, type=int, help="Number files+dirs that need to be added/deleted to trigger an alert (default=50)")
parser.add_argument("-q","--numrel", dest="num_rel", default=0.05, type=float, help="Fraction of files+dirs that need to be added/deleted to trigger an alert (default=0.05)")
parser.add_argument("-l","--logdir", dest="log_dir", default="logs/", help="The directory where log files and status information is kept (default logs/)")
parser.add_argument("--schedule", dest="daySchedule", default="",  help="Defines a series of times to run the script. The argument is a continuous string with no spaces, and times are comma-separated given in this form:  HH:MM,HH:MM,.. (NOT currently implemented)")
parser.add_argument("--persistentAlert", action='store_true',  help="If this flag is set then the alert to the user is a foreground window that requires pressing OK to dismiss")

args = parser.parse_args()

'''
In order to find all the changes done to a folder we have to keep a detailed snapshop of the folder structure.
This means we need to know all directories and files the main directory contains (along with individual item sizes) 
as well as the same information for all subdirs (recursively). Note that we only rely on file size to determine that
a file has changed or not, not on a hash generated by the file. This is acceptable because the main application of
this script is to track size changes. The script is designed to be run periodically as an agent/daemon, and thus 
should function after shut downs and restarts. Hence we use two files to store needed information. 
One is a .json file recording the last state snapshop:  <log_dir>/Track<dir path>.json
The other is a log of the changes we find over time:    <log_dir>/Track<dir path>changes.log
'''

class Tracker:

    def __init__(self, root, log_dir=''):
        
        self.root = root # the directory that we will monitor and track changes
        self.log_dir = log_dir # the directory where the logs and json files will be kept

        self.previous_state = self.readPrevState()  # a structure to hold all dir and file info of our previous snapshot
        self.current_state = {}  # a structure to hold all dir and file info of the current state
        
        self.added_dirs = []    # a list of directories added along with file and subdir info
        self.deleted_dirs = []  # a list of directories deleted along with the total size and number of files in them
        self.added_files = {}   # a dictionary of files added. filename is the key, size is the value
        self.deleted_files = {} # a dictionary of files deleted. filename is the key, size is the value
        self.changed_files = {} # a dictionary of files changed. filename is the key, a tuple (old_size, new_size) is the value 

        self.added_total_size = 0   # The total size in bytes of all files added
        self.deleted_total_size = 0 # The total size in bytes of all files deleted
        self.changed_total_size = 0 # The total size in bytes of all files changes
        self.added_total_num = 0   # The total number of all files added
        self.deleted_total_num = 0 # The total number of all files deleted
        self.changed_total_num = 0 # The total number of all files changed

        self.current_total_size = 0 # The total size of all files inside the tracked dir
        self.current_total_file_num = 0 # The total number of all files inside the tracked dir
        self.current_total_dir_num = 0 # The total number of all subdirs inside the tracked dir
    
        self.summary = ''  # A string to contain a summary of the additions/deletions/changes

    '''
    Read the previous state of the root directory from a special file. 
    If the file does not exist or is corrupted, return an empty dict
    '''
    def readPrevState(self):

        prev_state_filename = 'track{}.json'.format(self.root.replace('/','_'))
        try:
            with open(join(self.log_dir, prev_state_filename)) as state_file:
                return json.load(state_file) # note: strings are returned as unicode strings

        except (IOError, ValueError):
            return {}

    '''
    Write the current state of the root directory to the special file. Overwrite file.
    Note that all strings will be written as unicode. Non-ascii characters will be written as codepoints in ascii 
    (e.g., the letter alpha will be written as u'\u03b8'). We could write them as non-ascii utf-8 characters by 
    using the parameter ensure_ascii=False, but when reading this file the json.load() method will return unicode 
    strings anyway. Moreover, it is best practice to work with unicode strings, instead of a particular encoding.
    '''
    def writeCurrentState(self):
        prev_state_filename = 'track{}.json'.format(self.root.replace('/','_'))
        with open(join(self.log_dir, prev_state_filename), 'w') as state_file:
            json.dump(self.current_state, state_file, encoding='utf-8', separators=(',', ':'))
            

    '''
    A function to return the total size of files and number of files of a deleted directory.
    This implies that we are searching inside previous_state. The pathname should be a unicode string.
    '''
    def getSizeAndNum(self, pathname):

        # Check if pathname is unicode. Left unchecked, the rest of the function's code will silently fail
        # to find a regular string  with non-ascii chars inside the previous_state dict.
        if type(pathname) is not unicode: 
            raise ValueError('pathname needs to be a unicode string, you have passed:', type(pathname))

        if pathname not in self.previous_state: return (0, 0)

        dirs, files_with_sizes = self.previous_state[pathname]
        total_size = sum(files_with_sizes.values())
        total_num = len(files_with_sizes)
        # Recursively visit all subdirs
        for d in dirs:
            subdir_size, subdir_num = self.getSizeAndNum(join(pathname, d))
            total_size += subdir_size
            total_num += subdir_num

        return (total_size, total_num)


    '''
    A function to find all additions, deletions, and changes in all files and subdirs
    '''
    def findChanges(self):

        # reset the current state
        self.current_state = {}

        for path, curr_dirs, curr_files in walk(self.root):

            # convert the path to Unicode, keep the original path to be used in finding filesizes
            path_unicode = path.decode('utf-8')

            # convert dir names in Unicode
            curr_dirs = [d.decode('utf-8') for d in curr_dirs]

            # Find the sizes of all the files in this directory, and create a dictionary. 
            # Use a dictionary comprehension. Store filenames in Unicode
            curr_files_and_sizes = {fname.decode('utf-8'): getsize(join(path, fname)) for fname in curr_files}

            # update the total counts
            self.current_total_size += sum(curr_files_and_sizes.values())
            self.current_total_file_num += len(curr_files_and_sizes)
            self.current_total_dir_num += 1 

            # update the current state
            self.current_state[path_unicode] = [curr_dirs, curr_files_and_sizes]

            # check is this path existed in the previous state
            if path_unicode in self.previous_state:
                prev_dirs, prev_files_and_sizes = self.previous_state[path_unicode]

                # check if sub dirs are the same
                if prev_dirs != curr_dirs:
                    # we only want to get the deleted dirs here, since the 
                    # added ones will appear in the path as we walk the tree
                    deleted_dirs_list = list(set(prev_dirs)-set(curr_dirs))
                    for dname in deleted_dirs_list:
                        # use a recursive function to get the total size and file number in the deleted dir
                        dir_size , dir_file_num = self.getSizeAndNum(join(path_unicode, dname))
                        self.deleted_dirs.append([join(path_unicode, dname), dir_size, dir_file_num])


                # check if files are the same                
                if prev_files_and_sizes != curr_files_and_sizes:
                    # find the differences
                    deleted_files_set = set(prev_files_and_sizes) - set(curr_files_and_sizes)
                    for f in deleted_files_set:
                        self.deleted_files[join(path_unicode,f)] = prev_files_and_sizes[f]

                    added_files_set = set(curr_files_and_sizes) - set(prev_files_and_sizes)
                    for f in added_files_set:
                        self.added_files[join(path_unicode,f)] = curr_files_and_sizes[f]

                    common_files_set = set(curr_files_and_sizes) - added_files_set
                    for f in common_files_set:
                        if prev_files_and_sizes[f] != curr_files_and_sizes[f]:
                            self.changed_files[join(path_unicode,f)] = (prev_files_and_sizes[f], curr_files_and_sizes[f])

            else:
                self.added_dirs.append([path_unicode, curr_dirs, curr_files_and_sizes])

        # Finally calculate various aggregates:

        # sum up the deleted files sizes, and count the files
        self.deleted_total_size = sum(self.deleted_files.values())
        self.deleted_total_num = len(self.deleted_files)
        # add the total sizes and numbers of deleted directories
        self.deleted_total_size += sum([size for p, size, num in self.deleted_dirs])
        self.deleted_total_num  += sum([num for p, size, num in self.deleted_dirs])

        # sum up the added files sizes, and count the files
        self.added_total_size = sum(self.added_files.values())
        self.added_total_num = len(self.added_files)
        # add the total sizes and numbers of added directories
        self.added_total_size += sum([ sum(f.values()) for p, d, f in self.added_dirs])
        self.added_total_num += sum([ len(f) for p, d, f in self.added_dirs])

        # for changed files sum up the differences between old and new sizes for every changed file
        self.changed_total_size = sum(map(lambda (old_size, new_size):abs(old_size-new_size), self.changed_files.values()))
        self.changed_total_num = len(self.changed_files)

       
    '''
    Write all changes to the log file. Record the timestamp, summary of changes, and detailed list of changes.
    If no changes happened, only the timestamp is recorded. If no previous snapshot is present we record 
    summary information of the new directory tracked. 
    '''
    def writeChanges(self):

        change_log_filename = 'track{}changes.log'.format(self.root.replace('/','_') )
        with open(join(self.log_dir, change_log_filename), 'ab') as log_file: 
            # write a timestamp
            log_file.write('\n----------------  {}  ----------------\n'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            # check to see if we had no previous state, in which case we do not want to record all additions
            if not self.previous_state:
                log_file.write('New directory tracked.\n(Or .json file holding the previous state was deleted or corrupted)\n')
                log_file.write('{} dirs and {} files occupying {}\n'.format(self.current_total_dir_num,
                                                                            self.current_total_file_num,
                                                                            self.humanReadableSize(self.current_total_size)))    
            else:
                # provide a summary of the changes
                report_for_added = ''; report_for_deleted =''; report_for_changed = ''
                if self.added_total_size > 0: 
                    report_for_added = 'Added:   {} files totalling {}\n'.format(self.added_total_num, self.humanReadableSize(self.added_total_size))
                if self.deleted_total_size > 0: 
                    report_for_deleted = 'Deleted: {} files totalling {}\n'.format(self.deleted_total_num, self.humanReadableSize(self.deleted_total_size))
                if self.changed_total_size > 0: 
                    report_for_changed = 'Changed: {} files changed by {}\n'.format(self.changed_total_num, self.humanReadableSize(self.changed_total_size))

                self.summary = report_for_added + report_for_deleted + report_for_changed
                log_file.write(self.summary)

                # provide a detailed list of all changes
                # start with deleted dirs, 
                for d in self.deleted_dirs:
                    path, size, files_num = d
                    log_file.write('Deleted dir: {} contained {} in {} files\n'.format(path.encode('utf-8'), self.humanReadableSize(size), files_num))

                # then added dirs    
                for d in self.added_dirs:
                    path, dirs, files_with_sizes = d
                    size = sum(files_with_sizes.values()) 
                    files_num = len(files_with_sizes)
                    log_file.write('Added dir: {}, contains {} in {} files\n'.format(path.encode('utf-8'), self.humanReadableSize(size), files_num))

                # continuing with deleted/added/changed files
                for f, size in self.deleted_files.iteritems():
                    log_file.write('Deleted file: {} was {} bytes\n'.format(f.encode('utf-8'), size))

                for f, size in self.added_files.iteritems():
                    log_file.write('Added file: {} is {} bytes\n'.format(f.encode('utf-8'), size))     

                for f, (old_size, new_size) in self.changed_files.iteritems():
                    log_file.write('Changed file: {} from {} to {} bytes\n'.format(f.encode('utf-8'), old_size, new_size)) 


    '''
    A function to decide whether to alert the user and what kind of alert to present (persistent, or notification)
    Note that size_abs is expressed in bytes, not MB
    '''
    def alertUser(self, size_abs, size_rel, num_abs, num_rel, persistentAlert):

        if not self.previous_state: return 

        size = self.added_total_size + self.deleted_total_size + self.changed_total_size
        num  = self.added_total_num  + self.deleted_total_num  + self.changed_total_num
        
        if (size > size_abs or 
            size > size_rel * self.current_total_size or 
            num  > num_abs or 
            num  > num_rel * self.current_total_file_num):

            applescript = 'display notification "{}" with title "Boulis Directory Tracker"'.format(self.summary)
            alt_applescript = 'display dialog "{}" with title "Boulis Directory Tracker" with icon caution buttons {{"OK"}}'.format(self.summary)

            # check whether we need a persistent window or just a notification, and run the appropriate applescript
            if persistentAlert:
                subprocess.call("osascript -e '{}'".format(alt_applescript), shell=True)
            else:
                subprocess.call("osascript -e '{}'".format(applescript), shell=True)

    '''
    A function to print file sizes in a more human readable form (using KB, MB, GB)
    It also uses valiable decimal precision for different sizes. 0 decimals for bytes and KB, 2 for MB, 3 for GB
    '''
    def humanReadableSize(self, num):

        for unit, decimals_printed in zip(['bytes','KB','MB', 'GB'], [0, 0, 2, 3]):
            if abs(num) < 1024.0:
                if decimals_printed == 0:
                    return '{} {}'.format(int(round(num)), unit)
                else:
                    # create the format string to fit the desired decimal precision
                    # use {{ and }} to escape the special characters { and }. for decimals= 3 this will return '{:.3f} {}'
                    format_string = '{{:.{}f}} {{}}'.format(decimals_printed) 
                    return format_string.format(num, unit)
            num /= 1024.0
        # if the num is bigger than 1024 after all divisions, just use the larger unit     
        return '{.3f} GB'.format(num)



def singleRun(root, log_dir, size_abs, size_rel, num_abs, num_rel, persistentAlert):

    t = Tracker(root, log_dir)
    t.findChanges()
    t.writeChanges()
    t.writeCurrentState()
    t.alertUser(size_abs, size_rel, num_abs, num_rel, persistentAlert)


def main_loop():
    singleRun(args.dir, args.log_dir, args.size_abs *1024*1024, args.size_rel, args.num_abs, args.num_rel, args.persistentAlert)
    # There are provisions to include multiple scheduled runs in the future, hence the parameter --schedule. 
    # Not needed for the Mac OSX environment

if __name__ == '__main__':
     main_loop()