Automating Borg Backup#

I wanted to use a proper backup solution to replace my rsync script. I decided to use BorgBackup as it seemed to suit the bill. It is a repository based system that has very strong deduplication algorithms. Essentially, you create a backup repository in a particular path and then backup folders and files to the repository.

I liked the idea of using BorgBackup. However, I wanted to automate the process. My previous rsync backup script worked with removable media consisting of various USB hard disks and regular hard disks that would be plugged into a USB dock. It would detect which drive was inserted and sync the files to it. I wanted to make sure I could do the same thing with BorgBackup.

Initially, I wrote a shell script to handle it. There was a lot of repetition in the script so I decided to rewrite it in python.

Here is the resulting script:

  1#!/usr/bin/env python3
  2#-*- coding:utf-8 -*-
  3
  4"""
  5A script to automate a borg backup.
  6
  7
  8Copyright (c) 2018 Troy Williams
  9
 10License: The MIT License (http://www.opensource.org/licenses/mit-license.php)
 11"""
 12
 13# Constants
 14__uuid__ = ''
 15__author__ = 'Troy Williams'
 16__email__ = 'troy.williams@bluebill.net'
 17__copyright__ = 'Copyright (c) 2018, Troy Williams'
 18__date__ = '2018-10-01'
 19__maintainer__ = 'Troy Williams'
 20
 21import sys
 22import os
 23import subprocess
 24import platform
 25import datetime
 26from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 27
 28# ---------------
 29# This stuff with the drives, backup folders could be moved to a configuration
 30# file...
 31
 32# removable drives base folder
 33base_media_folder = "/media/troy"
 34
 35# removable drives that we are interested in backing up to
 36backup_media = []
 37backup_media.append(os.path.join(base_media_folder, "backup1"))
 38backup_media.append(os.path.join(base_media_folder, "backup2"))
 39backup_media.append(os.path.join(base_media_folder, "backup3"))
 40backup_media.append(os.path.join(base_media_folder,
 41                                 "c5817615-37c8-4765-bbc3-955ecd426db1/troy"))
 42
 43# repositories on the media that we want to backup too.
 44# The tuple contains the path in the file system and
 45# the target folder on the media.
 46repositories = [('/home/troy', 'home_backup'),
 47                ('/home/troy/music', 'music'),
 48                ('/home/troy/pictures', 'pictures'),
 49                ('/home/troy/videos', 'videos')]
 50
 51
 52def find_first_active_drive(paths):
 53    """
 54    """
 55    for path in paths:
 56        if os.path.isdir(path):
 57            return path
 58
 59    return None
 60
 61
 62def borg_create_command(backup_folder,
 63                        repository_folder,
 64                        backup_excludes=None):
 65
 66    command = ['borg',
 67               'create',
 68               '--verbose',
 69               '--progress',
 70               '--stats',
 71               '--compression', 'auto,lzma',
 72               "{}::'{}-{:%Y-%m-%d %H:%M}'".format(repository_folder,
 73                                                   platform.node(),
 74                                                   datetime.datetime.now()),
 75               backup_folder]
 76
 77    if backup_excludes:
 78        excludes = []
 79        for exclude in backup_excludes:
 80            excludes.append('--exclude')
 81            excludes.append('{}'.format(exclude))
 82
 83            # excludes.append('--exclude {}'.format(exclude))
 84
 85        command.extend(excludes)
 86
 87    return command
 88
 89
 90def borg_prune_command(repository_folder):
 91
 92    # borg prune -v $REPOSITORY --prefix '{hostname}-'         \
 93    #     --keep-hourly=6                                      \
 94    #     --keep-daily=7                                       \
 95    #     --keep-weekly=4                                      \
 96    #     --keep-monthly=6                                     \
 97
 98    command = ['borg',
 99               'prune',
100               '-v',
101               repository_folder,
102               '--prefix',
103               '{}-'.format(platform.node()),
104               '--keep-hourly=6',
105               '--keep-daily=7',
106               '--keep-weekly=4',
107               '--keep-monthly=6']
108
109    return command
110
111
112def borg_list_command(repository_folder):
113
114    return ['borg', 'list', repository_folder]
115
116
117def get_parser():
118    """Get parser object for script xy.py."""
119
120    parser = ArgumentParser(description=__doc__,
121                            formatter_class=ArgumentDefaultsHelpFormatter)
122
123    parser.add_argument('--init',
124                        dest='init',
125                        action='store_true',
126                        help=('Create folders and initialize',
127                              ' repositories on the target media.'))
128
129    parser.add_argument('--verify',
130                        dest='verify',
131                        action='store_true',
132                        help='Verify and Validate the repositories.')
133
134    # parser.add_argument('-r', '--result',
135    #                     dest='result_file',
136    #                     help='The file to contain the',
137    #                          'combined odd and even lines.',
138    #                     metavar=')
139
140    return parser
141
142
143def main():
144    """
145    This runs the rest of the functions in this module
146    """
147
148    # get the command line arguments
149    parser = get_parser()
150    args = parser.parse_args()
151
152    target_path = find_first_active_drive(backup_media)
153    if not target_path:
154        print('Could not find a backup drive.',
155              ' Please mount a drive and try again.')
156        sys.exit(1)
157
158    # we have a path to a repository
159    print('Backing up to: {}'.format(target_path))
160    print()
161
162# -----------------------------------------
163    if args.init:
164        # check to see if the repository folders exist on the  drive
165        # if not create them and initialize a borg repository
166        # https://borgbackup.readthedocs.io/en/stable/usage/init.html
167        print('Initializing target: {}'.format(target_path))
168
169        for r in repositories:
170            backup, folder = r
171
172            repo = os.path.join(target_path, folder)
173
174            if not os.path.exists(repo):
175                os.makedirs(repo)
176
177                result = subprocess.run(['borg',
178                                         'init',
179                                         '--encryption=none',
180                                         repo])
181
182                if result.returncode != 0:
183                    sys.exit(result.returncode)
184
185        print('{} has been initialized...'.format(target_path))
186        sys.exit(0)
187
188    if args.verify:
189        # verify the repositories on the target media
190        # https://borgbackup.readthedocs.io/en/stable/usage/check.html#
191
192        for r in repositories:
193            backup, folder = r
194
195            repo = os.path.join(target_path, folder)
196            print('Verifying: {}'.format(repo))
197
198            if not os.path.exists(repo):
199                os.makedirs(repo)
200
201                result = subprocess.run(['borg',
202                                         'check',
203                                         repo])
204
205                if result.returncode != 0:
206                    sys.exit(result.returncode)
207
208        print('Verification Complete...')
209        sys.exit(0)
210
211    # -------------------------
212    # Backup the folders to the repositories
213    for r in repositories:
214        backup, folder = r
215
216        repo = os.path.join(target_path, folder)
217
218        # see if there are any exclude files associated with the backup folder
219        # they will contain a list of folders and matches that we don't want to
220        # backup. If the file exists, it will be read in and processed
221        excludes_file = '{}.borg.excludes'.format(os.path.basename(backup))
222
223        excludes = None
224        try:
225            with open(excludes_file) as f:
226                excludes = [line.strip() for line in f]
227
228        except IOError:
229            pass
230
231        print()
232        print('Backing up: {}'.format(backup))
233        print('Repo:       {}'.format(repo))
234
235        if os.path.isdir(repo):
236            command = borg_create_command(backup,
237                                          repo,
238                                          excludes)
239
240            result = subprocess.run(command)
241
242            if result.returncode != 0:
243                sys.exit(result.returncode)
244
245            command = borg_prune_command(repo)
246            result = subprocess.run(command)
247
248            command = borg_list_command(repo)
249            result = subprocess.run(command)
250
251        else:
252            print("Repo Folder Doesn't exit. Skipping...")
253            print()
254
255    return 0  # success
256
257
258if __name__ == '__main__':
259    status = main()
260    sys.exit(status)

Usage#

In order to use the backup feature, you need to create and initialize a new repository on the target media. Do that by issuing the following command:

$ python3 backup_borg.py --init

To perform a backup on media that has already been initialized, issue the following command:

$ python3 backup_borg.py

Excludes#

If for some reason there are paths that you wish to exclude from the backup process. Create a file with the basename of the folder that you are backing up. For example, ‘/home/troy’, this path is set in the repositories variable. The file would be called “troy.borg.excludes” and it would contain the folders that you want to exclude. One folder/file filter per line:

/home/troy/.esd_auth
/home/troy/.mozilla/firefox/*/Cache
/home/troy/.mozilla/firefox/*/minidumps
/home/troy/.mozilla/firefox/*/.parentlock
/home/troy/.mozilla/firefox/*/urlclassifier3.sqlite
/home/troy/.mozilla/firefox/*/blocklist.xml
/home/troy/.mozilla/firefox/*/extensions.sqlite
/home/troy/.mozilla/firefox/*/extensions.sqlite-journal
/home/troy/.mozilla/firefox/*/extensions.rdf
/home/troy/.mozilla/firefox/*/extensions.ini
/home/troy/.mozilla/firefox/*/extensions.cache
/home/troy/.mozilla/firefox/*/XUL.mfasl
/home/troy/.mozilla/firefox/*/XPC.mfasl
/home/troy/.mozilla/firefox/*/xpti.dat
/home/troy/.mozilla/firefox/*/compreg.dat
/home/troy/.config/google-chrome/Default/Local Storage
/home/troy/.config/google-chrome/Default/Session Storage
/home/troy/.config/google-chrome/Default/Application Cache
/home/troy/.config/google-chrome/Default/History Index *