Automating Borg Backup#
I wanted to use a proper backup solution to replace my rsync script. I decided to use BorgBackup as it seemed to suit the bill. It is a repository based system that has very strong deduplication algorithms. Essentially, you create a backup repository in a particular path and then backup folders and files to the repository.
I liked the idea of using BorgBackup. However, I wanted to automate the process. My previous rsync backup script worked with removable media consisting of various USB hard disks and regular hard disks that would be plugged into a USB dock. It would detect which drive was inserted and sync the files to it. I wanted to make sure I could do the same thing with BorgBackup.
Initially, I wrote a shell script to handle it. There was a lot of repetition in the script so I decided to rewrite it in python.
Here is the resulting script:
1#!/usr/bin/env python3
2#-*- coding:utf-8 -*-
3
4"""
5A script to automate a borg backup.
6
7
8Copyright (c) 2018 Troy Williams
9
10License: The MIT License (http://www.opensource.org/licenses/mit-license.php)
11"""
12
13# Constants
14__uuid__ = ''
15__author__ = 'Troy Williams'
16__email__ = 'troy.williams@bluebill.net'
17__copyright__ = 'Copyright (c) 2018, Troy Williams'
18__date__ = '2018-10-01'
19__maintainer__ = 'Troy Williams'
20
21import sys
22import os
23import subprocess
24import platform
25import datetime
26from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
27
28# ---------------
29# This stuff with the drives, backup folders could be moved to a configuration
30# file...
31
32# removable drives base folder
33base_media_folder = "/media/troy"
34
35# removable drives that we are interested in backing up to
36backup_media = []
37backup_media.append(os.path.join(base_media_folder, "backup1"))
38backup_media.append(os.path.join(base_media_folder, "backup2"))
39backup_media.append(os.path.join(base_media_folder, "backup3"))
40backup_media.append(os.path.join(base_media_folder,
41 "c5817615-37c8-4765-bbc3-955ecd426db1/troy"))
42
43# repositories on the media that we want to backup too.
44# The tuple contains the path in the file system and
45# the target folder on the media.
46repositories = [('/home/troy', 'home_backup'),
47 ('/home/troy/music', 'music'),
48 ('/home/troy/pictures', 'pictures'),
49 ('/home/troy/videos', 'videos')]
50
51
52def find_first_active_drive(paths):
53 """
54 """
55 for path in paths:
56 if os.path.isdir(path):
57 return path
58
59 return None
60
61
62def borg_create_command(backup_folder,
63 repository_folder,
64 backup_excludes=None):
65
66 command = ['borg',
67 'create',
68 '--verbose',
69 '--progress',
70 '--stats',
71 '--compression', 'auto,lzma',
72 "{}::'{}-{:%Y-%m-%d %H:%M}'".format(repository_folder,
73 platform.node(),
74 datetime.datetime.now()),
75 backup_folder]
76
77 if backup_excludes:
78 excludes = []
79 for exclude in backup_excludes:
80 excludes.append('--exclude')
81 excludes.append('{}'.format(exclude))
82
83 # excludes.append('--exclude {}'.format(exclude))
84
85 command.extend(excludes)
86
87 return command
88
89
90def borg_prune_command(repository_folder):
91
92 # borg prune -v $REPOSITORY --prefix '{hostname}-' \
93 # --keep-hourly=6 \
94 # --keep-daily=7 \
95 # --keep-weekly=4 \
96 # --keep-monthly=6 \
97
98 command = ['borg',
99 'prune',
100 '-v',
101 repository_folder,
102 '--prefix',
103 '{}-'.format(platform.node()),
104 '--keep-hourly=6',
105 '--keep-daily=7',
106 '--keep-weekly=4',
107 '--keep-monthly=6']
108
109 return command
110
111
112def borg_list_command(repository_folder):
113
114 return ['borg', 'list', repository_folder]
115
116
117def get_parser():
118 """Get parser object for script xy.py."""
119
120 parser = ArgumentParser(description=__doc__,
121 formatter_class=ArgumentDefaultsHelpFormatter)
122
123 parser.add_argument('--init',
124 dest='init',
125 action='store_true',
126 help=('Create folders and initialize',
127 ' repositories on the target media.'))
128
129 parser.add_argument('--verify',
130 dest='verify',
131 action='store_true',
132 help='Verify and Validate the repositories.')
133
134 # parser.add_argument('-r', '--result',
135 # dest='result_file',
136 # help='The file to contain the',
137 # 'combined odd and even lines.',
138 # metavar=')
139
140 return parser
141
142
143def main():
144 """
145 This runs the rest of the functions in this module
146 """
147
148 # get the command line arguments
149 parser = get_parser()
150 args = parser.parse_args()
151
152 target_path = find_first_active_drive(backup_media)
153 if not target_path:
154 print('Could not find a backup drive.',
155 ' Please mount a drive and try again.')
156 sys.exit(1)
157
158 # we have a path to a repository
159 print('Backing up to: {}'.format(target_path))
160 print()
161
162# -----------------------------------------
163 if args.init:
164 # check to see if the repository folders exist on the drive
165 # if not create them and initialize a borg repository
166 # https://borgbackup.readthedocs.io/en/stable/usage/init.html
167 print('Initializing target: {}'.format(target_path))
168
169 for r in repositories:
170 backup, folder = r
171
172 repo = os.path.join(target_path, folder)
173
174 if not os.path.exists(repo):
175 os.makedirs(repo)
176
177 result = subprocess.run(['borg',
178 'init',
179 '--encryption=none',
180 repo])
181
182 if result.returncode != 0:
183 sys.exit(result.returncode)
184
185 print('{} has been initialized...'.format(target_path))
186 sys.exit(0)
187
188 if args.verify:
189 # verify the repositories on the target media
190 # https://borgbackup.readthedocs.io/en/stable/usage/check.html#
191
192 for r in repositories:
193 backup, folder = r
194
195 repo = os.path.join(target_path, folder)
196 print('Verifying: {}'.format(repo))
197
198 if not os.path.exists(repo):
199 os.makedirs(repo)
200
201 result = subprocess.run(['borg',
202 'check',
203 repo])
204
205 if result.returncode != 0:
206 sys.exit(result.returncode)
207
208 print('Verification Complete...')
209 sys.exit(0)
210
211 # -------------------------
212 # Backup the folders to the repositories
213 for r in repositories:
214 backup, folder = r
215
216 repo = os.path.join(target_path, folder)
217
218 # see if there are any exclude files associated with the backup folder
219 # they will contain a list of folders and matches that we don't want to
220 # backup. If the file exists, it will be read in and processed
221 excludes_file = '{}.borg.excludes'.format(os.path.basename(backup))
222
223 excludes = None
224 try:
225 with open(excludes_file) as f:
226 excludes = [line.strip() for line in f]
227
228 except IOError:
229 pass
230
231 print()
232 print('Backing up: {}'.format(backup))
233 print('Repo: {}'.format(repo))
234
235 if os.path.isdir(repo):
236 command = borg_create_command(backup,
237 repo,
238 excludes)
239
240 result = subprocess.run(command)
241
242 if result.returncode != 0:
243 sys.exit(result.returncode)
244
245 command = borg_prune_command(repo)
246 result = subprocess.run(command)
247
248 command = borg_list_command(repo)
249 result = subprocess.run(command)
250
251 else:
252 print("Repo Folder Doesn't exit. Skipping...")
253 print()
254
255 return 0 # success
256
257
258if __name__ == '__main__':
259 status = main()
260 sys.exit(status)
Usage#
In order to use the backup feature, you need to create and initialize a new repository on the target media. Do that by issuing the following command:
$ python3 backup_borg.py --init
To perform a backup on media that has already been initialized, issue the following command:
$ python3 backup_borg.py
Excludes#
If for some reason there are paths that you wish to exclude from the backup process. Create a file with the basename of the folder that you are backing up. For example, ‘/home/troy’, this path is set in the repositories variable. The file would be called “troy.borg.excludes” and it would contain the folders that you want to exclude. One folder/file filter per line:
/home/troy/.esd_auth
/home/troy/.mozilla/firefox/*/Cache
/home/troy/.mozilla/firefox/*/minidumps
/home/troy/.mozilla/firefox/*/.parentlock
/home/troy/.mozilla/firefox/*/urlclassifier3.sqlite
/home/troy/.mozilla/firefox/*/blocklist.xml
/home/troy/.mozilla/firefox/*/extensions.sqlite
/home/troy/.mozilla/firefox/*/extensions.sqlite-journal
/home/troy/.mozilla/firefox/*/extensions.rdf
/home/troy/.mozilla/firefox/*/extensions.ini
/home/troy/.mozilla/firefox/*/extensions.cache
/home/troy/.mozilla/firefox/*/XUL.mfasl
/home/troy/.mozilla/firefox/*/XPC.mfasl
/home/troy/.mozilla/firefox/*/xpti.dat
/home/troy/.mozilla/firefox/*/compreg.dat
/home/troy/.config/google-chrome/Default/Local Storage
/home/troy/.config/google-chrome/Default/Session Storage
/home/troy/.config/google-chrome/Default/Application Cache
/home/troy/.config/google-chrome/Default/History Index *