diff --git a/.dockerignore b/.dockerignore index f9bcfc9..32c14e3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,12 @@ -.gitignore -Dockerfile -LICENSE -*.md \ No newline at end of file +.gitignore +Dockerfile +LICENSE +*.md +drone.yml +.env +*.log +*.conf +*tests* +*dest* +*db* +*pycache* \ No newline at end of file diff --git a/.gitignore b/.gitignore index e69de29..0ada8d8 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,9 @@ +.env +seafile-backup.sh +*.log +*.conf +*tests* +*tests* +*dest* +*db* +*pycache* \ No newline at end of file diff --git a/AppriseClient.py b/AppriseClient.py new file mode 100644 index 0000000..e3606a5 --- /dev/null +++ b/AppriseClient.py @@ -0,0 +1,39 @@ +import requests as r +from tomllib import load +import os + +def apprise_notify(req_obj, apprise_url, aurls, title, body): + payload = {'urls': aurls,'title': title,'body': body,} + apprise_response = req_obj.post(apprise_url, json = payload ,verify=False) + return apprise_response + +class AppriseClient: + def __init__(self): + self.config = '' + try: + if os.environ["DOCKER"]: + self.host = os.environ["host"] + self.port = os.environ["port"] + self.aurls = os.environ["aurls"] + self.title = os.environ["title"] + self.body = os.environ["body"] + if os.environ["toml_path"]: + config_file_path=os.environ["toml_path"] + with open(config_file_path, 'rb') as c: + self.config = load(c) + except: + KeyError + if os.path.exists('./config.toml'): + config_file_path = './config.toml' + with open(config_file_path, 'rb') as c: + self.config = load(c) + if self.config: + self.host = self.config["apprise"]["host"] + self.port = self.config["apprise"]["port"] + self.aurls = self.config["apprise"]["aurls"] + self.title = self.config["apprise"]["title"] + self.body = self.config["apprise"]["body"] + self.apprise_response = apprise_notify(r,self.host,self.port,self.aurls,self.title,self.body) + +if __name__ == "__main__": + AppriseClient() \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7b41c2c..7cbb7ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM alpine:latest -RUN apk add --no-cache mariadb-client rclone curl supercronic docker -COPY entrypoint.sh opt +FROM python:alpine3.18 +RUN apk add --no-cache mariadb-client rclone supercronic docker restic +COPY . opt +RUN chmod +x /opt/entrypoint.sh +RUN chmod +x /opt/restic.sh +RUN pip install requests python-dotenv CMD ["/opt/entrypoint.sh"] \ No newline at end of file diff --git a/HealthchecksIO.py b/HealthchecksIO.py new file mode 100644 index 0000000..599db3a --- /dev/null +++ b/HealthchecksIO.py @@ -0,0 +1,6 @@ +def healthcheck_ping(req_obj, url): + try: + req_obj.get(url, timeout=10) + except req_obj.RequestException as e: + # Log ping failure here... + print("Ping failed: %s" % e) \ No newline at end of file diff --git a/LICENSE b/LICENSE index 0406f8e..f756b80 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,9 @@ -MIT License - -Copyright (c) 2023 jblu - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +MIT License + +Copyright (c) 2023 jblu + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 70c041a..8c4f24c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ -# seafile-backup - -backup seafile data and mariadb database \ No newline at end of file +# seafile-backup + +backup seafile data and mariadb database + +for restic +need 'RESTIC_REPOSITORY' environmental variable +need 'AWS_ACCESS_KEY_ID' environmental variable +need 'AWS_SECRET_ACCESS_KEY' environmental variable + diff --git a/drone.yml b/drone.yml new file mode 100644 index 0000000..e69de29 diff --git a/entrypoint.sh b/entrypoint.sh index 380b4a5..4d53e71 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,6 +2,6 @@ CRON_CONFIG_FILE="/opt/crontab" -echo "${CRON} sh /opt/seafile-backup.sh" > $CRON_CONFIG_FILE +echo "${CRON} python /opt/seafile-backup.py" > $CRON_CONFIG_FILE exec supercronic -passthrough-logs -quiet $CRON_CONFIG_FILE \ No newline at end of file diff --git a/restic.sh b/restic.sh new file mode 100644 index 0000000..c2c0d3b --- /dev/null +++ b/restic.sh @@ -0,0 +1,38 @@ +#!/bin/bash +: "${RESTIC_REPOSITORY:?Need the restic repository}" +: "${AWS_ACCESS_KEY_ID:?Need the access key id}" +: "${AWS_SECRET_ACCESS_KEY:?Need the secret access key}" +: "${RESTIC_PASSWORD:?Need the restic password}" +: "${LOG_PATH:-./restic-backup.log}" +: "${seafile_data_local:-/seafile}" + +# need to securely provide password: https://restic.readthedocs.io/en/latest/faq.html#how-can-i-specify-encryption-passwords-automatically +restic snapshots > /dev/null || restic init + +#Define a timestamp function +timestamp() { +date "+%b %d %Y %T %Z" +} + +# insert timestamp into log +printf "\n\n" +echo "-------------------------------------------------------------------------------" | tee -a $LOG_PATH +echo "$(timestamp): restic-backup.sh started" | tee -a $LOG_PATH + +# Run Backups +restic backup $seafile_data_local | tee -a $LOG_PATH + +# Remove snapshots according to policy +# If run cron more frequently, might add --keep-hourly 24 +restic forget --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 7 | tee -a $LOG_PATH + +# Remove unneeded data from the repository +restic prune | tee -a $LOG_PATH + +# Check the repository for errors +restic check | tee -a $LOG_PATH + +# insert timestamp into log +printf "\n\n" +echo "-------------------------------------------------------------------------------" | tee -a $LOG_PATH +echo "$(timestamp): restic-backup.sh finished" | tee -a $LOG_PATH \ No newline at end of file diff --git a/seafile-backup.py b/seafile-backup.py new file mode 100644 index 0000000..fb6e4c2 --- /dev/null +++ b/seafile-backup.py @@ -0,0 +1,132 @@ +import os +from datetime import datetime +from dotenv import load_dotenv +from AppriseClient import apprise_notify +from HealthchecksIO import healthcheck_ping +import requests as r +now = datetime.now() +r.packages.urllib3.disable_warnings() + +load_dotenv() + +def to_bool(value): + """ + Converts 'something' to boolean. Raises exception for invalid formats + Possible True values: 1, True, "1", "TRue", "yes", "y", "t" + Possible False values: 0, False, "0", "faLse", "no", "n", "f" + """ + if str(value).lower() in ("yes", "y", "true", "t", "1"): return True + if str(value).lower() in ("no", "n", "false", "f", "0"): return False + raise Exception('Invalid value for boolean conversion: ' + str(value) + \ + f'\nPossible True values: 1, True, "1", "TRue", "yes", "y", "t"\ + \nPossible False values: 0, False, "0", "faLse", "no", "n", "f"') + +# switches +docker_command = to_bool(os.getenv("docker_command")) +rclone_copy = to_bool(os.getenv("rclone_copy")) +rclone_push = to_bool(os.getenv("rclone_push")) +restic_push = to_bool(os.getenv("restic_push")) +db_dump = to_bool(os.getenv("db_dump")) +zip_db_files = to_bool(os.getenv("zip_db_files")) +offload_db_files = to_bool(os.getenv("offload_db_files")) +cleanup = to_bool(os.getenv("cleanup")) +healthcheck = to_bool(os.getenv("healthcheck")) +notify = to_bool(os.getenv("notify")) + +LOG_PATH = os.getenv("LOG_PATH") + +# docker +container_name = os.getenv("container_name") + +# data folders +seafile_data_local = os.getenv("seafile_data_local") +seafile_data_backup = os.getenv("seafile_data_backup") + +# databases +databases = os.getenv("databases") +db_dump_host = os.getenv("db_dump_host") +db_dump_user = os.getenv("db_dump_user") +db_dump_password = os.getenv("db_dump_password") +db_dump_tmp_path = os.getenv("db_dump_tmp_path") + +# Rclone remote +rclone_config_path = os.getenv("rclone_config_path") +rclone_remote = os.getenv("rclone_remote") +rclone_backend = os.getenv("rclone_backend") +rclone_provider = os.getenv("rclone_provider") +rclone_endpoint = os.getenv("rclone_endpoint") +rclone_remote_path = os.getenv("rclone_remote_path") +rclone_remote_db_path = os.getenv("rclone_remote_db_path") +rclone_environment_auth = os.getenv("rclone_environment_auth") +rclone_db_retention = os.getenv("rclone_db_retention") + +# Restic remote +RESTIC_REPOSITORY = os.getenv("RESTIC_REPOSITORY") +AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") +RESTIC_PASSWORD = os.getenv("RESTIC_PASSWORD") + +# healthchecks +healthcheck_url = os.getenv("healthcheck_url") + +# notify +apprise_apprise_url = os.getenv("apprise_apprise_url") +apprise_aurls = os.getenv("apprise_aurls") +apprise_title = os.getenv("apprise_title") +apprise_body = os.getenv("apprise_body") + +# Stop seafile and seafile hub +if docker_command: + os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seahub.sh stop') + os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seafile.sh stop') + +# Dump the databases +if db_dump: + for database in databases.split(','): + os.system(f'mariadb-dump -h {db_dump_host} -u {db_dump_user} -p{db_dump_password} --skip-opt\ + {database} > {db_dump_tmp_path}{database}.{now.strftime("%m-%d-%Y_%H-%M-%S")}.sql') + +# Local rclone backup +if rclone_copy: + os.system(f'rclone sync --config {rclone_config_path} --log-file={LOG_PATH} --log-level INFO {seafile_data_local} {seafile_data_backup}') + +# Remote rclone backup +if rclone_push: + if not os.path.exists(rclone_config_path): + os.system(f"rclone config create --config {rclone_config_path} {rclone_remote} {rclone_backend} provider={rclone_provider}\ + endpoint={rclone_endpoint} env_auth=true") + os.system(f'rclone sync --config {rclone_config_path} --log-file={LOG_PATH} --log-level INFO -P\ + {seafile_data_local} {rclone_remote}:{rclone_remote_path}') + +# Remote restic backup +if restic_push: + os.system("./restic.sh") + +# Start seafile and seafile hub +if docker_command: + os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seahub.sh start') + os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seafile.sh start') + +# compress db files +if zip_db_files: + os.system(f'zip -r {db_dump_tmp_path}/sfdb_{now.strftime("%m-%d-%Y_%H-%M-%S")} {db_dump_tmp_path}') + os.system(f'rm {db_dump_tmp_path}*.sql') + +# offload db file +if offload_db_files: + os.system(f'rclone copy --config {rclone_config_path} --log-file={LOG_PATH} --log-level INFO -P\ + {db_dump_tmp_path} {rclone_remote}:{rclone_remote_db_path}') + +# cleanup +if cleanup: + os.system(f'rm {db_dump_tmp_path}*sfdb_*') + os.system(f'Rclone delete --config {rclone_config_path} --log-file={LOG_PATH}\ + {rclone_db_retention} {rclone_remote}:{rclone_remote_db_path}') + +# healthcheck +if healthcheck: + healthcheck_ping(r, healthcheck_url) + +# notification +if notify: + apprise_notify(r, apprise_apprise_url, apprise_aurls, apprise_title, apprise_body) \ No newline at end of file diff --git a/seafile-backup.sh b/seafile-backup.sh deleted file mode 100644 index adecf35..0000000 --- a/seafile-backup.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -# Variables -DATE=`date +%F` -TIME=`date +%H%M` -BACKUPDIR=/backup -# /shared/seafile in seafile container -SEAFDIR=/seafile -BACKUPFILE=$BACKUPDIR/seafile-$DATE-$TIME.tar -TEMPDIR=/tmp/seafile-$DATE-$TIME -BACKUPDATADIR=/backupdata -# Shutdown seafile -docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seahub.sh stop -docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seafile.sh stop - -# Create directories -if [ ! -d $BACKUPDIR ] - then - echo Creating Backupdirectory $BACKUPDIR... - mkdir -pm 0600 $BACKUPDIR -fi -if [ ! -d $TEMPDIR ] - then - echo Create temporary directory $TEMPDIR... - mkdir -pm 0600 $TEMPDIR - mkdir -m 0600 $TEMPDIR/databases -fi - -# Dump data / copy data -echo Dumping ccnet database... -mysqldump -h $mysqlhost -u $mysqlusername -p $mysqlpassword --skip-opt ccnet-db > $TEMPDIR/databases/ccnet-db.sql.`date +"%Y-%m-%d-%H-%M-%S"` -if [ -e $TEMPDIR/databases/ccnet-db.sql.* ]; then echo ok.; else echo ERROR.; fi -echo Dumping SeaFile database... -mysqldump -h $mysqlhost -u $mysqlusername -p $mysqlpassword --skip-opt seafile-db > $TEMPDIR/databases/seafile-db.sql.`date +"%Y-%m-%d-%H-%M-%S"` -if [ -e $TEMPDIR/databases/seafile-db.sql.* ]; then echo ok.; else echo ERROR.; fi -echo Dumping SeaHub database... -mysqldump -h $mysqlhost -u $mysqlusername -p $mysqlpassword --skip-opt seahub-db > $TEMPDIR/databases/seahub-db.sql.`date +"%Y-%m-%d-%H-%M-%S"` -if [ -e $TEMPDIR/databases/seahub-db.sql.* ]; then echo ok.; else echo ERROR.; fi - -echo Copying seafile directory... -rclone sync $SEAFDIR/* $BACKUPDATADIR -if [ -d $TEMPDIR/data/seafile-data ]; then echo ok.; else echo ERROR.; fi - -# Start the server -docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seafile.sh start -docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seahub.sh start - -# compress data -echo Archive the backup... -cd $TEMPDIR -tar -cf $BACKUPFILE * -gzip $BACKUPFILE -if [ -e $BACKUPFILE.gz ]; then echo ok.; else echo ERROR.; fi - -# Cleanup -echo Deleting temporary files... -rm -Rf $TEMPDIR -if [ ! -d $TEMPDIR ]; then echo ok.; else echo ERROR.; fi \ No newline at end of file