working rc1

.dockerignore

@@ -1,4 +1,12 @@
-.gitignore
-Dockerfile
-LICENSE
-*.md
+.gitignore
+Dockerfile
+LICENSE
+*.md
+drone.yml
+.env
+*.log
+*.conf
+*tests*
+*dest*
+*db*
+*pycache*

.gitignore

@@ -0,0 +1,9 @@
+.env
+seafile-backup.sh
+*.log
+*.conf
+*tests*
+*tests*
+*dest*
+*db*
+*pycache*

AppriseClient.py

@@ -0,0 +1,39 @@
+import requests as r
+from tomllib import load
+import os
+
+def apprise_notify(req_obj, apprise_url, aurls, title, body):
+    payload = {'urls': aurls,'title': title,'body': body,}
+    apprise_response = req_obj.post(apprise_url, json = payload ,verify=False)
+    return apprise_response
+
+class AppriseClient:
+    def __init__(self):
+        self.config = ''
+        try:
+            if os.environ["DOCKER"]:
+                self.host = os.environ["host"]
+                self.port = os.environ["port"]
+                self.aurls = os.environ["aurls"]
+                self.title = os.environ["title"]
+                self.body = os.environ["body"]
+            if os.environ["toml_path"]:
+                config_file_path=os.environ["toml_path"]
+                with open(config_file_path, 'rb') as c:
+                    self.config = load(c)
+        except:
+            KeyError
+        if os.path.exists('./config.toml'):
+            config_file_path = './config.toml'
+            with open(config_file_path, 'rb') as c:
+                self.config = load(c)
+        if self.config:
+            self.host = self.config["apprise"]["host"]
+            self.port = self.config["apprise"]["port"]
+            self.aurls = self.config["apprise"]["aurls"]
+            self.title = self.config["apprise"]["title"]
+            self.body = self.config["apprise"]["body"]
+        self.apprise_response = apprise_notify(r,self.host,self.port,self.aurls,self.title,self.body)  
+
+if __name__ == "__main__":
+    AppriseClient()

Dockerfile

@@ -1,4 +1,7 @@
-FROM alpine:latest
-RUN apk add --no-cache mariadb-client rclone curl supercronic docker
-COPY entrypoint.sh opt
+FROM python:alpine3.18
+RUN apk add --no-cache mariadb-client rclone supercronic docker restic
+COPY . opt
+RUN chmod +x /opt/entrypoint.sh
+RUN chmod +x /opt/restic.sh
+RUN pip install requests python-dotenv
 CMD ["/opt/entrypoint.sh"]

HealthchecksIO.py

@@ -0,0 +1,6 @@
+def healthcheck_ping(req_obj, url):
+    try:
+        req_obj.get(url, timeout=10)
+    except req_obj.RequestException as e:
+        # Log ping failure here...
+        print("Ping failed: %s" % e)

LICENSE

@@ -1,9 +1,9 @@
-MIT License
-
-Copyright (c) 2023 jblu
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+MIT License
+
+Copyright (c) 2023 jblu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -1,3 +1,9 @@
-# seafile-backup
-
-backup seafile data and mariadb database
+# seafile-backup
+
+backup seafile data and mariadb database
+
+for restic
+need 'RESTIC_REPOSITORY' environmental variable
+need 'AWS_ACCESS_KEY_ID' environmental variable
+need 'AWS_SECRET_ACCESS_KEY' environmental variable
+

entrypoint.sh

@@ -2,6 +2,6 @@
 
 CRON_CONFIG_FILE="/opt/crontab"
 
-echo "${CRON} sh /opt/seafile-backup.sh" > $CRON_CONFIG_FILE
+echo "${CRON} python /opt/seafile-backup.py" > $CRON_CONFIG_FILE
 
 exec supercronic -passthrough-logs -quiet $CRON_CONFIG_FILE

restic.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+: "${RESTIC_REPOSITORY:?Need the restic repository}"
+: "${AWS_ACCESS_KEY_ID:?Need the access key id}"
+: "${AWS_SECRET_ACCESS_KEY:?Need the secret access key}"
+: "${RESTIC_PASSWORD:?Need the restic password}"
+: "${LOG_PATH:-./restic-backup.log}"
+: "${seafile_data_local:-/seafile}"
+
+# need to securely provide password: https://restic.readthedocs.io/en/latest/faq.html#how-can-i-specify-encryption-passwords-automatically
+restic snapshots > /dev/null || restic init
+
+#Define a timestamp function
+timestamp() {
+date "+%b %d %Y %T %Z"
+}
+
+# insert timestamp into log
+printf "\n\n"
+echo "-------------------------------------------------------------------------------" | tee -a $LOG_PATH
+echo "$(timestamp): restic-backup.sh started" | tee -a $LOG_PATH
+
+# Run Backups
+restic backup $seafile_data_local | tee -a $LOG_PATH
+
+# Remove snapshots according to policy
+# If run cron more frequently, might add --keep-hourly 24
+restic forget --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 7  | tee -a $LOG_PATH
+
+# Remove unneeded data from the repository
+restic prune | tee -a $LOG_PATH
+
+# Check the repository for errors
+restic check | tee -a $LOG_PATH
+
+# insert timestamp into log
+printf "\n\n"
+echo "-------------------------------------------------------------------------------" | tee -a $LOG_PATH
+echo "$(timestamp): restic-backup.sh finished" | tee -a $LOG_PATH

seafile-backup.py

@@ -0,0 +1,132 @@
+import os
+from datetime import datetime
+from dotenv import load_dotenv
+from AppriseClient import apprise_notify
+from HealthchecksIO import healthcheck_ping
+import requests as r
+now = datetime.now()
+r.packages.urllib3.disable_warnings()
+
+load_dotenv()
+
+def to_bool(value):
+    """
+       Converts 'something' to boolean. Raises exception for invalid formats
+           Possible True values: 1, True, "1", "TRue", "yes", "y", "t"
+           Possible False values: 0, False, "0", "faLse", "no", "n", "f"
+    """
+    if str(value).lower() in ("yes", "y", "true",  "t", "1"): return True
+    if str(value).lower() in ("no",  "n", "false", "f", "0"): return False
+    raise Exception('Invalid value for boolean conversion: ' + str(value) + \
+                    f'\nPossible True values: 1, True, "1", "TRue", "yes", "y", "t"\
+                        \nPossible False values: 0, False, "0", "faLse", "no", "n", "f"')
+
+# switches
+docker_command = to_bool(os.getenv("docker_command"))
+rclone_copy = to_bool(os.getenv("rclone_copy"))
+rclone_push = to_bool(os.getenv("rclone_push"))
+restic_push = to_bool(os.getenv("restic_push"))
+db_dump = to_bool(os.getenv("db_dump"))
+zip_db_files = to_bool(os.getenv("zip_db_files"))
+offload_db_files = to_bool(os.getenv("offload_db_files"))
+cleanup = to_bool(os.getenv("cleanup"))
+healthcheck = to_bool(os.getenv("healthcheck"))
+notify = to_bool(os.getenv("notify"))
+
+LOG_PATH = os.getenv("LOG_PATH")
+
+# docker
+container_name = os.getenv("container_name")
+
+# data folders
+seafile_data_local = os.getenv("seafile_data_local")
+seafile_data_backup = os.getenv("seafile_data_backup")
+
+# databases
+databases = os.getenv("databases")
+db_dump_host = os.getenv("db_dump_host")
+db_dump_user = os.getenv("db_dump_user")
+db_dump_password = os.getenv("db_dump_password")
+db_dump_tmp_path = os.getenv("db_dump_tmp_path")
+
+# Rclone remote
+rclone_config_path = os.getenv("rclone_config_path")
+rclone_remote = os.getenv("rclone_remote")
+rclone_backend = os.getenv("rclone_backend")
+rclone_provider = os.getenv("rclone_provider")
+rclone_endpoint = os.getenv("rclone_endpoint")
+rclone_remote_path = os.getenv("rclone_remote_path")
+rclone_remote_db_path = os.getenv("rclone_remote_db_path")
+rclone_environment_auth = os.getenv("rclone_environment_auth")
+rclone_db_retention = os.getenv("rclone_db_retention")
+
+# Restic remote
+RESTIC_REPOSITORY = os.getenv("RESTIC_REPOSITORY")
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
+RESTIC_PASSWORD = os.getenv("RESTIC_PASSWORD")
+
+# healthchecks
+healthcheck_url = os.getenv("healthcheck_url")
+
+# notify
+apprise_apprise_url = os.getenv("apprise_apprise_url")
+apprise_aurls = os.getenv("apprise_aurls")
+apprise_title = os.getenv("apprise_title")
+apprise_body = os.getenv("apprise_body")
+
+# Stop seafile and seafile hub
+if docker_command:
+    os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seahub.sh stop')
+    os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seafile.sh stop')
+
+# Dump the databases
+if db_dump:
+    for database in databases.split(','):
+        os.system(f'mariadb-dump -h {db_dump_host} -u {db_dump_user} -p{db_dump_password} --skip-opt\
+                   {database} > {db_dump_tmp_path}{database}.{now.strftime("%m-%d-%Y_%H-%M-%S")}.sql')
+
+# Local rclone backup
+if rclone_copy:
+    os.system(f'rclone sync --config {rclone_config_path} --log-file={LOG_PATH} --log-level INFO {seafile_data_local} {seafile_data_backup}')
+          
+# Remote rclone backup
+if rclone_push:
+    if not os.path.exists(rclone_config_path):
+        os.system(f"rclone config create --config {rclone_config_path} {rclone_remote} {rclone_backend} provider={rclone_provider}\
+                   endpoint={rclone_endpoint} env_auth=true")
+    os.system(f'rclone sync --config {rclone_config_path} --log-file={LOG_PATH} --log-level INFO -P\
+                {seafile_data_local} {rclone_remote}:{rclone_remote_path}')
+
+# Remote restic backup
+if restic_push:
+    os.system("./restic.sh")
+
+# Start seafile and seafile hub
+if docker_command:
+    os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seahub.sh start')
+    os.system(f'docker exec {container_name} /opt/seafile/seafile-server-latest/seafile.sh start')
+
+# compress db files
+if zip_db_files:
+    os.system(f'zip -r {db_dump_tmp_path}/sfdb_{now.strftime("%m-%d-%Y_%H-%M-%S")} {db_dump_tmp_path}')
+    os.system(f'rm {db_dump_tmp_path}*.sql')
+
+# offload db file
+if offload_db_files:
+    os.system(f'rclone copy --config {rclone_config_path} --log-file={LOG_PATH} --log-level INFO -P\
+               {db_dump_tmp_path} {rclone_remote}:{rclone_remote_db_path}')
+
+# cleanup
+if cleanup:
+    os.system(f'rm {db_dump_tmp_path}*sfdb_*')
+    os.system(f'Rclone delete --config {rclone_config_path} --log-file={LOG_PATH}\
+               {rclone_db_retention} {rclone_remote}:{rclone_remote_db_path}')
+
+# healthcheck
+if healthcheck:
+    healthcheck_ping(r, healthcheck_url)
+
+# notification
+if notify:
+    apprise_notify(r, apprise_apprise_url, apprise_aurls, apprise_title, apprise_body)

seafile-backup.sh

@@ -1,58 +0,0 @@
-#!/bin/sh
-
-# Variables
-DATE=`date +%F`
-TIME=`date +%H%M`
-BACKUPDIR=/backup
-# /shared/seafile in seafile container
-SEAFDIR=/seafile
-BACKUPFILE=$BACKUPDIR/seafile-$DATE-$TIME.tar
-TEMPDIR=/tmp/seafile-$DATE-$TIME
-BACKUPDATADIR=/backupdata
-# Shutdown seafile
-docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seahub.sh stop
-docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seafile.sh stop
-
-# Create directories
-if [ ! -d $BACKUPDIR ]
-  then
-  echo Creating Backupdirectory $BACKUPDIR...
-  mkdir -pm 0600 $BACKUPDIR
-fi
-if [ ! -d $TEMPDIR ]
-  then
-  echo Create temporary directory $TEMPDIR...
-  mkdir -pm 0600 $TEMPDIR
-  mkdir -m 0600 $TEMPDIR/databases
-fi
-
-# Dump data / copy data
-echo Dumping ccnet database...
-mysqldump -h $mysqlhost -u $mysqlusername -p $mysqlpassword --skip-opt ccnet-db > $TEMPDIR/databases/ccnet-db.sql.`date +"%Y-%m-%d-%H-%M-%S"`
-if [ -e $TEMPDIR/databases/ccnet-db.sql.* ]; then echo ok.; else echo ERROR.; fi
-echo Dumping SeaFile database...
-mysqldump -h $mysqlhost -u $mysqlusername -p $mysqlpassword --skip-opt seafile-db > $TEMPDIR/databases/seafile-db.sql.`date +"%Y-%m-%d-%H-%M-%S"`
-if [ -e $TEMPDIR/databases/seafile-db.sql.* ]; then echo ok.; else echo ERROR.; fi
-echo Dumping SeaHub database...
-mysqldump -h $mysqlhost -u $mysqlusername -p $mysqlpassword --skip-opt seahub-db > $TEMPDIR/databases/seahub-db.sql.`date +"%Y-%m-%d-%H-%M-%S"`
-if [ -e $TEMPDIR/databases/seahub-db.sql.* ]; then echo ok.; else echo ERROR.; fi
-
-echo Copying seafile directory...
-rclone sync $SEAFDIR/* $BACKUPDATADIR
-if [ -d $TEMPDIR/data/seafile-data ]; then echo ok.; else echo ERROR.; fi
-
-# Start the server
-docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seafile.sh start
-docker exec $seafilecontainer /opt/seafile/seafile-server-latest/seahub.sh start
-
-# compress data
-echo Archive the backup...
-cd $TEMPDIR
-tar -cf $BACKUPFILE *
-gzip $BACKUPFILE
-if [ -e $BACKUPFILE.gz ]; then echo ok.; else echo ERROR.; fi
-
-# Cleanup
-echo Deleting temporary files...
-rm -Rf $TEMPDIR
-if [ ! -d $TEMPDIR ]; then echo ok.; else echo ERROR.; fi