updated healthchecks to support starting and stopping a job

Reviewed-on: #59

.drone.yml

@@ -1,59 +1,55 @@
-kind: pipeline
-name: default
-steps:
-- name: docker
-  image: plugins/docker
-  settings:
-    registry: git.jonb.io
-    dry_run: false
-    username: jblu
-    password: 
-      from_secret: gittea_drone
-    repo: git.jonb.io/jblu/qbit-maid
-    tags:
-    - latest
-  when:
-    branch:
-      - main
-    event:
-      - push
-      - pull_request
-- name: docker-test
-  image: plugins/docker
-  settings:
-    registry: git.jonb.io
-    dry_run: false
-    username: jblu
-    password: 
-      from_secret: gittea_drone
-    repo: git.jonb.io/jblu/qbit-maid
-    tags:
-    - dev
-  when:
-    branch:
-      - dev*
-    event:
-      - push
-      - pull_request
-- name: test-main
-  image: git.jonb.io/jblu/qbit-maid:latest
-  commands:
-  - python test_qbitmaid.py
-  - python test_dragnet.py
-  when:
-    branch:
-      - main
-    event:
-      - push
-      - pull_request
-- name: test-dev
-  image: git.jonb.io/jblu/qbit-maid:dev
-  commands:
-  - python test_qbitmaid.py
-  - python test_dragnet.py
-  when:
-    branch:
-      - dev*
-    event:
-      - push
-      - pull_request
+kind: pipeline
+name: default
+steps:
+- name: docker
+  image: plugins/docker
+  settings:
+    registry: git.jonb.io
+    dry_run: false
+    username: jblu
+    password: 
+      from_secret: gittea_drone
+    repo: git.jonb.io/jblu/qbit-maid
+    tags:
+    - latest
+  when:
+    branch:
+      - main
+    event:
+      - push
+- name: docker-test
+  image: plugins/docker
+  settings:
+    registry: git.jonb.io
+    dry_run: false
+    username: jblu
+    password: 
+      from_secret: gittea_drone
+    repo: git.jonb.io/jblu/qbit-maid
+    tags:
+    - dev
+  when:
+    branch:
+      - dev*
+    event:
+      - push
+- name: test-main
+  image: git.jonb.io/jblu/qbit-maid:latest
+  commands:
+  - python test_qbitmaid.py
+  - python test_write_csv.py
+  when:
+    branch:
+      - main
+    event:
+      - push
+- name: test-dev
+  image: git.jonb.io/jblu/qbit-maid:dev
+  commands:
+  - python test_qbitmaid.py
+  - python test_write_csv.py
+  when:
+    branch:
+      - dev*
+    event:
+      - push

README.md

@@ -2,7 +2,7 @@
 # qbit-maid
 > Warning: This application removes torrents that are over the minimum age and that are not part of the ignored categories, domains or tags. Please use the delete_torrents feature set to false when first testing its functionality.
 
-![](https://drone.jbranan.com/api/badges/jblu/phc/status.svg)
+[![Build Status](https://drone.jonb.io/api/badges/jblu/qbit-maid/status.svg?ref=refs/heads/main)](https://drone.jonb.io/jblu/qbit-maid)
 
 The objective is to remove torrents based on the following criteria:
 - tracker domain name
@@ -13,15 +13,15 @@ The objective is to remove torrents based on the following criteria:
 ## Install
 ### Docker(Recommended)
 
-[package ](https://git.jbranan.com/jblu/-/packages/container/qbit-maid/latest)
+[package](https://git.jonb.io/jblu/-/packages/container/qbit-maid/latest)
 
-    docker pull git.jbranan.com/jblu/qbit-maid:latest
+    docker pull git.jonb.io/jblu/qbit-maid:latest
 
 #### Docker Run Command:
 
 > Please note it is best practice to escape spaces in variables. That is why there is backslashes in the cron schedule.
 
-    docker run --name qbit-maid -v /opt/qbit-maid:/config/ -e CRON=0\ 1\ *\ *\ * -e toml_path=/config/config.toml git.jbranan.com/jblu/qbit-maid
+    docker run --name qbit-maid -v /opt/qbit-maid:/config/ -e CRON=0\ 1\ *\ *\ * -e toml_path=/config/config.toml git.jonb.io/jblu/qbit-maid
 
 #### Docker Compose
 
@@ -29,7 +29,7 @@ The objective is to remove torrents based on the following criteria:
 version: '3.3'
 services:
     qbit-maid:
-        image: git.jbranan.com/jblu/qbit-maid
+        image: git.jonb.io/jblu/qbit-maid
         container_name: qbit-maid
         volumes:
             - /opt/qbit-maid:/config
@@ -38,7 +38,7 @@ services:
             - toml_path=/config/config.toml
 ```
 ### Via Git
-    git clone https://git.jbranan.com/jblu/qbit-maid.git
+    git clone https://git.jonb.io/jblu/qbit-maid.git
 
 Qbit-maid will look for an environment variable *toml_path* for its configuration.If it doesn't find it, it will look for a config.toml file in it's own directory.
 ##### config.toml
@@ -78,6 +78,10 @@ aurls = 'mailto://user:pass@gmail.com'
 enable_dragnet = false
 dragnet_outfile = "./orphaned.csv"
 
+[telemetry]
+enable_telemetry = false
+telemetry_outfile = "./telemetry.csv"
+
 [ignored_categories]
 tech = "tech"
 books = "books"

config.toml.example

@@ -14,8 +14,8 @@ protected_tag = "ipt"
 non_protected_tag = "public"
 
 [torrent]
-age = 2419200
-minimum_age = 432000
+max_age = 2419200
+min_age = 432000
 delete_torrents = false
 
 [pushover]

qbit-maid.py

@@ -55,8 +55,8 @@ class Qbt:
 
         #torrent
         self.delete_torrents = self.config["torrent"]["delete_torrents"]
-        self.minimum_age = self.config["torrent"]["minimum_age"]
-        self.age = self.config["torrent"]["age"]
+        self.min_age = self.config["torrent"]["min_age"]
+        self.max_age = self.config["torrent"]["max_age"]
 
         #pushover
         self.use_pushover = self.config["pushover"]["use_pushover"]
@@ -73,6 +73,10 @@ class Qbt:
         self.enable_dragnet = self.config["dragnet"]["enable_dragnet"]
         self.dragnet_outfile = self.config["dragnet"]["dragnet_outfile"]
 
+        #telemetry
+        self.enable_telemetry = self.config["telemetry"]["enable_telemetry"]
+        self.telemetry_outfile = self.config["telemetry"]["telemetry_outfile"]
+
         #ignored_categories
         self.cat_whitelist = self.config["ignored_categories"]
 
@@ -90,6 +94,11 @@ class Qbt:
         tor_log(self)
         tor_notify(self)
         self.t = time
+
+        #start healthcheck job
+        if self.use_healthcheck:
+            send_ping(self, r, self.healthcheck_url.rstrip("/") + "/start" )
+            
         # Pulling domain names to treat carefully
         self.tracker_list = []
         self.up_tor_counter = 0
@@ -112,6 +121,7 @@ class Qbt:
         if self.use_log:
             list_qbit_api_info(self)
             list_first_tor(self)
+            debug_torrent_list(self)
         build_tor_list(self)
         process_counts(self)
         if self.use_log:

qlist.py

@@ -12,7 +12,7 @@ def build_tor_list(self):
             # Need a way to tag when the tracker is blank
             if is_tracker_blank(torrent['tracker']):
                 if self.use_log:
-                    self.tl.warning(f'Torrent doesn\'t have a tracker ["{torrent["name"][0:20]}..."] [{torrent["tracker"]}]hash: {torrent["hash"]}')
+                    self.tl.warning(f'Torrent doesn\'t have a tracker ["{torrent["name"][0:20]}..."] hash: {torrent["hash"]}')
                 self.ignored_counter += 1
                 continue
             elif is_cat_ignored(torrent['category'], self.cat_whitelist.values()):
@@ -32,9 +32,9 @@ def build_tor_list(self):
                     self.qbt_client.torrents_add_tags(self.tracker_protected_tag,torrent['hash'])
                 elif is_not_protected_tracker(torrent['tracker'], self.tracker_whitelist.values()):
                     self.qbt_client.torrents_add_tags(self.tracker_non_protected_tag,torrent['hash'])
-            if is_preme(torrent['added_on'], self.minimum_age, self.t.time()):
+            if is_preme(torrent['seeding_time'], self.min_age):
                 self.preme_tor_counter += 1
-                self.tl.debug(f'Premature torrent: ["{torrent["name"][0:20]}..."] hash: {torrent["hash"]}')
+                self.tl.debug(f'Premature torrent: ["{torrent["name"][0:20]}..."] Seconds Seeded: [{torrent["seeding_time"]}] hash: {torrent["hash"]}')
                 continue
             elif is_protected_tracker(torrent['tracker'], self.tracker_whitelist.values()):
                 if is_tag_blank(torrent['tags']):
@@ -50,8 +50,8 @@ def build_tor_list(self):
                 self.tracker_list.append(torrent)
 
 
-def is_preme(added, minage, time):
-    if added + minage >= time:
+def is_preme(seeding_time, minage):
+    if seeding_time <= minage:
         return True
 
 def is_cat_ignored(cat, catlist):

qlogging.py

@@ -88,6 +88,9 @@ def get_script_runtime(self):
 
 def send_ping(self, req_obj, healthcheck_url):
     try:
-        req_obj.get(healthcheck_url, timeout=10)
+        req_obj.get(healthcheck_url, timeout=5)
     except req_obj.RequestException as e:
-        self.tl.info(f"Ping failed: {e}")
+        self.tl.info(f"Ping failed: {e}")
+
+def debug_torrent_list(self):
+    self.tl.debug(self.torrent_list)

qprocess.py

@@ -3,6 +3,10 @@ def tor_processor(self):
     If torrent meets criteria for deletion, its infohash_v1 will be appended to self.torrent_hash_delete_list
     """
     for canidate in self.tracker_list:
+        if self.enable_telemetry:
+            header = ['state','ratio','tags','added','hash','name','tracker']
+            row = [canidate['state'],canidate['ratio'],canidate["tags"],canidate['added_on'],canidate['infohash_v1'],canidate["name"][0:20],canidate['tracker']] 
+            write_csv(self.cv,self.telemetry_outfile,header,row)
         if self.use_log:
             self.tl.debug(f'---Reviewing canidate: ["{canidate["name"][0:20]}..."] {canidate["infohash_v1"]}---')
         if is_downloading(canidate['state']):
@@ -12,9 +16,9 @@ def tor_processor(self):
         elif is_protected_under_ratio(canidate['ratio'], 1.05, self.tracker_protected_tag, canidate["tags"]):
             if self.use_log:
                 self.tl.debug(f'["{canidate["name"][0:20]}..."] is below a 1.05 ratio({canidate["ratio"]})')
-            if is_old_tor(canidate['added_on'], self.age, self.t.time()):
+            if is_old_tor(canidate['time_active'], self.max_age):
                 if self.use_log:
-                    self.tl.debug(f'["{canidate["name"][0:20]}..."] Seconds old: {self.t.time() - self.age - canidate["added_on"]}')
+                    self.tl.debug(f'["{canidate["name"][0:20]}..."] Seconds old: {canidate["time_active"]} Delta: {canidate["time_active"] - self.max_age}')
                 self.torrent_hash_delete_list.append(canidate['infohash_v1'])
                 if self.use_log:
                     self.tl.info(f'Submitted ["{canidate["name"][0:20]}..."] for deletion from the protected list.') 
@@ -30,7 +34,9 @@ def tor_processor(self):
                 self.tl.info(f'Submitted ["{canidate["name"][0:20]}..."] for deletion.')
         else:
             if self.enable_dragnet:
-                dragnet(self.cv,self.dragnet_outfile,canidate['state'],canidate['ratio'],canidate["tags"],canidate['added_on'],self.age,self.t.time(),canidate['infohash_v1'],canidate["name"][0:20],canidate['tracker'])
+                header = ['state','ratio','tags','added','thash','tname','trname']
+                row = [canidate['state'],canidate['ratio'],canidate["tags"],canidate['added_on'],canidate['infohash_v1'],canidate["name"][0:20],canidate['tracker']] 
+                write_csv(self.cv,self.dragnet_outfile,header,row)
             self.tl.info(f'["{canidate["name"][0:20]}..."] is orphaned.')
             self.up_tor_counter += 1
             continue
@@ -56,8 +62,8 @@ def is_protected_under_ratio(torratio, setratio, settag, tortag):
     if torratio < float(setratio) and settag in tortag:
         return True
 
-def is_old_tor(toradd, setage, currenttime):
-    if toradd + setage <= currenttime:
+def is_old_tor(realage, maxage):
+    if realage >= maxage:
         return True
 
 def is_protected_over_ratio(torratio, setratio, settag, tortag):
@@ -68,9 +74,7 @@ def is_not_protected_tor(setnonprotectedtag, tortags):
     if setnonprotectedtag in tortags:
         return True
 
-def dragnet(csv_obj,outfile,state,ratio,tags,added,age,time,thash,tname,trname):
-    header = ['state','ratio','tags','added','age','time','thash','tname','trname']
-    row = [state,ratio,tags,added,age,time,thash,tname,trname]
+def write_csv(csv_obj,outfile,header,row):
     with open(outfile, 'a+', encoding='UTF8', newline='') as f:
         writer = csv_obj.writer(f)
         if f.tell() == 0:

test_dragnet.py

@@ -1,21 +0,0 @@
-from qprocess import dragnet
-import csv
-import unittest
-
-class TestDragnet(unittest.TestCase):
-    def test_dragnet_sanity(self):
-        self.cv = csv
-        outfile = './test_outfile.csv'
-        state = 'downloading'
-        ratio = 1.05
-        tags = 'ipt'
-        added = 1
-        age = 240000
-        time = 123456
-        thash = 'asfasdf23412adfqwer'
-        tname = 'thisismynamehahahah'
-        trname = 'https://localhost.stackoverflow.tech/317332f1c125bc9c1b9b14fb8e054908/announce'
-        dragnet(self.cv,outfile,state,ratio,tags,added,age,time,thash,tname,trname)
-
-if __name__ == '__main__':
-    unittest.main()

test_qbitmaid.py

@@ -1,11 +1,14 @@
 import unittest
+import requests as r
 from qlist import is_preme,is_cat_ignored,is_tracker_blank,is_protected_tracker,is_not_protected_tracker,is_tag_blank,is_ignored_tag
 from qprocess import is_downloading,is_protected_under_ratio,is_old_tor,is_protected_over_ratio,is_not_protected_tor
+from qlogging import send_ping
 
 class TestQbitmaid(unittest.TestCase):
     def test_ispreme_sanity(self):
-        self.assertTrue(is_preme(1,1,1))
-        self.assertFalse(is_preme(1,1,3))
+        self.assertTrue(is_preme(1,1))
+        self.assertTrue(is_preme(1,2))
+        self.assertFalse(is_preme(2,1))
     
     def test_ispreme(self):
         pass
@@ -68,12 +71,12 @@ class TestQbitmaid(unittest.TestCase):
         pass
 
     def test_isoldtor_sanity(self):
-        self.assertTrue(is_old_tor(1,2,4))
+        self.assertFalse(is_old_tor(1,2))
 
     def test_isoldtor(self):
-        self.assertFalse(is_old_tor(1661150664,2419200,1662049004.2101078))
-        self.assertFalse(is_old_tor(1661150664,2419200,1662049004))
-        self.assertFalse(is_old_tor(1661150664.000000,2419200.0000000,1662049004.2101078))
+        self.assertTrue(is_old_tor(1,1))
+        self.assertTrue(is_old_tor(2,1))
+        self.assertFalse(is_old_tor(1,2))
 
     def test_isprotectedoverratio_sanity(self):
         self.assertTrue(is_protected_over_ratio(2,1,'a','a,b,c'))
@@ -93,6 +96,17 @@ class TestQbitmaid(unittest.TestCase):
     def test_isignoredtag(self):
         self.assertTrue(is_ignored_tag(['save'], 'save,public,ipt'))
 
+    def test_sendpingstart_sanity(self):
+        send_ping(self, r, "https://thunder.jonb.io/ping/921625e5-5b76-4f45-a0c3-56145e16f3bb" + "/start")
+        url = "https://thunder.jonb.io/ping/921625e5-5b76-4f45-a0c3-56145e16f3bb"
+        send_ping(self, r, url)
+
+    def test_sendping_start(self):
+        url = "https://thunder.jonb.io/ping/921625e5-5b76-4f45-a0c3-56145e16f3bb/"
+        send_ping(self, r, url.strip("/") + "/start")
+        send_ping(self, r, "https://thunder.jonb.io/ping/921625e5-5b76-4f45-a0c3-56145e16f3bb")
+        
+
     # def test__sanity(self):
     #     pass
 

test_write_csv.py

@@ -0,0 +1,41 @@
+from qprocess import write_csv
+import csv
+import unittest
+import os
+# Torrent Items needed for testing
+# canidate['state'] {canidate["ratio"]} {torrent["tags"]} torrent["seeding_time"] {torrent["category"]} {torrent["name"][0:20]} {canidate["time_active"]}
+class TestWriteCSV(unittest.TestCase):
+    def test_write_csv_dragnet(self):
+        self.cv = csv
+        outfile = './test_dragnet_outfile.csv'
+        state = 'downloading'
+        ratio = 1.05
+        tags = 'ipt'
+        added = 1
+        age = 240000
+        time = 123456
+        thash = 'asfasdf23412adfqwer'
+        tname = 'thisismynamehahahah'
+        trname = 'https://localhost.stackoverflow.tech/317332f1c125bc9c1b9b14fb8e054908/announce'
+        header = ['state','ratio','tags','added','age','time','thash','tname','trname']
+        row = [state,ratio,tags,added,age,time,thash,tname,trname]
+        write_csv(self.cv,outfile,header,row)
+        self.assertTrue(os.path.exists(outfile))
+
+    def test_write_csv_telemetry(self):
+        self.cv = csv
+        outfile = './test_telemetry_outfile.csv'
+        state = 'downloading'
+        ratio = 1.05
+        tags = 'ipt'
+        added = 1
+        thash = 'asfasdf23412adfqwer'
+        tname = 'thisismynamehahahah'
+        trname = 'https://localhost.stackoverflow.tech/317332f1c125bc9c1b9b14fb8e054908/announce'
+        header = ['state','ratio','tags','added','hash','name','tracker']
+        row = [state,ratio,tags,added,thash,tname,trname]
+        write_csv(self.cv,outfile,header,row)
+        self.assertTrue(os.path.exists(outfile))
+
+if __name__ == '__main__':
+    unittest.main()