Merge pull request 'ci: auto-deploy qbit-maid to Komodo after image push' (#63) from dev-komodo-deploy into main

Reviewed-on: #63

.gitea/scripts/komodo-deploy.sh

@@ -0,0 +1,173 @@
+#!/bin/sh
+#
+# Trigger a Komodo deploy and optionally wait for it to finish.
+#
+# Required env:
+#   KOMODO_URL           Base URL of Komodo Core, e.g. https://k.jonb.io
+#   KOMODO_API_KEY       Komodo API key
+#   KOMODO_API_SECRET    Komodo API secret
+#   KOMODO_TARGET        Stack name/id (or Procedure name/id)
+#
+# Optional env:
+#   KOMODO_TARGET_TYPE   "stack" (default) or "procedure"
+#   KOMODO_REQUIRED      "true" to fail when KOMODO_TARGET is empty
+#   KOMODO_WAIT          "true" (default) to poll until the update completes
+#   KOMODO_WAIT_TIMEOUT  Seconds to wait for completion (default 600)
+#   KOMODO_WAIT_INTERVAL Seconds between polls (default 10)
+#
+# Exits non-zero if the request is rejected or the deploy fails.
+
+set -eu
+
+KOMODO_TARGET_TYPE="${KOMODO_TARGET_TYPE:-stack}"
+KOMODO_REQUIRED="${KOMODO_REQUIRED:-false}"
+KOMODO_WAIT="${KOMODO_WAIT:-true}"
+KOMODO_WAIT_TIMEOUT="${KOMODO_WAIT_TIMEOUT:-600}"
+KOMODO_WAIT_INTERVAL="${KOMODO_WAIT_INTERVAL:-10}"
+
+if [ -z "${KOMODO_TARGET:-}" ]; then
+  if [ "$KOMODO_REQUIRED" = "true" ]; then
+    echo "::error::KOMODO_TARGET is empty but KOMODO_REQUIRED=true" >&2
+    exit 1
+  fi
+  echo "KOMODO_TARGET is not set - skipping Komodo deploy."
+  exit 0
+fi
+
+# Explicit checks rather than ':?', whose exit status differs between
+# shells (bash exits 1, dash exits 2). Keep every failure path at exit 1.
+if [ -z "${KOMODO_URL:-}" ]; then
+  echo "::error::KOMODO_URL is required" >&2
+  exit 1
+fi
+if [ -z "${KOMODO_API_KEY:-}" ]; then
+  echo "::error::KOMODO_API_KEY is required" >&2
+  exit 1
+fi
+if [ -z "${KOMODO_API_SECRET:-}" ]; then
+  echo "::error::KOMODO_API_SECRET is required" >&2
+  exit 1
+fi
+
+# Normalise: strip any trailing slash so path joins are predictable.
+KOMODO_URL="${KOMODO_URL%/}"
+
+case "$KOMODO_TARGET_TYPE" in
+  stack)
+    EXECUTE="DeployStack"
+    PARAMS="{\"stack\": \"${KOMODO_TARGET}\"}"
+    ;;
+  procedure)
+    EXECUTE="RunProcedure"
+    PARAMS="{\"procedure\": \"${KOMODO_TARGET}\"}"
+    ;;
+  *)
+    echo "::error::KOMODO_TARGET_TYPE must be 'stack' or 'procedure', got '${KOMODO_TARGET_TYPE}'" >&2
+    exit 1
+    ;;
+esac
+
+tmp=$(mktemp)
+trap 'rm -f "$tmp"' EXIT
+
+# api <path> <json-body> -> writes body to $tmp, echoes HTTP status code
+api() {
+  curl -sS -o "$tmp" -w '%{http_code}' \
+    -X POST "${KOMODO_URL}$1" \
+    -H 'Content-Type: application/json' \
+    -H "x-api-key: ${KOMODO_API_KEY}" \
+    -H "x-api-secret: ${KOMODO_API_SECRET}" \
+    -d "$2"
+}
+
+echo "Requesting ${EXECUTE} for '${KOMODO_TARGET}' on ${KOMODO_URL}"
+
+if ! code=$(api "/execute/${EXECUTE}" "$PARAMS"); then
+  echo "::error::Could not reach Komodo at ${KOMODO_URL}" >&2
+  exit 1
+fi
+
+body=$(cat "$tmp")
+
+if [ "$code" != "200" ]; then
+  echo "::error::Komodo rejected ${EXECUTE} for '${KOMODO_TARGET}' (HTTP ${code})" >&2
+  echo "$body" >&2
+  exit 1
+fi
+
+echo "Komodo accepted the request (HTTP 200)."
+
+# The execute endpoint returns immediately with an Update; the deploy itself
+# runs asynchronously. Poll /read/GetUpdate until it reaches a terminal state.
+if [ "$KOMODO_WAIT" != "true" ]; then
+  echo "KOMODO_WAIT is not 'true' - not waiting for completion."
+  exit 0
+fi
+
+if ! command -v python3 >/dev/null 2>&1; then
+  echo "python3 not available - cannot poll for completion. Deploy was queued."
+  exit 0
+fi
+
+update_id=$(printf '%s' "$body" | python3 -c '
+import json, sys
+d = json.load(sys.stdin)
+print(d.get("_id", {}).get("$oid", ""))
+' 2>/dev/null || true)
+
+if [ -z "$update_id" ]; then
+  echo "Could not read update id from response - not polling. Deploy was queued."
+  exit 0
+fi
+
+echo "Waiting for update ${update_id} (timeout ${KOMODO_WAIT_TIMEOUT}s)..."
+
+elapsed=0
+while [ "$elapsed" -lt "$KOMODO_WAIT_TIMEOUT" ]; do
+  sleep "$KOMODO_WAIT_INTERVAL"
+  elapsed=$((elapsed + KOMODO_WAIT_INTERVAL))
+
+  if ! code=$(api "/read/GetUpdate" "{\"id\": \"${update_id}\"}"); then
+    echo "::warning::Could not reach Komodo while polling; retrying." >&2
+    continue
+  fi
+
+  if [ "$code" != "200" ]; then
+    echo "::warning::GetUpdate returned HTTP ${code}; retrying." >&2
+    continue
+  fi
+
+  status=$(python3 -c '
+import json, sys
+print(json.load(sys.stdin).get("status", ""))
+' < "$tmp" 2>/dev/null || true)
+
+  case "$status" in
+    Complete)
+      if python3 -c '
+import json, sys
+sys.exit(0 if json.load(sys.stdin).get("success") else 1)
+' < "$tmp" 2>/dev/null; then
+        echo "Deploy of '${KOMODO_TARGET}' completed successfully."
+        exit 0
+      fi
+      echo "::error::Deploy of '${KOMODO_TARGET}' failed. Komodo update log:" >&2
+      python3 -c '
+import json, sys
+d = json.load(sys.stdin)
+for log in d.get("logs", []):
+    print(f"  [{log.get(\"stage\", \"\")}] {log.get(\"msg\", \"\")}")
+' < "$tmp" >&2 2>/dev/null || cat "$tmp" >&2
+      exit 1
+      ;;
+    Queued|InProgress)
+      echo "  ${elapsed}s: ${status}"
+      ;;
+    *)
+      echo "  ${elapsed}s: unexpected status '${status}'"
+      ;;
+  esac
+done
+
+echo "::error::Timed out after ${KOMODO_WAIT_TIMEOUT}s waiting for update ${update_id}" >&2
+exit 1

.gitea/workflows/docker-dev.yml

@@ -27,4 +27,17 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           push: true
-          tags: git.jonb.io/jblu/qbit-maid:dev
+          tags: git.jonb.io/jblu/qbit-maid:dev
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Deploy to Komodo
+        env:
+          KOMODO_URL: '${{ secrets.KOMODO_URL }}'
+          KOMODO_API_KEY: '${{ secrets.KOMODO_API_KEY }}'
+          KOMODO_API_SECRET: '${{ secrets.KOMODO_API_SECRET }}'
+          # Set the KOMODO_STACK_DEV repository variable to your dev stack name.
+          # When it is empty the deploy step is skipped, so dev builds still pass.
+          KOMODO_TARGET: '${{ vars.KOMODO_STACK_DEV }}'
+        run: sh ./.gitea/scripts/komodo-deploy.sh

.gitea/workflows/docker.yml

@@ -27,4 +27,16 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           push: true
-          tags: git.jonb.io/jblu/qbit-maid:latest
+          tags: git.jonb.io/jblu/qbit-maid:latest
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Deploy to Komodo
+        env:
+          KOMODO_URL: '${{ secrets.KOMODO_URL }}'
+          KOMODO_API_KEY: '${{ secrets.KOMODO_API_KEY }}'
+          KOMODO_API_SECRET: '${{ secrets.KOMODO_API_SECRET }}'
+          KOMODO_TARGET: 'qbit-maid'
+          KOMODO_REQUIRED: 'true'
+        run: sh ./.gitea/scripts/komodo-deploy.sh

.gitea/workflows/test.yml

@@ -0,0 +1,17 @@
+name: test
+
+on:
+  push:
+    branches:
+      - main
+      - dev*
+  pull_request:
+
+jobs:
+  komodo-deploy-script:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run komodo-deploy.sh tests
+        run: python3 -m unittest -v test_komodo_deploy

docs/komodo-deploy-plan.md

@@ -0,0 +1,143 @@
+# Auto-deploy qbit-maid to Komodo
+
+Status: **implemented** on branch `dev-komodo-deploy`.
+Komodo instance: `https://k.jonb.io` · Stack: `qbit-maid`
+
+## What this does
+
+Every successful image build now triggers a Komodo stack deploy:
+
+| Workflow | Trigger | Image tag | Komodo target |
+|---|---|---|---|
+| `.gitea/workflows/docker.yml` | push to `main` | `:latest` | stack `qbit-maid` |
+| `.gitea/workflows/docker-dev.yml` | push to `dev*` | `:dev` | `vars.KOMODO_STACK_DEV` (skipped if unset) |
+
+The deploy step runs **after** `Build and push`, so the new image is already in
+the registry when Komodo pulls it. If the deploy fails, the workflow goes red.
+
+## Why not just use the existing webhook?
+
+You already have webhooks configured, and this is the important part: **a push
+webhook on the stack does not reliably deploy new container builds.**
+
+The webhook fires on `git push`. That happens *before* Gitea Actions has built
+and pushed the image — the build takes minutes. So Komodo receives the deploy
+request while the registry still holds the *previous* `:latest`. It pulls that,
+deploys it, and reports success. The new code never lands until something else
+triggers a second deploy.
+
+The ordering is the whole problem, and a webhook cannot fix it because the
+webhook is tied to the push event, not the build event.
+
+So the workflow calls the Komodo API directly instead:
+
+```
+POST https://k.jonb.io/execute/DeployStack
+x-api-key: <key>
+x-api-secret: <secret>
+{"stack": "qbit-maid"}
+```
+
+This is the same call the Komodo UI makes when you click **Deploy**, just
+triggered from CI at the right moment.
+
+### What to do with the existing webhook
+
+If you have a stack webhook pointing at
+`/listener/github/stack/qbit-maid/deploy`, it is now redundant and will cause a
+second, useless deploy on every push (deploying the stale image). **Remove it**
+in Gitea → repo Settings → Webhooks.
+
+Keep webhooks only if you later point them at a *Procedure* rather than the
+stack — see "Using a Procedure" below.
+
+## One Komodo setting to verify
+
+`DeployStack` runs `docker compose pull` before `up` **only when the stack has
+`auto_pull` enabled**. It defaults to `true`, so this is probably already fine —
+but if it has been turned off, deploys will silently reuse the cached image and
+you will see no change.
+
+Check in the Komodo UI: stack `qbit-maid` → Config → **Auto Pull** should be on.
+
+## Setup (one-time)
+
+### 1. Komodo API key
+
+Komodo → **Settings → API Keys → New**. The key's user needs **Execute**
+permission on the `qbit-maid` stack. Copy the key and secret — the secret is
+shown once.
+
+### 2. Gitea secrets
+
+Repo → **Settings → Actions → Secrets**:
+
+| Name | Value |
+|---|---|
+| `KOMODO_URL` | `https://k.jonb.io` |
+| `KOMODO_API_KEY` | from step 1 |
+| `KOMODO_API_SECRET` | from step 1 |
+
+### 3. Dev stack (optional)
+
+Repo → **Settings → Actions → Variables**:
+
+| Name | Value |
+|---|---|
+| `KOMODO_STACK_DEV` | your dev stack name, e.g. `qbit-maid-dev` |
+
+If this variable is unset, the dev workflow logs `KOMODO_TARGET is not set -
+skipping Komodo deploy` and still passes. Dev builds never fail because of a
+missing dev stack.
+
+## The deploy script
+
+`.gitea/scripts/komodo-deploy.sh` is shared by both workflows. It:
+
+1. Skips cleanly if `KOMODO_TARGET` is empty (unless `KOMODO_REQUIRED=true`).
+2. POSTs to `/execute/DeployStack` (or `/execute/RunProcedure`).
+3. Fails on any non-200, printing Komodo's error body.
+4. Polls `/read/GetUpdate` until the update reaches `Complete`, then checks
+   `success` and prints the Komodo log on failure.
+
+The execute endpoint returns immediately with an `Update` object — a `200` means
+"queued", not "deployed". That is why step 4 exists.
+
+Tunable via env: `KOMODO_TARGET_TYPE` (`stack`/`procedure`), `KOMODO_WAIT`
+(default `true`), `KOMODO_WAIT_TIMEOUT` (default 600s), `KOMODO_WAIT_INTERVAL`
+(default 10s).
+
+### Using a Procedure instead of a direct stack deploy
+
+If you later wrap the deploy in a Komodo Procedure (deploy + healthcheck +
+notification, say), point the workflow at it instead of editing the script:
+
+```yaml
+env:
+  KOMODO_TARGET_TYPE: 'procedure'
+  KOMODO_TARGET: 'qbit-maid deploy'
+```
+
+`RunProcedure` takes `{"procedure": "<name>"}` and returns the same `Update`
+shape, so the polling logic is unchanged.
+
+## Testing
+
+Verified locally against a mock of the Komodo API — 12 checks covering the
+success path, deploy failure, bad credentials (401), unknown stack (400),
+empty target skip, required-but-empty target, unreachable host, and the
+procedure target type. All pass.
+
+Live verification still needed:
+
+1. Push `dev-komodo-deploy` — it matches `dev*`, so `docker-dev.yml` runs. With
+   `KOMODO_STACK_DEV` unset it will skip the deploy; set the variable first if
+   you want to exercise it.
+2. Merge to `main` and confirm the Komodo UI shows a new `DeployStack` update
+   for `qbit-maid`, operator = the API key's user.
+3. Confirm the running container picked up the new image digest.
+
+## Rollback
+
+Delete the `Deploy to Komodo` step from the workflow(s). Nothing on the Komodo
+side is modified by CI beyond triggering a deploy.

test_komodo_deploy.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+"""Tests for .gitea/scripts/komodo-deploy.sh.
+
+Runs the real script against a local mock of the Komodo Core API, so no
+network access or live Komodo instance is required.
+
+Run with:  python3 -m unittest -v test_komodo_deploy
+"""
+
+import json
+import os
+import socket
+import subprocess
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+REPO_ROOT = os.path.dirname(os.path.abspath(__file__))
+SCRIPT = os.path.join(REPO_ROOT, ".gitea", "scripts", "komodo-deploy.sh")
+
+
+def _free_port():
+    """Return a port that is currently closed, for the unreachable-host test."""
+    s = socket.socket()
+    s.bind(("127.0.0.1", 0))
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+
+class MockKomodo:
+    """Minimal mock of the Komodo Core endpoints used by the deploy script.
+
+    ``responses`` is a list of ``(status, body)`` pairs consumed in order. A
+    single-element list repeats forever, which is what the timeout test needs.
+    Every request is recorded in ``requests`` for assertions.
+    """
+
+    def __init__(self, responses):
+        self.responses = list(responses)
+        self.requests = []
+        mock = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def do_POST(self):
+                length = int(self.headers.get("Content-Length", 0))
+                raw = self.rfile.read(length).decode() if length else ""
+                mock.requests.append(
+                    {
+                        "path": self.path,
+                        "key": self.headers.get("x-api-key"),
+                        "secret": self.headers.get("x-api-secret"),
+                        "body": raw,
+                    }
+                )
+                status, body = (
+                    mock.responses.pop(0)
+                    if len(mock.responses) > 1
+                    else mock.responses[0]
+                )
+                payload = json.dumps(body).encode()
+                self.send_response(status)
+                self.send_header("Content-Type", "application/json")
+                self.send_header("Content-Length", str(len(payload)))
+                self.end_headers()
+                self.wfile.write(payload)
+
+            def log_message(self, *args):
+                pass
+
+        self.server = HTTPServer(("127.0.0.1", 0), Handler)
+        self.port = self.server.server_address[1]
+        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, *exc):
+        self.server.shutdown()
+        self.server.server_close()
+
+    @property
+    def url(self):
+        return "http://127.0.0.1:%d" % self.port
+
+
+def run_script(env_overrides, timeout=60):
+    """Run the deploy script with a clean KOMODO_* environment."""
+    env = {k: v for k, v in os.environ.items() if not k.startswith("KOMODO_")}
+    env.update(env_overrides)
+    return subprocess.run(
+        ["sh", SCRIPT],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+    )
+
+
+def base_env(url, **extra):
+    env = {
+        "KOMODO_URL": url,
+        "KOMODO_API_KEY": "test-key",
+        "KOMODO_API_SECRET": "test-secret",
+        "KOMODO_TARGET": "qbit-maid",
+        "KOMODO_WAIT_INTERVAL": "1",
+    }
+    env.update(extra)
+    return env
+
+
+class TestKomodoDeployScript(unittest.TestCase):
+    # --- target selection -------------------------------------------------
+
+    def test_skips_when_target_empty(self):
+        r = run_script({"KOMODO_TARGET": ""})
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("skipping", r.stdout)
+
+    def test_fails_when_target_empty_and_required(self):
+        r = run_script({"KOMODO_TARGET": "", "KOMODO_REQUIRED": "true"})
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("KOMODO_REQUIRED=true", r.stderr)
+
+    def test_fails_when_url_missing(self):
+        r = run_script({"KOMODO_TARGET": "qbit-maid"})
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("KOMODO_URL is required", r.stderr)
+
+    def test_fails_when_api_key_missing(self):
+        r = run_script(
+            {
+                "KOMODO_URL": "http://127.0.0.1:1",
+                "KOMODO_TARGET": "qbit-maid",
+                "KOMODO_API_SECRET": "s",
+            }
+        )
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("KOMODO_API_KEY is required", r.stderr)
+
+    def test_invalid_target_type(self):
+        r = run_script(
+            {
+                "KOMODO_URL": "http://127.0.0.1:1",
+                "KOMODO_API_KEY": "k",
+                "KOMODO_API_SECRET": "s",
+                "KOMODO_TARGET": "qbit-maid",
+                "KOMODO_TARGET_TYPE": "bogus",
+            }
+        )
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("must be 'stack' or 'procedure'", r.stderr)
+
+    # --- request shape ----------------------------------------------------
+
+    def test_sends_auth_headers_and_stack_body(self):
+        with MockKomodo(
+            [
+                (200, {"_id": {"$oid": "u1"}}),
+                (200, {"status": "Complete", "success": True}),
+            ]
+        ) as mock:
+            r = run_script(base_env(mock.url))
+        self.assertEqual(r.returncode, 0, r.stderr)
+        first = mock.requests[0]
+        self.assertEqual(first["path"], "/execute/DeployStack")
+        self.assertEqual(first["key"], "test-key")
+        self.assertEqual(first["secret"], "test-secret")
+        self.assertEqual(json.loads(first["body"]), {"stack": "qbit-maid"})
+
+    def test_procedure_target_type(self):
+        with MockKomodo(
+            [
+                (200, {"_id": {"$oid": "u1"}}),
+                (200, {"status": "Complete", "success": True}),
+            ]
+        ) as mock:
+            r = run_script(base_env(mock.url, KOMODO_TARGET_TYPE="procedure"))
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertEqual(mock.requests[0]["path"], "/execute/RunProcedure")
+        self.assertEqual(json.loads(mock.requests[0]["body"]), {"procedure": "qbit-maid"})
+
+    # --- success and failure ---------------------------------------------
+
+    def test_success_path(self):
+        with MockKomodo(
+            [
+                (200, {"_id": {"$oid": "u1"}}),
+                (200, {"status": "Complete", "success": True}),
+            ]
+        ) as mock:
+            r = run_script(base_env(mock.url))
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("completed successfully", r.stdout)
+
+    def test_deploy_failure_surfaces_logs(self):
+        with MockKomodo(
+            [
+                (200, {"_id": {"$oid": "u1"}}),
+                (
+                    200,
+                    {
+                        "status": "Complete",
+                        "success": False,
+                        "logs": [{"stage": "Deploy", "msg": "image pull failed"}],
+                    },
+                ),
+            ]
+        ) as mock:
+            r = run_script(base_env(mock.url))
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("failed", r.stderr)
+        self.assertIn("image pull failed", r.stderr)
+
+    def test_unauthorized(self):
+        with MockKomodo([(401, {"error": "bad credentials"})]) as mock:
+            r = run_script(base_env(mock.url))
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("HTTP 401", r.stderr)
+
+    def test_bad_request(self):
+        with MockKomodo([(400, {"error": "stack not found"})]) as mock:
+            r = run_script(base_env(mock.url))
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("HTTP 400", r.stderr)
+
+    def test_unreachable_host(self):
+        r = run_script(base_env("http://127.0.0.1:%d" % _free_port()))
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("Could not reach Komodo", r.stderr)
+
+    # --- polling ----------------------------------------------------------
+
+    def test_no_wait_returns_after_accept(self):
+        with MockKomodo([(200, {"_id": {"$oid": "u1"}})]) as mock:
+            r = run_script(base_env(mock.url, KOMODO_WAIT="false"))
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("not waiting", r.stdout)
+        self.assertEqual(len(mock.requests), 1)
+
+    def test_polls_until_complete(self):
+        with MockKomodo(
+            [
+                (200, {"_id": {"$oid": "u1"}}),
+                (200, {"status": "Queued"}),
+                (200, {"status": "InProgress"}),
+                (200, {"status": "Complete", "success": True}),
+            ]
+        ) as mock:
+            r = run_script(base_env(mock.url))
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("Queued", r.stdout)
+        self.assertIn("InProgress", r.stdout)
+        self.assertIn("completed successfully", r.stdout)
+        self.assertEqual(len(mock.requests), 4)
+
+    def test_timeout(self):
+        with MockKomodo(
+            [
+                (200, {"_id": {"$oid": "u1"}}),
+                (200, {"status": "InProgress"}),
+            ]
+        ) as mock:
+            r = run_script(
+                base_env(mock.url, KOMODO_WAIT_TIMEOUT="2", KOMODO_WAIT_INTERVAL="1")
+            )
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("Timed out", r.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)