Merge pull request 'dev-make-query-performant' (#16) from dev-make-query-performant into main

Reviewed-on: #16

.gitea/workflows/build.yaml

@@ -1,21 +1,41 @@
-name: Build Inex Executable
+name: Build
-run-name: Deploy to ${{ inputs.deploy_target }} by @${{ gitea.actor }}
+on:
-on: [push]
+  push:
+    tags:
+    - '*'
 
 jobs:
-  Build:
+  build-linux-binary:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # release changes require contents write
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
         python-version: '3.12.4'
         cache: 'pip' # caching pip dependencies
+    - run: lsb_release -a
     - run: apt-get update
     - run: apt-get install unixodbc -y
     - run: pip install -r requirements.txt
     - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
-    - uses: actions/upload-artifact@v3
+    - uses: softprops/action-gh-release@v2
       with:
-        name: Inex
+        files: ${{ gitea.workspace }}/dist/inex
-        path: ${{ gitea.workspace }}/dist/inex
+
+  build-windows-binary:
+    runs-on: windows
+    permissions:
+      contents: write # release changes require contents write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12.4'
+          cache: 'pip' # caching pip dependencies
+      - run: python -m pip install -r requirements.txt
+      - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
+      - uses: softprops/action-gh-release@v2
+        with:
+          files: ${{ gitea.workspace }}/dist/inex.exe

.gitignore

@@ -1,4 +1,5 @@
-*config.toml
+*.toml
 *.json
 __pycache__/
 *.log
+*.token

LICENSE

@@ -209,7 +209,7 @@ If you develop a new program, and you want it to be of the greatest possible use
 To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
 
      Inex
-     Copyright (C) 2024  jblu
+     Copyright (C) 2024  Jonathan Branan
 
      This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 
@@ -221,7 +221,7 @@ Also add information on how to contact you by electronic and paper mail.
 
 If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
 
-     Inex  Copyright (C) 2024  jblu
+     Inex  Copyright (C) 2024  Jonathan Branan
      This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
      This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
 

config.toml.example

@@ -1,13 +1,53 @@
+[fortraPlatform]
+selectedPlatform = "dev"
+
+[fortraPlatform.dev]
+idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
+[fortraPlatform.stage]
+idp = "https://foundation.foundation-stage.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-stage.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
+[fortraPlatform.prod]
+idp ="https://foundation.foundation-prod.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-prod.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
 [database]
+overrideEmbeddedquery = false
 driver = "ODBC Driver 18 for SQL Server"
 server = "192.168.x.x"
 database = "EFTDB"
 user = "a"
 password = "a"
-query = "SELECT [Id],[Version] FROM [EFTDB].[dbo].[tbl_Schema_Version]"
+query = """DECLARE @stopTime DATETIME2
+SET @stopTime=DATEADD(DAY, -30, GETDATE())
+SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
+FROM tbl_Transactions t
+     Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
+     Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
+WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
+
+[immutables]
+prd_instance_id = 1
+product_guid = "asdf"
+product_name = "EFT"
+product_version ="8.1.0.9"
 
 [output]
+pushToplatform = true
+dumpTojson = true
 filename ="./data.json"
+token = "./.token"
 
 [logging]
 use_log = true

executable/config.toml.example

@@ -1,15 +0,0 @@
-[database]
-driver = "ODBC Driver 18 for SQL Server"
-server = "192.168.x.x"
-database = "EFTDB"
-user = "a"
-password = "a"
-query = "SELECT [Id],[Version] FROM [EFTDB].[dbo].[tbl_Schema_Version]"
-
-[output]
-filename ="./data.json"
-
-[logging]
-use_log = true
-logLevel = "debug"
-logPath = "./inex.log"

inex.py

@@ -1,68 +1,89 @@
 import pyodbc
 import os
 import logging
-import datetime
+import tomllib
-from tomllib import load
 from inexLogging import inexLog
 import inexConnect
 from inexDataModel import dataTemplate
 from inexDataProcessing import processData
 import json
-import decimal
+import requests
+import inexEncoder
+import inexSqlquery
 
 class Inex:
     def __init__(self):
         """Initilize config, calls functions from inex-connect.py and inex-logging.py"""
-        if os.path.exists('./config.toml'):
-            config_file_path = './config.toml'
-            with open(config_file_path, 'rb') as c:
-                self.config = load(c)
-
         # assign libraries
         self.db = pyodbc
-        self.tm = datetime
         self.il = logging
         self.ic = inexConnect
+        self.r = requests
+        self.tl = tomllib
+        self.os = os
+        self.j = json
+        self.e = inexEncoder.Encoder
+        self.sq = inexSqlquery
+
+        if self.os.path.exists('./config.toml'):
+            config_file_path = './config.toml'
+            with open(config_file_path, 'rb') as c:
+                self.config = self.tl.load(c)
         
         # set config
-        self.dbDriver = self.config["database"]["driver"]
+        try:
-        self.dbServer = self.config["database"]["server"]
+            if self.config:
-        self.dbDatabase = self.config["database"]["database"]
+                self.dbDriver = self.config["database"]["driver"]
-        self.dbUser = self.config["database"]["user"]
+                self.dbServer = self.config["database"]["server"]
-        self.dbPassword = self.config["database"]["password"]
+                self.dbDatabase = self.config["database"]["database"]
-        self.dbQuery = self.config["database"]["query"]
+                self.dbUser = self.config["database"]["user"]
-        self.outputFile = self.config["output"]["filename"]
+                self.dbPassword = self.config["database"]["password"]
-        self.useLog = self.config["logging"]["useLog"]
+                self.dbQuery = self.config["database"]["query"]
-        self.logPath = self.config["logging"]["logPath"]
+                self.outputFile = self.config["output"]["filename"]
-        self.logLevel = self.config["logging"]["logLevel"]
+                self.useLog = self.config["logging"]["useLog"]
+                self.logPath = self.config["logging"]["logPath"]
+                self.logLevel = self.config["logging"]["logLevel"]
+                self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
+                self.productGUID = self.config["immutables"]["product_guid"]
+                self.productName = self.config["immutables"]["product_name"]
+                self.productVersion = self.config["immutables"]["product_version"]
+                self.tokenFilepath = self.config["output"]["token"]
+                self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
+                self.writeJsonfile = self.config["output"]["dumpTojson"]
+                self.pushToplatform = self.config["output"]["pushToplatform"]
+                self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
+        except:
+            print("No config.toml. Please use example file and configure appropriately")
+            exit(1)
+
+        if "dev" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["dev"]
+        if "stag" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["stage"]
+        if "prod" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["prod"]
 
         #Setup logging
         inexLog(self)
 
         # create the connection to the database
-        self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
+        self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
 
-        self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery)
-        # print(f"returned data: {self.data}")
-        self.modifiedData = processData(self.data, dataTemplate)
 
-        print(self.modifiedData)
+        self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery))
+
+        self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
+                                         product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
+        
+        if self.pushToplatform:
+            inexConnect.fortraEFC.pushPayload(self)
 
         # TODO: move this to its own function
         if self.useLog:
             self.il.warning(f"Writing to '{self.outputFile}'.")
-
+        if self.writeJsonfile:
-        with open(self.outputFile, "w") as f:
+            with open(self.outputFile, "w") as f:
-            json.dump(self.modifiedData, f, indent = 2, cls=Encoder)
+                self.j.dump(self.modifiedData, f, indent = 2, cls=self.e)
-
-# TODO: Move this class to it's own file
-class Encoder(json.JSONEncoder):
-    def default(self, o):
-        if isinstance(o, decimal.Decimal):
-            return int(o)
-        if isinstance(o, datetime.datetime):
-            return str(o)
-        return super().default(o)
 
 # Run
 if  __name__== "__main__":

inexConnect.py

@@ -1,43 +1,74 @@
-def connectDatabase(self, lib, driver, server, database, user, password):
+class inexSql:
-    """Connects to the database. Requires a windows driver to do so.
+    def connectDatabase(self, lib, driver, server, database, user, password):
-    Typically there is one installed by default"""
+        """Connects to the database. Requires a windows driver to do so.
+        Typically there is one installed by default"""
 
-    connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
+        connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
-    if self.useLog:
-        self.il.debug(f"Connection String: connectionString")
-        self.il.info(f"Connecting to {database}@{server} with driver[{driver}].")
-    try:
-        connection = lib.connect(connectionString)
-    except lib.Error as ex:
-        sqlstate = ex.args[1]
         if self.useLog:
-            self.il.error(sqlstate)
+            self.il.debug(f"Connection String: connectionString")
-    if self.useLog:
+            self.il.info(f"Connecting to {database}@{server} with driver[{driver}].")
-        self.il.debug(f"Connected.")
+        try:
-    cursor = connection.cursor()
+            connection = lib.connect(connectionString)
-
+        except lib.Error as ex:
-    return  cursor
+            sqlstate = ex.args[1]
-
+            if self.useLog:
-def databaseQuery(self, cursor, query, args=()):
+                self.il.error(sqlstate)
-    if self.useLog:
-        self.il.debug(f"Query:")
-        self.il.debug(query)
-        self.il.info(f"Sending query:{query[0:20]}...")
-
-    try:
-        cur = cursor.execute(query, args)
-    except cur.Error as ex:
-        sqlstate = ex.args[1]
         if self.useLog:
-            self.il.error(sqlstate)
+            self.il.debug(f"Connected.")
+        cursor = connection.cursor()
 
-    if self.useLog:
+        return  cursor
-        self.il.debug(f"Processing database response...")
-    r = [dict((cur.description[i][0], value) \
-               for i, value in enumerate(row)) for row in cur.fetchall()]
 
-    cur.connection.close()
+    def databaseQuery(self, cursor, query, args=()):
-    if self.useLog:
+        if self.useLog:
-        self.il.debug(f"Database connection closed")
+            self.il.debug(f"Query:")
-    # return (r[0] if r else None) if one else r
+            self.il.debug(query)
-    return r
+            self.il.info(f"Sending query:{query[0:20]}...")
+
+        try:
+            cur = cursor.execute(query, args)
+        except cur.Error as ex:
+            sqlstate = ex.args[1]
+            if self.useLog:
+                self.il.error(sqlstate)
+
+        if self.useLog:
+            self.il.debug(f"Processing database response...")
+        r = [dict((cur.description[i][0], value) \
+                   for i, value in enumerate(row)) for row in cur.fetchall()]
+
+        cur.connection.close()
+        if self.useLog:
+            self.il.debug(f"Database connection closed")
+        return r
+
+class fortraEFC:
+    def getToken(self):
+        self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
+                                                                              "client_id": self.platformConfig["client_id"],\
+                                                                              "client_secret": self.platformConfig["secret"],})
+    def writeToken(self):
+        fortraEFC.getToken(self)
+        with open(self.tokenFilepath, "w") as f:
+            self.j.dump(self.tokenData.json(), f, indent = 2)
+
+    def readToken(self):
+        if self.os.path.exists(self.tokenFilepath):
+            with open(self.tokenFilepath, 'rb') as t:
+                self.tokenData = self.j.load(t)
+                # print(self.tokenData["access_token"])
+        else:
+            fortraEFC.writeToken(self)
+
+    def pushPayload(self):
+        fortraEFC.readToken(self)
+        try:
+            url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
+            pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\
+                                               json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e))
+            return pushPayloadResponse.status_code
+        except self.r.exceptions.HTTPError as errh:
+            print ("Http Error:",errh)
+            if "401" in errh:
+                fortraEFC.writeToken(self)
+                fortraEFC.pushPayload(self)

inexDataModel.py

@@ -1,48 +1,129 @@
-def dataTemplate(**kwargs):
+def dataTemplate(transactionType,**kwargs):
-    """Expects the following keyword arguments:
+    uploadDownload = {
-    status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name,
+        "bytes" : kwargs.get('bytes_out'),
-    guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name,
+        "dst_endpoint": { 
-    src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type,
+            "port": kwargs.get('dst_endpoint_port'),
-    session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype.
+            "ip": kwargs.get('dst_endpoint_ip'),
-    """
+            "type": kwargs.get('dst_endpoint_type')
-    template ={
+        },
-    "status": kwargs.get('status'),
+        "duration": kwargs.get('duration'),
-    "status_detail": kwargs.get('status_detail'),
+        "file": {
-	"status_code": kwargs.get('status_code'),
+            "created_time": kwargs.get('time'),
-    "file": {
+            "size": kwargs.get('file_size'),
-        "size": kwargs.get('file_size'),
+            "name": kwargs.get('file_name'),
-        "path": kwargs.get('file_path'),
+            "path": kwargs.get('file_path')
-        "virtual_path": kwargs.get('file_virtual_path'),
+        },
-        "name": kwargs.get('file_name')
+        "guid": kwargs.get('guid'),
-    },
+        "node_name": kwargs.get('node_name'),
-    "guid": kwargs.get('guid'),
+        "prd_ext_tenant_id": kwargs.get('tenant'),
-    "ref_id": kwargs.get('ref_id'),
+        "product_name": "GlobalScape EFT",
-    "prd_instance_id": kwargs.get('prd_instance_id'),
+        "prd_ext_tenant_name": "GlobalScape EFT",
-    "product_guid": kwargs.get('product_guid'),
+        "classifications": [{
-    "product_name": kwargs.get('product_name'),
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
-    "product_version": kwargs.get('product_version'),
+            "time": kwargs.get('time'),
-    "node_name":kwargs.get('node_name'),
+        }],
-    "src_endpoint": {
+        "session": {
-        "port": kwargs.get('src_endpoint_port'),
+           "created_time": kwargs.get('time'),
-        "ip": kwargs.get('src_endpoint_ip')
+           "uid": kwargs.get('session_uid')
-    },
+        },
-	"dst_endpoint": {
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "tenant": kwargs.get('tenant'),
+        "tenant_name":"GlobalScape",
+        "time": kwargs.get('time'),
+        "status_code": kwargs.get('status_code'),
+        "status_detail": kwargs.get('description'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+   
+    fileDeleted = {
+        "file": {
+                "size": kwargs.get('file_size'),
+                "name": kwargs.get('file_name'),
+                "path": kwargs.get('file_path')
+        },
+        "guid": kwargs.get('guid'),
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "prd_ext_tenant_name": "Globalscape EFT",
+        "prd_ext_tenant_id": kwargs.get('tenant'),
+        "product_name": "Globalscape EFT",
+        "session": {
+            "created_time": kwargs.get('time'),
+            "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "dst_endpoint": {
+            "port": kwargs.get('dst_endpoint_port'),
+            "ip": kwargs.get('dst_endpoint_ip'),
+            "type": kwargs.get('dst_endpoint_type')
+        },
+        "time": kwargs.get('time'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+    
+    logon ={
+    "classifications": [{
+        "ref_id": f"globalscape:{kwargs.get('guid')}",
+        "time": kwargs.get('time'),
+    }],
+    "dst_endpoint": {
         "port": kwargs.get('dst_endpoint_port'),
         "ip": kwargs.get('dst_endpoint_ip'),
         "type": kwargs.get('dst_endpoint_type')
     },
-    "session": {
+    "guid": kwargs.get('guid'),
-        "uid": kwargs.get('session_uid') 
+    "prd_ext_tenant_id": kwargs.get('tenant'),
+    "product_name": "GlobalScape EFT",
+    "prd_ext_tenant_name": "GlobalScape EFT",
+    "src_endpoint": {
+        "port": kwargs.get('src_endpoint_port'),
+        "ip": kwargs.get('src_endpoint_ip'),
+        "type": kwargs.get('src_endpoint_type')
     },
-	"bytes_out" : kwargs.get('bytes_out'),
-	"transfer_time" : kwargs.get('transfer_time'),
     "time": kwargs.get('time'),
     "user": {
+        "home_directory": kwargs.get('user_home_directory'),
+        "uuid": kwargs.get('guid'),
+        "uid": kwargs.get('uid'),
         "type": kwargs.get('user_type'),
-        "domain": kwargs.get('user_domain'),
         "name": kwargs.get('user_name')
     },
     "utype": kwargs.get('utype')
     }
 
+    if transactionType == "file_uploaded":
+        template = uploadDownload
+    if transactionType == "file_downloaded":
+        template = uploadDownload
+    if transactionType == "file_deleted":
+        template = fileDeleted
+    if transactionType == "user_logged_on":
+        template = logon
+    if transactionType == "other":
+        template = {}
+    
     return template

inexDataProcessing.py

@@ -1,32 +1,100 @@
-def processData(data, template):
+def processData(data, template, **kwargs):
+
     processedData = []
+    transactionLoginid = []
+
     for row in data:
         # print(f'Row: {row}')
-        processedData.append(template(status=row.get(''),\
+        if row.get('Command') == None:
-                            status_detail=row.get(''),\
+            continue
-                            status_code=row.get('ResultID'),\
+        try:
-                            file_size=row.get('FileSize'),\
+            processedData.append(template(identifyUtype(row.get('Command')),\
-                            file_path=row.get('PhysicalFolderName'),\
+                prd_ext_tenant_id='',\
-                            file_virtual_path=row.get('VirtualFolderName'),\
+                status_code=row.get('ResultID'),\
-                            file_name=row.get('FileName'),\
+                file_size=row.get('FileSize'),\
-                            guid=row.get('TransactionGUID'),\
+                file_path=row.get('PhysicalFolderName'),\
-                            ref_id=row.get('ProtocolCommandID'),\
+                file_virtual_path=row.get('VirtualFolderName'),\
-                            prd_instance_id=row.get(''),\
+                file_name=row.get('FileName'),\
-                            product_guid=row.get(''),\
+                guid=row.get('TransactionGUID'),\
-                            product_name=row.get(''),\
+                ref_id=row.get('ProtocolCommandID'),\
-                            product_version=row.get(''),\
+                prd_instance_id=kwargs.get('prd_instance_id'),\
-                            node_name=row.get('NodeName'),\
+                product_guid=kwargs.get('product_guid'),\
-                            src_endpoint_port=row.get('RemotePort'),\
+                product_name=kwargs.get('product_name'),\
-                            src_endpoint_ip=row.get('RemoteIP'),\
+                product_version=kwargs.get('product_version'),\
-                            dst_endpoint_port=row.get('LocalPort'),\
+                node_name=row.get('NodeName'),\
-                            dst_endpoint_ip=row.get('LocalIP'),\
+                src_endpoint_type=row.get('Protocol'),\
-                            dst_endpoint_type=row.get('Protocol'),\
+                src_endpoint_port=row.get('RemotePort'),\
-                            session_uid=row.get('TransactionID'),\
+                src_endpoint_ip=row.get('RemoteIP'),\
-                            bytes_out=row.get('BytesTransferred'),\
+                dst_endpoint_port=row.get('LocalPort'),\
-                            transfer_time=row.get('TransferTime'),\
+                dst_endpoint_ip=row.get('LocalIP'),\
-                            time=row.get('Time_stamp'),\
+                dst_endpoint_type=row.get('Protocol'),\
-                            user_type=row.get(''),\
+                session_uid=row.get('TransactionID'),\
-                            user_domain=row.get('SiteName'),\
+                bytes_out=row.get('BytesTransferred'),\
-                            user_name=row.get('Actor'),\
+                duration=row.get('TransferTime'),\
-                            utype=row.get('Command')))
+                time=row.get('Time_stamp'),\
+                user_type=identifyUserType(row.get('user_type')),\
+                user_domain=row.get('SiteName'),\
+                user_name=row.get('Actor'),\
+                user_home_directory=row.get('VirtualFolderName'),\
+                description=row.get('Description'),\
+                utype=identifyUtype(row.get('Command'))))
+        except UnboundLocalError:
+            print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+
+            continue
+
+        if row.get('TransactionGUID') not in transactionLoginid:
+            try:
+                processedData.append(template(identifyUtype(row.get('TransactionObject')),\
+                    guid=row.get('TransactionGUID'),\
+                    prd_instance_id=kwargs.get('prd_instance_id'),\
+                    product_guid=kwargs.get('product_guid'),\
+                    product_name=kwargs.get('product_name'),\
+                    product_version=kwargs.get('product_version'),\
+                    src_endpoint_type=row.get('Protocol'),\
+                    src_endpoint_port=row.get('RemotePort'),\
+                    src_endpoint_ip=row.get('RemoteIP'),\
+                    dst_endpoint_port=row.get('LocalPort'),\
+                    dst_endpoint_ip=row.get('LocalIP'),\
+                    dst_endpoint_type=row.get('Protocol'),\
+                    session_uid=row.get('TransactionID'),\
+                    bytes_out=row.get('BytesTransferred'),\
+                    transfer_time=row.get('TransferTime'),\
+                    time=row.get('Time_stamp'),\
+                    user_type=identifyUserType(row.get('user_type')),\
+                    user_domain=row.get('SiteName'),\
+                    user_name=row.get('Actor'),\
+                    user_home_directory=row.get('VirtualFolderName'),\
+                    utype=identifyUtype(row.get('TransactionObject'))\
+                    ))
+                transactionLoginid.append(row.get('TransactionGUID'))
+            except UnboundLocalError:
+                print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+                continue
+
     return processedData
+
+def identifyUserType(obj):
+    if obj:
+        if "Admin" in obj:
+            return "Administrator"
+        else:
+            return "User"
+    else:
+        return None
+def identifyUtype(obj):
+    user_logged_on = ['AUTH']
+    file_deleted = ["dele"]
+    file_uploaded = ["created"]
+    file_downloaded = ["sent"]
+
+    if obj in user_logged_on:
+        return "user_logged_on"
+    if obj in file_deleted:
+        return "file_deleted"
+    if obj in file_uploaded:
+        return "file_uploaded"
+    if obj in file_downloaded:
+        return "file_downloaded"
+    else:
+        return "other"

inexEncoder.py

@@ -0,0 +1,11 @@
+import json
+import decimal
+import datetime
+
+class Encoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, decimal.Decimal):
+            return int(o)
+        if isinstance(o, datetime.datetime):
+            return int(o.timestamp() * 1000)
+        return super().default(o)

inexSqlquery.py

@@ -0,0 +1,11 @@
+class sqlQuerymodel:
+    def queryData(overRideflag, configQuery):
+        """Embedded query data"""
+        q ="""DECLARE @stopTime DATETIME2
+        SET @stopTime=DATEADD(DAY, -30, GETDATE())
+        SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
+        FROM tbl_Transactions t
+             Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
+             Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)
+        WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
+        return configQuery if overRideflag else q

test.py

@@ -1,5 +1,21 @@
+import datetime
+
 def connectDatabase(driver, server, database, user, password):
     connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
     print(connectionString)
 
-a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
+# a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
+
+def converttimestamp(t):
+    print(int(t.timestamp()* 1000))
+
+def builddict(keys,*args,**kwargs):
+    dict = {}
+    for key in keys:
+        dict[key] = kwargs.get(key)
+    print(dict)
+
+testfolder = '/Usr/a/asdf/asf'
+user = 'a'
+
+print(testfolder.split(f"/{user}/"))