Merge pull request 'dev-make-query-performant' (#16) from dev-make-query-performant into main

Reviewed-on: #16

config.toml.example

@@ -23,41 +23,19 @@ client_id = "eft-event-generator-confidential"
 secret = ""
 
 [database]
+overrideEmbeddedquery = false
 driver = "ODBC Driver 18 for SQL Server"
 server = "192.168.x.x"
 database = "EFTDB"
 user = "a"
 password = "a"
-query = """DECLARE	@stopTime DATETIME2
+query = """DECLARE @stopTime DATETIME2
-SET @stopTime = DATEADD(DAY, -30, GETDATE())
+SET @stopTime=DATEADD(DAY, -30, GETDATE())
-SELECT p.[ProtocolCommandID]
+SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
-      ,t.[Time_stamp]
+FROM tbl_Transactions t
-      ,p.[RemoteIP]
+     Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
-      ,p.[RemotePort]
+     Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
-      ,p.[LocalIP]
+WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
-      ,p.[LocalPort]
-      ,p.[Protocol]
-      ,p.[SiteName]
-      ,p.[Command]
-      ,p.[CommandParameters]
-      ,p.[FileName]
-      ,p.[VirtualFolderName]
-      ,p.[PhysicalFolderName]
-      ,p.[IsInternal]
-      ,p.[FileSize]
-      ,p.[TransferTime]
-      ,p.[BytesTransferred]
-      ,p.[ResultID]
-      ,t.[TransactionID]
-      ,p.[Description]
-      ,p.[Actor]
-	  ,t.ParentTransactionID
-	  ,t.TransactionObject
-	  ,t.NodeName
-	  ,t.TransactionGUID
-	  ,a.Protocol user_type
-  FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
-  WHERE p.Time_stamp > @stopTime"""
 
 [immutables]
 prd_instance_id = 1

inex.py

@@ -10,6 +10,7 @@ import json
 import requests
 import inexEncoder
 import inexSqlquery
+
 class Inex:
     def __init__(self):
         """Initilize config, calls functions from inex-connect.py and inex-logging.py"""
@@ -30,24 +31,30 @@ class Inex:
                 self.config = self.tl.load(c)
         
         # set config
-        self.dbDriver = self.config["database"]["driver"]
+        try:
-        self.dbServer = self.config["database"]["server"]
+            if self.config:
-        self.dbDatabase = self.config["database"]["database"]
+                self.dbDriver = self.config["database"]["driver"]
-        self.dbUser = self.config["database"]["user"]
+                self.dbServer = self.config["database"]["server"]
-        self.dbPassword = self.config["database"]["password"]
+                self.dbDatabase = self.config["database"]["database"]
-        self.dbQuery = self.config["database"]["query"]
+                self.dbUser = self.config["database"]["user"]
-        self.outputFile = self.config["output"]["filename"]
+                self.dbPassword = self.config["database"]["password"]
-        self.useLog = self.config["logging"]["useLog"]
+                self.dbQuery = self.config["database"]["query"]
-        self.logPath = self.config["logging"]["logPath"]
+                self.outputFile = self.config["output"]["filename"]
-        self.logLevel = self.config["logging"]["logLevel"]
+                self.useLog = self.config["logging"]["useLog"]
-        self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
+                self.logPath = self.config["logging"]["logPath"]
-        self.productGUID = self.config["immutables"]["product_guid"]
+                self.logLevel = self.config["logging"]["logLevel"]
-        self.productName = self.config["immutables"]["product_name"]
+                self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
-        self.productVersion = self.config["immutables"]["product_version"]
+                self.productGUID = self.config["immutables"]["product_guid"]
-        self.tokenFilepath = self.config["output"]["token"]
+                self.productName = self.config["immutables"]["product_name"]
-        self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
+                self.productVersion = self.config["immutables"]["product_version"]
-        self.writeJsonfile = self.config["output"]["dumpTojson"]
+                self.tokenFilepath = self.config["output"]["token"]
-        self.pushToplatform = self.config["output"]["pushToplatform"]
+                self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
+                self.writeJsonfile = self.config["output"]["dumpTojson"]
+                self.pushToplatform = self.config["output"]["pushToplatform"]
+                self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
+        except:
+            print("No config.toml. Please use example file and configure appropriately")
+            exit(1)
 
         if "dev" in self.selectedPlatform.lower():
             self.platformConfig = self.config["fortraPlatform"]["dev"]
@@ -55,7 +62,6 @@ class Inex:
             self.platformConfig = self.config["fortraPlatform"]["stage"]
         if "prod" in self.selectedPlatform.lower():
             self.platformConfig = self.config["fortraPlatform"]["prod"]
-        # print(self.platformConfig)
 
         #Setup logging
         inexLog(self)
@@ -63,9 +69,8 @@ class Inex:
         # create the connection to the database
         self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
 
-        # self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.dbQuery)
 
-        self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData())
+        self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery))
 
         self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
                                          product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)

inexDataModel.py

@@ -35,7 +35,7 @@ def dataTemplate(transactionType,**kwargs):
         "tenant_name":"GlobalScape",
         "time": kwargs.get('time'),
         "status_code": kwargs.get('status_code'),
-        "status_detail": kwargs.get('status_detail'),
+        "status_detail": kwargs.get('description'),
         "user": {
             "home_directory": kwargs.get('user_home_directory'),
             "uuid": kwargs.get('guid'),
@@ -123,5 +123,7 @@ def dataTemplate(transactionType,**kwargs):
         template = fileDeleted
     if transactionType == "user_logged_on":
         template = logon
+    if transactionType == "other":
+        template = {}
     
     return template

inexDataProcessing.py

@@ -7,44 +7,21 @@ def processData(data, template, **kwargs):
         # print(f'Row: {row}')
         if row.get('Command') == None:
             continue
-
+        try:
-        processedData.append(template(identifyUtype(row.get('Command')),\
+            processedData.append(template(identifyUtype(row.get('Command')),\
-            prd_ext_tenant_id='',\
+                prd_ext_tenant_id='',\
-            status_code=row.get('ResultID'),\
+                status_code=row.get('ResultID'),\
-            file_size=row.get('FileSize'),\
+                file_size=row.get('FileSize'),\
-            file_path=row.get('PhysicalFolderName'),\
+                file_path=row.get('PhysicalFolderName'),\
-            file_virtual_path=row.get('VirtualFolderName'),\
+                file_virtual_path=row.get('VirtualFolderName'),\
-            file_name=row.get('FileName'),\
+                file_name=row.get('FileName'),\
-            guid=row.get('TransactionGUID'),\
-            ref_id=row.get('ProtocolCommandID'),\
-            prd_instance_id=kwargs.get('prd_instance_id'),\
-            product_guid=kwargs.get('product_guid'),\
-            product_name=kwargs.get('product_name'),\
-            product_version=kwargs.get('product_version'),\
-            node_name=row.get('NodeName'),\
-            src_endpoint_type=row.get('Protocol'),\
-            src_endpoint_port=row.get('RemotePort'),\
-            src_endpoint_ip=row.get('RemoteIP'),\
-            dst_endpoint_port=row.get('LocalPort'),\
-            dst_endpoint_ip=row.get('LocalIP'),\
-            dst_endpoint_type=row.get('Protocol'),\
-            session_uid=row.get('TransactionID'),\
-            bytes_out=row.get('BytesTransferred'),\
-            duration=row.get('TransferTime'),\
-            time=row.get('Time_stamp'),\
-            user_type=identifyUserType(row.get('user_type')),\
-            user_domain=row.get('SiteName'),\
-            user_name=row.get('Actor'),\
-            user_home_directory=row.get('VirtualFolderName'),\
-            utype=identifyUtype(row.get('Command'))))
-
-        if row.get('TransactionGUID') not in transactionLoginid:
-            processedData.append(template(identifyUtype(row.get('TransactionObject')),\
                 guid=row.get('TransactionGUID'),\
+                ref_id=row.get('ProtocolCommandID'),\
                 prd_instance_id=kwargs.get('prd_instance_id'),\
                 product_guid=kwargs.get('product_guid'),\
                 product_name=kwargs.get('product_name'),\
                 product_version=kwargs.get('product_version'),\
+                node_name=row.get('NodeName'),\
                 src_endpoint_type=row.get('Protocol'),\
                 src_endpoint_port=row.get('RemotePort'),\
                 src_endpoint_ip=row.get('RemoteIP'),\
@@ -53,15 +30,47 @@ def processData(data, template, **kwargs):
                 dst_endpoint_type=row.get('Protocol'),\
                 session_uid=row.get('TransactionID'),\
                 bytes_out=row.get('BytesTransferred'),\
-                transfer_time=row.get('TransferTime'),\
+                duration=row.get('TransferTime'),\
                 time=row.get('Time_stamp'),\
                 user_type=identifyUserType(row.get('user_type')),\
                 user_domain=row.get('SiteName'),\
                 user_name=row.get('Actor'),\
                 user_home_directory=row.get('VirtualFolderName'),\
-                utype=identifyUtype(row.get('TransactionObject'))\
+                description=row.get('Description'),\
-                ))
+                utype=identifyUtype(row.get('Command'))))
-            transactionLoginid.append(row.get('TransactionGUID'))
+        except UnboundLocalError:
+            print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+
+            continue
+
+        if row.get('TransactionGUID') not in transactionLoginid:
+            try:
+                processedData.append(template(identifyUtype(row.get('TransactionObject')),\
+                    guid=row.get('TransactionGUID'),\
+                    prd_instance_id=kwargs.get('prd_instance_id'),\
+                    product_guid=kwargs.get('product_guid'),\
+                    product_name=kwargs.get('product_name'),\
+                    product_version=kwargs.get('product_version'),\
+                    src_endpoint_type=row.get('Protocol'),\
+                    src_endpoint_port=row.get('RemotePort'),\
+                    src_endpoint_ip=row.get('RemoteIP'),\
+                    dst_endpoint_port=row.get('LocalPort'),\
+                    dst_endpoint_ip=row.get('LocalIP'),\
+                    dst_endpoint_type=row.get('Protocol'),\
+                    session_uid=row.get('TransactionID'),\
+                    bytes_out=row.get('BytesTransferred'),\
+                    transfer_time=row.get('TransferTime'),\
+                    time=row.get('Time_stamp'),\
+                    user_type=identifyUserType(row.get('user_type')),\
+                    user_domain=row.get('SiteName'),\
+                    user_name=row.get('Actor'),\
+                    user_home_directory=row.get('VirtualFolderName'),\
+                    utype=identifyUtype(row.get('TransactionObject'))\
+                    ))
+                transactionLoginid.append(row.get('TransactionGUID'))
+            except UnboundLocalError:
+                print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+                continue
 
     return processedData
 
@@ -88,4 +97,4 @@ def identifyUtype(obj):
     if obj in file_downloaded:
         return "file_downloaded"
     else:
-        return None
+        return "other"

inexSqlquery.py

@@ -1,11 +1,11 @@
 class sqlQuerymodel:
-    def queryData():
+    def queryData(overRideflag, configQuery):
         """Embedded query data"""
         q ="""DECLARE @stopTime DATETIME2
         SET @stopTime=DATEADD(DAY, -30, GETDATE())
-        SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.CommandParameters, p.FileName, p.VirtualFolderName, p.PhysicalFolderName, p.IsInternal, p.FileSize, p.TransferTime, p.BytesTransferred, p.ResultID, t.TransactionID, p.Description, p.Actor, t.ParentTransactionID, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
+        SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
         FROM tbl_Transactions t
              Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
              Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)
-        WHERE p.Time_stamp>@stopTime"""
+        WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
-        return q
+        return configQuery if overRideflag else q