Merge pull request 'dev-make-query-performant' (#16) from dev-make-query-performant into main

Reviewed-on: #16

.gitea/workflows/build.yaml

@@ -1,5 +1,8 @@
 name: Build
-on: push
+on:
+  push:
+    tags:
+    - '*'
 
 jobs:
   build-linux-binary:
@@ -18,7 +21,6 @@ jobs:
     - run: pip install -r requirements.txt
     - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
     - uses: softprops/action-gh-release@v2
-      if: startsWith(gitea.ref, 'refs/tags/')
       with:
         files: ${{ gitea.workspace }}/dist/inex
 
@@ -35,6 +37,5 @@ jobs:
       - run: python -m pip install -r requirements.txt
       - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
       - uses: softprops/action-gh-release@v2
-        if: startsWith(gitea.ref, 'refs/tags/')
         with:
           files: ${{ gitea.workspace }}/dist/inex.exe

config.toml.example

@@ -23,6 +23,7 @@ client_id = "eft-event-generator-confidential"
 secret = ""
 
 [database]
+overrideEmbeddedquery = false
 driver = "ODBC Driver 18 for SQL Server"
 server = "192.168.x.x"
 database = "EFTDB"
@@ -30,34 +31,11 @@ user = "a"
 password = "a"
 query = """DECLARE @stopTime DATETIME2
 SET @stopTime=DATEADD(DAY, -30, GETDATE())
-SELECT p.[ProtocolCommandID]
+SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
-      ,t.[Time_stamp]
+FROM tbl_Transactions t
-      ,p.[RemoteIP]
+     Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
-      ,p.[RemotePort]
+     Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
-      ,p.[LocalIP]
+WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
-      ,p.[LocalPort]
-      ,p.[Protocol]
-      ,p.[SiteName]
-      ,p.[Command]
-      ,p.[CommandParameters]
-      ,p.[FileName]
-      ,p.[VirtualFolderName]
-      ,p.[PhysicalFolderName]
-      ,p.[IsInternal]
-      ,p.[FileSize]
-      ,p.[TransferTime]
-      ,p.[BytesTransferred]
-      ,p.[ResultID]
-      ,t.[TransactionID]
-      ,p.[Description]
-      ,p.[Actor]
-	  ,t.ParentTransactionID
-	  ,t.TransactionObject
-	  ,t.NodeName
-	  ,t.TransactionGUID
-	  ,a.Protocol user_type
-  FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
-  WHERE p.Time_stamp > @stopTime"""
 
 [immutables]
 prd_instance_id = 1

inex.py

@@ -1,23 +1,21 @@
 import pyodbc
 import os
 import logging
-import datetime
 import tomllib
 from inexLogging import inexLog
 import inexConnect
 from inexDataModel import dataTemplate
 from inexDataProcessing import processData
 import json
-import decimal
 import requests
 import inexEncoder
+import inexSqlquery
 
 class Inex:
     def __init__(self):
         """Initilize config, calls functions from inex-connect.py and inex-logging.py"""
         # assign libraries
         self.db = pyodbc
-        self.tm = datetime
         self.il = logging
         self.ic = inexConnect
         self.r = requests
@@ -25,6 +23,7 @@ class Inex:
         self.os = os
         self.j = json
         self.e = inexEncoder.Encoder
+        self.sq = inexSqlquery
 
         if self.os.path.exists('./config.toml'):
             config_file_path = './config.toml'
@@ -32,6 +31,8 @@ class Inex:
                 self.config = self.tl.load(c)
         
         # set config
+        try:
+            if self.config:
                 self.dbDriver = self.config["database"]["driver"]
                 self.dbServer = self.config["database"]["server"]
                 self.dbDatabase = self.config["database"]["database"]
@@ -50,6 +51,10 @@ class Inex:
                 self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
                 self.writeJsonfile = self.config["output"]["dumpTojson"]
                 self.pushToplatform = self.config["output"]["pushToplatform"]
+                self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
+        except:
+            print("No config.toml. Please use example file and configure appropriately")
+            exit(1)
 
         if "dev" in self.selectedPlatform.lower():
             self.platformConfig = self.config["fortraPlatform"]["dev"]
@@ -57,15 +62,15 @@ class Inex:
             self.platformConfig = self.config["fortraPlatform"]["stage"]
         if "prod" in self.selectedPlatform.lower():
             self.platformConfig = self.config["fortraPlatform"]["prod"]
-        # print(self.platformConfig)
 
         #Setup logging
         inexLog(self)
 
         # create the connection to the database
-        self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
+        self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
 
-        self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery)
+
+        self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery))
 
         self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
                                          product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)

inexConnect.py

@@ -1,3 +1,4 @@
+class inexSql:
     def connectDatabase(self, lib, driver, server, database, user, password):
         """Connects to the database. Requires a windows driver to do so.
         Typically there is one installed by default"""

inexDataModel.py

@@ -1,48 +1,129 @@
-def dataTemplate(**kwargs):
+def dataTemplate(transactionType,**kwargs):
-    """Expects the following keyword arguments:
+    uploadDownload = {
-    status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name,
+        "bytes" : kwargs.get('bytes_out'),
-    guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name,
+        "dst_endpoint": { 
-    src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type,
+            "port": kwargs.get('dst_endpoint_port'),
-    session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype.
+            "ip": kwargs.get('dst_endpoint_ip'),
-    """
+            "type": kwargs.get('dst_endpoint_type')
-    template ={
+        },
-    "status": kwargs.get('status'),
+        "duration": kwargs.get('duration'),
-    "status_detail": kwargs.get('status_detail'),
-	"status_code": kwargs.get('status_code'),
         "file": {
+            "created_time": kwargs.get('time'),
             "size": kwargs.get('file_size'),
-        "path": kwargs.get('file_path'),
+            "name": kwargs.get('file_name'),
-        "virtual_path": kwargs.get('file_virtual_path'),
+            "path": kwargs.get('file_path')
-        "name": kwargs.get('file_name')
         },
         "guid": kwargs.get('guid'),
-    "ref_id": kwargs.get('ref_id'),
-    "prd_instance_id": kwargs.get('prd_instance_id'),
-    "product_guid": kwargs.get('product_guid'),
-    "product_name": kwargs.get('product_name'),
-    "product_version": kwargs.get('product_version'),
         "node_name": kwargs.get('node_name'),
+        "prd_ext_tenant_id": kwargs.get('tenant'),
+        "product_name": "GlobalScape EFT",
+        "prd_ext_tenant_name": "GlobalScape EFT",
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "session": {
+           "created_time": kwargs.get('time'),
+           "uid": kwargs.get('session_uid')
+        },
         "src_endpoint": {
             "port": kwargs.get('src_endpoint_port'),
-        "ip": kwargs.get('src_endpoint_ip')
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "tenant": kwargs.get('tenant'),
+        "tenant_name":"GlobalScape",
+        "time": kwargs.get('time'),
+        "status_code": kwargs.get('status_code'),
+        "status_detail": kwargs.get('description'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+   
+    fileDeleted = {
+        "file": {
+                "size": kwargs.get('file_size'),
+                "name": kwargs.get('file_name'),
+                "path": kwargs.get('file_path')
+        },
+        "guid": kwargs.get('guid'),
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "prd_ext_tenant_name": "Globalscape EFT",
+        "prd_ext_tenant_id": kwargs.get('tenant'),
+        "product_name": "Globalscape EFT",
+        "session": {
+            "created_time": kwargs.get('time'),
+            "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
         },
         "dst_endpoint": {
             "port": kwargs.get('dst_endpoint_port'),
             "ip": kwargs.get('dst_endpoint_ip'),
             "type": kwargs.get('dst_endpoint_type')
         },
-    "session": {
-        "uid": kwargs.get('session_uid') 
-    },
-	"bytes_out" : kwargs.get('bytes_out'),
-	"transfer_time" : kwargs.get('transfer_time'),
         "time": kwargs.get('time'),
         "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('uid'),
             "type": kwargs.get('user_type'),
-        "domain": kwargs.get('user_domain'),
             "name": kwargs.get('user_name')
         },
         "utype": kwargs.get('utype')
     }
     
+    logon ={
+    "classifications": [{
+        "ref_id": f"globalscape:{kwargs.get('guid')}",
+        "time": kwargs.get('time'),
+    }],
+    "dst_endpoint": {
+        "port": kwargs.get('dst_endpoint_port'),
+        "ip": kwargs.get('dst_endpoint_ip'),
+        "type": kwargs.get('dst_endpoint_type')
+    },
+    "guid": kwargs.get('guid'),
+    "prd_ext_tenant_id": kwargs.get('tenant'),
+    "product_name": "GlobalScape EFT",
+    "prd_ext_tenant_name": "GlobalScape EFT",
+    "src_endpoint": {
+        "port": kwargs.get('src_endpoint_port'),
+        "ip": kwargs.get('src_endpoint_ip'),
+        "type": kwargs.get('src_endpoint_type')
+    },
+    "time": kwargs.get('time'),
+    "user": {
+        "home_directory": kwargs.get('user_home_directory'),
+        "uuid": kwargs.get('guid'),
+        "uid": kwargs.get('uid'),
+        "type": kwargs.get('user_type'),
+        "name": kwargs.get('user_name')
+    },
+    "utype": kwargs.get('utype')
+    }
+
+    if transactionType == "file_uploaded":
+        template = uploadDownload
+    if transactionType == "file_downloaded":
+        template = uploadDownload
+    if transactionType == "file_deleted":
+        template = fileDeleted
+    if transactionType == "user_logged_on":
+        template = logon
+    if transactionType == "other":
+        template = {}
+    
     return template

inexDataProcessing.py

@@ -1,9 +1,15 @@
 def processData(data, template, **kwargs):
+
     processedData = []
+    transactionLoginid = []
+
     for row in data:
         # print(f'Row: {row}')
-        processedData.append(template(status=row.get(''),\
+        if row.get('Command') == None:
-                            status_detail=row.get(''),\
+            continue
+        try:
+            processedData.append(template(identifyUtype(row.get('Command')),\
+                prd_ext_tenant_id='',\
                 status_code=row.get('ResultID'),\
                 file_size=row.get('FileSize'),\
                 file_path=row.get('PhysicalFolderName'),\
@@ -16,6 +22,36 @@ def processData(data, template, **kwargs):
                 product_name=kwargs.get('product_name'),\
                 product_version=kwargs.get('product_version'),\
                 node_name=row.get('NodeName'),\
+                src_endpoint_type=row.get('Protocol'),\
+                src_endpoint_port=row.get('RemotePort'),\
+                src_endpoint_ip=row.get('RemoteIP'),\
+                dst_endpoint_port=row.get('LocalPort'),\
+                dst_endpoint_ip=row.get('LocalIP'),\
+                dst_endpoint_type=row.get('Protocol'),\
+                session_uid=row.get('TransactionID'),\
+                bytes_out=row.get('BytesTransferred'),\
+                duration=row.get('TransferTime'),\
+                time=row.get('Time_stamp'),\
+                user_type=identifyUserType(row.get('user_type')),\
+                user_domain=row.get('SiteName'),\
+                user_name=row.get('Actor'),\
+                user_home_directory=row.get('VirtualFolderName'),\
+                description=row.get('Description'),\
+                utype=identifyUtype(row.get('Command'))))
+        except UnboundLocalError:
+            print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+
+            continue
+
+        if row.get('TransactionGUID') not in transactionLoginid:
+            try:
+                processedData.append(template(identifyUtype(row.get('TransactionObject')),\
+                    guid=row.get('TransactionGUID'),\
+                    prd_instance_id=kwargs.get('prd_instance_id'),\
+                    product_guid=kwargs.get('product_guid'),\
+                    product_name=kwargs.get('product_name'),\
+                    product_version=kwargs.get('product_version'),\
+                    src_endpoint_type=row.get('Protocol'),\
                     src_endpoint_port=row.get('RemotePort'),\
                     src_endpoint_ip=row.get('RemoteIP'),\
                     dst_endpoint_port=row.get('LocalPort'),\
@@ -28,7 +64,14 @@ def processData(data, template, **kwargs):
                     user_type=identifyUserType(row.get('user_type')),\
                     user_domain=row.get('SiteName'),\
                     user_name=row.get('Actor'),\
-                            utype=identifyUtype(row.get('TransactionObject'))))
+                    user_home_directory=row.get('VirtualFolderName'),\
+                    utype=identifyUtype(row.get('TransactionObject'))\
+                    ))
+                transactionLoginid.append(row.get('TransactionGUID'))
+            except UnboundLocalError:
+                print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+                continue
+
     return processedData
 
 def identifyUserType(obj):
@@ -40,10 +83,10 @@ def identifyUserType(obj):
     else:
         return None
 def identifyUtype(obj):
-    user_logged_on = []
+    user_logged_on = ['AUTH']
-    file_deleted = []
+    file_deleted = ["dele"]
-    file_uploaded = []
+    file_uploaded = ["created"]
-    file_downloaded = []
+    file_downloaded = ["sent"]
 
     if obj in user_logged_on:
         return "user_logged_on"
@@ -54,4 +97,4 @@ def identifyUtype(obj):
     if obj in file_downloaded:
         return "file_downloaded"
     else:
-        return None
+        return "other"

inexSqlquery.py

@@ -0,0 +1,11 @@
+class sqlQuerymodel:
+    def queryData(overRideflag, configQuery):
+        """Embedded query data"""
+        q ="""DECLARE @stopTime DATETIME2
+        SET @stopTime=DATEADD(DAY, -30, GETDATE())
+        SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
+        FROM tbl_Transactions t
+             Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
+             Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)
+        WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
+        return configQuery if overRideflag else q

test.py

@@ -9,4 +9,13 @@ def connectDatabase(driver, server, database, user, password):
 def converttimestamp(t):
     print(int(t.timestamp()* 1000))
 
-a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))
+def builddict(keys,*args,**kwargs):
+    dict = {}
+    for key in keys:
+        dict[key] = kwargs.get(key)
+    print(dict)
+
+testfolder = '/Usr/a/asdf/asf'
+user = 'a'
+
+print(testfolder.split(f"/{user}/"))