Merge pull request 'fix(authentication): BUG #20 - Authentication isnt working with existing logic & updated model' (#21) from fix-first-time-auth-error into main

Reviewed-on: #21

README.md

@@ -1,34 +1,93 @@
 # Inex
 
-Information Exchange - Pull data from EFT ARM then dump it to a json file
+### Information Exchange - Ingest ARM data into EFC
+This application is designed to pull data from an Microsoft SQL database, reformat it, and then send that data via the web to the Event Fusion Center. It's designed to be performant. The embedded query doesn't return unneccesary data and data is written to memory before being pushed into EFC. Initially you may have to pull a large amount of days so the system is caught up but the sweet spot would be to pull a set of data every day. 
 
-# Data Map:
-## tbl_ProtocolCommands
-- ProtocolCommandID
-- Time_stamp
-- RemoteIP
-- RemotePort
-- LocalIP
-- LocalPort
-- Protocol
-- SiteName
-- Command
-- CommandParameters
-- FileName
-- VirtualFolderName
-- PhysicalFolderName
-- IsInternal
-- FileSize
-- TransferTime
-- BytesTransferred
-- ResultID
-- TransactionID
-- Description
-- Actor
-## tbl_Transactions
-- TransactionID
-- ParentTransactionID
-- TransactionObject
-- TransactionGUID
-- NodeName
-- time_stamp
+## Build from source
+You will need to install git and clone the repository:
+
+`git clone https://git.jonb.io/jblu/Inex.git`
+
+Alternitively you can download the files from [the releases page](https://git.jonb.io/jblu/Inex/releases).
+
+Then follow OS specific instructions. These are taken directly from the steps used in automation to build the executables found in the releases page.
+##### Windows
+
+You can find the requirements.txt in the repository.
+
+`python -m pip install -r requirements.txt`
+
+`pyinstaller --noconfirm --onefile --console path/to/inex.py`
+
+##### Linux
+Update Linux Distro. This is Ubuntu specific. Please check the package management tool in your specific version.
+
+`apt-get update`
+
+You need unixodbc or else pyodbc cannot install.
+
+`apt-get install unixodbc -y`
+
+You can find the requirements.txt in the repository.
+
+`pip install -r requirements.txt`
+
+`pyinstaller --noconfirm --onefile --console path/to/inex.py`
+
+## Setup
+You will need a *config.toml* file in the same directory where *inex.py* or inex executable is. It's recommended to rename the existing config.toml.example file to save sometime and avoid omitting any data.
+
+> If the application doesn't find the *config.toml* at the time of execution, it will not continue.
+
+#### Config.toml
+| Table | Purpose |
+|-|-|
+| fortraPlatform | Fortra Specific data |
+| database | MSSQL Configuration |
+| immutables | Data that must be included but is not provided by the source database |
+| output | If and where to write files|
+| logging | Set if, where and level of logging |
+
+The following settings are not obvious as to how they affect the application.
+> Note the #comments
+```
+[fortraPlatform]
+selectedPlatform = "dev" # This will modify which environment the data is pushed to. The tenant_id and secret must be manually modified.
+
+[database]
+overrideEmbeddedquery = true # Choose if embedded query should be overridden.
+daysTopull = 30 # This setting is only related to the embedded query. Please note this will not affect query provided in config.toml
+driver = "ODBC Driver 18 for SQL Server" # Select which windows driver should be used. This one is recommended.
+
+[output]
+pushToplatform = false # if true, send data to fortraPlatform setting to false is useful in dry-runs.
+dumpTojson = true # if true, write data to json file
+token = "./.token" # Recommended to leave as-is
+
+```
+
+## Usage
+
+### Windows
+##### Run inex.py 
+1. Download [source](https://git.jonb.io/jblu/Inex/releases)
+2. `cd C:\path\to\Inex`
+3. `python inex.py`
+
+OR
+
+##### Run inex.exe
+1. Download [source](https://git.jonb.io/jblu/Inex/releases)
+2. `cd C:\path\to\Inex.exe`
+3. CMD `.\inex.exe`
+
+### Linux
+##### Run inex.py
+1. Download [source](https://git.jonb.io/jblu/Inex/releases)
+2. `cd /path/to/Inex`
+3. `python ./inex.py`
+
+OR
+1. Download [source](https://git.jonb.io/jblu/Inex/releases)
+2. `cd /path/to/Inex`
+3. RUN `./inex`

config.toml.example

@@ -1,6 +1,33 @@
 [fortraPlatform]
 selectedPlatform = "dev"
 
+[output]
+pushToplatform = false
+dumpTojson = true
+filename ="./data.json"
+token = "./.token"
+
+[logging]
+useLog = true
+logLevel = "debug"
+logPath = "./inex.log"
+
+[database]
+overrideEmbeddedquery = false
+daysTopull = 20
+driver = "ODBC Driver 18 for SQL Server"
+server = "192.168.x.x"
+database = "EFTDB"
+user = "a"
+password = "a"
+query = """DECLARE @stopTime DATETIME2
+SET @stopTime=DATEADD(DAY, -30, GETDATE())
+SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
+FROM tbl_Transactions t
+     Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
+     Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
+WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
+
 [fortraPlatform.dev]
 idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
 efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
@@ -22,56 +49,6 @@ tenant_id = ""
 client_id = "eft-event-generator-confidential"
 secret = ""
 
-[database]
-driver = "ODBC Driver 18 for SQL Server"
-server = "192.168.x.x"
-database = "EFTDB"
-user = "a"
-password = "a"
-query = """DECLARE	@stopTime DATETIME2
-SET @stopTime = DATEADD(DAY, -30, GETDATE())
-SELECT p.[ProtocolCommandID]
-      ,t.[Time_stamp]
-      ,p.[RemoteIP]
-      ,p.[RemotePort]
-      ,p.[LocalIP]
-      ,p.[LocalPort]
-      ,p.[Protocol]
-      ,p.[SiteName]
-      ,p.[Command]
-      ,p.[CommandParameters]
-      ,p.[FileName]
-      ,p.[VirtualFolderName]
-      ,p.[PhysicalFolderName]
-      ,p.[IsInternal]
-      ,p.[FileSize]
-      ,p.[TransferTime]
-      ,p.[BytesTransferred]
-      ,p.[ResultID]
-      ,t.[TransactionID]
-      ,p.[Description]
-      ,p.[Actor]
-	  ,t.ParentTransactionID
-	  ,t.TransactionObject
-	  ,t.NodeName
-	  ,t.TransactionGUID
-	  ,a.Protocol user_type
-  FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
-  WHERE p.Time_stamp > @stopTime"""
-
-[immutables]
-prd_instance_id = 1
-product_guid = "asdf"
-product_name = "EFT"
-product_version ="8.1.0.9"
-
-[output]
-pushToplatform = true
-dumpTojson = true
-filename ="./data.json"
-token = "./.token"
-
-[logging]
-use_log = true
-logLevel = "debug"
-logPath = "./inex.log"
+[immutables] 
+product_name = "GlobalScape EFT"
+prd_ext_tenant_name = "GlobalScape EFT"

inex.py

@@ -10,6 +10,7 @@ import json
 import requests
 import inexEncoder
 import inexSqlquery
+
 class Inex:
     def __init__(self):
         """Initilize config, calls functions from inex-connect.py and inex-logging.py"""
@@ -30,24 +31,32 @@ class Inex:
                 self.config = self.tl.load(c)
         
         # set config
-        self.dbDriver = self.config["database"]["driver"]
-        self.dbServer = self.config["database"]["server"]
-        self.dbDatabase = self.config["database"]["database"]
-        self.dbUser = self.config["database"]["user"]
-        self.dbPassword = self.config["database"]["password"]
-        self.dbQuery = self.config["database"]["query"]
-        self.outputFile = self.config["output"]["filename"]
-        self.useLog = self.config["logging"]["useLog"]
-        self.logPath = self.config["logging"]["logPath"]
-        self.logLevel = self.config["logging"]["logLevel"]
-        self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
-        self.productGUID = self.config["immutables"]["product_guid"]
-        self.productName = self.config["immutables"]["product_name"]
-        self.productVersion = self.config["immutables"]["product_version"]
-        self.tokenFilepath = self.config["output"]["token"]
-        self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
-        self.writeJsonfile = self.config["output"]["dumpTojson"]
-        self.pushToplatform = self.config["output"]["pushToplatform"]
+        try:
+            if self.config:
+                self.dbDriver = self.config["database"]["driver"]
+                self.dbServer = self.config["database"]["server"]
+                self.dbDatabase = self.config["database"]["database"]
+                self.dbUser = self.config["database"]["user"]
+                self.dbPassword = self.config["database"]["password"]
+                self.dbQuery = self.config["database"]["query"]
+                self.outputFile = self.config["output"]["filename"]
+                self.useLog = self.config["logging"]["useLog"]
+                self.logPath = self.config["logging"]["logPath"]
+                self.logLevel = self.config["logging"]["logLevel"]
+                self.prdExttenantname = self.config["immutables"]["prd_ext_tenant_name"]
+                self.productName = self.config["immutables"]["product_name"]
+                self.tokenFilepath = self.config["output"]["token"]
+                self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
+                self.writeJsonfile = self.config["output"]["dumpTojson"]
+                self.pushToplatform = self.config["output"]["pushToplatform"]
+                self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
+                self.queryDaystopull = self.config["database"]["daysTopull"]
+        except Exception as e:
+            print("No config.toml or possibly missing settings in the file. Please use config.toml.example file and configure appropriately")
+            self.il.error(e)
+            print(e)
+
+            exit(1)
 
         if "dev" in self.selectedPlatform.lower():
             self.platformConfig = self.config["fortraPlatform"]["dev"]
@@ -55,7 +64,6 @@ class Inex:
             self.platformConfig = self.config["fortraPlatform"]["stage"]
         if "prod" in self.selectedPlatform.lower():
             self.platformConfig = self.config["fortraPlatform"]["prod"]
-        # print(self.platformConfig)
 
         #Setup logging
         inexLog(self)
@@ -63,15 +71,14 @@ class Inex:
         # create the connection to the database
         self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
 
-        # self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.dbQuery)
 
-        self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData())
+        self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull))
+
+        self.modifiedData = processData(self.data, dataTemplate, prd_ext_tenant_name=self.prdExttenantname,product_name=self.productName,\
+                                        prd_ext_tenant_id=self.platformConfig["tenant_id"])
 
-        self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
-                                         product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
-        
         if self.pushToplatform:
-            inexConnect.fortraEFC.pushPayload(self)
+            inexConnect.fortraEFC.__init__(self)
 
         # TODO: move this to its own function
         if self.useLog:

inexConnect.py

@@ -43,32 +43,46 @@ class inexSql:
         return r
 
 class fortraEFC:
-    def getToken(self):
-        self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
-                                                                              "client_id": self.platformConfig["client_id"],\
-                                                                              "client_secret": self.platformConfig["secret"],})
-    def writeToken(self):
-        fortraEFC.getToken(self)
-        with open(self.tokenFilepath, "w") as f:
-            self.j.dump(self.tokenData.json(), f, indent = 2)
+    def __init__(self):
+        # Check if .token file is present
+        if fortraEFC.readToken(self) == 1:
+            # Get fresh token. First run.
+            fortraEFC.getToken(self)
+            fortraEFC.writeToken(self)
+        # Push data with token
+        self.pushPayloadresponse = fortraEFC.pushPayload(self)
+        if self.pushPayloadresponse == 401:
+            fortraEFC.getToken(self)
+            fortraEFC.writeToken(self)
+            fortraEFC.pushPayload(self)
 
     def readToken(self):
         if self.os.path.exists(self.tokenFilepath):
             with open(self.tokenFilepath, 'rb') as t:
                 self.tokenData = self.j.load(t)
-                # print(self.tokenData["access_token"])
+                self.il.debug(f'readToken {self.tokenData["access_token"]}')
+            return 0
         else:
-            fortraEFC.writeToken(self)
+            return 1
+    
+    def getToken(self):
+        self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
+                                                                              "client_id": self.platformConfig["client_id"],\
+                                                                              "client_secret": self.platformConfig["secret"],})
+        self.tokenData = self.tokenData.json()
+        self.il.debug(f'getToken {self.tokenData["access_token"]}')
+
+    def writeToken(self):
+        fortraEFC.getToken(self)
+        with open(self.tokenFilepath, "w") as f:
+            self.j.dump(self.tokenData, f, indent = 2)
+            self.il.debug(f'writeToken {self.tokenData["access_token"]}')
 
     def pushPayload(self):
-        fortraEFC.readToken(self)
-        try:
-            url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
-            pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\
-                                               json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e))
-            return pushPayloadResponse.status_code
-        except self.r.exceptions.HTTPError as errh:
-            print ("Http Error:",errh)
-            if "401" in errh:
-                fortraEFC.writeToken(self)
-                fortraEFC.pushPayload(self)
+        self.il.debug(f'pushPayload {self.tokenData["access_token"]}')
+        url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
+        pushPayloadResponse = self.r.post(url, headers={'Authorization': f'Bearer {self.tokenData["access_token"]}'},\
+                                           data=self.j.dumps(self.modifiedData, cls=self.e))
+        self.il.debug(pushPayloadResponse.status_code)
+        self.il.debug(pushPayloadResponse.text)
+        return pushPayloadResponse.status_code

inexDataModel.py

@@ -1,211 +1,191 @@
-def dataTemplate(**kwargs):
-    """Expects the following keyword arguments:
-    status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name,
-    guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name,
-    src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type,
-    session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype.
-    """
-    template ={
-    "status": kwargs.get('status'),
-    "status_detail": kwargs.get('status_detail'),
-	"status_code": kwargs.get('status_code'),
-    "file": {
-        "size": kwargs.get('file_size'),
-        "path": kwargs.get('file_path'),
-        "virtual_path": kwargs.get('file_virtual_path'),
-        "name": kwargs.get('file_name')
-    },
-    "guid": kwargs.get('guid'),
-    "ref_id": kwargs.get('ref_id'),
-    "prd_instance_id": kwargs.get('prd_instance_id'),
-    "product_guid": kwargs.get('product_guid'),
-    "product_name": kwargs.get('product_name'),
-    "product_version": kwargs.get('product_version'),
-    "node_name":kwargs.get('node_name'),
-    "src_endpoint": {
-        "port": kwargs.get('src_endpoint_port'),
-        "ip": kwargs.get('src_endpoint_ip')
-    },
-	"dst_endpoint": {
+def dataTemplate(transactionType,**kwargs):
+    upload = {
+        "bytes" : kwargs.get('bytes'),
+        "dst_endpoint": { 
+            "port": kwargs.get('dst_endpoint_port'),
+            "ip": kwargs.get('dst_endpoint_ip'),
+            "type": kwargs.get('dst_endpoint_type')
+        },
+        "duration": kwargs.get('duration'),
+        "file": {
+            "created_time": kwargs.get('file_created_time'),
+            "uid": kwargs.get('file_uid'),
+            "size": kwargs.get('file_size'),
+            "name": kwargs.get('file_name'),
+            "path": kwargs.get('file_path')
+        },
+        "guid": kwargs.get('file_uid'),
+        "node_name": kwargs.get('node_name'),
+        "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
+        "product_name": kwargs.get('product_name'),
+        "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "session": {
+           "created_time": kwargs.get('time'),
+           "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "tenant": kwargs.get('prd_ext_tenant_id'),
+        "tenant_name":"GlobalScape",
+        "time": kwargs.get('time'),
+        "status_code": kwargs.get('status_code'),
+        "status_detail": kwargs.get('status_detail'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('user_uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+    download = {
+        "bytes" : kwargs.get('bytes'),
+        "dst_endpoint": { 
+            "port": kwargs.get('dst_endpoint_port'),
+            "ip": kwargs.get('dst_endpoint_ip'),
+            "type": kwargs.get('dst_endpoint_type')
+        },
+        "duration": kwargs.get('duration'),
+        "file": {
+            "uid": kwargs.get('file_uid'),
+            "size": kwargs.get('file_size'),
+            "name": kwargs.get('file_name'),
+            "path": kwargs.get('file_path')
+        },
+        "guid": kwargs.get('file_uid'),
+        "node_name": kwargs.get('node_name'),
+        "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
+        "product_name": kwargs.get('product_name'),
+        "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "session": {
+           "created_time": kwargs.get('time'),
+           "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "tenant": kwargs.get('prd_ext_tenant_id'),
+        "tenant_name":"GlobalScape",
+        "time": kwargs.get('time'),
+        "status_code": kwargs.get('status_code'),
+        "status_detail": kwargs.get('status_detail'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('user_uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+   
+    fileDeleted = {
+        "file": {
+                "size": kwargs.get('file_size'),
+                "name": kwargs.get('file_name'),
+                "path": kwargs.get('file_path'),
+                "uid": kwargs.get('file_uid'),
+        },
+        "guid": f'deleted:{kwargs.get("guid")}',
+        "node_name": kwargs.get('node_name'),
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "prd_ext_tenant_name": kwargs.get("prd_ext_tenant_name"),
+        "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
+        "product_name": kwargs.get("product_name"),
+        "session": {
+            "created_time": kwargs.get('time'),
+            "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "tenant": kwargs.get('prd_ext_tenant_id'),
+        "tenant_name":"GlobalScape",
+        "dst_endpoint": {
+            "port": kwargs.get('dst_endpoint_port'),
+            "ip": kwargs.get('dst_endpoint_ip'),
+            "type": kwargs.get('dst_endpoint_type')
+        },
+        "time": kwargs.get('time'),
+        "status_code": kwargs.get('status_code'),
+        "status_detail": kwargs.get('status_detail'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('user_session_uid'),
+            "uid": kwargs.get('user_uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+    
+    logon ={
+    "classifications": [{
+        "ref_id": f"globalscape:{kwargs.get('guid')}",
+        "time": kwargs.get('time'),
+    }],
+    "dst_endpoint": {
         "port": kwargs.get('dst_endpoint_port'),
         "ip": kwargs.get('dst_endpoint_ip'),
         "type": kwargs.get('dst_endpoint_type')
     },
-    "session": {
-        "uid": kwargs.get('session_uid') 
+    "guid": kwargs.get('guid'),
+    "node_name": kwargs.get('node_name'),
+    "tenant": kwargs.get('prd_ext_tenant_id'),
+    "tenant_name":"GlobalScape",
+    "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
+    "product_name": kwargs.get("product_name"),
+    "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
+    "status_code": kwargs.get('status_code'),
+    "status_detail": kwargs.get('status_detail'),
+    "src_endpoint": {
+        "port": kwargs.get('src_endpoint_port'),
+        "ip": kwargs.get('src_endpoint_ip'),
+        "type": kwargs.get('src_endpoint_type')
     },
-	"bytes_out" : kwargs.get('bytes_out'),
-	"transfer_time" : kwargs.get('transfer_time'),
     "time": kwargs.get('time'),
     "user": {
+        "home_directory": kwargs.get('user_home_directory'),
+        "uuid": kwargs.get('user_session_uid'),
+        "uid": kwargs.get('user_uid'),
         "type": kwargs.get('user_type'),
-        "domain": kwargs.get('user_domain'),
         "name": kwargs.get('user_name')
     },
+    "session": {
+        "created_time": kwargs.get('time'),
+        "uid": kwargs.get('session_uid')
+    },
     "utype": kwargs.get('utype')
     }
 
-    return template
-
-FileUploaded = {
-    "bytes" : 2490,
-    "dst_endpoint": { 
-        "port": 22,
-        "ip": "10.91.160.77",
-        "type": "SFTP"
-    },
-    "duration": 200,
-    "file": {
-        "created_time": 1722485724000,
-        "size": 2490,
-        "name": "Case9.vbs",
-        "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //<PhysicalFolderName>"
-    },
-    "guid": "48D9C7A3-2DC6-11EF-AA59-00155D641204",
-    "node_name":"PERF01-S2019-77",
-    "prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-    "product_name": "GlobalScape EFT",
-    "prd_ext_tenant_name": "GlobalScape EFT",
-    "classifications": [{
-        "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641204",
-        "time":1722485724000,
-    }],
-    "session": {
-       "created_time":1722485724000,
-       "uid": "3615136"
-    },
-    "src_endpoint": {
-        "port": 58868,
-        "ip": "10.91.160.45",
-        "type":"SFTP"
-    },
-    "tenant": "e71851c2-593f-4f49-9c07-91727b1be94b",
-    "tenant_name":"GlobalScape",
-    "time":1722485724000,
-    "status_code":226,
-    "status_detail":"Upload Successful",
-    "user": {
-        "home_directory": "/Usr/Ivan/",
-        "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641204",
-        "uid": "3978403",
-        "type": "User",
-        "name": "Ivan"
-    },
-    "utype": "file_uploaded"
-}
-
-FileDownloaded = {
-"bytes" : 4891,
-"dst_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.77",
-    "type": "HTTPS"
-},
-"duration": 200,
-"file": {
-    "created_time": 1722518124000,
-    "size": 4891,
-    "name": "FileDownload1.exe",
-    "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //<PhysicalFolderName>"
-},
-"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641205",
-"node_name":"PERF01-S2019-77",
-"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"product_name": "GlobalScape EFT",
-"prd_ext_tenant_name": "GlobalScape EFT",
-"classifications": [{
-    "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641205",
-    "time":1722518124000,
-}],
-"session": {
-   "created_time":1722518124000,
-   "uid": "3615137"
-},
-"src_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.45",
-    "type":"HTTPS"
-},
-"tenant": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"tenant_name":"GlobalScape",
-"time":1722518124000,
-"status_code":226,
-"status_detail":"Download Successful",
-"user": {
-    "home_directory": "/Usr/Ivan/",
-    "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641205",
-    "uid": "3978404",
-    "type": "User",
-    "name": "Ivan" 
-},
-"utype": "file_downloaded"}
-
-FileDeleted = {
-        "file": {
-        "size": 304673,
-        "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan",
-"name": "DeleteME.txt"
-},
-"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641207",
-"classifications": [{
-    "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641207",
-    "time":1722515664000,
-}],
-"prd_ext_tenant_name": "Globalscape EFT",
-"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"product_name": "Globalscape EFT",
-"session": {
-   "created_time":1722515664000,
-   "uid": "3615138"
-},
-"src_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.45",
-    "type":"HTTPS"
-},
-"dst_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.77",
-    "type": "HTTPS"
-},
-"time": 1722515664000,
-"user": {
-    "home_directory": "/Usr/Ivan/",
-    "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641207",
-    "uid": "3978406",
-    "type": "User",
-    "name": "Ivan"
-},
-"utype": "file_deleted"
-}
-
-Logon ={
-"classifications": [{
-    "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641206",
-    "time": 1722510924000,
+    if transactionType == "file_uploaded":
+        template = upload
+    if transactionType == "file_downloaded":
+        template = download
+    if transactionType == "file_deleted":
+        template = fileDeleted
+    if transactionType == "user_logged_on":
+        template = logon
+    if transactionType == "other":
+        template = {}
     
-}],
-"dst_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.77",
-    "type": "HTTPS"
-},
-"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641206",
-"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"product_name": "GlobalScape EFT",
-"prd_ext_tenant_name": "GlobalScape EFT",
-"src_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.45",
-    "type":"HTTPS"
-},
-"time": 1722510924000,
-"user": {
-    "home_directory": "/Usr/Ivan/",
-    "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641206",
-    "uid": "3978405",
-    "type": "User",
-    "name": "Ivan"
-},
-"utype": "user_logged_on"
-}
+    return template

inexDataProcessing.py

@@ -1,37 +1,97 @@
 def processData(data, template, **kwargs):
+    """Translates data from sql query to the appropriate place in the respective template.
+    Accepts data, which is the sql query output, the template function, and finally
+    additional data to insert into the template."""
     processedData = []
+    transactionLoginid = []
+    
     for row in data:
         # print(f'Row: {row}')
-        processedData.append(template(status=row.get(''),\
-                            status_detail=row.get(''),\
-                            status_code=row.get('ResultID'),\
-                            file_size=row.get('FileSize'),\
-                            file_path=row.get('PhysicalFolderName'),\
-                            file_virtual_path=row.get('VirtualFolderName'),\
-                            file_name=row.get('FileName'),\
-                            guid=row.get('TransactionGUID'),\
-                            ref_id=row.get('ProtocolCommandID'),\
-                            prd_instance_id=kwargs.get('prd_instance_id'),\
-                            product_guid=kwargs.get('product_guid'),\
-                            product_name=kwargs.get('product_name'),\
-                            product_version=kwargs.get('product_version'),\
-                            node_name=row.get('NodeName'),\
-                            src_endpoint_port=row.get('RemotePort'),\
-                            src_endpoint_ip=row.get('RemoteIP'),\
-                            dst_endpoint_port=row.get('LocalPort'),\
-                            dst_endpoint_ip=row.get('LocalIP'),\
-                            dst_endpoint_type=row.get('Protocol'),\
-                            session_uid=row.get('TransactionID'),\
-                            bytes_out=row.get('BytesTransferred'),\
-                            transfer_time=row.get('TransferTime'),\
-                            time=row.get('Time_stamp'),\
-                            user_type=identifyUserType(row.get('user_type')),\
-                            user_domain=row.get('SiteName'),\
-                            user_name=row.get('Actor'),\
-                            utype=identifyUtype(row.get('TransactionObject'))))
+        # must set variables for the different templates and do logic based on that. Do not call identifyUtype many times
+        identifyUtypecommand = identifyUtype(row.get('Command'))
+
+        if identifyUtypecommand == "other":
+            continue
+
+        if row.get('Command') == None:
+            continue
+
+        userType = identifyUserType(row.get('user_type'))
+        try:
+            processedData.append(template(identifyUtypecommand,\
+                prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
+                user_uid=row.get('TransactionID'),\
+                status_detail=row.get('Description'),\
+                prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
+                status_code=row.get('ResultID'),\
+                file_created_time=row.get('Time_stamp'),\
+                file_size=row.get('FileSize'),\
+                file_uid=row.get('ProtocolCommandID'),\
+                file_path=row.get('PhysicalFolderName'),\
+                file_name=row.get('FileName'),\
+                guid=row.get('TransactionGUID'),\
+                product_name=kwargs.get('product_name'),\
+                node_name=row.get('NodeName'),\
+                session_uid=row.get('TransactionID'),\
+                src_endpoint_type=row.get('Protocol'),\
+                src_endpoint_port=row.get('RemotePort'),\
+                src_endpoint_ip=row.get('RemoteIP'),\
+                dst_endpoint_port=row.get('LocalPort'),\
+                dst_endpoint_ip=row.get('LocalIP'),\
+                dst_endpoint_type=row.get('Protocol'),\
+                user_session_uid=row.get('TransactionID'),\
+                bytes=row.get('BytesTransferred'),\
+                time=row.get('Time_stamp'),\
+                duration=row.get('TransferTime'),\
+                user_type=userType,\
+                user_name=row.get('Actor'),\
+                user_home_directory=row.get('VirtualFolderName'),\
+                utype=identifyUtypecommand))
+        except UnboundLocalError:
+            print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+            continue
+
+        identifyUtypetransactionObject = identifyUtype(row.get('TransactionObject'))
+
+        if identifyUtypetransactionObject == "other":
+            continue
+
+        if row.get('TransactionGUID') not in transactionLoginid:
+            try:
+                processedData.append(template(identifyUtypetransactionObject,\
+                    prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
+                    prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
+                    status_detail=row.get('Description'),\
+                    guid=row.get('TransactionGUID'),\
+                    status_code=row.get('ResultID'),\
+                    node_name=row.get('NodeName'),\
+                    prd_instance_id=kwargs.get('prd_instance_id'),\
+                    product_name=kwargs.get('product_name'),\
+                    src_endpoint_type=row.get('Protocol'),\
+                    src_endpoint_port=row.get('RemotePort'),\
+                    src_endpoint_ip=row.get('RemoteIP'),\
+                    dst_endpoint_port=row.get('LocalPort'),\
+                    dst_endpoint_ip=row.get('LocalIP'),\
+                    dst_endpoint_type=row.get('Protocol'),\
+                    session_uid=row.get('TransactionID'),\
+                    transfer_time=row.get('TransferTime'),\
+                    time=row.get('Time_stamp'),\
+                    user_session_uid=row.get('TransactionID'),\
+                    user_uid=row.get('TransactionID'),\
+                    user_type=userType,\
+                    user_name=row.get('Actor'),\
+                    user_home_directory=row.get('PhysicalFolderName'),\
+                    utype=identifyUtypetransactionObject\
+                    ))
+                transactionLoginid.append(row.get('TransactionGUID'))
+            except UnboundLocalError:
+                print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
+                continue
+
     return processedData
 
 def identifyUserType(obj):
+    """Check string if it has Admin-> return Administrator else return User."""
     if obj:
         if "Admin" in obj:
             return "Administrator"
@@ -39,19 +99,22 @@ def identifyUserType(obj):
             return "User"
     else:
         return None
+
 def identifyUtype(obj):
-    user_logged_on = []
-    file_deleted = []
-    file_uploaded = []
-    file_downloaded = []
+    """Process Type of transaction based on string that passed in.
+    Return transaction type."""
+    user_logged_on = ['AUTH']
+    file_deleted = ["dele"]
+    file_uploaded = ["created"]
+    file_downloaded = ["sent"]
 
     if obj in user_logged_on:
         return "user_logged_on"
-    if obj in file_deleted:
+    elif obj in file_deleted:
         return "file_deleted"
-    if obj in file_uploaded:
+    elif obj in file_uploaded:
         return "file_uploaded"
-    if obj in file_downloaded:
+    elif obj in file_downloaded:
         return "file_downloaded"
     else:
-        return None
+        return "other"

inexEncoder.py

@@ -3,6 +3,8 @@ import decimal
 import datetime
 
 class Encoder(json.JSONEncoder):
+    """Encoder uses json.JSONEncoder and checks for instances of decimal and datetime. 
+    Changes decimal.Decimal to int and datetime.datetime to unix timestamp with miliseconds."""
     def default(self, o):
         if isinstance(o, decimal.Decimal):
             return int(o)

inexSqlquery.py

@@ -1,11 +1,11 @@
 class sqlQuerymodel:
-    def queryData():
+    def queryData(overRideflag, configQuery, daysTopull):
         """Embedded query data"""
         q ="""DECLARE @stopTime DATETIME2
         SET @stopTime=DATEADD(DAY, -30, GETDATE())
-        SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.CommandParameters, p.FileName, p.VirtualFolderName, p.PhysicalFolderName, p.IsInternal, p.FileSize, p.TransferTime, p.BytesTransferred, p.ResultID, t.TransactionID, p.Description, p.Actor, t.ParentTransactionID, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
+        SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
         FROM tbl_Transactions t
              Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
              Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)
-        WHERE p.Time_stamp>@stopTime"""
-        return q
+        WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL""".replace("30", str(daysTopull))
+        return configQuery if overRideflag else q

test.py

@@ -9,4 +9,34 @@ def connectDatabase(driver, server, database, user, password):
 def converttimestamp(t):
     print(int(t.timestamp()* 1000))
 
-a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))
+def builddict(keys,*args,**kwargs):
+    dict = {}
+    for key in keys:
+        dict[key] = kwargs.get(key)
+    print(dict)
+
+testfolder = '/Usr/a/asdf/asf'
+user = 'a'
+
+def identifyUtype(obj):
+    """Process Type of transaction based on string that passed in.
+    Return transaction type."""
+    user_logged_on = ['AUTH']
+    file_deleted = ["dele"]
+    file_uploaded = ["created"]
+    file_downloaded = ["sent"]
+
+    if obj in user_logged_on:
+        return "user_logged_on"
+    if obj in file_deleted:
+        return "file_deleted"
+    if obj in file_uploaded:
+        return "file_uploaded"
+    if obj in file_downloaded:
+        return "file_downloaded"
+    else:
+        return "other"
+    
+transactionType = 'file_uploaded'
+
+print(transactionType.split("_")[1].rstrip("d").rstrip("e"))