From 55d90cb2dccbfea94be9b3a39aeab7b37f865ee1 Mon Sep 17 00:00:00 2001 From: Jonathan Branan Date: Fri, 16 Aug 2024 15:31:33 -0500 Subject: [PATCH] Updated data models --- config.toml.example | 8 +++----- inex.py | 11 ++++------- inexDataModel.py | 38 +++++++++++++++++++------------------- inexDataProcessing.py | 33 ++++++++++++++++----------------- inexSqlquery.py | 2 +- 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/config.toml.example b/config.toml.example index eb131bc..48e4219 100644 --- a/config.toml.example +++ b/config.toml.example @@ -22,7 +22,7 @@ user = "a" password = "a" query = """DECLARE @stopTime DATETIME2 SET @stopTime=DATEADD(DAY, -30, GETDATE()) -SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type +SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type FROM tbl_Transactions t Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID) @@ -50,7 +50,5 @@ client_id = "eft-event-generator-confidential" secret = "" [immutables] -prd_instance_id = 1 -product_guid = "asdf" -product_name = "EFT" -product_version ="8.1.0.9" \ No newline at end of file +product_name = "GlobalScape EFT" +prd_ext_tenant_name = "GlobalScape EFT" \ No newline at end of file diff --git a/inex.py b/inex.py index 2c69a07..7b70b58 100644 --- a/inex.py +++ b/inex.py @@ -43,10 +43,8 @@ class Inex: self.useLog = self.config["logging"]["useLog"] self.logPath = self.config["logging"]["logPath"] self.logLevel = self.config["logging"]["logLevel"] - self.prdInstanceID = self.config["immutables"]["prd_instance_id"] - self.productGUID = self.config["immutables"]["product_guid"] + self.prdExttenantname = self.config["immutables"]["prd_ext_tenant_name"] self.productName = self.config["immutables"]["product_name"] - self.productVersion = self.config["immutables"]["product_version"] self.tokenFilepath = self.config["output"]["token"] self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] self.writeJsonfile = self.config["output"]["dumpTojson"] @@ -54,7 +52,7 @@ class Inex: self.queryOverride = self.config["database"]["overrideEmbeddedquery"] self.queryDaystopull = self.config["database"]["daysTopull"] except: - print("No config.toml or possibly missing settings in the file. Please use example file and configure appropriately") + print("No config.toml or possibly missing settings in the file. Please use config.toml.example file and configure appropriately") exit(1) if "dev" in self.selectedPlatform.lower(): @@ -73,9 +71,8 @@ class Inex: self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull)) - self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ - product_guid=self.productGUID,product_name=self.productName,\ - product_version=self.productVersion) + self.modifiedData = processData(self.data, dataTemplate, prd_ext_tenant_name=self.prdExttenantname,product_name=self.productName,\ + prd_ext_tenant_id=self.platformConfig["tenant_id"]) if self.pushToplatform: inexConnect.fortraEFC.__init__(self) diff --git a/inexDataModel.py b/inexDataModel.py index ccacaa8..82559d2 100644 --- a/inexDataModel.py +++ b/inexDataModel.py @@ -1,6 +1,6 @@ def dataTemplate(transactionType,**kwargs): uploadDownload = { - "bytes" : kwargs.get('bytes_out'), + "bytes" : kwargs.get('bytes'), "dst_endpoint": { "port": kwargs.get('dst_endpoint_port'), "ip": kwargs.get('dst_endpoint_ip'), @@ -8,7 +8,7 @@ def dataTemplate(transactionType,**kwargs): }, "duration": kwargs.get('duration'), "file": { - "created_time": kwargs.get('time'), + "created_time": kwargs.get('file_created_time'), "uid": kwargs.get('file_uid'), "size": kwargs.get('file_size'), "name": kwargs.get('file_name'), @@ -17,8 +17,8 @@ def dataTemplate(transactionType,**kwargs): "guid": f'{transactionType.split("_")[1].rstrip("d").rstrip("e")}:{kwargs.get("guid")}', "node_name": kwargs.get('node_name'), "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), - "product_name": "GlobalScape EFT", - "prd_ext_tenant_name": "GlobalScape EFT", + "product_name": kwargs.get('product_name'), + "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'), "classifications": [{ "ref_id": f"globalscape:{kwargs.get('guid')}", "time": kwargs.get('time'), @@ -32,15 +32,15 @@ def dataTemplate(transactionType,**kwargs): "ip": kwargs.get('src_endpoint_ip'), "type": kwargs.get('src_endpoint_type') }, - "tenant": kwargs.get('tenant'), + "tenant": kwargs.get('prd_ext_tenant_id'), "tenant_name":"GlobalScape", "time": kwargs.get('time'), "status_code": kwargs.get('status_code'), - "status_detail": kwargs.get('description'), + "status_detail": kwargs.get('status_detail'), "user": { "home_directory": kwargs.get('user_home_directory'), "uuid": kwargs.get('guid'), - "uid": kwargs.get('uid'), + "uid": kwargs.get('user_uid'), "type": kwargs.get('user_type'), "name": kwargs.get('user_name') }, @@ -60,9 +60,9 @@ def dataTemplate(transactionType,**kwargs): "ref_id": f"globalscape:{kwargs.get('guid')}", "time": kwargs.get('time'), }], - "prd_ext_tenant_name": "Globalscape EFT", + "prd_ext_tenant_name": kwargs.get("prd_ext_tenant_name"), "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), - "product_name": "Globalscape EFT", + "product_name": kwargs.get("product_name"), "session": { "created_time": kwargs.get('time'), "uid": kwargs.get('session_uid') @@ -72,7 +72,7 @@ def dataTemplate(transactionType,**kwargs): "ip": kwargs.get('src_endpoint_ip'), "type": kwargs.get('src_endpoint_type') }, - "tenant": kwargs.get('tenant'), + "tenant": kwargs.get('prd_ext_tenant_id'), "tenant_name":"GlobalScape", "dst_endpoint": { "port": kwargs.get('dst_endpoint_port'), @@ -81,11 +81,11 @@ def dataTemplate(transactionType,**kwargs): }, "time": kwargs.get('time'), "status_code": kwargs.get('status_code'), - "status_detail": kwargs.get('description'), + "status_detail": kwargs.get('status_detail'), "user": { "home_directory": kwargs.get('user_home_directory'), - "uuid": kwargs.get('session_uid'), - "uid": kwargs.get('uid'), + "uuid": kwargs.get('user_session_uid'), + "uid": kwargs.get('user_uid'), "type": kwargs.get('user_type'), "name": kwargs.get('user_name') }, @@ -104,13 +104,13 @@ def dataTemplate(transactionType,**kwargs): }, "guid": kwargs.get('guid'), "node_name": kwargs.get('node_name'), - "tenant": kwargs.get('tenant'), + "tenant": kwargs.get('prd_ext_tenant_id'), "tenant_name":"GlobalScape", "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), - "product_name": "GlobalScape EFT", - "prd_ext_tenant_name": "GlobalScape EFT", + "product_name": kwargs.get("product_name"), + "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'), "status_code": kwargs.get('status_code'), - "status_detail": kwargs.get('description'), + "status_detail": kwargs.get('status_detail'), "src_endpoint": { "port": kwargs.get('src_endpoint_port'), "ip": kwargs.get('src_endpoint_ip'), @@ -119,8 +119,8 @@ def dataTemplate(transactionType,**kwargs): "time": kwargs.get('time'), "user": { "home_directory": kwargs.get('user_home_directory'), - "uuid": kwargs.get('session_uid'), - "uid": kwargs.get('uid'), + "uuid": kwargs.get('user_session_uid'), + "uid": kwargs.get('user_uid'), "type": kwargs.get('user_type'), "name": kwargs.get('user_name') }, diff --git a/inexDataProcessing.py b/inexDataProcessing.py index f275ff9..45a79b6 100644 --- a/inexDataProcessing.py +++ b/inexDataProcessing.py @@ -13,36 +13,33 @@ def processData(data, template, **kwargs): continue try: processedData.append(template(identifyUtype(row.get('Command')),\ - status_detail='',\ - # this should be taken from self.platformConfig + prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\ + user_uid=row.get('TransactionID'),\ + status_detail=row.get('Description'),\ prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\ status_code=row.get('ResultID'),\ + file_created_time=row.get('Time_stamp'),\ file_size=row.get('FileSize'),\ + file_uid=row.get('ProtocolCommandID'),\ file_path=row.get('PhysicalFolderName'),\ - file_virtual_path=row.get('VirtualFolderName'),\ file_name=row.get('FileName'),\ guid=row.get('TransactionGUID'),\ - ref_id=row.get('TransactionGUID'),\ - prd_instance_id=kwargs.get('prd_instance_id'),\ - product_guid=kwargs.get('product_guid'),\ product_name=kwargs.get('product_name'),\ - product_version=kwargs.get('product_version'),\ node_name=row.get('NodeName'),\ + session_uid=row.get('TransactionID'),\ src_endpoint_type=row.get('Protocol'),\ src_endpoint_port=row.get('RemotePort'),\ src_endpoint_ip=row.get('RemoteIP'),\ dst_endpoint_port=row.get('LocalPort'),\ dst_endpoint_ip=row.get('LocalIP'),\ dst_endpoint_type=row.get('Protocol'),\ - session_uid=row.get('TransactionID'),\ - bytes_out=row.get('BytesTransferred'),\ - duration=row.get('TransferTime'),\ + user_session_uid=row.get('TransactionID'),\ + bytes=row.get('BytesTransferred'),\ time=row.get('Time_stamp'),\ + duration=row.get('TransferTime'),\ user_type=identifyUserType(row.get('user_type')),\ - user_domain=row.get('SiteName'),\ user_name=row.get('Actor'),\ user_home_directory=row.get('VirtualFolderName'),\ - description=row.get('Description'),\ utype=identifyUtype(row.get('Command')))) except UnboundLocalError: print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}') @@ -53,11 +50,13 @@ def processData(data, template, **kwargs): try: processedData.append(template(identifyUtype(row.get('TransactionObject')),\ prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\ + prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\ + status_detail=row.get('Description'),\ guid=row.get('TransactionGUID'),\ + status_code=row.get('ResultID'),\ + node_name=row.get('NodeName'),\ prd_instance_id=kwargs.get('prd_instance_id'),\ - product_guid=kwargs.get('product_guid'),\ product_name=kwargs.get('product_name'),\ - product_version=kwargs.get('product_version'),\ src_endpoint_type=row.get('Protocol'),\ src_endpoint_port=row.get('RemotePort'),\ src_endpoint_ip=row.get('RemoteIP'),\ @@ -65,13 +64,13 @@ def processData(data, template, **kwargs): dst_endpoint_ip=row.get('LocalIP'),\ dst_endpoint_type=row.get('Protocol'),\ session_uid=row.get('TransactionID'),\ - bytes_out=row.get('BytesTransferred'),\ transfer_time=row.get('TransferTime'),\ time=row.get('Time_stamp'),\ + user_session_uid=row.get('TransactionID'),\ + user_uid=row.get('TransactionID'),\ user_type=identifyUserType(row.get('user_type')),\ - user_domain=row.get('SiteName'),\ user_name=row.get('Actor'),\ - user_home_directory=row.get('VirtualFolderName'),\ + user_home_directory=row.get('PhysicalFolderName'),\ utype=identifyUtype(row.get('TransactionObject'))\ )) transactionLoginid.append(row.get('TransactionGUID')) diff --git a/inexSqlquery.py b/inexSqlquery.py index 9c10e87..f7a080e 100644 --- a/inexSqlquery.py +++ b/inexSqlquery.py @@ -3,7 +3,7 @@ class sqlQuerymodel: """Embedded query data""" q ="""DECLARE @stopTime DATETIME2 SET @stopTime=DATEADD(DAY, -30, GETDATE()) - SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type + SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type FROM tbl_Transactions t Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)