jblu/Inex
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 8 Wiki Activity

Updated data models

Browse Source
This commit is contained in:
Jonathan Branan committed 2024-08-16 15:31:33 -05:00
1 parent adcced2c1a
commit 55d90cb2dc
5 files changed
+43 -49
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

config.toml.example
+3 -5
View File
@@ -22,7 +22,7 @@ user = "a"
password = "a"
query = """DECLARE @stopTime DATETIME2
SET @stopTime=DATEADD(DAY, -30, GETDATE())
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
FROM tbl_Transactions t
Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
@@ -50,7 +50,5 @@ client_id = "eft-event-generator-confidential"
secret = ""
[immutables]
prd_instance_id = 1
product_guid = "asdf"
product_name = "EFT"
product_version ="8.1.0.9"
product_name = "GlobalScape EFT"
prd_ext_tenant_name = "GlobalScape EFT"
inex.py
+4 -7
View File
@@ -43,10 +43,8 @@ class Inex:
self.useLog = self.config["logging"]["useLog"]
self.logPath = self.config["logging"]["logPath"]
self.logLevel = self.config["logging"]["logLevel"]
self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
self.productGUID = self.config["immutables"]["product_guid"]
self.prdExttenantname = self.config["immutables"]["prd_ext_tenant_name"]
self.productName = self.config["immutables"]["product_name"]
self.productVersion = self.config["immutables"]["product_version"]
self.tokenFilepath = self.config["output"]["token"]
self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
self.writeJsonfile = self.config["output"]["dumpTojson"]
@@ -54,7 +52,7 @@ class Inex:
self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
self.queryDaystopull = self.config["database"]["daysTopull"]
except:
print("No config.toml or possibly missing settings in the file. Please use example file and configure appropriately")
print("No config.toml or possibly missing settings in the file. Please use config.toml.example file and configure appropriately")
exit(1)
if "dev" in self.selectedPlatform.lower():
@@ -73,9 +71,8 @@ class Inex:
self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull))
self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
product_guid=self.productGUID,product_name=self.productName,\
product_version=self.productVersion)
self.modifiedData = processData(self.data, dataTemplate, prd_ext_tenant_name=self.prdExttenantname,product_name=self.productName,\
prd_ext_tenant_id=self.platformConfig["tenant_id"])
if self.pushToplatform:
inexConnect.fortraEFC.__init__(self)
inexDataModel.py
+19 -19
View File
@@ -1,6 +1,6 @@
def dataTemplate(transactionType,**kwargs):
uploadDownload = {
"bytes" : kwargs.get('bytes_out'),
"bytes" : kwargs.get('bytes'),
"dst_endpoint": {
"port": kwargs.get('dst_endpoint_port'),
"ip": kwargs.get('dst_endpoint_ip'),
@@ -8,7 +8,7 @@ def dataTemplate(transactionType,**kwargs):
},
"duration": kwargs.get('duration'),
"file": {
"created_time": kwargs.get('time'),
"created_time": kwargs.get('file_created_time'),
"uid": kwargs.get('file_uid'),
"size": kwargs.get('file_size'),
"name": kwargs.get('file_name'),
@@ -17,8 +17,8 @@ def dataTemplate(transactionType,**kwargs):
"guid": f'{transactionType.split("_")[1].rstrip("d").rstrip("e")}:{kwargs.get("guid")}',
"node_name": kwargs.get('node_name'),
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
"product_name": "GlobalScape EFT",
"prd_ext_tenant_name": "GlobalScape EFT",
"product_name": kwargs.get('product_name'),
"prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
"classifications": [{
"ref_id": f"globalscape:{kwargs.get('guid')}",
"time": kwargs.get('time'),
@@ -32,15 +32,15 @@ def dataTemplate(transactionType,**kwargs):
"ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type')
},
"tenant": kwargs.get('tenant'),
"tenant": kwargs.get('prd_ext_tenant_id'),
"tenant_name":"GlobalScape",
"time": kwargs.get('time'),
"status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('description'),
"status_detail": kwargs.get('status_detail'),
"user": {
"home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('guid'),
"uid": kwargs.get('uid'),
"uid": kwargs.get('user_uid'),
"type": kwargs.get('user_type'),
"name": kwargs.get('user_name')
},
@@ -60,9 +60,9 @@ def dataTemplate(transactionType,**kwargs):
"ref_id": f"globalscape:{kwargs.get('guid')}",
"time": kwargs.get('time'),
}],
"prd_ext_tenant_name": "Globalscape EFT",
"prd_ext_tenant_name": kwargs.get("prd_ext_tenant_name"),
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
"product_name": "Globalscape EFT",
"product_name": kwargs.get("product_name"),
"session": {
"created_time": kwargs.get('time'),
"uid": kwargs.get('session_uid')
@@ -72,7 +72,7 @@ def dataTemplate(transactionType,**kwargs):
"ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type')
},
"tenant": kwargs.get('tenant'),
"tenant": kwargs.get('prd_ext_tenant_id'),
"tenant_name":"GlobalScape",
"dst_endpoint": {
"port": kwargs.get('dst_endpoint_port'),
@@ -81,11 +81,11 @@ def dataTemplate(transactionType,**kwargs):
},
"time": kwargs.get('time'),
"status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('description'),
"status_detail": kwargs.get('status_detail'),
"user": {
"home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('session_uid'),
"uid": kwargs.get('uid'),
"uuid": kwargs.get('user_session_uid'),
"uid": kwargs.get('user_uid'),
"type": kwargs.get('user_type'),
"name": kwargs.get('user_name')
},
@@ -104,13 +104,13 @@ def dataTemplate(transactionType,**kwargs):
},
"guid": kwargs.get('guid'),
"node_name": kwargs.get('node_name'),
"tenant": kwargs.get('tenant'),
"tenant": kwargs.get('prd_ext_tenant_id'),
"tenant_name":"GlobalScape",
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
"product_name": "GlobalScape EFT",
"prd_ext_tenant_name": "GlobalScape EFT",
"product_name": kwargs.get("product_name"),
"prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
"status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('description'),
"status_detail": kwargs.get('status_detail'),
"src_endpoint": {
"port": kwargs.get('src_endpoint_port'),
"ip": kwargs.get('src_endpoint_ip'),
@@ -119,8 +119,8 @@ def dataTemplate(transactionType,**kwargs):
"time": kwargs.get('time'),
"user": {
"home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('session_uid'),
"uid": kwargs.get('uid'),
"uuid": kwargs.get('user_session_uid'),
"uid": kwargs.get('user_uid'),
"type": kwargs.get('user_type'),
"name": kwargs.get('user_name')
},
inexDataProcessing.py
+16 -17
View File
@@ -13,36 +13,33 @@ def processData(data, template, **kwargs):
continue
try:
processedData.append(template(identifyUtype(row.get('Command')),\
status_detail='',\
# this should be taken from self.platformConfig
prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
user_uid=row.get('TransactionID'),\
status_detail=row.get('Description'),\
prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
status_code=row.get('ResultID'),\
file_created_time=row.get('Time_stamp'),\
file_size=row.get('FileSize'),\
file_uid=row.get('ProtocolCommandID'),\
file_path=row.get('PhysicalFolderName'),\
file_virtual_path=row.get('VirtualFolderName'),\
file_name=row.get('FileName'),\
guid=row.get('TransactionGUID'),\
ref_id=row.get('TransactionGUID'),\
prd_instance_id=kwargs.get('prd_instance_id'),\
product_guid=kwargs.get('product_guid'),\
product_name=kwargs.get('product_name'),\
product_version=kwargs.get('product_version'),\
node_name=row.get('NodeName'),\
session_uid=row.get('TransactionID'),\
src_endpoint_type=row.get('Protocol'),\
src_endpoint_port=row.get('RemotePort'),\
src_endpoint_ip=row.get('RemoteIP'),\
dst_endpoint_port=row.get('LocalPort'),\
dst_endpoint_ip=row.get('LocalIP'),\
dst_endpoint_type=row.get('Protocol'),\
session_uid=row.get('TransactionID'),\
bytes_out=row.get('BytesTransferred'),\
duration=row.get('TransferTime'),\
user_session_uid=row.get('TransactionID'),\
bytes=row.get('BytesTransferred'),\
time=row.get('Time_stamp'),\
duration=row.get('TransferTime'),\
user_type=identifyUserType(row.get('user_type')),\
user_domain=row.get('SiteName'),\
user_name=row.get('Actor'),\
user_home_directory=row.get('VirtualFolderName'),\
description=row.get('Description'),\
utype=identifyUtype(row.get('Command'))))
except UnboundLocalError:
print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
@@ -53,11 +50,13 @@ def processData(data, template, **kwargs):
try:
processedData.append(template(identifyUtype(row.get('TransactionObject')),\
prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
status_detail=row.get('Description'),\
guid=row.get('TransactionGUID'),\
status_code=row.get('ResultID'),\
node_name=row.get('NodeName'),\
prd_instance_id=kwargs.get('prd_instance_id'),\
product_guid=kwargs.get('product_guid'),\
product_name=kwargs.get('product_name'),\
product_version=kwargs.get('product_version'),\
src_endpoint_type=row.get('Protocol'),\
src_endpoint_port=row.get('RemotePort'),\
src_endpoint_ip=row.get('RemoteIP'),\
@@ -65,13 +64,13 @@ def processData(data, template, **kwargs):
dst_endpoint_ip=row.get('LocalIP'),\
dst_endpoint_type=row.get('Protocol'),\
session_uid=row.get('TransactionID'),\
bytes_out=row.get('BytesTransferred'),\
transfer_time=row.get('TransferTime'),\
time=row.get('Time_stamp'),\
user_session_uid=row.get('TransactionID'),\
user_uid=row.get('TransactionID'),\
user_type=identifyUserType(row.get('user_type')),\
user_domain=row.get('SiteName'),\
user_name=row.get('Actor'),\
user_home_directory=row.get('VirtualFolderName'),\
user_home_directory=row.get('PhysicalFolderName'),\
utype=identifyUtype(row.get('TransactionObject'))\
))
transactionLoginid.append(row.get('TransactionGUID'))
inexSqlquery.py
+1 -1
View File
@@ -3,7 +3,7 @@ class sqlQuerymodel:
"""Embedded query data"""
q ="""DECLARE @stopTime DATETIME2
SET @stopTime=DATEADD(DAY, -30, GETDATE())
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
FROM tbl_Transactions t
Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)