jblu/Inex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases 8 Wiki Activity

Updated data models

Browse Source
This commit is contained in:
Jonathan Branan 2024-08-16 15:31:33 -05:00
parent adcced2c1a
commit 55d90cb2dc
5 changed files with 43 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
config.toml.example
View File

@ -22,7 +22,7 @@ user = "a"
password = "a" password = "a"
query = """DECLARE @stopTime DATETIME2 query = """DECLARE @stopTime DATETIME2
SET @stopTime=DATEADD(DAY, -30, GETDATE()) SET @stopTime=DATEADD(DAY, -30, GETDATE())
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
FROM tbl_Transactions t FROM tbl_Transactions t
Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID) Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
@ -50,7 +50,5 @@ client_id = "eft-event-generator-confidential"
secret = "" secret = ""
[immutables] [immutables]
prd_instance_id = 1 product_name = "GlobalScape EFT"
product_guid = "asdf" prd_ext_tenant_name = "GlobalScape EFT"
product_name = "EFT"
product_version ="8.1.0.9"

11
inex.py
View File

@ -43,10 +43,8 @@ class Inex:
self.useLog = self.config["logging"]["useLog"] self.useLog = self.config["logging"]["useLog"]
self.logPath = self.config["logging"]["logPath"] self.logPath = self.config["logging"]["logPath"]
self.logLevel = self.config["logging"]["logLevel"] self.logLevel = self.config["logging"]["logLevel"]
self.prdInstanceID = self.config["immutables"]["prd_instance_id"] self.prdExttenantname = self.config["immutables"]["prd_ext_tenant_name"]
self.productGUID = self.config["immutables"]["product_guid"]
self.productName = self.config["immutables"]["product_name"] self.productName = self.config["immutables"]["product_name"]
self.productVersion = self.config["immutables"]["product_version"]
self.tokenFilepath = self.config["output"]["token"] self.tokenFilepath = self.config["output"]["token"]
self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
self.writeJsonfile = self.config["output"]["dumpTojson"] self.writeJsonfile = self.config["output"]["dumpTojson"]
@ -54,7 +52,7 @@ class Inex:
self.queryOverride = self.config["database"]["overrideEmbeddedquery"] self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
self.queryDaystopull = self.config["database"]["daysTopull"] self.queryDaystopull = self.config["database"]["daysTopull"]
except: except:
print("No config.toml or possibly missing settings in the file. Please use example file and configure appropriately") print("No config.toml or possibly missing settings in the file. Please use config.toml.example file and configure appropriately")
exit(1) exit(1)
if "dev" in self.selectedPlatform.lower(): if "dev" in self.selectedPlatform.lower():
@ -73,9 +71,8 @@ class Inex:
self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull)) self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull))
self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ self.modifiedData = processData(self.data, dataTemplate, prd_ext_tenant_name=self.prdExttenantname,product_name=self.productName,\
product_guid=self.productGUID,product_name=self.productName,\ prd_ext_tenant_id=self.platformConfig["tenant_id"])
product_version=self.productVersion)
if self.pushToplatform: if self.pushToplatform:
inexConnect.fortraEFC.__init__(self) inexConnect.fortraEFC.__init__(self)

38
inexDataModel.py
View File

@ -1,6 +1,6 @@
def dataTemplate(transactionType,**kwargs): def dataTemplate(transactionType,**kwargs):
uploadDownload = { uploadDownload = {
"bytes" : kwargs.get('bytes_out'), "bytes" : kwargs.get('bytes'),
"dst_endpoint": { "dst_endpoint": {
"port": kwargs.get('dst_endpoint_port'), "port": kwargs.get('dst_endpoint_port'),
"ip": kwargs.get('dst_endpoint_ip'), "ip": kwargs.get('dst_endpoint_ip'),
@ -8,7 +8,7 @@ def dataTemplate(transactionType,**kwargs):
}, },
"duration": kwargs.get('duration'), "duration": kwargs.get('duration'),
"file": { "file": {
"created_time": kwargs.get('time'), "created_time": kwargs.get('file_created_time'),
"uid": kwargs.get('file_uid'), "uid": kwargs.get('file_uid'),
"size": kwargs.get('file_size'), "size": kwargs.get('file_size'),
"name": kwargs.get('file_name'), "name": kwargs.get('file_name'),
@ -17,8 +17,8 @@ def dataTemplate(transactionType,**kwargs):
"guid": f'{transactionType.split("_")[1].rstrip("d").rstrip("e")}:{kwargs.get("guid")}', "guid": f'{transactionType.split("_")[1].rstrip("d").rstrip("e")}:{kwargs.get("guid")}',
"node_name": kwargs.get('node_name'), "node_name": kwargs.get('node_name'),
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
"product_name": "GlobalScape EFT", "product_name": kwargs.get('product_name'),
"prd_ext_tenant_name": "GlobalScape EFT", "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
"classifications": [{ "classifications": [{
"ref_id": f"globalscape:{kwargs.get('guid')}", "ref_id": f"globalscape:{kwargs.get('guid')}",
"time": kwargs.get('time'), "time": kwargs.get('time'),
@ -32,15 +32,15 @@ def dataTemplate(transactionType,**kwargs):
"ip": kwargs.get('src_endpoint_ip'), "ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type') "type": kwargs.get('src_endpoint_type')
}, },
"tenant": kwargs.get('tenant'), "tenant": kwargs.get('prd_ext_tenant_id'),
"tenant_name":"GlobalScape", "tenant_name":"GlobalScape",
"time": kwargs.get('time'), "time": kwargs.get('time'),
"status_code": kwargs.get('status_code'), "status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('description'), "status_detail": kwargs.get('status_detail'),
"user": { "user": {
"home_directory": kwargs.get('user_home_directory'), "home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('guid'), "uuid": kwargs.get('guid'),
"uid": kwargs.get('uid'), "uid": kwargs.get('user_uid'),
"type": kwargs.get('user_type'), "type": kwargs.get('user_type'),
"name": kwargs.get('user_name') "name": kwargs.get('user_name')
}, },
@ -60,9 +60,9 @@ def dataTemplate(transactionType,**kwargs):
"ref_id": f"globalscape:{kwargs.get('guid')}", "ref_id": f"globalscape:{kwargs.get('guid')}",
"time": kwargs.get('time'), "time": kwargs.get('time'),
}], }],
"prd_ext_tenant_name": "Globalscape EFT", "prd_ext_tenant_name": kwargs.get("prd_ext_tenant_name"),
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
"product_name": "Globalscape EFT", "product_name": kwargs.get("product_name"),
"session": { "session": {
"created_time": kwargs.get('time'), "created_time": kwargs.get('time'),
"uid": kwargs.get('session_uid') "uid": kwargs.get('session_uid')
@ -72,7 +72,7 @@ def dataTemplate(transactionType,**kwargs):
"ip": kwargs.get('src_endpoint_ip'), "ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type') "type": kwargs.get('src_endpoint_type')
}, },
"tenant": kwargs.get('tenant'), "tenant": kwargs.get('prd_ext_tenant_id'),
"tenant_name":"GlobalScape", "tenant_name":"GlobalScape",
"dst_endpoint": { "dst_endpoint": {
"port": kwargs.get('dst_endpoint_port'), "port": kwargs.get('dst_endpoint_port'),
@ -81,11 +81,11 @@ def dataTemplate(transactionType,**kwargs):
}, },
"time": kwargs.get('time'), "time": kwargs.get('time'),
"status_code": kwargs.get('status_code'), "status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('description'), "status_detail": kwargs.get('status_detail'),
"user": { "user": {
"home_directory": kwargs.get('user_home_directory'), "home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('session_uid'), "uuid": kwargs.get('user_session_uid'),
"uid": kwargs.get('uid'), "uid": kwargs.get('user_uid'),
"type": kwargs.get('user_type'), "type": kwargs.get('user_type'),
"name": kwargs.get('user_name') "name": kwargs.get('user_name')
}, },
@ -104,13 +104,13 @@ def dataTemplate(transactionType,**kwargs):
}, },
"guid": kwargs.get('guid'), "guid": kwargs.get('guid'),
"node_name": kwargs.get('node_name'), "node_name": kwargs.get('node_name'),
"tenant": kwargs.get('tenant'), "tenant": kwargs.get('prd_ext_tenant_id'),
"tenant_name":"GlobalScape", "tenant_name":"GlobalScape",
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
"product_name": "GlobalScape EFT", "product_name": kwargs.get("product_name"),
"prd_ext_tenant_name": "GlobalScape EFT", "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
"status_code": kwargs.get('status_code'), "status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('description'), "status_detail": kwargs.get('status_detail'),
"src_endpoint": { "src_endpoint": {
"port": kwargs.get('src_endpoint_port'), "port": kwargs.get('src_endpoint_port'),
"ip": kwargs.get('src_endpoint_ip'), "ip": kwargs.get('src_endpoint_ip'),
@ -119,8 +119,8 @@ def dataTemplate(transactionType,**kwargs):
"time": kwargs.get('time'), "time": kwargs.get('time'),
"user": { "user": {
"home_directory": kwargs.get('user_home_directory'), "home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('session_uid'), "uuid": kwargs.get('user_session_uid'),
"uid": kwargs.get('uid'), "uid": kwargs.get('user_uid'),
"type": kwargs.get('user_type'), "type": kwargs.get('user_type'),
"name": kwargs.get('user_name') "name": kwargs.get('user_name')
}, },

33
inexDataProcessing.py
View File

@ -13,36 +13,33 @@ def processData(data, template, **kwargs):
continue continue
try: try:
processedData.append(template(identifyUtype(row.get('Command')),\ processedData.append(template(identifyUtype(row.get('Command')),\
status_detail='',\ prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
# this should be taken from self.platformConfig user_uid=row.get('TransactionID'),\
status_detail=row.get('Description'),\
prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\ prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
status_code=row.get('ResultID'),\ status_code=row.get('ResultID'),\
file_created_time=row.get('Time_stamp'),\
file_size=row.get('FileSize'),\ file_size=row.get('FileSize'),\
file_uid=row.get('ProtocolCommandID'),\
file_path=row.get('PhysicalFolderName'),\ file_path=row.get('PhysicalFolderName'),\
file_virtual_path=row.get('VirtualFolderName'),\
file_name=row.get('FileName'),\ file_name=row.get('FileName'),\
guid=row.get('TransactionGUID'),\ guid=row.get('TransactionGUID'),\
ref_id=row.get('TransactionGUID'),\
prd_instance_id=kwargs.get('prd_instance_id'),\
product_guid=kwargs.get('product_guid'),\
product_name=kwargs.get('product_name'),\ product_name=kwargs.get('product_name'),\
product_version=kwargs.get('product_version'),\
node_name=row.get('NodeName'),\ node_name=row.get('NodeName'),\
session_uid=row.get('TransactionID'),\
src_endpoint_type=row.get('Protocol'),\ src_endpoint_type=row.get('Protocol'),\
src_endpoint_port=row.get('RemotePort'),\ src_endpoint_port=row.get('RemotePort'),\
src_endpoint_ip=row.get('RemoteIP'),\ src_endpoint_ip=row.get('RemoteIP'),\
dst_endpoint_port=row.get('LocalPort'),\ dst_endpoint_port=row.get('LocalPort'),\
dst_endpoint_ip=row.get('LocalIP'),\ dst_endpoint_ip=row.get('LocalIP'),\
dst_endpoint_type=row.get('Protocol'),\ dst_endpoint_type=row.get('Protocol'),\
session_uid=row.get('TransactionID'),\ user_session_uid=row.get('TransactionID'),\
bytes_out=row.get('BytesTransferred'),\ bytes=row.get('BytesTransferred'),\
duration=row.get('TransferTime'),\
time=row.get('Time_stamp'),\ time=row.get('Time_stamp'),\
duration=row.get('TransferTime'),\
user_type=identifyUserType(row.get('user_type')),\ user_type=identifyUserType(row.get('user_type')),\
user_domain=row.get('SiteName'),\
user_name=row.get('Actor'),\ user_name=row.get('Actor'),\
user_home_directory=row.get('VirtualFolderName'),\ user_home_directory=row.get('VirtualFolderName'),\
description=row.get('Description'),\
utype=identifyUtype(row.get('Command')))) utype=identifyUtype(row.get('Command'))))
except UnboundLocalError: except UnboundLocalError:
print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}') print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
@ -53,11 +50,13 @@ def processData(data, template, **kwargs):
try: try:
processedData.append(template(identifyUtype(row.get('TransactionObject')),\ processedData.append(template(identifyUtype(row.get('TransactionObject')),\
prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\ prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
status_detail=row.get('Description'),\
guid=row.get('TransactionGUID'),\ guid=row.get('TransactionGUID'),\
status_code=row.get('ResultID'),\
node_name=row.get('NodeName'),\
prd_instance_id=kwargs.get('prd_instance_id'),\ prd_instance_id=kwargs.get('prd_instance_id'),\
product_guid=kwargs.get('product_guid'),\
product_name=kwargs.get('product_name'),\ product_name=kwargs.get('product_name'),\
product_version=kwargs.get('product_version'),\
src_endpoint_type=row.get('Protocol'),\ src_endpoint_type=row.get('Protocol'),\
src_endpoint_port=row.get('RemotePort'),\ src_endpoint_port=row.get('RemotePort'),\
src_endpoint_ip=row.get('RemoteIP'),\ src_endpoint_ip=row.get('RemoteIP'),\
@ -65,13 +64,13 @@ def processData(data, template, **kwargs):
dst_endpoint_ip=row.get('LocalIP'),\ dst_endpoint_ip=row.get('LocalIP'),\
dst_endpoint_type=row.get('Protocol'),\ dst_endpoint_type=row.get('Protocol'),\
session_uid=row.get('TransactionID'),\ session_uid=row.get('TransactionID'),\
bytes_out=row.get('BytesTransferred'),\
transfer_time=row.get('TransferTime'),\ transfer_time=row.get('TransferTime'),\
time=row.get('Time_stamp'),\ time=row.get('Time_stamp'),\
user_session_uid=row.get('TransactionID'),\
user_uid=row.get('TransactionID'),\
user_type=identifyUserType(row.get('user_type')),\ user_type=identifyUserType(row.get('user_type')),\
user_domain=row.get('SiteName'),\
user_name=row.get('Actor'),\ user_name=row.get('Actor'),\
user_home_directory=row.get('VirtualFolderName'),\ user_home_directory=row.get('PhysicalFolderName'),\
utype=identifyUtype(row.get('TransactionObject'))\ utype=identifyUtype(row.get('TransactionObject'))\
)) ))
transactionLoginid.append(row.get('TransactionGUID')) transactionLoginid.append(row.get('TransactionGUID'))

2
inexSqlquery.py
View File

@ -3,7 +3,7 @@ class sqlQuerymodel:
"""Embedded query data""" """Embedded query data"""
q ="""DECLARE @stopTime DATETIME2 q ="""DECLARE @stopTime DATETIME2
SET @stopTime=DATEADD(DAY, -30, GETDATE()) SET @stopTime=DATEADD(DAY, -30, GETDATE())
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
FROM tbl_Transactions t FROM tbl_Transactions t
Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID) Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)