diff --git a/config.toml.example b/config.toml.example index 1ea5daf..37c4113 100644 --- a/config.toml.example +++ b/config.toml.example @@ -23,41 +23,19 @@ client_id = "eft-event-generator-confidential" secret = "" [database] +overrideEmbeddedquery = false driver = "ODBC Driver 18 for SQL Server" server = "192.168.x.x" database = "EFTDB" user = "a" password = "a" -query = """DECLARE @stopTime DATETIME2 -SET @stopTime = DATEADD(DAY, -30, GETDATE()) -SELECT p.[ProtocolCommandID] - ,t.[Time_stamp] - ,p.[RemoteIP] - ,p.[RemotePort] - ,p.[LocalIP] - ,p.[LocalPort] - ,p.[Protocol] - ,p.[SiteName] - ,p.[Command] - ,p.[CommandParameters] - ,p.[FileName] - ,p.[VirtualFolderName] - ,p.[PhysicalFolderName] - ,p.[IsInternal] - ,p.[FileSize] - ,p.[TransferTime] - ,p.[BytesTransferred] - ,p.[ResultID] - ,t.[TransactionID] - ,p.[Description] - ,p.[Actor] - ,t.ParentTransactionID - ,t.TransactionObject - ,t.NodeName - ,t.TransactionGUID - ,a.Protocol user_type - FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID) - WHERE p.Time_stamp > @stopTime""" +query = """DECLARE @stopTime DATETIME2 +SET @stopTime=DATEADD(DAY, -30, GETDATE()) +SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type +FROM tbl_Transactions t + Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) + Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID) +WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL""" [immutables] prd_instance_id = 1 diff --git a/inex.py b/inex.py index 8c4c5fb..1d1af51 100644 --- a/inex.py +++ b/inex.py @@ -10,6 +10,7 @@ import json import requests import inexEncoder import inexSqlquery + class Inex: def __init__(self): """Initilize config, calls functions from inex-connect.py and inex-logging.py""" @@ -30,24 +31,30 @@ class Inex: self.config = self.tl.load(c) # set config - self.dbDriver = self.config["database"]["driver"] - self.dbServer = self.config["database"]["server"] - self.dbDatabase = self.config["database"]["database"] - self.dbUser = self.config["database"]["user"] - self.dbPassword = self.config["database"]["password"] - self.dbQuery = self.config["database"]["query"] - self.outputFile = self.config["output"]["filename"] - self.useLog = self.config["logging"]["useLog"] - self.logPath = self.config["logging"]["logPath"] - self.logLevel = self.config["logging"]["logLevel"] - self.prdInstanceID = self.config["immutables"]["prd_instance_id"] - self.productGUID = self.config["immutables"]["product_guid"] - self.productName = self.config["immutables"]["product_name"] - self.productVersion = self.config["immutables"]["product_version"] - self.tokenFilepath = self.config["output"]["token"] - self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] - self.writeJsonfile = self.config["output"]["dumpTojson"] - self.pushToplatform = self.config["output"]["pushToplatform"] + try: + if self.config: + self.dbDriver = self.config["database"]["driver"] + self.dbServer = self.config["database"]["server"] + self.dbDatabase = self.config["database"]["database"] + self.dbUser = self.config["database"]["user"] + self.dbPassword = self.config["database"]["password"] + self.dbQuery = self.config["database"]["query"] + self.outputFile = self.config["output"]["filename"] + self.useLog = self.config["logging"]["useLog"] + self.logPath = self.config["logging"]["logPath"] + self.logLevel = self.config["logging"]["logLevel"] + self.prdInstanceID = self.config["immutables"]["prd_instance_id"] + self.productGUID = self.config["immutables"]["product_guid"] + self.productName = self.config["immutables"]["product_name"] + self.productVersion = self.config["immutables"]["product_version"] + self.tokenFilepath = self.config["output"]["token"] + self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] + self.writeJsonfile = self.config["output"]["dumpTojson"] + self.pushToplatform = self.config["output"]["pushToplatform"] + self.queryOverride = self.config["database"]["overrideEmbeddedquery"] + except: + print("No config.toml. Please use example file and configure appropriately") + exit(1) if "dev" in self.selectedPlatform.lower(): self.platformConfig = self.config["fortraPlatform"]["dev"] @@ -55,7 +62,6 @@ class Inex: self.platformConfig = self.config["fortraPlatform"]["stage"] if "prod" in self.selectedPlatform.lower(): self.platformConfig = self.config["fortraPlatform"]["prod"] - # print(self.platformConfig) #Setup logging inexLog(self) @@ -63,9 +69,8 @@ class Inex: # create the connection to the database self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) - # self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.dbQuery) - self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData()) + self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery)) self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) diff --git a/inexDataModel.py b/inexDataModel.py index 5190be5..24849f7 100644 --- a/inexDataModel.py +++ b/inexDataModel.py @@ -35,7 +35,7 @@ def dataTemplate(transactionType,**kwargs): "tenant_name":"GlobalScape", "time": kwargs.get('time'), "status_code": kwargs.get('status_code'), - "status_detail": kwargs.get('status_detail'), + "status_detail": kwargs.get('description'), "user": { "home_directory": kwargs.get('user_home_directory'), "uuid": kwargs.get('guid'), @@ -123,5 +123,7 @@ def dataTemplate(transactionType,**kwargs): template = fileDeleted if transactionType == "user_logged_on": template = logon + if transactionType == "other": + template = {} return template \ No newline at end of file diff --git a/inexDataProcessing.py b/inexDataProcessing.py index de01e0f..8b1e4d0 100644 --- a/inexDataProcessing.py +++ b/inexDataProcessing.py @@ -7,44 +7,21 @@ def processData(data, template, **kwargs): # print(f'Row: {row}') if row.get('Command') == None: continue - - processedData.append(template(identifyUtype(row.get('Command')),\ - prd_ext_tenant_id='',\ - status_code=row.get('ResultID'),\ - file_size=row.get('FileSize'),\ - file_path=row.get('PhysicalFolderName'),\ - file_virtual_path=row.get('VirtualFolderName'),\ - file_name=row.get('FileName'),\ - guid=row.get('TransactionGUID'),\ - ref_id=row.get('ProtocolCommandID'),\ - prd_instance_id=kwargs.get('prd_instance_id'),\ - product_guid=kwargs.get('product_guid'),\ - product_name=kwargs.get('product_name'),\ - product_version=kwargs.get('product_version'),\ - node_name=row.get('NodeName'),\ - src_endpoint_type=row.get('Protocol'),\ - src_endpoint_port=row.get('RemotePort'),\ - src_endpoint_ip=row.get('RemoteIP'),\ - dst_endpoint_port=row.get('LocalPort'),\ - dst_endpoint_ip=row.get('LocalIP'),\ - dst_endpoint_type=row.get('Protocol'),\ - session_uid=row.get('TransactionID'),\ - bytes_out=row.get('BytesTransferred'),\ - duration=row.get('TransferTime'),\ - time=row.get('Time_stamp'),\ - user_type=identifyUserType(row.get('user_type')),\ - user_domain=row.get('SiteName'),\ - user_name=row.get('Actor'),\ - user_home_directory=row.get('VirtualFolderName'),\ - utype=identifyUtype(row.get('Command')))) - - if row.get('TransactionGUID') not in transactionLoginid: - processedData.append(template(identifyUtype(row.get('TransactionObject')),\ + try: + processedData.append(template(identifyUtype(row.get('Command')),\ + prd_ext_tenant_id='',\ + status_code=row.get('ResultID'),\ + file_size=row.get('FileSize'),\ + file_path=row.get('PhysicalFolderName'),\ + file_virtual_path=row.get('VirtualFolderName'),\ + file_name=row.get('FileName'),\ guid=row.get('TransactionGUID'),\ + ref_id=row.get('ProtocolCommandID'),\ prd_instance_id=kwargs.get('prd_instance_id'),\ product_guid=kwargs.get('product_guid'),\ product_name=kwargs.get('product_name'),\ product_version=kwargs.get('product_version'),\ + node_name=row.get('NodeName'),\ src_endpoint_type=row.get('Protocol'),\ src_endpoint_port=row.get('RemotePort'),\ src_endpoint_ip=row.get('RemoteIP'),\ @@ -53,15 +30,47 @@ def processData(data, template, **kwargs): dst_endpoint_type=row.get('Protocol'),\ session_uid=row.get('TransactionID'),\ bytes_out=row.get('BytesTransferred'),\ - transfer_time=row.get('TransferTime'),\ + duration=row.get('TransferTime'),\ time=row.get('Time_stamp'),\ user_type=identifyUserType(row.get('user_type')),\ user_domain=row.get('SiteName'),\ user_name=row.get('Actor'),\ user_home_directory=row.get('VirtualFolderName'),\ - utype=identifyUtype(row.get('TransactionObject'))\ - )) - transactionLoginid.append(row.get('TransactionGUID')) + description=row.get('Description'),\ + utype=identifyUtype(row.get('Command')))) + except UnboundLocalError: + print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}') + + continue + + if row.get('TransactionGUID') not in transactionLoginid: + try: + processedData.append(template(identifyUtype(row.get('TransactionObject')),\ + guid=row.get('TransactionGUID'),\ + prd_instance_id=kwargs.get('prd_instance_id'),\ + product_guid=kwargs.get('product_guid'),\ + product_name=kwargs.get('product_name'),\ + product_version=kwargs.get('product_version'),\ + src_endpoint_type=row.get('Protocol'),\ + src_endpoint_port=row.get('RemotePort'),\ + src_endpoint_ip=row.get('RemoteIP'),\ + dst_endpoint_port=row.get('LocalPort'),\ + dst_endpoint_ip=row.get('LocalIP'),\ + dst_endpoint_type=row.get('Protocol'),\ + session_uid=row.get('TransactionID'),\ + bytes_out=row.get('BytesTransferred'),\ + transfer_time=row.get('TransferTime'),\ + time=row.get('Time_stamp'),\ + user_type=identifyUserType(row.get('user_type')),\ + user_domain=row.get('SiteName'),\ + user_name=row.get('Actor'),\ + user_home_directory=row.get('VirtualFolderName'),\ + utype=identifyUtype(row.get('TransactionObject'))\ + )) + transactionLoginid.append(row.get('TransactionGUID')) + except UnboundLocalError: + print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}') + continue return processedData @@ -88,4 +97,4 @@ def identifyUtype(obj): if obj in file_downloaded: return "file_downloaded" else: - return None \ No newline at end of file + return "other" \ No newline at end of file diff --git a/inexSqlquery.py b/inexSqlquery.py index 009d075..8db92ab 100644 --- a/inexSqlquery.py +++ b/inexSqlquery.py @@ -1,11 +1,11 @@ class sqlQuerymodel: - def queryData(): + def queryData(overRideflag, configQuery): """Embedded query data""" q ="""DECLARE @stopTime DATETIME2 SET @stopTime=DATEADD(DAY, -30, GETDATE()) - SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.CommandParameters, p.FileName, p.VirtualFolderName, p.PhysicalFolderName, p.IsInternal, p.FileSize, p.TransferTime, p.BytesTransferred, p.ResultID, t.TransactionID, p.Description, p.Actor, t.ParentTransactionID, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type + SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type FROM tbl_Transactions t Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID) - WHERE p.Time_stamp>@stopTime""" - return q \ No newline at end of file + WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL""" + return configQuery if overRideflag else q \ No newline at end of file