From c56c9ac208242bc8d1b70ef05a9e65ab0e5153fb Mon Sep 17 00:00:00 2001 From: Jonathan Branan Date: Thu, 15 Aug 2024 14:58:53 -0500 Subject: [PATCH] Fixed auth bug --- config.toml.example | 16 +++--- inex.py | 19 ++++---- inexConnect.py | 57 +++++++++++++--------- inexDataModel.py | 111 +++++++++++++++++++++++++++++++++--------- inexDataProcessing.py | 68 +++++++++++++++----------- inexSqlquery.py | 2 +- test.py | 6 ++- 7 files changed, 185 insertions(+), 94 deletions(-) diff --git a/config.toml.example b/config.toml.example index eb131bc..0ca5baf 100644 --- a/config.toml.example +++ b/config.toml.example @@ -2,19 +2,19 @@ selectedPlatform = "dev" [output] -pushToplatform = true +pushToplatform = false dumpTojson = true filename ="./data.json" token = "./.token" [logging] -use_log = true +useLog = true logLevel = "debug" logPath = "./inex.log" [database] overrideEmbeddedquery = false -daysTopull = 10 +daysTopull = 20 driver = "ODBC Driver 18 for SQL Server" server = "192.168.x.x" database = "EFTDB" @@ -22,7 +22,7 @@ user = "a" password = "a" query = """DECLARE @stopTime DATETIME2 SET @stopTime=DATEADD(DAY, -30, GETDATE()) -SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type +SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type FROM tbl_Transactions t Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID) @@ -49,8 +49,6 @@ tenant_id = "" client_id = "eft-event-generator-confidential" secret = "" -[immutables] -prd_instance_id = 1 -product_guid = "asdf" -product_name = "EFT" -product_version ="8.1.0.9" \ No newline at end of file +[immutables] +product_name = "GlobalScape EFT" +prd_ext_tenant_name = "GlobalScape EFT" \ No newline at end of file diff --git a/inex.py b/inex.py index fa90f49..42f04ee 100644 --- a/inex.py +++ b/inex.py @@ -43,18 +43,19 @@ class Inex: self.useLog = self.config["logging"]["useLog"] self.logPath = self.config["logging"]["logPath"] self.logLevel = self.config["logging"]["logLevel"] - self.prdInstanceID = self.config["immutables"]["prd_instance_id"] - self.productGUID = self.config["immutables"]["product_guid"] + self.prdExttenantname = self.config["immutables"]["prd_ext_tenant_name"] self.productName = self.config["immutables"]["product_name"] - self.productVersion = self.config["immutables"]["product_version"] self.tokenFilepath = self.config["output"]["token"] self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] self.writeJsonfile = self.config["output"]["dumpTojson"] self.pushToplatform = self.config["output"]["pushToplatform"] self.queryOverride = self.config["database"]["overrideEmbeddedquery"] self.queryDaystopull = self.config["database"]["daysTopull"] - except: - print("No config.toml. Please use example file and configure appropriately") + except Exception as e: + print("No config.toml or possibly missing settings in the file. Please use config.toml.example file and configure appropriately") + self.il.error(e) + print(e) + exit(1) if "dev" in self.selectedPlatform.lower(): @@ -73,11 +74,11 @@ class Inex: self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull)) - self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ - product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) - + self.modifiedData = processData(self.data, dataTemplate, prd_ext_tenant_name=self.prdExttenantname,product_name=self.productName,\ + prd_ext_tenant_id=self.platformConfig["tenant_id"]) + if self.pushToplatform: - inexConnect.fortraEFC.pushPayload(self) + inexConnect.fortraEFC.__init__(self) # TODO: move this to its own function if self.useLog: diff --git a/inexConnect.py b/inexConnect.py index 3eefad6..b42cbf4 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -43,33 +43,46 @@ class inexSql: return r class fortraEFC: - def getToken(self): - self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\ - "client_id": self.platformConfig["client_id"],\ - "client_secret": self.platformConfig["secret"],}) - def writeToken(self): - fortraEFC.getToken(self) - with open(self.tokenFilepath, "w") as f: - self.j.dump(self.tokenData.json(), f, indent = 2) + def __init__(self): + # Check if .token file is present + if fortraEFC.readToken(self) == 1: + # Get fresh token. First run. + fortraEFC.getToken(self) + fortraEFC.writeToken(self) + # Push data with token + self.pushPayloadresponse = fortraEFC.pushPayload(self) + if self.pushPayloadresponse == 401: + fortraEFC.getToken(self) + fortraEFC.writeToken(self) + fortraEFC.pushPayload(self) def readToken(self): if self.os.path.exists(self.tokenFilepath): with open(self.tokenFilepath, 'rb') as t: self.tokenData = self.j.load(t) - # print(self.tokenData["access_token"]) + self.il.debug(f'readToken {self.tokenData["access_token"]}') + return 0 else: - fortraEFC.writeToken(self) + return 1 + + def getToken(self): + self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\ + "client_id": self.platformConfig["client_id"],\ + "client_secret": self.platformConfig["secret"],}) + self.tokenData = self.tokenData.json() + self.il.debug(f'getToken {self.tokenData["access_token"]}') + + def writeToken(self): + fortraEFC.getToken(self) + with open(self.tokenFilepath, "w") as f: + self.j.dump(self.tokenData, f, indent = 2) + self.il.debug(f'writeToken {self.tokenData["access_token"]}') def pushPayload(self): - fortraEFC.readToken(self) - print(self.tokenData) - try: - url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event' - pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\ - json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e)) - return pushPayloadResponse.status_code - except self.r.exceptions.HTTPError as errh: - print ("Http Error:",errh) - if "401" in errh: - fortraEFC.writeToken(self) - fortraEFC.pushPayload(self) \ No newline at end of file + self.il.debug(f'pushPayload {self.tokenData["access_token"]}') + url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event' + pushPayloadResponse = self.r.post(url, headers={'Authorization': f'Bearer {self.tokenData["access_token"]}'},\ + data=self.j.dumps(self.modifiedData, cls=self.e)) + self.il.debug(pushPayloadResponse.status_code) + self.il.debug(pushPayloadResponse.text) + return pushPayloadResponse.status_code \ No newline at end of file diff --git a/inexDataModel.py b/inexDataModel.py index 24849f7..3db6542 100644 --- a/inexDataModel.py +++ b/inexDataModel.py @@ -1,6 +1,6 @@ def dataTemplate(transactionType,**kwargs): - uploadDownload = { - "bytes" : kwargs.get('bytes_out'), + upload = { + "bytes" : kwargs.get('bytes'), "dst_endpoint": { "port": kwargs.get('dst_endpoint_port'), "ip": kwargs.get('dst_endpoint_ip'), @@ -8,16 +8,17 @@ def dataTemplate(transactionType,**kwargs): }, "duration": kwargs.get('duration'), "file": { - "created_time": kwargs.get('time'), + "created_time": kwargs.get('file_created_time'), + "uid": kwargs.get('file_uid'), "size": kwargs.get('file_size'), "name": kwargs.get('file_name'), "path": kwargs.get('file_path') }, - "guid": kwargs.get('guid'), + "guid": f'{transactionType.split("_")[1].rstrip("d").rstrip("e")}:{kwargs.get("guid")}', "node_name": kwargs.get('node_name'), - "prd_ext_tenant_id": kwargs.get('tenant'), - "product_name": "GlobalScape EFT", - "prd_ext_tenant_name": "GlobalScape EFT", + "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), + "product_name": kwargs.get('product_name'), + "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'), "classifications": [{ "ref_id": f"globalscape:{kwargs.get('guid')}", "time": kwargs.get('time'), @@ -31,15 +32,62 @@ def dataTemplate(transactionType,**kwargs): "ip": kwargs.get('src_endpoint_ip'), "type": kwargs.get('src_endpoint_type') }, - "tenant": kwargs.get('tenant'), + "tenant": kwargs.get('prd_ext_tenant_id'), "tenant_name":"GlobalScape", "time": kwargs.get('time'), "status_code": kwargs.get('status_code'), - "status_detail": kwargs.get('description'), + "status_detail": kwargs.get('status_detail'), "user": { "home_directory": kwargs.get('user_home_directory'), "uuid": kwargs.get('guid'), - "uid": kwargs.get('uid'), + "uid": kwargs.get('user_uid'), + "type": kwargs.get('user_type'), + "name": kwargs.get('user_name') + }, + "utype": kwargs.get('utype') + } + download = { + "bytes" : kwargs.get('bytes'), + "dst_endpoint": { + "port": kwargs.get('dst_endpoint_port'), + "ip": kwargs.get('dst_endpoint_ip'), + "type": kwargs.get('dst_endpoint_type') + }, + "duration": kwargs.get('duration'), + "file": { + "created_time": kwargs.get('file_created_time'), + "uid": kwargs.get('file_uid'), + "size": kwargs.get('file_size'), + "name": kwargs.get('file_name'), + "path": kwargs.get('file_path') + }, + "guid": f'{transactionType.split("_")[1].rstrip("d").rstrip("e")}:{kwargs.get("guid")}', + "node_name": kwargs.get('node_name'), + "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), + "product_name": kwargs.get('product_name'), + "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'), + "classifications": [{ + "ref_id": f"globalscape:{kwargs.get('guid')}", + "time": kwargs.get('time'), + }], + "session": { + "created_time": kwargs.get('time'), + "uid": kwargs.get('session_uid') + }, + "src_endpoint": { + "port": kwargs.get('src_endpoint_port'), + "ip": kwargs.get('src_endpoint_ip'), + "type": kwargs.get('src_endpoint_type') + }, + "tenant": kwargs.get('prd_ext_tenant_id'), + "tenant_name":"GlobalScape", + "time": kwargs.get('time'), + "status_code": kwargs.get('status_code'), + "status_detail": kwargs.get('status_detail'), + "user": { + "home_directory": kwargs.get('user_home_directory'), + "uuid": kwargs.get('guid'), + "uid": kwargs.get('user_uid'), "type": kwargs.get('user_type'), "name": kwargs.get('user_name') }, @@ -50,16 +98,18 @@ def dataTemplate(transactionType,**kwargs): "file": { "size": kwargs.get('file_size'), "name": kwargs.get('file_name'), - "path": kwargs.get('file_path') + "path": kwargs.get('file_path'), + "uid": kwargs.get('file_uid'), }, - "guid": kwargs.get('guid'), + "guid": f'deleted:{kwargs.get("guid")}', + "node_name": kwargs.get('node_name'), "classifications": [{ "ref_id": f"globalscape:{kwargs.get('guid')}", "time": kwargs.get('time'), }], - "prd_ext_tenant_name": "Globalscape EFT", - "prd_ext_tenant_id": kwargs.get('tenant'), - "product_name": "Globalscape EFT", + "prd_ext_tenant_name": kwargs.get("prd_ext_tenant_name"), + "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), + "product_name": kwargs.get("product_name"), "session": { "created_time": kwargs.get('time'), "uid": kwargs.get('session_uid') @@ -69,16 +119,20 @@ def dataTemplate(transactionType,**kwargs): "ip": kwargs.get('src_endpoint_ip'), "type": kwargs.get('src_endpoint_type') }, + "tenant": kwargs.get('prd_ext_tenant_id'), + "tenant_name":"GlobalScape", "dst_endpoint": { "port": kwargs.get('dst_endpoint_port'), "ip": kwargs.get('dst_endpoint_ip'), "type": kwargs.get('dst_endpoint_type') }, "time": kwargs.get('time'), + "status_code": kwargs.get('status_code'), + "status_detail": kwargs.get('status_detail'), "user": { "home_directory": kwargs.get('user_home_directory'), - "uuid": kwargs.get('guid'), - "uid": kwargs.get('uid'), + "uuid": kwargs.get('user_session_uid'), + "uid": kwargs.get('user_uid'), "type": kwargs.get('user_type'), "name": kwargs.get('user_name') }, @@ -96,9 +150,14 @@ def dataTemplate(transactionType,**kwargs): "type": kwargs.get('dst_endpoint_type') }, "guid": kwargs.get('guid'), - "prd_ext_tenant_id": kwargs.get('tenant'), - "product_name": "GlobalScape EFT", - "prd_ext_tenant_name": "GlobalScape EFT", + "node_name": kwargs.get('node_name'), + "tenant": kwargs.get('prd_ext_tenant_id'), + "tenant_name":"GlobalScape", + "prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'), + "product_name": kwargs.get("product_name"), + "prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'), + "status_code": kwargs.get('status_code'), + "status_detail": kwargs.get('status_detail'), "src_endpoint": { "port": kwargs.get('src_endpoint_port'), "ip": kwargs.get('src_endpoint_ip'), @@ -107,18 +166,22 @@ def dataTemplate(transactionType,**kwargs): "time": kwargs.get('time'), "user": { "home_directory": kwargs.get('user_home_directory'), - "uuid": kwargs.get('guid'), - "uid": kwargs.get('uid'), + "uuid": kwargs.get('user_session_uid'), + "uid": kwargs.get('user_uid'), "type": kwargs.get('user_type'), "name": kwargs.get('user_name') }, + "session": { + "created_time": kwargs.get('time'), + "uid": kwargs.get('session_uid') + }, "utype": kwargs.get('utype') } if transactionType == "file_uploaded": - template = uploadDownload + template = upload if transactionType == "file_downloaded": - template = uploadDownload + template = download if transactionType == "file_deleted": template = fileDeleted if transactionType == "user_logged_on": diff --git a/inexDataProcessing.py b/inexDataProcessing.py index b0b7699..e17b08b 100644 --- a/inexDataProcessing.py +++ b/inexDataProcessing.py @@ -4,58 +4,69 @@ def processData(data, template, **kwargs): additional data to insert into the template.""" processedData = [] transactionLoginid = [] - + for row in data: # print(f'Row: {row}') - if identifyUtype(row.get('Command')) == "other": + # must set variables for the different templates and do logic based on that. Do not call identifyUtype many times + identifyUtypecommand = identifyUtype(row.get('Command')) + + if identifyUtypecommand == "other": continue - + if row.get('Command') == None: continue + + userType = identifyUserType(row.get('user_type')) try: - processedData.append(template(identifyUtype(row.get('Command')),\ - prd_ext_tenant_id='',\ + processedData.append(template(identifyUtypecommand,\ + prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\ + user_uid=row.get('TransactionID'),\ + status_detail=row.get('Description'),\ + prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\ status_code=row.get('ResultID'),\ + file_created_time=row.get('Time_stamp'),\ file_size=row.get('FileSize'),\ + file_uid=row.get('ProtocolCommandID'),\ file_path=row.get('PhysicalFolderName'),\ - file_virtual_path=row.get('VirtualFolderName'),\ file_name=row.get('FileName'),\ guid=row.get('TransactionGUID'),\ - ref_id=row.get('ProtocolCommandID'),\ - prd_instance_id=kwargs.get('prd_instance_id'),\ - product_guid=kwargs.get('product_guid'),\ product_name=kwargs.get('product_name'),\ - product_version=kwargs.get('product_version'),\ node_name=row.get('NodeName'),\ + session_uid=row.get('TransactionID'),\ src_endpoint_type=row.get('Protocol'),\ src_endpoint_port=row.get('RemotePort'),\ src_endpoint_ip=row.get('RemoteIP'),\ dst_endpoint_port=row.get('LocalPort'),\ dst_endpoint_ip=row.get('LocalIP'),\ dst_endpoint_type=row.get('Protocol'),\ - session_uid=row.get('TransactionID'),\ - bytes_out=row.get('BytesTransferred'),\ - duration=row.get('TransferTime'),\ + user_session_uid=row.get('TransactionID'),\ + bytes=row.get('BytesTransferred'),\ time=row.get('Time_stamp'),\ - user_type=identifyUserType(row.get('user_type')),\ - user_domain=row.get('SiteName'),\ + duration=row.get('TransferTime'),\ + user_type=userType,\ user_name=row.get('Actor'),\ user_home_directory=row.get('VirtualFolderName'),\ - description=row.get('Description'),\ - utype=identifyUtype(row.get('Command')))) + utype=identifyUtypecommand)) except UnboundLocalError: print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}') + continue + identifyUtypetransactionObject = identifyUtype(row.get('TransactionObject')) + + if identifyUtypetransactionObject == "other": continue if row.get('TransactionGUID') not in transactionLoginid: try: - processedData.append(template(identifyUtype(row.get('TransactionObject')),\ + processedData.append(template(identifyUtypetransactionObject,\ + prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\ + prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\ + status_detail=row.get('Description'),\ guid=row.get('TransactionGUID'),\ + status_code=row.get('ResultID'),\ + node_name=row.get('NodeName'),\ prd_instance_id=kwargs.get('prd_instance_id'),\ - product_guid=kwargs.get('product_guid'),\ product_name=kwargs.get('product_name'),\ - product_version=kwargs.get('product_version'),\ src_endpoint_type=row.get('Protocol'),\ src_endpoint_port=row.get('RemotePort'),\ src_endpoint_ip=row.get('RemoteIP'),\ @@ -63,14 +74,14 @@ def processData(data, template, **kwargs): dst_endpoint_ip=row.get('LocalIP'),\ dst_endpoint_type=row.get('Protocol'),\ session_uid=row.get('TransactionID'),\ - bytes_out=row.get('BytesTransferred'),\ transfer_time=row.get('TransferTime'),\ time=row.get('Time_stamp'),\ - user_type=identifyUserType(row.get('user_type')),\ - user_domain=row.get('SiteName'),\ + user_session_uid=row.get('TransactionID'),\ + user_uid=row.get('TransactionID'),\ + user_type=userType,\ user_name=row.get('Actor'),\ - user_home_directory=row.get('VirtualFolderName'),\ - utype=identifyUtype(row.get('TransactionObject'))\ + user_home_directory=row.get('PhysicalFolderName'),\ + utype=identifyUtypetransactionObject\ )) transactionLoginid.append(row.get('TransactionGUID')) except UnboundLocalError: @@ -88,6 +99,7 @@ def identifyUserType(obj): return "User" else: return None + def identifyUtype(obj): """Process Type of transaction based on string that passed in. Return transaction type.""" @@ -98,11 +110,11 @@ def identifyUtype(obj): if obj in user_logged_on: return "user_logged_on" - if obj in file_deleted: + elif obj in file_deleted: return "file_deleted" - if obj in file_uploaded: + elif obj in file_uploaded: return "file_uploaded" - if obj in file_downloaded: + elif obj in file_downloaded: return "file_downloaded" else: return "other" \ No newline at end of file diff --git a/inexSqlquery.py b/inexSqlquery.py index 9c10e87..f7a080e 100644 --- a/inexSqlquery.py +++ b/inexSqlquery.py @@ -3,7 +3,7 @@ class sqlQuerymodel: """Embedded query data""" q ="""DECLARE @stopTime DATETIME2 SET @stopTime=DATEADD(DAY, -30, GETDATE()) - SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type + SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type FROM tbl_Transactions t Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID) diff --git a/test.py b/test.py index c430b1f..4be6116 100644 --- a/test.py +++ b/test.py @@ -35,4 +35,8 @@ def identifyUtype(obj): if obj in file_downloaded: return "file_downloaded" else: - return "other" \ No newline at end of file + return "other" + +transactionType = 'file_uploaded' + +print(transactionType.split("_")[1].rstrip("d").rstrip("e")) \ No newline at end of file