diff --git a/inexDataModel.py b/inexDataModel.py index 86fe862..5190be5 100644 --- a/inexDataModel.py +++ b/inexDataModel.py @@ -1,211 +1,127 @@ -def dataTemplate(**kwargs): - """Expects the following keyword arguments: - status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name, - guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name, - src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type, - session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype. - """ - template ={ - "status": kwargs.get('status'), - "status_detail": kwargs.get('status_detail'), - "status_code": kwargs.get('status_code'), - "file": { - "size": kwargs.get('file_size'), - "path": kwargs.get('file_path'), - "virtual_path": kwargs.get('file_virtual_path'), - "name": kwargs.get('file_name') - }, - "guid": kwargs.get('guid'), - "ref_id": kwargs.get('ref_id'), - "prd_instance_id": kwargs.get('prd_instance_id'), - "product_guid": kwargs.get('product_guid'), - "product_name": kwargs.get('product_name'), - "product_version": kwargs.get('product_version'), - "node_name":kwargs.get('node_name'), - "src_endpoint": { - "port": kwargs.get('src_endpoint_port'), - "ip": kwargs.get('src_endpoint_ip') - }, - "dst_endpoint": { +def dataTemplate(transactionType,**kwargs): + uploadDownload = { + "bytes" : kwargs.get('bytes_out'), + "dst_endpoint": { + "port": kwargs.get('dst_endpoint_port'), + "ip": kwargs.get('dst_endpoint_ip'), + "type": kwargs.get('dst_endpoint_type') + }, + "duration": kwargs.get('duration'), + "file": { + "created_time": kwargs.get('time'), + "size": kwargs.get('file_size'), + "name": kwargs.get('file_name'), + "path": kwargs.get('file_path') + }, + "guid": kwargs.get('guid'), + "node_name": kwargs.get('node_name'), + "prd_ext_tenant_id": kwargs.get('tenant'), + "product_name": "GlobalScape EFT", + "prd_ext_tenant_name": "GlobalScape EFT", + "classifications": [{ + "ref_id": f"globalscape:{kwargs.get('guid')}", + "time": kwargs.get('time'), + }], + "session": { + "created_time": kwargs.get('time'), + "uid": kwargs.get('session_uid') + }, + "src_endpoint": { + "port": kwargs.get('src_endpoint_port'), + "ip": kwargs.get('src_endpoint_ip'), + "type": kwargs.get('src_endpoint_type') + }, + "tenant": kwargs.get('tenant'), + "tenant_name":"GlobalScape", + "time": kwargs.get('time'), + "status_code": kwargs.get('status_code'), + "status_detail": kwargs.get('status_detail'), + "user": { + "home_directory": kwargs.get('user_home_directory'), + "uuid": kwargs.get('guid'), + "uid": kwargs.get('uid'), + "type": kwargs.get('user_type'), + "name": kwargs.get('user_name') + }, + "utype": kwargs.get('utype') + } + + fileDeleted = { + "file": { + "size": kwargs.get('file_size'), + "name": kwargs.get('file_name'), + "path": kwargs.get('file_path') + }, + "guid": kwargs.get('guid'), + "classifications": [{ + "ref_id": f"globalscape:{kwargs.get('guid')}", + "time": kwargs.get('time'), + }], + "prd_ext_tenant_name": "Globalscape EFT", + "prd_ext_tenant_id": kwargs.get('tenant'), + "product_name": "Globalscape EFT", + "session": { + "created_time": kwargs.get('time'), + "uid": kwargs.get('session_uid') + }, + "src_endpoint": { + "port": kwargs.get('src_endpoint_port'), + "ip": kwargs.get('src_endpoint_ip'), + "type": kwargs.get('src_endpoint_type') + }, + "dst_endpoint": { + "port": kwargs.get('dst_endpoint_port'), + "ip": kwargs.get('dst_endpoint_ip'), + "type": kwargs.get('dst_endpoint_type') + }, + "time": kwargs.get('time'), + "user": { + "home_directory": kwargs.get('user_home_directory'), + "uuid": kwargs.get('guid'), + "uid": kwargs.get('uid'), + "type": kwargs.get('user_type'), + "name": kwargs.get('user_name') + }, + "utype": kwargs.get('utype') + } + + logon ={ + "classifications": [{ + "ref_id": f"globalscape:{kwargs.get('guid')}", + "time": kwargs.get('time'), + }], + "dst_endpoint": { "port": kwargs.get('dst_endpoint_port'), "ip": kwargs.get('dst_endpoint_ip'), "type": kwargs.get('dst_endpoint_type') }, - "session": { - "uid": kwargs.get('session_uid') + "guid": kwargs.get('guid'), + "prd_ext_tenant_id": kwargs.get('tenant'), + "product_name": "GlobalScape EFT", + "prd_ext_tenant_name": "GlobalScape EFT", + "src_endpoint": { + "port": kwargs.get('src_endpoint_port'), + "ip": kwargs.get('src_endpoint_ip'), + "type": kwargs.get('src_endpoint_type') }, - "bytes_out" : kwargs.get('bytes_out'), - "transfer_time" : kwargs.get('transfer_time'), "time": kwargs.get('time'), "user": { + "home_directory": kwargs.get('user_home_directory'), + "uuid": kwargs.get('guid'), + "uid": kwargs.get('uid'), "type": kwargs.get('user_type'), - "domain": kwargs.get('user_domain'), "name": kwargs.get('user_name') }, "utype": kwargs.get('utype') } - return template - -FileUploaded = { - "bytes" : 2490, - "dst_endpoint": { - "port": 22, - "ip": "10.91.160.77", - "type": "SFTP" - }, - "duration": 200, - "file": { - "created_time": 1722485724000, - "size": 2490, - "name": "Case9.vbs", - "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //" - }, - "guid": "48D9C7A3-2DC6-11EF-AA59-00155D641204", - "node_name":"PERF01-S2019-77", - "prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", - "product_name": "GlobalScape EFT", - "prd_ext_tenant_name": "GlobalScape EFT", - "classifications": [{ - "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641204", - "time":1722485724000, - }], - "session": { - "created_time":1722485724000, - "uid": "3615136" - }, - "src_endpoint": { - "port": 58868, - "ip": "10.91.160.45", - "type":"SFTP" - }, - "tenant": "e71851c2-593f-4f49-9c07-91727b1be94b", - "tenant_name":"GlobalScape", - "time":1722485724000, - "status_code":226, - "status_detail":"Upload Successful", - "user": { - "home_directory": "/Usr/Ivan/", - "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641204", - "uid": "3978403", - "type": "User", - "name": "Ivan" - }, - "utype": "file_uploaded" -} - -FileDownloaded = { -"bytes" : 4891, -"dst_endpoint": { - "port": 443, - "ip": "10.91.160.77", - "type": "HTTPS" -}, -"duration": 200, -"file": { - "created_time": 1722518124000, - "size": 4891, - "name": "FileDownload1.exe", - "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //" -}, -"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641205", -"node_name":"PERF01-S2019-77", -"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", -"product_name": "GlobalScape EFT", -"prd_ext_tenant_name": "GlobalScape EFT", -"classifications": [{ - "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641205", - "time":1722518124000, -}], -"session": { - "created_time":1722518124000, - "uid": "3615137" -}, -"src_endpoint": { - "port": 443, - "ip": "10.91.160.45", - "type":"HTTPS" -}, -"tenant": "e71851c2-593f-4f49-9c07-91727b1be94b", -"tenant_name":"GlobalScape", -"time":1722518124000, -"status_code":226, -"status_detail":"Download Successful", -"user": { - "home_directory": "/Usr/Ivan/", - "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641205", - "uid": "3978404", - "type": "User", - "name": "Ivan" -}, -"utype": "file_downloaded"} - -FileDeleted = { - "file": { - "size": 304673, - "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan", -"name": "DeleteME.txt" -}, -"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641207", -"classifications": [{ - "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641207", - "time":1722515664000, -}], -"prd_ext_tenant_name": "Globalscape EFT", -"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", -"product_name": "Globalscape EFT", -"session": { - "created_time":1722515664000, - "uid": "3615138" -}, -"src_endpoint": { - "port": 443, - "ip": "10.91.160.45", - "type":"HTTPS" -}, -"dst_endpoint": { - "port": 443, - "ip": "10.91.160.77", - "type": "HTTPS" -}, -"time": 1722515664000, -"user": { - "home_directory": "/Usr/Ivan/", - "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641207", - "uid": "3978406", - "type": "User", - "name": "Ivan" -}, -"utype": "file_deleted" -} - -Logon ={ -"classifications": [{ - "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641206", - "time": 1722510924000, + if transactionType == "file_uploaded": + template = uploadDownload + if transactionType == "file_downloaded": + template = uploadDownload + if transactionType == "file_deleted": + template = fileDeleted + if transactionType == "user_logged_on": + template = logon -}], -"dst_endpoint": { - "port": 443, - "ip": "10.91.160.77", - "type": "HTTPS" -}, -"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641206", -"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", -"product_name": "GlobalScape EFT", -"prd_ext_tenant_name": "GlobalScape EFT", -"src_endpoint": { - "port": 443, - "ip": "10.91.160.45", - "type":"HTTPS" -}, -"time": 1722510924000, -"user": { - "home_directory": "/Usr/Ivan/", - "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641206", - "uid": "3978405", - "type": "User", - "name": "Ivan" -}, -"utype": "user_logged_on" -} \ No newline at end of file + return template \ No newline at end of file diff --git a/inexDataProcessing.py b/inexDataProcessing.py index a64d964..de01e0f 100644 --- a/inexDataProcessing.py +++ b/inexDataProcessing.py @@ -1,34 +1,68 @@ def processData(data, template, **kwargs): + processedData = [] + transactionLoginid = [] + for row in data: # print(f'Row: {row}') - processedData.append(template(status=row.get(''),\ - status_detail=row.get(''),\ - status_code=row.get('ResultID'),\ - file_size=row.get('FileSize'),\ - file_path=row.get('PhysicalFolderName'),\ - file_virtual_path=row.get('VirtualFolderName'),\ - file_name=row.get('FileName'),\ - guid=row.get('TransactionGUID'),\ - ref_id=row.get('ProtocolCommandID'),\ - prd_instance_id=kwargs.get('prd_instance_id'),\ - product_guid=kwargs.get('product_guid'),\ - product_name=kwargs.get('product_name'),\ - product_version=kwargs.get('product_version'),\ - node_name=row.get('NodeName'),\ - src_endpoint_port=row.get('RemotePort'),\ - src_endpoint_ip=row.get('RemoteIP'),\ - dst_endpoint_port=row.get('LocalPort'),\ - dst_endpoint_ip=row.get('LocalIP'),\ - dst_endpoint_type=row.get('Protocol'),\ - session_uid=row.get('TransactionID'),\ - bytes_out=row.get('BytesTransferred'),\ - transfer_time=row.get('TransferTime'),\ - time=row.get('Time_stamp'),\ - user_type=identifyUserType(row.get('user_type')),\ - user_domain=row.get('SiteName'),\ - user_name=row.get('Actor'),\ - utype=identifyUtype(row.get('TransactionObject')))) + if row.get('Command') == None: + continue + + processedData.append(template(identifyUtype(row.get('Command')),\ + prd_ext_tenant_id='',\ + status_code=row.get('ResultID'),\ + file_size=row.get('FileSize'),\ + file_path=row.get('PhysicalFolderName'),\ + file_virtual_path=row.get('VirtualFolderName'),\ + file_name=row.get('FileName'),\ + guid=row.get('TransactionGUID'),\ + ref_id=row.get('ProtocolCommandID'),\ + prd_instance_id=kwargs.get('prd_instance_id'),\ + product_guid=kwargs.get('product_guid'),\ + product_name=kwargs.get('product_name'),\ + product_version=kwargs.get('product_version'),\ + node_name=row.get('NodeName'),\ + src_endpoint_type=row.get('Protocol'),\ + src_endpoint_port=row.get('RemotePort'),\ + src_endpoint_ip=row.get('RemoteIP'),\ + dst_endpoint_port=row.get('LocalPort'),\ + dst_endpoint_ip=row.get('LocalIP'),\ + dst_endpoint_type=row.get('Protocol'),\ + session_uid=row.get('TransactionID'),\ + bytes_out=row.get('BytesTransferred'),\ + duration=row.get('TransferTime'),\ + time=row.get('Time_stamp'),\ + user_type=identifyUserType(row.get('user_type')),\ + user_domain=row.get('SiteName'),\ + user_name=row.get('Actor'),\ + user_home_directory=row.get('VirtualFolderName'),\ + utype=identifyUtype(row.get('Command')))) + + if row.get('TransactionGUID') not in transactionLoginid: + processedData.append(template(identifyUtype(row.get('TransactionObject')),\ + guid=row.get('TransactionGUID'),\ + prd_instance_id=kwargs.get('prd_instance_id'),\ + product_guid=kwargs.get('product_guid'),\ + product_name=kwargs.get('product_name'),\ + product_version=kwargs.get('product_version'),\ + src_endpoint_type=row.get('Protocol'),\ + src_endpoint_port=row.get('RemotePort'),\ + src_endpoint_ip=row.get('RemoteIP'),\ + dst_endpoint_port=row.get('LocalPort'),\ + dst_endpoint_ip=row.get('LocalIP'),\ + dst_endpoint_type=row.get('Protocol'),\ + session_uid=row.get('TransactionID'),\ + bytes_out=row.get('BytesTransferred'),\ + transfer_time=row.get('TransferTime'),\ + time=row.get('Time_stamp'),\ + user_type=identifyUserType(row.get('user_type')),\ + user_domain=row.get('SiteName'),\ + user_name=row.get('Actor'),\ + user_home_directory=row.get('VirtualFolderName'),\ + utype=identifyUtype(row.get('TransactionObject'))\ + )) + transactionLoginid.append(row.get('TransactionGUID')) + return processedData def identifyUserType(obj): @@ -40,10 +74,10 @@ def identifyUserType(obj): else: return None def identifyUtype(obj): - user_logged_on = [] - file_deleted = [] - file_uploaded = [] - file_downloaded = [] + user_logged_on = ['AUTH'] + file_deleted = ["dele"] + file_uploaded = ["created"] + file_downloaded = ["sent"] if obj in user_logged_on: return "user_logged_on" diff --git a/test.py b/test.py index fb702bc..d1c1ce3 100644 --- a/test.py +++ b/test.py @@ -9,4 +9,13 @@ def connectDatabase(driver, server, database, user, password): def converttimestamp(t): print(int(t.timestamp()* 1000)) -a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000)) \ No newline at end of file +def builddict(keys,*args,**kwargs): + dict = {} + for key in keys: + dict[key] = kwargs.get(key) + print(dict) + +testfolder = '/Usr/a/asdf/asf' +user = 'a' + +print(testfolder.split(f"/{user}/")) \ No newline at end of file