jblu/Inex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases 8 Wiki Activity

dev-update-data-model #13

Merged
jblu merged 2 commits from dev-update-data-model into main 2024-08-10 00:03:55 -05:00
Conversation 0 Commits 2 Files Changed 3 +128 -206
3 changed files with 128 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7e5a8a2603 - Show all commits

312
inexDataModel.py
View File

@ -1,211 +1,125 @@
def dataTemplate(**kwargs): def dataTemplate(transactionType,**kwargs):
"""Expects the following keyword arguments: general = {
status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name, "bytes" : kwargs.get('bytes_out'),
guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name, "dst_endpoint": {
src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type, "port": kwargs.get('dst_endpoint_port'),
session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype. "ip": kwargs.get('dst_endpoint_ip'),
""" "type": kwargs.get('dst_endpoint_type')
template ={ },
"status": kwargs.get('status'), "duration": kwargs.get('duration'),
"status_detail": kwargs.get('status_detail'), "file": {
"status_code": kwargs.get('status_code'), "created_time": kwargs.get('time'),
"file": { "size": kwargs.get('file_size'),
"size": kwargs.get('file_size'), "name": kwargs.get('file_name'),
"path": kwargs.get('file_path'), "path": kwargs.get('file_path')
"virtual_path": kwargs.get('file_virtual_path'), },
"name": kwargs.get('file_name') "guid": kwargs.get('guid'),
}, "node_name": kwargs.get('node_name'),
"guid": kwargs.get('guid'), "prd_ext_tenant_id": kwargs.get('tenant'),
"ref_id": kwargs.get('ref_id'), "product_name": "GlobalScape EFT",
"prd_instance_id": kwargs.get('prd_instance_id'), "prd_ext_tenant_name": "GlobalScape EFT",
"product_guid": kwargs.get('product_guid'), "classifications": [{
"product_name": kwargs.get('product_name'), "ref_id": f"globalscape:{kwargs.get('guid')}",
"product_version": kwargs.get('product_version'), "time": kwargs.get('time'),
"node_name":kwargs.get('node_name'), }],
"src_endpoint": { "session": {
"port": kwargs.get('src_endpoint_port'), "created_time": kwargs.get('time'),
"ip": kwargs.get('src_endpoint_ip') "uid": kwargs.get('session_uid')
}, },
"dst_endpoint": { "src_endpoint": {
"port": kwargs.get('src_endpoint_port'),
"ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type')
},
"tenant": kwargs.get('tenant'),
"tenant_name":"GlobalScape",
"time": kwargs.get('time'),
"status_code": kwargs.get('status_code'),
"status_detail": kwargs.get('status_detail'),
"user": {
"home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('guid'),
"uid": kwargs.get('uid'),
"type": kwargs.get('user_type'),
"name": kwargs.get('user_name')
},
"utype": kwargs.get('utype')
}
fileDeleted = {
"file": {
"size": kwargs.get('file_size'),
"name": kwargs.get('file_name'),
"path": kwargs.get('file_path')
},
"guid": kwargs.get('guid'),
"classifications": [{
"ref_id": f"globalscape:{kwargs.get('guid')}",
"time": kwargs.get('time'),
}],
"prd_ext_tenant_name": "Globalscape EFT",
"prd_ext_tenant_id": kwargs.get('tenant'),
"product_name": "Globalscape EFT",
"session": {
"created_time": kwargs.get('time'),
"uid": kwargs.get('session_uid')
},
"src_endpoint": {
"port": kwargs.get('src_endpoint_port'),
"ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type')
},
"dst_endpoint": {
"port": kwargs.get('dst_endpoint_port'),
"ip": kwargs.get('dst_endpoint_ip'),
"type": kwargs.get('dst_endpoint_type')
},
"time": kwargs.get('time'),
"user": {
"home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('guid'),
"uid": kwargs.get('uid'),
"type": kwargs.get('user_type'),
"name": kwargs.get('user_name')
},
"utype": kwargs.get('utype')
}
logon ={
"classifications": [{
"ref_id": f"globalscape:{kwargs.get('guid')}",
"time": kwargs.get('time'),
}],
"dst_endpoint": {
"port": kwargs.get('dst_endpoint_port'), "port": kwargs.get('dst_endpoint_port'),
"ip": kwargs.get('dst_endpoint_ip'), "ip": kwargs.get('dst_endpoint_ip'),
"type": kwargs.get('dst_endpoint_type') "type": kwargs.get('dst_endpoint_type')
}, },
"session": { "guid": kwargs.get('guid'),
"uid": kwargs.get('session_uid') "prd_ext_tenant_id": kwargs.get('tenant'),
"product_name": "GlobalScape EFT",
"prd_ext_tenant_name": "GlobalScape EFT",
"src_endpoint": {
"port": kwargs.get('src_endpoint_port'),
"ip": kwargs.get('src_endpoint_ip'),
"type": kwargs.get('src_endpoint_type')
}, },
"bytes_out" : kwargs.get('bytes_out'),
"transfer_time" : kwargs.get('transfer_time'),
"time": kwargs.get('time'), "time": kwargs.get('time'),
"user": { "user": {
"home_directory": kwargs.get('user_home_directory'),
"uuid": kwargs.get('guid'),
"uid": kwargs.get('uid'),
"type": kwargs.get('user_type'), "type": kwargs.get('user_type'),
"domain": kwargs.get('user_domain'),
"name": kwargs.get('user_name') "name": kwargs.get('user_name')
}, },
"utype": kwargs.get('utype') "utype": kwargs.get('utype')
} }
if transactionType == "FileUploaded":
return template template = general
if transactionType == "FileDownloaded":
FileUploaded = { template = general
"bytes" : 2490, if transactionType == "FileDeleted":
"dst_endpoint": { template = fileDeleted
"port": 22, if transactionType == "Logon":
"ip": "10.91.160.77", template = logon
"type": "SFTP" return template
},
"duration": 200,
"file": {
"created_time": 1722485724000,
"size": 2490,
"name": "Case9.vbs",
"path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //<PhysicalFolderName>"
},
"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641204",
"node_name":"PERF01-S2019-77",
"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
"product_name": "GlobalScape EFT",
"prd_ext_tenant_name": "GlobalScape EFT",
"classifications": [{
"ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641204",
"time":1722485724000,
}],
"session": {
"created_time":1722485724000,
"uid": "3615136"
},
"src_endpoint": {
"port": 58868,
"ip": "10.91.160.45",
"type":"SFTP"
},
"tenant": "e71851c2-593f-4f49-9c07-91727b1be94b",
"tenant_name":"GlobalScape",
"time":1722485724000,
"status_code":226,
"status_detail":"Upload Successful",
"user": {
"home_directory": "/Usr/Ivan/",
"uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641204",
"uid": "3978403",
"type": "User",
"name": "Ivan"
},
"utype": "file_uploaded"
}
FileDownloaded = {
"bytes" : 4891,
"dst_endpoint": {
"port": 443,
"ip": "10.91.160.77",
"type": "HTTPS"
},
"duration": 200,
"file": {
"created_time": 1722518124000,
"size": 4891,
"name": "FileDownload1.exe",
"path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //<PhysicalFolderName>"
},
"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641205",
"node_name":"PERF01-S2019-77",
"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
"product_name": "GlobalScape EFT",
"prd_ext_tenant_name": "GlobalScape EFT",
"classifications": [{
"ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641205",
"time":1722518124000,
}],
"session": {
"created_time":1722518124000,
"uid": "3615137"
},
"src_endpoint": {
"port": 443,
"ip": "10.91.160.45",
"type":"HTTPS"
},
"tenant": "e71851c2-593f-4f49-9c07-91727b1be94b",
"tenant_name":"GlobalScape",
"time":1722518124000,
"status_code":226,
"status_detail":"Download Successful",
"user": {
"home_directory": "/Usr/Ivan/",
"uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641205",
"uid": "3978404",
"type": "User",
"name": "Ivan"
},
"utype": "file_downloaded"}
FileDeleted = {
"file": {
"size": 304673,
"path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan",
"name": "DeleteME.txt"
},
"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641207",
"classifications": [{
"ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641207",
"time":1722515664000,
}],
"prd_ext_tenant_name": "Globalscape EFT",
"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
"product_name": "Globalscape EFT",
"session": {
"created_time":1722515664000,
"uid": "3615138"
},
"src_endpoint": {
"port": 443,
"ip": "10.91.160.45",
"type":"HTTPS"
},
"dst_endpoint": {
"port": 443,
"ip": "10.91.160.77",
"type": "HTTPS"
},
"time": 1722515664000,
"user": {
"home_directory": "/Usr/Ivan/",
"uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641207",
"uid": "3978406",
"type": "User",
"name": "Ivan"
},
"utype": "file_deleted"
}
Logon ={
"classifications": [{
"ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641206",
"time": 1722510924000,
}],
"dst_endpoint": {
"port": 443,
"ip": "10.91.160.77",
"type": "HTTPS"
},
"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641206",
"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
"product_name": "GlobalScape EFT",
"prd_ext_tenant_name": "GlobalScape EFT",
"src_endpoint": {
"port": 443,
"ip": "10.91.160.45",
"type":"HTTPS"
},
"time": 1722510924000,
"user": {
"home_directory": "/Usr/Ivan/",
"uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641206",
"uid": "3978405",
"type": "User",
"name": "Ivan"
},
"utype": "user_logged_on"
}

14
inexDataProcessing.py
View File

@ -1,8 +1,10 @@
def processData(data, template, **kwargs): def processData(data, template, **kwargs):
processedData = [] processedData = []
for row in data: for row in data:
# print(f'Row: {row}') print(f'Row: {row}')
processedData.append(template(status=row.get(''),\ if row.get('Command') == None:
continue
processedData.append(template(identifyUtype(row.get('Command')),\
status_detail=row.get(''),\ status_detail=row.get(''),\
status_code=row.get('ResultID'),\ status_code=row.get('ResultID'),\
file_size=row.get('FileSize'),\ file_size=row.get('FileSize'),\
@ -28,7 +30,7 @@ def processData(data, template, **kwargs):
user_type=identifyUserType(row.get('user_type')),\ user_type=identifyUserType(row.get('user_type')),\
user_domain=row.get('SiteName'),\ user_domain=row.get('SiteName'),\
user_name=row.get('Actor'),\ user_name=row.get('Actor'),\
utype=identifyUtype(row.get('TransactionObject')))) utype=identifyUtype(row.get('Command'))))
return processedData return processedData
def identifyUserType(obj): def identifyUserType(obj):
@ -41,9 +43,9 @@ def identifyUserType(obj):
return None return None
def identifyUtype(obj): def identifyUtype(obj):
user_logged_on = [] user_logged_on = []
file_deleted = [] file_deleted = ["dele"]
file_uploaded = [] file_uploaded = ["created"]
file_downloaded = [] file_downloaded = ["sent"]
if obj in user_logged_on: if obj in user_logged_on:
return "user_logged_on" return "user_logged_on"

8
test.py
View File

@ -9,4 +9,10 @@ def connectDatabase(driver, server, database, user, password):
def converttimestamp(t): def converttimestamp(t):
print(int(t.timestamp()* 1000)) print(int(t.timestamp()* 1000))
a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000)) def builddict(keys,*args,**kwargs):
dict = {}
for key in keys:
dict[key] = kwargs.get(key)
print(dict)
a = builddict(["bytes","duration","dst_endpoint"],bytes=2490,duration=200,dst_enpoint={"port": 1,"ip": 1,"type":1})