Merge pull request 'dev-update-data-model' (#13) from dev-update-data-model into main

Reviewed-on: #13

inexDataModel.py

@@ -1,211 +1,127 @@
-def dataTemplate(**kwargs):
-    """Expects the following keyword arguments:
-    status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name,
-    guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name,
-    src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type,
-    session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype.
-    """
-    template ={
-    "status": kwargs.get('status'),
-    "status_detail": kwargs.get('status_detail'),
-	"status_code": kwargs.get('status_code'),
-    "file": {
-        "size": kwargs.get('file_size'),
-        "path": kwargs.get('file_path'),
-        "virtual_path": kwargs.get('file_virtual_path'),
-        "name": kwargs.get('file_name')
-    },
-    "guid": kwargs.get('guid'),
-    "ref_id": kwargs.get('ref_id'),
-    "prd_instance_id": kwargs.get('prd_instance_id'),
-    "product_guid": kwargs.get('product_guid'),
-    "product_name": kwargs.get('product_name'),
-    "product_version": kwargs.get('product_version'),
-    "node_name":kwargs.get('node_name'),
-    "src_endpoint": {
-        "port": kwargs.get('src_endpoint_port'),
-        "ip": kwargs.get('src_endpoint_ip')
-    },
-	"dst_endpoint": {
+def dataTemplate(transactionType,**kwargs):
+    uploadDownload = {
+        "bytes" : kwargs.get('bytes_out'),
+        "dst_endpoint": { 
+            "port": kwargs.get('dst_endpoint_port'),
+            "ip": kwargs.get('dst_endpoint_ip'),
+            "type": kwargs.get('dst_endpoint_type')
+        },
+        "duration": kwargs.get('duration'),
+        "file": {
+            "created_time": kwargs.get('time'),
+            "size": kwargs.get('file_size'),
+            "name": kwargs.get('file_name'),
+            "path": kwargs.get('file_path')
+        },
+        "guid": kwargs.get('guid'),
+        "node_name": kwargs.get('node_name'),
+        "prd_ext_tenant_id": kwargs.get('tenant'),
+        "product_name": "GlobalScape EFT",
+        "prd_ext_tenant_name": "GlobalScape EFT",
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "session": {
+           "created_time": kwargs.get('time'),
+           "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "tenant": kwargs.get('tenant'),
+        "tenant_name":"GlobalScape",
+        "time": kwargs.get('time'),
+        "status_code": kwargs.get('status_code'),
+        "status_detail": kwargs.get('status_detail'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+   
+    fileDeleted = {
+        "file": {
+                "size": kwargs.get('file_size'),
+                "name": kwargs.get('file_name'),
+                "path": kwargs.get('file_path')
+        },
+        "guid": kwargs.get('guid'),
+        "classifications": [{
+            "ref_id": f"globalscape:{kwargs.get('guid')}",
+            "time": kwargs.get('time'),
+        }],
+        "prd_ext_tenant_name": "Globalscape EFT",
+        "prd_ext_tenant_id": kwargs.get('tenant'),
+        "product_name": "Globalscape EFT",
+        "session": {
+            "created_time": kwargs.get('time'),
+            "uid": kwargs.get('session_uid')
+        },
+        "src_endpoint": {
+            "port": kwargs.get('src_endpoint_port'),
+            "ip": kwargs.get('src_endpoint_ip'),
+            "type": kwargs.get('src_endpoint_type')
+        },
+        "dst_endpoint": {
+            "port": kwargs.get('dst_endpoint_port'),
+            "ip": kwargs.get('dst_endpoint_ip'),
+            "type": kwargs.get('dst_endpoint_type')
+        },
+        "time": kwargs.get('time'),
+        "user": {
+            "home_directory": kwargs.get('user_home_directory'),
+            "uuid": kwargs.get('guid'),
+            "uid": kwargs.get('uid'),
+            "type": kwargs.get('user_type'),
+            "name": kwargs.get('user_name')
+        },
+        "utype": kwargs.get('utype')
+    }
+    
+    logon ={
+    "classifications": [{
+        "ref_id": f"globalscape:{kwargs.get('guid')}",
+        "time": kwargs.get('time'),
+    }],
+    "dst_endpoint": {
         "port": kwargs.get('dst_endpoint_port'),
         "ip": kwargs.get('dst_endpoint_ip'),
         "type": kwargs.get('dst_endpoint_type')
     },
-    "session": {
-        "uid": kwargs.get('session_uid') 
+    "guid": kwargs.get('guid'),
+    "prd_ext_tenant_id": kwargs.get('tenant'),
+    "product_name": "GlobalScape EFT",
+    "prd_ext_tenant_name": "GlobalScape EFT",
+    "src_endpoint": {
+        "port": kwargs.get('src_endpoint_port'),
+        "ip": kwargs.get('src_endpoint_ip'),
+        "type": kwargs.get('src_endpoint_type')
     },
-	"bytes_out" : kwargs.get('bytes_out'),
-	"transfer_time" : kwargs.get('transfer_time'),
     "time": kwargs.get('time'),
     "user": {
+        "home_directory": kwargs.get('user_home_directory'),
+        "uuid": kwargs.get('guid'),
+        "uid": kwargs.get('uid'),
         "type": kwargs.get('user_type'),
-        "domain": kwargs.get('user_domain'),
         "name": kwargs.get('user_name')
     },
     "utype": kwargs.get('utype')
     }
 
-    return template
-
-FileUploaded = {
-    "bytes" : 2490,
-    "dst_endpoint": { 
-        "port": 22,
-        "ip": "10.91.160.77",
-        "type": "SFTP"
-    },
-    "duration": 200,
-    "file": {
-        "created_time": 1722485724000,
-        "size": 2490,
-        "name": "Case9.vbs",
-        "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //<PhysicalFolderName>"
-    },
-    "guid": "48D9C7A3-2DC6-11EF-AA59-00155D641204",
-    "node_name":"PERF01-S2019-77",
-    "prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-    "product_name": "GlobalScape EFT",
-    "prd_ext_tenant_name": "GlobalScape EFT",
-    "classifications": [{
-        "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641204",
-        "time":1722485724000,
-    }],
-    "session": {
-       "created_time":1722485724000,
-       "uid": "3615136"
-    },
-    "src_endpoint": {
-        "port": 58868,
-        "ip": "10.91.160.45",
-        "type":"SFTP"
-    },
-    "tenant": "e71851c2-593f-4f49-9c07-91727b1be94b",
-    "tenant_name":"GlobalScape",
-    "time":1722485724000,
-    "status_code":226,
-    "status_detail":"Upload Successful",
-    "user": {
-        "home_directory": "/Usr/Ivan/",
-        "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641204",
-        "uid": "3978403",
-        "type": "User",
-        "name": "Ivan"
-    },
-    "utype": "file_uploaded"
-}
-
-FileDownloaded = {
-"bytes" : 4891,
-"dst_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.77",
-    "type": "HTTPS"
-},
-"duration": 200,
-"file": {
-    "created_time": 1722518124000,
-    "size": 4891,
-    "name": "FileDownload1.exe",
-    "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //<PhysicalFolderName>"
-},
-"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641205",
-"node_name":"PERF01-S2019-77",
-"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"product_name": "GlobalScape EFT",
-"prd_ext_tenant_name": "GlobalScape EFT",
-"classifications": [{
-    "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641205",
-    "time":1722518124000,
-}],
-"session": {
-   "created_time":1722518124000,
-   "uid": "3615137"
-},
-"src_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.45",
-    "type":"HTTPS"
-},
-"tenant": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"tenant_name":"GlobalScape",
-"time":1722518124000,
-"status_code":226,
-"status_detail":"Download Successful",
-"user": {
-    "home_directory": "/Usr/Ivan/",
-    "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641205",
-    "uid": "3978404",
-    "type": "User",
-    "name": "Ivan" 
-},
-"utype": "file_downloaded"}
-
-FileDeleted = {
-        "file": {
-        "size": 304673,
-        "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan",
-"name": "DeleteME.txt"
-},
-"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641207",
-"classifications": [{
-    "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641207",
-    "time":1722515664000,
-}],
-"prd_ext_tenant_name": "Globalscape EFT",
-"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"product_name": "Globalscape EFT",
-"session": {
-   "created_time":1722515664000,
-   "uid": "3615138"
-},
-"src_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.45",
-    "type":"HTTPS"
-},
-"dst_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.77",
-    "type": "HTTPS"
-},
-"time": 1722515664000,
-"user": {
-    "home_directory": "/Usr/Ivan/",
-    "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641207",
-    "uid": "3978406",
-    "type": "User",
-    "name": "Ivan"
-},
-"utype": "file_deleted"
-}
-
-Logon ={
-"classifications": [{
-    "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641206",
-    "time": 1722510924000,
+    if transactionType == "file_uploaded":
+        template = uploadDownload
+    if transactionType == "file_downloaded":
+        template = uploadDownload
+    if transactionType == "file_deleted":
+        template = fileDeleted
+    if transactionType == "user_logged_on":
+        template = logon
     
-}],
-"dst_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.77",
-    "type": "HTTPS"
-},
-"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641206",
-"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b",
-"product_name": "GlobalScape EFT",
-"prd_ext_tenant_name": "GlobalScape EFT",
-"src_endpoint": {
-    "port": 443,
-    "ip": "10.91.160.45",
-    "type":"HTTPS"
-},
-"time": 1722510924000,
-"user": {
-    "home_directory": "/Usr/Ivan/",
-    "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641206",
-    "uid": "3978405",
-    "type": "User",
-    "name": "Ivan"
-},
-"utype": "user_logged_on"
-}
+    return template

inexDataProcessing.py

@@ -1,34 +1,68 @@
 def processData(data, template, **kwargs):
+
     processedData = []
+    transactionLoginid = []
+
     for row in data:
         # print(f'Row: {row}')
-        processedData.append(template(status=row.get(''),\
-                            status_detail=row.get(''),\
-                            status_code=row.get('ResultID'),\
-                            file_size=row.get('FileSize'),\
-                            file_path=row.get('PhysicalFolderName'),\
-                            file_virtual_path=row.get('VirtualFolderName'),\
-                            file_name=row.get('FileName'),\
-                            guid=row.get('TransactionGUID'),\
-                            ref_id=row.get('ProtocolCommandID'),\
-                            prd_instance_id=kwargs.get('prd_instance_id'),\
-                            product_guid=kwargs.get('product_guid'),\
-                            product_name=kwargs.get('product_name'),\
-                            product_version=kwargs.get('product_version'),\
-                            node_name=row.get('NodeName'),\
-                            src_endpoint_port=row.get('RemotePort'),\
-                            src_endpoint_ip=row.get('RemoteIP'),\
-                            dst_endpoint_port=row.get('LocalPort'),\
-                            dst_endpoint_ip=row.get('LocalIP'),\
-                            dst_endpoint_type=row.get('Protocol'),\
-                            session_uid=row.get('TransactionID'),\
-                            bytes_out=row.get('BytesTransferred'),\
-                            transfer_time=row.get('TransferTime'),\
-                            time=row.get('Time_stamp'),\
-                            user_type=identifyUserType(row.get('user_type')),\
-                            user_domain=row.get('SiteName'),\
-                            user_name=row.get('Actor'),\
-                            utype=identifyUtype(row.get('TransactionObject'))))
+        if row.get('Command') == None:
+            continue
+
+        processedData.append(template(identifyUtype(row.get('Command')),\
+            prd_ext_tenant_id='',\
+            status_code=row.get('ResultID'),\
+            file_size=row.get('FileSize'),\
+            file_path=row.get('PhysicalFolderName'),\
+            file_virtual_path=row.get('VirtualFolderName'),\
+            file_name=row.get('FileName'),\
+            guid=row.get('TransactionGUID'),\
+            ref_id=row.get('ProtocolCommandID'),\
+            prd_instance_id=kwargs.get('prd_instance_id'),\
+            product_guid=kwargs.get('product_guid'),\
+            product_name=kwargs.get('product_name'),\
+            product_version=kwargs.get('product_version'),\
+            node_name=row.get('NodeName'),\
+            src_endpoint_type=row.get('Protocol'),\
+            src_endpoint_port=row.get('RemotePort'),\
+            src_endpoint_ip=row.get('RemoteIP'),\
+            dst_endpoint_port=row.get('LocalPort'),\
+            dst_endpoint_ip=row.get('LocalIP'),\
+            dst_endpoint_type=row.get('Protocol'),\
+            session_uid=row.get('TransactionID'),\
+            bytes_out=row.get('BytesTransferred'),\
+            duration=row.get('TransferTime'),\
+            time=row.get('Time_stamp'),\
+            user_type=identifyUserType(row.get('user_type')),\
+            user_domain=row.get('SiteName'),\
+            user_name=row.get('Actor'),\
+            user_home_directory=row.get('VirtualFolderName'),\
+            utype=identifyUtype(row.get('Command'))))
+
+        if row.get('TransactionGUID') not in transactionLoginid:
+            processedData.append(template(identifyUtype(row.get('TransactionObject')),\
+                guid=row.get('TransactionGUID'),\
+                prd_instance_id=kwargs.get('prd_instance_id'),\
+                product_guid=kwargs.get('product_guid'),\
+                product_name=kwargs.get('product_name'),\
+                product_version=kwargs.get('product_version'),\
+                src_endpoint_type=row.get('Protocol'),\
+                src_endpoint_port=row.get('RemotePort'),\
+                src_endpoint_ip=row.get('RemoteIP'),\
+                dst_endpoint_port=row.get('LocalPort'),\
+                dst_endpoint_ip=row.get('LocalIP'),\
+                dst_endpoint_type=row.get('Protocol'),\
+                session_uid=row.get('TransactionID'),\
+                bytes_out=row.get('BytesTransferred'),\
+                transfer_time=row.get('TransferTime'),\
+                time=row.get('Time_stamp'),\
+                user_type=identifyUserType(row.get('user_type')),\
+                user_domain=row.get('SiteName'),\
+                user_name=row.get('Actor'),\
+                user_home_directory=row.get('VirtualFolderName'),\
+                utype=identifyUtype(row.get('TransactionObject'))\
+                ))
+            transactionLoginid.append(row.get('TransactionGUID'))
+
     return processedData
 
 def identifyUserType(obj):
@@ -40,10 +74,10 @@ def identifyUserType(obj):
     else:
         return None
 def identifyUtype(obj):
-    user_logged_on = []
-    file_deleted = []
-    file_uploaded = []
-    file_downloaded = []
+    user_logged_on = ['AUTH']
+    file_deleted = ["dele"]
+    file_uploaded = ["created"]
+    file_downloaded = ["sent"]
 
     if obj in user_logged_on:
         return "user_logged_on"

test.py

@@ -9,4 +9,13 @@ def connectDatabase(driver, server, database, user, password):
 def converttimestamp(t):
     print(int(t.timestamp()* 1000))
 
-a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))
+def builddict(keys,*args,**kwargs):
+    dict = {}
+    for key in keys:
+        dict[key] = kwargs.get(key)
+    print(dict)
+
+testfolder = '/Usr/a/asdf/asf'
+user = 'a'
+
+print(testfolder.split(f"/{user}/"))