From cc8adbebad75094fca4d4c29155d9d7f2e3f5239 Mon Sep 17 00:00:00 2001 From: jblu Date: Thu, 1 Aug 2024 11:02:37 -0500 Subject: [PATCH 1/2] embedded queries --- inex.py | 12 +++--- inexConnect.py | 112 +++++++++++++++++++++++++++++++----------------- inexSqlquery.py | 11 +++++ 3 files changed, 90 insertions(+), 45 deletions(-) create mode 100644 inexSqlquery.py diff --git a/inex.py b/inex.py index d9f75f0..8c4c5fb 100644 --- a/inex.py +++ b/inex.py @@ -1,23 +1,20 @@ import pyodbc import os import logging -import datetime import tomllib from inexLogging import inexLog import inexConnect from inexDataModel import dataTemplate from inexDataProcessing import processData import json -import decimal import requests import inexEncoder - +import inexSqlquery class Inex: def __init__(self): """Initilize config, calls functions from inex-connect.py and inex-logging.py""" # assign libraries self.db = pyodbc - self.tm = datetime self.il = logging self.ic = inexConnect self.r = requests @@ -25,6 +22,7 @@ class Inex: self.os = os self.j = json self.e = inexEncoder.Encoder + self.sq = inexSqlquery if self.os.path.exists('./config.toml'): config_file_path = './config.toml' @@ -63,9 +61,11 @@ class Inex: inexLog(self) # create the connection to the database - self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) + self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) - self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery) + # self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.dbQuery) + + self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData()) self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) diff --git a/inexConnect.py b/inexConnect.py index 0ea9542..ca57b0e 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -1,45 +1,79 @@ -def connectDatabase(self, lib, driver, server, database, user, password): - """Connects to the database. Requires a windows driver to do so. - Typically there is one installed by default""" +class inexSql: + def connectDatabase(self, lib, driver, server, database, user, password): + """Connects to the database. Requires a windows driver to do so. + Typically there is one installed by default""" - connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes' - if self.useLog: - self.il.debug(f"Connection String: connectionString") - self.il.info(f"Connecting to {database}@{server} with driver[{driver}].") - try: - connection = lib.connect(connectionString) - except lib.Error as ex: - sqlstate = ex.args[1] + connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes' if self.useLog: - self.il.error(sqlstate) - if self.useLog: - self.il.debug(f"Connected.") - cursor = connection.cursor() - - return cursor - -def databaseQuery(self, cursor, query, args=()): - if self.useLog: - self.il.debug(f"Query:") - self.il.debug(query) - self.il.info(f"Sending query:{query[0:20]}...") - - try: - cur = cursor.execute(query, args) - except cur.Error as ex: - sqlstate = ex.args[1] + self.il.debug(f"Connection String: connectionString") + self.il.info(f"Connecting to {database}@{server} with driver[{driver}].") + try: + connection = lib.connect(connectionString) + except lib.Error as ex: + sqlstate = ex.args[1] + if self.useLog: + self.il.error(sqlstate) if self.useLog: - self.il.error(sqlstate) - - if self.useLog: - self.il.debug(f"Processing database response...") - r = [dict((cur.description[i][0], value) \ - for i, value in enumerate(row)) for row in cur.fetchall()] - - cur.connection.close() - if self.useLog: - self.il.debug(f"Database connection closed") - return r + self.il.debug(f"Connected.") + cursor = connection.cursor() + + return cursor + + def databaseQuery(self, cursor, query, args=()): + if self.useLog: + self.il.debug(f"Query:") + self.il.debug(query) + self.il.info(f"Sending query:{query[0:20]}...") + + try: + cur = cursor.execute(query, args) + except cur.Error as ex: + sqlstate = ex.args[1] + if self.useLog: + self.il.error(sqlstate) + + if self.useLog: + self.il.debug(f"Processing database response...") + r = [dict((cur.description[i][0], value) \ + for i, value in enumerate(row)) for row in cur.fetchall()] + + cur.connection.close() + if self.useLog: + self.il.debug(f"Database connection closed") + return r + def queryData(): + """Embedded query data""" + q ="""DECLARE @stopTime DATETIME2 + SET @stopTime = DATEADD(DAY, -30, GETDATE()) + SELECT p.[ProtocolCommandID] + ,t.[Time_stamp] + ,p.[RemoteIP] + ,p.[RemotePort] + ,p.[LocalIP] + ,p.[LocalPort] + ,p.[Protocol] + ,p.[SiteName] + ,p.[Command] + ,p.[CommandParameters] + ,p.[FileName] + ,p.[VirtualFolderName] + ,p.[PhysicalFolderName] + ,p.[IsInternal] + ,p.[FileSize] + ,p.[TransferTime] + ,p.[BytesTransferred] + ,p.[ResultID] + ,t.[TransactionID] + ,p.[Description] + ,p.[Actor] + ,t.ParentTransactionID + ,t.TransactionObject + ,t.NodeName + ,t.TransactionGUID + ,a.Protocol user_type + FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID) + WHERE p.Time_stamp > @stopTime""" + return q class fortraEFC: def getToken(self): diff --git a/inexSqlquery.py b/inexSqlquery.py new file mode 100644 index 0000000..009d075 --- /dev/null +++ b/inexSqlquery.py @@ -0,0 +1,11 @@ +class sqlQuerymodel: + def queryData(): + """Embedded query data""" + q ="""DECLARE @stopTime DATETIME2 + SET @stopTime=DATEADD(DAY, -30, GETDATE()) + SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.CommandParameters, p.FileName, p.VirtualFolderName, p.PhysicalFolderName, p.IsInternal, p.FileSize, p.TransferTime, p.BytesTransferred, p.ResultID, t.TransactionID, p.Description, p.Actor, t.ParentTransactionID, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type + FROM tbl_Transactions t + Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID) + Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID) + WHERE p.Time_stamp>@stopTime""" + return q \ No newline at end of file From 643aaa946e6e7d384b0356a55c8a71a1cff0613e Mon Sep 17 00:00:00 2001 From: Jonathan Branan Date: Fri, 2 Aug 2024 20:57:20 -0500 Subject: [PATCH 2/2] generalized db query and embedded it. --- inexConnect.py | 33 ---------- inexDataModel.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 34 deletions(-) diff --git a/inexConnect.py b/inexConnect.py index ca57b0e..e653ece 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -41,39 +41,6 @@ class inexSql: if self.useLog: self.il.debug(f"Database connection closed") return r - def queryData(): - """Embedded query data""" - q ="""DECLARE @stopTime DATETIME2 - SET @stopTime = DATEADD(DAY, -30, GETDATE()) - SELECT p.[ProtocolCommandID] - ,t.[Time_stamp] - ,p.[RemoteIP] - ,p.[RemotePort] - ,p.[LocalIP] - ,p.[LocalPort] - ,p.[Protocol] - ,p.[SiteName] - ,p.[Command] - ,p.[CommandParameters] - ,p.[FileName] - ,p.[VirtualFolderName] - ,p.[PhysicalFolderName] - ,p.[IsInternal] - ,p.[FileSize] - ,p.[TransferTime] - ,p.[BytesTransferred] - ,p.[ResultID] - ,t.[TransactionID] - ,p.[Description] - ,p.[Actor] - ,t.ParentTransactionID - ,t.TransactionObject - ,t.NodeName - ,t.TransactionGUID - ,a.Protocol user_type - FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID) - WHERE p.Time_stamp > @stopTime""" - return q class fortraEFC: def getToken(self): diff --git a/inexDataModel.py b/inexDataModel.py index 132ced3..86fe862 100644 --- a/inexDataModel.py +++ b/inexDataModel.py @@ -45,4 +45,167 @@ def dataTemplate(**kwargs): "utype": kwargs.get('utype') } - return template \ No newline at end of file + return template + +FileUploaded = { + "bytes" : 2490, + "dst_endpoint": { + "port": 22, + "ip": "10.91.160.77", + "type": "SFTP" + }, + "duration": 200, + "file": { + "created_time": 1722485724000, + "size": 2490, + "name": "Case9.vbs", + "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //" + }, + "guid": "48D9C7A3-2DC6-11EF-AA59-00155D641204", + "node_name":"PERF01-S2019-77", + "prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", + "product_name": "GlobalScape EFT", + "prd_ext_tenant_name": "GlobalScape EFT", + "classifications": [{ + "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641204", + "time":1722485724000, + }], + "session": { + "created_time":1722485724000, + "uid": "3615136" + }, + "src_endpoint": { + "port": 58868, + "ip": "10.91.160.45", + "type":"SFTP" + }, + "tenant": "e71851c2-593f-4f49-9c07-91727b1be94b", + "tenant_name":"GlobalScape", + "time":1722485724000, + "status_code":226, + "status_detail":"Upload Successful", + "user": { + "home_directory": "/Usr/Ivan/", + "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641204", + "uid": "3978403", + "type": "User", + "name": "Ivan" + }, + "utype": "file_uploaded" +} + +FileDownloaded = { +"bytes" : 4891, +"dst_endpoint": { + "port": 443, + "ip": "10.91.160.77", + "type": "HTTPS" +}, +"duration": 200, +"file": { + "created_time": 1722518124000, + "size": 4891, + "name": "FileDownload1.exe", + "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan //" +}, +"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641205", +"node_name":"PERF01-S2019-77", +"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", +"product_name": "GlobalScape EFT", +"prd_ext_tenant_name": "GlobalScape EFT", +"classifications": [{ + "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641205", + "time":1722518124000, +}], +"session": { + "created_time":1722518124000, + "uid": "3615137" +}, +"src_endpoint": { + "port": 443, + "ip": "10.91.160.45", + "type":"HTTPS" +}, +"tenant": "e71851c2-593f-4f49-9c07-91727b1be94b", +"tenant_name":"GlobalScape", +"time":1722518124000, +"status_code":226, +"status_detail":"Download Successful", +"user": { + "home_directory": "/Usr/Ivan/", + "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641205", + "uid": "3978404", + "type": "User", + "name": "Ivan" +}, +"utype": "file_downloaded"} + +FileDeleted = { + "file": { + "size": 304673, + "path": "\\\\10.255.255.9\\shared\\HASite\\InetPub\\EFTRoot\\MySite\\Usr\\Ivan", +"name": "DeleteME.txt" +}, +"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641207", +"classifications": [{ + "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641207", + "time":1722515664000, +}], +"prd_ext_tenant_name": "Globalscape EFT", +"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", +"product_name": "Globalscape EFT", +"session": { + "created_time":1722515664000, + "uid": "3615138" +}, +"src_endpoint": { + "port": 443, + "ip": "10.91.160.45", + "type":"HTTPS" +}, +"dst_endpoint": { + "port": 443, + "ip": "10.91.160.77", + "type": "HTTPS" +}, +"time": 1722515664000, +"user": { + "home_directory": "/Usr/Ivan/", + "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641207", + "uid": "3978406", + "type": "User", + "name": "Ivan" +}, +"utype": "file_deleted" +} + +Logon ={ +"classifications": [{ + "ref_id": "globalscape:48D9C7A3-2DC6-11EF-AA59-00155D641206", + "time": 1722510924000, + +}], +"dst_endpoint": { + "port": 443, + "ip": "10.91.160.77", + "type": "HTTPS" +}, +"guid": "48D9C7A3-2DC6-11EF-AA59-00155D641206", +"prd_ext_tenant_id": "e71851c2-593f-4f49-9c07-91727b1be94b", +"product_name": "GlobalScape EFT", +"prd_ext_tenant_name": "GlobalScape EFT", +"src_endpoint": { + "port": 443, + "ip": "10.91.160.45", + "type":"HTTPS" +}, +"time": 1722510924000, +"user": { + "home_directory": "/Usr/Ivan/", + "uuid":"48D9C7A3-2DC6-11EF-AA59-00155D641206", + "uid": "3978405", + "type": "User", + "name": "Ivan" +}, +"utype": "user_logged_on" +} \ No newline at end of file