jblu/Inex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases 8 Wiki Activity

moved encoder to seperate file and created .token refresh logic

Browse Source
This commit is contained in:
Jonathan Branan 2024-07-29 17:30:11 -05:00
parent 1800aafd72
commit 045467783f
7 changed files with 128 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
*config.toml
*.toml
*.json
__pycache__/
*.log

72
config.toml.example
View File

@ -1,3 +1,27 @@
[fortraPlatform]
selectedPlatform = "dev"
[fortraPlatform.dev]
idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
tenant_id = ""
client_id = "eft-event-generator-confidential"
secret = ""
[fortraPlatform.stage]
idp = "https://foundation.foundation-stage.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
efc_url = "https://efc.efc-stage.cloudops.fortradev.com"
tenant_id = ""
client_id = "eft-event-generator-confidential"
secret = ""
[fortraPlatform.prod]
idp ="https://foundation.foundation-prod.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
efc_url = "https://efc.efc-prod.cloudops.fortradev.com"
tenant_id = ""
client_id = "eft-event-generator-confidential"
secret = ""
[database]
driver = "ODBC Driver 18 for SQL Server"
server = "192.168.x.x"
@ -6,32 +30,33 @@ user = "a"
password = "a"
query = """DECLARE @stopTime DATETIME2
SET @stopTime = DATEADD(DAY, -30, GETDATE())
SELECT [ProtocolCommandID]
,p.[Time_stamp]
,[RemoteIP]
,[RemotePort]
,[LocalIP]
,[LocalPort]
,[Protocol]
,[SiteName]
,[Command]
,[CommandParameters]
,[FileName]
,[VirtualFolderName]
,[PhysicalFolderName]
,[IsInternal]
,[FileSize]
,[TransferTime]
,[BytesTransferred]
,[ResultID]
,p.[TransactionID]
,[Description]
,[Actor]
SELECT p.[ProtocolCommandID]
,t.[Time_stamp]
,p.[RemoteIP]
,p.[RemotePort]
,p.[LocalIP]
,p.[LocalPort]
,p.[Protocol]
,p.[SiteName]
,p.[Command]
,p.[CommandParameters]
,p.[FileName]
,p.[VirtualFolderName]
,p.[PhysicalFolderName]
,p.[IsInternal]
,p.[FileSize]
,p.[TransferTime]
,p.[BytesTransferred]
,p.[ResultID]
,t.[TransactionID]
,p.[Description]
,p.[Actor]
,t.ParentTransactionID
,t.TransactionObject
,t.NodeName
,t.TransactionGUID
FROM [EFTDB].[dbo].[tbl_ProtocolCommands] p Full JOIN tbl_Transactions t ON (p.TransactionID = t.TransactionID)
,a.Protocol user_type
FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
WHERE p.Time_stamp > @stopTime"""
[immutables]
@ -41,7 +66,10 @@ product_name = "EFT"
product_version ="8.1.0.9"
[output]
pushToplatform = true
dumpTojson = true
filename ="./data.json"
token = "./.token"
[logging]
use_log = true

36
inex.py
View File

@ -10,6 +10,7 @@ from inexDataProcessing import processData
import json
import decimal
import requests
import inexEncoder
class Inex:
def __init__(self):
@ -23,6 +24,7 @@ class Inex:
self.tl = tomllib
self.os = os
self.j = json
self.e = inexEncoder.Encoder
if self.os.path.exists('./config.toml'):
config_file_path = './config.toml'
@ -46,6 +48,8 @@ class Inex:
self.productVersion = self.config["immutables"]["product_version"]
self.tokenFilepath = self.config["output"]["token"]
self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
self.writeJsonfile = self.config["output"]["dumpTojson"]
self.pushToplatform = self.config["output"]["pushToplatform"]
if "dev" in self.selectedPlatform.lower():
self.platformConfig = self.config["fortraPlatform"]["dev"]
@ -53,34 +57,28 @@ class Inex:
self.platformConfig = self.config["fortraPlatform"]["stage"]
if "prod" in self.selectedPlatform.lower():
self.platformConfig = self.config["fortraPlatform"]["prod"]
print(self.platformConfig)
# print(self.platformConfig)
#Setup logging
inexLog(self)
# create the connection to the database
# self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
# self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery)
self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery)
# self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
# product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
# # TODO: move this to its own function
# if self.useLog:
# self.il.warning(f"Writing to '{self.outputFile}'.")
if self.pushToplatform:
inexConnect.fortraEFC.pushPayload(self)
# with open(self.outputFile, "w") as f:
# json.dump(self.modifiedData, f, indent = 2, cls=Encoder)
# TODO: Move this class to it's own file
class Encoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, decimal.Decimal):
return int(o)
if isinstance(o, datetime.datetime):
return str(o)
return super().default(o)
# TODO: move this to its own function
if self.useLog:
self.il.warning(f"Writing to '{self.outputFile}'.")
if self.writeJsonfile:
with open(self.outputFile, "w") as f:
self.j.dump(self.modifiedData, f, indent = 2, cls=self.e)
# Run
if __name__== "__main__":

37
inexConnect.py
View File

@ -42,21 +42,32 @@ def databaseQuery(self, cursor, query, args=()):
return r
class fortraEFC:
def __init__(self):
def getToken(self):
self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
"client_id": self.platformConfig["client_id"],\
"client_secret": self.platformConfig["secret"],})
def writeToken(self):
fortraEFC.getToken(self)
with open(self.tokenFilepath, "w") as f:
self.j.dump(self.tokenData.json(), f, indent = 2)
def readToken(self):
if self.os.path.exists(self.tokenFilepath):
with open(self.tokenFilepath, 'rb') as t:
self.token = self.j.load(t)
print(self.token["access_token"])
def saveToken(self):
with open(self.tokenFilepath, "w") as f:
self.j.dump(self.tokenData, f, indent = 2)
def getToken(self):
self.tokenData = self.r.post(self.platformConfig["idp"], headers={"client_id": self.platformConfig["client_id"],"client_secret": self.platformConfig["secret"]})
self.tokenData = self.j.load(t)
# print(self.tokenData["access_token"])
else:
fortraEFC.writeToken(self)
def pushPayload(self):
url = f'{self.host}/api/v1/unity/data/{self.tenant_id}/machine_event'
pushPayloadResponse = self.r.post(self.platformConfig["efc_url"], headers={'Authorization': f'bearer {self.token["access_token"]}'},\
payload=self.modifiedData)
fortraEFC.readToken(self)
try:
url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\
json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e))
return pushPayloadResponse.status_code
except self.r.exceptions.HTTPError as errh:
print ("Http Error:",errh)
if "401" in errh:
fortraEFC.writeToken(self)
fortraEFC.pushPayload(self)

16
inexDataProcessing.py
View File

@ -28,7 +28,7 @@ def processData(data, template, **kwargs):
user_type=identifyUserType(row.get('user_type')),\
user_domain=row.get('SiteName'),\
user_name=row.get('Actor'),\
utype=row.get('TransactionObject')))
utype=identifyUtype(row.get('TransactionObject'))))
return processedData
def identifyUserType(obj):
@ -39,3 +39,17 @@ def identifyUserType(obj):
return "User"
else:
return None
def identifyUtype(obj):
user_logged_on = []
file_deleted = []
file_uploaded = []
file_downloaded = []
if obj in user_logged_on:
return "user_logged_on"
if obj in file_deleted:
return "file_deleted"
if obj in file_uploaded:
return "file_uploaded"
if obj in file_downloaded:
return "file_downloaded"

11
inexEncoder.py Normal file
View File

@ -0,0 +1,11 @@
import json
import decimal
import datetime
class Encoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, decimal.Decimal):
return int(o)
if isinstance(o, datetime.datetime):
return int(o.timestamp() * 1000)
return super().default(o)

9
test.py
View File

@ -1,5 +1,12 @@
import datetime
def connectDatabase(driver, server, database, user, password):
connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
print(connectionString)
a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
# a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
def converttimestamp(t):
print(int(t.timestamp()* 1000))
a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))