From c030fc69a2b6a995f1064ecac25a9ccf8b0b0250 Mon Sep 17 00:00:00 2001 From: Jonathan Branan Date: Thu, 25 Jul 2024 13:30:21 -0500 Subject: [PATCH 1/7] updating build.yaml --- .gitea/workflows/build.yaml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index d5f7f1d..cdfb260 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -1,10 +1,10 @@ -name: Build Inex Executable -run-name: Deploy to ${{ inputs.deploy_target }} by @${{ gitea.actor }} -on: [push] +name: Build + +on: push jobs: - linux: - runs-on: ubuntu-22.04 + build-linux-binary: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -15,7 +15,13 @@ jobs: - run: apt-get install unixodbc -y - run: pip install -r requirements.txt - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py - - uses: actions/upload-artifact@v3 + - uses: softprops/action-gh-release@v2 + - name: Release + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') with: - name: Inex - path: ${{ gitea.workspace }}/dist/inex \ No newline at end of file + files: ${{ gitea.workspace }}/dist/inex + build-windows-binary: + runs-on: windows + steps: + \ No newline at end of file From 573cd651beb41674a4a8816fa9972ea282873ef8 Mon Sep 17 00:00:00 2001 From: jblu Date: Thu, 25 Jul 2024 16:09:09 -0500 Subject: [PATCH 2/7] automated release idea --- .gitea/workflows/build.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index cdfb260..ce6b73d 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -18,10 +18,20 @@ jobs: - uses: softprops/action-gh-release@v2 - name: Release uses: softprops/action-gh-release@v2 - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(gitea.ref, 'refs/tags/') with: files: ${{ gitea.workspace }}/dist/inex build-windows-binary: runs-on: windows + permissions: + contents: write # release changes require contents write + steps: + - uses: actions/checkout@v4 + - name: Upload Release Asset + env: + GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: pip install -r requirements.txt + run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py + run: gh release upload \ No newline at end of file From 1f255ec33a38faa7a768b29807d8efabf023c0e0 Mon Sep 17 00:00:00 2001 From: jblu Date: Thu, 25 Jul 2024 22:22:08 -0500 Subject: [PATCH 3/7] Added push payload --- inex.py | 2 ++ inexConnect.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/inex.py b/inex.py index 7e2e149..ad805c6 100644 --- a/inex.py +++ b/inex.py @@ -9,6 +9,7 @@ from inexDataModel import dataTemplate from inexDataProcessing import processData import json import decimal +import requests class Inex: def __init__(self): @@ -23,6 +24,7 @@ class Inex: self.tm = datetime self.il = logging self.ic = inexConnect + self.r = requests # set config self.dbDriver = self.config["database"]["driver"] diff --git a/inexConnect.py b/inexConnect.py index dad618a..cad9d37 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -40,4 +40,22 @@ def databaseQuery(self, cursor, query, args=()): if self.useLog: self.il.debug(f"Database connection closed") # return (r[0] if r else None) if one else r - return r \ No newline at end of file + return r + +def renewToken(func): + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except error: + getToken() + return func(*args, **kwargs) + return wrapper + +def getToken(idpUrl, id, secret): + pass + +@renewToken +def pushPayload(reqObj, targetUrl, token, payload): + pushPayloadResponse = reqObj.post(targetUrl, headers={"Bearer": token},\ + payload=payload,verify=False) + return pushPayloadResponse.status_code \ No newline at end of file From 11b312a1e8cf834f0912e73ba749312459516b5b Mon Sep 17 00:00:00 2001 From: jblu Date: Sat, 27 Jul 2024 18:24:15 -0500 Subject: [PATCH 4/7] updated client --- inexConnect.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/inexConnect.py b/inexConnect.py index cad9d37..fcb7474 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -42,20 +42,12 @@ def databaseQuery(self, cursor, query, args=()): # return (r[0] if r else None) if one else r return r -def renewToken(func): - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except error: - getToken() - return func(*args, **kwargs) - return wrapper +def getToken(reqObj,idpUrl, id, secret): + getTokenResponse = reqObj.post(idpUrl, headers={"client_id": id,"client_secret": secret}) + return getTokenResponse["access_token"] -def getToken(idpUrl, id, secret): - pass - -@renewToken -def pushPayload(reqObj, targetUrl, token, payload): - pushPayloadResponse = reqObj.post(targetUrl, headers={"Bearer": token},\ - payload=payload,verify=False) +def pushPayload(reqObj, host, token, tenant_id, payload): + url = f'{host}/api/v1/unity/data/{tenant_id}/machine_event' + pushPayloadResponse = reqObj.post(url, headers={'Authorization': f'bearer {token}'},\ + payload=payload) return pushPayloadResponse.status_code \ No newline at end of file From 1800aafd72deba7ea626cc6a7c06bcbaf52a21bc Mon Sep 17 00:00:00 2001 From: jblu Date: Mon, 29 Jul 2024 14:04:35 -0500 Subject: [PATCH 5/7] updated saving tokens --- .gitignore | 3 ++- inex.py | 48 ++++++++++++++++++++++++++++++------------------ inexConnect.py | 27 ++++++++++++++++++--------- 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index d13a43a..b0b8eda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *config.toml *.json __pycache__/ -*.log \ No newline at end of file +*.log +*.token \ No newline at end of file diff --git a/inex.py b/inex.py index ad805c6..4ac30ba 100644 --- a/inex.py +++ b/inex.py @@ -2,7 +2,7 @@ import pyodbc import os import logging import datetime -from tomllib import load +import tomllib from inexLogging import inexLog import inexConnect from inexDataModel import dataTemplate @@ -14,18 +14,21 @@ import requests class Inex: def __init__(self): """Initilize config, calls functions from inex-connect.py and inex-logging.py""" - if os.path.exists('./config.toml'): - config_file_path = './config.toml' - with open(config_file_path, 'rb') as c: - self.config = load(c) - # assign libraries self.db = pyodbc self.tm = datetime self.il = logging self.ic = inexConnect self.r = requests + self.tl = tomllib + self.os = os + self.j = json + if self.os.path.exists('./config.toml'): + config_file_path = './config.toml' + with open(config_file_path, 'rb') as c: + self.config = self.tl.load(c) + # set config self.dbDriver = self.config["database"]["driver"] self.dbServer = self.config["database"]["server"] @@ -41,25 +44,34 @@ class Inex: self.productGUID = self.config["immutables"]["product_guid"] self.productName = self.config["immutables"]["product_name"] self.productVersion = self.config["immutables"]["product_version"] - + self.tokenFilepath = self.config["output"]["token"] + self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] + + if "dev" in self.selectedPlatform.lower(): + self.platformConfig = self.config["fortraPlatform"]["dev"] + if "stag" in self.selectedPlatform.lower(): + self.platformConfig = self.config["fortraPlatform"]["stage"] + if "prod" in self.selectedPlatform.lower(): + self.platformConfig = self.config["fortraPlatform"]["prod"] + print(self.platformConfig) + #Setup logging inexLog(self) # create the connection to the database - self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) + # self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) - self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery) + # self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery) - self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ - product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) + # self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ + # product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) + + # # TODO: move this to its own function + # if self.useLog: + # self.il.warning(f"Writing to '{self.outputFile}'.") - - # TODO: move this to its own function - if self.useLog: - self.il.warning(f"Writing to '{self.outputFile}'.") - - with open(self.outputFile, "w") as f: - json.dump(self.modifiedData, f, indent = 2, cls=Encoder) + # with open(self.outputFile, "w") as f: + # json.dump(self.modifiedData, f, indent = 2, cls=Encoder) # TODO: Move this class to it's own file class Encoder(json.JSONEncoder): diff --git a/inexConnect.py b/inexConnect.py index fcb7474..c36fed0 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -39,15 +39,24 @@ def databaseQuery(self, cursor, query, args=()): cur.connection.close() if self.useLog: self.il.debug(f"Database connection closed") - # return (r[0] if r else None) if one else r return r -def getToken(reqObj,idpUrl, id, secret): - getTokenResponse = reqObj.post(idpUrl, headers={"client_id": id,"client_secret": secret}) - return getTokenResponse["access_token"] +class fortraEFC: + def __init__(self): + if self.os.path.exists(self.tokenFilepath): + with open(self.tokenFilepath, 'rb') as t: + self.token = self.j.load(t) + print(self.token["access_token"]) -def pushPayload(reqObj, host, token, tenant_id, payload): - url = f'{host}/api/v1/unity/data/{tenant_id}/machine_event' - pushPayloadResponse = reqObj.post(url, headers={'Authorization': f'bearer {token}'},\ - payload=payload) - return pushPayloadResponse.status_code \ No newline at end of file + def saveToken(self): + with open(self.tokenFilepath, "w") as f: + self.j.dump(self.tokenData, f, indent = 2) + + def getToken(self): + self.tokenData = self.r.post(self.platformConfig["idp"], headers={"client_id": self.platformConfig["client_id"],"client_secret": self.platformConfig["secret"]}) + + def pushPayload(self): + url = f'{self.host}/api/v1/unity/data/{self.tenant_id}/machine_event' + pushPayloadResponse = self.r.post(self.platformConfig["efc_url"], headers={'Authorization': f'bearer {self.token["access_token"]}'},\ + payload=self.modifiedData) + return pushPayloadResponse.status_code \ No newline at end of file From 045467783f083ebc012f191bfa4bfecb28648983 Mon Sep 17 00:00:00 2001 From: Jonathan Branan Date: Mon, 29 Jul 2024 17:30:11 -0500 Subject: [PATCH 6/7] moved encoder to seperate file and created .token refresh logic --- .gitignore | 2 +- config.toml.example | 72 ++++++++++++++++++++++++++++++------------- inex.py | 36 ++++++++++------------ inexConnect.py | 39 ++++++++++++++--------- inexDataProcessing.py | 18 +++++++++-- inexEncoder.py | 11 +++++++ test.py | 9 +++++- 7 files changed, 128 insertions(+), 59 deletions(-) create mode 100644 inexEncoder.py diff --git a/.gitignore b/.gitignore index b0b8eda..f37f297 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -*config.toml +*.toml *.json __pycache__/ *.log diff --git a/config.toml.example b/config.toml.example index 59bb7a0..1ea5daf 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,3 +1,27 @@ +[fortraPlatform] +selectedPlatform = "dev" + +[fortraPlatform.dev] +idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token" +efc_url = "https://efc.efc-dev.cloudops.fortradev.com" +tenant_id = "" +client_id = "eft-event-generator-confidential" +secret = "" + +[fortraPlatform.stage] +idp = "https://foundation.foundation-stage.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token" +efc_url = "https://efc.efc-stage.cloudops.fortradev.com" +tenant_id = "" +client_id = "eft-event-generator-confidential" +secret = "" + +[fortraPlatform.prod] +idp ="https://foundation.foundation-prod.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token" +efc_url = "https://efc.efc-prod.cloudops.fortradev.com" +tenant_id = "" +client_id = "eft-event-generator-confidential" +secret = "" + [database] driver = "ODBC Driver 18 for SQL Server" server = "192.168.x.x" @@ -6,32 +30,33 @@ user = "a" password = "a" query = """DECLARE @stopTime DATETIME2 SET @stopTime = DATEADD(DAY, -30, GETDATE()) -SELECT [ProtocolCommandID] - ,p.[Time_stamp] - ,[RemoteIP] - ,[RemotePort] - ,[LocalIP] - ,[LocalPort] - ,[Protocol] - ,[SiteName] - ,[Command] - ,[CommandParameters] - ,[FileName] - ,[VirtualFolderName] - ,[PhysicalFolderName] - ,[IsInternal] - ,[FileSize] - ,[TransferTime] - ,[BytesTransferred] - ,[ResultID] - ,p.[TransactionID] - ,[Description] - ,[Actor] +SELECT p.[ProtocolCommandID] + ,t.[Time_stamp] + ,p.[RemoteIP] + ,p.[RemotePort] + ,p.[LocalIP] + ,p.[LocalPort] + ,p.[Protocol] + ,p.[SiteName] + ,p.[Command] + ,p.[CommandParameters] + ,p.[FileName] + ,p.[VirtualFolderName] + ,p.[PhysicalFolderName] + ,p.[IsInternal] + ,p.[FileSize] + ,p.[TransferTime] + ,p.[BytesTransferred] + ,p.[ResultID] + ,t.[TransactionID] + ,p.[Description] + ,p.[Actor] ,t.ParentTransactionID ,t.TransactionObject ,t.NodeName ,t.TransactionGUID - FROM [EFTDB].[dbo].[tbl_ProtocolCommands] p Full JOIN tbl_Transactions t ON (p.TransactionID = t.TransactionID) + ,a.Protocol user_type + FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID) WHERE p.Time_stamp > @stopTime""" [immutables] @@ -41,7 +66,10 @@ product_name = "EFT" product_version ="8.1.0.9" [output] +pushToplatform = true +dumpTojson = true filename ="./data.json" +token = "./.token" [logging] use_log = true diff --git a/inex.py b/inex.py index 4ac30ba..d9f75f0 100644 --- a/inex.py +++ b/inex.py @@ -10,6 +10,7 @@ from inexDataProcessing import processData import json import decimal import requests +import inexEncoder class Inex: def __init__(self): @@ -23,6 +24,7 @@ class Inex: self.tl = tomllib self.os = os self.j = json + self.e = inexEncoder.Encoder if self.os.path.exists('./config.toml'): config_file_path = './config.toml' @@ -46,6 +48,8 @@ class Inex: self.productVersion = self.config["immutables"]["product_version"] self.tokenFilepath = self.config["output"]["token"] self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"] + self.writeJsonfile = self.config["output"]["dumpTojson"] + self.pushToplatform = self.config["output"]["pushToplatform"] if "dev" in self.selectedPlatform.lower(): self.platformConfig = self.config["fortraPlatform"]["dev"] @@ -53,34 +57,28 @@ class Inex: self.platformConfig = self.config["fortraPlatform"]["stage"] if "prod" in self.selectedPlatform.lower(): self.platformConfig = self.config["fortraPlatform"]["prod"] - print(self.platformConfig) + # print(self.platformConfig) #Setup logging inexLog(self) # create the connection to the database - # self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) + self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword) - # self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery) + self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery) - # self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ - # product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) + self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\ + product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion) - # # TODO: move this to its own function - # if self.useLog: - # self.il.warning(f"Writing to '{self.outputFile}'.") + if self.pushToplatform: + inexConnect.fortraEFC.pushPayload(self) - # with open(self.outputFile, "w") as f: - # json.dump(self.modifiedData, f, indent = 2, cls=Encoder) - -# TODO: Move this class to it's own file -class Encoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, decimal.Decimal): - return int(o) - if isinstance(o, datetime.datetime): - return str(o) - return super().default(o) + # TODO: move this to its own function + if self.useLog: + self.il.warning(f"Writing to '{self.outputFile}'.") + if self.writeJsonfile: + with open(self.outputFile, "w") as f: + self.j.dump(self.modifiedData, f, indent = 2, cls=self.e) # Run if __name__== "__main__": diff --git a/inexConnect.py b/inexConnect.py index c36fed0..0ea9542 100644 --- a/inexConnect.py +++ b/inexConnect.py @@ -42,21 +42,32 @@ def databaseQuery(self, cursor, query, args=()): return r class fortraEFC: - def __init__(self): + def getToken(self): + self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\ + "client_id": self.platformConfig["client_id"],\ + "client_secret": self.platformConfig["secret"],}) + def writeToken(self): + fortraEFC.getToken(self) + with open(self.tokenFilepath, "w") as f: + self.j.dump(self.tokenData.json(), f, indent = 2) + + def readToken(self): if self.os.path.exists(self.tokenFilepath): with open(self.tokenFilepath, 'rb') as t: - self.token = self.j.load(t) - print(self.token["access_token"]) + self.tokenData = self.j.load(t) + # print(self.tokenData["access_token"]) + else: + fortraEFC.writeToken(self) - def saveToken(self): - with open(self.tokenFilepath, "w") as f: - self.j.dump(self.tokenData, f, indent = 2) - - def getToken(self): - self.tokenData = self.r.post(self.platformConfig["idp"], headers={"client_id": self.platformConfig["client_id"],"client_secret": self.platformConfig["secret"]}) - def pushPayload(self): - url = f'{self.host}/api/v1/unity/data/{self.tenant_id}/machine_event' - pushPayloadResponse = self.r.post(self.platformConfig["efc_url"], headers={'Authorization': f'bearer {self.token["access_token"]}'},\ - payload=self.modifiedData) - return pushPayloadResponse.status_code \ No newline at end of file + fortraEFC.readToken(self) + try: + url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event' + pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\ + json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e)) + return pushPayloadResponse.status_code + except self.r.exceptions.HTTPError as errh: + print ("Http Error:",errh) + if "401" in errh: + fortraEFC.writeToken(self) + fortraEFC.pushPayload(self) \ No newline at end of file diff --git a/inexDataProcessing.py b/inexDataProcessing.py index 2b7c2ee..00ceddd 100644 --- a/inexDataProcessing.py +++ b/inexDataProcessing.py @@ -28,7 +28,7 @@ def processData(data, template, **kwargs): user_type=identifyUserType(row.get('user_type')),\ user_domain=row.get('SiteName'),\ user_name=row.get('Actor'),\ - utype=row.get('TransactionObject'))) + utype=identifyUtype(row.get('TransactionObject')))) return processedData def identifyUserType(obj): @@ -38,4 +38,18 @@ def identifyUserType(obj): else: return "User" else: - return None \ No newline at end of file + return None +def identifyUtype(obj): + user_logged_on = [] + file_deleted = [] + file_uploaded = [] + file_downloaded = [] + + if obj in user_logged_on: + return "user_logged_on" + if obj in file_deleted: + return "file_deleted" + if obj in file_uploaded: + return "file_uploaded" + if obj in file_downloaded: + return "file_downloaded" \ No newline at end of file diff --git a/inexEncoder.py b/inexEncoder.py new file mode 100644 index 0000000..09ee3a9 --- /dev/null +++ b/inexEncoder.py @@ -0,0 +1,11 @@ +import json +import decimal +import datetime + +class Encoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, decimal.Decimal): + return int(o) + if isinstance(o, datetime.datetime): + return int(o.timestamp() * 1000) + return super().default(o) \ No newline at end of file diff --git a/test.py b/test.py index 1d193f6..fb702bc 100644 --- a/test.py +++ b/test.py @@ -1,5 +1,12 @@ +import datetime + def connectDatabase(driver, server, database, user, password): connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes' print(connectionString) -a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e") \ No newline at end of file +# a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e") + +def converttimestamp(t): + print(int(t.timestamp()* 1000)) + +a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000)) \ No newline at end of file From 2b3a1a7078da5fdd8aeb43f117c153972eb702a4 Mon Sep 17 00:00:00 2001 From: jblu Date: Tue, 30 Jul 2024 12:46:19 -0500 Subject: [PATCH 7/7] adjusted utype handling --- inexDataProcessing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inexDataProcessing.py b/inexDataProcessing.py index 00ceddd..a64d964 100644 --- a/inexDataProcessing.py +++ b/inexDataProcessing.py @@ -52,4 +52,6 @@ def identifyUtype(obj): if obj in file_uploaded: return "file_uploaded" if obj in file_downloaded: - return "file_downloaded" \ No newline at end of file + return "file_downloaded" + else: + return None \ No newline at end of file