Merge pull request 'dev-create-rest-client' (#7) from dev-create-rest-client into main

Reviewed-on: #7

.gitea/workflows/build.yaml

@@ -1,10 +1,10 @@
-name: Build Inex Executable
-run-name: Deploy to ${{ inputs.deploy_target }} by @${{ gitea.actor }}
-on: [push]
+name: Build
+
+on: push
 
 jobs:
-  linux:
-    runs-on: ubuntu-22.04
+  build-linux-binary:
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
@@ -15,7 +15,23 @@ jobs:
     - run: apt-get install unixodbc -y
     - run: pip install -r requirements.txt
     - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
-    - uses: actions/upload-artifact@v3
+    - uses: softprops/action-gh-release@v2
+    - name: Release
+      uses: softprops/action-gh-release@v2
+      if: startsWith(gitea.ref, 'refs/tags/')
       with:
-        name: Inex
-        path: ${{ gitea.workspace }}/dist/inex
+        files: ${{ gitea.workspace }}/dist/inex
+  build-windows-binary:
+    runs-on: windows
+    permissions:
+      contents: write # release changes require contents write
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Upload Release Asset
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: pip install -r requirements.txt
+        run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
+        run: gh release upload <release_tag> <a_file>
+      

.gitignore

@@ -1,4 +1,5 @@
-*config.toml
+*.toml
 *.json
 __pycache__/
 *.log
+*.token

config.toml.example

@@ -1,3 +1,27 @@
+[fortraPlatform]
+selectedPlatform = "dev"
+
+[fortraPlatform.dev]
+idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
+[fortraPlatform.stage]
+idp = "https://foundation.foundation-stage.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-stage.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
+[fortraPlatform.prod]
+idp ="https://foundation.foundation-prod.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-prod.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
 [database]
 driver = "ODBC Driver 18 for SQL Server"
 server = "192.168.x.x"
@@ -6,32 +30,33 @@ user = "a"
 password = "a"
 query = """DECLARE	@stopTime DATETIME2
 SET @stopTime = DATEADD(DAY, -30, GETDATE())
-SELECT [ProtocolCommandID]
-      ,p.[Time_stamp]
-      ,[RemoteIP]
-      ,[RemotePort]
-      ,[LocalIP]
-      ,[LocalPort]
-      ,[Protocol]
-      ,[SiteName]
-      ,[Command]
-      ,[CommandParameters]
-      ,[FileName]
-      ,[VirtualFolderName]
-      ,[PhysicalFolderName]
-      ,[IsInternal]
-      ,[FileSize]
-      ,[TransferTime]
-      ,[BytesTransferred]
-      ,[ResultID]
-      ,p.[TransactionID]
-      ,[Description]
-      ,[Actor]
+SELECT p.[ProtocolCommandID]
+      ,t.[Time_stamp]
+      ,p.[RemoteIP]
+      ,p.[RemotePort]
+      ,p.[LocalIP]
+      ,p.[LocalPort]
+      ,p.[Protocol]
+      ,p.[SiteName]
+      ,p.[Command]
+      ,p.[CommandParameters]
+      ,p.[FileName]
+      ,p.[VirtualFolderName]
+      ,p.[PhysicalFolderName]
+      ,p.[IsInternal]
+      ,p.[FileSize]
+      ,p.[TransferTime]
+      ,p.[BytesTransferred]
+      ,p.[ResultID]
+      ,t.[TransactionID]
+      ,p.[Description]
+      ,p.[Actor]
 	  ,t.ParentTransactionID
 	  ,t.TransactionObject
 	  ,t.NodeName
 	  ,t.TransactionGUID
-  FROM [EFTDB].[dbo].[tbl_ProtocolCommands] p Full JOIN tbl_Transactions t ON (p.TransactionID = t.TransactionID)
+	  ,a.Protocol user_type
+  FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
   WHERE p.Time_stamp > @stopTime"""
 
 [immutables]
@@ -41,7 +66,10 @@ product_name = "EFT"
 product_version ="8.1.0.9"
 
 [output]
+pushToplatform = true
+dumpTojson = true
 filename ="./data.json"
+token = "./.token"
 
 [logging]
 use_log = true

inex.py

@@ -2,27 +2,34 @@ import pyodbc
 import os
 import logging
 import datetime
-from tomllib import load
+import tomllib
 from inexLogging import inexLog
 import inexConnect
 from inexDataModel import dataTemplate
 from inexDataProcessing import processData
 import json
 import decimal
+import requests
+import inexEncoder
 
 class Inex:
     def __init__(self):
         """Initilize config, calls functions from inex-connect.py and inex-logging.py"""
-        if os.path.exists('./config.toml'):
-            config_file_path = './config.toml'
-            with open(config_file_path, 'rb') as c:
-                self.config = load(c)
-
         # assign libraries
         self.db = pyodbc
         self.tm = datetime
         self.il = logging
         self.ic = inexConnect
+        self.r = requests
+        self.tl = tomllib
+        self.os = os
+        self.j = json
+        self.e = inexEncoder.Encoder
+
+        if self.os.path.exists('./config.toml'):
+            config_file_path = './config.toml'
+            with open(config_file_path, 'rb') as c:
+                self.config = self.tl.load(c)
         
         # set config
         self.dbDriver = self.config["database"]["driver"]
@@ -39,6 +46,18 @@ class Inex:
         self.productGUID = self.config["immutables"]["product_guid"]
         self.productName = self.config["immutables"]["product_name"]
         self.productVersion = self.config["immutables"]["product_version"]
+        self.tokenFilepath = self.config["output"]["token"]
+        self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
+        self.writeJsonfile = self.config["output"]["dumpTojson"]
+        self.pushToplatform = self.config["output"]["pushToplatform"]
+
+        if "dev" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["dev"]
+        if "stag" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["stage"]
+        if "prod" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["prod"]
+        # print(self.platformConfig)
 
         #Setup logging
         inexLog(self)
@@ -51,22 +70,15 @@ class Inex:
         self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
                                          product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
         
+        if self.pushToplatform:
+            inexConnect.fortraEFC.pushPayload(self)
 
         # TODO: move this to its own function
         if self.useLog:
             self.il.warning(f"Writing to '{self.outputFile}'.")
-
+        if self.writeJsonfile:
             with open(self.outputFile, "w") as f:
-            json.dump(self.modifiedData, f, indent = 2, cls=Encoder)
-
-# TODO: Move this class to it's own file
-class Encoder(json.JSONEncoder):
-    def default(self, o):
-        if isinstance(o, decimal.Decimal):
-            return int(o)
-        if isinstance(o, datetime.datetime):
-            return str(o)
-        return super().default(o)
+                self.j.dump(self.modifiedData, f, indent = 2, cls=self.e)
 
 # Run
 if  __name__== "__main__":

inexConnect.py

@@ -39,5 +39,35 @@ def databaseQuery(self, cursor, query, args=()):
     cur.connection.close()
     if self.useLog:
         self.il.debug(f"Database connection closed")
-    # return (r[0] if r else None) if one else r
     return r
+
+class fortraEFC:
+    def getToken(self):
+        self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
+                                                                              "client_id": self.platformConfig["client_id"],\
+                                                                              "client_secret": self.platformConfig["secret"],})
+    def writeToken(self):
+        fortraEFC.getToken(self)
+        with open(self.tokenFilepath, "w") as f:
+            self.j.dump(self.tokenData.json(), f, indent = 2)
+
+    def readToken(self):
+        if self.os.path.exists(self.tokenFilepath):
+            with open(self.tokenFilepath, 'rb') as t:
+                self.tokenData = self.j.load(t)
+                # print(self.tokenData["access_token"])
+        else:
+            fortraEFC.writeToken(self)
+
+    def pushPayload(self):
+        fortraEFC.readToken(self)
+        try:
+            url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
+            pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\
+                                               json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e))
+            return pushPayloadResponse.status_code
+        except self.r.exceptions.HTTPError as errh:
+            print ("Http Error:",errh)
+            if "401" in errh:
+                fortraEFC.writeToken(self)
+                fortraEFC.pushPayload(self)

inexDataProcessing.py

@@ -28,7 +28,7 @@ def processData(data, template, **kwargs):
                             user_type=identifyUserType(row.get('user_type')),\
                             user_domain=row.get('SiteName'),\
                             user_name=row.get('Actor'),\
-                            utype=row.get('TransactionObject')))
+                            utype=identifyUtype(row.get('TransactionObject'))))
     return processedData
 
 def identifyUserType(obj):
@@ -39,3 +39,19 @@ def identifyUserType(obj):
             return "User"
     else:
         return None
+def identifyUtype(obj):
+    user_logged_on = []
+    file_deleted = []
+    file_uploaded = []
+    file_downloaded = []
+
+    if obj in user_logged_on:
+        return "user_logged_on"
+    if obj in file_deleted:
+        return "file_deleted"
+    if obj in file_uploaded:
+        return "file_uploaded"
+    if obj in file_downloaded:
+        return "file_downloaded"
+    else:
+        return None

inexEncoder.py

@@ -0,0 +1,11 @@
+import json
+import decimal
+import datetime
+
+class Encoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, decimal.Decimal):
+            return int(o)
+        if isinstance(o, datetime.datetime):
+            return int(o.timestamp() * 1000)
+        return super().default(o)

test.py

@@ -1,5 +1,12 @@
+import datetime
+
 def connectDatabase(driver, server, database, user, password):
     connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
     print(connectionString)
 
-a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
+# a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
+
+def converttimestamp(t):
+    print(int(t.timestamp()* 1000))
+
+a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))