Merge pull request 'dev-automate-workflow' (#8) from dev-automate-workflow into main

Reviewed-on: #8

.gitea/workflows/build.yaml

@@ -1,21 +1,35 @@
-name: Build Inex Executable
+name: Build
-run-name: Deploy to ${{ inputs.deploy_target }} by @${{ gitea.actor }}
+on: push
-on: [push]
 
 jobs:
-  Build:
+  # build-linux-binary:
-    runs-on: ubuntu-22.04
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - uses: actions/checkout@v4
+  #   - uses: actions/setup-python@v5
+  #     with:
+  #       python-version: '3.12.4'
+  #       cache: 'pip' # caching pip dependencies
+  #   - run: apt-get update
+  #   - run: apt-get install unixodbc -y
+  #   - run: pip install -r requirements.txt
+  #   - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
+  #   - uses: softprops/action-gh-release@v2
+  #   - name: Release
+  #     uses: softprops/action-gh-release@v2
+  #     if: startsWith(gitea.ref, 'refs/tags/')
+  #     with:
+  #       files: ${{ gitea.workspace }}/dist/inex
+  build-windows-binary:
+    runs-on: windows
+    permissions:
+      contents: write # release changes require contents write
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
+      - name: Upload Release Asset
-      with:
+        env:
-        python-version: '3.12.4'
+          GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
-        cache: 'pip' # caching pip dependencies
+        run: pip install -r requirements.txt
-    - run: apt-get update
+        run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
-    - run: apt-get install unixodbc -y
+        # run: gh release upload <release_tag> <a_file>
-    - run: pip install -r requirements.txt
+      
-    - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
-    - uses: actions/upload-artifact@v3
-      with:
-        name: Inex
-        path: ${{ gitea.workspace }}/dist/inex

.gitignore

@@ -1,4 +1,5 @@
-*config.toml
+*.toml
 *.json
 __pycache__/
 *.log
+*.token

config.toml.example

@@ -1,13 +1,75 @@
+[fortraPlatform]
+selectedPlatform = "dev"
+
+[fortraPlatform.dev]
+idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
+[fortraPlatform.stage]
+idp = "https://foundation.foundation-stage.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-stage.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
+[fortraPlatform.prod]
+idp ="https://foundation.foundation-prod.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
+efc_url = "https://efc.efc-prod.cloudops.fortradev.com"
+tenant_id = ""
+client_id = "eft-event-generator-confidential"
+secret = ""
+
 [database]
 driver = "ODBC Driver 18 for SQL Server"
 server = "192.168.x.x"
 database = "EFTDB"
 user = "a"
 password = "a"
-query = "SELECT [Id],[Version] FROM [EFTDB].[dbo].[tbl_Schema_Version]"
+query = """DECLARE	@stopTime DATETIME2
+SET @stopTime = DATEADD(DAY, -30, GETDATE())
+SELECT p.[ProtocolCommandID]
+      ,t.[Time_stamp]
+      ,p.[RemoteIP]
+      ,p.[RemotePort]
+      ,p.[LocalIP]
+      ,p.[LocalPort]
+      ,p.[Protocol]
+      ,p.[SiteName]
+      ,p.[Command]
+      ,p.[CommandParameters]
+      ,p.[FileName]
+      ,p.[VirtualFolderName]
+      ,p.[PhysicalFolderName]
+      ,p.[IsInternal]
+      ,p.[FileSize]
+      ,p.[TransferTime]
+      ,p.[BytesTransferred]
+      ,p.[ResultID]
+      ,t.[TransactionID]
+      ,p.[Description]
+      ,p.[Actor]
+	  ,t.ParentTransactionID
+	  ,t.TransactionObject
+	  ,t.NodeName
+	  ,t.TransactionGUID
+	  ,a.Protocol user_type
+  FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
+  WHERE p.Time_stamp > @stopTime"""
+
+[immutables]
+prd_instance_id = 1
+product_guid = "asdf"
+product_name = "EFT"
+product_version ="8.1.0.9"
 
 [output]
+pushToplatform = true
+dumpTojson = true
 filename ="./data.json"
+token = "./.token"
 
 [logging]
 use_log = true

executable/config.toml.example

@@ -4,8 +4,28 @@ server = "192.168.x.x"
 database = "EFTDB"
 user = "a"
 password = "a"
-query = "SELECT [Id],[Version] FROM [EFTDB].[dbo].[tbl_Schema_Version]"
+query = """SELECT TOP (1) [ProtocolCommandID]
-
+      ,[Time_stamp]
+      ,[RemoteIP]
+      ,[RemotePort]
+      ,[LocalIP]
+      ,[LocalPort]
+      ,[Protocol]
+      ,[SiteName]
+      ,[Command]
+      ,[CommandParameters]
+      ,[FileName]
+      ,[VirtualFolderName]
+      ,[PhysicalFolderName]
+      ,[IsInternal]
+      ,[FileSize]
+      ,[TransferTime]
+      ,[BytesTransferred]
+      ,[ResultID]
+      ,[TransactionID]
+      ,[Description]
+      ,[Actor]
+  FROM [EFTDB].[dbo].[tbl_ProtocolCommands]"""
 [output]
 filename ="./data.json"
 

inex.py

@@ -2,27 +2,34 @@ import pyodbc
 import os
 import logging
 import datetime
-from tomllib import load
+import tomllib
 from inexLogging import inexLog
 import inexConnect
 from inexDataModel import dataTemplate
 from inexDataProcessing import processData
 import json
 import decimal
+import requests
+import inexEncoder
 
 class Inex:
     def __init__(self):
         """Initilize config, calls functions from inex-connect.py and inex-logging.py"""
-        if os.path.exists('./config.toml'):
-            config_file_path = './config.toml'
-            with open(config_file_path, 'rb') as c:
-                self.config = load(c)
-
         # assign libraries
         self.db = pyodbc
         self.tm = datetime
         self.il = logging
         self.ic = inexConnect
+        self.r = requests
+        self.tl = tomllib
+        self.os = os
+        self.j = json
+        self.e = inexEncoder.Encoder
+
+        if self.os.path.exists('./config.toml'):
+            config_file_path = './config.toml'
+            with open(config_file_path, 'rb') as c:
+                self.config = self.tl.load(c)
         
         # set config
         self.dbDriver = self.config["database"]["driver"]
@@ -35,6 +42,22 @@ class Inex:
         self.useLog = self.config["logging"]["useLog"]
         self.logPath = self.config["logging"]["logPath"]
         self.logLevel = self.config["logging"]["logLevel"]
+        self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
+        self.productGUID = self.config["immutables"]["product_guid"]
+        self.productName = self.config["immutables"]["product_name"]
+        self.productVersion = self.config["immutables"]["product_version"]
+        self.tokenFilepath = self.config["output"]["token"]
+        self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
+        self.writeJsonfile = self.config["output"]["dumpTojson"]
+        self.pushToplatform = self.config["output"]["pushToplatform"]
+
+        if "dev" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["dev"]
+        if "stag" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["stage"]
+        if "prod" in self.selectedPlatform.lower():
+            self.platformConfig = self.config["fortraPlatform"]["prod"]
+        # print(self.platformConfig)
 
         #Setup logging
         inexLog(self)
@@ -43,26 +66,19 @@ class Inex:
         self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
 
         self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery)
-        # print(f"returned data: {self.data}")
-        self.modifiedData = processData(self.data, dataTemplate)
 
-        print(self.modifiedData)
+        self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
+                                         product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
+        
+        if self.pushToplatform:
+            inexConnect.fortraEFC.pushPayload(self)
 
         # TODO: move this to its own function
         if self.useLog:
             self.il.warning(f"Writing to '{self.outputFile}'.")
-
+        if self.writeJsonfile:
-        with open(self.outputFile, "w") as f:
+            with open(self.outputFile, "w") as f:
-            json.dump(self.modifiedData, f, indent = 2, cls=Encoder)
+                self.j.dump(self.modifiedData, f, indent = 2, cls=self.e)
-
-# TODO: Move this class to it's own file
-class Encoder(json.JSONEncoder):
-    def default(self, o):
-        if isinstance(o, decimal.Decimal):
-            return int(o)
-        if isinstance(o, datetime.datetime):
-            return str(o)
-        return super().default(o)
 
 # Run
 if  __name__== "__main__":

inexConnect.py

@@ -39,5 +39,35 @@ def databaseQuery(self, cursor, query, args=()):
     cur.connection.close()
     if self.useLog:
         self.il.debug(f"Database connection closed")
-    # return (r[0] if r else None) if one else r
     return r
+
+class fortraEFC:
+    def getToken(self):
+        self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
+                                                                              "client_id": self.platformConfig["client_id"],\
+                                                                              "client_secret": self.platformConfig["secret"],})
+    def writeToken(self):
+        fortraEFC.getToken(self)
+        with open(self.tokenFilepath, "w") as f:
+            self.j.dump(self.tokenData.json(), f, indent = 2)
+
+    def readToken(self):
+        if self.os.path.exists(self.tokenFilepath):
+            with open(self.tokenFilepath, 'rb') as t:
+                self.tokenData = self.j.load(t)
+                # print(self.tokenData["access_token"])
+        else:
+            fortraEFC.writeToken(self)
+
+    def pushPayload(self):
+        fortraEFC.readToken(self)
+        try:
+            url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
+            pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\
+                                               json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e))
+            return pushPayloadResponse.status_code
+        except self.r.exceptions.HTTPError as errh:
+            print ("Http Error:",errh)
+            if "401" in errh:
+                fortraEFC.writeToken(self)
+                fortraEFC.pushPayload(self)

inexDataProcessing.py

@@ -1,4 +1,4 @@
-def processData(data, template):
+def processData(data, template, **kwargs):
     processedData = []
     for row in data:
         # print(f'Row: {row}')
@@ -11,10 +11,10 @@ def processData(data, template):
                             file_name=row.get('FileName'),\
                             guid=row.get('TransactionGUID'),\
                             ref_id=row.get('ProtocolCommandID'),\
-                            prd_instance_id=row.get(''),\
+                            prd_instance_id=kwargs.get('prd_instance_id'),\
-                            product_guid=row.get(''),\
+                            product_guid=kwargs.get('product_guid'),\
-                            product_name=row.get(''),\
+                            product_name=kwargs.get('product_name'),\
-                            product_version=row.get(''),\
+                            product_version=kwargs.get('product_version'),\
                             node_name=row.get('NodeName'),\
                             src_endpoint_port=row.get('RemotePort'),\
                             src_endpoint_ip=row.get('RemoteIP'),\
@@ -25,8 +25,33 @@ def processData(data, template):
                             bytes_out=row.get('BytesTransferred'),\
                             transfer_time=row.get('TransferTime'),\
                             time=row.get('Time_stamp'),\
-                            user_type=row.get(''),\
+                            user_type=identifyUserType(row.get('user_type')),\
                             user_domain=row.get('SiteName'),\
                             user_name=row.get('Actor'),\
-                            utype=row.get('Command')))
+                            utype=identifyUtype(row.get('TransactionObject'))))
     return processedData
+
+def identifyUserType(obj):
+    if obj:
+        if "Admin" in obj:
+            return "Administrator"
+        else:
+            return "User"
+    else:
+        return None
+def identifyUtype(obj):
+    user_logged_on = []
+    file_deleted = []
+    file_uploaded = []
+    file_downloaded = []
+
+    if obj in user_logged_on:
+        return "user_logged_on"
+    if obj in file_deleted:
+        return "file_deleted"
+    if obj in file_uploaded:
+        return "file_uploaded"
+    if obj in file_downloaded:
+        return "file_downloaded"
+    else:
+        return None

inexEncoder.py

@@ -0,0 +1,11 @@
+import json
+import decimal
+import datetime
+
+class Encoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, decimal.Decimal):
+            return int(o)
+        if isinstance(o, datetime.datetime):
+            return int(o.timestamp() * 1000)
+        return super().default(o)

test.py

@@ -1,5 +1,12 @@
+import datetime
+
 def connectDatabase(driver, server, database, user, password):
     connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
     print(connectionString)
 
-a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
+# a = connectDatabase("ODBC Driver 18 for SQL Server","b","c","d","e")
+
+def converttimestamp(t):
+    print(int(t.timestamp()* 1000))
+
+a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))