Compare commits
No commits in common. "main" and "v.0.0.2" have entirely different histories.
@ -1,28 +1,19 @@
|
|||||||
name: Build
|
name: Build
|
||||||
on:
|
on: push
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- '*'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-linux-binary:
|
# build-linux-binary:
|
||||||
runs-on: ubuntu-latest
|
# runs-on: ubuntu-latest
|
||||||
permissions:
|
# steps:
|
||||||
contents: write # release changes require contents write
|
# - uses: actions/checkout@v4
|
||||||
steps:
|
# - uses: actions/setup-python@v5
|
||||||
- uses: actions/checkout@v4
|
# with:
|
||||||
- uses: actions/setup-python@v5
|
# python-version: '3.12.4'
|
||||||
with:
|
# cache: 'pip' # caching pip dependencies
|
||||||
python-version: '3.12.4'
|
# - run: apt-get update
|
||||||
cache: 'pip' # caching pip dependencies
|
# - run: apt-get install unixodbc -y
|
||||||
- run: lsb_release -a
|
# - run: pip install -r requirements.txt
|
||||||
- run: apt-get update
|
# - run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
|
||||||
- run: apt-get install unixodbc -y
|
|
||||||
- run: pip install -r requirements.txt
|
|
||||||
- run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
|
|
||||||
- uses: softprops/action-gh-release@v2
|
|
||||||
with:
|
|
||||||
files: ${{ gitea.workspace }}/dist/inex
|
|
||||||
|
|
||||||
build-windows-binary:
|
build-windows-binary:
|
||||||
runs-on: windows
|
runs-on: windows
|
||||||
@ -37,7 +28,6 @@ jobs:
|
|||||||
- run: python -m pip install -r requirements.txt
|
- run: python -m pip install -r requirements.txt
|
||||||
- run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
|
- run: pyinstaller --noconfirm --onefile --console ${{ gitea.workspace }}/inex.py
|
||||||
- uses: softprops/action-gh-release@v2
|
- uses: softprops/action-gh-release@v2
|
||||||
|
if: startsWith(gitea.ref, 'refs/tags/')
|
||||||
with:
|
with:
|
||||||
files: |
|
files: ${{ gitea.workspace }}/dist/inex.exe
|
||||||
${{ gitea.workspace }}/dist/inex.exe
|
|
||||||
${{ gitea.workspace }}/config.toml.example
|
|
4
LICENSE
4
LICENSE
@ -209,7 +209,7 @@ If you develop a new program, and you want it to be of the greatest possible use
|
|||||||
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
|
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
|
||||||
|
|
||||||
Inex
|
Inex
|
||||||
Copyright (C) 2024 Jonathan Branan
|
Copyright (C) 2024 jblu
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
||||||
|
|
||||||
@ -221,7 +221,7 @@ Also add information on how to contact you by electronic and paper mail.
|
|||||||
|
|
||||||
If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
|
If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
|
||||||
|
|
||||||
Inex Copyright (C) 2024 Jonathan Branan
|
Inex Copyright (C) 2024 jblu
|
||||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||||
This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
|
This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
|
||||||
|
|
||||||
|
121
README.md
121
README.md
@ -1,93 +1,34 @@
|
|||||||
# Inex
|
# Inex
|
||||||
|
|
||||||
### Information Exchange - Ingest ARM data into EFC
|
Information Exchange - Pull data from EFT ARM then dump it to a json file
|
||||||
This application is designed to pull data from an Microsoft SQL database, reformat it, and then send that data via the web to the Event Fusion Center. It's designed to be performant. The embedded query doesn't return unneccesary data and data is written to memory before being pushed into EFC. Initially you may have to pull a large amount of days so the system is caught up but the sweet spot would be to pull a set of data every day.
|
|
||||||
|
|
||||||
## Build from source
|
# Data Map:
|
||||||
You will need to install git and clone the repository:
|
## tbl_ProtocolCommands
|
||||||
|
- ProtocolCommandID
|
||||||
`git clone https://git.jonb.io/jblu/Inex.git`
|
- Time_stamp
|
||||||
|
- RemoteIP
|
||||||
Alternitively you can download the files from [the releases page](https://git.jonb.io/jblu/Inex/releases).
|
- RemotePort
|
||||||
|
- LocalIP
|
||||||
Then follow OS specific instructions. These are taken directly from the steps used in automation to build the executables found in the releases page.
|
- LocalPort
|
||||||
##### Windows
|
- Protocol
|
||||||
|
- SiteName
|
||||||
You can find the requirements.txt in the repository.
|
- Command
|
||||||
|
- CommandParameters
|
||||||
`python -m pip install -r requirements.txt`
|
- FileName
|
||||||
|
- VirtualFolderName
|
||||||
`pyinstaller --noconfirm --onefile --console path/to/inex.py`
|
- PhysicalFolderName
|
||||||
|
- IsInternal
|
||||||
##### Linux
|
- FileSize
|
||||||
Update Linux Distro. This is Ubuntu specific. Please check the package management tool in your specific version.
|
- TransferTime
|
||||||
|
- BytesTransferred
|
||||||
`apt-get update`
|
- ResultID
|
||||||
|
- TransactionID
|
||||||
You need unixodbc or else pyodbc cannot install.
|
- Description
|
||||||
|
- Actor
|
||||||
`apt-get install unixodbc -y`
|
## tbl_Transactions
|
||||||
|
- TransactionID
|
||||||
You can find the requirements.txt in the repository.
|
- ParentTransactionID
|
||||||
|
- TransactionObject
|
||||||
`pip install -r requirements.txt`
|
- TransactionGUID
|
||||||
|
- NodeName
|
||||||
`pyinstaller --noconfirm --onefile --console path/to/inex.py`
|
- time_stamp
|
||||||
|
|
||||||
## Setup
|
|
||||||
You will need a *config.toml* file in the same directory where *inex.py* or inex executable is. It's recommended to rename the existing config.toml.example file to save sometime and avoid omitting any data.
|
|
||||||
|
|
||||||
> If the application doesn't find the *config.toml* at the time of execution, it will not continue.
|
|
||||||
|
|
||||||
#### Config.toml
|
|
||||||
| Table | Purpose |
|
|
||||||
|-|-|
|
|
||||||
| fortraPlatform | Fortra Specific data |
|
|
||||||
| database | MSSQL Configuration |
|
|
||||||
| immutables | Data that must be included but is not provided by the source database |
|
|
||||||
| output | If and where to write files|
|
|
||||||
| logging | Set if, where and level of logging |
|
|
||||||
|
|
||||||
The following settings are not obvious as to how they affect the application.
|
|
||||||
> Note the #comments
|
|
||||||
```
|
|
||||||
[fortraPlatform]
|
|
||||||
selectedPlatform = "dev" # This will modify which environment the data is pushed to. The tenant_id and secret must be manually modified.
|
|
||||||
|
|
||||||
[database]
|
|
||||||
overrideEmbeddedquery = true # Choose if embedded query should be overridden.
|
|
||||||
daysTopull = 30 # This setting is only related to the embedded query. Please note this will not affect query provided in config.toml
|
|
||||||
driver = "ODBC Driver 18 for SQL Server" # Select which windows driver should be used. This one is recommended.
|
|
||||||
|
|
||||||
[output]
|
|
||||||
pushToplatform = false # if true, send data to fortraPlatform setting to false is useful in dry-runs.
|
|
||||||
dumpTojson = true # if true, write data to json file
|
|
||||||
token = "./.token" # Recommended to leave as-is
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
### Windows
|
|
||||||
##### Run inex.py
|
|
||||||
1. Download [source](https://git.jonb.io/jblu/Inex/releases)
|
|
||||||
2. `cd C:\path\to\Inex`
|
|
||||||
3. `python inex.py`
|
|
||||||
|
|
||||||
OR
|
|
||||||
|
|
||||||
##### Run inex.exe
|
|
||||||
1. Download [source](https://git.jonb.io/jblu/Inex/releases)
|
|
||||||
2. `cd C:\path\to\Inex.exe`
|
|
||||||
3. CMD `.\inex.exe`
|
|
||||||
|
|
||||||
### Linux
|
|
||||||
##### Run inex.py
|
|
||||||
1. Download [source](https://git.jonb.io/jblu/Inex/releases)
|
|
||||||
2. `cd /path/to/Inex`
|
|
||||||
3. `python ./inex.py`
|
|
||||||
|
|
||||||
OR
|
|
||||||
1. Download [source](https://git.jonb.io/jblu/Inex/releases)
|
|
||||||
2. `cd /path/to/Inex`
|
|
||||||
3. RUN `./inex`
|
|
@ -1,33 +1,6 @@
|
|||||||
[fortraPlatform]
|
[fortraPlatform]
|
||||||
selectedPlatform = "dev"
|
selectedPlatform = "dev"
|
||||||
|
|
||||||
[output]
|
|
||||||
pushToplatform = false
|
|
||||||
dumpTojson = true
|
|
||||||
filename ="./data.json"
|
|
||||||
token = "./.token"
|
|
||||||
|
|
||||||
[logging]
|
|
||||||
useLog = true
|
|
||||||
logLevel = "debug"
|
|
||||||
logPath = "./inex.log"
|
|
||||||
|
|
||||||
[database]
|
|
||||||
overrideEmbeddedquery = false
|
|
||||||
daysTopull = 20
|
|
||||||
driver = "ODBC Driver 18 for SQL Server"
|
|
||||||
server = "192.168.x.x"
|
|
||||||
database = "EFTDB"
|
|
||||||
user = "a"
|
|
||||||
password = "a"
|
|
||||||
query = """DECLARE @stopTime DATETIME2
|
|
||||||
SET @stopTime=DATEADD(DAY, -30, GETDATE())
|
|
||||||
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
|
|
||||||
FROM tbl_Transactions t
|
|
||||||
Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
|
|
||||||
Full JOIN tbl_Authentications a ON(t.TransactionID=a.TransactionID)
|
|
||||||
WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL"""
|
|
||||||
|
|
||||||
[fortraPlatform.dev]
|
[fortraPlatform.dev]
|
||||||
idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
|
idp = "https://foundation.foundation-dev.cloudops.fortradev.com/idp/realms/products/protocol/openid-connect/token"
|
||||||
efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
|
efc_url = "https://efc.efc-dev.cloudops.fortradev.com"
|
||||||
@ -49,6 +22,56 @@ tenant_id = ""
|
|||||||
client_id = "eft-event-generator-confidential"
|
client_id = "eft-event-generator-confidential"
|
||||||
secret = ""
|
secret = ""
|
||||||
|
|
||||||
|
[database]
|
||||||
|
driver = "ODBC Driver 18 for SQL Server"
|
||||||
|
server = "192.168.x.x"
|
||||||
|
database = "EFTDB"
|
||||||
|
user = "a"
|
||||||
|
password = "a"
|
||||||
|
query = """DECLARE @stopTime DATETIME2
|
||||||
|
SET @stopTime = DATEADD(DAY, -30, GETDATE())
|
||||||
|
SELECT p.[ProtocolCommandID]
|
||||||
|
,t.[Time_stamp]
|
||||||
|
,p.[RemoteIP]
|
||||||
|
,p.[RemotePort]
|
||||||
|
,p.[LocalIP]
|
||||||
|
,p.[LocalPort]
|
||||||
|
,p.[Protocol]
|
||||||
|
,p.[SiteName]
|
||||||
|
,p.[Command]
|
||||||
|
,p.[CommandParameters]
|
||||||
|
,p.[FileName]
|
||||||
|
,p.[VirtualFolderName]
|
||||||
|
,p.[PhysicalFolderName]
|
||||||
|
,p.[IsInternal]
|
||||||
|
,p.[FileSize]
|
||||||
|
,p.[TransferTime]
|
||||||
|
,p.[BytesTransferred]
|
||||||
|
,p.[ResultID]
|
||||||
|
,t.[TransactionID]
|
||||||
|
,p.[Description]
|
||||||
|
,p.[Actor]
|
||||||
|
,t.ParentTransactionID
|
||||||
|
,t.TransactionObject
|
||||||
|
,t.NodeName
|
||||||
|
,t.TransactionGUID
|
||||||
|
,a.Protocol user_type
|
||||||
|
FROM [EFTDB].[dbo].[tbl_Transactions] t Full JOIN tbl_ProtocolCommands p ON (t.TransactionID = p.TransactionID) Full join tbl_Authentications a ON (t.TransactionID = a.TransactionID)
|
||||||
|
WHERE p.Time_stamp > @stopTime"""
|
||||||
|
|
||||||
[immutables]
|
[immutables]
|
||||||
product_name = "GlobalScape EFT"
|
prd_instance_id = 1
|
||||||
prd_ext_tenant_name = "GlobalScape EFT"
|
product_guid = "asdf"
|
||||||
|
product_name = "EFT"
|
||||||
|
product_version ="8.1.0.9"
|
||||||
|
|
||||||
|
[output]
|
||||||
|
pushToplatform = true
|
||||||
|
dumpTojson = true
|
||||||
|
filename ="./data.json"
|
||||||
|
token = "./.token"
|
||||||
|
|
||||||
|
[logging]
|
||||||
|
use_log = true
|
||||||
|
logLevel = "debug"
|
||||||
|
logPath = "./inex.log"
|
35
executable/config.toml.example
Normal file
35
executable/config.toml.example
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
[database]
|
||||||
|
driver = "ODBC Driver 18 for SQL Server"
|
||||||
|
server = "192.168.x.x"
|
||||||
|
database = "EFTDB"
|
||||||
|
user = "a"
|
||||||
|
password = "a"
|
||||||
|
query = """SELECT TOP (1) [ProtocolCommandID]
|
||||||
|
,[Time_stamp]
|
||||||
|
,[RemoteIP]
|
||||||
|
,[RemotePort]
|
||||||
|
,[LocalIP]
|
||||||
|
,[LocalPort]
|
||||||
|
,[Protocol]
|
||||||
|
,[SiteName]
|
||||||
|
,[Command]
|
||||||
|
,[CommandParameters]
|
||||||
|
,[FileName]
|
||||||
|
,[VirtualFolderName]
|
||||||
|
,[PhysicalFolderName]
|
||||||
|
,[IsInternal]
|
||||||
|
,[FileSize]
|
||||||
|
,[TransferTime]
|
||||||
|
,[BytesTransferred]
|
||||||
|
,[ResultID]
|
||||||
|
,[TransactionID]
|
||||||
|
,[Description]
|
||||||
|
,[Actor]
|
||||||
|
FROM [EFTDB].[dbo].[tbl_ProtocolCommands]"""
|
||||||
|
[output]
|
||||||
|
filename ="./data.json"
|
||||||
|
|
||||||
|
[logging]
|
||||||
|
use_log = true
|
||||||
|
logLevel = "debug"
|
||||||
|
logPath = "./inex.log"
|
BIN
executable/inex.exe
Normal file
BIN
executable/inex.exe
Normal file
Binary file not shown.
71
inex.py
71
inex.py
@ -1,24 +1,23 @@
|
|||||||
import pyodbc
|
import pyodbc
|
||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
|
import datetime
|
||||||
import tomllib
|
import tomllib
|
||||||
from inexLogging import inexLog
|
from inexLogging import inexLog
|
||||||
import inexConnect
|
import inexConnect
|
||||||
from inexDataModel import dataTemplate
|
from inexDataModel import dataTemplate
|
||||||
from inexDataProcessing import processData
|
from inexDataProcessing import processData
|
||||||
import json
|
import json
|
||||||
|
import decimal
|
||||||
import requests
|
import requests
|
||||||
import inexEncoder
|
import inexEncoder
|
||||||
import inexSqlquery
|
|
||||||
|
|
||||||
class Inex:
|
class Inex:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
"""Initilize config, calls functions from inexConnect.py, inexLogging.py
|
"""Initilize config, calls functions from inex-connect.py and inex-logging.py"""
|
||||||
inexDataModel.py, inexDataProcessing.py, inexEncoder.py and inexSqlquery.py
|
|
||||||
Main logic of the program. Requires a config.toml in the same directory it's
|
|
||||||
being run from."""
|
|
||||||
# assign libraries
|
# assign libraries
|
||||||
self.db = pyodbc
|
self.db = pyodbc
|
||||||
|
self.tm = datetime
|
||||||
self.il = logging
|
self.il = logging
|
||||||
self.ic = inexConnect
|
self.ic = inexConnect
|
||||||
self.r = requests
|
self.r = requests
|
||||||
@ -26,41 +25,31 @@ class Inex:
|
|||||||
self.os = os
|
self.os = os
|
||||||
self.j = json
|
self.j = json
|
||||||
self.e = inexEncoder.Encoder
|
self.e = inexEncoder.Encoder
|
||||||
self.sq = inexSqlquery
|
|
||||||
|
|
||||||
# Check if local config file exists.
|
|
||||||
if self.os.path.exists('./config.toml'):
|
if self.os.path.exists('./config.toml'):
|
||||||
config_file_path = './config.toml'
|
config_file_path = './config.toml'
|
||||||
with open(config_file_path, 'rb') as c:
|
with open(config_file_path, 'rb') as c:
|
||||||
self.config = self.tl.load(c)
|
self.config = self.tl.load(c)
|
||||||
|
|
||||||
# set config
|
# set config
|
||||||
try:
|
self.dbDriver = self.config["database"]["driver"]
|
||||||
if self.config:
|
self.dbServer = self.config["database"]["server"]
|
||||||
self.dbDriver = self.config["database"]["driver"]
|
self.dbDatabase = self.config["database"]["database"]
|
||||||
self.dbServer = self.config["database"]["server"]
|
self.dbUser = self.config["database"]["user"]
|
||||||
self.dbDatabase = self.config["database"]["database"]
|
self.dbPassword = self.config["database"]["password"]
|
||||||
self.dbUser = self.config["database"]["user"]
|
self.dbQuery = self.config["database"]["query"]
|
||||||
self.dbPassword = self.config["database"]["password"]
|
self.outputFile = self.config["output"]["filename"]
|
||||||
self.dbQuery = self.config["database"]["query"]
|
self.useLog = self.config["logging"]["useLog"]
|
||||||
self.outputFile = self.config["output"]["filename"]
|
self.logPath = self.config["logging"]["logPath"]
|
||||||
self.useLog = self.config["logging"]["useLog"]
|
self.logLevel = self.config["logging"]["logLevel"]
|
||||||
self.logPath = self.config["logging"]["logPath"]
|
self.prdInstanceID = self.config["immutables"]["prd_instance_id"]
|
||||||
self.logLevel = self.config["logging"]["logLevel"]
|
self.productGUID = self.config["immutables"]["product_guid"]
|
||||||
self.prdExttenantname = self.config["immutables"]["prd_ext_tenant_name"]
|
self.productName = self.config["immutables"]["product_name"]
|
||||||
self.productName = self.config["immutables"]["product_name"]
|
self.productVersion = self.config["immutables"]["product_version"]
|
||||||
self.tokenFilepath = self.config["output"]["token"]
|
self.tokenFilepath = self.config["output"]["token"]
|
||||||
self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
|
self.selectedPlatform = self.config["fortraPlatform"]["selectedPlatform"]
|
||||||
self.writeJsonfile = self.config["output"]["dumpTojson"]
|
self.writeJsonfile = self.config["output"]["dumpTojson"]
|
||||||
self.pushToplatform = self.config["output"]["pushToplatform"]
|
self.pushToplatform = self.config["output"]["pushToplatform"]
|
||||||
self.queryOverride = self.config["database"]["overrideEmbeddedquery"]
|
|
||||||
self.queryDaystopull = self.config["database"]["daysTopull"]
|
|
||||||
except Exception as e:
|
|
||||||
print("No config.toml or possibly missing settings in the file. Please use config.toml.example file and configure appropriately")
|
|
||||||
self.il.error(e)
|
|
||||||
print(e)
|
|
||||||
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
if "dev" in self.selectedPlatform.lower():
|
if "dev" in self.selectedPlatform.lower():
|
||||||
self.platformConfig = self.config["fortraPlatform"]["dev"]
|
self.platformConfig = self.config["fortraPlatform"]["dev"]
|
||||||
@ -68,27 +57,25 @@ class Inex:
|
|||||||
self.platformConfig = self.config["fortraPlatform"]["stage"]
|
self.platformConfig = self.config["fortraPlatform"]["stage"]
|
||||||
if "prod" in self.selectedPlatform.lower():
|
if "prod" in self.selectedPlatform.lower():
|
||||||
self.platformConfig = self.config["fortraPlatform"]["prod"]
|
self.platformConfig = self.config["fortraPlatform"]["prod"]
|
||||||
|
# print(self.platformConfig)
|
||||||
|
|
||||||
#Setup logging
|
#Setup logging
|
||||||
inexLog(self)
|
inexLog(self)
|
||||||
|
|
||||||
# create the connection to the database
|
# create the connection to the database
|
||||||
self.cursor = self.ic.inexSql.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
|
self.cursor = self.ic.connectDatabase(self, self.db, self.dbDriver, self.dbServer, self.dbDatabase, self.dbUser, self.dbPassword)
|
||||||
|
|
||||||
# Query the database
|
self.data = self.ic.databaseQuery(self, self.cursor, self.dbQuery)
|
||||||
self.data = self.ic.inexSql.databaseQuery(self, self.cursor, self.sq.sqlQuerymodel.queryData(self.queryOverride,self.dbQuery, self.queryDaystopull))
|
|
||||||
|
|
||||||
# Modify the data to meet EFC requirements
|
self.modifiedData = processData(self.data, dataTemplate, prd_instance_id=self.prdInstanceID,\
|
||||||
self.modifiedData = processData(self.data, dataTemplate, prd_ext_tenant_name=self.prdExttenantname,product_name=self.productName,\
|
product_guid=self.productGUID,product_name=self.productName,product_version=self.productVersion)
|
||||||
prd_ext_tenant_id=self.platformConfig["tenant_id"])
|
|
||||||
|
|
||||||
# Push data to EFC. Check for local Auth token -> Authenticate if needed -> push data
|
|
||||||
if self.pushToplatform:
|
if self.pushToplatform:
|
||||||
inexConnect.fortraEFC.__init__(self)
|
inexConnect.fortraEFC.pushPayload(self)
|
||||||
|
|
||||||
|
# TODO: move this to its own function
|
||||||
if self.useLog:
|
if self.useLog:
|
||||||
self.il.warning(f"Writing to '{self.outputFile}'.")
|
self.il.warning(f"Writing to '{self.outputFile}'.")
|
||||||
# Write data to json
|
|
||||||
if self.writeJsonfile:
|
if self.writeJsonfile:
|
||||||
with open(self.outputFile, "w") as f:
|
with open(self.outputFile, "w") as f:
|
||||||
self.j.dump(self.modifiedData, f, indent = 2, cls=self.e)
|
self.j.dump(self.modifiedData, f, indent = 2, cls=self.e)
|
||||||
|
138
inexConnect.py
138
inexConnect.py
@ -1,97 +1,73 @@
|
|||||||
class inexSql:
|
def connectDatabase(self, lib, driver, server, database, user, password):
|
||||||
def connectDatabase(self, lib, driver, server, database, user, password):
|
"""Connects to the database. Requires a windows driver to do so.
|
||||||
"""Connects to the database. Requires a windows driver to do so.
|
Typically there is one installed by default"""
|
||||||
Typically there is one installed by default"""
|
|
||||||
|
|
||||||
connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
|
connectionString = f'DRIVER={{{driver}}};SERVER={server};DATABASE={database};UID={user};PWD={password};TrustServerCertificate=yes'
|
||||||
|
if self.useLog:
|
||||||
|
self.il.debug(f"Connection String: connectionString")
|
||||||
|
self.il.info(f"Connecting to {database}@{server} with driver[{driver}].")
|
||||||
|
try:
|
||||||
|
connection = lib.connect(connectionString)
|
||||||
|
except lib.Error as ex:
|
||||||
|
sqlstate = ex.args[1]
|
||||||
if self.useLog:
|
if self.useLog:
|
||||||
self.il.debug(f"Connection String: connectionString")
|
self.il.error(sqlstate)
|
||||||
self.il.info(f"Connecting to {database}@{server} with driver[{driver}].")
|
if self.useLog:
|
||||||
try:
|
self.il.debug(f"Connected.")
|
||||||
connection = lib.connect(connectionString)
|
cursor = connection.cursor()
|
||||||
except lib.Error as ex:
|
|
||||||
sqlstate = ex.args[1]
|
return cursor
|
||||||
if self.useLog:
|
|
||||||
self.il.error(sqlstate)
|
def databaseQuery(self, cursor, query, args=()):
|
||||||
|
if self.useLog:
|
||||||
|
self.il.debug(f"Query:")
|
||||||
|
self.il.debug(query)
|
||||||
|
self.il.info(f"Sending query:{query[0:20]}...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
cur = cursor.execute(query, args)
|
||||||
|
except cur.Error as ex:
|
||||||
|
sqlstate = ex.args[1]
|
||||||
if self.useLog:
|
if self.useLog:
|
||||||
self.il.debug(f"Connected.")
|
self.il.error(sqlstate)
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
return cursor
|
if self.useLog:
|
||||||
|
self.il.debug(f"Processing database response...")
|
||||||
|
r = [dict((cur.description[i][0], value) \
|
||||||
|
for i, value in enumerate(row)) for row in cur.fetchall()]
|
||||||
|
|
||||||
def databaseQuery(self, cursor, query, args=()):
|
cur.connection.close()
|
||||||
"""Use the database connection to send a query."""
|
if self.useLog:
|
||||||
if self.useLog:
|
self.il.debug(f"Database connection closed")
|
||||||
self.il.debug(f"Query:")
|
return r
|
||||||
self.il.debug(query)
|
|
||||||
self.il.info(f"Sending query:{query[0:20]}...")
|
|
||||||
|
|
||||||
try:
|
|
||||||
cur = cursor.execute(query, args)
|
|
||||||
except cur.Error as ex:
|
|
||||||
sqlstate = ex.args[1]
|
|
||||||
if self.useLog:
|
|
||||||
self.il.error(sqlstate)
|
|
||||||
|
|
||||||
if self.useLog:
|
|
||||||
self.il.debug(f"Processing database response...")
|
|
||||||
r = [dict((cur.description[i][0], value) \
|
|
||||||
for i, value in enumerate(row)) for row in cur.fetchall()]
|
|
||||||
|
|
||||||
cur.connection.close()
|
|
||||||
if self.useLog:
|
|
||||||
self.il.debug(f"Database connection closed")
|
|
||||||
return r
|
|
||||||
|
|
||||||
class fortraEFC:
|
class fortraEFC:
|
||||||
"""Class to connect to fortra EFC. It will authenticate and push rest payloads.
|
|
||||||
Writes a .token file to the same directory script was run in."""
|
|
||||||
def __init__(self):
|
|
||||||
"""This is the logic for how authentication is handled"""
|
|
||||||
# Check if .token file is present
|
|
||||||
if fortraEFC.readToken(self) == 1:
|
|
||||||
# Get fresh token. First run.
|
|
||||||
fortraEFC.getToken(self)
|
|
||||||
fortraEFC.writeToken(self)
|
|
||||||
# Push data with token
|
|
||||||
self.pushPayloadresponse = fortraEFC.pushPayload(self)
|
|
||||||
if self.pushPayloadresponse == 401:
|
|
||||||
fortraEFC.getToken(self)
|
|
||||||
fortraEFC.writeToken(self)
|
|
||||||
fortraEFC.pushPayload(self)
|
|
||||||
|
|
||||||
def readToken(self):
|
|
||||||
"""Looks locally for a .token file. Returns a numeral code
|
|
||||||
for logic in the init method."""
|
|
||||||
if self.os.path.exists(self.tokenFilepath):
|
|
||||||
with open(self.tokenFilepath, 'rb') as t:
|
|
||||||
self.tokenData = self.j.load(t)
|
|
||||||
self.il.debug(f'readToken {self.tokenData["access_token"]}')
|
|
||||||
return 0
|
|
||||||
else:
|
|
||||||
return 1
|
|
||||||
|
|
||||||
def getToken(self):
|
def getToken(self):
|
||||||
"""Gets a token from fortra idp."""
|
|
||||||
self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
|
self.tokenData = self.r.post(self.platformConfig["idp"], data={"grant_type":"client_credentials",\
|
||||||
"client_id": self.platformConfig["client_id"],\
|
"client_id": self.platformConfig["client_id"],\
|
||||||
"client_secret": self.platformConfig["secret"],})
|
"client_secret": self.platformConfig["secret"],})
|
||||||
self.tokenData = self.tokenData.json()
|
|
||||||
self.il.debug(f'getToken {self.tokenData["access_token"]}')
|
|
||||||
|
|
||||||
def writeToken(self):
|
def writeToken(self):
|
||||||
"""Writes a token to a local file named '.token'."""
|
|
||||||
fortraEFC.getToken(self)
|
fortraEFC.getToken(self)
|
||||||
with open(self.tokenFilepath, "w") as f:
|
with open(self.tokenFilepath, "w") as f:
|
||||||
self.j.dump(self.tokenData, f, indent = 2)
|
self.j.dump(self.tokenData.json(), f, indent = 2)
|
||||||
self.il.debug(f'writeToken {self.tokenData["access_token"]}')
|
|
||||||
|
def readToken(self):
|
||||||
|
if self.os.path.exists(self.tokenFilepath):
|
||||||
|
with open(self.tokenFilepath, 'rb') as t:
|
||||||
|
self.tokenData = self.j.load(t)
|
||||||
|
# print(self.tokenData["access_token"])
|
||||||
|
else:
|
||||||
|
fortraEFC.writeToken(self)
|
||||||
|
|
||||||
def pushPayload(self):
|
def pushPayload(self):
|
||||||
"""Sends data to fortra EFC. Requires a token from the idp."""
|
fortraEFC.readToken(self)
|
||||||
self.il.debug(f'pushPayload {self.tokenData["access_token"]}')
|
try:
|
||||||
url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
|
url = f'{self.platformConfig["efc_url"]}/api/v1/unity/data/{self.platformConfig["tenant_id"]}/machine_event'
|
||||||
pushPayloadResponse = self.r.post(url, headers={'Authorization': f'Bearer {self.tokenData["access_token"]}'},\
|
pushPayloadResponse = self.r.post(url, headers={'Authorization': f'bearer {self.tokenData["access_token"]}'},\
|
||||||
data=self.j.dumps(self.modifiedData, cls=self.e))
|
json=self.j.dumps(self.modifiedData,indent = 2, cls=self.e))
|
||||||
self.il.debug(pushPayloadResponse.status_code)
|
return pushPayloadResponse.status_code
|
||||||
self.il.debug(pushPayloadResponse.text)
|
except self.r.exceptions.HTTPError as errh:
|
||||||
return pushPayloadResponse.status_code
|
print ("Http Error:",errh)
|
||||||
|
if "401" in errh:
|
||||||
|
fortraEFC.writeToken(self)
|
||||||
|
fortraEFC.pushPayload(self)
|
215
inexDataModel.py
215
inexDataModel.py
@ -1,195 +1,48 @@
|
|||||||
def dataTemplate(transactionType,**kwargs):
|
def dataTemplate(**kwargs):
|
||||||
"""Created templates for use. This function forms json data into an
|
"""Expects the following keyword arguments:
|
||||||
appropriate model for EFC. It returnes the appropriate template based
|
status,status_detail,status_code,file_size,file_path,file_virtual_path,file_name,
|
||||||
on the transaction type passed into the function. The logic to process
|
guid,ref_id,prd_instance_id,product_guid,product_name,product_version,node_name,
|
||||||
this is at the bottom of the function."""
|
src_endpoint_port,src_endpoint_ip,dst_endpoint_port,dst_endpoint_ip,dst_endpoint_type,
|
||||||
upload = {
|
session_uid,bytes_out,transfer_time,time,user_type,user_domain,user_name and utype.
|
||||||
"bytes" : kwargs.get('bytes'),
|
"""
|
||||||
"dst_endpoint": {
|
template ={
|
||||||
"port": kwargs.get('dst_endpoint_port'),
|
"status": kwargs.get('status'),
|
||||||
"ip": kwargs.get('dst_endpoint_ip'),
|
"status_detail": kwargs.get('status_detail'),
|
||||||
"type": kwargs.get('dst_endpoint_type')
|
"status_code": kwargs.get('status_code'),
|
||||||
},
|
"file": {
|
||||||
"duration": kwargs.get('duration'),
|
"size": kwargs.get('file_size'),
|
||||||
"file": {
|
"path": kwargs.get('file_path'),
|
||||||
"created_time": kwargs.get('file_created_time'),
|
"virtual_path": kwargs.get('file_virtual_path'),
|
||||||
"uid": kwargs.get('file_uid'),
|
"name": kwargs.get('file_name')
|
||||||
"size": kwargs.get('file_size'),
|
},
|
||||||
"name": kwargs.get('file_name'),
|
"guid": kwargs.get('guid'),
|
||||||
"path": kwargs.get('file_path')
|
"ref_id": kwargs.get('ref_id'),
|
||||||
},
|
"prd_instance_id": kwargs.get('prd_instance_id'),
|
||||||
"guid": kwargs.get('file_uid'),
|
"product_guid": kwargs.get('product_guid'),
|
||||||
"node_name": kwargs.get('node_name'),
|
"product_name": kwargs.get('product_name'),
|
||||||
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
|
"product_version": kwargs.get('product_version'),
|
||||||
"product_name": kwargs.get('product_name'),
|
"node_name":kwargs.get('node_name'),
|
||||||
"prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
|
"src_endpoint": {
|
||||||
"classifications": [{
|
"port": kwargs.get('src_endpoint_port'),
|
||||||
"ref_id": f"globalscape:{kwargs.get('guid')}",
|
"ip": kwargs.get('src_endpoint_ip')
|
||||||
"time": kwargs.get('time'),
|
},
|
||||||
}],
|
"dst_endpoint": {
|
||||||
"session": {
|
|
||||||
"created_time": kwargs.get('time'),
|
|
||||||
"uid": kwargs.get('session_uid')
|
|
||||||
},
|
|
||||||
"src_endpoint": {
|
|
||||||
"port": kwargs.get('src_endpoint_port'),
|
|
||||||
"ip": kwargs.get('src_endpoint_ip'),
|
|
||||||
"type": kwargs.get('src_endpoint_type')
|
|
||||||
},
|
|
||||||
"tenant": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"tenant_name":"GlobalScape",
|
|
||||||
"time": kwargs.get('time'),
|
|
||||||
"status_code": kwargs.get('status_code'),
|
|
||||||
"status_detail": kwargs.get('status_detail'),
|
|
||||||
"user": {
|
|
||||||
"home_directory": kwargs.get('user_home_directory'),
|
|
||||||
"uuid": kwargs.get('guid'),
|
|
||||||
"uid": kwargs.get('user_uid'),
|
|
||||||
"type": kwargs.get('user_type'),
|
|
||||||
"name": kwargs.get('user_name')
|
|
||||||
},
|
|
||||||
"utype": kwargs.get('utype')
|
|
||||||
}
|
|
||||||
download = {
|
|
||||||
"bytes" : kwargs.get('bytes'),
|
|
||||||
"dst_endpoint": {
|
|
||||||
"port": kwargs.get('dst_endpoint_port'),
|
|
||||||
"ip": kwargs.get('dst_endpoint_ip'),
|
|
||||||
"type": kwargs.get('dst_endpoint_type')
|
|
||||||
},
|
|
||||||
"duration": kwargs.get('duration'),
|
|
||||||
"file": {
|
|
||||||
"uid": kwargs.get('file_uid'),
|
|
||||||
"size": kwargs.get('file_size'),
|
|
||||||
"name": kwargs.get('file_name'),
|
|
||||||
"path": kwargs.get('file_path')
|
|
||||||
},
|
|
||||||
"guid": kwargs.get('file_uid'),
|
|
||||||
"node_name": kwargs.get('node_name'),
|
|
||||||
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"product_name": kwargs.get('product_name'),
|
|
||||||
"prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
|
|
||||||
"classifications": [{
|
|
||||||
"ref_id": f"globalscape:{kwargs.get('guid')}",
|
|
||||||
"time": kwargs.get('time'),
|
|
||||||
}],
|
|
||||||
"session": {
|
|
||||||
"created_time": kwargs.get('time'),
|
|
||||||
"uid": kwargs.get('session_uid')
|
|
||||||
},
|
|
||||||
"src_endpoint": {
|
|
||||||
"port": kwargs.get('src_endpoint_port'),
|
|
||||||
"ip": kwargs.get('src_endpoint_ip'),
|
|
||||||
"type": kwargs.get('src_endpoint_type')
|
|
||||||
},
|
|
||||||
"tenant": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"tenant_name":"GlobalScape",
|
|
||||||
"time": kwargs.get('time'),
|
|
||||||
"status_code": kwargs.get('status_code'),
|
|
||||||
"status_detail": kwargs.get('status_detail'),
|
|
||||||
"user": {
|
|
||||||
"home_directory": kwargs.get('user_home_directory'),
|
|
||||||
"uuid": kwargs.get('guid'),
|
|
||||||
"uid": kwargs.get('user_uid'),
|
|
||||||
"type": kwargs.get('user_type'),
|
|
||||||
"name": kwargs.get('user_name')
|
|
||||||
},
|
|
||||||
"utype": kwargs.get('utype')
|
|
||||||
}
|
|
||||||
|
|
||||||
fileDeleted = {
|
|
||||||
"file": {
|
|
||||||
"size": kwargs.get('file_size'),
|
|
||||||
"name": kwargs.get('file_name'),
|
|
||||||
"path": kwargs.get('file_path'),
|
|
||||||
"uid": kwargs.get('file_uid'),
|
|
||||||
},
|
|
||||||
"guid": f'deleted:{kwargs.get("guid")}',
|
|
||||||
"node_name": kwargs.get('node_name'),
|
|
||||||
"classifications": [{
|
|
||||||
"ref_id": f"globalscape:{kwargs.get('guid')}",
|
|
||||||
"time": kwargs.get('time'),
|
|
||||||
}],
|
|
||||||
"prd_ext_tenant_name": kwargs.get("prd_ext_tenant_name"),
|
|
||||||
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"product_name": kwargs.get("product_name"),
|
|
||||||
"session": {
|
|
||||||
"created_time": kwargs.get('time'),
|
|
||||||
"uid": kwargs.get('session_uid')
|
|
||||||
},
|
|
||||||
"src_endpoint": {
|
|
||||||
"port": kwargs.get('src_endpoint_port'),
|
|
||||||
"ip": kwargs.get('src_endpoint_ip'),
|
|
||||||
"type": kwargs.get('src_endpoint_type')
|
|
||||||
},
|
|
||||||
"tenant": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"tenant_name":"GlobalScape",
|
|
||||||
"dst_endpoint": {
|
|
||||||
"port": kwargs.get('dst_endpoint_port'),
|
|
||||||
"ip": kwargs.get('dst_endpoint_ip'),
|
|
||||||
"type": kwargs.get('dst_endpoint_type')
|
|
||||||
},
|
|
||||||
"time": kwargs.get('time'),
|
|
||||||
"status_code": kwargs.get('status_code'),
|
|
||||||
"status_detail": kwargs.get('status_detail'),
|
|
||||||
"user": {
|
|
||||||
"home_directory": kwargs.get('user_home_directory'),
|
|
||||||
"uuid": kwargs.get('user_session_uid'),
|
|
||||||
"uid": kwargs.get('user_uid'),
|
|
||||||
"type": kwargs.get('user_type'),
|
|
||||||
"name": kwargs.get('user_name')
|
|
||||||
},
|
|
||||||
"utype": kwargs.get('utype')
|
|
||||||
}
|
|
||||||
|
|
||||||
logon ={
|
|
||||||
"classifications": [{
|
|
||||||
"ref_id": f"globalscape:{kwargs.get('guid')}",
|
|
||||||
"time": kwargs.get('time'),
|
|
||||||
}],
|
|
||||||
"dst_endpoint": {
|
|
||||||
"port": kwargs.get('dst_endpoint_port'),
|
"port": kwargs.get('dst_endpoint_port'),
|
||||||
"ip": kwargs.get('dst_endpoint_ip'),
|
"ip": kwargs.get('dst_endpoint_ip'),
|
||||||
"type": kwargs.get('dst_endpoint_type')
|
"type": kwargs.get('dst_endpoint_type')
|
||||||
},
|
},
|
||||||
"guid": kwargs.get('guid'),
|
"session": {
|
||||||
"node_name": kwargs.get('node_name'),
|
"uid": kwargs.get('session_uid')
|
||||||
"tenant": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"tenant_name":"GlobalScape",
|
|
||||||
"prd_ext_tenant_id": kwargs.get('prd_ext_tenant_id'),
|
|
||||||
"product_name": kwargs.get("product_name"),
|
|
||||||
"prd_ext_tenant_name": kwargs.get('prd_ext_tenant_name'),
|
|
||||||
"status_code": kwargs.get('status_code'),
|
|
||||||
"status_detail": kwargs.get('status_detail'),
|
|
||||||
"src_endpoint": {
|
|
||||||
"port": kwargs.get('src_endpoint_port'),
|
|
||||||
"ip": kwargs.get('src_endpoint_ip'),
|
|
||||||
"type": kwargs.get('src_endpoint_type')
|
|
||||||
},
|
},
|
||||||
|
"bytes_out" : kwargs.get('bytes_out'),
|
||||||
|
"transfer_time" : kwargs.get('transfer_time'),
|
||||||
"time": kwargs.get('time'),
|
"time": kwargs.get('time'),
|
||||||
"user": {
|
"user": {
|
||||||
"home_directory": kwargs.get('user_home_directory'),
|
|
||||||
"uuid": kwargs.get('user_session_uid'),
|
|
||||||
"uid": kwargs.get('user_uid'),
|
|
||||||
"type": kwargs.get('user_type'),
|
"type": kwargs.get('user_type'),
|
||||||
|
"domain": kwargs.get('user_domain'),
|
||||||
"name": kwargs.get('user_name')
|
"name": kwargs.get('user_name')
|
||||||
},
|
},
|
||||||
"session": {
|
|
||||||
"created_time": kwargs.get('time'),
|
|
||||||
"uid": kwargs.get('session_uid')
|
|
||||||
},
|
|
||||||
"utype": kwargs.get('utype')
|
"utype": kwargs.get('utype')
|
||||||
}
|
}
|
||||||
|
|
||||||
if transactionType == "file_uploaded":
|
|
||||||
template = upload
|
|
||||||
if transactionType == "file_downloaded":
|
|
||||||
template = download
|
|
||||||
if transactionType == "file_deleted":
|
|
||||||
template = fileDeleted
|
|
||||||
if transactionType == "user_logged_on":
|
|
||||||
template = logon
|
|
||||||
if transactionType == "other":
|
|
||||||
template = {}
|
|
||||||
|
|
||||||
return template
|
return template
|
@ -1,99 +1,37 @@
|
|||||||
def processData(data, template, **kwargs):
|
def processData(data, template, **kwargs):
|
||||||
"""Translates data from sql query to the appropriate place in the respective template.
|
|
||||||
Accepts data, which is the sql query output, the template function, and finally
|
|
||||||
additional data to insert into the template. Uses other functions to further
|
|
||||||
process row data."""
|
|
||||||
processedData = []
|
processedData = []
|
||||||
transactionLoginid = []
|
|
||||||
|
|
||||||
for row in data:
|
for row in data:
|
||||||
# print(f'Row: {row}')
|
# print(f'Row: {row}')
|
||||||
# must set variables for the different templates and do logic based on that. Do not call identifyUtype many times
|
processedData.append(template(status=row.get(''),\
|
||||||
identifyUtypecommand = identifyUtype(row.get('Command'))
|
status_detail=row.get(''),\
|
||||||
|
status_code=row.get('ResultID'),\
|
||||||
if identifyUtypecommand == "other":
|
file_size=row.get('FileSize'),\
|
||||||
continue
|
file_path=row.get('PhysicalFolderName'),\
|
||||||
|
file_virtual_path=row.get('VirtualFolderName'),\
|
||||||
if row.get('Command') == None:
|
file_name=row.get('FileName'),\
|
||||||
continue
|
guid=row.get('TransactionGUID'),\
|
||||||
|
ref_id=row.get('ProtocolCommandID'),\
|
||||||
userType = identifyUserType(row.get('user_type'))
|
prd_instance_id=kwargs.get('prd_instance_id'),\
|
||||||
userHome = parseHomefolder(row.get('Actor'),row.get('VirtualFolderName'))
|
product_guid=kwargs.get('product_guid'),\
|
||||||
try:
|
product_name=kwargs.get('product_name'),\
|
||||||
processedData.append(template(identifyUtypecommand,\
|
product_version=kwargs.get('product_version'),\
|
||||||
prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
|
node_name=row.get('NodeName'),\
|
||||||
user_uid=row.get('TransactionID'),\
|
src_endpoint_port=row.get('RemotePort'),\
|
||||||
status_detail=row.get('Description'),\
|
src_endpoint_ip=row.get('RemoteIP'),\
|
||||||
prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
|
dst_endpoint_port=row.get('LocalPort'),\
|
||||||
status_code=row.get('ResultID'),\
|
dst_endpoint_ip=row.get('LocalIP'),\
|
||||||
file_created_time=row.get('Time_stamp'),\
|
dst_endpoint_type=row.get('Protocol'),\
|
||||||
file_size=row.get('FileSize'),\
|
session_uid=row.get('TransactionID'),\
|
||||||
file_uid=row.get('ProtocolCommandID'),\
|
bytes_out=row.get('BytesTransferred'),\
|
||||||
file_path=row.get('PhysicalFolderName'),\
|
transfer_time=row.get('TransferTime'),\
|
||||||
file_name=row.get('FileName'),\
|
time=row.get('Time_stamp'),\
|
||||||
guid=row.get('TransactionGUID'),\
|
user_type=identifyUserType(row.get('user_type')),\
|
||||||
product_name=kwargs.get('product_name'),\
|
user_domain=row.get('SiteName'),\
|
||||||
node_name=row.get('NodeName'),\
|
user_name=row.get('Actor'),\
|
||||||
session_uid=row.get('TransactionID'),\
|
utype=identifyUtype(row.get('TransactionObject'))))
|
||||||
src_endpoint_type=row.get('Protocol'),\
|
|
||||||
src_endpoint_port=row.get('RemotePort'),\
|
|
||||||
src_endpoint_ip=row.get('RemoteIP'),\
|
|
||||||
dst_endpoint_port=row.get('LocalPort'),\
|
|
||||||
dst_endpoint_ip=row.get('LocalIP'),\
|
|
||||||
dst_endpoint_type=row.get('Protocol'),\
|
|
||||||
user_session_uid=row.get('TransactionID'),\
|
|
||||||
bytes=row.get('BytesTransferred'),\
|
|
||||||
time=row.get('Time_stamp'),\
|
|
||||||
duration=row.get('TransferTime'),\
|
|
||||||
user_type=userType,\
|
|
||||||
user_name=row.get('Actor'),\
|
|
||||||
user_home_directory=userHome,\
|
|
||||||
utype=identifyUtypecommand))
|
|
||||||
except UnboundLocalError:
|
|
||||||
print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
|
|
||||||
continue
|
|
||||||
|
|
||||||
identifyUtypetransactionObject = identifyUtype(row.get('TransactionObject'))
|
|
||||||
|
|
||||||
if identifyUtypetransactionObject == "other":
|
|
||||||
continue
|
|
||||||
|
|
||||||
if row.get('TransactionGUID') not in transactionLoginid:
|
|
||||||
try:
|
|
||||||
processedData.append(template(identifyUtypetransactionObject,\
|
|
||||||
prd_ext_tenant_id=kwargs.get('prd_ext_tenant_id'),\
|
|
||||||
prd_ext_tenant_name=kwargs.get('prd_ext_tenant_name'),\
|
|
||||||
status_detail=row.get('Description'),\
|
|
||||||
guid=row.get('TransactionGUID'),\
|
|
||||||
status_code=row.get('ResultID'),\
|
|
||||||
node_name=row.get('NodeName'),\
|
|
||||||
prd_instance_id=kwargs.get('prd_instance_id'),\
|
|
||||||
product_name=kwargs.get('product_name'),\
|
|
||||||
src_endpoint_type=row.get('Protocol'),\
|
|
||||||
src_endpoint_port=row.get('RemotePort'),\
|
|
||||||
src_endpoint_ip=row.get('RemoteIP'),\
|
|
||||||
dst_endpoint_port=row.get('LocalPort'),\
|
|
||||||
dst_endpoint_ip=row.get('LocalIP'),\
|
|
||||||
dst_endpoint_type=row.get('Protocol'),\
|
|
||||||
session_uid=row.get('TransactionID'),\
|
|
||||||
transfer_time=row.get('TransferTime'),\
|
|
||||||
time=row.get('Time_stamp'),\
|
|
||||||
user_session_uid=row.get('TransactionID'),\
|
|
||||||
user_uid=row.get('TransactionID'),\
|
|
||||||
user_type=userType,\
|
|
||||||
user_name=row.get('Actor'),\
|
|
||||||
user_home_directory=userHome,\
|
|
||||||
utype=identifyUtypetransactionObject\
|
|
||||||
))
|
|
||||||
transactionLoginid.append(row.get('TransactionGUID'))
|
|
||||||
except UnboundLocalError:
|
|
||||||
print(f'Problem row GUID:{row.get("TransactionGUID")} ::: TransactionObject:{row.get("TransactionObject")} Command: {row.get("Command")}')
|
|
||||||
continue
|
|
||||||
|
|
||||||
return processedData
|
return processedData
|
||||||
|
|
||||||
def identifyUserType(obj):
|
def identifyUserType(obj):
|
||||||
"""Check string if it has Admin-> return Administrator else return User."""
|
|
||||||
if obj:
|
if obj:
|
||||||
if "Admin" in obj:
|
if "Admin" in obj:
|
||||||
return "Administrator"
|
return "Administrator"
|
||||||
@ -101,36 +39,19 @@ def identifyUserType(obj):
|
|||||||
return "User"
|
return "User"
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def parseHomefolder(user, virtualfolder):
|
|
||||||
"""Extract users home folder using the username. Will not work on edge cases
|
|
||||||
such as when a users home folder does not have the user name. When that occurs
|
|
||||||
it is impossible to know based on the arm data what the home folder is.
|
|
||||||
This function is an assumption so it may return the incorrect home folder.
|
|
||||||
This function finds the user name and takes the path from the left of the folder
|
|
||||||
as the home folder. There are cases where this may not be accurate."""
|
|
||||||
if user:
|
|
||||||
userSplit = f'/{user}/'
|
|
||||||
if virtualfolder:
|
|
||||||
if userSplit in virtualfolder:
|
|
||||||
home = virtualfolder.split(userSplit)[0] + userSplit
|
|
||||||
return home if home else None
|
|
||||||
|
|
||||||
def identifyUtype(obj):
|
def identifyUtype(obj):
|
||||||
"""Process Type of transaction based on string that passed in.
|
user_logged_on = []
|
||||||
Return transaction type."""
|
file_deleted = []
|
||||||
user_logged_on = ['AUTH']
|
file_uploaded = []
|
||||||
file_deleted = ["dele"]
|
file_downloaded = []
|
||||||
file_uploaded = ["created"]
|
|
||||||
file_downloaded = ["sent"]
|
|
||||||
|
|
||||||
if obj in user_logged_on:
|
if obj in user_logged_on:
|
||||||
return "user_logged_on"
|
return "user_logged_on"
|
||||||
elif obj in file_deleted:
|
if obj in file_deleted:
|
||||||
return "file_deleted"
|
return "file_deleted"
|
||||||
elif obj in file_uploaded:
|
if obj in file_uploaded:
|
||||||
return "file_uploaded"
|
return "file_uploaded"
|
||||||
elif obj in file_downloaded:
|
if obj in file_downloaded:
|
||||||
return "file_downloaded"
|
return "file_downloaded"
|
||||||
else:
|
else:
|
||||||
return "other"
|
return None
|
@ -3,8 +3,6 @@ import decimal
|
|||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
class Encoder(json.JSONEncoder):
|
class Encoder(json.JSONEncoder):
|
||||||
"""Encoder uses json.JSONEncoder and checks for instances of decimal and datetime.
|
|
||||||
Changes decimal.Decimal to int and datetime.datetime to unix timestamp with miliseconds."""
|
|
||||||
def default(self, o):
|
def default(self, o):
|
||||||
if isinstance(o, decimal.Decimal):
|
if isinstance(o, decimal.Decimal):
|
||||||
return int(o)
|
return int(o)
|
||||||
|
@ -1,11 +0,0 @@
|
|||||||
class sqlQuerymodel:
|
|
||||||
def queryData(overRideflag, configQuery, daysTopull):
|
|
||||||
"""Embedded query data. Data is slightly modified to change the amount of days to pull."""
|
|
||||||
q ="""DECLARE @stopTime DATETIME2
|
|
||||||
SET @stopTime=DATEADD(DAY, -30, GETDATE())
|
|
||||||
SELECT p.ProtocolCommandID, t.Time_stamp, p.RemoteIP, p.RemotePort, p.LocalIP, p.LocalPort, p.Protocol, p.SiteName, p.Command, p.FileName, p.PhysicalFolderName, p.VirtualFolderName, p.FileSize, p.TransferTime, p.BytesTransferred, p.Description, p.ResultID, t.TransactionID, p.Actor, t.TransactionObject, t.NodeName, t.TransactionGUID, a.Protocol user_type
|
|
||||||
FROM tbl_Transactions t
|
|
||||||
Full JOIN tbl_ProtocolCommands p ON(t.TransactionID=p.TransactionID)
|
|
||||||
Full join tbl_Authentications a ON(t.TransactionID=a.TransactionID)
|
|
||||||
WHERE p.Time_stamp>@stopTime AND p.Command IS NOT NULL""".replace("30", str(daysTopull))
|
|
||||||
return configQuery if overRideflag else q
|
|
38
test.py
38
test.py
@ -9,40 +9,4 @@ def connectDatabase(driver, server, database, user, password):
|
|||||||
def converttimestamp(t):
|
def converttimestamp(t):
|
||||||
print(int(t.timestamp()* 1000))
|
print(int(t.timestamp()* 1000))
|
||||||
|
|
||||||
def builddict(keys,*args,**kwargs):
|
a = converttimestamp(datetime.datetime(2024, 7, 23, 14, 26, 38, 214000))
|
||||||
dict = {}
|
|
||||||
for key in keys:
|
|
||||||
dict[key] = kwargs.get(key)
|
|
||||||
print(dict)
|
|
||||||
|
|
||||||
|
|
||||||
def identifyUtype(obj):
|
|
||||||
"""Process Type of transaction based on string that passed in.
|
|
||||||
Return transaction type."""
|
|
||||||
user_logged_on = ['AUTH']
|
|
||||||
file_deleted = ["dele"]
|
|
||||||
file_uploaded = ["created"]
|
|
||||||
file_downloaded = ["sent"]
|
|
||||||
|
|
||||||
if obj in user_logged_on:
|
|
||||||
return "user_logged_on"
|
|
||||||
if obj in file_deleted:
|
|
||||||
return "file_deleted"
|
|
||||||
if obj in file_uploaded:
|
|
||||||
return "file_uploaded"
|
|
||||||
if obj in file_downloaded:
|
|
||||||
return "file_downloaded"
|
|
||||||
else:
|
|
||||||
return "other"
|
|
||||||
|
|
||||||
|
|
||||||
testfolder = '/Usr/a/asdf/asf/asdfas/asdfasdf/'
|
|
||||||
user = 'a'
|
|
||||||
|
|
||||||
def parsehomefolder(user, virtualfolder):
|
|
||||||
userSplit = f'/{user}/'
|
|
||||||
home = virtualfolder.split(userSplit)[0] + userSplit
|
|
||||||
print(home)
|
|
||||||
return home
|
|
||||||
|
|
||||||
a = parsehomefolder(user, testfolder)
|
|
Loading…
Reference in New Issue
Block a user