diff --git a/filter-plugin/logstash-filter-oua-guardium/Makefile b/filter-plugin/logstash-filter-oua-guardium/Makefile index 7cb2fc6f5..fec00969e 100644 --- a/filter-plugin/logstash-filter-oua-guardium/Makefile +++ b/filter-plugin/logstash-filter-oua-guardium/Makefile @@ -1,5 +1,5 @@ -logstash_dir=/home/marockar/oua_uc/logstash-7.5.2 -java_home_dir=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-2.el8_5.x86_64 +logstash_dir=${HOME}/tools/logstash_8.13.4 +java_home_dir=/Library/Java/JavaVirtualMachines/ibm-jdk-8//Contents/Home/ version=$(shell cat VERSION) all: jar gem diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackage/OracleUnifiedAudit/filter.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackage/OracleUnifiedAudit/filter.conf deleted file mode 100644 index 37660df7a..000000000 --- a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackage/OracleUnifiedAudit/filter.conf +++ /dev/null @@ -1,26 +0,0 @@ -filter { - if [type] == "oua" { - ruby { - code => " - if event.get('message') - event.set('message', event.get('message').gsub('\\\\\\', '\\\\\\\\')) - end - " - } - mutate { - gsub => [ - "command", "\w+/[^\s@]+@", "*****@" - ] - } - json { - source => "message" - } - mutate { - add_field => {"[HostName]" => "%{SERVER_ADDRESS}" } - add_field => {"[PortNumber]" => "%{SERVER_PORT}" } - } - if "_jsonparsefailure" not in [tags] { - oua_filter {} - } - } -} diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverCloudwatch/OuaCloudwatch.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverCloudwatch/OuaCloudwatch.conf new file mode 100644 index 000000000..da2d5b6e8 --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverCloudwatch/OuaCloudwatch.conf @@ -0,0 +1,216 @@ +#/* +#Copyright 2023-24 IBM Inc. - All Rights Reserved. +#SPDX-License-Identifier: Apache-2.0 +#*/ + +input{ + cloudwatch_logs { + #Mandatory arguments: + #Insert the log group that is created for the data instance for example, /aws/rds/instance//postgresql + log_group => [""] + #Insert the access key and secret that has access to log group + access_key_id => "***" + secret_access_key => "***" + region => "us-east-1" #Region that has the DB, Default value: us-east-1 + start_position => "end" + interval => 5 + event_filter => "" + type => "Oua" + add_field => {"account_id" => ""} + #Unmask is an optional parameter. To get unmasked logs, you need to set unmask = true. + #unmask => true + } + } + + +filter { + if [type] == "Oua" { + + xml { + source => "message" + store_xml => false + xpath => [ + "/AuditRecord/Audit_Type/text()", "Audit_Type", + "/AuditRecord/Session_Id/text()", "Session_Id", + "/AuditRecord/StatementId/text()", "StatementId", + "/AuditRecord/EntryId/text()", "EntryId", + "/AuditRecord/Extended_Timestamp/text()", "Extended_Timestamp", #thisisTimeStamp + "/AuditRecord/DB_User/text()", "DB_User", #this is DBUser + "/AuditRecord/OS_User/text()", "OS_User", + "/AuditRecord/Userhost/text()", "Userhost", #this is clienIP or clientHostname + "/AuditRecord/OS_Process/text()", "OS_Process", + "/AuditRecord/Instance_Number/text()", "Instance_Number", + "/AuditRecord/Object_Schema/text()", "Object_Schema", #this is databaseName + "/AuditRecord/Object_Name/text()", "Object_Name", #this is tableName + "/AuditRecord/Action/text()", "Action", + "/AuditRecord/Returncode/text()", "Returncode", + "/AuditRecord/Comment_Text/text()", "Comment_text", #this is to captureloginfailed. + "/AuditRecord/Scn/text()", "Scn", + "/AuditRecord/DBID/text()", "DBID", + "/AuditRecord/Current_User/text()", "Current_User", + "/AuditRecord/Sql_Bind/text()", "Sql_Bind", + "/AuditRecord/Sql_Text/text()", "Sql_Text" + ] + } + mutate { + add_field => { "Check_DB_User" => "%{DB_User}" } + add_field => { "Check_Sql_Text" => "%{Sql_Text}" } + add_field => { "Check_Comment_text" => "%{Comment_text}" } + } + + + if [Check_DB_User] == "RDSADMIN" or [Check_DB_User] == "SYS" + or [Check_DB_User] == "/" or [Sql_Text] == "" or [Sql_Text] == "/" or (![Comment_text] and ![Sql_Text]) + { + drop {} + } + + # Clean up Sql_Text + mutate { + gsub => [ + "Sql_Text", ":\\\"SYS_B_([0-9]+)\\\"", ":\"SYS_B_\\1\"" + ] + } + + # Clean up Sql_Bind + grok { + match => { "Sql_Bind" => "^ #1\(1\):: %{GREEDYDATA:cleaned_sql_bind}" } + } + mutate { + replace => { "Sql_Bind" => "%{cleaned_sql_bind}" } + } + + mutate { add_field => { "log_group" => "%{[cloudwatch_logs][log_group]}" } } + + grok { match => { "log_group" => "(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)" } } + + + #Build GuardRecord + + mutate { + add_field => + { + "[GuardRecord][accessor][serverHostName]" => "%{account_id}_%{instance}" + "[GuardRecord][accessor][dbProtocol]" => "ORACLE" + "[GuardRecord][accessor][dataType]" => "TEXT" + "[GuardRecord][accessor][language]" => "ORACLE" + "[GuardRecord][accessor][serverType]" => "Oracle" + "[GuardRecord][dbName]" => "NA" + "[GuardRecord][time][minOffsetFromGMT]" => "0" + "[GuardRecord][time][minDst]" => "0" + "[GuardRecord][sessionLocator][clientPort]" => "-1" + "[GuardRecord][sessionLocator][serverIp]" => "0.0.0.0" + "[GuardRecord][sessionLocator][isIpv6]" => "false" + "[GuardRecord][sessionLocator][serverPort]" => "-1" + "[GuardRecord][accessor][dbUser]" => "NA" + "[GuardRecord][accessor][dbProtocolVersion]" => "" + "[GuardRecord][accessor][clientMac]" => "" + "[GuardRecord][accessor][serverOs]" => "" + "[GuardRecord][accessor][clientOs]" => "" + "[GuardRecord][accessor][osUser]" => "" + "[GuardRecord][appUserName]" => "" + + } + } + + ruby { code => 'event.set("[GuardRecord][sessionLocator][clientIpv6]", nil)' } + ruby { code => 'event.set("[GuardRecord][sessionLocator][serverIpv6]", nil)' } + + mutate { + add_field => { "check_action_code" => "%{Action}" } + add_field => { "check_return_code" => "%{Returncode}" } + + } + + + if [check_return_code] != "0" + { + if [check_action_code] == "12" + { + mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "SQL_ERROR" }} + mutate { replace => { "[GuardRecord][exception][sqlString]" => "%{Sql_Text}" }} + mutate { replace => { "[GuardRecord][exception][description]" => "SQL_ERROR" }} + ruby { code => 'event.set("[GuardRecord][data]", nil)' } + + } + else + { + mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" }} + mutate { replace => { "[GuardRecord][exception][sqlString]" => "%{Comment_text}" }} + mutate { replace => { "[GuardRecord][exception][description]" => "LOGIN_FAILED" }} + ruby { code => 'event.set("[GuardRecord][data]", nil)' } + } + } + else + { + ruby { code => 'event.set("[GuardRecord][data][construct]", nil)' } + mutate { add_field => { "[GuardRecord][data][originalSqlCommand]" => "%{Sql_Text}" }} + ruby { code => 'event.set("[GuardRecord][exception]", nil)' } + + } + + + if [Session_Id]{ + mutate { add_field => { "[GuardRecord][sessionId]" => "%{Session_Id}" }} + } + + if [Object_Schema]{ + mutate { replace => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{Object_Schema}" }} + mutate { replace => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{instance}:%{Object_Schema}" }} + + } + if[DB_User] + { + mutate { replace => { "[GuardRecord][accessor][dbUser]" => "%{DB_User}" }} + } + + + if [Userhost] { + mutate { + add_field => { "[GuardRecord][accessor][clientHostName]" => "NA" } + add_field => { "[GuardRecord][sessionLocator][clientIp]" => "0.0.0.0" } + } + grok { + match => { "Userhost" => "(?:%{WORD}[-_])?(?:%{IP:myclientIp}|%{HOSTNAME:myclientHostName})" } + } + if [myclientHostName] { + mutate { + replace => { "[GuardRecord][accessor][clientHostName]" => "%{myclientHostName}" } + } + } else { + mutate { + replace => { "[GuardRecord][sessionLocator][clientIp]" => "%{myclientIp}" } + } + } + } + + # Date conversion + if [Extended_Timestamp] { + date { + match => ["[Extended_Timestamp][0]", "ISO8601"] + target => "new_timeStamp" + } + ruby { + code => ' + if event.get("[new_timeStamp]") + event.set("[GuardRecord][time][timestamp]", event.get("[new_timeStamp]").time.to_i * 1000) + else + event.tag("timestamp_missing_error") + end + ' + } + } + + + # Remove unnecessary fields + prune { + whitelist_names => [ "GuardRecord"] + } + mutate { + convert => { "[GuardRcecord]" => "string" } + } + json_encode { + source => "[GuardRecord]" + } + } +} diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverCloudwatch/ouaCloudwatch-test.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverCloudwatch/ouaCloudwatch-test.conf new file mode 100644 index 000000000..5c63ed364 --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverCloudwatch/ouaCloudwatch-test.conf @@ -0,0 +1,219 @@ +#/* +#Copyright 2023-24 IBM Inc. - All Rights Reserved. +#SPDX-License-Identifier: Apache-2.0 +#*/ + +input { + generator { + lines => [ + '11221661575952024-01-03T15:12:47.288741ZADMINapurvabirajdarApurvas-MacBook-Pro.local21229unknown04408001A007BBA000001678517840ADMINCreate table OracleDB (DayId int,DayName varchar(50))', #ValidLog + '1121823766352024-01-03T09:23:04.079928ZADMINapurvabirajdarApurvas-MacBook-Pro.local7715unknown0ADMINDEMO1120000000000000000942374764461678517840ADMINDROP TABLE demo1', #SQLERROR + '1122358112024-01-03T17:18:45.538741ZBBBBBapurvabirajdarApurvas-MacBook-Pro.local15787unknown010000000000000000001017Authenticated by: DATABASE;AUTHENTICATED IDENTITY: BBBBB; Client address: (ADDRESS=(PROTOCOL=tcp)(HOST=49.156.71.123)(PORT=55984))1678517840BBBBB', #loginFailed + '1112832122023-12-18T13:08:16.902914ZADMINapurvabirajdarApurvas-MacBook-Pro.local2761unknown010201678517840\n', #sql_text blank + '44294967295242023-12-18T19:27:55.087565Z/rdsdbrdsdbip-10-1-5-7855500SYSDBA1678517840SYS\nCOMMIT\n' #system-generted + ] + count => 1 + type => "Oua" + add_field => { log_group => ["/aws/rds/instance/database-2/audit"] } + add_field => { "account_id" => "34682495****" } + } +} + + +filter { + if [type] == "Oua" { + + xml { + source => "message" + store_xml => false + xpath => [ + "/AuditRecord/Audit_Type/text()", "Audit_Type", + "/AuditRecord/Session_Id/text()", "Session_Id", + "/AuditRecord/StatementId/text()", "StatementId", + "/AuditRecord/EntryId/text()", "EntryId", + "/AuditRecord/Extended_Timestamp/text()", "Extended_Timestamp", #thisisTimeStamp + "/AuditRecord/DB_User/text()", "DB_User", #this is DBUser + "/AuditRecord/OS_User/text()", "OS_User", + "/AuditRecord/Userhost/text()", "Userhost", #this is clienIP or clientHostname + "/AuditRecord/OS_Process/text()", "OS_Process", + "/AuditRecord/Instance_Number/text()", "Instance_Number", + "/AuditRecord/Object_Schema/text()", "Object_Schema", #this is databaseName + "/AuditRecord/Object_Name/text()", "Object_Name", #this is tableName + "/AuditRecord/Action/text()", "Action", + "/AuditRecord/Returncode/text()", "Returncode", + "/AuditRecord/Comment_Text/text()", "Comment_text", #this is to captureloginfailed. + "/AuditRecord/Scn/text()", "Scn", + "/AuditRecord/DBID/text()", "DBID", + "/AuditRecord/Current_User/text()", "Current_User", + "/AuditRecord/Sql_Bind/text()", "Sql_Bind", + "/AuditRecord/Sql_Text/text()", "Sql_Text" + ] + } + mutate { + add_field => { "Check_DB_User" => "%{DB_User}" } + add_field => { "Check_Sql_Text" => "%{Sql_Text}" } + add_field => { "Check_Comment_text" => "%{Comment_text}" } + } + + + if [Check_DB_User] == "RDSADMIN" or [Check_DB_User] == "SYS" + or [Check_DB_User] == "/" or [Sql_Text] == "" or [Sql_Text] == "/" or (![Comment_text] and ![Sql_Text]) + { + drop {} + } + + # Clean up Sql_Text + mutate { + gsub => [ + "Sql_Text", ":\\\"SYS_B_([0-9]+)\\\"", ":\"SYS_B_\\1\"" + ] + } + + # Clean up Sql_Bind + grok { + match => { "Sql_Bind" => "^ #1\(1\):: %{GREEDYDATA:cleaned_sql_bind}" } + } + mutate { + replace => { "Sql_Bind" => "%{cleaned_sql_bind}" } + } + + mutate { add_field => { "log_group" => "%{[cloudwatch_logs][log_group]}" } } + + grok { match => { "log_group" => "(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)" } } + + + #Build GuardRecord + + mutate { + add_field => + { + "[GuardRecord][accessor][serverHostName]" => "%{account_id}_%{instance}" + "[GuardRecord][accessor][dbProtocol]" => "ORACLE" + "[GuardRecord][accessor][dataType]" => "TEXT" + "[GuardRecord][accessor][language]" => "ORACLE" + "[GuardRecord][accessor][serverType]" => "Oracle" + "[GuardRecord][dbName]" => "NA" + "[GuardRecord][time][minOffsetFromGMT]" => "0" + "[GuardRecord][time][minDst]" => "0" + "[GuardRecord][sessionLocator][clientPort]" => "-1" + "[GuardRecord][sessionLocator][serverIp]" => "0.0.0.0" + "[GuardRecord][sessionLocator][isIpv6]" => "false" + "[GuardRecord][sessionLocator][serverPort]" => "-1" + "[GuardRecord][accessor][dbUser]" => "NA" + "[GuardRecord][accessor][dbProtocolVersion]" => "" + "[GuardRecord][accessor][clientMac]" => "" + "[GuardRecord][accessor][serverOs]" => "" + "[GuardRecord][accessor][clientOs]" => "" + "[GuardRecord][accessor][osUser]" => "" + "[GuardRecord][appUserName]" => "" + + } + } + + ruby { code => 'event.set("[GuardRecord][sessionLocator][clientIpv6]", nil)' } + ruby { code => 'event.set("[GuardRecord][sessionLocator][serverIpv6]", nil)' } + + mutate { + add_field => { "check_action_code" => "%{Action}" } + add_field => { "check_return_code" => "%{Returncode}" } + + } + + + if [check_return_code] != "0" + { + if [check_action_code] == "12" + { + mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "SQL_ERROR" }} + mutate { replace => { "[GuardRecord][exception][sqlString]" => "%{Sql_Text}" }} + mutate { replace => { "[GuardRecord][exception][description]" => "SQL_ERROR" }} + ruby { code => 'event.set("[GuardRecord][data]", nil)' } + + } + else + { + mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" }} + mutate { replace => { "[GuardRecord][exception][sqlString]" => "%{Comment_text}" }} + mutate { replace => { "[GuardRecord][exception][description]" => "LOGIN_FAILED" }} + ruby { code => 'event.set("[GuardRecord][data]", nil)' } + } + } + else + { + ruby { code => 'event.set("[GuardRecord][data][construct]", nil)' } + mutate { add_field => { "[GuardRecord][data][originalSqlCommand]" => "%{Sql_Text}" }} + ruby { code => 'event.set("[GuardRecord][exception]", nil)' } + + } + + + if [Session_Id]{ + mutate { add_field => { "[GuardRecord][sessionId]" => "%{Session_Id}" }} + } + + if [Object_Schema]{ + mutate { replace => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{Object_Schema}" }} + mutate { replace => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{instance}:%{Object_Schema}" }} + + } + if[DB_User] + { + mutate { replace => { "[GuardRecord][accessor][dbUser]" => "%{DB_User}" }} + } + + + if [Userhost] { + mutate { + add_field => { "[GuardRecord][accessor][clientHostName]" => "NA" } + add_field => { "[GuardRecord][sessionLocator][clientIp]" => "0.0.0.0" } + } + grok { + match => { "Userhost" => "(?:%{WORD}[-_])?(?:%{IP:myclientIp}|%{HOSTNAME:myclientHostName})" } + } + if [myclientHostName] { + mutate { + replace => { "[GuardRecord][accessor][clientHostName]" => "%{myclientHostName}" } + } + } else { + mutate { + replace => { "[GuardRecord][sessionLocator][clientIp]" => "%{myclientIp}" } + } + } + } + + # Date conversion + if [Extended_Timestamp] { + date { + match => ["[Extended_Timestamp][0]", "ISO8601"] + target => "new_timeStamp" + } + ruby { + code => ' + if event.get("[new_timeStamp]") + event.set("[GuardRecord][time][timestamp]", event.get("[new_timeStamp]").time.to_i * 1000) + else + event.tag("timestamp_missing_error") + end + ' + } + } + + + # Remove unnecessary fields + prune { + whitelist_names => [ "message","GuardRecord"] + } + mutate { + convert => { "[GuardRcecord]" => "string" } + } + json_encode { + source => "[GuardRecord]" + } + } +} + +output { + stdout { + codec => rubydebug + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/filter.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/filter.conf new file mode 100644 index 000000000..bfcf7e64e --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/filter.conf @@ -0,0 +1,21 @@ + +filter { + if [type] == "oua" { + json { + source => "message" + } + if [obj_owner] == "SYS" or [obj_owner] == "AUDSYS" or [obj_owner] == "RDSADMIN" + or [current_user] == "RDSADMIN" or [current_user] == "SYS" or [sql_text] =~ "DBMS_OUTPUT.GET_LINE" + + { + drop{} + } + mutate { + add_field => {"[HostName]" => "%{SERVER_ADDRESS}" } + add_field => {"[PortNumber]" => "%{SERVER_PORT}" } + } + if "_jsonparsefailure" not in [tags] { + oua_filter {} + } + } +} diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/guardium-oua-uc.zip b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/guardium-oua-uc.zip new file mode 100644 index 000000000..7be1fa835 Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/guardium-oua-uc.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackage/OracleUnifiedAudit/manifest.json b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/manifest.json similarity index 100% rename from filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackage/OracleUnifiedAudit/manifest.json rename to filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/OracleUnifiedAudit/manifest.json diff --git a/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/ouaPipe.conf similarity index 78% rename from filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf rename to filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/ouaPipe.conf index 0addbff73..b784beb0b 100644 --- a/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf +++ b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/ouaPipe.conf @@ -13,17 +13,17 @@ input { filter { if [type] == "oua" { ruby { - code => " - if event.get('message') - event.set('message', event.get('message').gsub('\\\\\\', '\\\\\\\\')) - end - " - } - mutate { - gsub => [ - "command", "\w+/[^\s@]+@", "*****@" - ] - } + code => " + if event.get('message') + event.set('message', event.get('message').gsub('\\\\\\', '\\\\\\\\')) + end + " + } + mutate { + gsub => [ + "command", "\w+/[^\s@]+@", "*****@" + ] + } json { source => "message" } @@ -46,4 +46,4 @@ filter { output { stdout { codec => rubydebug } -} \ No newline at end of file +} diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageWithConnectJdbc/OuaConnectJdbc.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageWithConnectJdbc/OuaConnectJdbc.conf new file mode 100644 index 000000000..52061a732 --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageWithConnectJdbc/OuaConnectJdbc.conf @@ -0,0 +1,42 @@ +#/* +#Copyright 2024, 2025 IBM Inc. - All Rights Reserved. +#SPDX-License-Identifier: Apache-2.0 +#*/ + +input{ + kafka + { + # topic name should be same as the connector name + topics => ["oua-db1-kafka"] + type => "oua-db1-kafka" + + # kafka brokers + bootstrap_servers => ":" + partition_assignment_strategy => "cooperative_sticky" + group_id => "" + + add_field => { "uc_load" => "large" } + + # section kafka-secure + security_protocol=> "SSL" + ssl_truststore_location => "${LOGSTASH_KAFKA_CERTS_DIR}/.serverkeystore.jks" + ssl_truststore_password => "${ssl_truststore_password_key}" + ssl_truststore_type => "jks" + # end section kafka-secure + + } + } + + +filter { + if [type] == "oua-db1-kafka" { + mutate { + add_field => {"SERVER_ADDRESS" => ""} + add_field => {"SERVER_PORT" => ""} + } + + if "_jsonparsefailure" not in [tags] { + oua_filter {} + } + } +} diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageWithConnectJdbc/ouaConnectJdbc-test.conf b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageWithConnectJdbc/ouaConnectJdbc-test.conf new file mode 100644 index 000000000..13cb70920 --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageWithConnectJdbc/ouaConnectJdbc-test.conf @@ -0,0 +1,34 @@ +#/* +#Copyright 2023-24 IBM Inc. - All Rights Reserved. +#SPDX-License-Identifier: Apache-2.0 +#*/ + +input { + generator { + lines => [ + '{"schema":{"type":"struct","fields":[{"type":"int64","optional":true,"name":"org.apache.kafka.connect.data.Timestamp","version":1,"field":"EVENT_TIMESTAMP_UTC"},{"type":"string","optional":true,"field":"SESSIONID"},{"type":"string","optional":true,"field":"USERID"},{"type":"string","optional":true,"field":"CON_NAME"},{"type":"string","optional":true,"field":"OS_USER"},{"type":"string","optional":true,"field":"CLIENT_HOST_IP"},{"type":"string","optional":true,"field":"CLIENT_HOST_NAME"},{"type":"string","optional":true,"field":"SERVER_HOST_IP"},{"type":"string","optional":true,"field":"CLIENT_PROGRAM_NAME"},{"type":"string","optional":true,"field":"DBID"},{"type":"string","optional":true,"field":"DBNAME"},{"type":"string","optional":true,"field":"RETURN_CODE"},{"type":"string","optional":true,"field":"SQL_TEXT"}],"optional":false},"payload":{"EVENT_TIMESTAMP_UTC":1716303794152,"SESSIONID":"2967627425","USERID":"SUPERUSER","CON_NAME":"TESTDB","OS_USER":"Administrator","CLIENT_HOST_IP":"129.41.87.3","CLIENT_HOST_NAME":"sys-auto-fyre-ws161","SERVER_HOST_IP":"127.0.0.1","CLIENT_PROGRAM_NAME":"JDBC Thin Client","DBID":"2951224659","DBNAME":"TESTDB","RETURN_CODE":"0","SQL_TEXT":"INSERT INTO ACCOUNT_MASTER_SHOP_ACCOUNT_MASTER_DEPARTMENT(DEALER,ARTICLE ,PRICE) VALUES('D',4,19.95)\u0000"}}', #INSERT + '{"schema":{"type":"struct","fields":[{"type":"int64","optional":true,"name":"org.apache.kafka.connect.data.Timestamp","version":1,"field":"EVENT_TIMESTAMP_UTC"},{"type":"string","optional":true,"field":"SESSIONID"},{"type":"string","optional":true,"field":"USERID"},{"type":"string","optional":true,"field":"CON_NAME"},{"type":"string","optional":true,"field":"OS_USER"},{"type":"string","optional":true,"field":"CLIENT_HOST_IP"},{"type":"string","optional":true,"field":"CLIENT_HOST_NAME"},{"type":"string","optional":true,"field":"SERVER_HOST_IP"},{"type":"string","optional":true,"field":"CLIENT_PROGRAM_NAME"},{"type":"string","optional":true,"field":"DBID"},{"type":"string","optional":true,"field":"DBNAME"},{"type":"string","optional":true,"field":"RETURN_CODE"},{"type":"string","optional":true,"field":"SQL_TEXT"}],"optional":false},"payload":{"EVENT_TIMESTAMP_UTC":1716217082712,"SESSIONID":"788787356","USERID":"SUPERUSER","CON_NAME":"TESTDB","OS_USER":"Administrator","CLIENT_HOST_IP":"170.225.223.20","CLIENT_HOST_NAME":"sys-auto-fyre-ws91","SERVER_HOST_IP":"127.0.0.1","CLIENT_PROGRAM_NAME":"JDBC Thin Client","DBID":"2951224659","DBNAME":"TESTDB","RETURN_CODE":"0","SQL_TEXT":"DELETE FROM ACCOUNT_MASTER_SUPPLIERS_ACCOUNT_MASTER_DEPARTMENT WHERE SUPPLIERID IN('501')\u0000"}}', #DELETE + '{"schema":{"type":"struct","fields":[{"type":"int64","optional":true,"name":"org.apache.kafka.connect.data.Timestamp","version":1,"field":"EVENT_TIMESTAMP_UTC"},{"type":"string","optional":true,"field":"SESSIONID"},{"type":"string","optional":true,"field":"USERID"},{"type":"string","optional":true,"field":"CON_NAME"},{"type":"string","optional":true,"field":"OS_USER"},{"type":"string","optional":true,"field":"CLIENT_HOST_IP"},{"type":"string","optional":true,"field":"CLIENT_HOST_NAME"},{"type":"string","optional":true,"field":"SERVER_HOST_IP"},{"type":"string","optional":true,"field":"CLIENT_PROGRAM_NAME"},{"type":"string","optional":true,"field":"DBID"},{"type":"string","optional":true,"field":"DBNAME"},{"type":"string","optional":true,"field":"RETURN_CODE"},{"type":"string","optional":true,"field":"SQL_TEXT"}],"optional":false},"payload":{"EVENT_TIMESTAMP_UTC":1715934746826,"SESSIONID":"3738814242","USERID":"SUPERUSER","CON_NAME":"TESTDB","OS_USER":"Administrator","CLIENT_HOST_IP":"170.225.223.19","CLIENT_HOST_NAME":"sys-auto-fyre-ws51","SERVER_HOST_IP":"127.0.0.1","CLIENT_PROGRAM_NAME":"JDBC Thin Client","DBID":"2951224659","DBNAME":"TESTDB","RETURN_CODE":"1","SQL_TEXT":"INSERT INTO ACCOUNT_MASTER_SHOP_ACCOUNT_MASTER_DEPARTMENT(DEALER,ARTICLE,PRICE) VALUES('A',1,3.45)\u0000"}}', # RETURN CODE 1 + ] + count => 1 + type => "Oua" + } +} + + +filter { + if [type] == "oua" { + mutate { + add_field => {"SERVER_ADDRESS" => "auto-oua-01.cxmxwdil903v.us-west-1.rds.amazonaws.com"} + add_field => {"SERVER_PORT" => "1521"} + } + if "_jsonparsefailure" not in [tags] { + oua_filter {} + } + } + +output { + stdout { + codec => rubydebug + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverCloudwatch.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverCloudwatch.md new file mode 100644 index 000000000..04f30401d --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OuaOverCloudwatch.md @@ -0,0 +1,178 @@ +## 1. Configuring AWS RDS Oracle + +### Procedure + + 1. Browse to the Amazon AWS console at https://console.aws.amazon.com/. + a. Click on Services in the top left menu. + b. In the Database section, click on RDS. + c. Select the required region in the top right corner. + d. On the Amazon RDS Dashboard in the central panel, click on Create database. + e. Select the database creation method. + f. In Engine Options, select Oracle and the Oracle version. + g. Select the appropriate template (such as Production or Dev/Test). + h. In the settings section, enter a name for this database instance and create the master account with username and password that you use to log in to the database. + i. Select the database instance size as required. + j. Select the storage options (enabling Storage auto scaling, and so on). + k. Select Availability and durability options. + l. Depending on the connectivity requirements, select the settings for connectivity. + m. Select the type of Authentication to be enabled for database. Available option are Password Authentication, Password and IAM database authentication, and Password and Kerberos authentication. + n. Expand Additional Configuration options, as follows: + i. Configure the database options. + ii. Select options for backup. + iii. Select whether to enable encryption on the database instance. + iv. Select the options for deletion protection. + o. Click Create Database. + p. To view the database, click on Databases under Amazon RDS in the left panel. + q. To access the database instance from the remote client, take the following steps: + i. After the database is created, you can see the VPC security group associated with it on the summary page in the Connectivity and Security tab, under the Security section. + ii. Edit this security group to allow traffic on port 1521. + iii. Click on the group name to enable editing auditing. + iv. In the panel that appears underneath, go to Inbound Rules section, click on Edit Inbound Rules. + v. Select type as Oracle-RDS, protocol as TCP, Port Range as 1521. Per the required source, you can either set the range to a specific IP address or opened to all hosts. + +2. In the Additional Configuration section, under Log exports, select the newly created group, and then select the log type '**Audit**'from Amazon CloudWatch log options. +3. Click on Add Rule and Save changes. + **Note:** You might need to restart the database. + +## 2. Enabling auditing + + 1. Enable auditing by setting up a few parameters on the Parameter Group and associating the same parameters on the Database instance. + a. Select Parameter Groups from the left pane on Amazon RDS + b. Select the newly created Parameter Group. + c. Click on Edit parameters button on right corner. + d. Add the following setting: + + ``` + audit_trail = XML, EXTENDED + ``` + + + 2. Associating the DB Parameter Group to database Instance + a. Click on RDS and then on Databases from the left panel + b. Click on the Oracle database instance to be updated + c. Click on Modify button + d. In the Additional Configuration section, under database options, in the DB Parameter Group drop-down, select the newly created group + e. Click on continue + f. Select the database instance that, in its configuration section, the status shown for the DB Parameter Group is pending-reboot + g. Reboot the Database instance for the changes to take affect + + + 3. Applying different policies. + Depending on the requirement, rather than enabling auditing for all the tables and operations, you can choose to audit only selected tables or operations. +You can perform all operations from the master user which you created when you created the database. + +a. To audit every action for all users: + + ``` + create audit policy MyPolicy1 + actions all; + AUDIT policy MyPolicy1 + ``` + +b. To audit every action for admin users: + + ``` + create audit policy MyPolicy2 + actions all + when q'~ sys_context('userenv', 'session_user') = 'ADMIN' ~' + evaluate per SESSION; + Audit policy MyPolicy2; + ``` + +c. Audit policies which have mentioned explicitly to capture[user can add and delete according to requirements]: + + ``` + CREATE AUDIT POLICY MyPolicy3 ACTIONS UPDATE, INSERT, SELECT, DELETE; + AUDIT POLICY MyPolicy3; + ``` + +d. To enable login failed events in DB: + + ``` + CREATE AUDIT POLICY ORA_LOGIN_LOGOUT ACTIONS LOGON; + AUDIT POLICY ORA_LOGIN_LOGOUT WHENEVER NOT SUCCESSFUL; + ``` + +e. To remove an existing policy: + + ``` + NOAUDIT POLICY MyPolicy1; + DROP AUDIT POLICY MyPolicy1; + ``` + +f. To check audit logs below view name can be used it shows all audit operation that is performed on database. + + ``` + SELECT * FROM UNIFIED_AUDIT_TRAIL WHERE DBUSERNAME ='ADMIN'AND OBJECT_SCHEMA ='ADMIN' order by event_timestamp desc; + ``` + +g. To check login failed events captured in our database: + + ``` + SELECT * FROM UNIFIED_AUDIT_TRAIL + WHERE ACTION_NAME = 'LOGON' + AND RETURN_CODE != 0 + ORDER BY EVENT_TIMESTAMP DESC; + ``` + +h. To delete the content: + + ``` + BEGIN + dbms_audit_mgmt.clean_audit_trail(dbms_audit_mgmt.audit_trail_unified,false); + END; + ``` + +## Limitations + There will be delay in data being observed for reports due to limitaion of ORACLE RDS DB instance. + +## 3. Configuring the Oracle Unified Auditing filters in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the MSSQL template. + + +#### Procedure + +#### Before you begin + +• Configure the policies you require. See [policies](/docs/#policies) for more information. + +• You must have permission for the S-Tap Management role. The admin user includes this role by default. + +• Download the [logstash-json-encode.zip](./logstash-json-encode.zip) plug-in. This is not necessary for Guardium Data Protection v12.0 and later. + +• Choose the appropriate plugin download based on your Guardium version.
+I. If you are using Guardium 11.4 with patch p485 or earlier, +Download [logstash-filter-xml-4.1.3-1.zip](./logstash-filter-xml-4.1.3-1.zip).
+II. If you are using Guardium 11.5 with patch p535 or earlier, +Download [logstash-filter-xml-4.1.3-1.zip](./logstash-filter-xml-4.1.3-1.zip).
+III. If you are using Guardium 12.0 with patch p5 or earlier, +Download [logstash-filter-xml-4.2.0-1.zip](./logstash-filter-xml-4.2.0-1.zip).
+IV. For the Guardium 11.4(p490 or later),11.5(p540 or later) and 12.0(p10 or later), +Download [logstash-filter-xml-4.2.0-2.zip](./logstash-filter-xml-4.2.0-2.zip).
+ +#### Procedure: + +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. Enable the universal connector if it is currently disabled. +3. Click **Upload File** and select the offline [logstash-json-encode.zip](./logstash-json-encode.zip) plug-in. After it uploads, click **OK**.This is not necessary for Guardium Data Protection v12.0 and later. +4. Upload the relevant plugin based on the version of the Guardium. + I. If you are using Guardium 11.4 with patch p485 or earlier, + Download [logstash-filter-xml-4.1.3-1.zip](./logstash-filter-xml-4.1.3-1.zip).
+ II. If you are using Guardium 11.5 with patch p535 or earlier, + Download [logstash-filter-xml-4.1.3-1.zip](./logstash-filter-xml-4.1.3-1.zip).
+ III. If you are using Guardium 12.0 with patch p5 or earlier, + Download [logstash-filter-xml-4.2.0-1.zip](./logstash-filter-xml-4.2.0-1.zip).
+ IV. For the Guardium 11.4(p490 or later),11.5(p540 or later) and 12.0(p10 or later), + Download [logstash-filter-xml-4.2.0-2.zip](./logstash-filter-xml-4.2.0-2.zip).
+ +6. Click the Plus sign to open the Connector Configuration dialog box. +7. Type a name in the **Connector name** field. +8. Update the input section to add the details from [OuaCloudwatch.conf](OracleUnifiedAuditPackageOverCloudwatch/OuaCloudwatch.conf) for AWS OUA. +9. Update the filter section to add the details from [OuaCloudwatch.conf](OracleUnifiedAuditPackageOverCloudwatch/OuaCloudwatch.conf) for AWS OUA. +10. The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. This is no longer required starting v12p20 and v12.1. +12. Click **Save**. Guardium validates the new connector and enables the universal connector if it was + disabled. After it is validated, it appears in the Configure Universal Connector page. + + + diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbc.md similarity index 86% rename from filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md rename to filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbc.md index c600a5a8b..739f7fda2 100644 --- a/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md +++ b/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbc.md @@ -1,21 +1,23 @@ -# Oracle Unified Audit Universal Connector Over JDBC Connect +# Oracle Unified Audit Universal Connector ## Meet Oracle Unified Audit Over JDBC Connect -* Tested versions: 19,21 -* Environments: On-prem, RDS in AWS +* Tested versions: 19, 21 +* Environments: On-prem, RDS in AWS, Oracle Base Database Service in OCI * Supported inputs: Kafka Input (pull) -* Supported Oracle versions: 19,21 +* Supported Oracle versions: 19, 21 * Supported Guardium versions: - * Guardium Data Protection: appliance bundle 12.1p105 or later. - + * Guardium Data Protection: appliance bundle 12.1p105 or later. + **Note**: This readme is also applicable for **OUA over JDBC connect 2.0** and **OUA Multitenant over JDBC connect 2.0** plug-ins. -Kafka-connect is framework for streaming data between Apache Kafka and other systems. +## Functional Overview + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. Detailed breakdown: -1. Kafka-connect JDBC Connector: pulls data from `UNIFIED_AUDIT_TRAIL`. -2. Kafka-connect JDBC Connector for OUA Multitenant over JDBC connect 2.0: pulls data from `CDB_UNIFIED_AUDIT_TRAIL`. -3. The data in the Kafka topic is consumed by kafka-input plugin and process by the 'guardium-oua-uc' filter plug-in, +1. Kafka-connect JDBC Connector: Used to pull data from `UNIFIED_AUDIT_TRAIL`. +2. Produce to Kafka: The queried data is then sent (produced) to a Kafka topic. +3. Consume with UC: The data in the Kafka topic is consumed by kafka-input plugin and process by guardium-oua-uc, a specific Unified Connector designed for your use case. **Tip**: IBM recommends creating a Kafka cluster only after your environment is patched with appliance bundle 12.0p120+ 12.0p5002 for Guardium Data Protection version 12.1, as using a Kafka cluster before appliance bundle 12.0p120 + UC 12.0p5002 may provide undesirable results and does not support disaster recovery scenarios. This ensures that profiles using the Kafka cluster are applied correctly. @@ -23,14 +25,12 @@ Detailed breakdown: ### GDP versions available with OUA over JDBC credential support | Credential types | Patch details for availability | |--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **JDBC** | GDP version 12.1 + Appliance bundle patch 105 + Universal connector patch 1006 and later | -| **JDBC & Kerbseros** | GDP version 12.1 + Appliance bundle patch 115 + Universal connector patch 5002 and later | -| **JDBC & Kerbseros for OUA 2.0** | GDP version 12.1 + Appliance bundle patch 120 + Universal connector patch p5002 | - +| **JDBC** | GDP version 12.1 + Appliance bundle patch 105 or 115 + Universal connector patch 1006 and later | +| **JDBC & Kerberos** | GDP version 12.1 + Appliance bundle patch 115 + Universal connector patch 5002 and later | +| **JDBC & Kerberos for OUA 2.0** | GDP version 12.1 + Appliance bundle patch 120 + Universal connector patch p5002 | - -### Requirements -1. This feature is currently supported only in environments that is using central manager workflow and a Kafka cluster, in Guardium Data Protection version 12.1. +## Requirements +1. This feature is currently supported only in environments that use a central manager workflow and a Kafka cluster, in Guardium Data Protection version 12.1. 2. Unified auditing must be enabled in an Oracle database that will be monitored by this method 3. Download Oracle JDBC driver according to the details mentioned in the **[JDBC driver library](https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md#configuring-universal-connector-profile)** field. @@ -111,7 +111,7 @@ Detailed breakdown: ### Configuring Universal Connector Profile 1. To create a datasource profile, see [Creating data source profiles](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_datasource_profile_management.html). -2. Select '**OUA over JDBC connect**' in the plug-ins list +2. Select '**OUA over JDBC connect**' in the plug-ins list. 3. Update the parameters as follows: | Field | Description | @@ -123,8 +123,10 @@ Detailed breakdown: | **Hostname** | Specifies the hostname or IP address of the Oracle database server. It is the address where the Oracle instance can be accessed for establishing a JDBC connection. | | **JDBC driver library** | The Oracle JDBC driver JAR file (e.g., `ojdbc8.jar`) is required for the connector to communicate with the Oracle database. Download the [Oracle JDBC driver JAR file](https://download.oracle.com/otn-pub/otn_software/jdbc/234/ojdbc8.jar) and upload it to the Kafka Connect environment. | | **Port** | Specifies the port number used to connect to the Oracle database. The default port number is 1521, but it can vary depending on the Oracle configuration. Port 1521 must be open and accessible for the connection. | -| **Service Name / SID** | Specifies the Oracle service name (or SID if it's an older configuration) for the Kafka connector to connect. The service name uniquely identifies a database service within an Oracle environment and is provided by the database administrator. For OUA over JDBC data is retrived from the service itself: unified_audit_trail . | -| **CDB Service Name / SID** | OUA over JDBC Connect 2.0 and OUA multitenant over JDBC Connect, data is retrived from CDB service audit log: cdb_unified_audit_trail. | +| **Service Name / SID** | Specifies the Oracle service name (or SID if it's an older configuration) for the Kafka connector to connect. The service name uniquely identifies a database service within an Oracle environment and is provided by the database administrator. For OUA over JDBC data is retrieved from the service itself: unified_audit_trail . | +| **CDB Service Name / SID** | OUA over JDBC Connect 2.0 and OUA multitenant over JDBC Connect, data is retrieved from CDB service audit log: cdb_unified_audit_trail. | 4. Continue from step 3 of [Creating data source profile topic](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_datasource_profile_management.html) to complete creating a datasource profile. + + diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverPipe.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipe.md new file mode 100644 index 000000000..4f517959a --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipe.md @@ -0,0 +1,165 @@ +# Oracle Unified Audit Universal Connector + +## Meet Oracle Unified Audit + +* Tested versions: 18,19 +* Environments: On-prem, RDS in AWS, Oracle Autonomous Database in OCI + + **Note**: + * Autonomous Database in OCI is supported only by Guardium Data Protection SqlGuard-12.0p7015_Bundle_May_20_2024 and SqlGuard-11.0p545_Bundle_Jul_09_2024. + * For Oracle Autonomous Database, TCP is the only supported protocol, which does not provide built-in encryption. +* Supported inputs: Oracle Unified Audit (pull) +* Supported Oracle versions: 18, 19, and 21 +* Supported Guardium versions: + * Guardium Data Protection: 11.4 and above + +## Requirements + +1. Unified auditing must be enabled in an Oracle database that will be monitored by this method +2. Download the Basic Instant client package from Oracle. + **Note:** : In this release only specific Instant client will be supported from v21.1.0.0.0 Download [here](https://download.oracle.com/otn_software/linux/instantclient/211000/oracle-instantclient-basic-21.1.0.0.0-1.x86_64.rpm) +3. Download the OUA universal connector plug-in `guardium-oua-uc.zip` from [here](./guardium-oua-uc.zip) +4. A designated user for OUA UC should be created for Oracle database access. An existing user with sysdba privileges can also be used +5. A secret containing the user’s password for OUA universal connector must be created + - Example: `grdapi universal_connector_keystore_add key=OUA_USER_PASS password=` where `` is the OUA universal connector user’s password for the database. `OUA_USER_PASS` will be used in the plug-in configuration as a variable for password secret + +Currently, this plug-in will work only on IBM Security Guardium Data Protection, not Guardium Insights + +## Building + +Update the variables in Makefile for your environment's Java home and Logstash location + +## Setup + +1. Create a designated Database User for OUA UC to retrieve audit data with minimal privileges (using DBA help) as follows: + - Assuming the name for the designated Oracle Unified Audit user with minimal permissions will be "guardium" with password "password" + - Connect to Oracle using sysdba account and execute the following commands: + + ``` + CREATE USER guardium IDENTIFIED BY password; + GRANT CONNECT, RESOURCE to guardium; + GRANT SELECT ANY DICTIONARY TO guardium; + exec DBMS_NETWORK_ACL_ADMIN.APPEND_HOST_ACE(host => 'localhost', + ace => xs$ace_type(privilege_list => xs$name_list('connect', + 'resolve'), principal_name => 'guardium', principal_type => xs_acl.ptype_db)); + ``` + + - To verify your new user's privileges, connect to the Oracle instance that you planning to monitor using the name and credentials for your designated user and run the following statements: + + ``` + select count(*) from AUDSYS.AUD$UNIFIED; + SELECT UTL_INADDR.get_host_address FROM DUAL; + ``` + + - If there are no errors that means you can use this new user for this UC method + +2. Enable the universal collector feature on the designated Guardium collectors or the stand-alone system. See [here](https://www.ibm.com/docs/en/guardium/11.4?topic=connector-enabling-guardium-universal-collectors) + +3. On the collector, go to Setup > Tools and Views > Configure Universal Connector + +4. Click on the "UPLOAD” button and upload downloaded `oracle-instantclient-basic-21.1.0.0.0-1.x86_64.rpm` and then `guardium-oua-uc.zip` + + 5. Click on "+". The Connector Configuration dialog opens + + - Type any unique name in the "Connector name" field + + - Paste content of "[ouaPipe.conf](https://github.ibm.com/Activity-Insights/universal-connectors/blob/master/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/ouaPipe.conf)" in the **Input configuration** field + + - Paste content of "[ouaPipe.conf](https://github.ibm.com/Activity-Insights/universal-connectors/blob/master/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackageOverPipe/ouaPipe.conf)" in the **Filter configuration** field + + + - **NOTE**: The type specified for the filters must be unique among Universal Connectors and be identical in the input and filter configurations + + 6. Click **Save**. Guardium validates the new connector and enables plugin. After it is validated, it appears in the **Configure Universal Connector** page. + +**Note**: The following arguments are used in configuration for the OUA plug-in +- -c path : path to instantclient libraries +- -a path : path to tnsnames.ora (directory, not the filename) +- -r rows : number of rows to fetch at a single time (default 100) +- -t timeout : timeout for all operations (connect, execute, fetch, etc) specified in milliseconds (default 300000) +- -p period : period of time between passes specified in seconds (default 300) +- -s path : path to directory where state should be saved +- -j : output audits in JSON format + +## Limitations + Normally, the "Statement Type" attribute for the "FULL SQL" entity in Guardium reports shows us whether a full SQL statement is a prepared statement. However, because OUA doesn't give us information about whether a statement is a prepared statement or not, the "Statement type" attribute is not applicable for the OUA universal connector plug-in. + +## Troubleshooting + +### Issue: Oracle DB has too many audit logs + +If you are encountering issues related to excessive audit logs in the Oracle database, it might be causing performance issues or S-TAP not coming up and no traffic in collector. The following steps can help resolve this issue: + +1. **Clean up Oracle DB audit logs:** + + It's possible that the Oracle database has accumulated too many audit logs. You can clean them up using the following command: + + ```sql + exec dbms_audit_mgmt.clean_audit_trail(audit_trail_type => dbms_audit_mgmt.audit_trail_unified, use_last_arch_timestamp => FALSE); + ``` + + **Note:** This command should be executed with a user that has admin privileges. + +2. **Restart the UC from GUI:** + + After cleaning up the audit logs, restart the UC (Universal Connector) for the changes to take effect. You can restart the UC by disabling and then enabling it through the GUI. + + - Go to Configure Universal Connector page. Click on the Disable button if it is enabled already. + - Wait for a few seconds till it loads. + - Enable the Universal Guardium connector by clicking enable button. + +This should resolve any issues related to the audit log overload. + +### Issue: Change the user password (OUA_USER_PASS) in Keystore + +Complete the following steps to change the OUA user password in Keystore. + +1. Remove the existing password from the **OUA_USER_PASS** parameter. + + ```grdapi universal_connector_keystore_remove key=OUA_USER_PASS``` + +3. Add new user password. + + ```grdapi universal_connector_keystore_add key=OUA_USER_PASS password=password``` + +4. Restart the Universal Connector. + + ```grdapi stop_universal_connector``` + + ```grdapi run_universal_connector``` + +5. Verify the password update status by using the following grdapis. + + a. Verify if **OUA_USER_PASS** is defined in the Keystore. + + grdapi universal_connector_keystore_list + + b. Verify the Universal Connector status. + + grdapi get_universal_connector_status + +### Issue: UC OUA Connection Failure Caused by Special Character in Oracle Password + +#### Root Cause: +The issue occurs when a special character, specifically `@`, is used in the Oracle database password. This interferes with the connection string and leads to connection failures. + +#### Symptoms: +- No audit traffic is seen in the Guardium collector. +- UC fails to connect to the database. +- S-TAP not showing in the S-TAP status monitor page. + +#### Example Logs: +ORA-12154: TNS:could not resolve the connect identifier specified +can't connect to: rdxxxxx1d@infdbrds001awsd...:1521/DBMINFD1 + +#### Resolution: +1. **Update the Oracle Password** + - Remove or replace the `@` character from password for the database. + - Update the password in the Universal Connector configuration. + +2. **Restart the UC Connector** + - Go to *Configure Universal Connector* in the GUI. + - Click **Disable**, wait a few seconds, then click **Enable** to restart the connector. + +#### Note: +Special characters like `@` are reserved in connection strings. Avoid using them in database passwords when configuring UC with Guardium. diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md deleted file mode 100644 index 18014eb88..000000000 --- a/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md +++ /dev/null @@ -1,106 +0,0 @@ -# Oracle Unified Audit Universal Connector - -## Meet Oracle Unified Audit - -* Tested versions: 18,19 -* Environments: On-prem, RDS in AWS, Oracle Autonomous Database in OCI - - **Note**: - * Autonomous Database in OCI is supported only by Guardium Data Protection SqlGuard-12.0p7015_Bundle_May_20_2024 and SqlGuard-11.0p545_Bundle_Jul_09_2024. - * For Oracle Autonomous Database, TCP is the only supported protocol, which does not provide built-in encryption. -* Supported inputs: Oracle Unified Audit (pull) -* Supported Oracle versions: 18, 19, and 21 -* Supported Guardium versions: - * Guardium Data Protection: 11.4 and above - - -## Requirements - -1. Unified auditing must be enabled in an Oracle database that will be monitored by this method -2. Download the Basic Instant client package from Oracle. - **Note:** : In this release,only specific instance clients will be supported from starting v21.1.0.0.0 Download [here](https://download.oracle.com/otn_software/linux/instantclient/211000/oracle-instantclient-basic-21.1.0.0.0-1.x86_64.rpm) -3. Oracle Unified Audit Universal Connector is automatically available with Guardium Data Protection versions 12.x, 11.4 with appliance bundle 11.0p490 or later or Guardium Data Protection version 11.5 with appliance bundle 11.0p540 or later releases. -**Note**: For Guardium Data Protection version 11.4 without appliance bundle 11.0p490 or prior or Guardium Data Protection version 11.5 without appliance bundle 11.0p540 or prior, download the [OUA](https://github.com/IBM/universal-connectors/raw/release-v1.2.0/filter-plugin/logstash-filter-oua-guardium/OracleUnifiedAuditPackage/OracleUnifiedAudit/guardium-oua-uc.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). - -4. A designated user for OUA UC should be created for Oracle database access. An existing user with sysdba privileges can also be used - -5. You must create a secret containing your OUA universal connector password. - - Example: `grdapi universal_connector_keystore_add key=OUA_USER_PASS password=` where `` is the OUA universal connector user’s password for the database. `OUA_USER_PASS` will be used in the plug-in configuration as a variable for the password secret. - -6. Configure the policies you require. See [policies](https://github.com/IBM/universal-connectors/tree/main/docs#policies) for more information. - -Currently, this plug-in will work only on IBM Security Guardium Data Protection, not Guardium Insights - -## Building - -Update the variables in Makefile for your environment's Java home and Logstash location - -## Setup - -1. Create a designated Database User for OUA UC to retrieve audit data with minimal privileges (using DBA help) as follows: - - Assuming the name for the designated Oracle Unified Audit user with minimal permissions will be "guardium" with password "password" - - Connect to Oracle using sysdba account and execute the following commands: - - - For Oracle Autonomous Database in OCI, run the following commands to create a user with the required grants: - ``` - CREATE USER guardium IDENTIFIED BY password; - GRANT CONNECT to guardium; - GRANT SELECT ANY DICTIONARY to guardium; - GRANT AUDIT_VIEWER to guardium; - GRANT SELECT ON v$INSTANCE to guardium; - GRANT SELECT ON v$DATABASE to guardium; - GRANT SELECT ON v$MYSTAT to guardium; - ``` - - - For other environments including RDS in AWS and Oracle Databases On-Premises run the following commands: - ``` - CREATE USER IDENTIFIED BY ; - GRANT CONNECT to ; - GRANT AUDIT_VIEWER to ; - GRANT SELECT ON v_$INSTANCE to ; - GRANT SELECT ON v_$DATABASE to ; - GRANT SELECT ON v_$MYSTAT to ; - ``` - - - To verify your new user's privileges, connect to the Oracle instance that you planning to monitor using the name and credentials for your designated user and run the following statements: - - ``` - select count(*) from AUDSYS.AUD$UNIFIED; - SELECT UTL_INADDR.get_host_address FROM DUAL; - ``` - - - If there are no errors that means you can use this new user for this UC method - -2. Enable the universal collector feature on the designated Guardium collectors or the stand-alone system. See [here](/docs/Guardium%20Data%20Protection/uc_config_gdp.md). - -3. On the collector, go to Setup > Tools and Views > Configure Universal Connector - -4. Click on the "UPLOAD” button and upload the downloaded `oracle-instantclient-basic-21.1.0.0.0-1.x86_64.rpm`. - -6. Then, upload the `guardium-oua-uc.zip` file. This step is not necessary for Guardium Data Protection v12.0 and later. - - 5. Click on the "+". The Connector Configuration dialog opens. - - - Type any unique name in the **Connector name** field. - - - Paste the content from the "[ouaPipe.conf](https://github.com/IBM/universal-connectors/raw/main/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf)" in the **Input configuration** field - - - Paste the content from the "[ouaPipe.conf](https://github.com/IBM/universal-connectors/raw/main/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf)" in the **Filter configuration** field - - - **NOTE**: The type specified for the filters must be unique among Universal Connectors and be identical in the input and filter configurations - - 6. Click **Save**. Guardium validates the new connector and enables the plug-in. After it is validated, it appears on the **Configure Universal Connector** page. - -**Note**: The following arguments are used in the OUA plug-in configuration. -- -c path : path to instantclient libraries -- -a path : path to tnsnames.ora (directory, not the filename) -- -r rows : number of rows to fetch at a single time (default 100) -- -t timeout : timeout for all operations (connect, execute, fetch, etc) specified in milliseconds (default 300000) -- -p period : period of time between passes specified in seconds (default 300) -- -s path : path to directory where state should be saved -- -j : output audits in JSON format - -## Limitation -- Normally, the "statement type" attribute for the "FULL SQL" entity in Guardium reports shows us whether a full SQL statement is a prepared statement. However, because OUA doesn't give us information about whether a statement is a prepared statement or not, the "Statement type" attribute is not applicable for the OUA universal connector plug-in. -- The "record affected" field is not supported when using Oracle with the universal connector. -- If you restart the database, then any Universal Connector that uses the OUA plugin must also be restarted. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-oua-guardium/README.md b/filter-plugin/logstash-filter-oua-guardium/README.md index 8e4edc7c4..5eac46296 100644 --- a/filter-plugin/logstash-filter-oua-guardium/README.md +++ b/filter-plugin/logstash-filter-oua-guardium/README.md @@ -1,9 +1,19 @@ -# Oracle Unified Audit Universal Connector +## OUA-Guardium Logstash filter plug-in -## Follow this link to set up and use Oracle Unified Audit Universal Connector over Pipe +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Postgres audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. -[OuaOverPipeReadme](./OuaOverPipeReadme.md) +This plugin is written in java, and so is a script that can be directly copied into the Guardium configuration for Universal Connectors. There is no need to upload the plugin code. However, in order to support a few features one zip has to be added with the name "postgres-offline-plugins-7.5.2.zip". -## Follow this link to set up and use Oracle Unified Audit Universal Connector over JDBC Connect +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for the Guardium universal connector. -[OuaOverConnectJdbc](./OuaOverConnectJdbcReadme.md) \ No newline at end of file +## Follow the below link to setup and use AWS OUA using Cloudwatch input. + +[AWS OUA-Cloudwatch README](./OuaOverCloudwatch.md) + +## Follow the below link to setup and use AWS OUA using Pipe input. + +[AWS Oua-Pipe README](./OuaOverPipe.md) + +## Follow the below link to setup and use kafka-connect with Aiven jdbc plugin. + +[JDBC_Oua-kafka-connect README](./OuaOverConnectJdbc.md) \ No newline at end of file diff --git a/filter-plugin/logstash-filter-oua-guardium/VERSION b/filter-plugin/logstash-filter-oua-guardium/VERSION index 3eefcb9dd..26aaba0e8 100644 --- a/filter-plugin/logstash-filter-oua-guardium/VERSION +++ b/filter-plugin/logstash-filter-oua-guardium/VERSION @@ -1 +1 @@ -1.0.0 +1.2.0 diff --git a/filter-plugin/logstash-filter-oua-guardium/build.gradle b/filter-plugin/logstash-filter-oua-guardium/build.gradle index 3051b47dd..60d6b3af4 100644 --- a/filter-plugin/logstash-filter-oua-guardium/build.gradle +++ b/filter-plugin/logstash-filter-oua-guardium/build.gradle @@ -44,13 +44,10 @@ buildscript { dependencies { classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } } -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) - repositories { mavenCentral() } @@ -74,17 +71,20 @@ shadowJar { } dependencies { - implementation group: 'commons-validator', name: 'commons-validator', version: versions.dependencies.commonsValidator - implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore - implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang - implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-validator', name: 'commons-validator', version: '1.7' + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' + implementation 'org.apache.commons:commons-lang3:3.7' + implementation 'com.google.code.gson:gson:2.8.9' implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") - implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-*.*.*.jar") + //implementation 'com.google.code.gson:gson:2.8.9' + + testImplementation 'junit:junit:4.12' + testImplementation 'org.jruby:jruby-complete:9.2.7.0' + - testImplementation 'junit:junit:' + versions.dependencies.junit - testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-*.*.*.jar") - testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") } clean { diff --git a/filter-plugin/logstash-filter-oua-guardium/guardium-oua-uc.zip b/filter-plugin/logstash-filter-oua-guardium/guardium-oua-uc.zip new file mode 100644 index 000000000..3b2530f5e Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/guardium-oua-uc.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.1.3-1.zip b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.1.3-1.zip new file mode 100644 index 000000000..0746464b0 Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.1.3-1.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.2.0-1.zip b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.2.0-1.zip new file mode 100644 index 000000000..21c30ec36 Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.2.0-1.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.2.0-2.zip b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.2.0-2.zip new file mode 100644 index 000000000..069df731d Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-xml-4.2.0-2.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/logstash-json-encode.zip b/filter-plugin/logstash-filter-oua-guardium/logstash-json-encode.zip new file mode 100644 index 000000000..46b740624 Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/logstash-json-encode.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java b/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java index e14a4eb69..94c28098d 100644 --- a/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java +++ b/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java @@ -12,6 +12,7 @@ import com.ibm.guardium.universalconnector.commons.Util; import com.ibm.guardium.universalconnector.commons.structures.*; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; @@ -23,6 +24,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; + // class name must match plugin name @LogstashPlugin(name = "oua_filter") public class OuaFilter implements Filter { @@ -44,38 +46,72 @@ public class OuaFilter implements Filter { public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); public static final String LOGSTASH_TAG_PARSE_ERROR = "_ouaguardium_parse_error"; - + private String id; - private String sourceField; + private String sourceField; - // first part is days since Unix Epoch - // second part is UTC time since Unix Epoch - // +000018533 22:00:03.981871 + // first part is days since Unix Epoch + // second part is UTC time since Unix Epoch + // +000018533 22:00:03.981871 private static String AUDIT_DATE_FORMAT = "HH:mm:ss.SSSSSS"; private static final DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ofPattern(AUDIT_DATE_FORMAT)); private static final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); - public static final String SERVER_TYPE_STRING = "Oracle"; - public static final String DATA_PROTOCOL_STRING = "Oracle Unified Audit"; - public static final String LANGUAGE_STRING = "ORACLE"; - public static final String UNKNOWN_STRING = ""; - - public static final String CLIENT_IP_TAG = "client_host_ip"; - public static final String SERVER_IP_TAG = "server_host_ip"; - public static final String CLIENT_HOST_TAG = "client_host_name"; - public static final String SERVER_HOST_TAG = "HostName"; - - public static final String SERVER_HOST_PORT = "PortNumber"; - public static final String OS_USER_TAG = "os_user"; - public static final String SOURCE_PROGRAM_TAG = "client_program_name"; - public static final String DB_NAME_TAG = "dbname"; - public static final String TIMESTAMP_TAG = "event_timestamp"; - public static final String SQL_TAG = "sql_text"; - public static final String DB_USER_TAG = "userid"; - public static final String SESSION_ID_TAG = "sessionid"; - public static final String CON_NAME_TAG = "con_name"; - public static final String RETURN_CODE_TAG = "return_code"; + public static final String SERVER_TYPE_STRING = "Oracle"; + public static final String DATA_PROTOCOL_STRING = "ORACLE"; + public static final String LANGUAGE_STRING = "ORACLE"; + public static final String UNKNOWN_STRING = ""; + + public static final String SERVER_ACCOUNT_ID = "accountId"; + + public static final String SERVER_ADDRESS_TAG = "SERVER_ADDRESS"; + public static final String SERVER_HOST_TAG = "HostName"; + + public static final String SERVER_PORT_TAG = "SERVER_PORT"; + public static final String SERVER_HOST_PORT = "PortNumber"; + + public static final String CLIENT_IP_TAG = "client_host_ip"; + public static final String CLIENT_IP_UPPER_TAG = "CLIENT_HOST_IP"; + + public static final String SERVER_IP_TAG = "server_host_ip"; + public static final String SERVER_IP_UPPER_TAG = "SERVER_HOST_IP"; + + public static final String CLIENT_HOST_TAG = "client_host_name"; + public static final String CLIENT_HOST_UPPER_TAG = "CLIENT_HOST_NAME"; + + public static final String OS_USER_TAG = "os_user"; + public static final String OS_USER_UPPER_TAG = "OS_USER"; + + public static final String SOURCE_PROGRAM_TAG = "client_program_name"; + public static final String SOURCE_PROGRAM_UPPER_TAG = "CLIENT_PROGRAM_NAME"; + + public static final String DB_NAME_TAG = "dbname"; + public static final String DB_NAME_UPPER_TAG = "DBNAME"; + + public static final String TIMESTAMP_TAG = "event_timestamp"; + public static final String TIMESTAMP_UTC_UPPER_TAG = "EVENT_TIMESTAMP_UTC"; + + public static final String SQL_TAG = "sql_text"; + public static final String SQL_UPPER_TAG = "SQL_TEXT"; + + public static final String DB_USER_TAG = "userid"; + public static final String DB_USER_UPPER_TAG = "USERID"; + + public static final String SESSION_ID_TAG = "sessionid"; + public static final String SESSION_ID_UPPER_TAG = "SESSIONID"; + + public static final String CON_NAME_TAG = "con_name"; + public static final String CON_NAME_UPPER_TAG = "CON_NAME"; + + public static final String RETURN_CODE_TAG = "return_code"; + public static final String RETURN_CODE_UPPER_TAG = "RETURN_CODE"; + + public static final String PAYLOAD = "payload"; + + public static final String MESSAGE = "message"; + + public static final String SOURCE_SYSTEM = "source_system"; public OuaFilter(String id, Configuration config, Context context) { // constructors should validate configuration options @@ -91,51 +127,62 @@ public Collection filter(Collection events, FilterMatchListener ma ArrayList outputEvents = new ArrayList<>(); for (Event e : events) { if (log.isDebugEnabled()) { - log.debug("Event: "+logEvent(e)); + log.debug("Event: "+logEvent(e)); + } + // removing all events that not contains Message fields + if ( !(e.getField(MESSAGE) instanceof String)) { + if (log.isDebugEnabled()) { + log.debug("Event without message field removed: " + logEvent(e)); + } + skippedEvents.add(e); + continue; + } + try { + if( (e.includes(SOURCE_SYSTEM) && "kafka-connect".equalsIgnoreCase((String) e.getField(SOURCE_SYSTEM))) || + e.getField(MESSAGE).toString().contains("\"payload\":")) { + processConnectMessage(e); + } + + int return_code = -1; + + if (e.getField(OuaFilter.RETURN_CODE_TAG) instanceof Long) { + return_code = Integer.parseInt(e.getField(OuaFilter.RETURN_CODE_TAG).toString()); + } + + if (return_code == 0 ) { + Record record = OuaFilter.parseRecord(e); + + final GsonBuilder builder = new GsonBuilder(); + builder.serializeNulls(); + final Gson gson = builder.create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + if (log.isDebugEnabled()) { + log.debug("Record Event: "+logEvent(e)); + } + addServerFields(e); + matchListener.filterMatched(e); // Flag OK for filter input/parsing/out + + outputEvents.add(e); + } else { + Event exception_event = e.clone(); + Record exception_record = OuaFilter.parseExceptionRecord(exception_event); + + final GsonBuilder exception_builder = new GsonBuilder(); + exception_builder.serializeNulls(); + final Gson exception_gson = exception_builder.create(); + exception_event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, exception_gson.toJson(exception_record)); + + if (log.isDebugEnabled()) { + log.debug("Record Event: "+logEvent(exception_event)); + } + matchListener.filterMatched(exception_event); // Flag OK for filter input/parsing/out + outputEvents.add(exception_event); + } + } catch (Exception exception) { + log.error("Error parsing OUA event "+logEvent(e), exception); + e.tag(LOGSTASH_TAG_PARSE_ERROR); } - if (e.getField("message") instanceof String) { - try { - int return_code = -1; - - if (e.getField(OuaFilter.RETURN_CODE_TAG) instanceof Long) { - return_code = Integer.parseInt(e.getField(OuaFilter.RETURN_CODE_TAG).toString()); - } - - if (return_code == 0 || !OuaFilter.isLoginFailedReturnCode(return_code)) { - Record record = OuaFilter.parseRecord(e); - - final GsonBuilder builder = new GsonBuilder(); - builder.serializeNulls(); - final Gson gson = builder.create(); - e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); - - if (log.isDebugEnabled()) { - log.debug("Record Event: "+logEvent(e)); - } - matchListener.filterMatched(e); // Flag OK for filter input/parsing/out - outputEvents.add(e); - } - - if (return_code != 0) { - Event exception_event = e.clone(); - Record exception_record = OuaFilter.parseExceptionRecord(exception_event); - - final GsonBuilder exception_builder = new GsonBuilder(); - exception_builder.serializeNulls(); - final Gson exception_gson = exception_builder.create(); - exception_event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, exception_gson.toJson(exception_record)); - - if (log.isDebugEnabled()) { - log.debug("Record Event: "+logEvent(exception_event)); - } - matchListener.filterMatched(exception_event); // Flag OK for filter input/parsing/out - outputEvents.add(exception_event); - } - } catch (Exception exception) { - log.error("Error parsing OUA event "+logEvent(e), exception); - e.tag(LOGSTASH_TAG_PARSE_ERROR); - } - } } // Remove skipped events from reaching output @@ -143,127 +190,163 @@ public Collection filter(Collection events, FilterMatchListener ma return outputEvents; } + private void addServerFields(Event e) { + if (e.includes(OuaFilter.SERVER_ADDRESS_TAG)) { + e.setField(OuaFilter.SERVER_HOST_TAG, e.getField(OuaFilter.SERVER_ADDRESS_TAG)); + } + if (e.includes(OuaFilter.SERVER_PORT_TAG)) { + e.setField(OuaFilter.SERVER_HOST_PORT, e.getField(OuaFilter.SERVER_PORT_TAG)); + } + } + + public static String dbServiceName(final Event event) + { + // Check if DB_NAME field exists and is not null + if (event.getField(OuaFilter.DB_NAME_TAG) instanceof String && + event.getField(OuaFilter.DB_NAME_TAG) != null && + !event.getField(OuaFilter.DB_NAME_TAG).toString().isEmpty()) + { + String dbName = event.getField(OuaFilter.DB_NAME_TAG).toString(); + + if (event.getField(OuaFilter.SERVER_ACCOUNT_ID) instanceof String && + event.getField(OuaFilter.SERVER_ACCOUNT_ID) != null && + !event.getField(OuaFilter.SERVER_ACCOUNT_ID).toString().isEmpty()) + { + String accountId = event.getField(OuaFilter.SERVER_ACCOUNT_ID).toString(); + return accountId + ":" + dbName; // Concatenate accountId and dbName with a colon separator + } + else + { + return dbName; + } + } + else + { + return OuaFilter.UNKNOWN_STRING; + } + } + + public static Record parseRecord(final Event event) throws ParseException { Record record = new Record(); - - if (event.getField(OuaFilter.DB_NAME_TAG) instanceof String) { - record.setDbName(event.getField(OuaFilter.DB_NAME_TAG).toString()); - } else { - record.setDbName(OuaFilter.UNKNOWN_STRING); - } - - if (event.getField(OuaFilter.SESSION_ID_TAG) instanceof Long) { - record.setSessionId(event.getField(OuaFilter.SESSION_ID_TAG).toString()); - } else { - record.setSessionId(OuaFilter.UNKNOWN_STRING); - } - - record.setSessionLocator(OuaFilter.parseSessionLocator(event)); + + record.setDbName(dbServiceName(event)); + + if (event.getField(OuaFilter.SESSION_ID_TAG) instanceof Long) { + record.setSessionId(event.getField(OuaFilter.SESSION_ID_TAG).toString()); + } else { + record.setSessionId(OuaFilter.UNKNOWN_STRING); + } + + record.setSessionLocator(OuaFilter.parseSessionLocator(event)); record.setAccessor(OuaFilter.parseAccessor(event)); record.setData(OuaFilter.parseData(event)); record.setException(null); - record.setTime(OuaFilter.getTime(event)); + record.setTime(OuaFilter.getTime(event)); - return record; - } + return record; + } - public static Record parseExceptionRecord(final Event event) throws ParseException { + public static Record parseExceptionRecord(final Event event) throws ParseException { Record record = new Record(); - if (event.getField(OuaFilter.DB_NAME_TAG) instanceof String) { - record.setDbName(event.getField(OuaFilter.DB_NAME_TAG).toString()); - } else { - record.setDbName(OuaFilter.UNKNOWN_STRING); - } + record.setDbName(dbServiceName(event)); - if (event.getField(OuaFilter.SESSION_ID_TAG) instanceof Long) { - record.setSessionId(event.getField(OuaFilter.SESSION_ID_TAG).toString()); - } else { - record.setSessionId(OuaFilter.UNKNOWN_STRING); - } - record.setSessionLocator(OuaFilter.parseSessionLocator(event)); + if (event.getField(OuaFilter.SESSION_ID_TAG) instanceof Long) { + record.setSessionId(event.getField(OuaFilter.SESSION_ID_TAG).toString()); + } else { + record.setSessionId(OuaFilter.UNKNOWN_STRING); + } + + record.setSessionLocator(OuaFilter.parseSessionLocator(event)); record.setAccessor(OuaFilter.parseAccessor(event)); record.setData(null); - record.setException(OuaFilter.parseException(event)); - - record.setTime(OuaFilter.getTime(event)); - - return record; - } - - private static ExceptionRecord parseException(final Event event) { - ExceptionRecord exception_record = null; - int return_code = 0; - - if (event.getField(OuaFilter.RETURN_CODE_TAG) instanceof Long) { - return_code = Integer.parseInt(event.getField(OuaFilter.RETURN_CODE_TAG).toString()); - } - - if (return_code != 0) { - exception_record = new ExceptionRecord(); - - exception_record.setDescription(String.format("ORA-%05d", return_code)); - if (OuaFilter.isLoginFailedReturnCode(return_code)) { - exception_record.setExceptionTypeId("LOGIN_FAILED"); - } else { - exception_record.setExceptionTypeId("SQL_ERROR"); - } - if (event.getField(OuaFilter.SQL_TAG) instanceof String) { - exception_record.setSqlString(event.getField(OuaFilter.SQL_TAG).toString()); - } else { - exception_record.setSqlString(OuaFilter.UNKNOWN_STRING); - } - } - - return exception_record; - } - - private static Boolean isLoginFailedReturnCode(int code) { - if (code == 1017 || code == 1004 || code == 1005 || - code == 1040 || code == 1045 || code == 1988 || - code == 12317 || code == 1267 || code == 28000 || - code == 28001 || code == 28030 || code == 28273 || - code == 28009) { - return true; - } else { - return false; - } - } - - private static Data parseData(final Event event) { - Data data = new Data(); - - if (event.getField(OuaFilter.SQL_TAG) instanceof String) { - data.setOriginalSqlCommand(event.getField(OuaFilter.SQL_TAG).toString()); - } else { - data.setOriginalSqlCommand(OuaFilter.UNKNOWN_STRING); - } - - return data; - } - - private static Time getTime(final Event event) { - if (event.getField(OuaFilter.TIMESTAMP_TAG) instanceof String) { - String[] timestamp_parts = event.getField(OuaFilter.TIMESTAMP_TAG).toString().split(" "); - - if (timestamp_parts.length != 2) { - return new Time(0, 0, 0); - } - - LocalTime time = LocalTime.parse(timestamp_parts[1], DATE_TIME_FORMATTER); - long millis = ((Long.parseLong(timestamp_parts[0].substring(1)) * 24) + time.getHour()) * 60 * 60; - millis += (time.getMinute() * 60) + time.getSecond(); - millis *= 1000; - millis += time.getNano() / 1000000; - - - return new Time(millis, 0, 0); - } else { - return new Time(0, 0, 0); - } - } + record.setException(OuaFilter.parseException(event)); + + record.setTime(OuaFilter.getTime(event)); + + return record; + } + + private static ExceptionRecord parseException(final Event event) { + ExceptionRecord exception_record = null; + int return_code = 0; + + if (event.getField(OuaFilter.RETURN_CODE_TAG) instanceof Long) { + return_code = Integer.parseInt(event.getField(OuaFilter.RETURN_CODE_TAG).toString()); + } + + if (return_code != 0) { + exception_record = new ExceptionRecord(); + + exception_record.setDescription(String.format("ORA-%05d", return_code)); + if (OuaFilter.isLoginFailedReturnCode(return_code)) { + exception_record.setExceptionTypeId("LOGIN_FAILED"); + } else { + exception_record.setExceptionTypeId("SQL_ERROR"); + } + if (event.getField(OuaFilter.SQL_TAG) instanceof String) { + exception_record.setSqlString(event.getField(OuaFilter.SQL_TAG).toString()); + } else { + exception_record.setSqlString(OuaFilter.UNKNOWN_STRING); + } + } + + return exception_record; + } + + private static Boolean isLoginFailedReturnCode(int code) { + if (code == 1017 || code == 1004 || code == 1005 || + code == 1040 || code == 1045 || code == 1988 || + code == 12317 || code == 1267 || code == 28000 || + code == 28001 || code == 28030 || code == 28273 || + code == 28009) { + return true; + } else { + return false; + } + } + + private static Data parseData(final Event event) { + Data data = new Data(); + + if (event.getField(OuaFilter.SQL_TAG) instanceof String) { + data.setOriginalSqlCommand(event.getField(OuaFilter.SQL_TAG).toString()); + } else { + data.setOriginalSqlCommand(OuaFilter.UNKNOWN_STRING); + } + + return data; + } + + private static Time getTime(final Event event) { + if (event.getField(OuaFilter.TIMESTAMP_TAG) instanceof String) { + String[] timestamp_parts = event.getField(OuaFilter.TIMESTAMP_TAG).toString().split(" "); + + if (timestamp_parts.length != 2) { + return new Time(0, 0, 0); + } + + LocalTime time = LocalTime.parse(timestamp_parts[1], DATE_TIME_FORMATTER); + long millis = ((Long.parseLong(timestamp_parts[0].substring(1)) * 24) + time.getHour()) * 60 * 60; + millis += (time.getMinute() * 60) + time.getSecond(); + millis *= 1000; + millis += time.getNano() / 1000000; + + + return new Time(millis, 0, 0); + } + // case of timestamp arrive as unix time in millis event_timestamp_utc like 1719231526755 + Object timestampUtcObj = event.getField(TIMESTAMP_UTC_UPPER_TAG); + if(timestampUtcObj instanceof Long) { + return new Time((Long) timestampUtcObj, 0, 0); + } + // case we didn't receive any timestamp we set to default value 1970 + return new Time(0, 0, 0); + + } private static SessionLocator parseSessionLocator(final Event event) { SessionLocator sessionLocator = new SessionLocator(); @@ -274,78 +357,74 @@ private static SessionLocator parseSessionLocator(final Event event) { sessionLocator.setServerIp(OuaFilter.UNKNOWN_STRING); sessionLocator.setServerIpv6(OuaFilter.UNKNOWN_STRING); - if (event.getField(OuaFilter.CLIENT_IP_TAG) instanceof String) { - String address = event.getField(OuaFilter.CLIENT_IP_TAG).toString(); - if (Util.isIPv6(address)) { - sessionLocator.setIpv6(true); - sessionLocator.setClientIpv6(address); - } else { - sessionLocator.setClientIp(address); - } - } - if (event.getField(OuaFilter.SERVER_IP_TAG) instanceof String) { - String address = event.getField(OuaFilter.SERVER_IP_TAG).toString(); - if (Util.isIPv6(address)) { - sessionLocator.setIpv6(true); - sessionLocator.setServerIpv6(address); - } else { - sessionLocator.setServerIp(address); - } - } - - if (event.getField(OuaFilter.SERVER_HOST_PORT) instanceof String) { - sessionLocator.setServerPort(Integer.parseInt(event.getField(OuaFilter.SERVER_HOST_PORT).toString())); - } else { - sessionLocator.setServerPort(-1); - } - - - return sessionLocator; + if (event.getField(OuaFilter.CLIENT_IP_TAG) instanceof String) { + String address = event.getField(OuaFilter.CLIENT_IP_TAG).toString(); + if (Util.isIPv6(address)) { + sessionLocator.setIpv6(true); + sessionLocator.setClientIpv6(address); + } else { + sessionLocator.setClientIp(address); + } + } + if (event.getField(OuaFilter.SERVER_IP_TAG) instanceof String) { + String address = event.getField(OuaFilter.SERVER_IP_TAG).toString(); + if (Util.isIPv6(address)) { + sessionLocator.setIpv6(true); + sessionLocator.setServerIpv6(address); + } else { + sessionLocator.setServerIp(address); + } + } + + if (event.getField(OuaFilter.SERVER_HOST_PORT) instanceof String) { + sessionLocator.setServerPort(Integer.parseInt(event.getField(OuaFilter.SERVER_HOST_PORT).toString())); + } else { + sessionLocator.setServerPort(-1); + } + + + return sessionLocator; } - private static Accessor parseAccessor(final Event event) { + private static Accessor parseAccessor(final Event event) { Accessor accessor = new Accessor(); - accessor.setLanguage(OuaFilter.LANGUAGE_STRING); + accessor.setLanguage(OuaFilter.LANGUAGE_STRING); accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); accessor.setDbProtocol(OuaFilter.DATA_PROTOCOL_STRING); accessor.setServerType(OuaFilter.SERVER_TYPE_STRING); - if (event.getField(OuaFilter.CLIENT_HOST_TAG) instanceof String) { - accessor.setClientHostName(event.getField(OuaFilter.CLIENT_HOST_TAG).toString()); - } else { - accessor.setClientHostName(OuaFilter.UNKNOWN_STRING); - } - - if (event.getField(OuaFilter.SERVER_HOST_TAG) instanceof String) { - accessor.setServerHostName(event.getField(OuaFilter.SERVER_HOST_TAG).toString()); - } else { - accessor.setServerHostName(OuaFilter.UNKNOWN_STRING); - } - - if (event.getField(OuaFilter.OS_USER_TAG) instanceof String) { - accessor.setOsUser(event.getField(OuaFilter.OS_USER_TAG).toString()); - } else { - accessor.setOsUser(OuaFilter.UNKNOWN_STRING); - } - - if (event.getField(OuaFilter.SOURCE_PROGRAM_TAG) instanceof String) { - accessor.setSourceProgram(event.getField(OuaFilter.SOURCE_PROGRAM_TAG).toString()); - } else { - accessor.setSourceProgram(OuaFilter.UNKNOWN_STRING); - } - - if (event.getField(OuaFilter.DB_USER_TAG) instanceof String) { - accessor.setDbUser(event.getField(OuaFilter.DB_USER_TAG).toString()); - } else { - accessor.setDbUser(OuaFilter.UNKNOWN_STRING); - } - - if (event.getField(OuaFilter.CON_NAME_TAG) instanceof String) { - accessor.setServiceName(event.getField(OuaFilter.CON_NAME_TAG).toString()); - } else { - accessor.setServiceName(OuaFilter.UNKNOWN_STRING); - } + if (event.getField(OuaFilter.CLIENT_HOST_TAG) instanceof String) { + accessor.setClientHostName(event.getField(OuaFilter.CLIENT_HOST_TAG).toString()); + } else { + accessor.setClientHostName(OuaFilter.UNKNOWN_STRING); + } + + if (event.getField(OuaFilter.SERVER_HOST_TAG) instanceof String) { + accessor.setServerHostName(event.getField(OuaFilter.SERVER_HOST_TAG).toString()); + } else { + accessor.setServerHostName(OuaFilter.UNKNOWN_STRING); + } + + if (event.getField(OuaFilter.OS_USER_TAG) instanceof String) { + accessor.setOsUser(event.getField(OuaFilter.OS_USER_TAG).toString()); + } else { + accessor.setOsUser(OuaFilter.UNKNOWN_STRING); + } + + if (event.getField(OuaFilter.SOURCE_PROGRAM_TAG) instanceof String) { + accessor.setSourceProgram(event.getField(OuaFilter.SOURCE_PROGRAM_TAG).toString()); + } else { + accessor.setSourceProgram(OuaFilter.UNKNOWN_STRING); + } + + if (event.getField(OuaFilter.DB_USER_TAG) instanceof String) { + accessor.setDbUser(event.getField(OuaFilter.DB_USER_TAG).toString()); + } else { + accessor.setDbUser(OuaFilter.UNKNOWN_STRING); + } + + accessor.setServiceName(dbServiceName(event)); return accessor; } @@ -380,4 +459,65 @@ public Collection> configSchema() { public String getId() { return this.id; } -} + + public static void processConnectMessage(Event event) { + String message = (String) event.getField(MESSAGE); + JsonObject payload = JsonParser.parseString(message).getAsJsonObject(); + if(payload.has(PAYLOAD)) { + payload = payload.get(PAYLOAD).getAsJsonObject(); + } + + + if(payload.has(CLIENT_IP_UPPER_TAG) && !payload.get(CLIENT_IP_UPPER_TAG).isJsonNull()) { + String clientIp = payload.get(CLIENT_IP_UPPER_TAG).getAsString(); + event.setField(CLIENT_IP_TAG, clientIp); + } + if(payload.has(CLIENT_HOST_UPPER_TAG) && !payload.get(CLIENT_HOST_UPPER_TAG).isJsonNull()) { + String clientHost = payload.get(CLIENT_HOST_UPPER_TAG).getAsString(); + event.setField(CLIENT_HOST_TAG, clientHost); + } + if(payload.has(SERVER_IP_UPPER_TAG) && !payload.get(SERVER_IP_UPPER_TAG).isJsonNull()) { + String serverIp = payload.get(SERVER_IP_UPPER_TAG).getAsString(); + event.setField(SERVER_IP_TAG, serverIp); + } + if(payload.has(SOURCE_PROGRAM_UPPER_TAG) && !payload.get(SOURCE_PROGRAM_UPPER_TAG).isJsonNull()) { + String programName = payload.get(SOURCE_PROGRAM_UPPER_TAG).getAsString(); + event.setField(SOURCE_PROGRAM_TAG, programName); + } + if(payload.has(CON_NAME_UPPER_TAG) && !payload.get(CON_NAME_UPPER_TAG).isJsonNull()) { + String conName = payload.get(CON_NAME_UPPER_TAG).getAsString(); + event.setField(CON_NAME_TAG, conName); + } + if(payload.has(DB_NAME_UPPER_TAG) && !payload.get(DB_NAME_UPPER_TAG).isJsonNull()) { + String dbName = payload.get(DB_NAME_UPPER_TAG).getAsString(); + event.setField(DB_NAME_TAG, dbName); + } + if(payload.has(OS_USER_UPPER_TAG) && !payload.get(OS_USER_UPPER_TAG).isJsonNull()) { + String osUser = payload.get(OS_USER_UPPER_TAG).getAsString(); + event.setField(OS_USER_TAG, osUser); + } + if(payload.has(DB_USER_UPPER_TAG) && !payload.get(DB_USER_UPPER_TAG).isJsonNull() ) { + String userid = payload.get(DB_USER_UPPER_TAG).getAsString(); + event.setField(DB_USER_TAG, userid); + } + if(payload.has(RETURN_CODE_UPPER_TAG) && !payload.get(RETURN_CODE_UPPER_TAG).isJsonNull()) { + long returnCode = payload.get(RETURN_CODE_UPPER_TAG).getAsLong(); + event.setField(RETURN_CODE_TAG, returnCode); + } + if(payload.has(SESSION_ID_UPPER_TAG) && !payload.get(SESSION_ID_UPPER_TAG).isJsonNull()) { + long sessionId = payload.get(SESSION_ID_UPPER_TAG).getAsLong(); + event.setField(SESSION_ID_TAG, sessionId); + } + if(payload.has(SQL_UPPER_TAG) && !payload.get(SQL_UPPER_TAG).isJsonNull()) { + String sqlText = payload.get(SQL_UPPER_TAG).getAsString(); + // remove null character from string + sqlText = StringUtils.replace(sqlText, "\u0000", ""); + event.setField(SQL_TAG, sqlText); + } + //EVENT_TIMESTAMP_UTC + if(payload.has(TIMESTAMP_UTC_UPPER_TAG) && !payload.get(TIMESTAMP_UTC_UPPER_TAG).isJsonNull()) { + long eventTimeStamp = payload.get(TIMESTAMP_UTC_UPPER_TAG).getAsLong(); + event.setField(TIMESTAMP_UTC_UPPER_TAG, eventTimeStamp); + } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-oua-guardium/src/test/java/com/ibm/guardium/OuaFilterTest.java b/filter-plugin/logstash-filter-oua-guardium/src/test/java/com/ibm/guardium/OuaFilterTest.java index 2f146c17a..bb979ed59 100644 --- a/filter-plugin/logstash-filter-oua-guardium/src/test/java/com/ibm/guardium/OuaFilterTest.java +++ b/filter-plugin/logstash-filter-oua-guardium/src/test/java/com/ibm/guardium/OuaFilterTest.java @@ -3,7 +3,9 @@ import co.elastic.logstash.api.Context; import co.elastic.logstash.api.Event; import co.elastic.logstash.api.FilterMatchListener; +import com.google.gson.Gson; import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; import org.junit.Assert; import org.junit.Test; import org.logstash.plugins.ContextImpl; @@ -11,13 +13,10 @@ import java.util.Collections; import java.util.concurrent.atomic.AtomicInteger; - - public class OuaFilterTest { @Test public void testParse() { final String msg = "{\"sessionid\":373114511,\"proxy_sessionid\":0,\"instance_id\":1,\"instance_name\":\"on8crh7u\",\"authentication_type\":\"(TYPE=(DATABASE));(CLIENT ADDRESS=((ADDRESS=(PROTOCOL=tcp)(HOST=9.70.157.206)(PORT=46511))));\",\"userid\":\"SYSTEM\",\"os_user\":\"oracle18\",\"current_user\":\"SYSTEM\",\"client_host_ip\":\"9.70.157.206\",\"client_host_name\":\"rh7u1x64t-ktap.guard.swg.usma.ibm.com\",\"server_host_ip\":\"9.70.157.206\",\"server_host_name\":\"rh7u1x64t-ktap.guard.swg.usma.ibm.com\",\"client_program_name\":\"sqlplus@rh7u1x64t-ktap.guard.swg.usma.ibm.com (\",\"terminal\":\"pts/1\",\"os_process\":\"15767\",\"dbid\":2287733986,\"dbname\":\"ON8CRH7U\",\"statement_id\":14,\"return_code\":0,\"server_nls_characterset_id\":873,\"server_nls_nchar_characterset_id\":2000,\"event_timestamp\":\"+000017794 22:56:57.111795\",\"sql_text\":\" alter user QA_TEST default role ALL\"}"; - final long session_id = 373114511; final String db_name = "ON8CRH7U"; final String db_user = "SYSTEM"; @@ -30,6 +29,7 @@ public void testParse() { final String timestamp = "+000017794 22:56:57.111795"; final String sql_text = " alter user QA_TEST default role ALL"; final int server_port = 1521; + final int return_code =0; @@ -38,6 +38,61 @@ public void testParse() { TestMatchListener matchListener = new TestMatchListener(); Event e = new org.logstash.Event(); + e.setField(OuaFilter.MESSAGE, msg); + e.setField(OuaFilter.SESSION_ID_TAG, session_id); + e.setField(OuaFilter.DB_NAME_TAG, db_name); + e.setField(OuaFilter.DB_USER_TAG, db_user); + e.setField(OuaFilter.OS_USER_TAG, os_user); + e.setField(OuaFilter.CLIENT_IP_TAG, client_ip); + e.setField(OuaFilter.SERVER_IP_TAG, server_ip); + e.setField(OuaFilter.CLIENT_HOST_TAG, client_host); + e.setField(OuaFilter.SERVER_HOST_TAG, server_host); + e.setField(OuaFilter.SERVER_HOST_PORT, server_port); + e.setField(OuaFilter.SOURCE_PROGRAM_TAG, source_program); + e.setField(OuaFilter.TIMESTAMP_TAG, timestamp); + e.setField(OuaFilter.SQL_TAG, sql_text); + e.setField(OuaFilter.RETURN_CODE_TAG,return_code); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + + Record record = new Gson().fromJson((String)e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), Record.class); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + + Assert.assertEquals(1, matchListener.getMatchCount()); + Assert.assertEquals("ON8CRH7U", record.getDbName()); + Assert.assertEquals("ON8CRH7U", record.getAccessor().getServiceName()); + + } + + @Test + public void testParseCloud() { + final String msg = "{\"sessionid\":373114511,\"proxy_sessionid\":0,\"instance_id\":1,\"instance_name\":\"on8crh7u\",\"authentication_type\":\"(TYPE=(DATABASE));(CLIENT ADDRESS=((ADDRESS=(PROTOCOL=tcp)(HOST=9.70.157.206)(PORT=46511))));\",\"userid\":\"SYSTEM\",\"os_user\":\"oracle18\",\"current_user\":\"SYSTEM\",\"client_host_ip\":\"9.70.157.206\",\"client_host_name\":\"rh7u1x64t-ktap.guard.swg.usma.ibm.com\",\"server_host_ip\":\"9.70.157.206\",\"server_host_name\":\"rh7u1x64t-ktap.guard.swg.usma.ibm.com\",\"client_program_name\":\"sqlplus@rh7u1x64t-ktap.guard.swg.usma.ibm.com (\",\"terminal\":\"pts/1\",\"os_process\":\"15767\",\"dbid\":2287733986,\"dbname\":\"testdb\",\"statement_id\":14,\"return_code\":0,\"server_nls_characterset_id\":873,\"server_nls_nchar_characterset_id\":2000,\"event_timestamp\":\"+000017794 22:56:57.111795\",\"sql_text\":\" alter user QA_TEST default role ALL\"}"; + + final long session_id = 373114511; + final String db_name = "testdb"; + final String db_user = "SYSTEM"; + final String os_user = "oracle18"; + final String client_ip = "9.70.157.206"; + final String server_ip = "9.70.157.206"; + final String client_host = "rh7u1x64t-ktap.guard.swg.usma.ibm.com"; + final String server_host = "rh7u1x64t-ktap.guard.swg.usma.ibm.com"; + final String source_program = "sqlplus@rh7u1x64t-ktap.guard.swg.usma.ibm.com ("; + final String timestamp = "+000017794 22:56:57.111795"; + final String sql_text = " alter user QA_TEST default role ALL"; + final int server_port = 1521; + final String account_id = "346824953529"; + final int return_code =0; + + + + Context context = new ContextImpl(null, null); + OuaFilter filter = new OuaFilter("test-id", null, context); + TestMatchListener matchListener = new TestMatchListener(); + Event e = new org.logstash.Event(); + e.setField("message", msg); e.setField(OuaFilter.SESSION_ID_TAG, session_id); e.setField(OuaFilter.DB_NAME_TAG, db_name); @@ -51,13 +106,121 @@ public void testParse() { e.setField(OuaFilter.SOURCE_PROGRAM_TAG, source_program); e.setField(OuaFilter.TIMESTAMP_TAG, timestamp); e.setField(OuaFilter.SQL_TAG, sql_text); + e.setField(OuaFilter.SERVER_ACCOUNT_ID,account_id); + e.setField(OuaFilter.RETURN_CODE_TAG,return_code); Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(2, results.size()); + Assert.assertEquals(1, results.size()); Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(2, matchListener.getMatchCount()); + + Record record = new Gson().fromJson((String)e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), Record.class); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + + Assert.assertEquals(1, matchListener.getMatchCount()); + Assert.assertEquals("346824953529:testdb", record.getDbName()); + Assert.assertEquals("346824953529:testdb", record.getAccessor().getServiceName()); + } + @Test + public void testParseJdbcConnect() { + final String msg = "{\"payload\":{\"CLIENT_HOST_IP\":\"9.147.163.45\",\"CLIENT_HOST_NAME\":\"dors-mbp.givatayim.il.ibm.com\",\"CLIENT_PROGRAM_NAME\":\"DBeaver 24?1?0 ? SQLEditor ?Script?16?sql?\",\"CON_NAME\":\"ON9PRH8X\",\"DBID\":\"1440598309\",\"DBNAME\":\"ON9CRH8X\",\"EVENT_TIMESTAMP\":\"+000019898 12:18:46.755743\",\"EVENT_TIMESTAMP_UTC\":1719231526755,\"OS_USER\":\"il017920\",\"RETURN_CODE\":\"0\",\"SERVER_HOST_IP\":\"9.46.226.158\",\"SESSIONID\":\"782118933\",\"SQL_TEXT\":\"INSERT INTO DOR_TEST VALUES ('97', 'aa', 'aa', 'aa', 'aa')\\u0000\",\"USERID\":\"SCOTT\"},\"schema\":{\"fields\":[{\"field\":\"EVENT_TIMESTAMP_UTC\",\"name\":\"org.apache.kafka.connect.data.Timestamp\",\"optional\":true,\"type\":\"int64\",\"version\":1},{\"field\":\"EVENT_TIMESTAMP\",\"optional\":true,\"type\":\"string\"},{\"field\":\"SESSIONID\",\"optional\":true,\"type\":\"string\"},{\"field\":\"USERID\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CON_NAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"OS_USER\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CLIENT_HOST_IP\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CLIENT_HOST_NAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"SERVER_HOST_IP\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CLIENT_PROGRAM_NAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"DBID\",\"optional\":true,\"type\":\"string\"},{\"field\":\"DBNAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"RETURN_CODE\",\"optional\":true,\"type\":\"string\"},{\"field\":\"SQL_TEXT\",\"optional\":true,\"type\":\"string\"}],\"optional\":false,\"type\":\"struct\"}}"; + + Context context = new ContextImpl(null, null); + OuaFilter filter = new OuaFilter("test-id", null, context); + TestMatchListener matchListener = new TestMatchListener(); + Event e = new org.logstash.Event(); + + e.setField(OuaFilter.SERVER_ADDRESS_TAG, "auto-oua-01.cxmxwdil903v.us-west-1.rds.amazonaws.com"); + e.setField(OuaFilter.SERVER_HOST_PORT, 1521L); + e.setField("message", msg); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + + Record record = new Gson().fromJson((String)e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), Record.class); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + + // verify \u0000 removed from sql text + Assert.assertEquals("INSERT INTO DOR_TEST VALUES ('97', 'aa', 'aa', 'aa', 'aa')", record.getData().getOriginalSqlCommand()); + + //verify time + Assert.assertEquals(1719231526755L, record.getTime().getTimstamp()); + + Assert.assertEquals(1521L, e.getField(OuaFilter.SERVER_HOST_PORT)); + Assert.assertEquals("auto-oua-01.cxmxwdil903v.us-west-1.rds.amazonaws.com", e.getField(OuaFilter.SERVER_HOST_TAG)); + + Assert.assertEquals(1, matchListener.getMatchCount()); + Assert.assertEquals("ON9CRH8X", record.getDbName()); + Assert.assertEquals("ON9CRH8X", record.getAccessor().getServiceName()); + } + + @Test + public void testParseJdbcConnect_userid_null() { + final String msg = "{\"payload\":{\"CLIENT_HOST_IP\":\"9.147.163.45\",\"CLIENT_HOST_NAME\":\"dors-mbp.givatayim.il.ibm.com\",\"CLIENT_PROGRAM_NAME\":\"DBeaver 24?1?0 ? SQLEditor ?Script?16?sql?\",\"CON_NAME\":\"ON9PRH8X\",\"DBID\":\"1440598309\",\"DBNAME\":\"ON9CRH8X\",\"EVENT_TIMESTAMP\":\"+000019898 12:18:46.755743\",\"EVENT_TIMESTAMP_UTC\":1719231526755,\"OS_USER\":\"il017920\",\"RETURN_CODE\":\"0\",\"SERVER_HOST_IP\":\"9.46.226.158\",\"SESSIONID\":\"782118933\",\"SQL_TEXT\":\"INSERT INTO DOR_TEST VALUES ('97', 'aa', 'aa', 'aa', 'aa')\\u0000\",\"USERID\":null},\"schema\":{\"fields\":[{\"field\":\"EVENT_TIMESTAMP_UTC\",\"name\":\"org.apache.kafka.connect.data.Timestamp\",\"optional\":true,\"type\":\"int64\",\"version\":1},{\"field\":\"EVENT_TIMESTAMP\",\"optional\":true,\"type\":\"string\"},{\"field\":\"SESSIONID\",\"optional\":true,\"type\":\"string\"},{\"field\":\"USERID\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CON_NAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"OS_USER\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CLIENT_HOST_IP\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CLIENT_HOST_NAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"SERVER_HOST_IP\",\"optional\":true,\"type\":\"string\"},{\"field\":\"CLIENT_PROGRAM_NAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"DBID\",\"optional\":true,\"type\":\"string\"},{\"field\":\"DBNAME\",\"optional\":true,\"type\":\"string\"},{\"field\":\"RETURN_CODE\",\"optional\":true,\"type\":\"string\"},{\"field\":\"SQL_TEXT\",\"optional\":true,\"type\":\"string\"}],\"optional\":false,\"type\":\"struct\"}}"; + + Context context = new ContextImpl(null, null); + OuaFilter filter = new OuaFilter("test-id", null, context); + TestMatchListener matchListener = new TestMatchListener(); + Event e = new org.logstash.Event(); + + // init + e.setField(OuaFilter.SERVER_ADDRESS_TAG, "auto-oua-01.cxmxwdil903v.us-west-1.rds.amazonaws.com"); + e.setField(OuaFilter.SERVER_HOST_PORT, 1521L); + e.setField("message", msg); + + // action + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + + Record record = new Gson().fromJson((String)e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), Record.class); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + + // result + Assert.assertEquals("",record.getAccessor().getDbUser()); // verify jsonNull not throw exception + } + + @Test + public void testParseJdbcConnectNoSchema() { + String msg ="{\"EVENT_TIMESTAMP_UTC\":1733051171821,\"SESSIONID\":\"1276239794\",\"USERID\":\"SCOTT\",\"CON_NAME\":\"ON9PDBQA\",\"OS_USER\":\"il017920\",\"CLIENT_HOST_IP\":\"9.147.173.67\",\"CLIENT_HOST_NAME\":\"dors-mbp.givatayim.il.ibm.com\",\"SERVER_HOST_IP\":\"9.46.92.230\",\"CLIENT_PROGRAM_NAME\":\"SQL Developer\",\"DBID\":\"2505005351\",\"DBNAME\":\"ON9CDBQA\",\"RETURN_CODE\":\"0\",\"SQL_TEXT\":\"insert into persons values ('first3', 'last3')\\u0000\"}"; + + Context context = new ContextImpl(null, null); + OuaFilter filter = new OuaFilter("test-id", null, context); + TestMatchListener matchListener = new TestMatchListener(); + Event e = new org.logstash.Event(); + + e.setField(OuaFilter.SERVER_ADDRESS_TAG, "uc.test.demo-server.ibm.com"); + e.setField(OuaFilter.SERVER_HOST_PORT, 1521L); + e.setField("message", msg); + e.setField("source_system", "kafka-connect"); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + + Record record = new Gson().fromJson((String)e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), Record.class); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + + // verify \u0000 removed from sql text + Assert.assertEquals("insert into persons values ('first3', 'last3')", record.getData().getOriginalSqlCommand()); + + //verify time + Assert.assertEquals(1733051171821L, record.getTime().getTimstamp()); + + Assert.assertEquals(1521L, e.getField(OuaFilter.SERVER_HOST_PORT)); + Assert.assertEquals("uc.test.demo-server.ibm.com", e.getField(OuaFilter.SERVER_HOST_TAG)); + + Assert.assertEquals(1, matchListener.getMatchCount()); + Assert.assertEquals("ON9CDBQA", record.getDbName()); + Assert.assertEquals("ON9CDBQA", record.getAccessor().getServiceName()); + } + + } class TestMatchListener implements FilterMatchListener {