GRD-106316 Adding the missing dependencies to build/lib

.github/CODEOWNERS

@@ -5,5 +5,4 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-*       @devanshah2 @Tal-Daniel @ofer-haim @natalygmkibm @apurva-birajdar @itai-g-weather-com @RefaelAdi @PratikshaSonawane @mwnovak-ibm @hadarkorny @doryo @chirag-ibm @rasikashete3 @pankajkumaribm @taees-eimouri @pankajkumar @piyush-desai-ibm @ShalakaKulkarni15 @JingqiuDu
-
+*       @devanshah2 @Tal-Daniel @ofer-haim @natalygmkibm @apurva-birajdar @itai-g-weather-com @RefaelAdi @PratikshaSonawane @mwnovak-ibm @hadarkorny @doryo @chirag-ibm @rasikashete3 @pankajkumaribm @taees-eimouri @pankajkumar @piyush-desai-ibm @ShalakaKulkarni15 @JingqiuDu @rupathil @zeeIBM @Rose-Kaur

build/defaultOfflinePackagePlugins.txt

@@ -36,9 +36,11 @@ filter-plugin/logstash-filter-intersystems-iris-guardium/logstash-filter-intersy
 filter-plugin/logstash-filter-postgres-ibmcloud-guardium/logstash-filter-icd_postgresql_guardium_filter
 filter-plugin/logstash-filter-mysql-azure-guardium/logstash-filter-azure_mysql_guardium_filter
 filter-plugin/logstash-filter-scylldb-guardium/logstash-filter-scylladb_guardium_filter
+filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks_guardium_filter
+filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_filter
+filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella_guardium_filter
+filter-plugin/logstash-filter-opensearch-guardium/logstash-filter-opensearch_guardium_filter
+input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input
+input-plugin/logstash-input-http/logstash-offline-input-http-plugins
 input-plugin/logstash-input-mongo-atlas/logstash-input-mongo_atlas_input
-input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase-capella_input
-input-plugin/logstash-input-http/logstash-input-http_input
-filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella-guardium_filter
-filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks-guardium_filter
-filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino-guardium_filter
+

build/pluginsToBuild_GDP.txt

@@ -36,9 +36,10 @@ filter-plugin/logstash-filter-intersystems-iris-guardium
 filter-plugin/logstash-filter-postgres-ibmcloud-guardium
 filter-plugin/logstash-filter-mysql-azure-guardium
 filter-plugin/logstash-filter-scylldb-guardium
-input-plugin/logstash-input-mongo-atlas
+filter-plugin/logstash-filter-databricks-guardium
+filter-plugin/logstash-filter-trino-guardium
+filter-plugin/logstash-filter-capella-guardium
+filter-plugin/logstash-filter-opensearch-guardium
 input-plugin/logstash-input-couchbase-capella
 input-plugin/logstash-input-http
-filter-plugin/logstash-filter-capella-guardium
-filter-plugin/logstash-filter-databricks-guardium
-filter-plugin/logstash-filter-trino-guardium
+input-plugin/logstash-input-mongo-atlas

build/verifiedUCPlugins_gdp.txt

@@ -43,12 +43,18 @@ filter-plugin/logstash-filter-mysql-guardium/MySQLOverSyslogPackage
 #Other
 filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage
 filter-plugin/logstash-filter-azure-postgresql-guardium/AzurePostgresqlOverAzureEventHub
+filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub
+filter-plugin/logstash-filter-trino-guardium/TrinoOverSyslogPackage
+filter-plugin/logstash-filter-capella-guardium/CapellaCouchbaseOverCapellaPackage
+filter-plugin/logstash-filter-opensearch-guardium/OpenSearchOverCloudwatchPackage
 #Input plug-ins
 input-plugin/logstash-input-azure-event-hubs/AzureEventHubsInputPackage
 input-plugin/logstash-input-beats/FilebeatInputPackage
+input-plugin/logstash-input-couchbase-capella/InputCouchbaseCapellaPackage
 input-plugin/logstash-input-cloudwatch-logs/CloudwatchLogsInputPackage
 input-plugin/logstash-input-jdbc/JdbcInputPackage
 input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage
 input-plugin/logstash-input-sqs/SQSInputPackage
 input-plugin/logstash-input-tcp-syslog/SyslogInputPackage
-input-plugin/logstash-input-google-pubsub/GooglePubSubPackage
+input-plugin/logstash-input-http/httpInputPackage
+input-plugin/logstash-input-google-pubsub/GooglePubSubPackage

common/build.gradle

@@ -64,3 +64,11 @@ artifacts {
      archives sourcesJar
      archives javadocJar
 }
+
+task copyDependencies(type: Copy) {
+    description 'Copies all runtime dependencies into build/libs directory'
+    from configurations.runtimeClasspath
+    into "${buildDir}/libs"
+}
+
+jar.finalizedBy(copyDependencies)

filter-plugin/logstash-filter-azure-sql-guardium/README.md

@@ -108,7 +108,7 @@ The Guardium universal connector is the Guardium entry point for native audit lo
 
 **Note**: For Guardium Data Protection version 11.4 without appliance bundle 11.0p490 or prior or Guardium Data Protection version 11.5 without appliance bundle 11.0p540 or prior, download the [Azure-SQL-Offline-Package.zip](https://github.com/IBM/universal-connectors/releases/download/v1.5.6/logstash-filter-azuresql_guardium_plugin_filter.zip) plug-in. (Do not unzip the offline-package file throughout the procedure).
 
-• Download the mssql-jdbc-7.4.1.jre8 from [here](https://jar-download.com/artifacts/com.microsoft.sqlserver/mssql-jdbc/7.4.1.jre8)
+• Download the mssql-jdbc-7.4.1.jre8 from [here](https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/7.4.1.jre8/mssql-jdbc-7.4.1.jre8.jar)
 
 #### Configuration
 

filter-plugin/logstash-filter-capella-guardium/README.md

@@ -118,7 +118,7 @@ enforcements. Configure Guardium to read the native audit/data_access logs by cu
   the  [logstash-input-couchbase_capella_input](../../input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input.zip)
   plug-in.
 * Download
-  the [logstash-filter-capella_guardium_filter](capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip)
+  the [logstash-filter-capella_guardium_filter](logstash-filter-capella_guardium_filter.zip)
   plug-in.
 * Capella-Guardium Logstash filter plug-in is automatically available with Guardium Data Protection versions 12.x, 11.4
   with appliance bundle 11.0p490 or later or Guardium Data Protection version 11.5 with appliance bundle 11.0p540 or
@@ -129,15 +129,15 @@ enforcements. Configure Guardium to read the native audit/data_access logs by cu
 1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```.
 2. Enable the universal connector if it is disabled.
 3. Click ```Upload File``` and select the
-   offline  [logstash-filter-capella_guardium_filter](capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip)
+   offline  [logstash-filter-capella_guardium_filter](logstash-filter-capella_guardium_filter.zip)
    plug-in. After it is uploaded, click ```OK```.
 4. Click the Plus sign to open the Connector Configuration dialog box.
 5. Type a name in the Connector name field.
 6. Update the input section to add the details from
-   the [capellaCouchbase.conf](capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf) file's input part,
+   the [capellaCouchbase.conf](CapellaCouchbaseOverCapellaPackage/capellaCouchbase.conf) file's input part,
    omitting the keyword "input{" at the beginning and its corresponding "}" at the end.
 7. Update the filter section to add the details from
-   the [capellaCouchbase.conf](capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf) file's filter part,
+   the [capellaCouchbase.conf](CapellaCouchbaseOverCapellaPackage/capellaCouchbase.conf) file's filter part,
    omitting the keyword "filter{" at the beginning and its corresponding "}" at the end.
 8. The 'type' fields should match in the input and filter configuration sections. This field should be unique for every
    individual connector added.

filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/Parser.java

@@ -130,7 +130,7 @@ protected ExceptionRecord getException(String payload, String sqlString) {
       return exceptionRecord;
     } else if (statement != null && !status.contains(SUCCESS_STATUS)) {
       exceptionRecord.setDescription(serviceName);
-      exceptionRecord.setSqlString(sqlString);
+      exceptionRecord.setSqlString(statement);
       exceptionRecord.setExceptionTypeId(SQL_ERROR);
       return exceptionRecord;
     }

filter-plugin/logstash-filter-capella-guardium/src/test/java/com/ibm/guardium/capella/ParserTest.java

@@ -753,5 +753,6 @@ void testSQLError() {
     assertEquals("COUCHB", record.getAccessor().getLanguage());
     assertEquals("UNRECOGNIZED statement", record.getException().getDescription());
     assertEquals("SQL_ERROR", record.getException().getExceptionTypeId());
+    assertEquals("select * fro test;", record.getException().getSqlString());
   }
 }

filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/mongodbSyslog.conf

@@ -0,0 +1,54 @@
+#/*
+#Copyright 2020-2021 IBM Inc. All rights reserved
+#SPDX-License-Identifier: Apache-2.0
+#*/
+
+input {
+ tcp {
+      port => 5001
+      type => "syslog-mongodb"
+      dns_reverse_lookup_enabled => false
+      ssl_enable => true
+      # ssl_certificate_authorities => SSL_CERT_AUTH
+      ssl_cert => "/service/certs/external/tls-syslog.crt"
+      ssl_key => "/service/certs/external/tls-syslog.key"
+      ssl_verify => true
+ }
+}
+
+
+
+filter {
+if [type] == "syslog-mongodb" {
+	# break apart the message and prepare for what filter expects
+	grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:server_hostname} %{SYSLOGPROG:source_program}(?:[%{POSINT:syslog_pid}])?: %{GREEDYDATA:syslog_message}" }
+	}
+
+    date {
+      match => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+	}
+
+    mutate { rename => { "host" => "server_ip" } }
+
+	# send to filter
+	mongodb_guardium_filter {}
+
+	# keep original event fields, for debugging
+	if "_mongoguardium_json_parse_error" not in [tags] {
+		mutate { remove_field => [
+			"message", "syslog_timestamp", "source_program", "program",
+			"syslog_pid", "syslog_message",
+			"server_hostname", "client_hostname", "host",
+			"ecs", "log", "agent", "input"]
+		}
+	}
+}
+
+
+# uncomment to test events/sec
+# 	metrics {
+#		meter => "events"
+#		add_tag => "metric"
+# 	}
+}

filter-plugin/logstash-filter-mongodb-guardium/MongodbOverFilebeatPackage/mongodbFilebeat.conf

@@ -0,0 +1,36 @@
+#/*
+#Copyright 2020-2021 IBM Inc. All rights reserved
+#SPDX-License-Identifier: Apache-2.0
+#*/
+
+input {
+    beats {
+ 	    port => <port>
+ 	    type => "filebeat"
+ 	    # For SSL over Filebeat, uncomment the following lines after generating an SSL key and a certificate authority (CA) using GuardAPI (see documentation), copy the public certificate authority (CA) to your data source and adjust Filebeat configuration:
+ 	    #ssl => true
+ 	    #ssl_certificate => "${SSL_DIR}/cert.pem"
+ 	    #ssl_key => "${SSL_DIR}/key.pem"
+    }
+}
+filter{
+  if [type] == "filebeat" and "guc_filter_param_datasource_tag" in [tags] {
+    mutate { add_field => { "source_program" => "mongod" } }
+    mutate { add_field => { "server_hostname" => "%{[host][name]}" } }
+    mutate { add_field => { "server_ip" => "%{[host][ip][0]}" } }
+    mutate { replace => { "message" => "%{source_program}: %{message}" } }
+
+    mongodb_guardium_filter {}
+
+    if "_mongoguardium_json_parse_error" not in [tags] {
+      mutate { remove_field => ["message","syslog_timestamp","source_program","program","syslog_pid","syslog_message","server_hostname","client_hostname","host","ecs","log","agent","input"] }
+    }
+  }
+
+# uncomment to test events/sec
+# 	metrics {
+#		meter => "events"
+#		add_tag => "metric"
+# 	}
+}
+

filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage/mongodbAtlas.conf

@@ -0,0 +1,34 @@
+input {
+    mongo_atlas_input{
+        interval => 300
+        public-key => "<PUBLIC-KEY>"
+        private-key => "<PRIVATE-KEY>"
+        group-id => "<GROUP-ID>" # example, 61f8b9021d9dcc4b97fbfcf1
+        hostname => "<HOSTNAME>" # example, cluster1-shard-00-02.i2jq9.mongodb.net
+        type => "mongodbatlas"
+    }
+}
+
+filter { 
+	if [type] == "mongodbatlas" {
+     mutate { add_field => { "source_program" => "mongod" } }
+     mutate { add_field => { "client_hostname" => "%{[agent][hostname]}" } }
+     mutate { add_field => { "server_hostname" => "%{hostname}" } }
+     mutate { add_field => { "server_ip" => "%{[host][ip][0]}" } }
+         mutate { replace => { "message" => "%{source_program}: %{message}" } }
+
+         mongodb_guardium_filter {}
+
+    # keep original event fields, for debugging
+    if "_mongoguardium_json_parse_error" not in [tags] {
+    	mutate { remove_field => [
+    		"message", "syslog_timestamp",
+    		"source_program", "program",
+    		"syslog_pid", "syslog_message",
+    		"server_hostname", "client_hostname", "host",
+    		"ecs", "log", "agent", "input"]
+    	}
+        }
+    }
+}
+