From e1fb5d25db2a35ee46cd0c623fcaee31a6536f0f Mon Sep 17 00:00:00 2001 From: "Andy.Chen" Date: Mon, 25 Aug 2025 21:28:10 -0400 Subject: [PATCH 1/4] GRD-106525 fix Neo4j outofbounds exception Signed-off-by: Andy.Chen --- .../guardium/neodb/NeodbGuardiumFilter.java | 277 ++-- .../java/com/ibm/guardium/neodb/Parser.java | 1299 ++++++++--------- .../com/ibm/guardium/neodb/ParserTest.java | 583 ++++---- 3 files changed, 1094 insertions(+), 1065 deletions(-) diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java index 03deb7def..0b043b275 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java @@ -9,6 +9,7 @@ import java.text.ParseException; import java.util.Collection; import java.util.Collections; + import com.google.gson.JsonElement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,140 +32,142 @@ @LogstashPlugin(name = "neodb_guardium_filter") public class NeodbGuardiumFilter implements Filter { - private static Logger logger = LogManager.getLogger(NeodbGuardiumFilter.class); - - - public static final String LOG42_CONF = "log4j2uc.properties"; - static { - try { - String uc_etc = System.getenv("UC_ETC"); - LoggerContext context = (LoggerContext) LogManager.getContext(false); - File file = new File(uc_etc + File.separator + LOG42_CONF); - context.setConfigLocation(file.toURI()); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private String id; - public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); - private static Logger log = LogManager.getLogger(NeodbGuardiumFilter.class); - - public NeodbGuardiumFilter(String id, Configuration config, Context context) { - this.id = id; - } - - @Override - public Collection> configSchema() { - // should return a list of all configuration options for this plugin - return Collections.singletonList(SOURCE_CONFIG); - } - - @Override - public String getId() { - return this.id; - } - - @Override - public Collection filter(Collection events, FilterMatchListener matchListener) { - - for (Event e : events) { - if(logger.isDebugEnabled()){ - logger.debug("Event Now: {}", e.getData()); - } - - try { - JsonObject inputData = inputData(e); - - if (isNotSystemGeneratedEvent(inputData)) { - - Record record = Parser.parseRecord(inputData); - - final GsonBuilder builder = new GsonBuilder(); - builder.serializeNulls(); - final Gson gson = builder.create(); - e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); - - matchListener.filterMatched(e); - } else { - e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_NEO); - } - - } catch (ParseException pe) { - log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), pe); - e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); - } catch (Exception exception) { - log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), - exception); - e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); - } - } - return events; - } - - private boolean isNotSystemGeneratedEvent(JsonObject inputData) { - if (inputData == null || !inputData.has(Constants.QUERY_STATEMENT)) { - return false; // Treat as system-generated if missing QUERY_STATEMENT or inputData is null - } - - JsonElement queryElement = inputData.get(Constants.QUERY_STATEMENT); - if (queryElement == null || queryElement.isJsonNull()) { - return false; // Treat as system-generated if queryElement is null or JsonNull - } - - String queryStatement = queryElement.getAsString(); - if (queryStatement == null || queryStatement.isEmpty() || queryStatement.startsWith("EXPLAIN")) { - return false; // Treat as system-generated if queryStatement is null, empty, or starts with "EXPLAIN" - } - - return true; // Otherwise, it is not a system-generated event - } - - private JsonObject inputData(Event e) { - JsonObject data = new JsonObject(); - - if (e.getField(Constants.CLIENT_IP) != null - && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { - data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); - } - if (e.getField(Constants.SERVER_IP) != null - && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { - data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); - } - if (e.getField(Constants.DB_PROTOCOL) != null - && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { - data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); - } - if (e.getField(Constants.TIMESTAMP) != null - && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { - data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); - } - if (e.getField(Constants.LOG_LEVEL) != null - && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { - data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); - } - if (e.getField(Constants.DB_USER) != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { - data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); - } - if (e.getField(Constants.DB_NAME) != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { - data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); - } - if (e.getField(Constants.SOURCE_PROGRAM) != null - && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { - data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); - } - if (e.getField(Constants.QUERY_STATEMENT) != null - && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { - data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); - } - if (e.getField(Constants.SERVER_HOSTNAME) != null - && !e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { - data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); - } - if (e.getField(Constants.MESSAGE) != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { - data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); - } - return data; - } - -} \ No newline at end of file + private static Logger logger = LogManager.getLogger(NeodbGuardiumFilter.class); + + + public static final String LOG42_CONF = "log4j2uc.properties"; + + static { + try { + String uc_etc = System.getenv("UC_ETC"); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + File file = new File(uc_etc + File.separator + LOG42_CONF); + context.setConfigLocation(file.toURI()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private String id; + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + private static Logger log = LogManager.getLogger(NeodbGuardiumFilter.class); + Parser parser = new Parser(); + + public NeodbGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + + for (Event e : events) { + if (logger.isDebugEnabled()) { + logger.debug("Event Now: {}", e.getData()); + } + + try { + JsonObject inputData = inputData(e); + + if (isNotSystemGeneratedEvent(inputData)) { + + Record record = parser.parseRecord(inputData); + + final GsonBuilder builder = new GsonBuilder(); + builder.serializeNulls(); + final Gson gson = builder.create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + matchListener.filterMatched(e); + } else { + e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_NEO); + } + + } catch (ParseException pe) { + log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), pe); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } catch (Exception exception) { + log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), + exception); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } + } + return events; + } + + private boolean isNotSystemGeneratedEvent(JsonObject inputData) { + if (inputData == null || !inputData.has(Constants.QUERY_STATEMENT)) { + return false; // Treat as system-generated if missing QUERY_STATEMENT or inputData is null + } + + JsonElement queryElement = inputData.get(Constants.QUERY_STATEMENT); + if (queryElement == null || queryElement.isJsonNull()) { + return false; // Treat as system-generated if queryElement is null or JsonNull + } + + String queryStatement = queryElement.getAsString(); + if (queryStatement == null || queryStatement.isEmpty() || queryStatement.startsWith("EXPLAIN")) { + return false; // Treat as system-generated if queryStatement is null, empty, or starts with "EXPLAIN" + } + + return true; // Otherwise, it is not a system-generated event + } + + private JsonObject inputData(Event e) { + JsonObject data = new JsonObject(); + + if (e.getField(Constants.CLIENT_IP) != null + && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { + data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); + } + if (e.getField(Constants.SERVER_IP) != null + && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { + data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); + } + if (e.getField(Constants.DB_PROTOCOL) != null + && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { + data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); + } + if (e.getField(Constants.TIMESTAMP) != null + && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { + data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); + } + if (e.getField(Constants.LOG_LEVEL) != null + && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { + data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); + } + if (e.getField(Constants.DB_USER) != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { + data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); + } + if (e.getField(Constants.DB_NAME) != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { + data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); + } + if (e.getField(Constants.SOURCE_PROGRAM) != null + && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { + data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); + } + if (e.getField(Constants.QUERY_STATEMENT) != null + && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { + data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); + } + if (e.getField(Constants.SERVER_HOSTNAME) != null + && !e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { + data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); + } + if (e.getField(Constants.MESSAGE) != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { + data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); + } + return data; + } + +} diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java index 84a3f9952..d3993f428 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java @@ -4,6 +4,7 @@ // package com.ibm.guardium.neodb; + import java.net.InetAddress; import java.net.UnknownHostException; import java.text.ParseException; @@ -12,11 +13,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -35,688 +32,672 @@ public class Parser { - private static Logger log = LogManager.getLogger(Parser.class); + private Logger log = LogManager.getLogger(Parser.class); + + private final DateTimeFormatterBuilder dateTimeFormatterBuilder = + new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS[[XXX][X]]")); + + private final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); + + LinkedHashMap> operations = null; + LinkedHashMap variables = null; + static Character randomValue = 65; + + public Record parseRecord(final JsonObject data) throws ParseException { + Record record = new Record(); + + if (data != null) { + + record.setAppUserName(Constants.NOT_AVAILABLE); + + // DB Name + String dbName = Constants.NOT_AVAILABLE; + + if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { + dbName = data.get(Constants.DB_NAME).getAsString(); + } + + record.setDbName(dbName); + + // Time + String dateString = parseTimestamp(data); + String timeZone = null; + if (data.has(Constants.MIN_OFF)) { + timeZone = data.get(Constants.MIN_OFF).getAsString(); + } + Time time = getTime(dateString, timeZone); + + if (time != null) { + record.setTime(time); + } + // SessionLocator + record.setSessionLocator(parseSessionLocator(data)); + + // Accessor + record.setAccessor(parseAccessor(data)); + + record.setSessionId(Constants.UNKNOWN_STRING); + + // Data + if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { + record.setData(parseData(data)); + } else { + record.setException(parseException(data)); + } + + } + return record; + } - private static final DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS[[XXX][X]]")); + // -----------------------------------------------Accessor----------------- - private static final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); + public Accessor parseAccessor(JsonObject data) { + Accessor accessor = new Accessor(); - static LinkedHashMap> operations = null; - static LinkedHashMap variables = null; - static Character randomValue = 65; + accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); + accessor.setServerType(Constants.SERVER_TYPE_STRING); - public static Record parseRecord(final JsonObject data) throws ParseException { - Record record = new Record(); + String dbUser = Constants.NOT_AVAILABLE; + if (data.has(Constants.DB_USER) && data.get(Constants.DB_USER) != null) { + dbUser = data.get(Constants.DB_USER).getAsString(); + } + accessor.setDbUser(dbUser); - if (data != null) { + if (data.has(Constants.SERVER_HOSTNAME) && data.get(Constants.SERVER_HOSTNAME) != null) + accessor.setServerHostName(data.get(Constants.SERVER_HOSTNAME).getAsString()); - record.setAppUserName(Constants.NOT_AVAILABLE); + String sourceProgram = Constants.UNKNOWN_STRING; + if (data.has(Constants.SOURCE_PROGRAM) && data.get(Constants.SOURCE_PROGRAM) != null) { + sourceProgram = data.get(Constants.SOURCE_PROGRAM).getAsString(); + } + accessor.setSourceProgram(sourceProgram); - // DB Name - String dbName = Constants.NOT_AVAILABLE; + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.COMM_PROTOCOL); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + + String serviceName = Constants.NOT_AVAILABLE; + + if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { + serviceName = data.get(Constants.DB_NAME).getAsString(); + } + + accessor.setServiceName(serviceName); + + return accessor; + } + + // -----------------------------------------------SessionLocator----------------- + + public SessionLocator parseSessionLocator(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setIpv6(false); + + int clientPort = Constants.DEFAULT_PORT; + int serverPort = Constants.DEFAULT_PORT; + String clientIpAdd = Constants.NOT_AVAILABLE; + String serverIpAdd = Constants.NOT_AVAILABLE; + + if (data.has(Constants.CLIENT_IP)) { + String clientIpPortDetails = data.get(Constants.CLIENT_IP).getAsString(); + String clientIp = clientIpPortDetails.substring(7); + String[] clientIpPort = clientIp.split(":"); + clientIpAdd = clientIpPort[0]; + + if (isIPv6Address(clientIpAdd)) { + sessionLocator.setIpv6(true); + sessionLocator.setClientIpv6(clientIpAdd); + clientIpAdd = Constants.UNKNOWN_STRING; + } else { + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + } + + } + + if (data.has(Constants.SERVER_IP)) { + String serverIPPortDetails = data.get(Constants.SERVER_IP).getAsString(); + String serverIp = serverIPPortDetails.substring(7); + String[] serverIpPort = serverIp.split(":"); + serverIpAdd = serverIpPort[0]; + + + if (isIPv6Address(serverIpAdd)) { + sessionLocator.setIpv6(true); + sessionLocator.setServerIpv6(serverIpAdd); + serverIpAdd = Constants.UNKNOWN_STRING; + } else { + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + } + + } + + sessionLocator.setClientIp(clientIpAdd); + sessionLocator.setClientPort(clientPort); + sessionLocator.setServerIp(serverIpAdd); + sessionLocator.setServerPort(serverPort); + + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + + return sessionLocator; + } + + private boolean isIPv6Address(String ip) { + try { + InetAddress inetAddress = InetAddress.getByName(ip); + return inetAddress instanceof java.net.Inet6Address; + } catch (UnknownHostException e) { + return false; + } + } + + // -----------------------------------------------Timestamp----------------- + + public String parseTimestamp(final JsonObject data) { + String dateString = null; + if (data.has(Constants.TIMESTAMP)) { + dateString = data.get(Constants.TIMESTAMP).getAsString(); + } + return dateString; + } + + public Time getTime(String dateString, String timeZone) { + if (dateString != null) { + JsonObject data = new JsonObject(); + LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); + ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); + if (timeZone != null) { + offset = ZoneOffset.of(timeZone); + } + ZonedDateTime zdt = dt.atOffset(offset).toZonedDateTime(); + long millis = zdt.toInstant().toEpochMilli(); + int minOffset = zdt.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); + } + return null; + } + + // -----------------------------------------------Exception----------------- + + public ExceptionRecord parseException(JsonObject data) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + + String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); + String[] queryData = queryStatement.split("'} - "); + if (queryData.length == 1) { + queryData = queryStatement.split("} - "); + } + + String query = queryData[0].toString(); + String exceptionDes = queryData[queryData.length - 1].toString(); + exceptionRecord.setDescription(exceptionDes); + + exceptionRecord.setSqlString(query); + return exceptionRecord; + } + + // -----------------------------------------------Data Construct---------------- + + public Data parseData(JsonObject inputJSON) { + Data data = new Data(); + try { + Construct construct = parseAsConstruct(inputJSON); + if (construct != null) { + data.setConstruct(construct); + + if (construct.getFullSql() == null || construct.getFullSql().isEmpty()) { + construct.setFullSql(Constants.UNKNOWN_STRING); + } + if (construct.getRedactedSensitiveDataSql() == null || construct.redactedSensitiveDataSql.isEmpty()) { + construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); + } + } + } catch (Exception e) { + log.error("Neo4j filter: Error parsing JSon in parser" + inputJSON, e); + throw e; + } + return data; + } + + public Construct parseAsConstruct(final JsonObject data) { + try { + final Sentence sentence = parseSentence(data); + + final Construct construct = new Construct(); + construct.sentences.add(sentence); + + String queryStatement = data.get(Constants.QUERY_STATEMENT).toString(); + String[] fullSql = queryStatement.split("runtime"); + String query[] = fullSql[0].split("- \\{\\} - "); + if (query[0].isEmpty()) + construct.setFullSql(fullSql[0].substring(1).trim()); + else + construct.setFullSql(query[0].substring(1).trim()); + construct.setRedactedSensitiveDataSql(parseRedactedSensitiveDataSql(data)); + return construct; + } catch (final Exception e) { + throw e; + } + } + + protected Sentence parseSentence(final JsonObject data) { + + Sentence sentence = null; + + if (data.has(Constants.QUERY_STATEMENT)) { + String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); + + sentence = parseQuery(queryStatement.trim()); + + //For Match (n) return n + if (sentence == null) { + Sentence validSentence = new Sentence("MATCH"); + SentenceObject sentenceObject = new SentenceObject("AllNodesOrRelationships"); + sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + validSentence.getObjects().add(sentenceObject); + sentence = validSentence; + } + + } + + return sentence; + } + + Sentence parseQuery(String query) { + List gfg = + new ArrayList<>( + Arrays.asList( + Constants.ON_CREAT_ST, + Constants.ON_MATC_ST, + Constants.MATCH, + Constants.CREATE, + Constants.MERGE, + Constants.DELETE, + Constants.DETACH_DELET, + Constants.REMOVE, + Constants.SET, + Constants.RETURN, + Constants.WHERE, + Constants.FOREACH)); + + if (query.contains(Constants.ON_CREATE_SET)) + query = query.replace(Constants.ON_CREATE_SET, Constants.ON_CREAT_ST); + if (query.contains(Constants.ON_MATCH_SET)) + query = query.replace(Constants.ON_MATCH_SET, Constants.ON_MATC_ST); + if (query.contains(Constants.DETACH_DELETE)) + query = query.replace(Constants.DETACH_DELETE, Constants.DETACH_DELET); + + // find the index of all the supported operations in the query. + Map ops = new TreeMap<>(); + + for (String operation : gfg) { + + int index = 0; + + while (true) { + index = query.indexOf(operation, index); + if (index == -1) break; + ops.put(index, operation); + index++; + } + } + + // split the main query into multiple queries using index. + List queries = new ArrayList<>(); + + int i = 0; + int lastKeyValue = -1; + for (Map.Entry entry : ops.entrySet()) { + if (i == 0) { + i++; + lastKeyValue = entry.getKey(); + continue; + } + + String subquery = query.substring(lastKeyValue, entry.getKey()); + queries.add(subquery.trim()); + lastKeyValue = entry.getKey(); + } + + if (lastKeyValue != -1) { + String subquery = query.substring(lastKeyValue); + queries.add(subquery); + } + + operations = new LinkedHashMap<>(); + variables = new LinkedHashMap<>(); + + for (String queryString : queries) { + if (queryString.startsWith(Constants.RETURN) + || queryString.startsWith(Constants.WHERE) + || queryString.startsWith(Constants.FOREACH)) { + continue; + } else if (queryString.startsWith(Constants.ON_CREAT_ST)) onCreateSet(queryString); + else if (queryString.startsWith(Constants.ON_MATC_ST)) onMatchSet(queryString); + else if (queryString.startsWith(Constants.MATCH)) match(queryString); + else if (queryString.startsWith(Constants.CREATE)) create(queryString); + else if (queryString.startsWith(Constants.MERGE)) merge(queryString); + else if (queryString.startsWith(Constants.SET)) set(queryString); + else if (queryString.startsWith(Constants.DELETE)) delete(queryString); + else if (queryString.startsWith(Constants.REMOVE)) remove(queryString); + else if (queryString.startsWith(Constants.DETACH_DELET)) detachDelete(queryString); + } + + // Form the sentences out of different verb and objects + + // convert to ArrayList of key set + List alKeys = new ArrayList<>(operations.keySet()); + + Sentence originalSentence = null; + if (!operations.isEmpty()) { + int j = 1; + SentenceObject sentenceObject; + for (String keys : alKeys) { + ArrayList operation = operations.get(keys); + if (operation == null || operation.isEmpty()) continue; + + for (String string : operation) { + if (j == 1) { + originalSentence = new Sentence(string); + sentenceObject = new SentenceObject(variables.get(keys)); + if (sentenceObject.getName() == null) sentenceObject.setName(Constants.EVERYTHING); + sentenceObject.setType( + Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + originalSentence.getObjects().add(sentenceObject); + j++; + } else { + Sentence decendant = new Sentence(string); + sentenceObject = new SentenceObject(variables.get(keys)); + if (sentenceObject.getName() == null) sentenceObject.setName(Constants.EVERYTHING); + sentenceObject.setType( + Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + decendant.getObjects().add(sentenceObject); + originalSentence.getDescendants().add(decendant); + } + } + } + } + + return originalSentence; + } + + private void remove(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.REMOVE); + String alias = ""; + if (arr[1].contains(".")) { + String arrs[] = arr[1].split("\\.", 2); + alias = arrs[0].trim(); + } else if (arr[1].contains(":")) { + String arrs[] = arr[1].split("\\:", 2); + alias = arrs[0].trim(); + } + + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.REMOVE); + operations.put(alias, operationPerfomed); + } + + private void delete(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.DELETE); + String alias = arr[1].trim(); + if (alias.contains("-")) { + String arrs[] = alias.split("-", 2); + alias = arrs[0].trim(); + } + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.DELETE); + operations.put(alias, operationPerfomed); + } + + private void detachDelete(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.DETACH_DELET); + String alias = arr[1].trim(); + if (alias.contains("-")) { + String arrs[] = alias.split("-", 2); + alias = arrs[0].trim(); + } + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.DETACH_DELETE); + operations.put(alias, operationPerfomed); + } + + private void onMatchSet(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.ON_MATC_ST); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.ON_MATCH_SET); + operations.put(alias, operationPerfomed); + } + + private void onCreateSet(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.ON_CREAT_ST); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.ON_CREATE_SET); + operations.put(alias, operationPerfomed); + + } + + private void set(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.SET); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.SET); + operations.put(alias, operationPerfomed); + } + + private void match(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.MATCH).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.MATCH).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + private void create(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.CREATE).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.CREATE).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + private void merge(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.MERGE).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.MERGE).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + /* + * In order to fetch the alias and the node we use the indexing. On the + * basis of index, function/method is called. Variable map is used to store the alias + * and nodeName. Operation map is used to store the alias and operation + * performed. + */ + + private String squareBracket(String query, String operation) { + + String closingBracket = ""; + String value = ""; + String operationValue = operation; + ArrayList operationPerfomed = new ArrayList(); + + String[] arr = query.split("\\[", 2); - if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { - dbName = data.get(Constants.DB_NAME).getAsString(); - } + String[] arrs = arr[1].split(":", 2); + + if (arrs.length == 2) { + Pattern pattern = Pattern.compile("^[A-Za-z_|]+[\\sA-Za-z_|]*"); + Matcher matcher = pattern.matcher(arrs[1]); + if (matcher.find()) { + value = matcher.group(0); + } - record.setDbName(dbName); + // in order to get the alias only and remove all the extra characters. + // EX : -[rel returns rel + arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); - // Time - String dateString = Parser.parseTimestamp(data); - String timeZone = null; - if (data.has(Constants.MIN_OFF)) { - timeZone = data.get(Constants.MIN_OFF).getAsString(); - } - Time time = Parser.getTime(dateString, timeZone); + pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); + matcher = pattern.matcher(arrs[0]); + if (!matcher.find()) { + arrs[0] = "" + randomValue; + randomValue++; + } - if (time != null) { - record.setTime(time); - } - // SeessionLocator - record.setSessionLocator(Parser.parseSessionLocator(data)); + if (operations.containsKey(arrs[0].trim())) { + operationPerfomed = operations.get(arrs[0]); + operationPerfomed.add(operationValue); + operations.replace(arrs[0].trim(), operationPerfomed); + } else { + operationPerfomed.add(operationValue); + operations.put(arrs[0].trim(), operationPerfomed); + } - // Accessor - record.setAccessor(Parser.parseAccessor(data)); + variables.put(arrs[0].trim(), value.trim()); - Parser.parseSessionId(record); + String[] roundValue = arrs[1].split("\\]", 2); + closingBracket = roundValue[1]; + } - // Data - if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { - record.setData(Parser.parseData(data)); - } else { - record.setException(Parser.parseException(data)); - } - - } - return record; - } + return closingBracket; -// ---------------------- Session Id ----------------------- - - public static void parseSessionId(Record record) { + } - Integer hashCode = (record.getSessionLocator().getClientIp() + record.getSessionLocator().getClientPort() - + record.getDbName()).hashCode(); - record.setSessionId(hashCode.toString()); - } - -// -----------------------------------------------Accessor----------------- - - public static Accessor parseAccessor(JsonObject data) { - Accessor accessor = new Accessor(); - - accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); - accessor.setServerType(Constants.SERVER_TYPE_STRING); - - String dbUser = Constants.NOT_AVAILABLE; - if (data.has(Constants.DB_USER) && data.get(Constants.DB_USER) != null) { - dbUser = data.get(Constants.DB_USER).getAsString(); - } - accessor.setDbUser(dbUser); - - if(data.has(Constants.SERVER_HOSTNAME) && data.get(Constants.SERVER_HOSTNAME) != null) - accessor.setServerHostName(data.get(Constants.SERVER_HOSTNAME).getAsString()); - - String sourceProgram = Constants.UNKNOWN_STRING; - if (data.has(Constants.SOURCE_PROGRAM) && data.get(Constants.SOURCE_PROGRAM) != null) { - sourceProgram = data.get(Constants.SOURCE_PROGRAM).getAsString(); - } - accessor.setSourceProgram(sourceProgram); - - accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); - accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); - - accessor.setClient_mac(Constants.UNKNOWN_STRING); - accessor.setClientHostName(Constants.UNKNOWN_STRING); - accessor.setClientOs(Constants.UNKNOWN_STRING); - accessor.setCommProtocol(Constants.COMM_PROTOCOL); - accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); - accessor.setOsUser(Constants.UNKNOWN_STRING); - accessor.setServerDescription(Constants.UNKNOWN_STRING); - accessor.setServerOs(Constants.UNKNOWN_STRING); - accessor.setServiceName(Constants.UNKNOWN_STRING); - - return accessor; - } - -// -----------------------------------------------SessionLocator----------------- - - public static SessionLocator parseSessionLocator(JsonObject data) { - SessionLocator sessionLocator = new SessionLocator(); - sessionLocator.setIpv6(false); - - int clientPort = 0; - int serverPort = 0; - String clientIpAdd = Constants.NOT_AVAILABLE; - String serverIpAdd = Constants.NOT_AVAILABLE; - - if (data.has(Constants.CLIENT_IP)) { - String clientIpPortDetails = data.get(Constants.CLIENT_IP).getAsString(); - String clientIp = clientIpPortDetails.substring(7); - String[] clientIpPort = clientIp.split(":"); - clientIpAdd = clientIpPort[0]; - clientPort = Integer.parseInt(clientIpPort[1]); - - if (isIPv6Address(clientIpAdd)) { - sessionLocator.setIpv6(true); - sessionLocator.setClientIpv6(clientIpAdd); - clientIpAdd = Constants.UNKNOWN_STRING; - } else { - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - } - - } - - if (data.has(Constants.SERVER_IP)) { - String serverIPPortDetails = data.get(Constants.SERVER_IP).getAsString(); - String serverIp = serverIPPortDetails.substring(7); - String[] serverIpPort = serverIp.split(":"); - serverIpAdd = serverIpPort[0]; - serverPort = Integer.parseInt(serverIpPort[1]); - - if (isIPv6Address(serverIpAdd)) { - sessionLocator.setIpv6(true); - sessionLocator.setServerIpv6(serverIpAdd); - serverIpAdd = Constants.UNKNOWN_STRING; - } else { - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - } - } - - sessionLocator.setClientIp(clientIpAdd); - sessionLocator.setClientPort(clientPort); - sessionLocator.setServerIp(serverIpAdd); - sessionLocator.setServerPort(serverPort); - - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - - return sessionLocator; - } - - private static boolean isIPv6Address(String ip) { - try { - InetAddress inetAddress = InetAddress.getByName(ip); - return inetAddress instanceof java.net.Inet6Address; - } catch (UnknownHostException e) { - return false; - } - } - -// -----------------------------------------------Timestamp----------------- - - public static String parseTimestamp(final JsonObject data) { - String dateString = null; - if (data.has(Constants.TIMESTAMP)) { - dateString = data.get(Constants.TIMESTAMP).getAsString(); - } - return dateString; - } - - - public static Time getTime(String dateString, String timeZone) { - if (dateString != null) { - JsonObject data = new JsonObject(); - LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); - ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); - if (timeZone != null) { - offset = ZoneOffset.of(timeZone); - } - ZonedDateTime zdt = dt.atOffset(offset).toZonedDateTime(); - long millis = zdt.toInstant().toEpochMilli(); - int minOffset = zdt.getOffset().getTotalSeconds() / 60; - return new Time(millis, minOffset, 0); - } - return null; - } - -// -----------------------------------------------Exception----------------- - - public static ExceptionRecord parseException(JsonObject data) { - ExceptionRecord exceptionRecord = new ExceptionRecord(); - - exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); - - String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); - String[] queryData = queryStatement.split("'} - "); - if(queryData.length == 1){ - queryData = queryStatement.split("} - "); - } - - String query = queryData[0].toString(); - String exceptionDes = queryData[queryData.length - 1].toString(); - exceptionRecord.setDescription(exceptionDes); - - exceptionRecord.setSqlString(query); - return exceptionRecord; - } - -// -----------------------------------------------Data Construct---------------- - - public static Data parseData(JsonObject inputJSON) { - Data data = new Data(); - try { - Construct construct = parseAsConstruct(inputJSON); - if (construct != null) { - data.setConstruct(construct); - - if (construct.getFullSql() == null || construct.getFullSql().isEmpty()) { - construct.setFullSql(Constants.UNKNOWN_STRING); - } - if (construct.getRedactedSensitiveDataSql() == null || construct.redactedSensitiveDataSql.isEmpty()) { - construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); - } - } - } catch (Exception e) { - log.error("Neo4j filter: Error parsing JSon in parser" + inputJSON, e); - throw e; - } - return data; - } - - public static Construct parseAsConstruct(final JsonObject data) { - try { - final Sentence sentence = Parser.parseSentence(data); - - final Construct construct = new Construct(); - construct.sentences.add(sentence); - - String queryStatement = data.get(Constants.QUERY_STATEMENT).toString(); - String[] fullSql = queryStatement.split("runtime"); - String query[] = fullSql[0].split("- \\{\\} - "); - if(query[0].isEmpty()) - construct.setFullSql(fullSql[0].substring(1).trim()); - else - construct.setFullSql(query[0].substring(1).trim()); - construct.setRedactedSensitiveDataSql(Parser.parseRedactedSensitiveDataSql(data)); - return construct; - } catch (final Exception e) { - throw e; - } - } - - protected static Sentence parseSentence(final JsonObject data) { - - Sentence sentence = null; - - if (data.has(Constants.QUERY_STATEMENT)) { - String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); - - sentence = parseQuery(queryStatement.trim()); - - //For Match (n) return n - if(sentence == null) { - Sentence validSentence = new Sentence("MATCH"); - SentenceObject sentenceObject = new SentenceObject("AllNodesOrRelationships"); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - validSentence.getObjects().add(sentenceObject); - sentence = validSentence; - } - - } - - return sentence; - } - - public static Sentence parseQuery(String query) { - - ArrayList gfg = new ArrayList() { - { - add(Constants.ON_CREAT_ST); - add(Constants.ON_MATC_ST); - add(Constants.MATCH); - add(Constants.CREATE); - add(Constants.MERGE); - add(Constants.DELETE); - add(Constants.DETACH_DELET); - add(Constants.REMOVE); - add(Constants.SET); - add(Constants.RETURN); - add(Constants.WHERE); - add(Constants.FOREACH); - } - }; - - if (query.contains(Constants.ON_CREATE_SET)) - query = query.replace(Constants.ON_CREATE_SET, Constants.ON_CREAT_ST); - if (query.contains(Constants.ON_MATCH_SET)) - query = query.replace(Constants.ON_MATCH_SET, Constants.ON_MATC_ST); - if (query.contains(Constants.DETACH_DELETE)) - query = query.replace(Constants.DETACH_DELETE, Constants.DETACH_DELET); - - // find the index of all the supported operations in the query. - Map ops = new TreeMap<>(); - - for (String operation : gfg) { - - int index = 0; - - while (true) { - index = query.indexOf(operation, index); - if (index == -1) - break; - ops.put(index, operation); - index++; - } - } - - // split the main query into multiple queries using index. - List queries = new ArrayList(); - - int i = 0; - int lastKeyValue = -1; - for (Map.Entry entry : ops.entrySet()) { - if (i == 0) { - i++; - lastKeyValue = entry.getKey(); - continue; - } - - String subquery = query.substring(lastKeyValue, entry.getKey()); - queries.add(subquery.trim()); - lastKeyValue = entry.getKey(); - - } - - if (lastKeyValue != -1) { - String subquery = query.substring(lastKeyValue); - queries.add(subquery); - } - - operations = new LinkedHashMap>(); - variables = new LinkedHashMap(); - - for (String queryString : queries) { - if (queryString.startsWith(Constants.RETURN) || queryString.startsWith(Constants.WHERE) - || queryString.startsWith(Constants.FOREACH)) - continue; - else if (queryString.startsWith(Constants.ON_CREAT_ST)) - onCreateSet(queryString); - else if (queryString.startsWith(Constants.ON_MATC_ST)) - onMatchSet(queryString); - else if (queryString.startsWith(Constants.MATCH)) - match(queryString); - else if (queryString.startsWith(Constants.CREATE)) - create(queryString); - else if (queryString.startsWith(Constants.MERGE)) - merge(queryString); - else if (queryString.startsWith(Constants.SET)) - set(queryString); - else if (queryString.startsWith(Constants.DELETE)) - delete(queryString); - else if (queryString.startsWith(Constants.REMOVE)) - remove(queryString); - else if (queryString.startsWith(Constants.DETACH_DELET)) - detachDelete(queryString); - } - - // Form the sentences out of different verb and objects - - // convert to ArrayList of key set - List alKeys = new ArrayList(operations.keySet()); - - SentenceObject sentenceObject = null; - - int j = 1; - Sentence originalSentence = null; - - for (String keys : alKeys) { - if (!operations.isEmpty()) { - ArrayList operation = operations.get(keys); - - for (String string : operation) { - - if (j == 1) { - originalSentence = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if (sentenceObject.getName() == null) - sentenceObject.setName(Constants.EVERYTHING); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - originalSentence.getObjects().add(sentenceObject); - j++; - } else { - Sentence decendant = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if (sentenceObject.getName() == null) - sentenceObject.setName(Constants.EVERYTHING); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - decendant.getObjects().add(sentenceObject); - originalSentence.getDescendants().add(decendant); - } - } - } - } - - return originalSentence; - } - - private static void remove(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.REMOVE); - String alias = ""; - if (arr[1].contains(".")) { - String arrs[] = arr[1].split("\\.", 2); - alias = arrs[0].trim(); - } else if (arr[1].contains(":")) { - String arrs[] = arr[1].split("\\:", 2); - alias = arrs[0].trim(); - } - - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.REMOVE); - operations.put(alias, operationPerfomed); - } - - private static void delete(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.DELETE); - String alias = arr[1].trim(); - if (alias.contains("-")) { - String arrs[] = alias.split("-", 2); - alias = arrs[0].trim(); - } - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.DELETE); - operations.put(alias, operationPerfomed); - } - - private static void detachDelete(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.DETACH_DELET); - String alias = arr[1].trim(); - if (alias.contains("-")) { - String arrs[] = alias.split("-", 2); - alias = arrs[0].trim(); - } - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.DETACH_DELETE); - operations.put(alias, operationPerfomed); - } - - private static void onMatchSet(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.ON_MATC_ST); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.ON_MATCH_SET); - operations.put(alias, operationPerfomed); - } - - private static void onCreateSet(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.ON_CREAT_ST); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.ON_CREATE_SET); - operations.put(alias, operationPerfomed); - - } - - private static void set(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.SET); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.SET); - operations.put(alias, operationPerfomed); - } - - private static void match(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.MATCH).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.MATCH).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - private static void create(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.CREATE).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.CREATE).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - private static void merge(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.MERGE).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.MERGE).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - /* - * In order to fetch the alias and the node we use the indexing. On the - * basis of index, function/method is called. Variable map is used to store the alias - * and nodeName. Operation map is used to store the alias and operation - * performed. - */ - - private static String squareBracket(String query, String operation) { - - String closingBracket = ""; - String value = ""; - String operationValue = operation; - ArrayList operationPerfomed = new ArrayList(); - - String[] arr = query.split("\\[", 2); - - String[] arrs = arr[1].split(":", 2); - - if (arrs.length == 2) { - Pattern pattern = Pattern.compile("^[A-Za-z_|]+[\\sA-Za-z_|]*"); - Matcher matcher = pattern.matcher(arrs[1]); - if (matcher.find()) { - value = matcher.group(0); - } - - // in order to get the alias only and remove all the extra characters. - // EX : -[rel returns rel - arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); + protected String roundBracket(String query, String operation) { - pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); - matcher = pattern.matcher(arrs[0]); - if (!matcher.find()) { - arrs[0] = "" + randomValue; - randomValue++; - } - - if (operations.containsKey(arrs[0].trim())) { - operationPerfomed = operations.get(arrs[0]); - operationPerfomed.add(operationValue); - operations.replace(arrs[0].trim(), operationPerfomed); - } else { - operationPerfomed.add(operationValue); - operations.put(arrs[0].trim(), operationPerfomed); - } + String closingBracket = ""; + String value = ""; + String operationValue = operation; + ArrayList operationPerfomed = new ArrayList(); - variables.put(arrs[0].trim(), value.trim()); + String[] arr = query.split("\\(", 2); - String[] roundValue = arrs[1].split("\\]", 2); - closingBracket = roundValue[1]; - } - - return closingBracket; + if (arr[1].indexOf(")") < arr[1].indexOf(":")) { + String temp[] = arr[1].split("\\)", 2); + arr[1] = temp[1]; + } - } - - protected static String roundBracket(String query, String operation) { + String[] arrs = arr[1].split(":", 2); - String closingBracket = ""; - String value = ""; - String operationValue = operation; - ArrayList operationPerfomed = new ArrayList(); + if (arrs.length == 2) { + Pattern pattern = Pattern.compile("^[A-Za-z_]+[\\sA-Za-z_]*"); + Matcher matcher = pattern.matcher(arrs[1]); + if (matcher.find()) { + value = matcher.group(0); + } - String[] arr = query.split("\\(", 2); + // in order to get the alias only and remove all the extra characters. + // EX : -[rel returns rel + arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); - if (arr[1].indexOf(")") < arr[1].indexOf(":")) { - String temp[] = arr[1].split("\\)", 2); - arr[1] = temp[1]; - } - - String[] arrs = arr[1].split(":", 2); - - if (arrs.length == 2) { - Pattern pattern = Pattern.compile("^[A-Za-z_]+[\\sA-Za-z_]*"); - Matcher matcher = pattern.matcher(arrs[1]); - if (matcher.find()) { - value = matcher.group(0); - } + pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); + matcher = pattern.matcher(arrs[0]); + if (!matcher.find()) { + arrs[0] = "" + randomValue; + randomValue++; + } - // in order to get the alias only and remove all the extra characters. - // EX : -[rel returns rel - arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); + if (operations.containsKey(arrs[0].trim())) { + operationPerfomed = operations.get(arrs[0]); + operations.remove(arrs[0]); + } + operationPerfomed.add(operationValue); + operations.put(arrs[0].trim(), operationPerfomed); - pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); - matcher = pattern.matcher(arrs[0]); - if (!matcher.find()) { - arrs[0] = "" + randomValue; - randomValue++; - } - - if (operations.containsKey(arrs[0].trim())) { - operationPerfomed = operations.get(arrs[0]); - operations.remove(arrs[0]); - } - operationPerfomed.add(operationValue); - operations.put(arrs[0].trim(), operationPerfomed); + variables.put(arrs[0].trim(), value.trim()); - variables.put(arrs[0].trim(), value.trim()); + String[] roundValue = arrs[1].split("\\)", 2); + closingBracket = roundValue[1]; + } - String[] roundValue = arrs[1].split("\\)", 2); - closingBracket = roundValue[1]; - } - - return closingBracket; + return closingBracket; - } + } - protected static String parseRedactedSensitiveDataSql(JsonObject data) { - String redactedData = ""; - redactedData = data.get(Constants.MESSAGE).getAsString(); + protected String parseRedactedSensitiveDataSql(JsonObject data) { + String redactedData = ""; + redactedData = data.get(Constants.MESSAGE).getAsString(); - return redactedData; - } + return redactedData; + } -} \ No newline at end of file +} diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java index 0c69b8428..5f6083cd4 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java @@ -11,283 +11,328 @@ import org.junit.Assert; import org.junit.Test; +import java.text.ParseException; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; public class ParserTest { + Parser parser; + String neoSuccessString = "2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoSuccessString_grokOutput = "{\n" + + " \"ts\": \"\\\"2021-08-06 17:09:40.008+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:51372\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\\\";\"\n" + + " }"; + + @Test + public void testParseAsConstruct_Match() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + final Construct result = parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("MATCH", sentence.getVerb().trim()); + Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testParseAsConstruct_Create() { + parser = new Parser(); + String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + + JsonObject inputData = inputData(e); + + final Construct result = parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("CREATE", sentence.getVerb().trim()); + Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + + @Test + public void testDBNameAndServiceNameEqual() throws ParseException { + parser = new Parser(); + String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4jDB \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + + Event e = getParsedEvent(neoString_grokOutput, neoString); + e.setField("minoff", "+07:00"); + + JsonObject inputData = inputData(e); + + final Record record = parser.parseRecord(inputData); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + } + + @Test + public void testParseAsConstruct_Merge() { + parser = new Parser(); + String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:56:12.097+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + e.setField("minoff", "+04:00"); + + JsonObject inputData = inputData(e); + + final Construct result = parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("MERGE", sentence.getVerb().trim()); + Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testParseException_DELETE() { + parser = new Parser(); + String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-03-03 08:49:32.367+0000\",\n" + + " \"log_level\": \"ERROR\",\n" + + " \"metadata1\": \"7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.2.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:62845\",\n" + + " \"server_ip\": \"server/127.0.0.1:7687\",\n" + + " \"dbname\": \" \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))\\\";\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + JsonObject inputData = inputData(e); + e.setField("minoff", "+07:00"); + + final ExceptionRecord exceptionRecord = parser.parseException(inputData); + + Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); + Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); + } + + @Test + public void testParseAccessor() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Accessor accessor = parser.parseAccessor(inputData); + + Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); + Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); + Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); + Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); + } + + @Test + public void testParseSessionLocator() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final SessionLocator sessionLocator = parser.parseSessionLocator(inputData); + + Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); + Assert.assertEquals(-1, sessionLocator.getClientPort()); + Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); + Assert.assertEquals(-1, sessionLocator.getServerPort()); + + } + + @Test + public void testParseSessionId() throws ParseException { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Record record = parser.parseRecord(inputData); + + Assert.assertEquals(Constants.UNKNOWN_STRING, record.getSessionId()); + + } + + @Test + public void testParseTimestamp() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); + JsonObject inputData = inputData(e); + + final String timestamp = parser.parseTimestamp(inputData); + + Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); + + } + + @Test + public void testGetTime() { + parser = new Parser(); + String dateString = "2021-01-25 11:17:09.099+0000"; + String timeZone = "-04:00"; + final Time time = parser.getTime(dateString, timeZone); + + Assert.assertEquals(0, time.getMinDst()); + Assert.assertEquals(-240, time.getMinOffsetFromGMT()); + Assert.assertEquals(1611587829099L, time.getTimstamp()); + + } + + @Test + public void testParseSentence() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Sentence sentence = parser.parseSentence(inputData); + + Assert.assertEquals("MATCH", sentence.getVerb()); + Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + + } + + @Test + public void testParseRedactedSensitiveDataSql() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final String redacted = parser.parseRedactedSensitiveDataSql(inputData); + + Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); + + } - String neoSuccessString = "2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoSuccessString_grokOutput = "{\n" + - " \"ts\": \"\\\"2021-08-06 17:09:40.008+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:51372\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\\\";\"\n" + - " }"; - - @Test - public void testParseAsConstruct_Match() { - - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - final Construct result = Parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("MATCH", sentence.getVerb().trim()); - Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseAsConstruct_Create() { - String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput,neoString); - - JsonObject inputData = inputData(e); - - final Construct result = Parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("CREATE", sentence.getVerb().trim()); - Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseAsConstruct_Merge() { - String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:56:12.097+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput,neoString); - e.setField("minoff", "+04:00"); - JsonObject inputData = inputData(e); - - final Construct result = Parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("MERGE", sentence.getVerb().trim()); - Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseException_DELETE() { - String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-03-03 08:49:32.367+0000\",\n" + - " \"log_level\": \"ERROR\",\n" + - " \"metadata1\": \"7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.2.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:62845\",\n" + - " \"server_ip\": \"server/127.0.0.1:7687\",\n" + - " \"dbname\": \" \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))\\\";\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput, neoString); - e.setField("minoff", "+07:00"); - JsonObject inputData = inputData(e); - - final ExceptionRecord exceptionRecord = Parser.parseException(inputData); - - Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); - Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); - } - - @Test - public void testParseAccessor() { - - Event e = getParsedEvent(neoSuccessString_grokOutput,neoSuccessString); - - JsonObject inputData = inputData(e); - - final Accessor accessor = Parser.parseAccessor(inputData); - - Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); - Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); - Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); - Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); - } - - @Test - public void testParseSessionLocator() { - - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final SessionLocator sessionLocator = Parser.parseSessionLocator(inputData); - - Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); - Assert.assertEquals(51372, sessionLocator.getClientPort()); - Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); - Assert.assertEquals(11004, sessionLocator.getServerPort()); - - } - - @Test - public void testParseTimestamp() { - - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); - JsonObject inputData = inputData(e); - - final String timestamp = Parser.parseTimestamp(inputData); - - Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); - - } - - @Test - public void testGetTime() { - - String dateString = "2021-01-25 11:17:09.099+0000"; - String timeZone = "-04:00"; - final Time time = Parser.getTime(dateString, timeZone); - - Assert.assertEquals(0, time.getMinDst()); - Assert.assertEquals(-240, time.getMinOffsetFromGMT()); - Assert.assertEquals(1611587829099L, time.getTimstamp()); - - } - - @Test - public void testParseSentence() { - - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final Sentence sentence = Parser.parseSentence(inputData); - - Assert.assertEquals("MATCH", sentence.getVerb()); - Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - - } - - @Test - public void testParseRedactedSensitiveDataSql() { - - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final String redacted = Parser.parseRedactedSensitiveDataSql(inputData); - - Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); - - } - // ----------------------------------- --------------------------------------------------- - - private JsonObject inputData(Event e){ - JsonObject data = new JsonObject(); - - if(e.getField(Constants.CLIENT_IP).toString() != null && !e.getField(Constants.CLIENT_IP).toString().isEmpty()){ - data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); - } - if(e.getField(Constants.SERVER_IP).toString() != null && !e.getField(Constants.SERVER_IP).toString().isEmpty()){ - data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); - } - if(e.getField(Constants.DB_PROTOCOL).toString() != null && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()){ - data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); - } - if(e.getField(Constants.TIMESTAMP).toString() != null && !e.getField(Constants.TIMESTAMP).toString().isEmpty()){ - data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); - } - if(e.getField(Constants.LOG_LEVEL).toString() != null && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()){ - data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); - } - if(e.getField(Constants.DB_USER).toString() != null && !e.getField(Constants.DB_USER).toString().isEmpty()){ - data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); - } - if(e.getField(Constants.DB_NAME).toString() != null && !e.getField(Constants.DB_NAME).toString().isEmpty()){ - data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); - } - if(e.getField(Constants.SOURCE_PROGRAM).toString() != null && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()){ - data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); - } - if(e.getField(Constants.QUERY_STATEMENT).toString() != null && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()){ - data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); - } - if(e.getField(Constants.MESSAGE).toString() != null && !e.getField(Constants.MESSAGE).toString().isEmpty()){ - data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); - } - return data; - } - - - public static Event getParsedEvent(String logEvent_json , String logEvent) { - - Map results = parseJson(logEvent_json); - Event e = new org.logstash.Event(); - e.setField("message", logEvent); - - e.setField(Constants.CLIENT_IP, results.get(Constants.CLIENT_IP)); - e.setField(Constants.SERVER_IP, results.get(Constants.SERVER_IP)); - e.setField(Constants.DB_PROTOCOL, results.get(Constants.DB_PROTOCOL)); - e.setField(Constants.TIMESTAMP, results.get(Constants.TIMESTAMP)); - e.setField(Constants.LOG_LEVEL, results.get(Constants.LOG_LEVEL)); - e.setField(Constants.DB_USER, results.get(Constants.DB_USER)); - e.setField(Constants.DB_NAME, results.get(Constants.DB_NAME)); - e.setField(Constants.SOURCE_PROGRAM, results.get(Constants.SOURCE_PROGRAM)); - e.setField(Constants.QUERY_STATEMENT, results.get(Constants.QUERY_STATEMENT)); - - return e; - } - - - public static Map parseJson(String jsonString) { - Map map = new HashMap<>(); - - // Use a regular expression to match each key-value pair in the JSON string - String pattern = "\"(.*?)\":\\s*\"(.*?)\""; - Pattern r = Pattern.compile(pattern); - Matcher m = r.matcher(jsonString); - - while (m.find()) { - // Add each key-value pair to the map - String key = m.group(1); - String value = m.group(2).replaceAll("(? results = parseJson(logEvent_json); + Event e = new org.logstash.Event(); + e.setField("message", logEvent); + + e.setField(Constants.CLIENT_IP, results.get(Constants.CLIENT_IP)); + e.setField(Constants.SERVER_IP, results.get(Constants.SERVER_IP)); + e.setField(Constants.DB_PROTOCOL, results.get(Constants.DB_PROTOCOL)); + e.setField(Constants.TIMESTAMP, results.get(Constants.TIMESTAMP)); + e.setField(Constants.LOG_LEVEL, results.get(Constants.LOG_LEVEL)); + e.setField(Constants.DB_USER, results.get(Constants.DB_USER)); + e.setField(Constants.DB_NAME, results.get(Constants.DB_NAME)); + e.setField(Constants.SOURCE_PROGRAM, results.get(Constants.SOURCE_PROGRAM)); + e.setField(Constants.QUERY_STATEMENT, results.get(Constants.QUERY_STATEMENT)); + + return e; + } + + + public static Map parseJson(String jsonString) { + Map map = new HashMap<>(); + + // Use a regular expression to match each key-value pair in the JSON string + String pattern = "\"(.*?)\":\\s*\"(.*?)\""; + Pattern r = Pattern.compile(pattern); + Matcher m = r.matcher(jsonString); + + while (m.find()) { + // Add each key-value pair to the map + String key = m.group(1); + String value = m.group(2).replaceAll("(? Date: Mon, 25 Aug 2025 22:05:18 -0400 Subject: [PATCH 2/4] GRD-106525 fix Neo4j outofbounds exception Signed-off-by: Andy.Chen --- .../guardium/neodb/NeodbGuardiumFilter.java | 269 ++-- .../java/com/ibm/guardium/neodb/Parser.java | 1293 +++++++++-------- 2 files changed, 788 insertions(+), 774 deletions(-) diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java index 0b043b275..b2b32cb19 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java @@ -9,7 +9,6 @@ import java.text.ParseException; import java.util.Collection; import java.util.Collections; - import com.google.gson.JsonElement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,142 +31,140 @@ @LogstashPlugin(name = "neodb_guardium_filter") public class NeodbGuardiumFilter implements Filter { - private static Logger logger = LogManager.getLogger(NeodbGuardiumFilter.class); - + private static Logger logger = LogManager.getLogger(NeodbGuardiumFilter.class); - public static final String LOG42_CONF = "log4j2uc.properties"; - static { - try { - String uc_etc = System.getenv("UC_ETC"); - LoggerContext context = (LoggerContext) LogManager.getContext(false); - File file = new File(uc_etc + File.separator + LOG42_CONF); - context.setConfigLocation(file.toURI()); - } catch (Exception e) { - e.printStackTrace(); - } - } + public static final String LOG42_CONF = "log4j2uc.properties"; + static { + try { + String uc_etc = System.getenv("UC_ETC"); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + File file = new File(uc_etc + File.separator + LOG42_CONF); + context.setConfigLocation(file.toURI()); + } catch (Exception e) { + e.printStackTrace(); + } + } - private String id; - public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); - private static Logger log = LogManager.getLogger(NeodbGuardiumFilter.class); + private String id; + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + private static Logger log = LogManager.getLogger(NeodbGuardiumFilter.class); Parser parser = new Parser(); - - public NeodbGuardiumFilter(String id, Configuration config, Context context) { - this.id = id; - } - - @Override - public Collection> configSchema() { - // should return a list of all configuration options for this plugin - return Collections.singletonList(SOURCE_CONFIG); - } - - @Override - public String getId() { - return this.id; - } - - @Override - public Collection filter(Collection events, FilterMatchListener matchListener) { - - for (Event e : events) { - if (logger.isDebugEnabled()) { - logger.debug("Event Now: {}", e.getData()); - } - - try { - JsonObject inputData = inputData(e); - - if (isNotSystemGeneratedEvent(inputData)) { - - Record record = parser.parseRecord(inputData); - - final GsonBuilder builder = new GsonBuilder(); - builder.serializeNulls(); - final Gson gson = builder.create(); - e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); - - matchListener.filterMatched(e); - } else { - e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_NEO); - } - - } catch (ParseException pe) { - log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), pe); - e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); - } catch (Exception exception) { - log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), - exception); - e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); - } - } - return events; - } - - private boolean isNotSystemGeneratedEvent(JsonObject inputData) { - if (inputData == null || !inputData.has(Constants.QUERY_STATEMENT)) { - return false; // Treat as system-generated if missing QUERY_STATEMENT or inputData is null - } - - JsonElement queryElement = inputData.get(Constants.QUERY_STATEMENT); - if (queryElement == null || queryElement.isJsonNull()) { - return false; // Treat as system-generated if queryElement is null or JsonNull - } - - String queryStatement = queryElement.getAsString(); - if (queryStatement == null || queryStatement.isEmpty() || queryStatement.startsWith("EXPLAIN")) { - return false; // Treat as system-generated if queryStatement is null, empty, or starts with "EXPLAIN" - } - - return true; // Otherwise, it is not a system-generated event - } - - private JsonObject inputData(Event e) { - JsonObject data = new JsonObject(); - - if (e.getField(Constants.CLIENT_IP) != null - && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { - data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); - } - if (e.getField(Constants.SERVER_IP) != null - && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { - data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); - } - if (e.getField(Constants.DB_PROTOCOL) != null - && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { - data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); - } - if (e.getField(Constants.TIMESTAMP) != null - && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { - data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); - } - if (e.getField(Constants.LOG_LEVEL) != null - && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { - data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); - } - if (e.getField(Constants.DB_USER) != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { - data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); - } - if (e.getField(Constants.DB_NAME) != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { - data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); - } - if (e.getField(Constants.SOURCE_PROGRAM) != null - && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { - data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); - } - if (e.getField(Constants.QUERY_STATEMENT) != null - && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { - data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); - } - if (e.getField(Constants.SERVER_HOSTNAME) != null - && !e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { - data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); - } - if (e.getField(Constants.MESSAGE) != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { - data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); - } - return data; - } - -} + public NeodbGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + + for (Event e : events) { + if(logger.isDebugEnabled()){ + logger.debug("Event Now: {}", e.getData()); + } + + try { + JsonObject inputData = inputData(e); + + if (isNotSystemGeneratedEvent(inputData)) { + + Record record = Parser.parseRecord(inputData); + + final GsonBuilder builder = new GsonBuilder(); + builder.serializeNulls(); + final Gson gson = builder.create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + matchListener.filterMatched(e); + } else { + e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_NEO); + } + + } catch (ParseException pe) { + log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), pe); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } catch (Exception exception) { + log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), + exception); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } + } + return events; + } + + private boolean isNotSystemGeneratedEvent(JsonObject inputData) { + if (inputData == null || !inputData.has(Constants.QUERY_STATEMENT)) { + return false; // Treat as system-generated if missing QUERY_STATEMENT or inputData is null + } + + JsonElement queryElement = inputData.get(Constants.QUERY_STATEMENT); + if (queryElement == null || queryElement.isJsonNull()) { + return false; // Treat as system-generated if queryElement is null or JsonNull + } + + String queryStatement = queryElement.getAsString(); + if (queryStatement == null || queryStatement.isEmpty() || queryStatement.startsWith("EXPLAIN")) { + return false; // Treat as system-generated if queryStatement is null, empty, or starts with "EXPLAIN" + } + + return true; // Otherwise, it is not a system-generated event + } + + private JsonObject inputData(Event e) { + JsonObject data = new JsonObject(); + + if (e.getField(Constants.CLIENT_IP) != null + && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { + data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); + } + if (e.getField(Constants.SERVER_IP) != null + && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { + data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); + } + if (e.getField(Constants.DB_PROTOCOL) != null + && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { + data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); + } + if (e.getField(Constants.TIMESTAMP) != null + && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { + data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); + } + if (e.getField(Constants.LOG_LEVEL) != null + && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { + data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); + } + if (e.getField(Constants.DB_USER) != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { + data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); + } + if (e.getField(Constants.DB_NAME) != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { + data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); + } + if (e.getField(Constants.SOURCE_PROGRAM) != null + && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { + data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); + } + if (e.getField(Constants.QUERY_STATEMENT) != null + && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { + data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); + } + if (e.getField(Constants.SERVER_HOSTNAME) != null + && !e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { + data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); + } + if (e.getField(Constants.MESSAGE) != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { + data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); + } + return data; + } + +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java index d3993f428..3fb548757 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java @@ -4,7 +4,6 @@ // package com.ibm.guardium.neodb; - import java.net.InetAddress; import java.net.UnknownHostException; import java.text.ParseException; @@ -13,7 +12,11 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -32,672 +35,686 @@ public class Parser { - private Logger log = LogManager.getLogger(Parser.class); - - private final DateTimeFormatterBuilder dateTimeFormatterBuilder = - new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS[[XXX][X]]")); - - private final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); - - LinkedHashMap> operations = null; - LinkedHashMap variables = null; - static Character randomValue = 65; - - public Record parseRecord(final JsonObject data) throws ParseException { - Record record = new Record(); - - if (data != null) { - - record.setAppUserName(Constants.NOT_AVAILABLE); - - // DB Name - String dbName = Constants.NOT_AVAILABLE; - - if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { - dbName = data.get(Constants.DB_NAME).getAsString(); - } - - record.setDbName(dbName); - - // Time - String dateString = parseTimestamp(data); - String timeZone = null; - if (data.has(Constants.MIN_OFF)) { - timeZone = data.get(Constants.MIN_OFF).getAsString(); - } - Time time = getTime(dateString, timeZone); - - if (time != null) { - record.setTime(time); - } - // SessionLocator - record.setSessionLocator(parseSessionLocator(data)); - - // Accessor - record.setAccessor(parseAccessor(data)); - - record.setSessionId(Constants.UNKNOWN_STRING); + private Logger log = LogManager.getLogger(Parser.class); - // Data - if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { - record.setData(parseData(data)); - } else { - record.setException(parseException(data)); - } + private final DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS[[XXX][X]]")); - } - return record; - } + private final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); - // -----------------------------------------------Accessor----------------- + static LinkedHashMap> operations = null; + static LinkedHashMap variables = null; + static Character randomValue = 65; - public Accessor parseAccessor(JsonObject data) { - Accessor accessor = new Accessor(); + public Record parseRecord(final JsonObject data) throws ParseException { + Record record = new Record(); - accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); - accessor.setServerType(Constants.SERVER_TYPE_STRING); + if (data != null) { - String dbUser = Constants.NOT_AVAILABLE; - if (data.has(Constants.DB_USER) && data.get(Constants.DB_USER) != null) { - dbUser = data.get(Constants.DB_USER).getAsString(); - } - accessor.setDbUser(dbUser); + record.setAppUserName(Constants.NOT_AVAILABLE); - if (data.has(Constants.SERVER_HOSTNAME) && data.get(Constants.SERVER_HOSTNAME) != null) - accessor.setServerHostName(data.get(Constants.SERVER_HOSTNAME).getAsString()); + // DB Name + String dbName = Constants.NOT_AVAILABLE; - String sourceProgram = Constants.UNKNOWN_STRING; - if (data.has(Constants.SOURCE_PROGRAM) && data.get(Constants.SOURCE_PROGRAM) != null) { - sourceProgram = data.get(Constants.SOURCE_PROGRAM).getAsString(); - } - accessor.setSourceProgram(sourceProgram); + if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { + dbName = data.get(Constants.DB_NAME).getAsString(); + } - accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); - accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); - - accessor.setClient_mac(Constants.UNKNOWN_STRING); - accessor.setClientHostName(Constants.UNKNOWN_STRING); - accessor.setClientOs(Constants.UNKNOWN_STRING); - accessor.setCommProtocol(Constants.COMM_PROTOCOL); - accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); - accessor.setOsUser(Constants.UNKNOWN_STRING); - accessor.setServerDescription(Constants.UNKNOWN_STRING); - accessor.setServerOs(Constants.UNKNOWN_STRING); - - String serviceName = Constants.NOT_AVAILABLE; - - if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { - serviceName = data.get(Constants.DB_NAME).getAsString(); - } - - accessor.setServiceName(serviceName); - - return accessor; - } - - // -----------------------------------------------SessionLocator----------------- - - public SessionLocator parseSessionLocator(JsonObject data) { - SessionLocator sessionLocator = new SessionLocator(); - sessionLocator.setIpv6(false); - - int clientPort = Constants.DEFAULT_PORT; - int serverPort = Constants.DEFAULT_PORT; - String clientIpAdd = Constants.NOT_AVAILABLE; - String serverIpAdd = Constants.NOT_AVAILABLE; - - if (data.has(Constants.CLIENT_IP)) { - String clientIpPortDetails = data.get(Constants.CLIENT_IP).getAsString(); - String clientIp = clientIpPortDetails.substring(7); - String[] clientIpPort = clientIp.split(":"); - clientIpAdd = clientIpPort[0]; - - if (isIPv6Address(clientIpAdd)) { - sessionLocator.setIpv6(true); - sessionLocator.setClientIpv6(clientIpAdd); - clientIpAdd = Constants.UNKNOWN_STRING; - } else { - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - } - - } - - if (data.has(Constants.SERVER_IP)) { - String serverIPPortDetails = data.get(Constants.SERVER_IP).getAsString(); - String serverIp = serverIPPortDetails.substring(7); - String[] serverIpPort = serverIp.split(":"); - serverIpAdd = serverIpPort[0]; - - - if (isIPv6Address(serverIpAdd)) { - sessionLocator.setIpv6(true); - sessionLocator.setServerIpv6(serverIpAdd); - serverIpAdd = Constants.UNKNOWN_STRING; - } else { - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - } - - } - - sessionLocator.setClientIp(clientIpAdd); - sessionLocator.setClientPort(clientPort); - sessionLocator.setServerIp(serverIpAdd); - sessionLocator.setServerPort(serverPort); - - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - - return sessionLocator; - } - - private boolean isIPv6Address(String ip) { - try { - InetAddress inetAddress = InetAddress.getByName(ip); - return inetAddress instanceof java.net.Inet6Address; - } catch (UnknownHostException e) { - return false; - } - } - - // -----------------------------------------------Timestamp----------------- - - public String parseTimestamp(final JsonObject data) { - String dateString = null; - if (data.has(Constants.TIMESTAMP)) { - dateString = data.get(Constants.TIMESTAMP).getAsString(); - } - return dateString; - } - - public Time getTime(String dateString, String timeZone) { - if (dateString != null) { - JsonObject data = new JsonObject(); - LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); - ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); - if (timeZone != null) { - offset = ZoneOffset.of(timeZone); - } - ZonedDateTime zdt = dt.atOffset(offset).toZonedDateTime(); - long millis = zdt.toInstant().toEpochMilli(); - int minOffset = zdt.getOffset().getTotalSeconds() / 60; - return new Time(millis, minOffset, 0); - } - return null; - } - - // -----------------------------------------------Exception----------------- - - public ExceptionRecord parseException(JsonObject data) { - ExceptionRecord exceptionRecord = new ExceptionRecord(); - - exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); - - String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); - String[] queryData = queryStatement.split("'} - "); - if (queryData.length == 1) { - queryData = queryStatement.split("} - "); - } - - String query = queryData[0].toString(); - String exceptionDes = queryData[queryData.length - 1].toString(); - exceptionRecord.setDescription(exceptionDes); - - exceptionRecord.setSqlString(query); - return exceptionRecord; - } - - // -----------------------------------------------Data Construct---------------- - - public Data parseData(JsonObject inputJSON) { - Data data = new Data(); - try { - Construct construct = parseAsConstruct(inputJSON); - if (construct != null) { - data.setConstruct(construct); - - if (construct.getFullSql() == null || construct.getFullSql().isEmpty()) { - construct.setFullSql(Constants.UNKNOWN_STRING); - } - if (construct.getRedactedSensitiveDataSql() == null || construct.redactedSensitiveDataSql.isEmpty()) { - construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); - } - } - } catch (Exception e) { - log.error("Neo4j filter: Error parsing JSon in parser" + inputJSON, e); - throw e; - } - return data; - } - - public Construct parseAsConstruct(final JsonObject data) { - try { - final Sentence sentence = parseSentence(data); - - final Construct construct = new Construct(); - construct.sentences.add(sentence); - - String queryStatement = data.get(Constants.QUERY_STATEMENT).toString(); - String[] fullSql = queryStatement.split("runtime"); - String query[] = fullSql[0].split("- \\{\\} - "); - if (query[0].isEmpty()) - construct.setFullSql(fullSql[0].substring(1).trim()); - else - construct.setFullSql(query[0].substring(1).trim()); - construct.setRedactedSensitiveDataSql(parseRedactedSensitiveDataSql(data)); - return construct; - } catch (final Exception e) { - throw e; - } - } - - protected Sentence parseSentence(final JsonObject data) { - - Sentence sentence = null; - - if (data.has(Constants.QUERY_STATEMENT)) { - String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); - - sentence = parseQuery(queryStatement.trim()); - - //For Match (n) return n - if (sentence == null) { - Sentence validSentence = new Sentence("MATCH"); - SentenceObject sentenceObject = new SentenceObject("AllNodesOrRelationships"); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - validSentence.getObjects().add(sentenceObject); - sentence = validSentence; - } - - } - - return sentence; - } - - Sentence parseQuery(String query) { - List gfg = - new ArrayList<>( - Arrays.asList( - Constants.ON_CREAT_ST, - Constants.ON_MATC_ST, - Constants.MATCH, - Constants.CREATE, - Constants.MERGE, - Constants.DELETE, - Constants.DETACH_DELET, - Constants.REMOVE, - Constants.SET, - Constants.RETURN, - Constants.WHERE, - Constants.FOREACH)); - - if (query.contains(Constants.ON_CREATE_SET)) - query = query.replace(Constants.ON_CREATE_SET, Constants.ON_CREAT_ST); - if (query.contains(Constants.ON_MATCH_SET)) - query = query.replace(Constants.ON_MATCH_SET, Constants.ON_MATC_ST); - if (query.contains(Constants.DETACH_DELETE)) - query = query.replace(Constants.DETACH_DELETE, Constants.DETACH_DELET); - - // find the index of all the supported operations in the query. - Map ops = new TreeMap<>(); - - for (String operation : gfg) { - - int index = 0; - - while (true) { - index = query.indexOf(operation, index); - if (index == -1) break; - ops.put(index, operation); - index++; - } - } - - // split the main query into multiple queries using index. - List queries = new ArrayList<>(); - - int i = 0; - int lastKeyValue = -1; - for (Map.Entry entry : ops.entrySet()) { - if (i == 0) { - i++; - lastKeyValue = entry.getKey(); - continue; - } - - String subquery = query.substring(lastKeyValue, entry.getKey()); - queries.add(subquery.trim()); - lastKeyValue = entry.getKey(); - } - - if (lastKeyValue != -1) { - String subquery = query.substring(lastKeyValue); - queries.add(subquery); - } - - operations = new LinkedHashMap<>(); - variables = new LinkedHashMap<>(); - - for (String queryString : queries) { - if (queryString.startsWith(Constants.RETURN) - || queryString.startsWith(Constants.WHERE) - || queryString.startsWith(Constants.FOREACH)) { - continue; - } else if (queryString.startsWith(Constants.ON_CREAT_ST)) onCreateSet(queryString); - else if (queryString.startsWith(Constants.ON_MATC_ST)) onMatchSet(queryString); - else if (queryString.startsWith(Constants.MATCH)) match(queryString); - else if (queryString.startsWith(Constants.CREATE)) create(queryString); - else if (queryString.startsWith(Constants.MERGE)) merge(queryString); - else if (queryString.startsWith(Constants.SET)) set(queryString); - else if (queryString.startsWith(Constants.DELETE)) delete(queryString); - else if (queryString.startsWith(Constants.REMOVE)) remove(queryString); - else if (queryString.startsWith(Constants.DETACH_DELET)) detachDelete(queryString); - } - - // Form the sentences out of different verb and objects - - // convert to ArrayList of key set - List alKeys = new ArrayList<>(operations.keySet()); - - Sentence originalSentence = null; - if (!operations.isEmpty()) { - int j = 1; - SentenceObject sentenceObject; - for (String keys : alKeys) { - ArrayList operation = operations.get(keys); - if (operation == null || operation.isEmpty()) continue; - - for (String string : operation) { - if (j == 1) { - originalSentence = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if (sentenceObject.getName() == null) sentenceObject.setName(Constants.EVERYTHING); - sentenceObject.setType( - Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - originalSentence.getObjects().add(sentenceObject); - j++; - } else { - Sentence decendant = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if (sentenceObject.getName() == null) sentenceObject.setName(Constants.EVERYTHING); - sentenceObject.setType( - Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - decendant.getObjects().add(sentenceObject); - originalSentence.getDescendants().add(decendant); - } - } - } - } - - return originalSentence; - } - - private void remove(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.REMOVE); - String alias = ""; - if (arr[1].contains(".")) { - String arrs[] = arr[1].split("\\.", 2); - alias = arrs[0].trim(); - } else if (arr[1].contains(":")) { - String arrs[] = arr[1].split("\\:", 2); - alias = arrs[0].trim(); - } - - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.REMOVE); - operations.put(alias, operationPerfomed); - } - - private void delete(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.DELETE); - String alias = arr[1].trim(); - if (alias.contains("-")) { - String arrs[] = alias.split("-", 2); - alias = arrs[0].trim(); - } - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.DELETE); - operations.put(alias, operationPerfomed); - } - - private void detachDelete(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.DETACH_DELET); - String alias = arr[1].trim(); - if (alias.contains("-")) { - String arrs[] = alias.split("-", 2); - alias = arrs[0].trim(); - } - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.DETACH_DELETE); - operations.put(alias, operationPerfomed); - } - - private void onMatchSet(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.ON_MATC_ST); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.ON_MATCH_SET); - operations.put(alias, operationPerfomed); - } - - private void onCreateSet(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.ON_CREAT_ST); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.ON_CREATE_SET); - operations.put(alias, operationPerfomed); - - } - - private void set(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.SET); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.SET); - operations.put(alias, operationPerfomed); - } - - private void match(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.MATCH).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.MATCH).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - private void create(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.CREATE).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.CREATE).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - private void merge(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.MERGE).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.MERGE).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - /* - * In order to fetch the alias and the node we use the indexing. On the - * basis of index, function/method is called. Variable map is used to store the alias - * and nodeName. Operation map is used to store the alias and operation - * performed. - */ - - private String squareBracket(String query, String operation) { - - String closingBracket = ""; - String value = ""; - String operationValue = operation; - ArrayList operationPerfomed = new ArrayList(); - - String[] arr = query.split("\\[", 2); + record.setDbName(dbName); - String[] arrs = arr[1].split(":", 2); - - if (arrs.length == 2) { - Pattern pattern = Pattern.compile("^[A-Za-z_|]+[\\sA-Za-z_|]*"); - Matcher matcher = pattern.matcher(arrs[1]); - if (matcher.find()) { - value = matcher.group(0); - } + // Time + String dateString = parseTimestamp(data); + String timeZone = null; + if (data.has(Constants.MIN_OFF)) { + timeZone = data.get(Constants.MIN_OFF).getAsString(); + } + Time time = getTime(dateString, timeZone); - // in order to get the alias only and remove all the extra characters. - // EX : -[rel returns rel - arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); + if (time != null) { + record.setTime(time); + } + // SeessionLocator + record.setSessionLocator(parseSessionLocator(data)); - pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); - matcher = pattern.matcher(arrs[0]); - if (!matcher.find()) { - arrs[0] = "" + randomValue; - randomValue++; - } + // Accessor + record.setAccessor(parseAccessor(data)); - if (operations.containsKey(arrs[0].trim())) { - operationPerfomed = operations.get(arrs[0]); - operationPerfomed.add(operationValue); - operations.replace(arrs[0].trim(), operationPerfomed); - } else { - operationPerfomed.add(operationValue); - operations.put(arrs[0].trim(), operationPerfomed); - } + Parser.parseSessionId(record); - variables.put(arrs[0].trim(), value.trim()); + // Data + if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { + record.setData(parseData(data)); + } else { + record.setException(parseException(data)); + } - String[] roundValue = arrs[1].split("\\]", 2); - closingBracket = roundValue[1]; - } + } + return record; + } - return closingBracket; +// ---------------------- Session Id ----------------------- - } + public void parseSessionId(Record record) { - protected String roundBracket(String query, String operation) { + Integer hashCode = (record.getSessionLocator().getClientIp() + record.getSessionLocator().getClientPort() + + record.getDbName()).hashCode(); + record.setSessionId(hashCode.toString()); + } + +// -----------------------------------------------Accessor----------------- + + public Accessor parseAccessor(JsonObject data) { + Accessor accessor = new Accessor(); + + accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); + accessor.setServerType(Constants.SERVER_TYPE_STRING); + + String dbUser = Constants.NOT_AVAILABLE; + if (data.has(Constants.DB_USER) && data.get(Constants.DB_USER) != null) { + dbUser = data.get(Constants.DB_USER).getAsString(); + } + accessor.setDbUser(dbUser); + + if(data.has(Constants.SERVER_HOSTNAME) && data.get(Constants.SERVER_HOSTNAME) != null) + accessor.setServerHostName(data.get(Constants.SERVER_HOSTNAME).getAsString()); + + String sourceProgram = Constants.UNKNOWN_STRING; + if (data.has(Constants.SOURCE_PROGRAM) && data.get(Constants.SOURCE_PROGRAM) != null) { + sourceProgram = data.get(Constants.SOURCE_PROGRAM).getAsString(); + } + accessor.setSourceProgram(sourceProgram); + + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.COMM_PROTOCOL); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + accessor.setServiceName(Constants.UNKNOWN_STRING); + + return accessor; + } + +// -----------------------------------------------SessionLocator----------------- + + public SessionLocator parseSessionLocator(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setIpv6(false); + + int clientPort = 0; + int serverPort = 0; + String clientIpAdd = Constants.NOT_AVAILABLE; + String serverIpAdd = Constants.NOT_AVAILABLE; + + if (data.has(Constants.CLIENT_IP)) { + String clientIpPortDetails = data.get(Constants.CLIENT_IP).getAsString(); + String clientIp = clientIpPortDetails.substring(7); + String[] clientIpPort = clientIp.split(":"); + clientIpAdd = clientIpPort[0]; + + if (isIPv6Address(clientIpAdd)) { + sessionLocator.setIpv6(true); + sessionLocator.setClientIpv6(clientIpAdd); + clientIpAdd = Constants.UNKNOWN_STRING; + } else { + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + } + + } + + if (data.has(Constants.SERVER_IP)) { + String serverIPPortDetails = data.get(Constants.SERVER_IP).getAsString(); + String serverIp = serverIPPortDetails.substring(7); + String[] serverIpPort = serverIp.split(":"); + serverIpAdd = serverIpPort[0]; + + if (isIPv6Address(serverIpAdd)) { + sessionLocator.setIpv6(true); + sessionLocator.setServerIpv6(serverIpAdd); + serverIpAdd = Constants.UNKNOWN_STRING; + } else { + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + } + } + + sessionLocator.setClientIp(clientIpAdd); + sessionLocator.setClientPort(clientPort); + sessionLocator.setServerIp(serverIpAdd); + sessionLocator.setServerPort(serverPort); + + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + + return sessionLocator; + } + + private boolean isIPv6Address(String ip) { + try { + InetAddress inetAddress = InetAddress.getByName(ip); + return inetAddress instanceof java.net.Inet6Address; + } catch (UnknownHostException e) { + return false; + } + } + +// -----------------------------------------------Timestamp----------------- + + public String parseTimestamp(final JsonObject data) { + String dateString = null; + if (data.has(Constants.TIMESTAMP)) { + dateString = data.get(Constants.TIMESTAMP).getAsString(); + } + return dateString; + } + + + public Time getTime(String dateString, String timeZone) { + if (dateString != null) { + JsonObject data = new JsonObject(); + LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); + ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); + if (timeZone != null) { + offset = ZoneOffset.of(timeZone); + } + ZonedDateTime zdt = dt.atOffset(offset).toZonedDateTime(); + long millis = zdt.toInstant().toEpochMilli(); + int minOffset = zdt.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); + } + return null; + } + +// -----------------------------------------------Exception----------------- + + public ExceptionRecord parseException(JsonObject data) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + + String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); + String[] queryData = queryStatement.split("'} - "); + if(queryData.length == 1){ + queryData = queryStatement.split("} - "); + } + + String query = queryData[0].toString(); + String exceptionDes = queryData[queryData.length - 1].toString(); + exceptionRecord.setDescription(exceptionDes); + + exceptionRecord.setSqlString(query); + return exceptionRecord; + } + +// -----------------------------------------------Data Construct---------------- + + public Data parseData(JsonObject inputJSON) { + Data data = new Data(); + try { + Construct construct = parseAsConstruct(inputJSON); + if (construct != null) { + data.setConstruct(construct); + + if (construct.getFullSql() == null || construct.getFullSql().isEmpty()) { + construct.setFullSql(Constants.UNKNOWN_STRING); + } + if (construct.getRedactedSensitiveDataSql() == null || construct.redactedSensitiveDataSql.isEmpty()) { + construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); + } + } + } catch (Exception e) { + log.error("Neo4j filter: Error parsing JSon in parser" + inputJSON, e); + throw e; + } + return data; + } + + public Construct parseAsConstruct(final JsonObject data) { + try { + final Sentence sentence = Parser.parseSentence(data); + + final Construct construct = new Construct(); + construct.sentences.add(sentence); + + String queryStatement = data.get(Constants.QUERY_STATEMENT).toString(); + String[] fullSql = queryStatement.split("runtime"); + String query[] = fullSql[0].split("- \\{\\} - "); + if(query[0].isEmpty()) + construct.setFullSql(fullSql[0].substring(1).trim()); + else + construct.setFullSql(query[0].substring(1).trim()); + construct.setRedactedSensitiveDataSql(Parser.parseRedactedSensitiveDataSql(data)); + return construct; + } catch (final Exception e) { + throw e; + } + } + + protected Sentence parseSentence(final JsonObject data) { + + Sentence sentence = null; + + if (data.has(Constants.QUERY_STATEMENT)) { + String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); + + sentence = parseQuery(queryStatement.trim()); + + //For Match (n) return n + if(sentence == null) { + Sentence validSentence = new Sentence("MATCH"); + SentenceObject sentenceObject = new SentenceObject("AllNodesOrRelationships"); + sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + validSentence.getObjects().add(sentenceObject); + sentence = validSentence; + } + + } + + return sentence; + } + + public static Sentence parseQuery(String query) { + + ArrayList gfg = new ArrayList() { + { + add(Constants.ON_CREAT_ST); + add(Constants.ON_MATC_ST); + add(Constants.MATCH); + add(Constants.CREATE); + add(Constants.MERGE); + add(Constants.DELETE); + add(Constants.DETACH_DELET); + add(Constants.REMOVE); + add(Constants.SET); + add(Constants.RETURN); + add(Constants.WHERE); + add(Constants.FOREACH); + } + }; + + if (query.contains(Constants.ON_CREATE_SET)) + query = query.replace(Constants.ON_CREATE_SET, Constants.ON_CREAT_ST); + if (query.contains(Constants.ON_MATCH_SET)) + query = query.replace(Constants.ON_MATCH_SET, Constants.ON_MATC_ST); + if (query.contains(Constants.DETACH_DELETE)) + query = query.replace(Constants.DETACH_DELETE, Constants.DETACH_DELET); + + // find the index of all the supported operations in the query. + Map ops = new TreeMap<>(); + + for (String operation : gfg) { + + int index = 0; + + while (true) { + index = query.indexOf(operation, index); + if (index == -1) + break; + ops.put(index, operation); + index++; + } + } + + // split the main query into multiple queries using index. + List queries = new ArrayList(); + + int i = 0; + int lastKeyValue = -1; + for (Map.Entry entry : ops.entrySet()) { + if (i == 0) { + i++; + lastKeyValue = entry.getKey(); + continue; + } + + String subquery = query.substring(lastKeyValue, entry.getKey()); + queries.add(subquery.trim()); + lastKeyValue = entry.getKey(); + + } + + if (lastKeyValue != -1) { + String subquery = query.substring(lastKeyValue); + queries.add(subquery); + } + + operations = new LinkedHashMap>(); + variables = new LinkedHashMap(); + + for (String queryString : queries) { + if (queryString.startsWith(Constants.RETURN) || queryString.startsWith(Constants.WHERE) + || queryString.startsWith(Constants.FOREACH)) + continue; + else if (queryString.startsWith(Constants.ON_CREAT_ST)) + onCreateSet(queryString); + else if (queryString.startsWith(Constants.ON_MATC_ST)) + onMatchSet(queryString); + else if (queryString.startsWith(Constants.MATCH)) + match(queryString); + else if (queryString.startsWith(Constants.CREATE)) + create(queryString); + else if (queryString.startsWith(Constants.MERGE)) + merge(queryString); + else if (queryString.startsWith(Constants.SET)) + set(queryString); + else if (queryString.startsWith(Constants.DELETE)) + delete(queryString); + else if (queryString.startsWith(Constants.REMOVE)) + remove(queryString); + else if (queryString.startsWith(Constants.DETACH_DELET)) + detachDelete(queryString); + } + + // Form the sentences out of different verb and objects + + // convert to ArrayList of key set + List alKeys = new ArrayList(operations.keySet()); + + SentenceObject sentenceObject = null; + + int j = 1; + Sentence originalSentence = null; + + for (String keys : alKeys) { + if (!operations.isEmpty()) { + ArrayList operation = operations.get(keys); + + for (String string : operation) { + + if (j == 1) { + originalSentence = new Sentence(string); + sentenceObject = new SentenceObject(variables.get(keys)); + if (sentenceObject.getName() == null) + sentenceObject.setName(Constants.EVERYTHING); + sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + originalSentence.getObjects().add(sentenceObject); + j++; + } else { + Sentence decendant = new Sentence(string); + sentenceObject = new SentenceObject(variables.get(keys)); + if (sentenceObject.getName() == null) + sentenceObject.setName(Constants.EVERYTHING); + sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + decendant.getObjects().add(sentenceObject); + originalSentence.getDescendants().add(decendant); + } + } + } + } + + return originalSentence; + } + + private void remove(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.REMOVE); + String alias = ""; + if (arr[1].contains(".")) { + String arrs[] = arr[1].split("\\.", 2); + alias = arrs[0].trim(); + } else if (arr[1].contains(":")) { + String arrs[] = arr[1].split("\\:", 2); + alias = arrs[0].trim(); + } + + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.REMOVE); + operations.put(alias, operationPerfomed); + } + + private void delete(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.DELETE); + String alias = arr[1].trim(); + if (alias.contains("-")) { + String arrs[] = alias.split("-", 2); + alias = arrs[0].trim(); + } + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.DELETE); + operations.put(alias, operationPerfomed); + } + + private void detachDelete(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.DETACH_DELET); + String alias = arr[1].trim(); + if (alias.contains("-")) { + String arrs[] = alias.split("-", 2); + alias = arrs[0].trim(); + } + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.DETACH_DELETE); + operations.put(alias, operationPerfomed); + } + + private void onMatchSet(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.ON_MATC_ST); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.ON_MATCH_SET); + operations.put(alias, operationPerfomed); + } + + private void onCreateSet(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.ON_CREAT_ST); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.ON_CREATE_SET); + operations.put(alias, operationPerfomed); + + } + + private void set(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.SET); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.SET); + operations.put(alias, operationPerfomed); + } + + private void match(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.MATCH).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.MATCH).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + private void create(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.CREATE).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.CREATE).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + private void merge(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.MERGE).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.MERGE).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + /* + * In order to fetch the alias and the node we use the indexing. On the + * basis of index, function/method is called. Variable map is used to store the alias + * and nodeName. Operation map is used to store the alias and operation + * performed. + */ + + private String squareBracket(String query, String operation) { + + String closingBracket = ""; + String value = ""; + String operationValue = operation; + ArrayList operationPerfomed = new ArrayList(); + + String[] arr = query.split("\\[", 2); + + String[] arrs = arr[1].split(":", 2); + + if (arrs.length == 2) { + Pattern pattern = Pattern.compile("^[A-Za-z_|]+[\\sA-Za-z_|]*"); + Matcher matcher = pattern.matcher(arrs[1]); + if (matcher.find()) { + value = matcher.group(0); + } + + // in order to get the alias only and remove all the extra characters. + // EX : -[rel returns rel + arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); - String closingBracket = ""; - String value = ""; - String operationValue = operation; - ArrayList operationPerfomed = new ArrayList(); + pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); + matcher = pattern.matcher(arrs[0]); + if (!matcher.find()) { + arrs[0] = "" + randomValue; + randomValue++; + } + + if (operations.containsKey(arrs[0].trim())) { + operationPerfomed = operations.get(arrs[0]); + operationPerfomed.add(operationValue); + operations.replace(arrs[0].trim(), operationPerfomed); + } else { + operationPerfomed.add(operationValue); + operations.put(arrs[0].trim(), operationPerfomed); + } - String[] arr = query.split("\\(", 2); + variables.put(arrs[0].trim(), value.trim()); - if (arr[1].indexOf(")") < arr[1].indexOf(":")) { - String temp[] = arr[1].split("\\)", 2); - arr[1] = temp[1]; - } + String[] roundValue = arrs[1].split("\\]", 2); + closingBracket = roundValue[1]; + } + + return closingBracket; - String[] arrs = arr[1].split(":", 2); + } + + protected String roundBracket(String query, String operation) { - if (arrs.length == 2) { - Pattern pattern = Pattern.compile("^[A-Za-z_]+[\\sA-Za-z_]*"); - Matcher matcher = pattern.matcher(arrs[1]); - if (matcher.find()) { - value = matcher.group(0); - } + String closingBracket = ""; + String value = ""; + String operationValue = operation; + ArrayList operationPerfomed = new ArrayList(); - // in order to get the alias only and remove all the extra characters. - // EX : -[rel returns rel - arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); + String[] arr = query.split("\\(", 2); - pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); - matcher = pattern.matcher(arrs[0]); - if (!matcher.find()) { - arrs[0] = "" + randomValue; - randomValue++; - } + if (arr[1].indexOf(")") < arr[1].indexOf(":")) { + String temp[] = arr[1].split("\\)", 2); + arr[1] = temp[1]; + } + + String[] arrs = arr[1].split(":", 2); + + if (arrs.length == 2) { + Pattern pattern = Pattern.compile("^[A-Za-z_]+[\\sA-Za-z_]*"); + Matcher matcher = pattern.matcher(arrs[1]); + if (matcher.find()) { + value = matcher.group(0); + } - if (operations.containsKey(arrs[0].trim())) { - operationPerfomed = operations.get(arrs[0]); - operations.remove(arrs[0]); - } - operationPerfomed.add(operationValue); - operations.put(arrs[0].trim(), operationPerfomed); + // in order to get the alias only and remove all the extra characters. + // EX : -[rel returns rel + arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); - variables.put(arrs[0].trim(), value.trim()); + pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); + matcher = pattern.matcher(arrs[0]); + if (!matcher.find()) { + arrs[0] = "" + randomValue; + randomValue++; + } + + if (operations.containsKey(arrs[0].trim())) { + operationPerfomed = operations.get(arrs[0]); + operations.remove(arrs[0]); + } + operationPerfomed.add(operationValue); + operations.put(arrs[0].trim(), operationPerfomed); - String[] roundValue = arrs[1].split("\\)", 2); - closingBracket = roundValue[1]; - } + variables.put(arrs[0].trim(), value.trim()); - return closingBracket; + String[] roundValue = arrs[1].split("\\)", 2); + closingBracket = roundValue[1]; + } + + return closingBracket; - } + } - protected String parseRedactedSensitiveDataSql(JsonObject data) { - String redactedData = ""; - redactedData = data.get(Constants.MESSAGE).getAsString(); + protected String parseRedactedSensitiveDataSql(JsonObject data) { + String redactedData = ""; + redactedData = data.get(Constants.MESSAGE).getAsString(); - return redactedData; - } + return redactedData; + } -} +} \ No newline at end of file From 565b158beebcbc8a25d38790934b0a5046a7b290 Mon Sep 17 00:00:00 2001 From: "Andy.Chen" Date: Mon, 25 Aug 2025 22:12:09 -0400 Subject: [PATCH 3/4] GRD-106525 fix Neo4j outofbounds exception Signed-off-by: Andy.Chen --- .../com/ibm/guardium/neodb/ParserTest.java | 613 +++++++++--------- 1 file changed, 298 insertions(+), 315 deletions(-) diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java index 5f6083cd4..183ef4e44 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java @@ -11,328 +11,311 @@ import org.junit.Assert; import org.junit.Test; -import java.text.ParseException; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; public class ParserTest { - Parser parser; - String neoSuccessString = "2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoSuccessString_grokOutput = "{\n" + - " \"ts\": \"\\\"2021-08-06 17:09:40.008+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:51372\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\\\";\"\n" + - " }"; - - @Test - public void testParseAsConstruct_Match() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - final Construct result = parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("MATCH", sentence.getVerb().trim()); - Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseAsConstruct_Create() { - parser = new Parser(); - String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput, neoString); - - JsonObject inputData = inputData(e); - - final Construct result = parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("CREATE", sentence.getVerb().trim()); - Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - - @Test - public void testDBNameAndServiceNameEqual() throws ParseException { - parser = new Parser(); - String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4jDB \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - - Event e = getParsedEvent(neoString_grokOutput, neoString); - e.setField("minoff", "+07:00"); - - JsonObject inputData = inputData(e); - - final Record record = parser.parseRecord(inputData); - Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); - } - - @Test - public void testParseAsConstruct_Merge() { - parser = new Parser(); - String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:56:12.097+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput, neoString); - e.setField("minoff", "+04:00"); - - JsonObject inputData = inputData(e); - - final Construct result = parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("MERGE", sentence.getVerb().trim()); - Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseException_DELETE() { - parser = new Parser(); - String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-03-03 08:49:32.367+0000\",\n" + - " \"log_level\": \"ERROR\",\n" + - " \"metadata1\": \"7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.2.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:62845\",\n" + - " \"server_ip\": \"server/127.0.0.1:7687\",\n" + - " \"dbname\": \" \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))\\\";\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput, neoString); - JsonObject inputData = inputData(e); - e.setField("minoff", "+07:00"); - - final ExceptionRecord exceptionRecord = parser.parseException(inputData); - - Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); - Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); - } - - @Test - public void testParseAccessor() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final Accessor accessor = parser.parseAccessor(inputData); - - Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); - Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); - Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); - Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); - } - - @Test - public void testParseSessionLocator() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final SessionLocator sessionLocator = parser.parseSessionLocator(inputData); - - Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); - Assert.assertEquals(-1, sessionLocator.getClientPort()); - Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); - Assert.assertEquals(-1, sessionLocator.getServerPort()); - - } - - @Test - public void testParseSessionId() throws ParseException { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final Record record = parser.parseRecord(inputData); - - Assert.assertEquals(Constants.UNKNOWN_STRING, record.getSessionId()); - - } - - @Test - public void testParseTimestamp() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); - JsonObject inputData = inputData(e); - - final String timestamp = parser.parseTimestamp(inputData); - - Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); - - } - - @Test - public void testGetTime() { - parser = new Parser(); - String dateString = "2021-01-25 11:17:09.099+0000"; - String timeZone = "-04:00"; - final Time time = parser.getTime(dateString, timeZone); - - Assert.assertEquals(0, time.getMinDst()); - Assert.assertEquals(-240, time.getMinOffsetFromGMT()); - Assert.assertEquals(1611587829099L, time.getTimstamp()); - - } - - @Test - public void testParseSentence() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final Sentence sentence = parser.parseSentence(inputData); - - Assert.assertEquals("MATCH", sentence.getVerb()); - Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - - } - - @Test - public void testParseRedactedSensitiveDataSql() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final String redacted = parser.parseRedactedSensitiveDataSql(inputData); - - Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); - - } - + Parser parser; + String neoSuccessString = "2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoSuccessString_grokOutput = "{\n" + + " \"ts\": \"\\\"2021-08-06 17:09:40.008+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:51372\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\\\";\"\n" + + " }"; + + @Test + public void testParseAsConstruct_Match() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + final Construct result = Parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("MATCH", sentence.getVerb().trim()); + Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testParseAsConstruct_Create() { + parser = new Parser(); + String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput,neoString); + + JsonObject inputData = inputData(e); + + final Construct result = Parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("CREATE", sentence.getVerb().trim()); + Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testDBNameAndServiceNameEqual() throws ParseException { + parser = new Parser(); + String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4jDB \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + + Event e = getParsedEvent(neoString_grokOutput, neoString); + e.setField("minoff", "+07:00"); + + JsonObject inputData = inputData(e); + + final Record record = parser.parseRecord(inputData); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + } + + @Test + public void testParseAsConstruct_Merge() { + parser = new Parser(); + String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:56:12.097+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput,neoString); + e.setField("minoff", "+04:00"); + JsonObject inputData = inputData(e); + + final Construct result = Parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("MERGE", sentence.getVerb().trim()); + Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testParseException_DELETE() { + String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-03-03 08:49:32.367+0000\",\n" + + " \"log_level\": \"ERROR\",\n" + + " \"metadata1\": \"7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.2.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:62845\",\n" + + " \"server_ip\": \"server/127.0.0.1:7687\",\n" + + " \"dbname\": \" \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))\\\";\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + JsonObject inputData = inputData(e); + e.setField("minoff", "+07:00"); + + final ExceptionRecord exceptionRecord = Parser.parseException(inputData); + + Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); + Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); + } + + @Test + public void testParseAccessor() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput,neoSuccessString); + + JsonObject inputData = inputData(e); + + final Accessor accessor = Parser.parseAccessor(inputData); + + Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); + Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); + Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); + Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); + } + + @Test + public void testParseSessionLocator() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final SessionLocator sessionLocator = Parser.parseSessionLocator(inputData); + + Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); + Assert.assertEquals(51372, sessionLocator.getClientPort()); + Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); + Assert.assertEquals(11004, sessionLocator.getServerPort()); + + } + + @Test + public void testParseTimestamp() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); + JsonObject inputData = inputData(e); + + final String timestamp = Parser.parseTimestamp(inputData); + + Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); + + } + + @Test + public void testGetTime() { + parser = new Parser(); + String dateString = "2021-01-25 11:17:09.099+0000"; + String timeZone = "-04:00"; + final Time time = Parser.getTime(dateString, timeZone); + + Assert.assertEquals(0, time.getMinDst()); + Assert.assertEquals(-240, time.getMinOffsetFromGMT()); + Assert.assertEquals(1611587829099L, time.getTimstamp()); + + } + + @Test + public void testParseSentence() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Sentence sentence = Parser.parseSentence(inputData); + + Assert.assertEquals("MATCH", sentence.getVerb()); + Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + + } + + @Test + public void testParseRedactedSensitiveDataSql() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final String redacted = Parser.parseRedactedSensitiveDataSql(inputData); + + Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); + + } + // ----------------------------------- --------------------------------------------------- - - private JsonObject inputData(Event e) { - JsonObject data = new JsonObject(); - - if (e.getField(Constants.CLIENT_IP).toString() != null && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { - data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); - } - if (e.getField(Constants.SERVER_IP).toString() != null && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { - data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); - } - if (e.getField(Constants.DB_PROTOCOL).toString() != null && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { - data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); - } - if (e.getField(Constants.TIMESTAMP).toString() != null && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { - data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); - } - if (e.getField(Constants.LOG_LEVEL).toString() != null && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { - data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); - } - if (e.getField(Constants.DB_USER).toString() != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { - data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); - } - if (e.getField(Constants.DB_NAME).toString() != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { - data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); - } - if (e.getField(Constants.SOURCE_PROGRAM).toString() != null && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { - data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); - } - if (e.getField(Constants.QUERY_STATEMENT).toString() != null && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { - data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); - } - if (e.getField(Constants.MESSAGE).toString() != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { - data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); - } - return data; - } - - - public static Event getParsedEvent(String logEvent_json, String logEvent) { - - Map results = parseJson(logEvent_json); - Event e = new org.logstash.Event(); - e.setField("message", logEvent); - - e.setField(Constants.CLIENT_IP, results.get(Constants.CLIENT_IP)); - e.setField(Constants.SERVER_IP, results.get(Constants.SERVER_IP)); - e.setField(Constants.DB_PROTOCOL, results.get(Constants.DB_PROTOCOL)); - e.setField(Constants.TIMESTAMP, results.get(Constants.TIMESTAMP)); - e.setField(Constants.LOG_LEVEL, results.get(Constants.LOG_LEVEL)); - e.setField(Constants.DB_USER, results.get(Constants.DB_USER)); - e.setField(Constants.DB_NAME, results.get(Constants.DB_NAME)); - e.setField(Constants.SOURCE_PROGRAM, results.get(Constants.SOURCE_PROGRAM)); - e.setField(Constants.QUERY_STATEMENT, results.get(Constants.QUERY_STATEMENT)); - - return e; - } - - - public static Map parseJson(String jsonString) { - Map map = new HashMap<>(); - - // Use a regular expression to match each key-value pair in the JSON string - String pattern = "\"(.*?)\":\\s*\"(.*?)\""; - Pattern r = Pattern.compile(pattern); - Matcher m = r.matcher(jsonString); - - while (m.find()) { - // Add each key-value pair to the map - String key = m.group(1); - String value = m.group(2).replaceAll("(? results = parseJson(logEvent_json); + Event e = new org.logstash.Event(); + e.setField("message", logEvent); + + e.setField(Constants.CLIENT_IP, results.get(Constants.CLIENT_IP)); + e.setField(Constants.SERVER_IP, results.get(Constants.SERVER_IP)); + e.setField(Constants.DB_PROTOCOL, results.get(Constants.DB_PROTOCOL)); + e.setField(Constants.TIMESTAMP, results.get(Constants.TIMESTAMP)); + e.setField(Constants.LOG_LEVEL, results.get(Constants.LOG_LEVEL)); + e.setField(Constants.DB_USER, results.get(Constants.DB_USER)); + e.setField(Constants.DB_NAME, results.get(Constants.DB_NAME)); + e.setField(Constants.SOURCE_PROGRAM, results.get(Constants.SOURCE_PROGRAM)); + e.setField(Constants.QUERY_STATEMENT, results.get(Constants.QUERY_STATEMENT)); + + return e; + } + + + public static Map parseJson(String jsonString) { + Map map = new HashMap<>(); + + // Use a regular expression to match each key-value pair in the JSON string + String pattern = "\"(.*?)\":\\s*\"(.*?)\""; + Pattern r = Pattern.compile(pattern); + Matcher m = r.matcher(jsonString); + + while (m.find()) { + // Add each key-value pair to the map + String key = m.group(1); + String value = m.group(2).replaceAll("(? Date: Mon, 25 Aug 2025 22:31:34 -0400 Subject: [PATCH 4/4] GRD-106525 fix Neo4j outofbounds exception Signed-off-by: Andy.Chen --- .../logstash-filter-neo4j-guardium/gradlew | 0 .../guardium/neodb/NeodbGuardiumFilter.java | 269 ++-- .../java/com/ibm/guardium/neodb/Parser.java | 1293 ++++++++--------- .../com/ibm/guardium/neodb/ParserTest.java | 613 ++++---- 4 files changed, 1089 insertions(+), 1086 deletions(-) mode change 100644 => 100755 filter-plugin/logstash-filter-neo4j-guardium/gradlew diff --git a/filter-plugin/logstash-filter-neo4j-guardium/gradlew b/filter-plugin/logstash-filter-neo4j-guardium/gradlew old mode 100644 new mode 100755 diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java index b2b32cb19..0b043b275 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/NeodbGuardiumFilter.java @@ -9,6 +9,7 @@ import java.text.ParseException; import java.util.Collection; import java.util.Collections; + import com.google.gson.JsonElement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,140 +32,142 @@ @LogstashPlugin(name = "neodb_guardium_filter") public class NeodbGuardiumFilter implements Filter { - private static Logger logger = LogManager.getLogger(NeodbGuardiumFilter.class); + private static Logger logger = LogManager.getLogger(NeodbGuardiumFilter.class); + + public static final String LOG42_CONF = "log4j2uc.properties"; - public static final String LOG42_CONF = "log4j2uc.properties"; - static { - try { - String uc_etc = System.getenv("UC_ETC"); - LoggerContext context = (LoggerContext) LogManager.getContext(false); - File file = new File(uc_etc + File.separator + LOG42_CONF); - context.setConfigLocation(file.toURI()); - } catch (Exception e) { - e.printStackTrace(); - } - } + static { + try { + String uc_etc = System.getenv("UC_ETC"); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + File file = new File(uc_etc + File.separator + LOG42_CONF); + context.setConfigLocation(file.toURI()); + } catch (Exception e) { + e.printStackTrace(); + } + } - private String id; - public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); - private static Logger log = LogManager.getLogger(NeodbGuardiumFilter.class); + private String id; + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + private static Logger log = LogManager.getLogger(NeodbGuardiumFilter.class); Parser parser = new Parser(); - public NeodbGuardiumFilter(String id, Configuration config, Context context) { - this.id = id; - } - - @Override - public Collection> configSchema() { - // should return a list of all configuration options for this plugin - return Collections.singletonList(SOURCE_CONFIG); - } - - @Override - public String getId() { - return this.id; - } - - @Override - public Collection filter(Collection events, FilterMatchListener matchListener) { - - for (Event e : events) { - if(logger.isDebugEnabled()){ - logger.debug("Event Now: {}", e.getData()); - } - - try { - JsonObject inputData = inputData(e); - - if (isNotSystemGeneratedEvent(inputData)) { - - Record record = Parser.parseRecord(inputData); - - final GsonBuilder builder = new GsonBuilder(); - builder.serializeNulls(); - final Gson gson = builder.create(); - e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); - - matchListener.filterMatched(e); - } else { - e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_NEO); - } - - } catch (ParseException pe) { - log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), pe); - e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); - } catch (Exception exception) { - log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), - exception); - e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); - } - } - return events; - } - - private boolean isNotSystemGeneratedEvent(JsonObject inputData) { - if (inputData == null || !inputData.has(Constants.QUERY_STATEMENT)) { - return false; // Treat as system-generated if missing QUERY_STATEMENT or inputData is null - } - - JsonElement queryElement = inputData.get(Constants.QUERY_STATEMENT); - if (queryElement == null || queryElement.isJsonNull()) { - return false; // Treat as system-generated if queryElement is null or JsonNull - } - - String queryStatement = queryElement.getAsString(); - if (queryStatement == null || queryStatement.isEmpty() || queryStatement.startsWith("EXPLAIN")) { - return false; // Treat as system-generated if queryStatement is null, empty, or starts with "EXPLAIN" - } - - return true; // Otherwise, it is not a system-generated event - } - - private JsonObject inputData(Event e) { - JsonObject data = new JsonObject(); - - if (e.getField(Constants.CLIENT_IP) != null - && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { - data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); - } - if (e.getField(Constants.SERVER_IP) != null - && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { - data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); - } - if (e.getField(Constants.DB_PROTOCOL) != null - && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { - data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); - } - if (e.getField(Constants.TIMESTAMP) != null - && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { - data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); - } - if (e.getField(Constants.LOG_LEVEL) != null - && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { - data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); - } - if (e.getField(Constants.DB_USER) != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { - data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); - } - if (e.getField(Constants.DB_NAME) != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { - data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); - } - if (e.getField(Constants.SOURCE_PROGRAM) != null - && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { - data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); - } - if (e.getField(Constants.QUERY_STATEMENT) != null - && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { - data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); - } - if (e.getField(Constants.SERVER_HOSTNAME) != null - && !e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { - data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); - } - if (e.getField(Constants.MESSAGE) != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { - data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); - } - return data; - } - -} \ No newline at end of file + + public NeodbGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + + for (Event e : events) { + if (logger.isDebugEnabled()) { + logger.debug("Event Now: {}", e.getData()); + } + + try { + JsonObject inputData = inputData(e); + + if (isNotSystemGeneratedEvent(inputData)) { + + Record record = parser.parseRecord(inputData); + + final GsonBuilder builder = new GsonBuilder(); + builder.serializeNulls(); + final Gson gson = builder.create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + matchListener.filterMatched(e); + } else { + e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_NEO); + } + + } catch (ParseException pe) { + log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), pe); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } catch (Exception exception) { + log.error("Neo4j filter: Error parsing neo4j event " + e.getField(Constants.MESSAGE).toString(), + exception); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } + } + return events; + } + + private boolean isNotSystemGeneratedEvent(JsonObject inputData) { + if (inputData == null || !inputData.has(Constants.QUERY_STATEMENT)) { + return false; // Treat as system-generated if missing QUERY_STATEMENT or inputData is null + } + + JsonElement queryElement = inputData.get(Constants.QUERY_STATEMENT); + if (queryElement == null || queryElement.isJsonNull()) { + return false; // Treat as system-generated if queryElement is null or JsonNull + } + + String queryStatement = queryElement.getAsString(); + if (queryStatement == null || queryStatement.isEmpty() || queryStatement.startsWith("EXPLAIN")) { + return false; // Treat as system-generated if queryStatement is null, empty, or starts with "EXPLAIN" + } + + return true; // Otherwise, it is not a system-generated event + } + + private JsonObject inputData(Event e) { + JsonObject data = new JsonObject(); + + if (e.getField(Constants.CLIENT_IP) != null + && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { + data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); + } + if (e.getField(Constants.SERVER_IP) != null + && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { + data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); + } + if (e.getField(Constants.DB_PROTOCOL) != null + && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()) { + data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); + } + if (e.getField(Constants.TIMESTAMP) != null + && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { + data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); + } + if (e.getField(Constants.LOG_LEVEL) != null + && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()) { + data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); + } + if (e.getField(Constants.DB_USER) != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { + data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); + } + if (e.getField(Constants.DB_NAME) != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { + data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); + } + if (e.getField(Constants.SOURCE_PROGRAM) != null + && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()) { + data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); + } + if (e.getField(Constants.QUERY_STATEMENT) != null + && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { + data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); + } + if (e.getField(Constants.SERVER_HOSTNAME) != null + && !e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { + data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); + } + if (e.getField(Constants.MESSAGE) != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { + data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); + } + return data; + } + +} diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java index 3fb548757..d3993f428 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java @@ -4,6 +4,7 @@ // package com.ibm.guardium.neodb; + import java.net.InetAddress; import java.net.UnknownHostException; import java.text.ParseException; @@ -12,11 +13,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -35,686 +32,672 @@ public class Parser { - private Logger log = LogManager.getLogger(Parser.class); + private Logger log = LogManager.getLogger(Parser.class); + + private final DateTimeFormatterBuilder dateTimeFormatterBuilder = + new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS[[XXX][X]]")); + + private final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); + + LinkedHashMap> operations = null; + LinkedHashMap variables = null; + static Character randomValue = 65; + + public Record parseRecord(final JsonObject data) throws ParseException { + Record record = new Record(); + + if (data != null) { + + record.setAppUserName(Constants.NOT_AVAILABLE); + + // DB Name + String dbName = Constants.NOT_AVAILABLE; + + if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { + dbName = data.get(Constants.DB_NAME).getAsString(); + } + + record.setDbName(dbName); + + // Time + String dateString = parseTimestamp(data); + String timeZone = null; + if (data.has(Constants.MIN_OFF)) { + timeZone = data.get(Constants.MIN_OFF).getAsString(); + } + Time time = getTime(dateString, timeZone); + + if (time != null) { + record.setTime(time); + } + // SessionLocator + record.setSessionLocator(parseSessionLocator(data)); + + // Accessor + record.setAccessor(parseAccessor(data)); + + record.setSessionId(Constants.UNKNOWN_STRING); - private final DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS[[XXX][X]]")); + // Data + if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { + record.setData(parseData(data)); + } else { + record.setException(parseException(data)); + } - private final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); + } + return record; + } - static LinkedHashMap> operations = null; - static LinkedHashMap variables = null; - static Character randomValue = 65; + // -----------------------------------------------Accessor----------------- - public Record parseRecord(final JsonObject data) throws ParseException { - Record record = new Record(); + public Accessor parseAccessor(JsonObject data) { + Accessor accessor = new Accessor(); - if (data != null) { + accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); + accessor.setServerType(Constants.SERVER_TYPE_STRING); - record.setAppUserName(Constants.NOT_AVAILABLE); + String dbUser = Constants.NOT_AVAILABLE; + if (data.has(Constants.DB_USER) && data.get(Constants.DB_USER) != null) { + dbUser = data.get(Constants.DB_USER).getAsString(); + } + accessor.setDbUser(dbUser); - // DB Name - String dbName = Constants.NOT_AVAILABLE; + if (data.has(Constants.SERVER_HOSTNAME) && data.get(Constants.SERVER_HOSTNAME) != null) + accessor.setServerHostName(data.get(Constants.SERVER_HOSTNAME).getAsString()); - if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { - dbName = data.get(Constants.DB_NAME).getAsString(); - } + String sourceProgram = Constants.UNKNOWN_STRING; + if (data.has(Constants.SOURCE_PROGRAM) && data.get(Constants.SOURCE_PROGRAM) != null) { + sourceProgram = data.get(Constants.SOURCE_PROGRAM).getAsString(); + } + accessor.setSourceProgram(sourceProgram); - record.setDbName(dbName); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.COMM_PROTOCOL); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + + String serviceName = Constants.NOT_AVAILABLE; + + if (data.has(Constants.DB_NAME) && data.get(Constants.DB_NAME) != null) { + serviceName = data.get(Constants.DB_NAME).getAsString(); + } + + accessor.setServiceName(serviceName); + + return accessor; + } + + // -----------------------------------------------SessionLocator----------------- + + public SessionLocator parseSessionLocator(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setIpv6(false); + + int clientPort = Constants.DEFAULT_PORT; + int serverPort = Constants.DEFAULT_PORT; + String clientIpAdd = Constants.NOT_AVAILABLE; + String serverIpAdd = Constants.NOT_AVAILABLE; + + if (data.has(Constants.CLIENT_IP)) { + String clientIpPortDetails = data.get(Constants.CLIENT_IP).getAsString(); + String clientIp = clientIpPortDetails.substring(7); + String[] clientIpPort = clientIp.split(":"); + clientIpAdd = clientIpPort[0]; + + if (isIPv6Address(clientIpAdd)) { + sessionLocator.setIpv6(true); + sessionLocator.setClientIpv6(clientIpAdd); + clientIpAdd = Constants.UNKNOWN_STRING; + } else { + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + } + + } + + if (data.has(Constants.SERVER_IP)) { + String serverIPPortDetails = data.get(Constants.SERVER_IP).getAsString(); + String serverIp = serverIPPortDetails.substring(7); + String[] serverIpPort = serverIp.split(":"); + serverIpAdd = serverIpPort[0]; + + + if (isIPv6Address(serverIpAdd)) { + sessionLocator.setIpv6(true); + sessionLocator.setServerIpv6(serverIpAdd); + serverIpAdd = Constants.UNKNOWN_STRING; + } else { + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + } + + } + + sessionLocator.setClientIp(clientIpAdd); + sessionLocator.setClientPort(clientPort); + sessionLocator.setServerIp(serverIpAdd); + sessionLocator.setServerPort(serverPort); + + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + + return sessionLocator; + } + + private boolean isIPv6Address(String ip) { + try { + InetAddress inetAddress = InetAddress.getByName(ip); + return inetAddress instanceof java.net.Inet6Address; + } catch (UnknownHostException e) { + return false; + } + } + + // -----------------------------------------------Timestamp----------------- + + public String parseTimestamp(final JsonObject data) { + String dateString = null; + if (data.has(Constants.TIMESTAMP)) { + dateString = data.get(Constants.TIMESTAMP).getAsString(); + } + return dateString; + } + + public Time getTime(String dateString, String timeZone) { + if (dateString != null) { + JsonObject data = new JsonObject(); + LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); + ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); + if (timeZone != null) { + offset = ZoneOffset.of(timeZone); + } + ZonedDateTime zdt = dt.atOffset(offset).toZonedDateTime(); + long millis = zdt.toInstant().toEpochMilli(); + int minOffset = zdt.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); + } + return null; + } + + // -----------------------------------------------Exception----------------- + + public ExceptionRecord parseException(JsonObject data) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + + String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); + String[] queryData = queryStatement.split("'} - "); + if (queryData.length == 1) { + queryData = queryStatement.split("} - "); + } + + String query = queryData[0].toString(); + String exceptionDes = queryData[queryData.length - 1].toString(); + exceptionRecord.setDescription(exceptionDes); + + exceptionRecord.setSqlString(query); + return exceptionRecord; + } + + // -----------------------------------------------Data Construct---------------- + + public Data parseData(JsonObject inputJSON) { + Data data = new Data(); + try { + Construct construct = parseAsConstruct(inputJSON); + if (construct != null) { + data.setConstruct(construct); + + if (construct.getFullSql() == null || construct.getFullSql().isEmpty()) { + construct.setFullSql(Constants.UNKNOWN_STRING); + } + if (construct.getRedactedSensitiveDataSql() == null || construct.redactedSensitiveDataSql.isEmpty()) { + construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); + } + } + } catch (Exception e) { + log.error("Neo4j filter: Error parsing JSon in parser" + inputJSON, e); + throw e; + } + return data; + } + + public Construct parseAsConstruct(final JsonObject data) { + try { + final Sentence sentence = parseSentence(data); + + final Construct construct = new Construct(); + construct.sentences.add(sentence); + + String queryStatement = data.get(Constants.QUERY_STATEMENT).toString(); + String[] fullSql = queryStatement.split("runtime"); + String query[] = fullSql[0].split("- \\{\\} - "); + if (query[0].isEmpty()) + construct.setFullSql(fullSql[0].substring(1).trim()); + else + construct.setFullSql(query[0].substring(1).trim()); + construct.setRedactedSensitiveDataSql(parseRedactedSensitiveDataSql(data)); + return construct; + } catch (final Exception e) { + throw e; + } + } + + protected Sentence parseSentence(final JsonObject data) { + + Sentence sentence = null; + + if (data.has(Constants.QUERY_STATEMENT)) { + String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); + + sentence = parseQuery(queryStatement.trim()); + + //For Match (n) return n + if (sentence == null) { + Sentence validSentence = new Sentence("MATCH"); + SentenceObject sentenceObject = new SentenceObject("AllNodesOrRelationships"); + sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + validSentence.getObjects().add(sentenceObject); + sentence = validSentence; + } + + } + + return sentence; + } + + Sentence parseQuery(String query) { + List gfg = + new ArrayList<>( + Arrays.asList( + Constants.ON_CREAT_ST, + Constants.ON_MATC_ST, + Constants.MATCH, + Constants.CREATE, + Constants.MERGE, + Constants.DELETE, + Constants.DETACH_DELET, + Constants.REMOVE, + Constants.SET, + Constants.RETURN, + Constants.WHERE, + Constants.FOREACH)); + + if (query.contains(Constants.ON_CREATE_SET)) + query = query.replace(Constants.ON_CREATE_SET, Constants.ON_CREAT_ST); + if (query.contains(Constants.ON_MATCH_SET)) + query = query.replace(Constants.ON_MATCH_SET, Constants.ON_MATC_ST); + if (query.contains(Constants.DETACH_DELETE)) + query = query.replace(Constants.DETACH_DELETE, Constants.DETACH_DELET); + + // find the index of all the supported operations in the query. + Map ops = new TreeMap<>(); + + for (String operation : gfg) { + + int index = 0; + + while (true) { + index = query.indexOf(operation, index); + if (index == -1) break; + ops.put(index, operation); + index++; + } + } + + // split the main query into multiple queries using index. + List queries = new ArrayList<>(); + + int i = 0; + int lastKeyValue = -1; + for (Map.Entry entry : ops.entrySet()) { + if (i == 0) { + i++; + lastKeyValue = entry.getKey(); + continue; + } + + String subquery = query.substring(lastKeyValue, entry.getKey()); + queries.add(subquery.trim()); + lastKeyValue = entry.getKey(); + } + + if (lastKeyValue != -1) { + String subquery = query.substring(lastKeyValue); + queries.add(subquery); + } + + operations = new LinkedHashMap<>(); + variables = new LinkedHashMap<>(); + + for (String queryString : queries) { + if (queryString.startsWith(Constants.RETURN) + || queryString.startsWith(Constants.WHERE) + || queryString.startsWith(Constants.FOREACH)) { + continue; + } else if (queryString.startsWith(Constants.ON_CREAT_ST)) onCreateSet(queryString); + else if (queryString.startsWith(Constants.ON_MATC_ST)) onMatchSet(queryString); + else if (queryString.startsWith(Constants.MATCH)) match(queryString); + else if (queryString.startsWith(Constants.CREATE)) create(queryString); + else if (queryString.startsWith(Constants.MERGE)) merge(queryString); + else if (queryString.startsWith(Constants.SET)) set(queryString); + else if (queryString.startsWith(Constants.DELETE)) delete(queryString); + else if (queryString.startsWith(Constants.REMOVE)) remove(queryString); + else if (queryString.startsWith(Constants.DETACH_DELET)) detachDelete(queryString); + } + + // Form the sentences out of different verb and objects + + // convert to ArrayList of key set + List alKeys = new ArrayList<>(operations.keySet()); + + Sentence originalSentence = null; + if (!operations.isEmpty()) { + int j = 1; + SentenceObject sentenceObject; + for (String keys : alKeys) { + ArrayList operation = operations.get(keys); + if (operation == null || operation.isEmpty()) continue; + + for (String string : operation) { + if (j == 1) { + originalSentence = new Sentence(string); + sentenceObject = new SentenceObject(variables.get(keys)); + if (sentenceObject.getName() == null) sentenceObject.setName(Constants.EVERYTHING); + sentenceObject.setType( + Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + originalSentence.getObjects().add(sentenceObject); + j++; + } else { + Sentence decendant = new Sentence(string); + sentenceObject = new SentenceObject(variables.get(keys)); + if (sentenceObject.getName() == null) sentenceObject.setName(Constants.EVERYTHING); + sentenceObject.setType( + Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS + + decendant.getObjects().add(sentenceObject); + originalSentence.getDescendants().add(decendant); + } + } + } + } + + return originalSentence; + } + + private void remove(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.REMOVE); + String alias = ""; + if (arr[1].contains(".")) { + String arrs[] = arr[1].split("\\.", 2); + alias = arrs[0].trim(); + } else if (arr[1].contains(":")) { + String arrs[] = arr[1].split("\\:", 2); + alias = arrs[0].trim(); + } + + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.REMOVE); + operations.put(alias, operationPerfomed); + } + + private void delete(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.DELETE); + String alias = arr[1].trim(); + if (alias.contains("-")) { + String arrs[] = alias.split("-", 2); + alias = arrs[0].trim(); + } + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.DELETE); + operations.put(alias, operationPerfomed); + } + + private void detachDelete(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.DETACH_DELET); + String alias = arr[1].trim(); + if (alias.contains("-")) { + String arrs[] = alias.split("-", 2); + alias = arrs[0].trim(); + } + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.DETACH_DELETE); + operations.put(alias, operationPerfomed); + } + + private void onMatchSet(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.ON_MATC_ST); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.ON_MATCH_SET); + operations.put(alias, operationPerfomed); + } + + private void onCreateSet(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.ON_CREAT_ST); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.ON_CREATE_SET); + operations.put(alias, operationPerfomed); + + } + + private void set(String queryString) { + + ArrayList operationPerfomed = new ArrayList<>(); + String arr[] = queryString.split(Constants.SET); + String arrs[] = arr[1].split("\\.", 2); + String alias = arrs[0].trim(); + if (operations.containsKey(alias)) { + operationPerfomed = operations.get(alias); + operations.remove(alias); + } + operationPerfomed.add(Constants.SET); + operations.put(alias, operationPerfomed); + } + + private void match(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.MATCH).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.MATCH).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + private void create(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.CREATE).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.CREATE).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + private void merge(String query) { + + String closingBracket = query; + + do { + int roundIndex = closingBracket.indexOf("("); + int squareIndex = closingBracket.indexOf("["); + + if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) + closingBracket = roundBracket(closingBracket, Constants.MERGE).trim(); + else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) + closingBracket = squareBracket(closingBracket, Constants.MERGE).trim(); + else + break; + } while (!closingBracket.isEmpty()); + + } + + /* + * In order to fetch the alias and the node we use the indexing. On the + * basis of index, function/method is called. Variable map is used to store the alias + * and nodeName. Operation map is used to store the alias and operation + * performed. + */ + + private String squareBracket(String query, String operation) { + + String closingBracket = ""; + String value = ""; + String operationValue = operation; + ArrayList operationPerfomed = new ArrayList(); + + String[] arr = query.split("\\[", 2); - // Time - String dateString = parseTimestamp(data); - String timeZone = null; - if (data.has(Constants.MIN_OFF)) { - timeZone = data.get(Constants.MIN_OFF).getAsString(); - } - Time time = getTime(dateString, timeZone); + String[] arrs = arr[1].split(":", 2); + + if (arrs.length == 2) { + Pattern pattern = Pattern.compile("^[A-Za-z_|]+[\\sA-Za-z_|]*"); + Matcher matcher = pattern.matcher(arrs[1]); + if (matcher.find()) { + value = matcher.group(0); + } - if (time != null) { - record.setTime(time); - } - // SeessionLocator - record.setSessionLocator(parseSessionLocator(data)); + // in order to get the alias only and remove all the extra characters. + // EX : -[rel returns rel + arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); - // Accessor - record.setAccessor(parseAccessor(data)); + pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); + matcher = pattern.matcher(arrs[0]); + if (!matcher.find()) { + arrs[0] = "" + randomValue; + randomValue++; + } - Parser.parseSessionId(record); + if (operations.containsKey(arrs[0].trim())) { + operationPerfomed = operations.get(arrs[0]); + operationPerfomed.add(operationValue); + operations.replace(arrs[0].trim(), operationPerfomed); + } else { + operationPerfomed.add(operationValue); + operations.put(arrs[0].trim(), operationPerfomed); + } - // Data - if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { - record.setData(parseData(data)); - } else { - record.setException(parseException(data)); - } + variables.put(arrs[0].trim(), value.trim()); - } - return record; - } + String[] roundValue = arrs[1].split("\\]", 2); + closingBracket = roundValue[1]; + } -// ---------------------- Session Id ----------------------- + return closingBracket; - public void parseSessionId(Record record) { + } - Integer hashCode = (record.getSessionLocator().getClientIp() + record.getSessionLocator().getClientPort() - + record.getDbName()).hashCode(); - record.setSessionId(hashCode.toString()); - } - -// -----------------------------------------------Accessor----------------- - - public Accessor parseAccessor(JsonObject data) { - Accessor accessor = new Accessor(); - - accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); - accessor.setServerType(Constants.SERVER_TYPE_STRING); - - String dbUser = Constants.NOT_AVAILABLE; - if (data.has(Constants.DB_USER) && data.get(Constants.DB_USER) != null) { - dbUser = data.get(Constants.DB_USER).getAsString(); - } - accessor.setDbUser(dbUser); - - if(data.has(Constants.SERVER_HOSTNAME) && data.get(Constants.SERVER_HOSTNAME) != null) - accessor.setServerHostName(data.get(Constants.SERVER_HOSTNAME).getAsString()); - - String sourceProgram = Constants.UNKNOWN_STRING; - if (data.has(Constants.SOURCE_PROGRAM) && data.get(Constants.SOURCE_PROGRAM) != null) { - sourceProgram = data.get(Constants.SOURCE_PROGRAM).getAsString(); - } - accessor.setSourceProgram(sourceProgram); - - accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); - accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); - - accessor.setClient_mac(Constants.UNKNOWN_STRING); - accessor.setClientHostName(Constants.UNKNOWN_STRING); - accessor.setClientOs(Constants.UNKNOWN_STRING); - accessor.setCommProtocol(Constants.COMM_PROTOCOL); - accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); - accessor.setOsUser(Constants.UNKNOWN_STRING); - accessor.setServerDescription(Constants.UNKNOWN_STRING); - accessor.setServerOs(Constants.UNKNOWN_STRING); - accessor.setServiceName(Constants.UNKNOWN_STRING); - - return accessor; - } - -// -----------------------------------------------SessionLocator----------------- - - public SessionLocator parseSessionLocator(JsonObject data) { - SessionLocator sessionLocator = new SessionLocator(); - sessionLocator.setIpv6(false); - - int clientPort = 0; - int serverPort = 0; - String clientIpAdd = Constants.NOT_AVAILABLE; - String serverIpAdd = Constants.NOT_AVAILABLE; - - if (data.has(Constants.CLIENT_IP)) { - String clientIpPortDetails = data.get(Constants.CLIENT_IP).getAsString(); - String clientIp = clientIpPortDetails.substring(7); - String[] clientIpPort = clientIp.split(":"); - clientIpAdd = clientIpPort[0]; - - if (isIPv6Address(clientIpAdd)) { - sessionLocator.setIpv6(true); - sessionLocator.setClientIpv6(clientIpAdd); - clientIpAdd = Constants.UNKNOWN_STRING; - } else { - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - } - - } - - if (data.has(Constants.SERVER_IP)) { - String serverIPPortDetails = data.get(Constants.SERVER_IP).getAsString(); - String serverIp = serverIPPortDetails.substring(7); - String[] serverIpPort = serverIp.split(":"); - serverIpAdd = serverIpPort[0]; - - if (isIPv6Address(serverIpAdd)) { - sessionLocator.setIpv6(true); - sessionLocator.setServerIpv6(serverIpAdd); - serverIpAdd = Constants.UNKNOWN_STRING; - } else { - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - } - } - - sessionLocator.setClientIp(clientIpAdd); - sessionLocator.setClientPort(clientPort); - sessionLocator.setServerIp(serverIpAdd); - sessionLocator.setServerPort(serverPort); - - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - - return sessionLocator; - } - - private boolean isIPv6Address(String ip) { - try { - InetAddress inetAddress = InetAddress.getByName(ip); - return inetAddress instanceof java.net.Inet6Address; - } catch (UnknownHostException e) { - return false; - } - } - -// -----------------------------------------------Timestamp----------------- - - public String parseTimestamp(final JsonObject data) { - String dateString = null; - if (data.has(Constants.TIMESTAMP)) { - dateString = data.get(Constants.TIMESTAMP).getAsString(); - } - return dateString; - } - - - public Time getTime(String dateString, String timeZone) { - if (dateString != null) { - JsonObject data = new JsonObject(); - LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); - ZoneOffset offset = ZoneOffset.of(ZoneOffset.UTC.getId()); - if (timeZone != null) { - offset = ZoneOffset.of(timeZone); - } - ZonedDateTime zdt = dt.atOffset(offset).toZonedDateTime(); - long millis = zdt.toInstant().toEpochMilli(); - int minOffset = zdt.getOffset().getTotalSeconds() / 60; - return new Time(millis, minOffset, 0); - } - return null; - } - -// -----------------------------------------------Exception----------------- - - public ExceptionRecord parseException(JsonObject data) { - ExceptionRecord exceptionRecord = new ExceptionRecord(); - - exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); - - String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); - String[] queryData = queryStatement.split("'} - "); - if(queryData.length == 1){ - queryData = queryStatement.split("} - "); - } - - String query = queryData[0].toString(); - String exceptionDes = queryData[queryData.length - 1].toString(); - exceptionRecord.setDescription(exceptionDes); - - exceptionRecord.setSqlString(query); - return exceptionRecord; - } - -// -----------------------------------------------Data Construct---------------- - - public Data parseData(JsonObject inputJSON) { - Data data = new Data(); - try { - Construct construct = parseAsConstruct(inputJSON); - if (construct != null) { - data.setConstruct(construct); - - if (construct.getFullSql() == null || construct.getFullSql().isEmpty()) { - construct.setFullSql(Constants.UNKNOWN_STRING); - } - if (construct.getRedactedSensitiveDataSql() == null || construct.redactedSensitiveDataSql.isEmpty()) { - construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); - } - } - } catch (Exception e) { - log.error("Neo4j filter: Error parsing JSon in parser" + inputJSON, e); - throw e; - } - return data; - } - - public Construct parseAsConstruct(final JsonObject data) { - try { - final Sentence sentence = Parser.parseSentence(data); - - final Construct construct = new Construct(); - construct.sentences.add(sentence); - - String queryStatement = data.get(Constants.QUERY_STATEMENT).toString(); - String[] fullSql = queryStatement.split("runtime"); - String query[] = fullSql[0].split("- \\{\\} - "); - if(query[0].isEmpty()) - construct.setFullSql(fullSql[0].substring(1).trim()); - else - construct.setFullSql(query[0].substring(1).trim()); - construct.setRedactedSensitiveDataSql(Parser.parseRedactedSensitiveDataSql(data)); - return construct; - } catch (final Exception e) { - throw e; - } - } - - protected Sentence parseSentence(final JsonObject data) { - - Sentence sentence = null; - - if (data.has(Constants.QUERY_STATEMENT)) { - String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); - - sentence = parseQuery(queryStatement.trim()); - - //For Match (n) return n - if(sentence == null) { - Sentence validSentence = new Sentence("MATCH"); - SentenceObject sentenceObject = new SentenceObject("AllNodesOrRelationships"); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - validSentence.getObjects().add(sentenceObject); - sentence = validSentence; - } - - } - - return sentence; - } - - public static Sentence parseQuery(String query) { - - ArrayList gfg = new ArrayList() { - { - add(Constants.ON_CREAT_ST); - add(Constants.ON_MATC_ST); - add(Constants.MATCH); - add(Constants.CREATE); - add(Constants.MERGE); - add(Constants.DELETE); - add(Constants.DETACH_DELET); - add(Constants.REMOVE); - add(Constants.SET); - add(Constants.RETURN); - add(Constants.WHERE); - add(Constants.FOREACH); - } - }; - - if (query.contains(Constants.ON_CREATE_SET)) - query = query.replace(Constants.ON_CREATE_SET, Constants.ON_CREAT_ST); - if (query.contains(Constants.ON_MATCH_SET)) - query = query.replace(Constants.ON_MATCH_SET, Constants.ON_MATC_ST); - if (query.contains(Constants.DETACH_DELETE)) - query = query.replace(Constants.DETACH_DELETE, Constants.DETACH_DELET); - - // find the index of all the supported operations in the query. - Map ops = new TreeMap<>(); - - for (String operation : gfg) { - - int index = 0; - - while (true) { - index = query.indexOf(operation, index); - if (index == -1) - break; - ops.put(index, operation); - index++; - } - } - - // split the main query into multiple queries using index. - List queries = new ArrayList(); - - int i = 0; - int lastKeyValue = -1; - for (Map.Entry entry : ops.entrySet()) { - if (i == 0) { - i++; - lastKeyValue = entry.getKey(); - continue; - } - - String subquery = query.substring(lastKeyValue, entry.getKey()); - queries.add(subquery.trim()); - lastKeyValue = entry.getKey(); - - } - - if (lastKeyValue != -1) { - String subquery = query.substring(lastKeyValue); - queries.add(subquery); - } - - operations = new LinkedHashMap>(); - variables = new LinkedHashMap(); - - for (String queryString : queries) { - if (queryString.startsWith(Constants.RETURN) || queryString.startsWith(Constants.WHERE) - || queryString.startsWith(Constants.FOREACH)) - continue; - else if (queryString.startsWith(Constants.ON_CREAT_ST)) - onCreateSet(queryString); - else if (queryString.startsWith(Constants.ON_MATC_ST)) - onMatchSet(queryString); - else if (queryString.startsWith(Constants.MATCH)) - match(queryString); - else if (queryString.startsWith(Constants.CREATE)) - create(queryString); - else if (queryString.startsWith(Constants.MERGE)) - merge(queryString); - else if (queryString.startsWith(Constants.SET)) - set(queryString); - else if (queryString.startsWith(Constants.DELETE)) - delete(queryString); - else if (queryString.startsWith(Constants.REMOVE)) - remove(queryString); - else if (queryString.startsWith(Constants.DETACH_DELET)) - detachDelete(queryString); - } - - // Form the sentences out of different verb and objects - - // convert to ArrayList of key set - List alKeys = new ArrayList(operations.keySet()); - - SentenceObject sentenceObject = null; - - int j = 1; - Sentence originalSentence = null; - - for (String keys : alKeys) { - if (!operations.isEmpty()) { - ArrayList operation = operations.get(keys); - - for (String string : operation) { - - if (j == 1) { - originalSentence = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if (sentenceObject.getName() == null) - sentenceObject.setName(Constants.EVERYTHING); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - originalSentence.getObjects().add(sentenceObject); - j++; - } else { - Sentence decendant = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if (sentenceObject.getName() == null) - sentenceObject.setName(Constants.EVERYTHING); - sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS - - decendant.getObjects().add(sentenceObject); - originalSentence.getDescendants().add(decendant); - } - } - } - } - - return originalSentence; - } - - private void remove(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.REMOVE); - String alias = ""; - if (arr[1].contains(".")) { - String arrs[] = arr[1].split("\\.", 2); - alias = arrs[0].trim(); - } else if (arr[1].contains(":")) { - String arrs[] = arr[1].split("\\:", 2); - alias = arrs[0].trim(); - } - - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.REMOVE); - operations.put(alias, operationPerfomed); - } - - private void delete(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.DELETE); - String alias = arr[1].trim(); - if (alias.contains("-")) { - String arrs[] = alias.split("-", 2); - alias = arrs[0].trim(); - } - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.DELETE); - operations.put(alias, operationPerfomed); - } - - private void detachDelete(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.DETACH_DELET); - String alias = arr[1].trim(); - if (alias.contains("-")) { - String arrs[] = alias.split("-", 2); - alias = arrs[0].trim(); - } - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.DETACH_DELETE); - operations.put(alias, operationPerfomed); - } - - private void onMatchSet(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.ON_MATC_ST); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.ON_MATCH_SET); - operations.put(alias, operationPerfomed); - } - - private void onCreateSet(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.ON_CREAT_ST); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.ON_CREATE_SET); - operations.put(alias, operationPerfomed); - - } - - private void set(String queryString) { - - ArrayList operationPerfomed = new ArrayList<>(); - String arr[] = queryString.split(Constants.SET); - String arrs[] = arr[1].split("\\.", 2); - String alias = arrs[0].trim(); - if (operations.containsKey(alias)) { - operationPerfomed = operations.get(alias); - operations.remove(alias); - } - operationPerfomed.add(Constants.SET); - operations.put(alias, operationPerfomed); - } - - private void match(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.MATCH).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.MATCH).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - private void create(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.CREATE).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.CREATE).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - private void merge(String query) { - - String closingBracket = query; - - do { - int roundIndex = closingBracket.indexOf("("); - int squareIndex = closingBracket.indexOf("["); - - if (roundIndex != -1 && (roundIndex < squareIndex || squareIndex == -1)) - closingBracket = roundBracket(closingBracket, Constants.MERGE).trim(); - else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) - closingBracket = squareBracket(closingBracket, Constants.MERGE).trim(); - else - break; - } while (!closingBracket.isEmpty()); - - } - - /* - * In order to fetch the alias and the node we use the indexing. On the - * basis of index, function/method is called. Variable map is used to store the alias - * and nodeName. Operation map is used to store the alias and operation - * performed. - */ - - private String squareBracket(String query, String operation) { - - String closingBracket = ""; - String value = ""; - String operationValue = operation; - ArrayList operationPerfomed = new ArrayList(); - - String[] arr = query.split("\\[", 2); - - String[] arrs = arr[1].split(":", 2); - - if (arrs.length == 2) { - Pattern pattern = Pattern.compile("^[A-Za-z_|]+[\\sA-Za-z_|]*"); - Matcher matcher = pattern.matcher(arrs[1]); - if (matcher.find()) { - value = matcher.group(0); - } - - // in order to get the alias only and remove all the extra characters. - // EX : -[rel returns rel - arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); + protected String roundBracket(String query, String operation) { - pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); - matcher = pattern.matcher(arrs[0]); - if (!matcher.find()) { - arrs[0] = "" + randomValue; - randomValue++; - } - - if (operations.containsKey(arrs[0].trim())) { - operationPerfomed = operations.get(arrs[0]); - operationPerfomed.add(operationValue); - operations.replace(arrs[0].trim(), operationPerfomed); - } else { - operationPerfomed.add(operationValue); - operations.put(arrs[0].trim(), operationPerfomed); - } + String closingBracket = ""; + String value = ""; + String operationValue = operation; + ArrayList operationPerfomed = new ArrayList(); - variables.put(arrs[0].trim(), value.trim()); + String[] arr = query.split("\\(", 2); - String[] roundValue = arrs[1].split("\\]", 2); - closingBracket = roundValue[1]; - } - - return closingBracket; + if (arr[1].indexOf(")") < arr[1].indexOf(":")) { + String temp[] = arr[1].split("\\)", 2); + arr[1] = temp[1]; + } - } - - protected String roundBracket(String query, String operation) { + String[] arrs = arr[1].split(":", 2); - String closingBracket = ""; - String value = ""; - String operationValue = operation; - ArrayList operationPerfomed = new ArrayList(); + if (arrs.length == 2) { + Pattern pattern = Pattern.compile("^[A-Za-z_]+[\\sA-Za-z_]*"); + Matcher matcher = pattern.matcher(arrs[1]); + if (matcher.find()) { + value = matcher.group(0); + } - String[] arr = query.split("\\(", 2); + // in order to get the alias only and remove all the extra characters. + // EX : -[rel returns rel + arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); - if (arr[1].indexOf(")") < arr[1].indexOf(":")) { - String temp[] = arr[1].split("\\)", 2); - arr[1] = temp[1]; - } - - String[] arrs = arr[1].split(":", 2); - - if (arrs.length == 2) { - Pattern pattern = Pattern.compile("^[A-Za-z_]+[\\sA-Za-z_]*"); - Matcher matcher = pattern.matcher(arrs[1]); - if (matcher.find()) { - value = matcher.group(0); - } + pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); + matcher = pattern.matcher(arrs[0]); + if (!matcher.find()) { + arrs[0] = "" + randomValue; + randomValue++; + } - // in order to get the alias only and remove all the extra characters. - // EX : -[rel returns rel - arrs[0] = arrs[0].replaceAll("[^A-Za-z]+", ""); + if (operations.containsKey(arrs[0].trim())) { + operationPerfomed = operations.get(arrs[0]); + operations.remove(arrs[0]); + } + operationPerfomed.add(operationValue); + operations.put(arrs[0].trim(), operationPerfomed); - pattern = Pattern.compile("^[A-Za-z]+[\\sA-Za-z]*"); - matcher = pattern.matcher(arrs[0]); - if (!matcher.find()) { - arrs[0] = "" + randomValue; - randomValue++; - } - - if (operations.containsKey(arrs[0].trim())) { - operationPerfomed = operations.get(arrs[0]); - operations.remove(arrs[0]); - } - operationPerfomed.add(operationValue); - operations.put(arrs[0].trim(), operationPerfomed); + variables.put(arrs[0].trim(), value.trim()); - variables.put(arrs[0].trim(), value.trim()); + String[] roundValue = arrs[1].split("\\)", 2); + closingBracket = roundValue[1]; + } - String[] roundValue = arrs[1].split("\\)", 2); - closingBracket = roundValue[1]; - } - - return closingBracket; + return closingBracket; - } + } - protected String parseRedactedSensitiveDataSql(JsonObject data) { - String redactedData = ""; - redactedData = data.get(Constants.MESSAGE).getAsString(); + protected String parseRedactedSensitiveDataSql(JsonObject data) { + String redactedData = ""; + redactedData = data.get(Constants.MESSAGE).getAsString(); - return redactedData; - } + return redactedData; + } -} \ No newline at end of file +} diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java index 183ef4e44..5f6083cd4 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java @@ -11,311 +11,328 @@ import org.junit.Assert; import org.junit.Test; +import java.text.ParseException; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; public class ParserTest { - Parser parser; - String neoSuccessString = "2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoSuccessString_grokOutput = "{\n" + - " \"ts\": \"\\\"2021-08-06 17:09:40.008+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:51372\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\\\";\"\n" + - " }"; - - @Test - public void testParseAsConstruct_Match() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - final Construct result = Parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("MATCH", sentence.getVerb().trim()); - Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseAsConstruct_Create() { - parser = new Parser(); - String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput,neoString); - - JsonObject inputData = inputData(e); - - final Construct result = Parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("CREATE", sentence.getVerb().trim()); - Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testDBNameAndServiceNameEqual() throws ParseException { - parser = new Parser(); - String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4jDB \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - - Event e = getParsedEvent(neoString_grokOutput, neoString); - e.setField("minoff", "+07:00"); - - JsonObject inputData = inputData(e); - - final Record record = parser.parseRecord(inputData); - Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); - } - - @Test - public void testParseAsConstruct_Merge() { - parser = new Parser(); - String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-08-06 15:56:12.097+0000\",\n" + - " \"log_level\": \"INFO\",\n" + - " \"metadata1\": \" 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:54356\",\n" + - " \"server_ip\": \"server/127.0.0.1:11004\",\n" + - " \"dbname\": \"neo4j \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput,neoString); - e.setField("minoff", "+04:00"); - JsonObject inputData = inputData(e); - - final Construct result = Parser.parseAsConstruct(inputData); - - final Sentence sentence = result.sentences.get(0); - - Assert.assertEquals("MERGE", sentence.getVerb().trim()); - Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - } - - @Test - public void testParseException_DELETE() { - String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; - String neoString_grokOutput = "{\n" + - " \"ts\": \"2021-03-03 08:49:32.367+0000\",\n" + - " \"log_level\": \"ERROR\",\n" + - " \"metadata1\": \"7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session\",\n" + - " \"protocol\": \"bolt\",\n" + - " \"driverVersion\": \"neo4j-browser/v4.2.1\",\n" + - " \"client_ip\": \"client/127.0.0.1:62845\",\n" + - " \"server_ip\": \"server/127.0.0.1:7687\",\n" + - " \"dbname\": \" \",\n" + - " \"dbuser\": \"neo4j \",\n" + - " \"queryStatement\": \"DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))\\\";\"\n" + - " }"; - Event e = getParsedEvent(neoString_grokOutput, neoString); - JsonObject inputData = inputData(e); - e.setField("minoff", "+07:00"); - - final ExceptionRecord exceptionRecord = Parser.parseException(inputData); - - Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); - Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); - } - - @Test - public void testParseAccessor() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput,neoSuccessString); - - JsonObject inputData = inputData(e); - - final Accessor accessor = Parser.parseAccessor(inputData); - - Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); - Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); - Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); - Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); - } - - @Test - public void testParseSessionLocator() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final SessionLocator sessionLocator = Parser.parseSessionLocator(inputData); - - Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); - Assert.assertEquals(51372, sessionLocator.getClientPort()); - Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); - Assert.assertEquals(11004, sessionLocator.getServerPort()); - - } - - @Test - public void testParseTimestamp() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); - JsonObject inputData = inputData(e); - - final String timestamp = Parser.parseTimestamp(inputData); - - Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); - - } - - @Test - public void testGetTime() { - parser = new Parser(); - String dateString = "2021-01-25 11:17:09.099+0000"; - String timeZone = "-04:00"; - final Time time = Parser.getTime(dateString, timeZone); - - Assert.assertEquals(0, time.getMinDst()); - Assert.assertEquals(-240, time.getMinOffsetFromGMT()); - Assert.assertEquals(1611587829099L, time.getTimstamp()); - - } - - @Test - public void testParseSentence() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final Sentence sentence = Parser.parseSentence(inputData); - - Assert.assertEquals("MATCH", sentence.getVerb()); - Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); - Assert.assertEquals("graph", sentence.getObjects().get(0).type); - - } - - @Test - public void testParseRedactedSensitiveDataSql() { - parser = new Parser(); - Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - - JsonObject inputData = inputData(e); - - final String redacted = Parser.parseRedactedSensitiveDataSql(inputData); - - Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); - - } - + Parser parser; + String neoSuccessString = "2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoSuccessString_grokOutput = "{\n" + + " \"ts\": \"\\\"2021-08-06 17:09:40.008+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:51372\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\\\";\"\n" + + " }"; + + @Test + public void testParseAsConstruct_Match() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + final Construct result = parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("MATCH", sentence.getVerb().trim()); + Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testParseAsConstruct_Create() { + parser = new Parser(); + String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + + JsonObject inputData = inputData(e); + + final Construct result = parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("CREATE", sentence.getVerb().trim()); + Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + + @Test + public void testDBNameAndServiceNameEqual() throws ParseException { + parser = new Parser(); + String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:57:11.502+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4jDB \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + + Event e = getParsedEvent(neoString_grokOutput, neoString); + e.setField("minoff", "+07:00"); + + JsonObject inputData = inputData(e); + + final Record record = parser.parseRecord(inputData); + Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + } + + @Test + public void testParseAsConstruct_Merge() { + parser = new Parser(); + String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-08-06 15:56:12.097+0000\",\n" + + " \"log_level\": \"INFO\",\n" + + " \"metadata1\": \" 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.3.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:54356\",\n" + + " \"server_ip\": \"server/127.0.0.1:11004\",\n" + + " \"dbname\": \"neo4j \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + e.setField("minoff", "+04:00"); + + JsonObject inputData = inputData(e); + + final Construct result = parser.parseAsConstruct(inputData); + + final Sentence sentence = result.sentences.get(0); + + Assert.assertEquals("MERGE", sentence.getVerb().trim()); + Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + } + + @Test + public void testParseException_DELETE() { + parser = new Parser(); + String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; + String neoString_grokOutput = "{\n" + + " \"ts\": \"2021-03-03 08:49:32.367+0000\",\n" + + " \"log_level\": \"ERROR\",\n" + + " \"metadata1\": \"7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session\",\n" + + " \"protocol\": \"bolt\",\n" + + " \"driverVersion\": \"neo4j-browser/v4.2.1\",\n" + + " \"client_ip\": \"client/127.0.0.1:62845\",\n" + + " \"server_ip\": \"server/127.0.0.1:7687\",\n" + + " \"dbname\": \" \",\n" + + " \"dbuser\": \"neo4j \",\n" + + " \"queryStatement\": \"DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))\\\";\"\n" + + " }"; + Event e = getParsedEvent(neoString_grokOutput, neoString); + JsonObject inputData = inputData(e); + e.setField("minoff", "+07:00"); + + final ExceptionRecord exceptionRecord = parser.parseException(inputData); + + Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); + Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); + } + + @Test + public void testParseAccessor() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Accessor accessor = parser.parseAccessor(inputData); + + Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); + Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); + Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); + Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); + } + + @Test + public void testParseSessionLocator() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final SessionLocator sessionLocator = parser.parseSessionLocator(inputData); + + Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); + Assert.assertEquals(-1, sessionLocator.getClientPort()); + Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); + Assert.assertEquals(-1, sessionLocator.getServerPort()); + + } + + @Test + public void testParseSessionId() throws ParseException { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Record record = parser.parseRecord(inputData); + + Assert.assertEquals(Constants.UNKNOWN_STRING, record.getSessionId()); + + } + + @Test + public void testParseTimestamp() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); + JsonObject inputData = inputData(e); + + final String timestamp = parser.parseTimestamp(inputData); + + Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); + + } + + @Test + public void testGetTime() { + parser = new Parser(); + String dateString = "2021-01-25 11:17:09.099+0000"; + String timeZone = "-04:00"; + final Time time = parser.getTime(dateString, timeZone); + + Assert.assertEquals(0, time.getMinDst()); + Assert.assertEquals(-240, time.getMinOffsetFromGMT()); + Assert.assertEquals(1611587829099L, time.getTimstamp()); + + } + + @Test + public void testParseSentence() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final Sentence sentence = parser.parseSentence(inputData); + + Assert.assertEquals("MATCH", sentence.getVerb()); + Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); + Assert.assertEquals("graph", sentence.getObjects().get(0).type); + + } + + @Test + public void testParseRedactedSensitiveDataSql() { + parser = new Parser(); + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); + + JsonObject inputData = inputData(e); + + final String redacted = parser.parseRedactedSensitiveDataSql(inputData); + + Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); + + } + // ----------------------------------- --------------------------------------------------- - - private JsonObject inputData(Event e){ - JsonObject data = new JsonObject(); - - if(e.getField(Constants.CLIENT_IP).toString() != null && !e.getField(Constants.CLIENT_IP).toString().isEmpty()){ - data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); - } - if(e.getField(Constants.SERVER_IP).toString() != null && !e.getField(Constants.SERVER_IP).toString().isEmpty()){ - data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); - } - if(e.getField(Constants.DB_PROTOCOL).toString() != null && e.getField(Constants.DB_PROTOCOL).toString().isEmpty()){ - data.addProperty(Constants.DB_PROTOCOL, e.getField(Constants.DB_PROTOCOL).toString()); - } - if(e.getField(Constants.TIMESTAMP).toString() != null && !e.getField(Constants.TIMESTAMP).toString().isEmpty()){ - data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); - } - if(e.getField(Constants.LOG_LEVEL).toString() != null && !e.getField(Constants.LOG_LEVEL).toString().isEmpty()){ - data.addProperty(Constants.LOG_LEVEL, e.getField(Constants.LOG_LEVEL).toString()); - } - if(e.getField(Constants.DB_USER).toString() != null && !e.getField(Constants.DB_USER).toString().isEmpty()){ - data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); - } - if(e.getField(Constants.DB_NAME).toString() != null && !e.getField(Constants.DB_NAME).toString().isEmpty()){ - data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); - } - if(e.getField(Constants.SOURCE_PROGRAM).toString() != null && !e.getField(Constants.SOURCE_PROGRAM).toString().isEmpty()){ - data.addProperty(Constants.SOURCE_PROGRAM, e.getField(Constants.SOURCE_PROGRAM).toString()); - } - if(e.getField(Constants.QUERY_STATEMENT).toString() != null && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()){ - data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); - } - if(e.getField(Constants.MESSAGE).toString() != null && !e.getField(Constants.MESSAGE).toString().isEmpty()){ - data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); - } - return data; - } - - - public static Event getParsedEvent(String logEvent_json , String logEvent) { - - Map results = parseJson(logEvent_json); - Event e = new org.logstash.Event(); - e.setField("message", logEvent); - - e.setField(Constants.CLIENT_IP, results.get(Constants.CLIENT_IP)); - e.setField(Constants.SERVER_IP, results.get(Constants.SERVER_IP)); - e.setField(Constants.DB_PROTOCOL, results.get(Constants.DB_PROTOCOL)); - e.setField(Constants.TIMESTAMP, results.get(Constants.TIMESTAMP)); - e.setField(Constants.LOG_LEVEL, results.get(Constants.LOG_LEVEL)); - e.setField(Constants.DB_USER, results.get(Constants.DB_USER)); - e.setField(Constants.DB_NAME, results.get(Constants.DB_NAME)); - e.setField(Constants.SOURCE_PROGRAM, results.get(Constants.SOURCE_PROGRAM)); - e.setField(Constants.QUERY_STATEMENT, results.get(Constants.QUERY_STATEMENT)); - - return e; - } - - - public static Map parseJson(String jsonString) { - Map map = new HashMap<>(); - - // Use a regular expression to match each key-value pair in the JSON string - String pattern = "\"(.*?)\":\\s*\"(.*?)\""; - Pattern r = Pattern.compile(pattern); - Matcher m = r.matcher(jsonString); - - while (m.find()) { - // Add each key-value pair to the map - String key = m.group(1); - String value = m.group(2).replaceAll("(? results = parseJson(logEvent_json); + Event e = new org.logstash.Event(); + e.setField("message", logEvent); + + e.setField(Constants.CLIENT_IP, results.get(Constants.CLIENT_IP)); + e.setField(Constants.SERVER_IP, results.get(Constants.SERVER_IP)); + e.setField(Constants.DB_PROTOCOL, results.get(Constants.DB_PROTOCOL)); + e.setField(Constants.TIMESTAMP, results.get(Constants.TIMESTAMP)); + e.setField(Constants.LOG_LEVEL, results.get(Constants.LOG_LEVEL)); + e.setField(Constants.DB_USER, results.get(Constants.DB_USER)); + e.setField(Constants.DB_NAME, results.get(Constants.DB_NAME)); + e.setField(Constants.SOURCE_PROGRAM, results.get(Constants.SOURCE_PROGRAM)); + e.setField(Constants.QUERY_STATEMENT, results.get(Constants.QUERY_STATEMENT)); + + return e; + } + + + public static Map parseJson(String jsonString) { + Map map = new HashMap<>(); + + // Use a regular expression to match each key-value pair in the JSON string + String pattern = "\"(.*?)\":\\s*\"(.*?)\""; + Pattern r = Pattern.compile(pattern); + Matcher m = r.matcher(jsonString); + + while (m.find()) { + // Add each key-value pair to the map + String key = m.group(1); + String value = m.group(2).replaceAll("(?