From da335951c6f0eb74d63b4a04549f1904aa4c7833 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 11 May 2023 10:02:32 +0200 Subject: [PATCH 01/12] output/alert: rewrite code for app-layer properties Especially fix setup-app-layer script to not forget this part This allows, for simple loggers, to have a unique definition of the actual logging function with the jsonbuilder. This way, alerts, files, and app-layer event can share the code to output the same data. Ticket: #3827 --- rust/src/applayertemplate/logger.rs | 2 + rust/src/bittorrent_dht/logger.rs | 2 + rust/src/http2/logger.rs | 5 +- rust/src/quic/logger.rs | 4 +- rust/src/snmp/log.rs | 4 +- rust/src/ssh/logger.rs | 2 + scripts/setup-app-layer.py | 4 + src/app-layer-ftp.c | 11 +- src/app-layer-ftp.h | 2 +- src/output-json-alert.c | 229 ++-------------------------- src/output-json-bittorrent-dht.c | 2 - src/output-json-dnp3.c | 21 +++ src/output-json-dnp3.h | 1 + src/output-json-dns.c | 21 ++- src/output-json-dns.h | 3 +- src/output-json-file.c | 21 ++- src/output-json-ftp.c | 11 +- src/output-json-http2.c | 15 -- src/output-json-http2.h | 1 - src/output-json-modbus.c | 13 -- src/output-json-modbus.h | 1 - src/output-json-mqtt.c | 12 +- src/output-json-mqtt.h | 2 +- src/output-json-quic.c | 13 -- src/output-json-quic.h | 1 - src/output-json-rfb.c | 13 -- src/output-json-rfb.h | 2 - src/output-json-sip.c | 11 -- src/output-json-sip.h | 2 - src/output-json-snmp.c | 4 +- src/output-json-ssh.c | 2 - src/output-json-template.c | 2 - src/output-json-tls.c | 14 +- src/output-json-tls.h | 2 +- src/output.c | 52 +++++++ src/output.h | 9 ++ 36 files changed, 177 insertions(+), 339 deletions(-) diff --git a/rust/src/applayertemplate/logger.rs b/rust/src/applayertemplate/logger.rs index 0105526fc0f1..766a07acdb9d 100644 --- a/rust/src/applayertemplate/logger.rs +++ b/rust/src/applayertemplate/logger.rs @@ -20,12 +20,14 @@ use crate::jsonbuilder::{JsonBuilder, JsonError}; use std; fn log_template(tx: &TemplateTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> { + js.open_object("template")?; if let Some(ref request) = tx.request { js.set_string("request", request)?; } if let Some(ref response) = tx.response { js.set_string("response", response)?; } + js.close()?; Ok(()) } diff --git a/rust/src/bittorrent_dht/logger.rs b/rust/src/bittorrent_dht/logger.rs index 2cfb9270ef67..74ea7c59ba57 100644 --- a/rust/src/bittorrent_dht/logger.rs +++ b/rust/src/bittorrent_dht/logger.rs @@ -48,6 +48,7 @@ fn print_ip_addr(addr: &[u8]) -> std::string::String { fn log_bittorrent_dht( tx: &BitTorrentDHTTransaction, js: &mut JsonBuilder, ) -> Result<(), JsonError> { + js.open_object("bittorrent_dht")?; js.set_hex("transaction_id", &tx.transaction_id)?; if let Some(client_version) = &tx.client_version { js.set_hex("client_version", client_version)?; @@ -125,6 +126,7 @@ fn log_bittorrent_dht( } js.close()?; }; + js.close()?; Ok(()) } diff --git a/rust/src/http2/logger.rs b/rust/src/http2/logger.rs index d25f852c43ab..099112b1aeb4 100644 --- a/rust/src/http2/logger.rs +++ b/rust/src/http2/logger.rs @@ -192,6 +192,7 @@ fn log_http2_frames(frames: &[HTTP2Frame], js: &mut JsonBuilder) -> Result Result { + js.open_object("http")?; js.set_string("version", "2")?; let mut common: HashMap> = HashMap::new(); @@ -261,8 +262,8 @@ fn log_http2(tx: &HTTP2Transaction, js: &mut JsonBuilder) -> Result Option { } } -fn log_template(tx: &QuicTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> { +fn log_quic(tx: &QuicTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> { js.open_object("quic")?; if tx.header.ty != QuicType::Short { js.set_string("version", String::from(tx.header.version).as_str())?; @@ -153,5 +153,5 @@ pub unsafe extern "C" fn rs_quic_to_json( tx: *mut std::os::raw::c_void, js: &mut JsonBuilder, ) -> bool { let tx = cast_pointer!(tx, QuicTransaction); - log_template(tx, js).is_ok() + log_quic(tx, js).is_ok() } diff --git a/rust/src/snmp/log.rs b/rust/src/snmp/log.rs index 83414816c466..5707f30ccb4e 100644 --- a/rust/src/snmp/log.rs +++ b/rust/src/snmp/log.rs @@ -39,6 +39,7 @@ fn str_of_pdu_type(t:&PduType) -> Cow { fn snmp_log_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> Result<(), JsonError> { + jsb.open_object("snmp")?; jsb.set_uint("version", tx.version as u64)?; if tx.encrypted { jsb.set_string("pdu_type", "encrypted")?; @@ -71,11 +72,12 @@ fn snmp_log_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> Result< } } + jsb.close()?; return Ok(()); } #[no_mangle] -pub extern "C" fn rs_snmp_log_json_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> bool +pub extern "C" fn rs_snmp_log_json_response(tx: &mut SNMPTransaction, jsb: &mut JsonBuilder) -> bool { snmp_log_response(jsb, tx).is_ok() } diff --git a/rust/src/ssh/logger.rs b/rust/src/ssh/logger.rs index 9bc7d7c33f39..008c6cb4517a 100644 --- a/rust/src/ssh/logger.rs +++ b/rust/src/ssh/logger.rs @@ -19,6 +19,7 @@ use super::ssh::SSHTransaction; use crate::jsonbuilder::{JsonBuilder, JsonError}; fn log_ssh(tx: &SSHTransaction, js: &mut JsonBuilder) -> Result { + js.open_object("ssh")?; if tx.cli_hdr.protover.is_empty() && tx.srv_hdr.protover.is_empty() { return Ok(false); } @@ -58,6 +59,7 @@ fn log_ssh(tx: &SSHTransaction, js: &mut JsonBuilder) -> Result } js.close()?; } + js.close()?; return Ok(true); } diff --git a/scripts/setup-app-layer.py b/scripts/setup-app-layer.py index 72f28c986c66..d8426634bca8 100755 --- a/scripts/setup-app-layer.py +++ b/scripts/setup-app-layer.py @@ -200,6 +200,10 @@ def logger_patch_output_c(proto): output = io.StringIO() inlines = open(filename).readlines() for i, line in enumerate(inlines): + if line.find("ALPROTO_TEMPLATE") > -1: + new_line = line.replace("TEMPLATE", proto.upper()).replace( + "template", proto.lower()) + output.write(new_line) if line.find("output-json-template.h") > -1: output.write(line.replace("template", proto.lower())) if line.find("/* Template JSON logger.") > -1: diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index 3db448279073..93962210d6ce 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -1405,13 +1405,10 @@ uint16_t JsonGetNextLineFromBuffer(const char *buffer, const uint16_t len) return c == NULL ? len : (uint16_t)(c - buffer + 1); } -void EveFTPDataAddMetadata(const Flow *f, JsonBuilder *jb) +bool EveFTPDataAddMetadata(void *vtx, JsonBuilder *jb) { - const FtpDataState *ftp_state = NULL; - if (f->alstate == NULL) - return; - - ftp_state = (FtpDataState *)f->alstate; + const FtpDataState *ftp_state = (FtpDataState *)vtx; + jb_open_object(jb, "ftp_data"); if (ftp_state->file_name) { jb_set_string_from_bytes(jb, "filename", ftp_state->file_name, ftp_state->file_len); @@ -1426,6 +1423,8 @@ void EveFTPDataAddMetadata(const Flow *f, JsonBuilder *jb) default: break; } + jb_close(jb); + return true; } /** diff --git a/src/app-layer-ftp.h b/src/app-layer-ftp.h index f79c5c9e7675..fb71d6b52de7 100644 --- a/src/app-layer-ftp.h +++ b/src/app-layer-ftp.h @@ -190,7 +190,7 @@ uint64_t FTPMemuseGlobalCounter(void); uint64_t FTPMemcapGlobalCounter(void); uint16_t JsonGetNextLineFromBuffer(const char *buffer, const uint16_t len); -void EveFTPDataAddMetadata(const Flow *f, JsonBuilder *jb); +bool EveFTPDataAddMetadata(void *vtx, JsonBuilder *jb); #endif /* __APP_LAYER_FTP_H__ */ diff --git a/src/output-json-alert.c b/src/output-json-alert.c index a7df1065509e..aa066619bd8d 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -137,164 +137,6 @@ static int AlertJsonDumpStreamSegmentCallback( return 1; } -static void AlertJsonTls(const Flow *f, JsonBuilder *js) -{ - SSLState *ssl_state = (SSLState *)FlowGetAppState(f); - if (ssl_state) { - jb_open_object(js, "tls"); - - JsonTlsLogJSONExtended(js, ssl_state); - - jb_close(js); - } - - return; -} - -static void AlertJsonSsh(const Flow *f, JsonBuilder *js) -{ - void *ssh_state = FlowGetAppState(f); - if (ssh_state) { - JsonBuilderMark mark = { 0, 0, 0 }; - void *tx_ptr = rs_ssh_state_get_tx(ssh_state, 0); - jb_get_mark(js, &mark); - jb_open_object(js, "ssh"); - if (rs_ssh_log_json(tx_ptr, js)) { - jb_close(js); - } else { - jb_restore_mark(js, &mark); - } - } - - return; -} - -static void AlertJsonHttp2(const Flow *f, const uint64_t tx_id, JsonBuilder *js) -{ - void *h2_state = FlowGetAppState(f); - if (h2_state) { - void *tx_ptr = rs_http2_state_get_tx(h2_state, tx_id); - if (tx_ptr) { - JsonBuilderMark mark = { 0, 0, 0 }; - jb_get_mark(js, &mark); - jb_open_object(js, "http"); - if (rs_http2_log_json(tx_ptr, js)) { - jb_close(js); - } else { - jb_restore_mark(js, &mark); - } - } - } - - return; -} - -static void AlertJsonDnp3(const Flow *f, const uint64_t tx_id, JsonBuilder *js) -{ - DNP3State *dnp3_state = (DNP3State *)FlowGetAppState(f); - if (dnp3_state) { - DNP3Transaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_DNP3, - dnp3_state, tx_id); - if (tx) { - JsonBuilderMark mark = { 0, 0, 0 }; - jb_get_mark(js, &mark); - bool logged = false; - jb_open_object(js, "dnp3"); - if (tx->is_request && tx->done) { - jb_open_object(js, "request"); - JsonDNP3LogRequest(js, tx); - jb_close(js); - logged = true; - } - if (!tx->is_request && tx->done) { - jb_open_object(js, "response"); - JsonDNP3LogResponse(js, tx); - jb_close(js); - logged = true; - } - if (logged) { - /* Close dnp3 object. */ - jb_close(js); - } else { - jb_restore_mark(js, &mark); - } - } - } -} - -static void AlertJsonDns(const Flow *f, const uint64_t tx_id, JsonBuilder *js) -{ - void *dns_state = (void *)FlowGetAppState(f); - if (dns_state) { - void *txptr = AppLayerParserGetTx(f->proto, ALPROTO_DNS, - dns_state, tx_id); - if (txptr) { - jb_open_object(js, "dns"); - JsonBuilder *qjs = JsonDNSLogQuery(txptr); - if (qjs != NULL) { - jb_set_object(js, "query", qjs); - jb_free(qjs); - } - JsonBuilder *ajs = JsonDNSLogAnswer(txptr); - if (ajs != NULL) { - jb_set_object(js, "answer", ajs); - jb_free(ajs); - } - jb_close(js); - } - } - return; -} - -static void AlertJsonSNMP(const Flow *f, const uint64_t tx_id, JsonBuilder *js) -{ - void *snmp_state = (void *)FlowGetAppState(f); - if (snmp_state != NULL) { - void *tx = AppLayerParserGetTx(f->proto, ALPROTO_SNMP, snmp_state, - tx_id); - if (tx != NULL) { - jb_open_object(js, "snmp"); - rs_snmp_log_json_response(js, tx); - jb_close(js); - } - } -} - -static void AlertJsonRDP(const Flow *f, const uint64_t tx_id, JsonBuilder *js) -{ - void *rdp_state = (void *)FlowGetAppState(f); - if (rdp_state != NULL) { - void *tx = AppLayerParserGetTx(f->proto, ALPROTO_RDP, rdp_state, - tx_id); - if (tx != NULL) { - JsonBuilderMark mark = { 0, 0, 0 }; - jb_get_mark(js, &mark); - if (!rs_rdp_to_json(tx, js)) { - jb_restore_mark(js, &mark); - } - } - } -} - -static void AlertJsonBitTorrentDHT(const Flow *f, const uint64_t tx_id, JsonBuilder *js) -{ - void *bittorrent_dht_state = (void *)FlowGetAppState(f); - if (bittorrent_dht_state != NULL) { - void *tx = - AppLayerParserGetTx(f->proto, ALPROTO_BITTORRENT_DHT, bittorrent_dht_state, tx_id); - if (tx != NULL) { - JsonBuilderMark mark = { 0, 0, 0 }; - jb_get_mark(js, &mark); - jb_open_object(js, "bittorrent_dht"); - if (rs_bittorrent_dht_logger_log(tx, js)) { - jb_close(js); - } else { - jb_restore_mark(js, &mark); - } - } - } -} - static void AlertJsonSourceTarget(const Packet *p, const PacketAlert *pa, JsonBuilder *js, JsonAddrInfo *addr) { @@ -471,7 +313,21 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb, const uint64_t tx_id, const uint16_t option_flags) { const AppProto proto = FlowGetAppProtocol(p->flow); + SimpleJsonAppLayerLogger *al = GetAppProtoSimpleJsonLogger(proto); JsonBuilderMark mark = { 0, 0, 0 }; + if (al && al->LogTx) { + void *state = FlowGetAppState(p->flow); + if (state) { + void *tx = AppLayerParserGetTx(p->flow->proto, proto, state, tx_id); + if (tx) { + jb_get_mark(jb, &mark); + if (!al->LogTx(tx, jb)) { + jb_restore_mark(jb, &mark); + } + } + } + return; + } switch (proto) { case ALPROTO_HTTP1: // TODO: Could result in an empty http object being logged. @@ -486,12 +342,6 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb, } jb_close(jb); break; - case ALPROTO_TLS: - AlertJsonTls(p->flow, jb); - break; - case ALPROTO_SSH: - AlertJsonSsh(p->flow, jb); - break; case ALPROTO_SMTP: jb_get_mark(jb, &mark); jb_open_object(jb, "smtp"); @@ -535,63 +385,12 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb, jb_restore_mark(jb, &mark); } break; - case ALPROTO_SIP: - JsonSIPAddMetadata(jb, p->flow, tx_id); - break; - case ALPROTO_RFB: - jb_get_mark(jb, &mark); - if (!JsonRFBAddMetadata(p->flow, tx_id, jb)) { - jb_restore_mark(jb, &mark); - } - break; - case ALPROTO_FTPDATA: - jb_get_mark(jb, &mark); - jb_open_object(jb, "ftp_data"); - EveFTPDataAddMetadata(p->flow, jb); - jb_close(jb); - break; - case ALPROTO_DNP3: - AlertJsonDnp3(p->flow, tx_id, jb); - break; - case ALPROTO_HTTP2: - AlertJsonHttp2(p->flow, tx_id, jb); - break; - case ALPROTO_DNS: - AlertJsonDns(p->flow, tx_id, jb); - break; case ALPROTO_IKE: jb_get_mark(jb, &mark); if (!EveIKEAddMetadata(p->flow, tx_id, jb)) { jb_restore_mark(jb, &mark); } break; - case ALPROTO_MQTT: - jb_get_mark(jb, &mark); - if (!JsonMQTTAddMetadata(p->flow, tx_id, jb)) { - jb_restore_mark(jb, &mark); - } - break; - case ALPROTO_QUIC: - jb_get_mark(jb, &mark); - if (!JsonQuicAddMetadata(p->flow, tx_id, jb)) { - jb_restore_mark(jb, &mark); - } - break; - case ALPROTO_SNMP: - AlertJsonSNMP(p->flow, tx_id, jb); - break; - case ALPROTO_RDP: - AlertJsonRDP(p->flow, tx_id, jb); - break; - case ALPROTO_MODBUS: - jb_get_mark(jb, &mark); - if (!JsonModbusAddMetadata(p->flow, tx_id, jb)) { - jb_restore_mark(jb, &mark); - } - break; - case ALPROTO_BITTORRENT_DHT: - AlertJsonBitTorrentDHT(p->flow, tx_id, jb); - break; default: break; } diff --git a/src/output-json-bittorrent-dht.c b/src/output-json-bittorrent-dht.c index 08b7dc4d722c..066df78f61fb 100644 --- a/src/output-json-bittorrent-dht.c +++ b/src/output-json-bittorrent-dht.c @@ -65,11 +65,9 @@ static int JsonBitTorrentDHTLogger(ThreadVars *tv, void *thread_data, const Pack return TM_ECODE_FAILED; } - jb_open_object(js, "bittorrent_dht"); if (!rs_bittorrent_dht_logger_log(tx, js)) { goto error; } - jb_close(js); OutputJsonBuilderBuffer(js, thread->ctx); jb_free(js); diff --git a/src/output-json-dnp3.c b/src/output-json-dnp3.c index 97b1e92e00ce..4336e04e070c 100644 --- a/src/output-json-dnp3.c +++ b/src/output-json-dnp3.c @@ -210,6 +210,27 @@ void JsonDNP3LogResponse(JsonBuilder *js, DNP3Transaction *dnp3tx) jb_close(js); } +bool AlertJsonDnp3(void *vtx, JsonBuilder *js) +{ + DNP3Transaction *tx = (DNP3Transaction *)vtx; + bool logged = false; + jb_open_object(js, "dnp3"); + if (tx->is_request && tx->done) { + jb_open_object(js, "request"); + JsonDNP3LogRequest(js, tx); + jb_close(js); + logged = true; + } + if (!tx->is_request && tx->done) { + jb_open_object(js, "response"); + JsonDNP3LogResponse(js, tx); + jb_close(js); + logged = true; + } + jb_close(js); + return logged; +} + static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) { diff --git a/src/output-json-dnp3.h b/src/output-json-dnp3.h index 85d02ff1011c..6f81026780d9 100644 --- a/src/output-json-dnp3.h +++ b/src/output-json-dnp3.h @@ -24,5 +24,6 @@ void JsonDNP3LogRequest(JsonBuilder *js, DNP3Transaction *); void JsonDNP3LogResponse(JsonBuilder *js, DNP3Transaction *); void JsonDNP3LogRegister(void); +bool AlertJsonDnp3(void *vtx, JsonBuilder *js); #endif /* __OUTPUT_JSON_DNP3_H__ */ diff --git a/src/output-json-dns.c b/src/output-json-dns.c index 020e27853a9e..b27c67feb240 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -263,7 +263,7 @@ typedef struct LogDnsLogThread_ { OutputJsonThreadCtx *ctx; } LogDnsLogThread; -JsonBuilder *JsonDNSLogQuery(void *txptr) +static JsonBuilder *JsonDNSLogQuery(void *txptr) { JsonBuilder *queryjb = jb_new_array(); if (queryjb == NULL) { @@ -292,7 +292,7 @@ JsonBuilder *JsonDNSLogQuery(void *txptr) return queryjb; } -JsonBuilder *JsonDNSLogAnswer(void *txptr) +static JsonBuilder *JsonDNSLogAnswer(void *txptr) { if (!rs_dns_do_log_answer(txptr, LOG_ALL_RRTYPES)) { return NULL; @@ -304,6 +304,23 @@ JsonBuilder *JsonDNSLogAnswer(void *txptr) } } +bool AlertJsonDns(void *txptr, JsonBuilder *js) +{ + jb_open_object(js, "dns"); + JsonBuilder *qjs = JsonDNSLogQuery(txptr); + if (qjs != NULL) { + jb_set_object(js, "query", qjs); + jb_free(qjs); + } + JsonBuilder *ajs = JsonDNSLogAnswer(txptr); + if (ajs != NULL) { + jb_set_object(js, "answer", ajs); + jb_free(ajs); + } + jb_close(js); + return true; +} + static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id) { diff --git a/src/output-json-dns.h b/src/output-json-dns.h index 1e19427361b8..f46cad011089 100644 --- a/src/output-json-dns.h +++ b/src/output-json-dns.h @@ -26,7 +26,6 @@ void JsonDnsLogRegister(void); -JsonBuilder *JsonDNSLogQuery(void *txptr) __attribute__((nonnull)); -JsonBuilder *JsonDNSLogAnswer(void *txptr) __attribute__((nonnull)); +bool AlertJsonDns(void *vtx, JsonBuilder *js); #endif /* __OUTPUT_JSON_DNS_H__ */ diff --git a/src/output-json-file.c b/src/output-json-file.c index 3b015ea88e06..f81fc0908ebc 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -123,6 +123,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, return NULL; JsonBuilderMark mark = { 0, 0, 0 }; + SimpleJsonAppLayerLogger *al; switch (p->flow->alproto) { case ALPROTO_HTTP1: jb_open_object(js, "http"); @@ -172,13 +173,19 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, jb_restore_mark(js, &mark); } break; - case ALPROTO_HTTP2: - jb_get_mark(js, &mark); - jb_open_object(js, "http"); - if (EveHTTP2AddMetadata(p->flow, tx_id, js)) { - jb_close(js); - } else { - jb_restore_mark(js, &mark); + default: + al = GetAppProtoSimpleJsonLogger(p->flow->alproto); + if (al && al->LogTx) { + void *state = FlowGetAppState(p->flow); + if (state) { + tx = AppLayerParserGetTx(p->flow->proto, p->flow->alproto, state, tx_id); + if (tx) { + jb_get_mark(js, &mark); + if (!al->LogTx(tx, js)) { + jb_restore_mark(js, &mark); + } + } + } } break; } diff --git a/src/output-json-ftp.c b/src/output-json-ftp.c index ece9344bf409..9fb8b8c622db 100644 --- a/src/output-json-ftp.c +++ b/src/output-json-ftp.c @@ -57,6 +57,7 @@ static void EveFTPLogCommand(FTPTransaction *tx, JsonBuilder *jb) return; } } + jb_open_object(jb, "ftp"); jb_set_string(jb, "command", tx->command_descriptor->command_name); uint32_t min_length = tx->command_descriptor->command_length + 1; /* command + space */ if (tx->request_length > min_length) { @@ -149,6 +150,7 @@ static void EveFTPLogCommand(FTPTransaction *tx, JsonBuilder *jb) } else { JB_SET_FALSE(jb, "reply_truncated"); } + jb_close(jb); } @@ -169,17 +171,14 @@ static int JsonFTPLogger(ThreadVars *tv, void *thread_data, JsonBuilder *jb = CreateEveHeaderWithTxId(p, LOG_DIR_FLOW, event_type, NULL, tx_id, thread->ctx); if (likely(jb)) { - jb_open_object(jb, event_type); if (f->alproto == ALPROTO_FTPDATA) { - EveFTPDataAddMetadata(f, jb); + if (!EveFTPDataAddMetadata(vtx, jb)) { + goto fail; + } } else { EveFTPLogCommand(tx, jb); } - if (!jb_close(jb)) { - goto fail; - } - OutputJsonBuilderBuffer(jb, thread); jb_free(jb); diff --git a/src/output-json-http2.c b/src/output-json-http2.c index d762e76d0665..7165ae8f6302 100644 --- a/src/output-json-http2.c +++ b/src/output-json-http2.c @@ -61,19 +61,6 @@ typedef struct JsonHttp2LogThread_ { OutputJsonThreadCtx *ctx; } JsonHttp2LogThread; - -bool EveHTTP2AddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *jb) -{ - void *state = FlowGetAppState(f); - if (state) { - void *tx = AppLayerParserGetTx(f->proto, ALPROTO_HTTP2, state, tx_id); - if (tx) { - return rs_http2_log_json(tx, jb); - } - } - return false; -} - static int JsonHttp2Logger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *txptr, uint64_t tx_id) { @@ -88,11 +75,9 @@ static int JsonHttp2Logger(ThreadVars *tv, void *thread_data, const Packet *p, if (unlikely(js == NULL)) return 0; - jb_open_object(js, "http"); if (!rs_http2_log_json(txptr, js)) { goto end; } - jb_close(js); OutputJsonBuilderBuffer(js, aft->ctx); end: jb_free(js); diff --git a/src/output-json-http2.h b/src/output-json-http2.h index 66bf2ade968e..88ba420ab2df 100644 --- a/src/output-json-http2.h +++ b/src/output-json-http2.h @@ -25,6 +25,5 @@ #define __OUTPUT_JSON_HTTP2_H__ void JsonHttp2LogRegister(void); -bool EveHTTP2AddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *jb); #endif /* __OUTPUT_JSON_HTTP2_H__ */ diff --git a/src/output-json-modbus.c b/src/output-json-modbus.c index ace8c061f92d..9e508ead9acc 100644 --- a/src/output-json-modbus.c +++ b/src/output-json-modbus.c @@ -136,19 +136,6 @@ static TmEcode JsonModbusLogThreadDeinit(ThreadVars *t, void *data) return TM_ECODE_OK; } -bool JsonModbusAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js) -{ - void *state = FlowGetAppState(f); - if (state) { - void *tx = AppLayerParserGetTx(f->proto, ALPROTO_MODBUS, state, tx_id); - if (tx) { - return rs_modbus_to_json(tx, js); - } - } - - return false; -} - void JsonModbusLogRegister(void) { /* Register as an eve sub-module. */ diff --git a/src/output-json-modbus.h b/src/output-json-modbus.h index 9bde2dae57a3..2b07e4eb2d5c 100644 --- a/src/output-json-modbus.h +++ b/src/output-json-modbus.h @@ -19,6 +19,5 @@ #define __OUTPUT_JSON_MODBUS_H__ void JsonModbusLogRegister(void); -bool JsonModbusAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js); #endif /* __OUTPUT_JSON_MODBUS_H__ */ diff --git a/src/output-json-mqtt.c b/src/output-json-mqtt.c index 9ea890508070..2f600343e20d 100644 --- a/src/output-json-mqtt.c +++ b/src/output-json-mqtt.c @@ -59,17 +59,9 @@ typedef struct LogMQTTLogThread_ { OutputJsonThreadCtx *ctx; } LogMQTTLogThread; -bool JsonMQTTAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js) +bool JsonMQTTAddMetadata(void *vtx, JsonBuilder *js) { - MQTTState *state = FlowGetAppState(f); - if (state) { - MQTTTransaction *tx = AppLayerParserGetTx(f->proto, ALPROTO_MQTT, state, tx_id); - if (tx) { - return rs_mqtt_logger_log(tx, MQTT_DEFAULTS, js); - } - } - - return false; + return rs_mqtt_logger_log(vtx, MQTT_DEFAULTS, js); } static int JsonMQTTLogger(ThreadVars *tv, void *thread_data, diff --git a/src/output-json-mqtt.h b/src/output-json-mqtt.h index 1acb4e107faf..42d66f48680d 100644 --- a/src/output-json-mqtt.h +++ b/src/output-json-mqtt.h @@ -25,6 +25,6 @@ #define __OUTPUT_JSON_MQTT_H__ void JsonMQTTLogRegister(void); -bool JsonMQTTAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js); +bool JsonMQTTAddMetadata(void *vtx, JsonBuilder *js); #endif /* __OUTPUT_JSON_MQTT_H__ */ diff --git a/src/output-json-quic.c b/src/output-json-quic.c index fdf2d0f09340..830ac78fdfbb 100644 --- a/src/output-json-quic.c +++ b/src/output-json-quic.c @@ -140,19 +140,6 @@ static TmEcode JsonQuicLogThreadDeinit(ThreadVars *t, void *data) return TM_ECODE_OK; } -bool JsonQuicAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js) -{ - void *state = FlowGetAppState(f); - if (state) { - void *tx = AppLayerParserGetTx(f->proto, ALPROTO_QUIC, state, tx_id); - if (tx) { - return rs_quic_to_json(tx, js); - } - } - - return false; -} - void JsonQuicLogRegister(void) { /* Register as an eve sub-module. */ diff --git a/src/output-json-quic.h b/src/output-json-quic.h index 2448d5063a34..48e38185f2bd 100644 --- a/src/output-json-quic.h +++ b/src/output-json-quic.h @@ -22,7 +22,6 @@ #ifndef __OUTPUT_JSON_QUIC_H__ #define __OUTPUT_JSON_QUIC_H__ -bool JsonQuicAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js); void JsonQuicLogRegister(void); #endif /* __OUTPUT_JSON_QUIC_H__ */ diff --git a/src/output-json-rfb.c b/src/output-json-rfb.c index cc12d2f1bbdc..e2b832bece13 100644 --- a/src/output-json-rfb.c +++ b/src/output-json-rfb.c @@ -46,19 +46,6 @@ #include "rust-bindings.h" -bool JsonRFBAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js) -{ - void *state = FlowGetAppState(f); - if (state) { - RFBTransaction *tx = AppLayerParserGetTx(f->proto, ALPROTO_RFB, state, tx_id); - if (tx) { - return rs_rfb_logger_log(tx, js); - } - } - - return false; -} - static int JsonRFBLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) { diff --git a/src/output-json-rfb.h b/src/output-json-rfb.h index 1264ee3f6b4b..7e4e48ebd4c8 100644 --- a/src/output-json-rfb.h +++ b/src/output-json-rfb.h @@ -26,6 +26,4 @@ void JsonRFBLogRegister(void); -bool JsonRFBAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js); - #endif /* __OUTPUT_JSON_RFB_H__ */ diff --git a/src/output-json-sip.c b/src/output-json-sip.c index 8297be1cc3eb..7dd442cf6aba 100644 --- a/src/output-json-sip.c +++ b/src/output-json-sip.c @@ -48,17 +48,6 @@ #include "rust.h" -void JsonSIPAddMetadata(JsonBuilder *js, const Flow *f, uint64_t tx_id) -{ - SIPState *state = FlowGetAppState(f); - if (state) { - SIPTransaction *tx = AppLayerParserGetTx(f->proto, ALPROTO_SIP, state, tx_id); - if (tx) { - rs_sip_log_json(tx, js); - } - } -} - static int JsonSIPLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) { diff --git a/src/output-json-sip.h b/src/output-json-sip.h index 60145dab5b98..0d2c53fa50df 100644 --- a/src/output-json-sip.h +++ b/src/output-json-sip.h @@ -26,6 +26,4 @@ void JsonSIPLogRegister(void); -void JsonSIPAddMetadata(JsonBuilder *js, const Flow *f, uint64_t tx_id); - #endif /* __OUTPUT_JSON_SIP_H__ */ diff --git a/src/output-json-snmp.c b/src/output-json-snmp.c index 27545b6f6903..cbf0a7c992e4 100644 --- a/src/output-json-snmp.c +++ b/src/output-json-snmp.c @@ -59,11 +59,9 @@ static int JsonSNMPLogger(ThreadVars *tv, void *thread_data, return TM_ECODE_FAILED; } - jb_open_object(jb, "snmp"); - if (!rs_snmp_log_json_response(jb, snmptx)) { + if (!rs_snmp_log_json_response(snmptx, jb)) { goto error; } - jb_close(jb); OutputJsonBuilderBuffer(jb, thread); diff --git a/src/output-json-ssh.c b/src/output-json-ssh.c index 5ec70142f634..45a8d8eab333 100644 --- a/src/output-json-ssh.c +++ b/src/output-json-ssh.c @@ -64,11 +64,9 @@ static int JsonSshLogger(ThreadVars *tv, void *thread_data, const Packet *p, if (unlikely(js == NULL)) return 0; - jb_open_object(js, "ssh"); if (!rs_ssh_log_json(txptr, js)) { goto end; } - jb_close(js); OutputJsonBuilderBuffer(js, thread); end: diff --git a/src/output-json-template.c b/src/output-json-template.c index 76d42ad834e6..2ca48b7ae373 100644 --- a/src/output-json-template.c +++ b/src/output-json-template.c @@ -74,11 +74,9 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, const Packet *p return TM_ECODE_FAILED; } - jb_open_object(js, "template"); if (!rs_template_logger_log(tx, js)) { goto error; } - jb_close(js); OutputJsonBuilderBuffer(js, thread->ctx); jb_free(js); diff --git a/src/output-json-tls.c b/src/output-json-tls.c index 9771f4d1cd7c..7460a32f2574 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -392,8 +392,9 @@ static void JsonTlsLogJSONCustom(OutputTlsCtx *tls_ctx, JsonBuilder *js, } } -void JsonTlsLogJSONExtended(JsonBuilder *tjs, SSLState * state) +static bool JsonTlsLogJSONExtendedAux(void *vtx, JsonBuilder *tjs) { + SSLState *state = (SSLState *)vtx; JsonTlsLogJSONBasic(tjs, state); /* tls serial */ @@ -425,6 +426,15 @@ void JsonTlsLogJSONExtended(JsonBuilder *tjs, SSLState * state) JsonTlsLogClientCert(tjs, &state->client_connp, false, false); jb_close(tjs); } + return true; +} + +bool JsonTlsLogJSONExtended(void *vtx, JsonBuilder *tjs) +{ + jb_open_object(tjs, "tls"); + bool r = JsonTlsLogJSONExtendedAux(vtx, tjs); + jb_close(tjs); + return r; } static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, @@ -459,7 +469,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, } /* log extended */ else if (tls_ctx->flags & LOG_TLS_EXTENDED) { - JsonTlsLogJSONExtended(js, ssl_state); + JsonTlsLogJSONExtendedAux(ssl_state, js); } /* log basic */ else { diff --git a/src/output-json-tls.h b/src/output-json-tls.h index 737e6233ef10..42f706b91d3f 100644 --- a/src/output-json-tls.h +++ b/src/output-json-tls.h @@ -29,6 +29,6 @@ void JsonTlsLogRegister(void); #include "app-layer-ssl.h" void JsonTlsLogJSONBasic(JsonBuilder *js, SSLState *ssl_state); -void JsonTlsLogJSONExtended(JsonBuilder *js, SSLState *ssl_state); +bool JsonTlsLogJSONExtended(void *vtx, JsonBuilder *js); #endif /* __OUTPUT_JSON_TLS_H__ */ diff --git a/src/output.c b/src/output.c index c13ab4862eda..bdc905e58bfd 100644 --- a/src/output.c +++ b/src/output.c @@ -67,6 +67,8 @@ #include "log-stats.h" #include "output-json-nfs.h" #include "output-json-ftp.h" +// for misplaced EveFTPDataAddMetadata +#include "app-layer-ftp.h" #include "output-json-tftp.h" #include "output-json-smb.h" #include "output-json-ike.h" @@ -1126,3 +1128,53 @@ void OutputRegisterLoggers(void) /* BitTorrent DHT JSON logger */ JsonBitTorrentDHTLogRegister(); } + +static SimpleJsonAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { + { ALPROTO_UNKNOWN, NULL }, + { ALPROTO_HTTP1, NULL }, // special: uses some options flags + { ALPROTO_FTP, NULL }, // TODO missing + { ALPROTO_SMTP, NULL }, // special: uses state + { ALPROTO_TLS, JsonTlsLogJSONExtended }, + { ALPROTO_SSH, rs_ssh_log_json }, + { ALPROTO_IMAP, NULL }, // protocol detection only + { ALPROTO_JABBER, NULL }, // no parser, no logging + { ALPROTO_SMB, NULL }, // special: uses state + { ALPROTO_DCERPC, NULL }, // TODO missing + { ALPROTO_IRC, NULL }, // no parser, no logging + { ALPROTO_DNS, AlertJsonDns }, + { ALPROTO_MODBUS, (SimpleJsonTxLogFunc)rs_modbus_to_json }, + { ALPROTO_ENIP, NULL }, // no logging + { ALPROTO_DNP3, AlertJsonDnp3 }, + { ALPROTO_NFS, NULL }, // special: uses state + { ALPROTO_NTP, NULL }, // no logging + { ALPROTO_FTPDATA, EveFTPDataAddMetadata }, + { ALPROTO_TFTP, NULL }, // TODO missing + { ALPROTO_IKE, NULL }, // special: uses state + { ALPROTO_KRB5, NULL }, // TODO missing + { ALPROTO_QUIC, rs_quic_to_json }, + { ALPROTO_DHCP, NULL }, // TODO missing + { ALPROTO_SNMP, (SimpleJsonTxLogFunc)rs_snmp_log_json_response }, + { ALPROTO_SIP, (SimpleJsonTxLogFunc)rs_sip_log_json }, + { ALPROTO_RFB, rs_rfb_logger_log }, + { ALPROTO_MQTT, JsonMQTTAddMetadata }, + { ALPROTO_PGSQL, NULL }, // TODO missing + { ALPROTO_TELNET, NULL }, // no logging + { ALPROTO_TEMPLATE, rs_template_logger_log }, + { ALPROTO_RDP, (SimpleJsonTxLogFunc)rs_rdp_to_json }, + { ALPROTO_HTTP2, rs_http2_log_json }, + { ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log }, + { ALPROTO_HTTP, NULL }, // signature protocol, not for app-layer logging + { ALPROTO_FAILED, NULL }, +#ifdef UNITTESTS + { ALPROTO_TEST, NULL }, +#endif /* UNITESTS */ +}; + +SimpleJsonAppLayerLogger *GetAppProtoSimpleJsonLogger(AppProto alproto) +{ + if (alproto < ALPROTO_MAX) { + BUG_ON(simple_json_applayer_loggers[alproto].proto != alproto); + return &simple_json_applayer_loggers[alproto]; + } + return NULL; +} diff --git a/src/output.h b/src/output.h index 5c2d7bc90e62..3daba06a33d3 100644 --- a/src/output.h +++ b/src/output.h @@ -208,4 +208,13 @@ void OutputLoggerExitPrintStats(ThreadVars *, void *); void OutputSetupActiveLoggers(void); void OutputClearActiveLoggers(void); +typedef bool (*SimpleJsonTxLogFunc)(void *, struct JsonBuilder *); + +typedef struct SimpleJsonAppLayerLogger { + AppProto proto; + SimpleJsonTxLogFunc LogTx; +} SimpleJsonAppLayerLogger; + +SimpleJsonAppLayerLogger *GetAppProtoSimpleJsonLogger(AppProto alproto); + #endif /* ! __OUTPUT_H__ */ From 405d00c8c987918e4559900e9c5f4d7ff8adb950 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 09:49:58 +0100 Subject: [PATCH 02/12] output/ftp: have ftp properties in alerts Ticket: 6500 --- src/output-json-ftp.c | 9 +++++---- src/output-json-ftp.h | 1 + src/output.c | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/output-json-ftp.c b/src/output-json-ftp.c index 9fb8b8c622db..34422f72f4af 100644 --- a/src/output-json-ftp.c +++ b/src/output-json-ftp.c @@ -46,15 +46,16 @@ #include "app-layer-ftp.h" #include "output-json-ftp.h" -static void EveFTPLogCommand(FTPTransaction *tx, JsonBuilder *jb) +bool EveFTPLogCommand(void *vtx, JsonBuilder *jb) { + FTPTransaction *tx = vtx; /* Preallocate array objects to simplify failure case */ JsonBuilder *js_resplist = NULL; if (!TAILQ_EMPTY(&tx->response_list)) { js_resplist = jb_new_array(); if (unlikely(js_resplist == NULL)) { - return; + return false; } } jb_open_object(jb, "ftp"); @@ -151,6 +152,7 @@ static void EveFTPLogCommand(FTPTransaction *tx, JsonBuilder *jb) JB_SET_FALSE(jb, "reply_truncated"); } jb_close(jb); + return true; } @@ -166,7 +168,6 @@ static int JsonFTPLogger(ThreadVars *tv, void *thread_data, } else { event_type = "ftp"; } - FTPTransaction *tx = vtx; JsonBuilder *jb = CreateEveHeaderWithTxId(p, LOG_DIR_FLOW, event_type, NULL, tx_id, thread->ctx); @@ -176,7 +177,7 @@ static int JsonFTPLogger(ThreadVars *tv, void *thread_data, goto fail; } } else { - EveFTPLogCommand(tx, jb); + EveFTPLogCommand(vtx, jb); } OutputJsonBuilderBuffer(jb, thread); diff --git a/src/output-json-ftp.h b/src/output-json-ftp.h index acba5539e1c6..704defd9585c 100644 --- a/src/output-json-ftp.h +++ b/src/output-json-ftp.h @@ -25,5 +25,6 @@ #define __OUTPUT_JSON_FTP_H__ void JsonFTPLogRegister(void); +bool EveFTPLogCommand(void *vtx, JsonBuilder *js); #endif /* __OUTPUT_JSON_FTP_H__ */ diff --git a/src/output.c b/src/output.c index bdc905e58bfd..a6391bdf3b78 100644 --- a/src/output.c +++ b/src/output.c @@ -1132,8 +1132,8 @@ void OutputRegisterLoggers(void) static SimpleJsonAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { { ALPROTO_UNKNOWN, NULL }, { ALPROTO_HTTP1, NULL }, // special: uses some options flags - { ALPROTO_FTP, NULL }, // TODO missing - { ALPROTO_SMTP, NULL }, // special: uses state + { ALPROTO_FTP, EveFTPLogCommand }, + { ALPROTO_SMTP, NULL }, // special: uses state { ALPROTO_TLS, JsonTlsLogJSONExtended }, { ALPROTO_SSH, rs_ssh_log_json }, { ALPROTO_IMAP, NULL }, // protocol detection only From 604b68259a0f5c6aa644308a998832dcb8a9a2a1 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 09:52:12 +0100 Subject: [PATCH 03/12] output/tftp: have tftp properties in alerts Ticket: 6501 --- rust/src/tftp/log.rs | 2 ++ src/output-json-tftp.c | 2 -- src/output.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/src/tftp/log.rs b/rust/src/tftp/log.rs index b4837036a156..f6e63531a07e 100644 --- a/rust/src/tftp/log.rs +++ b/rust/src/tftp/log.rs @@ -24,6 +24,7 @@ fn tftp_log_request(tx: &mut TFTPTransaction, jb: &mut JsonBuilder) -> Result<(), JsonError> { + jb.open_object("tftp")?; match tx.opcode { 1 => jb.set_string("packet", "read")?, 2 => jb.set_string("packet", "write")?, @@ -31,6 +32,7 @@ fn tftp_log_request(tx: &mut TFTPTransaction, }; jb.set_string("file", tx.filename.as_str())?; jb.set_string("mode", tx.mode.as_str())?; + jb.close()?; Ok(()) } diff --git a/src/output-json-tftp.c b/src/output-json-tftp.c index 4fff67a8b696..a0bc9ee1809e 100644 --- a/src/output-json-tftp.c +++ b/src/output-json-tftp.c @@ -58,11 +58,9 @@ static int JsonTFTPLogger(ThreadVars *tv, void *thread_data, return TM_ECODE_FAILED; } - jb_open_object(jb, "tftp"); if (unlikely(!rs_tftp_log_json_request(tx, jb))) { goto error; } - jb_close(jb); OutputJsonBuilderBuffer(jb, thread); diff --git a/src/output.c b/src/output.c index a6391bdf3b78..43a10d92bf85 100644 --- a/src/output.c +++ b/src/output.c @@ -1148,7 +1148,7 @@ static SimpleJsonAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { { ALPROTO_NFS, NULL }, // special: uses state { ALPROTO_NTP, NULL }, // no logging { ALPROTO_FTPDATA, EveFTPDataAddMetadata }, - { ALPROTO_TFTP, NULL }, // TODO missing + { ALPROTO_TFTP, (SimpleJsonTxLogFunc)rs_tftp_log_json_request }, { ALPROTO_IKE, NULL }, // special: uses state { ALPROTO_KRB5, NULL }, // TODO missing { ALPROTO_QUIC, rs_quic_to_json }, From 172a9bca51297702ffa7369612efb36febd62776 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 09:55:03 +0100 Subject: [PATCH 04/12] output/krb5: have krb5 properties in alerts Ticket: 5977 --- rust/src/krb/log.rs | 4 +++- src/output-json-krb5.c | 4 +--- src/output.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rust/src/krb/log.rs b/rust/src/krb/log.rs index 7cb952581bc7..58c0d64b4893 100644 --- a/rust/src/krb/log.rs +++ b/rust/src/krb/log.rs @@ -22,6 +22,7 @@ use crate::krb::krb5::{KRB5Transaction,test_weak_encryption}; fn krb5_log_response(jsb: &mut JsonBuilder, tx: &mut KRB5Transaction) -> Result<(), JsonError> { + jsb.open_object("krb5")?; match tx.error_code { Some(c) => { jsb.set_string("msg_type", &format!("{:?}", tx.msg_type))?; @@ -63,12 +64,13 @@ fn krb5_log_response(jsb: &mut JsonBuilder, tx: &mut KRB5Transaction) -> Result< jsb.set_string("ticket_encryption", &refs)?; jsb.set_bool("ticket_weak_encryption", test_weak_encryption(x))?; } + jsb.close()?; return Ok(()); } #[no_mangle] -pub extern "C" fn rs_krb5_log_json_response(jsb: &mut JsonBuilder, tx: &mut KRB5Transaction) -> bool +pub extern "C" fn rs_krb5_log_json_response(tx: &mut KRB5Transaction, jsb: &mut JsonBuilder) -> bool { krb5_log_response(jsb, tx).is_ok() } diff --git a/src/output-json-krb5.c b/src/output-json-krb5.c index 5e6fbad5ecd1..9fc45c5d3c53 100644 --- a/src/output-json-krb5.c +++ b/src/output-json-krb5.c @@ -59,11 +59,9 @@ static int JsonKRB5Logger(ThreadVars *tv, void *thread_data, return TM_ECODE_FAILED; } - jb_open_object(jb, "krb5"); - if (!rs_krb5_log_json_response(jb, krb5tx)) { + if (!rs_krb5_log_json_response(krb5tx, jb)) { goto error; } - jb_close(jb); OutputJsonBuilderBuffer(jb, thread); diff --git a/src/output.c b/src/output.c index 43a10d92bf85..087f2b9a18e3 100644 --- a/src/output.c +++ b/src/output.c @@ -1149,8 +1149,8 @@ static SimpleJsonAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { { ALPROTO_NTP, NULL }, // no logging { ALPROTO_FTPDATA, EveFTPDataAddMetadata }, { ALPROTO_TFTP, (SimpleJsonTxLogFunc)rs_tftp_log_json_request }, - { ALPROTO_IKE, NULL }, // special: uses state - { ALPROTO_KRB5, NULL }, // TODO missing + { ALPROTO_IKE, NULL }, // special: uses state + { ALPROTO_KRB5, (SimpleJsonTxLogFunc)rs_krb5_log_json_response }, { ALPROTO_QUIC, rs_quic_to_json }, { ALPROTO_DHCP, NULL }, // TODO missing { ALPROTO_SNMP, (SimpleJsonTxLogFunc)rs_snmp_log_json_response }, From 7c7aa8a6814edee80e1fc88d6b25991444195f4f Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Mon, 15 May 2023 09:43:14 +0200 Subject: [PATCH 05/12] output/dnp3: restrict function scope to one file --- src/output-json-dnp3.c | 4 ++-- src/output-json-dnp3.h | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/output-json-dnp3.c b/src/output-json-dnp3.c index 4336e04e070c..1387d6213632 100644 --- a/src/output-json-dnp3.c +++ b/src/output-json-dnp3.c @@ -140,7 +140,7 @@ static void JsonDNP3LogObjects(JsonBuilder *js, DNP3ObjectList *objects) } } -void JsonDNP3LogRequest(JsonBuilder *js, DNP3Transaction *dnp3tx) +static void JsonDNP3LogRequest(JsonBuilder *js, DNP3Transaction *dnp3tx) { JB_SET_STRING(js, "type", "request"); @@ -171,7 +171,7 @@ void JsonDNP3LogRequest(JsonBuilder *js, DNP3Transaction *dnp3tx) jb_close(js); } -void JsonDNP3LogResponse(JsonBuilder *js, DNP3Transaction *dnp3tx) +static void JsonDNP3LogResponse(JsonBuilder *js, DNP3Transaction *dnp3tx) { if (dnp3tx->ah.function_code == DNP3_APP_FC_UNSOLICITED_RESP) { JB_SET_STRING(js, "type", "unsolicited_response"); diff --git a/src/output-json-dnp3.h b/src/output-json-dnp3.h index 6f81026780d9..5b5f56236abd 100644 --- a/src/output-json-dnp3.h +++ b/src/output-json-dnp3.h @@ -20,9 +20,6 @@ #include "app-layer-dnp3.h" -void JsonDNP3LogRequest(JsonBuilder *js, DNP3Transaction *); -void JsonDNP3LogResponse(JsonBuilder *js, DNP3Transaction *); - void JsonDNP3LogRegister(void); bool AlertJsonDnp3(void *vtx, JsonBuilder *js); From 1acb710bc6944ad5db35c288f8273caba4aa0a1a Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Mon, 5 Jun 2023 08:43:05 +0200 Subject: [PATCH 06/12] output/dns: do not add empty app-layer metadata --- src/output-json-dns.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/output-json-dns.c b/src/output-json-dns.c index b27c67feb240..b5bf865ae34c 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -306,19 +306,22 @@ static JsonBuilder *JsonDNSLogAnswer(void *txptr) bool AlertJsonDns(void *txptr, JsonBuilder *js) { + bool r = false; jb_open_object(js, "dns"); JsonBuilder *qjs = JsonDNSLogQuery(txptr); if (qjs != NULL) { jb_set_object(js, "query", qjs); jb_free(qjs); + r = true; } JsonBuilder *ajs = JsonDNSLogAnswer(txptr); if (ajs != NULL) { jb_set_object(js, "answer", ajs); jb_free(ajs); + r = true; } jb_close(js); - return true; + return r; } static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data, From 4391e059f6ed253b7432653be46931efb0a50a83 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Sun, 14 May 2023 20:03:02 +0200 Subject: [PATCH 07/12] output: generic simple tx json logger Ticket: 3827 --- scripts/setup-app-layer.py | 34 ++--- src/Makefile.am | 24 ---- src/output-json-alert.c | 5 - src/output-json-bittorrent-dht.c | 163 ----------------------- src/output-json-bittorrent-dht.h | 27 ---- src/output-json-file.c | 1 - src/output-json-ftp.c | 57 -------- src/output-json-ftp.h | 1 - src/output-json-http2.c | 169 ------------------------ src/output-json-http2.h | 29 ---- src/output-json-krb5.c | 92 ------------- src/output-json-krb5.h | 29 ---- src/output-json-modbus.c | 147 --------------------- src/output-json-modbus.h | 23 ---- src/output-json-quic.c | 151 --------------------- src/output-json-quic.h | 27 ---- src/output-json-rdp.c | 79 ----------- src/output-json-rdp.h | 29 ---- src/output-json-rfb.c | 86 ------------ src/output-json-rfb.h | 29 ---- src/output-json-sip.c | 91 ------------- src/output-json-sip.h | 29 ---- src/output-json-snmp.c | 91 ------------- src/output-json-snmp.h | 29 ---- src/output-json-ssh.c | 89 ------------- src/output-json-ssh.h | 29 ---- src/output-json-template.c | 176 ------------------------- src/output-json-template.h | 29 ---- src/output-json-tftp.c | 90 ------------- src/output-json-tftp.h | 29 ---- src/output.c | 220 +++++++++++++++++++++++++++---- 31 files changed, 206 insertions(+), 1898 deletions(-) delete mode 100644 src/output-json-bittorrent-dht.c delete mode 100644 src/output-json-bittorrent-dht.h delete mode 100644 src/output-json-http2.c delete mode 100644 src/output-json-http2.h delete mode 100644 src/output-json-krb5.c delete mode 100644 src/output-json-krb5.h delete mode 100644 src/output-json-modbus.c delete mode 100644 src/output-json-modbus.h delete mode 100644 src/output-json-quic.c delete mode 100644 src/output-json-quic.h delete mode 100644 src/output-json-rdp.c delete mode 100644 src/output-json-rdp.h delete mode 100644 src/output-json-rfb.c delete mode 100644 src/output-json-rfb.h delete mode 100644 src/output-json-sip.c delete mode 100644 src/output-json-sip.h delete mode 100644 src/output-json-snmp.c delete mode 100644 src/output-json-snmp.h delete mode 100644 src/output-json-ssh.c delete mode 100644 src/output-json-ssh.h delete mode 100644 src/output-json-template.c delete mode 100644 src/output-json-template.h delete mode 100644 src/output-json-tftp.c delete mode 100644 src/output-json-tftp.h diff --git a/scripts/setup-app-layer.py b/scripts/setup-app-layer.py index d8426634bca8..26d9892b87f7 100755 --- a/scripts/setup-app-layer.py +++ b/scripts/setup-app-layer.py @@ -200,15 +200,21 @@ def logger_patch_output_c(proto): output = io.StringIO() inlines = open(filename).readlines() for i, line in enumerate(inlines): - if line.find("ALPROTO_TEMPLATE") > -1: - new_line = line.replace("TEMPLATE", proto.upper()).replace( - "template", proto.lower()) - output.write(new_line) - if line.find("output-json-template.h") > -1: - output.write(line.replace("template", proto.lower())) if line.find("/* Template JSON logger.") > -1: output.write(inlines[i].replace("Template", proto)) output.write(inlines[i+1].replace("Template", proto)) + output.write(inlines[i+2].replace("TEMPLATE", proto.upper()).replace( + "template", proto.lower()).replace("Template", proto)) + output.write(inlines[i+3]) + if line.find("rs_template_logger_log") > -1: + output.write(inlines[i].replace("TEMPLATE", proto.upper()).replace( + "template", proto.lower())) + if line.find("OutputTemplateLogInitSub(") > -1: + output.write(inlines[i].replace("Template", proto)) + output.write(inlines[i+1]) + output.write(inlines[i+2].replace("TEMPLATE", proto.upper())) + output.write(inlines[i+3]) + output.write(inlines[i+4]) output.write(line) open(filename, "w").write(output.getvalue()) @@ -216,27 +222,12 @@ def logger_copy_templates(proto): lower = proto.lower() pairs = ( - ("src/output-json-template.h", - "src/output-json-%s.h" % (lower)), - ("src/output-json-template.c", - "src/output-json-%s.c" % (lower)), ("rust/src/applayertemplate/logger.rs", "rust/src/applayer%s/logger.rs" % (lower)), ) common_copy_templates(proto, pairs) -def logger_patch_makefile_am(protoname): - filename = "src/Makefile.am" - print("Patching %s." % (filename)) - output = io.StringIO() - with open(filename) as infile: - for line in infile: - if line.lstrip().startswith("output-json-template."): - output.write(line.replace("template", protoname.lower())) - output.write(line) - open(filename, "w").write(output.getvalue()) - def detect_copy_templates(proto, buffername): lower = proto.lower() @@ -398,7 +389,6 @@ def main(): raise SetupError("no app-layer parser exists for %s" % (proto)) logger_copy_templates(proto) patch_rust_applayer_mod_rs(proto) - logger_patch_makefile_am(proto) logger_patch_output_c(proto) logger_patch_suricata_yaml_in(proto) diff --git a/src/Makefile.am b/src/Makefile.am index 48a5ce850ce2..ba11f704ecc8 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -392,7 +392,6 @@ noinst_HEADERS = \ output.h \ output-json-alert.h \ output-json-anomaly.h \ - output-json-bittorrent-dht.h \ output-json-dcerpc.h \ output-json-dhcp.h \ output-json-dnp3.h \ @@ -405,27 +404,16 @@ noinst_HEADERS = \ output-json-frame.h \ output-json-ftp.h \ output-json.h \ - output-json-http2.h \ output-json-http.h \ output-json-ike.h \ - output-json-krb5.h \ output-json-metadata.h \ - output-json-modbus.h \ - output-json-quic.h \ output-json-mqtt.h \ output-json-netflow.h \ output-json-nfs.h \ output-json-pgsql.h \ - output-json-rdp.h \ - output-json-rfb.h \ - output-json-sip.h \ output-json-smb.h \ output-json-smtp.h \ - output-json-snmp.h \ - output-json-ssh.h \ output-json-stats.h \ - output-json-template.h \ - output-json-tftp.h \ output-json-tls.h \ output-eve-syslog.h \ output-lua.h \ @@ -1000,7 +988,6 @@ libsuricata_c_a_SOURCES = \ output-flow.c \ output-json-alert.c \ output-json-anomaly.c \ - output-json-bittorrent-dht.c \ output-json.c \ output-json-common.c \ output-json-dcerpc.c \ @@ -1014,27 +1001,16 @@ libsuricata_c_a_SOURCES = \ output-json-flow.c \ output-json-frame.c \ output-json-ftp.c \ - output-json-http2.c \ output-json-http.c \ output-json-ike.c \ - output-json-krb5.c \ output-json-metadata.c \ - output-json-modbus.c \ - output-json-quic.c \ output-json-mqtt.c \ output-json-netflow.c \ output-json-nfs.c \ output-json-pgsql.c \ - output-json-rdp.c \ - output-json-rfb.c \ - output-json-sip.c \ output-json-smb.c \ output-json-smtp.c \ - output-json-snmp.c \ - output-json-ssh.c \ output-json-stats.c \ - output-json-template.c \ - output-json-tftp.c \ output-json-tls.c \ output-eve-syslog.c \ output-lua.c \ diff --git a/src/output-json-alert.c b/src/output-json-alert.c index aa066619bd8d..ee514f65f076 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -64,20 +64,15 @@ #include "output-json-dns.h" #include "output-json-http.h" #include "output-json-tls.h" -#include "output-json-ssh.h" #include "rust.h" #include "output-json-smtp.h" #include "output-json-email-common.h" #include "output-json-nfs.h" #include "output-json-smb.h" #include "output-json-flow.h" -#include "output-json-sip.h" -#include "output-json-rfb.h" #include "output-json-mqtt.h" #include "output-json-ike.h" -#include "output-json-modbus.h" #include "output-json-frame.h" -#include "output-json-quic.h" #include "util-byte.h" #include "util-privs.h" diff --git a/src/output-json-bittorrent-dht.c b/src/output-json-bittorrent-dht.c deleted file mode 100644 index 066df78f61fb..000000000000 --- a/src/output-json-bittorrent-dht.c +++ /dev/null @@ -1,163 +0,0 @@ -/* Copyright (C) 2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * Implement JSON/eve logging app-layer BitTorrent DHT. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "output-json-bittorrent-dht.h" -#include "rust.h" - -typedef struct LogBitTorrentDHTFileCtx_ { - uint32_t flags; - OutputJsonCtx *eve_ctx; -} LogBitTorrentDHTFileCtx; - -typedef struct LogBitTorrentDHTLogThread_ { - LogBitTorrentDHTFileCtx *bittorrent_dht_log_ctx; - OutputJsonThreadCtx *ctx; -} LogBitTorrentDHTLogThread; - -static int JsonBitTorrentDHTLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, - void *state, void *tx, uint64_t tx_id) -{ - LogBitTorrentDHTLogThread *thread = thread_data; - - JsonBuilder *js = CreateEveHeader( - p, LOG_DIR_PACKET, "bittorrent_dht", NULL, thread->bittorrent_dht_log_ctx->eve_ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_FAILED; - } - - if (!rs_bittorrent_dht_logger_log(tx, js)) { - goto error; - } - - OutputJsonBuilderBuffer(js, thread->ctx); - jb_free(js); - - return TM_ECODE_OK; - -error: - jb_free(js); - return TM_ECODE_FAILED; -} - -static void OutputBitTorrentDHTLogDeInitCtxSub(OutputCtx *output_ctx) -{ - LogBitTorrentDHTFileCtx *bittorrent_dht_log_ctx = (LogBitTorrentDHTFileCtx *)output_ctx->data; - SCFree(bittorrent_dht_log_ctx); - SCFree(output_ctx); -} - -static OutputInitResult OutputBitTorrentDHTLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) -{ - OutputInitResult result = { NULL, false }; - OutputJsonCtx *ajt = parent_ctx->data; - - LogBitTorrentDHTFileCtx *bittorrent_dht_log_ctx = SCCalloc(1, sizeof(*bittorrent_dht_log_ctx)); - if (unlikely(bittorrent_dht_log_ctx == NULL)) { - return result; - } - bittorrent_dht_log_ctx->eve_ctx = ajt; - - OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx)); - if (unlikely(output_ctx == NULL)) { - SCFree(bittorrent_dht_log_ctx); - return result; - } - output_ctx->data = bittorrent_dht_log_ctx; - output_ctx->DeInit = OutputBitTorrentDHTLogDeInitCtxSub; - - AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_BITTORRENT_DHT); - - result.ctx = output_ctx; - result.ok = true; - return result; -} - -static TmEcode JsonBitTorrentDHTLogThreadInit(ThreadVars *t, const void *initdata, void **data) -{ - LogBitTorrentDHTLogThread *thread = SCCalloc(1, sizeof(*thread)); - if (unlikely(thread == NULL)) { - return TM_ECODE_FAILED; - } - - if (initdata == NULL) { - SCLogDebug("Error getting context for EveLogBitTorrentDHT. \"initdata\" is NULL."); - goto error_exit; - } - - thread->bittorrent_dht_log_ctx = ((OutputCtx *)initdata)->data; - thread->ctx = CreateEveThreadCtx(t, thread->bittorrent_dht_log_ctx->eve_ctx); - if (!thread->ctx) { - goto error_exit; - } - *data = (void *)thread; - - return TM_ECODE_OK; - -error_exit: - SCFree(thread); - return TM_ECODE_FAILED; -} - -static TmEcode JsonBitTorrentDHTLogThreadDeinit(ThreadVars *t, void *data) -{ - LogBitTorrentDHTLogThread *thread = (LogBitTorrentDHTLogThread *)data; - if (thread == NULL) { - return TM_ECODE_OK; - } - FreeEveThreadCtx(thread->ctx); - SCFree(thread); - return TM_ECODE_OK; -} - -void JsonBitTorrentDHTLogRegister(void) -{ - if (ConfGetNode("app-layer.protocols.bittorrent-dht") == NULL) { - return; - } - - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonBitTorrentDHTLog", - "eve-log.bittorrent-dht", OutputBitTorrentDHTLogInitSub, ALPROTO_BITTORRENT_DHT, - JsonBitTorrentDHTLogger, JsonBitTorrentDHTLogThreadInit, - JsonBitTorrentDHTLogThreadDeinit, NULL); -} diff --git a/src/output-json-bittorrent-dht.h b/src/output-json-bittorrent-dht.h deleted file mode 100644 index 8927f4d15996..000000000000 --- a/src/output-json-bittorrent-dht.h +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright (C) 2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - */ - -#ifndef __OUTPUT_JSON_BITTORRENT_DHT_H__ -#define __OUTPUT_JSON_BITTORRENT_DHT_H__ - -void JsonBitTorrentDHTLogRegister(void); - -#endif /* __OUTPUT_JSON_BITTORRENT_DHT_H__ */ diff --git a/src/output-json-file.c b/src/output-json-file.c index f81fc0908ebc..bba9e35373b5 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -61,7 +61,6 @@ #include "output-json-email-common.h" #include "output-json-nfs.h" #include "output-json-smb.h" -#include "output-json-http2.h" #include "app-layer-htp.h" #include "app-layer-htp-xff.h" diff --git a/src/output-json-ftp.c b/src/output-json-ftp.c index 34422f72f4af..14232bdfe393 100644 --- a/src/output-json-ftp.c +++ b/src/output-json-ftp.c @@ -154,60 +154,3 @@ bool EveFTPLogCommand(void *vtx, JsonBuilder *jb) jb_close(jb); return true; } - - -static int JsonFTPLogger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) -{ - SCEnter(); - OutputJsonThreadCtx *thread = thread_data; - - const char *event_type; - if (f->alproto == ALPROTO_FTPDATA) { - event_type = "ftp_data"; - } else { - event_type = "ftp"; - } - - JsonBuilder *jb = - CreateEveHeaderWithTxId(p, LOG_DIR_FLOW, event_type, NULL, tx_id, thread->ctx); - if (likely(jb)) { - if (f->alproto == ALPROTO_FTPDATA) { - if (!EveFTPDataAddMetadata(vtx, jb)) { - goto fail; - } - } else { - EveFTPLogCommand(vtx, jb); - } - - OutputJsonBuilderBuffer(jb, thread); - - jb_free(jb); - } - return TM_ECODE_OK; - -fail: - jb_free(jb); - return TM_ECODE_FAILED; -} - -static OutputInitResult OutputFTPLogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTP); - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTPDATA); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonFTPLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonFTPLog", "eve-log.ftp", - OutputFTPLogInitSub, ALPROTO_FTP, JsonFTPLogger, JsonLogThreadInit, JsonLogThreadDeinit, - NULL); - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonFTPLog", "eve-log.ftp", - OutputFTPLogInitSub, ALPROTO_FTPDATA, JsonFTPLogger, JsonLogThreadInit, - JsonLogThreadDeinit, NULL); - - SCLogDebug("FTP JSON logger registered."); -} diff --git a/src/output-json-ftp.h b/src/output-json-ftp.h index 704defd9585c..61922795ce6f 100644 --- a/src/output-json-ftp.h +++ b/src/output-json-ftp.h @@ -24,7 +24,6 @@ #ifndef __OUTPUT_JSON_FTP_H__ #define __OUTPUT_JSON_FTP_H__ -void JsonFTPLogRegister(void); bool EveFTPLogCommand(void *vtx, JsonBuilder *js); #endif /* __OUTPUT_JSON_FTP_H__ */ diff --git a/src/output-json-http2.c b/src/output-json-http2.c deleted file mode 100644 index 7165ae8f6302..000000000000 --- a/src/output-json-http2.c +++ /dev/null @@ -1,169 +0,0 @@ -/* Copyright (C) 2020-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Philippe Antoine - * - * Implements HTTP2 JSON logging portion of the engine. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-print.h" -#include "util-unittest.h" - -#include "util-debug.h" -#include "app-layer-parser.h" -#include "output.h" -#include "app-layer-http2.h" -#include "app-layer.h" -#include "util-privs.h" -#include "util-buffer.h" - -#include "util-logopenfile.h" - -#include "output-json.h" -#include "output-json-http2.h" -#include "rust.h" - -#define MODULE_NAME "LogHttp2Log" - -typedef struct OutputHttp2Ctx_ { - OutputJsonCtx *eve_ctx; -} OutputHttp2Ctx; - - -typedef struct JsonHttp2LogThread_ { - OutputHttp2Ctx *http2log_ctx; - OutputJsonThreadCtx *ctx; -} JsonHttp2LogThread; - -static int JsonHttp2Logger(ThreadVars *tv, void *thread_data, const Packet *p, - Flow *f, void *state, void *txptr, uint64_t tx_id) -{ - JsonHttp2LogThread *aft = (JsonHttp2LogThread *)thread_data; - - if (unlikely(state == NULL)) { - return 0; - } - - JsonBuilder *js = CreateEveHeaderWithTxId( - p, LOG_DIR_FLOW, "http", NULL, tx_id, aft->http2log_ctx->eve_ctx); - if (unlikely(js == NULL)) - return 0; - - if (!rs_http2_log_json(txptr, js)) { - goto end; - } - OutputJsonBuilderBuffer(js, aft->ctx); -end: - jb_free(js); - return 0; -} - -static TmEcode JsonHttp2LogThreadInit(ThreadVars *t, const void *initdata, void **data) -{ - JsonHttp2LogThread *aft = SCCalloc(1, sizeof(JsonHttp2LogThread)); - if (unlikely(aft == NULL)) - return TM_ECODE_FAILED; - - if(initdata == NULL) - { - SCLogDebug("Error getting context for EveLogHTTP2. \"initdata\" argument NULL"); - goto error_exit; - } - - /* Use the Output Context (file pointer and mutex) */ - aft->http2log_ctx = ((OutputCtx *)initdata)->data; - aft->ctx = CreateEveThreadCtx(t, aft->http2log_ctx->eve_ctx); - if (!aft->ctx) { - goto error_exit; - } - - *data = (void *)aft; - return TM_ECODE_OK; - -error_exit: - SCFree(aft); - return TM_ECODE_FAILED; -} - -static TmEcode JsonHttp2LogThreadDeinit(ThreadVars *t, void *data) -{ - JsonHttp2LogThread *aft = (JsonHttp2LogThread *)data; - if (aft == NULL) { - return TM_ECODE_OK; - } - - FreeEveThreadCtx(aft->ctx); - /* clear memory */ - memset(aft, 0, sizeof(JsonHttp2LogThread)); - - SCFree(aft); - return TM_ECODE_OK; -} - -static void OutputHttp2LogDeinitSub(OutputCtx *output_ctx) -{ - OutputHttp2Ctx *http2_ctx = output_ctx->data; - SCFree(http2_ctx); - SCFree(output_ctx); -} - -static OutputInitResult OutputHttp2LogInitSub(ConfNode *conf, OutputCtx *parent_ctx) -{ - OutputInitResult result = { NULL, false }; - OutputJsonCtx *ojc = parent_ctx->data; - - OutputHttp2Ctx *http2_ctx = SCMalloc(sizeof(OutputHttp2Ctx)); - if (unlikely(http2_ctx == NULL)) - return result; - - OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); - if (unlikely(output_ctx == NULL)) { - SCFree(http2_ctx); - return result; - } - - http2_ctx->eve_ctx = ojc; - - output_ctx->data = http2_ctx; - output_ctx->DeInit = OutputHttp2LogDeinitSub; - - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP2); - - result.ctx = output_ctx; - result.ok = true; - return result; -} - -void JsonHttp2LogRegister (void) -{ - /* also register as child of eve-log */ - OutputRegisterTxSubModuleWithProgress(LOGGER_JSON_TX, "eve-log", MODULE_NAME, "eve-log.http2", - OutputHttp2LogInitSub, ALPROTO_HTTP2, JsonHttp2Logger, HTTP2StateClosed, - HTTP2StateClosed, JsonHttp2LogThreadInit, JsonHttp2LogThreadDeinit, NULL); -} diff --git a/src/output-json-http2.h b/src/output-json-http2.h deleted file mode 100644 index 88ba420ab2df..000000000000 --- a/src/output-json-http2.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2020 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Philippe Antoine - */ - -#ifndef __OUTPUT_JSON_HTTP2_H__ -#define __OUTPUT_JSON_HTTP2_H__ - -void JsonHttp2LogRegister(void); - -#endif /* __OUTPUT_JSON_HTTP2_H__ */ diff --git a/src/output-json-krb5.c b/src/output-json-krb5.c deleted file mode 100644 index 9fc45c5d3c53..000000000000 --- a/src/output-json-krb5.c +++ /dev/null @@ -1,92 +0,0 @@ -/* Copyright (C) 2018-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Pierre Chifflier - * - * Implement JSON/eve logging app-layer KRB5. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "app-layer-krb5.h" -#include "output-json-krb5.h" - -#include "rust.h" - -static int JsonKRB5Logger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - KRB5Transaction *krb5tx = tx; - OutputJsonThreadCtx *thread = thread_data; - - JsonBuilder *jb = CreateEveHeader(p, LOG_DIR_PACKET, "krb5", NULL, thread->ctx); - if (unlikely(jb == NULL)) { - return TM_ECODE_FAILED; - } - - if (!rs_krb5_log_json_response(krb5tx, jb)) { - goto error; - } - - OutputJsonBuilderBuffer(jb, thread); - - jb_free(jb); - return TM_ECODE_OK; - -error: - jb_free(jb); - return TM_ECODE_FAILED; -} - -static OutputInitResult OutputKRB5LogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_KRB5); - AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_KRB5); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonKRB5LogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonKRB5Log", "eve-log.krb5", - OutputKRB5LogInitSub, ALPROTO_KRB5, JsonKRB5Logger, JsonLogThreadInit, - JsonLogThreadDeinit, NULL); - - SCLogDebug("KRB5 JSON logger registered."); -} diff --git a/src/output-json-krb5.h b/src/output-json-krb5.h deleted file mode 100644 index 87f5b79136cb..000000000000 --- a/src/output-json-krb5.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2015 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Pierre Chifflier - */ - -#ifndef __OUTPUT_JSON_KRB5_H__ -#define __OUTPUT_JSON_KRB5_H__ - -void JsonKRB5LogRegister(void); - -#endif /* __OUTPUT_JSON_KRB5_H__ */ diff --git a/src/output-json-modbus.c b/src/output-json-modbus.c deleted file mode 100644 index 9e508ead9acc..000000000000 --- a/src/output-json-modbus.c +++ /dev/null @@ -1,147 +0,0 @@ -/* Copyright (C) 2019-2020 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" -#include "output.h" -#include "output-json.h" -#include "app-layer.h" -#include "app-layer-parser.h" -#include "output-json-modbus.h" -#include "rust.h" - -typedef struct LogModbusFileCtx_ { - LogFileCtx *file_ctx; - OutputJsonCtx *eve_ctx; -} LogModbusFileCtx; - -typedef struct JsonModbusLogThread_ { - LogModbusFileCtx *modbuslog_ctx; - OutputJsonThreadCtx *ctx; -} JsonModbusLogThread; - -static int JsonModbusLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, - void *state, void *tx, uint64_t tx_id) -{ - JsonModbusLogThread *thread = thread_data; - - JsonBuilder *js = - CreateEveHeader(p, LOG_DIR_FLOW, "modbus", NULL, thread->modbuslog_ctx->eve_ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_OK; - } - if (!rs_modbus_to_json(tx, js)) { - jb_free(js); - return TM_ECODE_FAILED; - } - OutputJsonBuilderBuffer(js, thread->ctx); - - jb_free(js); - return TM_ECODE_OK; -} - -static void OutputModbusLogDeInitCtxSub(OutputCtx *output_ctx) -{ - LogModbusFileCtx *modbuslog_ctx = (LogModbusFileCtx *)output_ctx->data; - SCFree(modbuslog_ctx); - SCFree(output_ctx); -} - -static OutputInitResult OutputModbusLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) -{ - OutputInitResult result = { NULL, false }; - OutputJsonCtx *ajt = parent_ctx->data; - - LogModbusFileCtx *modbuslog_ctx = SCCalloc(1, sizeof(*modbuslog_ctx)); - if (unlikely(modbuslog_ctx == NULL)) { - return result; - } - modbuslog_ctx->file_ctx = ajt->file_ctx; - modbuslog_ctx->eve_ctx = ajt; - - OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx)); - if (unlikely(output_ctx == NULL)) { - SCFree(modbuslog_ctx); - return result; - } - output_ctx->data = modbuslog_ctx; - output_ctx->DeInit = OutputModbusLogDeInitCtxSub; - - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_MODBUS); - - SCLogDebug("modbus log sub-module initialized."); - - result.ctx = output_ctx; - result.ok = true; - return result; -} - -static TmEcode JsonModbusLogThreadInit(ThreadVars *t, const void *initdata, void **data) -{ - if (initdata == NULL) { - SCLogDebug("Error getting context for EveLogModbus. \"initdata\" is NULL."); - return TM_ECODE_FAILED; - } - - JsonModbusLogThread *thread = SCCalloc(1, sizeof(*thread)); - if (unlikely(thread == NULL)) { - return TM_ECODE_FAILED; - } - - thread->modbuslog_ctx = ((OutputCtx *)initdata)->data; - thread->ctx = CreateEveThreadCtx(t, thread->modbuslog_ctx->eve_ctx); - if (thread->ctx == NULL) { - goto error_exit; - } - - *data = (void *)thread; - return TM_ECODE_OK; - -error_exit: - SCFree(thread); - return TM_ECODE_FAILED; -} - -static TmEcode JsonModbusLogThreadDeinit(ThreadVars *t, void *data) -{ - JsonModbusLogThread *thread = (JsonModbusLogThread *)data; - if (thread == NULL) { - return TM_ECODE_OK; - } - FreeEveThreadCtx(thread->ctx); - SCFree(thread); - return TM_ECODE_OK; -} - -void JsonModbusLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonModbusLog", "eve-log.modbus", - OutputModbusLogInitSub, ALPROTO_MODBUS, JsonModbusLogger, JsonModbusLogThreadInit, - JsonModbusLogThreadDeinit, NULL); - - SCLogDebug("modbus json logger registered."); -} diff --git a/src/output-json-modbus.h b/src/output-json-modbus.h deleted file mode 100644 index 2b07e4eb2d5c..000000000000 --- a/src/output-json-modbus.h +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright (C) 2019 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -#ifndef __OUTPUT_JSON_MODBUS_H__ -#define __OUTPUT_JSON_MODBUS_H__ - -void JsonModbusLogRegister(void); - -#endif /* __OUTPUT_JSON_MODBUS_H__ */ diff --git a/src/output-json-quic.c b/src/output-json-quic.c deleted file mode 100644 index 830ac78fdfbb..000000000000 --- a/src/output-json-quic.c +++ /dev/null @@ -1,151 +0,0 @@ -/* Copyright (C) 2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * Implements JSON/eve logging for Quic app-layer. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" -#include "output.h" -#include "output-json.h" -#include "app-layer.h" -#include "app-layer-parser.h" -#include "output-json-quic.h" -#include "rust.h" - -typedef struct LogQuicFileCtx_ { - LogFileCtx *file_ctx; - OutputJsonCtx *eve_ctx; -} LogQuicFileCtx; - -typedef struct JsonQuicLogThread_ { - LogQuicFileCtx *quiclog_ctx; - OutputJsonThreadCtx *ctx; -} JsonQuicLogThread; - -static int JsonQuicLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, - void *tx, uint64_t tx_id) -{ - JsonQuicLogThread *thread = thread_data; - - JsonBuilder *js = - CreateEveHeader(p, LOG_DIR_PACKET, "quic", NULL, thread->quiclog_ctx->eve_ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_OK; - } - if (!rs_quic_to_json(tx, js)) { - jb_free(js); - return TM_ECODE_FAILED; - } - OutputJsonBuilderBuffer(js, thread->ctx); - - jb_free(js); - return TM_ECODE_OK; -} - -static void OutputQuicLogDeInitCtxSub(OutputCtx *output_ctx) -{ - LogQuicFileCtx *quiclog_ctx = (LogQuicFileCtx *)output_ctx->data; - SCFree(quiclog_ctx); - SCFree(output_ctx); -} - -static OutputInitResult OutputQuicLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) -{ - OutputInitResult result = { NULL, false }; - OutputJsonCtx *ajt = parent_ctx->data; - - LogQuicFileCtx *quiclog_ctx = SCCalloc(1, sizeof(*quiclog_ctx)); - if (unlikely(quiclog_ctx == NULL)) { - return result; - } - quiclog_ctx->file_ctx = ajt->file_ctx; - quiclog_ctx->eve_ctx = ajt; - - OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx)); - if (unlikely(output_ctx == NULL)) { - SCFree(quiclog_ctx); - return result; - } - output_ctx->data = quiclog_ctx; - output_ctx->DeInit = OutputQuicLogDeInitCtxSub; - - AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_QUIC); - - result.ctx = output_ctx; - result.ok = true; - return result; -} - -static TmEcode JsonQuicLogThreadInit(ThreadVars *t, const void *initdata, void **data) -{ - if (initdata == NULL) { - SCLogDebug("Error getting context for EveLogQuic. \"initdata\" is NULL."); - return TM_ECODE_FAILED; - } - - JsonQuicLogThread *thread = SCCalloc(1, sizeof(*thread)); - if (unlikely(thread == NULL)) { - return TM_ECODE_FAILED; - } - - thread->quiclog_ctx = ((OutputCtx *)initdata)->data; - thread->ctx = CreateEveThreadCtx(t, thread->quiclog_ctx->eve_ctx); - if (thread->ctx == NULL) { - goto error_exit; - } - - *data = (void *)thread; - return TM_ECODE_OK; - -error_exit: - SCFree(thread); - return TM_ECODE_FAILED; -} - -static TmEcode JsonQuicLogThreadDeinit(ThreadVars *t, void *data) -{ - JsonQuicLogThread *thread = (JsonQuicLogThread *)data; - if (thread == NULL) { - return TM_ECODE_OK; - } - FreeEveThreadCtx(thread->ctx); - SCFree(thread); - return TM_ECODE_OK; -} - -void JsonQuicLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonQuicLog", "eve-log.quic", - OutputQuicLogInitSub, ALPROTO_QUIC, JsonQuicLogger, JsonQuicLogThreadInit, - JsonQuicLogThreadDeinit, NULL); - - SCLogDebug("quic json logger registered."); -} diff --git a/src/output-json-quic.h b/src/output-json-quic.h deleted file mode 100644 index 48e38185f2bd..000000000000 --- a/src/output-json-quic.h +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright (C) 2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - */ - -#ifndef __OUTPUT_JSON_QUIC_H__ -#define __OUTPUT_JSON_QUIC_H__ - -void JsonQuicLogRegister(void); - -#endif /* __OUTPUT_JSON_QUIC_H__ */ diff --git a/src/output-json-rdp.c b/src/output-json-rdp.c deleted file mode 100644 index bc5d9ae9df89..000000000000 --- a/src/output-json-rdp.c +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright (C) 2019-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Zach Kelly - * - * Application layer logger for RDP - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" -#include "output.h" -#include "output-json.h" -#include "app-layer.h" -#include "app-layer-parser.h" -#include "app-layer-rdp.h" -#include "output-json-rdp.h" -#include "rust.h" - -static int JsonRdpLogger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - OutputJsonThreadCtx *thread = thread_data; - - JsonBuilder *js = CreateEveHeader(p, LOG_DIR_PACKET, "rdp", NULL, thread->ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_OK; - } - if (!rs_rdp_to_json(tx, js)) { - jb_free(js); - return TM_ECODE_FAILED; - } - OutputJsonBuilderBuffer(js, thread); - - jb_free(js); - return TM_ECODE_OK; -} - -static OutputInitResult OutputRdpLogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_RDP); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonRdpLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonRdpLog", "eve-log.rdp", - OutputRdpLogInitSub, ALPROTO_RDP, JsonRdpLogger, JsonLogThreadInit, JsonLogThreadDeinit, - NULL); - - SCLogDebug("rdp json logger registered."); -} diff --git a/src/output-json-rdp.h b/src/output-json-rdp.h deleted file mode 100644 index 5dc9237691ea..000000000000 --- a/src/output-json-rdp.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2019 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Zach Kelly - */ - -#ifndef __OUTPUT_JSON_RDP_H__ -#define __OUTPUT_JSON_RDP_H__ - -void JsonRdpLogRegister(void); - -#endif /* __OUTPUT_JSON_RDP_H__ */ diff --git a/src/output-json-rfb.c b/src/output-json-rfb.c deleted file mode 100644 index e2b832bece13..000000000000 --- a/src/output-json-rfb.c +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright (C) 2020-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Frank Honza - * - * Implement JSON/eve logging app-layer RFB. - */ - -#include "suricata-common.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "app-layer-rfb.h" -#include "output-json-rfb.h" - -#include "rust-bindings.h" - -static int JsonRFBLogger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - OutputJsonThreadCtx *thread = thread_data; - - JsonBuilder *js = CreateEveHeader(p, LOG_DIR_FLOW, "rfb", NULL, thread->ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_FAILED; - } - - if (!rs_rfb_logger_log(tx, js)) { - goto error; - } - - OutputJsonBuilderBuffer(js, thread); - jb_free(js); - - return TM_ECODE_OK; - -error: - jb_free(js); - return TM_ECODE_FAILED; -} - -static OutputInitResult OutputRFBLogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_RFB); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonRFBLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonRFBLog", "eve-log.rfb", - OutputRFBLogInitSub, ALPROTO_RFB, JsonRFBLogger, JsonLogThreadInit, JsonLogThreadDeinit, - NULL); -} diff --git a/src/output-json-rfb.h b/src/output-json-rfb.h deleted file mode 100644 index 7e4e48ebd4c8..000000000000 --- a/src/output-json-rfb.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2020 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Frank Honza - */ - -#ifndef __OUTPUT_JSON_RFB_H__ -#define __OUTPUT_JSON_RFB_H__ - -void JsonRFBLogRegister(void); - -#endif /* __OUTPUT_JSON_RFB_H__ */ diff --git a/src/output-json-sip.c b/src/output-json-sip.c deleted file mode 100644 index 7dd442cf6aba..000000000000 --- a/src/output-json-sip.c +++ /dev/null @@ -1,91 +0,0 @@ -/* Copyright (C) 2018-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Giuseppe Longo - * - * Implement JSON/eve logging app-layer SIP. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "app-layer-sip.h" -#include "output-json-sip.h" - -#include "rust.h" - -static int JsonSIPLogger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - SIPTransaction *siptx = tx; - OutputJsonThreadCtx *thread = thread_data; - - JsonBuilder *js = CreateEveHeader((Packet *)p, LOG_DIR_PACKET, "sip", NULL, thread->ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_OK; - } - - if (!rs_sip_log_json(siptx, js)) { - goto error; - } - - OutputJsonBuilderBuffer(js, thread); - jb_free(js); - - return TM_ECODE_OK; - -error: - jb_free(js); - return TM_ECODE_FAILED; -} - -static OutputInitResult OutputSIPLogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_SIP); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonSIPLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonSIPLog", "eve-log.sip", - OutputSIPLogInitSub, ALPROTO_SIP, JsonSIPLogger, JsonLogThreadInit, JsonLogThreadDeinit, - NULL); - - SCLogDebug("SIP JSON logger registered."); -} diff --git a/src/output-json-sip.h b/src/output-json-sip.h deleted file mode 100644 index 0d2c53fa50df..000000000000 --- a/src/output-json-sip.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2015 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Giuseppe Longo - */ - -#ifndef __OUTPUT_JSON_SIP_H__ -#define __OUTPUT_JSON_SIP_H__ - -void JsonSIPLogRegister(void); - -#endif /* __OUTPUT_JSON_SIP_H__ */ diff --git a/src/output-json-snmp.c b/src/output-json-snmp.c deleted file mode 100644 index cbf0a7c992e4..000000000000 --- a/src/output-json-snmp.c +++ /dev/null @@ -1,91 +0,0 @@ -/* Copyright (C) 2018-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Pierre Chifflier - * - * Implement JSON/eve logging app-layer SNMP. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "app-layer-snmp.h" -#include "output-json-snmp.h" - -#include "rust.h" - -static int JsonSNMPLogger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - SNMPTransaction *snmptx = tx; - OutputJsonThreadCtx *thread = thread_data; - - JsonBuilder *jb = CreateEveHeader(p, LOG_DIR_PACKET, "snmp", NULL, thread->ctx); - if (unlikely(jb == NULL)) { - return TM_ECODE_FAILED; - } - - if (!rs_snmp_log_json_response(snmptx, jb)) { - goto error; - } - - OutputJsonBuilderBuffer(jb, thread); - - jb_free(jb); - return TM_ECODE_OK; - -error: - jb_free(jb); - return TM_ECODE_FAILED; -} - -static OutputInitResult OutputSNMPLogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_SNMP); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonSNMPLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonSNMPLog", "eve-log.snmp", - OutputSNMPLogInitSub, ALPROTO_SNMP, JsonSNMPLogger, JsonLogThreadInit, - JsonLogThreadDeinit, NULL); - - SCLogDebug("SNMP JSON logger registered."); -} diff --git a/src/output-json-snmp.h b/src/output-json-snmp.h deleted file mode 100644 index 4c88db32317d..000000000000 --- a/src/output-json-snmp.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2015-2019 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Pierre Chifflier - */ - -#ifndef __OUTPUT_JSON_SNMP_H__ -#define __OUTPUT_JSON_SNMP_H__ - -void JsonSNMPLogRegister(void); - -#endif /* __OUTPUT_JSON_SNMP_H__ */ diff --git a/src/output-json-ssh.c b/src/output-json-ssh.c deleted file mode 100644 index 45a8d8eab333..000000000000 --- a/src/output-json-ssh.c +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright (C) 2014-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Victor Julien - * - * Implements SSH JSON logging portion of the engine. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-print.h" -#include "util-unittest.h" - -#include "util-debug.h" -#include "app-layer-parser.h" -#include "output.h" -#include "app-layer-ssh.h" -#include "app-layer.h" -#include "util-privs.h" -#include "util-buffer.h" - -#include "util-logopenfile.h" - -#include "output-json.h" -#include "output-json-ssh.h" -#include "rust.h" - -#define MODULE_NAME "LogSshLog" - -static int JsonSshLogger(ThreadVars *tv, void *thread_data, const Packet *p, - Flow *f, void *state, void *txptr, uint64_t tx_id) -{ - OutputJsonThreadCtx *thread = thread_data; - - if (unlikely(state == NULL)) { - return 0; - } - - JsonBuilder *js = CreateEveHeaderWithTxId(p, LOG_DIR_FLOW, "ssh", NULL, tx_id, thread->ctx); - if (unlikely(js == NULL)) - return 0; - - if (!rs_ssh_log_json(txptr, js)) { - goto end; - } - OutputJsonBuilderBuffer(js, thread); - -end: - jb_free(js); - return 0; -} - -static OutputInitResult OutputSshLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_SSH); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonSshLogRegister (void) -{ - /* register as child of eve-log */ - OutputRegisterTxSubModuleWithCondition(LOGGER_JSON_TX, "eve-log", "JsonSshLog", "eve-log.ssh", - OutputSshLogInitSub, ALPROTO_SSH, JsonSshLogger, SSHTxLogCondition, JsonLogThreadInit, - JsonLogThreadDeinit, NULL); -} diff --git a/src/output-json-ssh.h b/src/output-json-ssh.h deleted file mode 100644 index d0f9d3fc7dde..000000000000 --- a/src/output-json-ssh.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2014 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Victor Julien - */ - -#ifndef __OUTPUT_JSON_SSH_H__ -#define __OUTPUT_JSON_SSH_H__ - -void JsonSshLogRegister(void); - -#endif /* __OUTPUT_JSON_SSH_H__ */ diff --git a/src/output-json-template.c b/src/output-json-template.c deleted file mode 100644 index 2ca48b7ae373..000000000000 --- a/src/output-json-template.c +++ /dev/null @@ -1,176 +0,0 @@ -/* Copyright (C) 2018-2022 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/* - * TODO: Update \author in this file and in output-json-template.h. - * TODO: Remove SCLogNotice statements, or convert to debug. - * TODO: Implement your app-layers logging. - */ - -/** - * \file - * - * \author FirstName LastName - * - * Implement JSON/eve logging app-layer Template. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "output-json-template.h" -#include "rust.h" - -typedef struct LogTemplateFileCtx_ { - uint32_t flags; - OutputJsonCtx *eve_ctx; -} LogTemplateFileCtx; - -typedef struct LogTemplateLogThread_ { - LogTemplateFileCtx *templatelog_ctx; - OutputJsonThreadCtx *ctx; -} LogTemplateLogThread; - -static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, - void *state, void *tx, uint64_t tx_id) -{ - SCLogNotice("JsonTemplateLogger"); - LogTemplateLogThread *thread = thread_data; - - JsonBuilder *js = - CreateEveHeader(p, LOG_DIR_PACKET, "template", NULL, thread->templatelog_ctx->eve_ctx); - if (unlikely(js == NULL)) { - return TM_ECODE_FAILED; - } - - if (!rs_template_logger_log(tx, js)) { - goto error; - } - - OutputJsonBuilderBuffer(js, thread->ctx); - jb_free(js); - - return TM_ECODE_OK; - -error: - jb_free(js); - return TM_ECODE_FAILED; -} - -static void OutputTemplateLogDeInitCtxSub(OutputCtx *output_ctx) -{ - LogTemplateFileCtx *templatelog_ctx = (LogTemplateFileCtx *)output_ctx->data; - SCFree(templatelog_ctx); - SCFree(output_ctx); -} - -static OutputInitResult OutputTemplateLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) -{ - OutputInitResult result = { NULL, false }; - OutputJsonCtx *ajt = parent_ctx->data; - - LogTemplateFileCtx *templatelog_ctx = SCCalloc(1, sizeof(*templatelog_ctx)); - if (unlikely(templatelog_ctx == NULL)) { - return result; - } - templatelog_ctx->eve_ctx = ajt; - - OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx)); - if (unlikely(output_ctx == NULL)) { - SCFree(templatelog_ctx); - return result; - } - output_ctx->data = templatelog_ctx; - output_ctx->DeInit = OutputTemplateLogDeInitCtxSub; - - SCLogNotice("Template log sub-module initialized."); - - AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_TEMPLATE); - - result.ctx = output_ctx; - result.ok = true; - return result; -} - -static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, const void *initdata, void **data) -{ - LogTemplateLogThread *thread = SCCalloc(1, sizeof(*thread)); - if (unlikely(thread == NULL)) { - return TM_ECODE_FAILED; - } - - if (initdata == NULL) { - SCLogDebug("Error getting context for EveLogTemplate. \"initdata\" is NULL."); - goto error_exit; - } - - thread->templatelog_ctx = ((OutputCtx *)initdata)->data; - thread->ctx = CreateEveThreadCtx(t, thread->templatelog_ctx->eve_ctx); - if (!thread->ctx) { - goto error_exit; - } - *data = (void *)thread; - - return TM_ECODE_OK; - -error_exit: - SCFree(thread); - return TM_ECODE_FAILED; -} - -static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data) -{ - LogTemplateLogThread *thread = (LogTemplateLogThread *)data; - if (thread == NULL) { - return TM_ECODE_OK; - } - FreeEveThreadCtx(thread->ctx); - SCFree(thread); - return TM_ECODE_OK; -} - -void JsonTemplateLogRegister(void) -{ - /* TEMPLATE_START_REMOVE */ - if (ConfGetNode("app-layer.protocols.template") == NULL) { - return; - } - /* TEMPLATE_END_REMOVE */ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonTemplateLog", "eve-log.template", - OutputTemplateLogInitSub, ALPROTO_TEMPLATE, JsonTemplateLogger, - JsonTemplateLogThreadInit, JsonTemplateLogThreadDeinit, NULL); - - SCLogNotice("Template JSON logger registered."); -} diff --git a/src/output-json-template.h b/src/output-json-template.h deleted file mode 100644 index d27b8d7c0e05..000000000000 --- a/src/output-json-template.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2018 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author FirstName LastName - */ - -#ifndef __OUTPUT_JSON_TEMPLATE_RUST_H__ -#define __OUTPUT_JSON_TEMPLATE_RUST_H__ - -void JsonTemplateLogRegister(void); - -#endif /* __OUTPUT_JSON_TEMPLATE_RUST_H__ */ diff --git a/src/output-json-tftp.c b/src/output-json-tftp.c deleted file mode 100644 index a0bc9ee1809e..000000000000 --- a/src/output-json-tftp.c +++ /dev/null @@ -1,90 +0,0 @@ -/* Copyright (C) 2020-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Clément Galland - * - * Implement JSON/eve logging app-layer TFTP. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-unittest.h" -#include "util-buffer.h" -#include "util-debug.h" -#include "util-byte.h" - -#include "output.h" -#include "output-json.h" - -#include "app-layer.h" -#include "app-layer-parser.h" - -#include "app-layer-tftp.h" -#include "output-json-tftp.h" - -#include "rust.h" - -static int JsonTFTPLogger(ThreadVars *tv, void *thread_data, - const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - OutputJsonThreadCtx *thread = thread_data; - - JsonBuilder *jb = CreateEveHeader(p, LOG_DIR_PACKET, "tftp", NULL, thread->ctx); - if (unlikely(jb == NULL)) { - return TM_ECODE_FAILED; - } - - if (unlikely(!rs_tftp_log_json_request(tx, jb))) { - goto error; - } - - OutputJsonBuilderBuffer(jb, thread); - - jb_free(jb); - return TM_ECODE_OK; - -error: - jb_free(jb); - return TM_ECODE_FAILED; -} - -static OutputInitResult OutputTFTPLogInitSub(ConfNode *conf, - OutputCtx *parent_ctx) -{ - AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_TFTP); - return OutputJsonLogInitSub(conf, parent_ctx); -} - -void JsonTFTPLogRegister(void) -{ - /* Register as an eve sub-module. */ - OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonTFTPLog", "eve-log.tftp", - OutputTFTPLogInitSub, ALPROTO_TFTP, JsonTFTPLogger, JsonLogThreadInit, - JsonLogThreadDeinit, NULL); - - SCLogDebug("TFTP JSON logger registered."); -} diff --git a/src/output-json-tftp.h b/src/output-json-tftp.h deleted file mode 100644 index 3db4ba06cd55..000000000000 --- a/src/output-json-tftp.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2017 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Clément Galland - */ - -#ifndef __OUTPUT_JSON_TFTP_H__ -#define __OUTPUT_JSON_TFTP_H__ - -void JsonTFTPLogRegister(void); - -#endif /* __OUTPUT_JSON_TFTP_H__ */ diff --git a/src/output.c b/src/output.c index 087f2b9a18e3..d789586e9f1c 100644 --- a/src/output.c +++ b/src/output.c @@ -54,12 +54,12 @@ #include "log-httplog.h" #include "output-json-http.h" #include "output-json-dns.h" -#include "output-json-modbus.h" #include "log-tlslog.h" #include "log-tlsstore.h" #include "output-json-tls.h" -#include "output-json-ssh.h" #include "log-pcap.h" +// for SSHTxLogCondition +#include "app-layer-ssh.h" #include "output-json-file.h" #include "output-json-smtp.h" #include "output-json-stats.h" @@ -69,26 +69,17 @@ #include "output-json-ftp.h" // for misplaced EveFTPDataAddMetadata #include "app-layer-ftp.h" -#include "output-json-tftp.h" #include "output-json-smb.h" #include "output-json-ike.h" -#include "output-json-krb5.h" -#include "output-json-quic.h" #include "output-json-dhcp.h" -#include "output-json-snmp.h" -#include "output-json-sip.h" -#include "output-json-rfb.h" #include "output-json-mqtt.h" #include "output-json-pgsql.h" -#include "output-json-template.h" -#include "output-json-rdp.h" -#include "output-json-http2.h" #include "output-lua.h" #include "output-json-dnp3.h" #include "output-json-metadata.h" #include "output-json-dcerpc.h" #include "output-json-frame.h" -#include "output-json-bittorrent-dht.h" +#include "app-layer-parser.h" #include "output-filestore.h" typedef struct RootLogger_ { @@ -1034,6 +1025,136 @@ void OutputRegisterRootLoggers(void) OutputStreamingLoggerRegister(); } +static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, + void *state, void *tx, uint64_t tx_id) +{ + OutputJsonThreadCtx *thread = thread_data; + SimpleJsonAppLayerLogger *al = GetAppProtoSimpleJsonLogger(f->alproto); + if (al == NULL) { + return TM_ECODE_FAILED; + } + + const char *name; + switch (al->proto) { + case ALPROTO_HTTP2: + // special case + name = "http"; + break; + case ALPROTO_FTPDATA: + // underscore instead of dash + name = "ftp_data"; + break; + case ALPROTO_BITTORRENT_DHT: + // underscore instead of dash + name = "bittorrent_dht"; + break; + default: + name = AppProtoToString(al->proto); + } + JsonBuilder *js = CreateEveHeader(p, LOG_DIR_PACKET, name, NULL, thread->ctx); + if (unlikely(js == NULL)) { + return TM_ECODE_FAILED; + } + + if (!al->LogTx(tx, js)) { + goto error; + } + + OutputJsonBuilderBuffer(js, thread); + jb_free(js); + + return TM_ECODE_OK; + +error: + jb_free(js); + return TM_ECODE_FAILED; +} + +static OutputInitResult OutputBitTorrentDHTLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_BITTORRENT_DHT); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputRdpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_RDP); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputRFBLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_RFB); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputTemplateLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_TEMPLATE); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputSIPLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_SIP); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputSNMPLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_SNMP); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputQuicLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_QUIC); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputKRB5LogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_KRB5); + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_KRB5); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputTFTPLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_TFTP); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputModbusLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_MODBUS); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputHttp2LogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP2); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputSshLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_SSH); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputFTPLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTP); + return OutputJsonLogInitSub(conf, parent_ctx); +} + +static OutputInitResult OutputFTPDataLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTPDATA); + return OutputJsonLogInitSub(conf, parent_ctx); +} + /** * \brief Register all non-root logging modules. */ @@ -1058,13 +1179,17 @@ void OutputRegisterLoggers(void) /* http log */ LogHttpLogRegister(); JsonHttpLogRegister(); - JsonHttp2LogRegister(); + OutputRegisterTxSubModuleWithProgress(LOGGER_JSON_TX, "eve-log", "LogHttp2Log", "eve-log.http2", + OutputHttp2LogInitSub, ALPROTO_HTTP2, JsonGenericLogger, HTTP2StateClosed, + HTTP2StateClosed, JsonLogThreadInit, JsonLogThreadDeinit, NULL); /* tls log */ LogTlsLogRegister(); JsonTlsLogRegister(); LogTlsStoreRegister(); /* ssh */ - JsonSshLogRegister(); + OutputRegisterTxSubModuleWithCondition(LOGGER_JSON_TX, "eve-log", "JsonSshLog", "eve-log.ssh", + OutputSshLogInitSub, ALPROTO_SSH, JsonGenericLogger, SSHTxLogCondition, + JsonLogThreadInit, JsonLogThreadDeinit, NULL); /* pcap log */ PcapLogRegister(); /* file log */ @@ -1073,7 +1198,11 @@ void OutputRegisterLoggers(void) /* dns */ JsonDnsLogRegister(); /* modbus */ - JsonModbusLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonModbusLog", "eve-log.modbus", + OutputModbusLogInitSub, ALPROTO_MODBUS, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + + SCLogDebug("modbus json logger registered."); /* tcp streaming data */ LogTcpDataLogRegister(); /* log stats */ @@ -1094,39 +1223,78 @@ void OutputRegisterLoggers(void) /* NFS JSON logger. */ JsonNFSLogRegister(); /* TFTP JSON logger. */ - JsonTFTPLogRegister(); - /* FTP JSON logger. */ - JsonFTPLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonTFTPLog", "eve-log.tftp", + OutputTFTPLogInitSub, ALPROTO_TFTP, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + + SCLogDebug("TFTP JSON logger registered."); + /* FTP and FTP-DATA JSON loggers. */ + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonFTPLog", "eve-log.ftp", + OutputFTPLogInitSub, ALPROTO_FTP, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonFTPLog", "eve-log.ftp", + OutputFTPDataLogInitSub, ALPROTO_FTPDATA, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + SCLogDebug("FTP JSON logger registered."); + /* SMB JSON logger. */ JsonSMBLogRegister(); /* IKE JSON logger. */ JsonIKELogRegister(); /* KRB5 JSON logger. */ - JsonKRB5LogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonKRB5Log", "eve-log.krb5", + OutputKRB5LogInitSub, ALPROTO_KRB5, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + + SCLogDebug("KRB5 JSON logger registered."); /* QUIC JSON logger. */ - JsonQuicLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonQuicLog", "eve-log.quic", + OutputQuicLogInitSub, ALPROTO_QUIC, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + + SCLogDebug("quic json logger registered."); /* DHCP JSON logger. */ JsonDHCPLogRegister(); /* SNMP JSON logger. */ - JsonSNMPLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonSNMPLog", "eve-log.snmp", + OutputSNMPLogInitSub, ALPROTO_SNMP, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + + SCLogDebug("SNMP JSON logger registered."); /* SIP JSON logger. */ - JsonSIPLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonSIPLog", "eve-log.sip", + OutputSIPLogInitSub, ALPROTO_SIP, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + + SCLogDebug("SIP JSON logger registered."); /* RFB JSON logger. */ - JsonRFBLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonRFBLog", "eve-log.rfb", + OutputRFBLogInitSub, ALPROTO_RFB, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); /* MQTT JSON logger. */ JsonMQTTLogRegister(); /* Pgsql JSON logger. */ JsonPgsqlLogRegister(); /* Template JSON logger. */ - JsonTemplateLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonTemplateLog", "eve-log.template", + OutputTemplateLogInitSub, ALPROTO_TEMPLATE, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); /* RDP JSON logger. */ - JsonRdpLogRegister(); + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonRdpLog", "eve-log.rdp", + OutputRdpLogInitSub, ALPROTO_RDP, JsonGenericLogger, JsonLogThreadInit, + JsonLogThreadDeinit, NULL); + SCLogDebug("rdp json logger registered."); /* DCERPC JSON logger. */ JsonDCERPCLogRegister(); /* app layer frames */ JsonFrameLogRegister(); /* BitTorrent DHT JSON logger */ - JsonBitTorrentDHTLogRegister(); + if (ConfGetNode("app-layer.protocols.bittorrent-dht") != NULL) { + /* Register as an eve sub-module. */ + OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonBitTorrentDHTLog", + "eve-log.bittorrent-dht", OutputBitTorrentDHTLogInitSub, ALPROTO_BITTORRENT_DHT, + JsonGenericLogger, JsonLogThreadInit, JsonLogThreadDeinit, NULL); + } } static SimpleJsonAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { From 56362699032cce0539d43d76bc2d50b9946e31f3 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 13:40:10 +0100 Subject: [PATCH 08/12] app-layer: do not require probing parser as fixed patterns can be enough --- src/app-layer-register.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app-layer-register.c b/src/app-layer-register.c index c4441d9f7c5b..c51073543b91 100644 --- a/src/app-layer-register.c +++ b/src/app-layer-register.c @@ -58,7 +58,6 @@ AppProto AppLayerRegisterProtocolDetection(const struct AppLayerParser *p, int e AppLayerProtoDetectRegisterProtocol(alproto, p->name); if (p->ProbeTS == NULL && p->ProbeTC == NULL) { - BUG_ON(p->default_port != NULL); return alproto; } From 3ddee5e0ecedc8044c555d636564297a420ad1cf Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 13:42:30 +0100 Subject: [PATCH 09/12] detect: make number of keywords dynamic --- src/detect-engine-register.c | 7 +++++-- src/detect-engine-register.h | 4 +++- src/detect-engine.c | 10 ++++++++++ src/detect-parse.c | 7 ++++--- src/detect-parse.h | 2 +- src/detect.h | 6 +++--- 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index df6e4a738ffc..34a839c71784 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -306,6 +306,9 @@ #include "util-mpm-ac.h" #include "runmodes.h" +int DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC; +int DETECT_TBLSIZE_IDX = DETECT_TBLSIZE_STATIC; + static void PrintFeatureList(const SigTableElmt *e, char sep) { const uint16_t flags = e->flags; @@ -374,7 +377,7 @@ static void SigMultilinePrint(int i, const char *prefix) int SigTableList(const char *keyword) { - size_t size = sizeof(sigmatch_table) / sizeof(SigTableElmt); + size_t size = DETECT_TBLSIZE; size_t i; if (keyword == NULL) { @@ -457,7 +460,7 @@ static void DetectFileHandlerRegister(void) void SigTableSetup(void) { - memset(sigmatch_table, 0, sizeof(sigmatch_table)); + memset(sigmatch_table, 0, DETECT_TBLSIZE * sizeof(SigTableElmt)); DetectSidRegister(); DetectPriorityRegister(); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 7d6c457ef9b0..e29b8220191f 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -333,9 +333,11 @@ enum DetectKeywordId { DETECT_AL_IKE_KEY_EXCHANGE, /* make sure this stays last */ - DETECT_TBLSIZE, + DETECT_TBLSIZE_STATIC, }; +extern int DETECT_TBLSIZE; +extern int DETECT_TBLSIZE_IDX; int SigTableList(const char *keyword); void SigTableSetup(void); void SigTableRegisterTests(void); diff --git a/src/detect-engine.c b/src/detect-engine.c index d8f9f1880e56..67d7dcb25da4 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2503,6 +2503,14 @@ static DetectEngineCtx *DetectEngineCtxInitReal(enum DetectEngineType type, cons goto error; } + de_ctx->sm_types_prefilter = SCCalloc(DETECT_TBLSIZE, sizeof(bool)); + if (de_ctx->sm_types_prefilter == NULL) { + goto error; + } + de_ctx->sm_types_silent_error = SCCalloc(DETECT_TBLSIZE, sizeof(bool)); + if (de_ctx->sm_types_silent_error == NULL) { + goto error; + } if (DetectEngineCtxLoadConf(de_ctx) == -1) { goto error; } @@ -2636,6 +2644,8 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) SigGroupCleanup(de_ctx); SpmDestroyGlobalThreadCtx(de_ctx->spm_global_thread_ctx); + SCFree(de_ctx->sm_types_prefilter); + SCFree(de_ctx->sm_types_silent_error); MpmFactoryDeRegisterAllMpmCtxProfiles(de_ctx); diff --git a/src/detect-parse.c b/src/detect-parse.c index d9800f0a2f34..e6127fff956e 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -74,7 +74,7 @@ #include "util-validate.h" /* Table with all filehandler registrations */ -DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE]; +DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE_STATIC]; void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg) { @@ -126,7 +126,8 @@ void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg) } /* Table with all SigMatch registrations */ -SigTableElmt sigmatch_table[DETECT_TBLSIZE]; +SigTableElmt sigmatch_table_static[DETECT_TBLSIZE_STATIC]; +SigTableElmt *sigmatch_table = sigmatch_table_static; extern int sc_set_caps; @@ -397,7 +398,7 @@ bool SigMatchSilentErrorEnabled(const DetectEngineCtx *de_ctx, bool SigMatchStrictEnabled(const enum DetectKeywordId id) { - if (id < DETECT_TBLSIZE) { + if ((int)id < DETECT_TBLSIZE) { return ((sigmatch_table[id].flags & SIGMATCH_STRICT_PARSING) != 0); } return false; diff --git a/src/detect-parse.h b/src/detect-parse.h index a7f2c4d17df7..b83747838065 100644 --- a/src/detect-parse.h +++ b/src/detect-parse.h @@ -42,7 +42,7 @@ typedef struct DetectFileHandlerTableElmt_ { void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *entry); /* File registration table */ -extern DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE]; +extern DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE_STATIC]; /** Flags to indicate if the Signature parsing must be done * switching the source and dest (for ip addresses and ports) diff --git a/src/detect.h b/src/detect.h index 04dd49a65a75..edb84bbcc388 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1007,8 +1007,8 @@ typedef struct DetectEngineCtx_ { /** per keyword flag indicating if a prefilter has been * set for it. If true, the setup function will have to * run. */ - bool sm_types_prefilter[DETECT_TBLSIZE]; - bool sm_types_silent_error[DETECT_TBLSIZE]; + bool *sm_types_prefilter; + bool *sm_types_silent_error; /* classification config parsing */ @@ -1545,7 +1545,7 @@ typedef struct DetectEngineMasterCtx_ { } DetectEngineMasterCtx; /* Table with all SigMatch registrations */ -extern SigTableElmt sigmatch_table[DETECT_TBLSIZE]; +extern SigTableElmt *sigmatch_table; /** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */ From e6637ac4372961622511d0ef7f723ecdaf4d5aee Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 13:55:01 +0100 Subject: [PATCH 10/12] detect: helper to have pure rust keywords Only implemented for snmp.version and mqtt.password But should be implemented for more --- rust/src/mqtt/detect.rs | 2 +- src/Makefile.am | 2 + src/detect-engine-helper.c | 116 +++++++++++++++++++++++++++++ src/detect-engine-helper.h | 53 +++++++++++++ src/detect-mqtt-connect-password.c | 35 ++------- src/detect-snmp-version.c | 38 +++------- 6 files changed, 188 insertions(+), 58 deletions(-) create mode 100644 src/detect-engine-helper.c create mode 100644 src/detect-engine-helper.h diff --git a/rust/src/mqtt/detect.rs b/rust/src/mqtt/detect.rs index b47a84f74409..98eb74c932a9 100644 --- a/rust/src/mqtt/detect.rs +++ b/rust/src/mqtt/detect.rs @@ -168,7 +168,7 @@ pub unsafe extern "C" fn rs_mqtt_tx_get_connect_username( #[no_mangle] pub unsafe extern "C" fn rs_mqtt_tx_get_connect_password( - tx: &MQTTTransaction, buffer: *mut *const u8, buffer_len: *mut u32, + tx: &MQTTTransaction, _flow_flags: u8, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { for msg in tx.msg.iter() { if let MQTTOperation::CONNECT(ref cv) = msg.op { diff --git a/src/Makefile.am b/src/Makefile.am index ba11f704ecc8..78a6c7acbf5f 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -137,6 +137,7 @@ noinst_HEADERS = \ detect-engine-file.h \ detect-engine-frame.h \ detect-engine.h \ + detect-engine-helper.h \ detect-engine-iponly.h \ detect-engine-loader.h \ detect-engine-mpm.h \ @@ -736,6 +737,7 @@ libsuricata_c_a_SOURCES = \ detect-engine-event.c \ detect-engine-file.c \ detect-engine-frame.c \ + detect-engine-helper.c \ detect-engine-iponly.c \ detect-engine-loader.c \ detect-engine-mpm.c \ diff --git a/src/detect-engine-helper.c b/src/detect-engine-helper.c new file mode 100644 index 000000000000..938a14834b8b --- /dev/null +++ b/src/detect-engine-helper.c @@ -0,0 +1,116 @@ +/* Copyright (C) 2023 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Philippe Antoine + * + */ + +#include "suricata-common.h" +#include "detect-engine.h" +#include "detect-engine-helper.h" +#include "detect-engine-mpm.h" +#include "detect-engine-prefilter.h" +#include "detect-parse.h" + +int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver) +{ + if (toserver) { + DetectAppLayerInspectEngineRegister2( + name, alproto, SIG_FLAG_TOSERVER, 0, DetectEngineInspectGenericList, NULL); + } + if (toclient) { + DetectAppLayerInspectEngineRegister2( + name, alproto, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectGenericList, NULL); + } + return DetectBufferTypeRegister(name); +} + +InspectionBuffer *DetectHelperGetData(struct DetectEngineThreadCtx_ *det_ctx, + const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv, + const int list_id, + bool (*GetBuf)(void *txv, const uint8_t flow_flags, const uint8_t **buf, uint32_t *buf_len)) +{ + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); + if (buffer->inspect == NULL) { + const uint8_t *b = NULL; + uint32_t b_len = 0; + + if (!GetBuf(txv, flow_flags, &b, &b_len)) + return NULL; + + InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len); + InspectionBufferApplyTransforms(buffer, transforms); + } + return buffer; +} + +int DetectHelperBufferMpmRegister(const char *name, const char *desc, AppProto alproto, + bool toclient, bool toserver, InspectionBufferGetDataPtr GetData) +{ + if (toserver) { + DetectAppLayerInspectEngineRegister2( + name, alproto, SIG_FLAG_TOSERVER, 0, DetectEngineInspectBufferGeneric, GetData); + DetectAppLayerMpmRegister2( + name, SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, GetData, alproto, 0); + } + if (toclient) { + DetectAppLayerInspectEngineRegister2( + name, alproto, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectBufferGeneric, GetData); + DetectAppLayerMpmRegister2( + name, SIG_FLAG_TOCLIENT, 2, PrefilterGenericMpmRegister, GetData, alproto, 0); + } + DetectBufferTypeSetDescriptionByName(name, desc); + return DetectBufferTypeGetByName(name); +} + +int DetectHelperKeywordRegister(const SCPluginSigTableElmt *kw) +{ + if (DETECT_TBLSIZE_IDX < DETECT_TBLSIZE) { + sigmatch_table[DETECT_TBLSIZE_IDX].name = kw->name; + sigmatch_table[DETECT_TBLSIZE_IDX].desc = kw->desc; + sigmatch_table[DETECT_TBLSIZE_IDX].flags = kw->flags; + sigmatch_table[DETECT_TBLSIZE_IDX].AppLayerTxMatch = kw->AppLayerTxMatch; + sigmatch_table[DETECT_TBLSIZE_IDX].Setup = kw->Setup; + sigmatch_table[DETECT_TBLSIZE_IDX].Free = kw->Free; + DETECT_TBLSIZE_IDX++; + return DETECT_TBLSIZE_IDX - 1; + } + return -1; +} + +int DetectHelperKeywordSetup(AppProto alproto, int kw_id, int buf_id, Signature *s, void *ctx) +{ + SigMatch *sm = NULL; + + if (DetectSignatureSetAppProto(s, alproto) != 0) + return -1; + + /* okay so far so good, lets get this into a SigMatch + * and put it in the Signature. */ + sm = SigMatchAlloc(); + if (sm == NULL) + return -1; + + sm->type = (uint16_t)kw_id; + sm->ctx = ctx; + + SigMatchAppendSMToList(s, sm, buf_id); + return 0; +} diff --git a/src/detect-engine-helper.h b/src/detect-engine-helper.h new file mode 100644 index 000000000000..1c37b23b817d --- /dev/null +++ b/src/detect-engine-helper.h @@ -0,0 +1,53 @@ +/* Copyright (C) 2023 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Philippe Antoine + */ + +#ifndef __DETECT_ENGINE_HELPER_H +#define __DETECT_ENGINE_HELPER_H + +#include "app-layer-protos.h" +#include "detect.h" + +// Structure for keyword dynamic registration by plugin +typedef struct SCPluginSigTableElmt { + const char *name; + const char *desc; + uint16_t flags; + int (*Setup)(DetectEngineCtx *, Signature *, const char *); + void (*Free)(DetectEngineCtx *, void *); + int (*AppLayerTxMatch)(DetectEngineThreadCtx *, Flow *, uint8_t flags, void *alstate, void *txv, + const Signature *, const SigMatchCtx *); +} SCPluginSigTableElmt; + +int DetectHelperKeywordRegister(const SCPluginSigTableElmt *kw); +int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver); + +typedef bool (*SimpleGetTxBuffer)(void *, uint8_t, const uint8_t **, uint32_t *); + +int DetectHelperKeywordSetup(AppProto alproto, int kw_id, int buf_id, Signature *s, void *ctx); +InspectionBuffer *DetectHelperGetData(struct DetectEngineThreadCtx_ *det_ctx, + const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv, + const int list_id, SimpleGetTxBuffer GetBuf); +int DetectHelperBufferMpmRegister(const char *name, const char *desc, AppProto alproto, + bool toclient, bool toserver, InspectionBufferGetDataPtr GetData); + +#endif /* __DETECT_ENGINE_HELPER_H */ diff --git a/src/detect-mqtt-connect-password.c b/src/detect-mqtt-connect-password.c index c08390748fe0..6749629a017e 100644 --- a/src/detect-mqtt-connect-password.c +++ b/src/detect-mqtt-connect-password.c @@ -26,6 +26,7 @@ #include "detect.h" #include "detect-parse.h" #include "detect-engine.h" +#include "detect-engine-helper.h" #include "detect-engine-mpm.h" #include "detect-engine-prefilter.h" #include "detect-mqtt-connect-password.h" @@ -49,24 +50,11 @@ static int DetectMQTTConnectPasswordSetup(DetectEngineCtx *de_ctx, Signature *s, } static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, - const DetectEngineTransforms *transforms, - Flow *_f, const uint8_t _flow_flags, - void *txv, const int list_id) + const DetectEngineTransforms *transforms, Flow *_f, const uint8_t flow_flags, void *txv, + const int list_id) { - InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); - if (buffer->inspect == NULL) { - const uint8_t *b = NULL; - uint32_t b_len = 0; - - if (rs_mqtt_tx_get_connect_password(txv, &b, &b_len) != 1) - return NULL; - if (b == NULL || b_len == 0) - return NULL; - - InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len); - InspectionBufferApplyTransforms(buffer, transforms); - } - return buffer; + return DetectHelperGetData(det_ctx, transforms, _f, flow_flags, txv, list_id, + (SimpleGetTxBuffer)rs_mqtt_tx_get_connect_password); } void DetectMQTTConnectPasswordRegister(void) @@ -78,17 +66,8 @@ void DetectMQTTConnectPasswordRegister(void) sigmatch_table[DETECT_AL_MQTT_CONNECT_PASSWORD].Setup = DetectMQTTConnectPasswordSetup; sigmatch_table[DETECT_AL_MQTT_CONNECT_PASSWORD].flags |= SIGMATCH_NOOPT; - DetectAppLayerInspectEngineRegister2(BUFFER_NAME, ALPROTO_MQTT, - SIG_FLAG_TOSERVER, 0, - DetectEngineInspectBufferGeneric, GetData); - - DetectAppLayerMpmRegister2(BUFFER_NAME, SIG_FLAG_TOSERVER, 2, - PrefilterGenericMpmRegister, GetData, ALPROTO_MQTT, - 1); - - DetectBufferTypeSetDescriptionByName(BUFFER_NAME, BUFFER_DESC); - - g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME); + g_buffer_id = DetectHelperBufferMpmRegister( + BUFFER_NAME, BUFFER_DESC, ALPROTO_MQTT, false, true, GetData); SCLogDebug("registering " BUFFER_NAME " rule option"); } diff --git a/src/detect-snmp-version.c b/src/detect-snmp-version.c index 57359c091bd6..8501c2690186 100644 --- a/src/detect-snmp-version.c +++ b/src/detect-snmp-version.c @@ -29,6 +29,7 @@ #include "detect-engine-content-inspection.h" #include "detect-snmp-version.h" #include "detect-engine-uint.h" +#include "detect-engine-helper.h" #include "app-layer-parser.h" #include "rust.h" @@ -60,13 +61,7 @@ void DetectSNMPVersionRegister (void) sigmatch_table[DETECT_AL_SNMP_VERSION].RegisterTests = DetectSNMPVersionRegisterTests; #endif - DetectAppLayerInspectEngineRegister2("snmp.version", ALPROTO_SNMP, SIG_FLAG_TOSERVER, 0, - DetectEngineInspectGenericList, NULL); - - DetectAppLayerInspectEngineRegister2("snmp.version", ALPROTO_SNMP, SIG_FLAG_TOCLIENT, 0, - DetectEngineInspectGenericList, NULL); - - g_snmp_version_buffer_id = DetectBufferTypeGetByName("snmp.version"); + g_snmp_version_buffer_id = DetectHelperBufferRegister("snmp.version", ALPROTO_SNMP, true, true); } /** @@ -131,34 +126,19 @@ static DetectU32Data *DetectSNMPVersionParse(const char *rawstr) static int DetectSNMPVersionSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr) { - DetectU32Data *dd = NULL; - SigMatch *sm = NULL; - - if (DetectSignatureSetAppProto(s, ALPROTO_SNMP) != 0) - return -1; - - dd = DetectSNMPVersionParse(rawstr); + DetectU32Data *dd = DetectSNMPVersionParse(rawstr); if (dd == NULL) { SCLogError("Parsing \'%s\' failed", rawstr); - goto error; + return -1; + } + if (DetectHelperKeywordSetup( + ALPROTO_SNMP, DETECT_AL_SNMP_VERSION, g_snmp_version_buffer_id, s, dd) < 0) { + DetectSNMPVersionFree(de_ctx, dd); + return -1; } - - /* okay so far so good, lets get this into a SigMatch - * and put it in the Signature. */ - sm = SigMatchAlloc(); - if (sm == NULL) - goto error; - - sm->type = DETECT_AL_SNMP_VERSION; - sm->ctx = (void *)dd; SCLogDebug("snmp.version %d", dd->arg1); - SigMatchAppendSMToList(s, sm, g_snmp_version_buffer_id); return 0; - -error: - DetectSNMPVersionFree(de_ctx, dd); - return -1; } /** From acd4ac010d4b85164656d7680856b1bc19aa00a3 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 14:26:36 +0100 Subject: [PATCH 11/12] app-layer: allow at compile-time more app-layer So that we can have dynamically registered protocols. Doing it at compile time, with CFLAGS=-DALPROTO_DYNAMIC_NB=1, allows to keep the rest of the code using ALPROTO_MAX Ticket: 5053 --- rust/src/core.rs | 7 ++++ src/app-layer-detect-proto.c | 4 +-- src/app-layer-protos.c | 19 +++++++++-- src/app-layer-protos.h | 14 +++++--- src/app-layer.c | 63 +++++++++++++++++------------------- src/output.c | 2 +- 6 files changed, 67 insertions(+), 42 deletions(-) diff --git a/rust/src/core.rs b/rust/src/core.rs index abb27ea578fe..5d8de0ce1c77 100644 --- a/rust/src/core.rs +++ b/rust/src/core.rs @@ -253,6 +253,13 @@ pub extern "C" fn rs_init(context: &'static SuricataContext) init_ffi(context); } +#[no_mangle] +pub extern "C" fn rs_update_alproto_failed(alproto: AppProto) { + unsafe { + ALPROTO_FAILED = alproto; + } +} + /// DetectEngineStateFree wrapper. pub fn sc_detect_engine_state_free(state: *mut DetectEngineState) { diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index c7f902edc22f..012a5a6e1ba9 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -489,8 +489,8 @@ static inline AppProto PPGetProto(const AppLayerProtoDetectProbingParserElement if (AppProtoIsValid(alproto)) { SCReturnUInt(alproto); } - if (alproto == ALPROTO_FAILED || - (pe->max_depth != 0 && buflen > pe->max_depth)) { + if (alproto == ALPROTO_FAILED || alproto == ALPROTO_INVALID || + (pe->max_depth != 0 && buflen > pe->max_depth)) { alproto_masks[0] |= pe->alproto_mask; } pe = pe->next; diff --git a/src/app-layer-protos.c b/src/app-layer-protos.c index 368efacd88d7..3dd290f754a6 100644 --- a/src/app-layer-protos.c +++ b/src/app-layer-protos.c @@ -24,13 +24,16 @@ #include "suricata-common.h" #include "app-layer-protos.h" +#include "rust.h" + +AppProto ALPROTO_FAILED = ALPROTO_MAX_STATIC; typedef struct AppProtoStringTuple { AppProto alproto; const char *str; } AppProtoStringTuple; -const AppProtoStringTuple AppProtoStrings[ALPROTO_MAX] = { +AppProtoStringTuple AppProtoStrings[ALPROTO_MAX] = { { ALPROTO_UNKNOWN, "unknown" }, { ALPROTO_HTTP1, "http1" }, { ALPROTO_FTP, "ftp" }, @@ -65,7 +68,7 @@ const AppProtoStringTuple AppProtoStrings[ALPROTO_MAX] = { { ALPROTO_HTTP2, "http2" }, { ALPROTO_BITTORRENT_DHT, "bittorrent-dht" }, { ALPROTO_HTTP, "http" }, - { ALPROTO_FAILED, "failed" }, + { ALPROTO_MAX_STATIC, "failed" }, #ifdef UNITTESTS { ALPROTO_TEST, "test" }, #endif @@ -104,3 +107,15 @@ AppProto StringToAppProto(const char *proto_name) return ALPROTO_UNKNOWN; } + +void RegisterAppProtoString(AppProto alproto, const char *proto_name) +{ + if (alproto == ALPROTO_FAILED && alproto + 1 < ALPROTO_MAX) { + AppProtoStrings[alproto].str = proto_name; + AppProtoStrings[alproto].alproto = alproto; + ALPROTO_FAILED++; + rs_update_alproto_failed(ALPROTO_FAILED); + AppProtoStrings[ALPROTO_FAILED].str = "failed"; + AppProtoStrings[ALPROTO_FAILED].alproto = ALPROTO_FAILED; + } +} diff --git a/src/app-layer-protos.h b/src/app-layer-protos.h index dd372550cbf5..4b4a79269285 100644 --- a/src/app-layer-protos.h +++ b/src/app-layer-protos.h @@ -65,20 +65,24 @@ enum AppProtoEnum { // HTTP for any version (ALPROTO_HTTP1 (version 1) or ALPROTO_HTTP2) ALPROTO_HTTP, - /* used by the probing parser when alproto detection fails - * permanently for that particular stream */ - ALPROTO_FAILED, #ifdef UNITTESTS ALPROTO_TEST, #endif /* UNITESTS */ /* keep last */ - ALPROTO_MAX, + ALPROTO_MAX_STATIC, + ALPROTO_INVALID = 0xffff, }; // NOTE: if ALPROTO's get >= 256, update SignatureNonPrefilterStore /* not using the enum as that is a unsigned int, so 4 bytes */ typedef uint16_t AppProto; +extern AppProto ALPROTO_FAILED; +#ifdef ALPROTO_DYNAMIC_NB +#define ALPROTO_MAX ALPROTO_MAX_STATIC + 1 + ALPROTO_DYNAMIC_NB +#else +#define ALPROTO_MAX ALPROTO_MAX_STATIC + 1 +#endif static inline bool AppProtoIsValid(AppProto a) { return ((a > ALPROTO_UNKNOWN && a < ALPROTO_FAILED)); @@ -115,4 +119,6 @@ const char *AppProtoToString(AppProto alproto); */ AppProto StringToAppProto(const char *proto_name); +void RegisterAppProtoString(AppProto alproto, const char *proto_name); + #endif /* __APP_LAYER_PROTOS_H__ */ diff --git a/src/app-layer.c b/src/app-layer.c index b031afce8ac8..433d232bfcce 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -839,42 +839,39 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow * tctx->alpd_tctx, f, p->payload, p->payload_len, IPPROTO_UDP, flags, &reverse_flow); PACKET_PROFILING_APP_PD_END(tctx); - switch (*alproto) { - case ALPROTO_UNKNOWN: - if (*alproto_otherdir != ALPROTO_UNKNOWN) { - // Use recognized side - f->alproto = *alproto_otherdir; - // do not keep ALPROTO_UNKNOWN for this side so as not to loop - *alproto = *alproto_otherdir; - if (*alproto_otherdir == ALPROTO_FAILED) { - SCLogDebug("ALPROTO_UNKNOWN flow %p", f); - } - } else { - // First side of protocol is unknown - *alproto = ALPROTO_FAILED; + if (*alproto == ALPROTO_UNKNOWN) { + if (*alproto_otherdir != ALPROTO_UNKNOWN) { + // Use recognized side + f->alproto = *alproto_otherdir; + // do not keep ALPROTO_UNKNOWN for this side so as not to loop + *alproto = *alproto_otherdir; + if (*alproto_otherdir == ALPROTO_FAILED) { + SCLogDebug("ALPROTO_UNKNOWN flow %p", f); } - break; - case ALPROTO_FAILED: - if (*alproto_otherdir != ALPROTO_UNKNOWN) { - // Use recognized side - f->alproto = *alproto_otherdir; - if (*alproto_otherdir == ALPROTO_FAILED) { - SCLogDebug("ALPROTO_UNKNOWN flow %p", f); - } + } else { + // First side of protocol is unknown + *alproto = ALPROTO_FAILED; + } + } else if (*alproto == ALPROTO_FAILED) { + if (*alproto_otherdir != ALPROTO_UNKNOWN) { + // Use recognized side + f->alproto = *alproto_otherdir; + if (*alproto_otherdir == ALPROTO_FAILED) { + SCLogDebug("ALPROTO_UNKNOWN flow %p", f); } - // else wait for second side of protocol - break; - default: - if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != ALPROTO_FAILED) { - if (*alproto_otherdir != *alproto) { - AppLayerDecoderEventsSetEventRaw( - &p->app_layer_events, APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS); - // data already sent to parser, we cannot change the protocol to use the one - // of the server - } - } else { - f->alproto = *alproto; + } + // else wait for second side of protocol + } else { + if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != ALPROTO_FAILED) { + if (*alproto_otherdir != *alproto) { + AppLayerDecoderEventsSetEventRaw( + &p->app_layer_events, APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS); + // data already sent to parser, we cannot change the protocol to use the one + // of the server } + } else { + f->alproto = *alproto; + } } if (*alproto_otherdir == ALPROTO_UNKNOWN) { if (f->alproto == ALPROTO_UNKNOWN) { diff --git a/src/output.c b/src/output.c index d789586e9f1c..398604d554ec 100644 --- a/src/output.c +++ b/src/output.c @@ -1332,7 +1332,7 @@ static SimpleJsonAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { { ALPROTO_HTTP2, rs_http2_log_json }, { ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log }, { ALPROTO_HTTP, NULL }, // signature protocol, not for app-layer logging - { ALPROTO_FAILED, NULL }, + { ALPROTO_MAX_STATIC, NULL }, #ifdef UNITTESTS { ALPROTO_TEST, NULL }, #endif /* UNITESTS */ From 0143503d15e38a23407be69cc7b758f2c4fcb338 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 16 Nov 2023 14:35:49 +0100 Subject: [PATCH 12/12] plugins: app-layer plugins Ticket: 5053 --- src/app-layer-parser.c | 14 +++++++++ src/app-layer-protos.c | 2 +- src/detect-engine-file.h | 11 ++++++++ src/detect-engine-register.c | 49 +++++++++++++++++++++++++++++++- src/detect-engine-register.h | 1 + src/detect-parse.c | 55 +++++++++++++++++++++--------------- src/detect.h | 1 + src/output.c | 8 ++++++ src/output.h | 1 + src/suricata-plugin.h | 12 ++++++++ src/suricata.c | 7 +++-- src/util-plugin.c | 29 +++++++++++++++++++ src/util-plugin.h | 2 ++ 13 files changed, 164 insertions(+), 28 deletions(-) diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 7783c076b65b..238ca52e0eab 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -63,6 +63,10 @@ #include "app-layer-rdp.h" #include "app-layer-http2.h" +#ifdef ALPROTO_DYNAMIC_NB +#include "util-plugin.h" +#endif + struct AppLayerParserThreadCtx_ { void *alproto_local_storage[FLOW_PROTO_MAX][ALPROTO_MAX]; }; @@ -1789,6 +1793,16 @@ void AppLayerParserRegisterProtocolParsers(void) "imap"); } +#ifdef ALPROTO_DYNAMIC_NB + for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) { + SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i); + if (app_layer_plugin == NULL) { + break; + } + app_layer_plugin->Register(); + } +#endif + ValidateParsers(); return; } diff --git a/src/app-layer-protos.c b/src/app-layer-protos.c index 3dd290f754a6..e78022216b01 100644 --- a/src/app-layer-protos.c +++ b/src/app-layer-protos.c @@ -101,7 +101,7 @@ AppProto StringToAppProto(const char *proto_name) // We could use a Multi Pattern Matcher for (size_t i = 0; i < ARRAY_SIZE(AppProtoStrings); i++) { - if (strcmp(proto_name, AppProtoStrings[i].str) == 0) + if (AppProtoStrings[i].str != NULL && strcmp(proto_name, AppProtoStrings[i].str) == 0) return AppProtoStrings[i].alproto; } diff --git a/src/detect-engine-file.h b/src/detect-engine-file.h index 3705a8f6e601..3b58ac9deee4 100644 --- a/src/detect-engine-file.h +++ b/src/detect-engine-file.h @@ -28,4 +28,15 @@ uint8_t DetectFileInspectGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx const struct DetectEngineAppInspectionEngine_ *engine, const Signature *s, Flow *f, uint8_t flags, void *_alstate, void *tx, uint64_t tx_id); +// file protocols with common file handling +typedef struct { + AppProto al_proto; + int direction; + int to_client_progress; + int to_server_progress; +} DetectFileHandlerProtocol_t; + +void DetectFileRegisterProto( + AppProto alproto, int direction, int to_client_progress, int to_server_progress); + #endif /* __DETECT_ENGINE_FILE_H__ */ diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 34a839c71784..49323417ceb0 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -305,6 +305,7 @@ #include "util-path.h" #include "util-mpm-ac.h" #include "runmodes.h" +#include "util-plugin.h" int DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC; int DETECT_TBLSIZE_IDX = DETECT_TBLSIZE_STATIC; @@ -452,14 +453,48 @@ int SigTableList(const char *keyword) static void DetectFileHandlerRegister(void) { - for (int i = 0; i < DETECT_TBLSIZE; i++) { + for (int i = 0; i < DETECT_TBLSIZE_STATIC; i++) { if (filehandler_table[i].name) DetectFileRegisterFileProtocols(&filehandler_table[i]); } } +void SigTableCleanup(void) +{ + if (sigmatch_table != sigmatch_table_static) { + SCFree(sigmatch_table); + sigmatch_table = sigmatch_table_static; + DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC; + } +} + void SigTableSetup(void) { +#ifdef ALPROTO_DYNAMIC_NB + bool to_grow = false; + for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) { + SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i); + if (app_layer_plugin == NULL) { + break; + } + if (app_layer_plugin->keywords_nb > 0) { + DETECT_TBLSIZE += app_layer_plugin->keywords_nb; + to_grow = true; + } + } + if (to_grow) { + if (sigmatch_table == sigmatch_table_static) { + sigmatch_table = SCMalloc(DETECT_TBLSIZE * sizeof(SigTableElmt)); + } else { + sigmatch_table = SCRealloc(sigmatch_table, DETECT_TBLSIZE * sizeof(SigTableElmt)); + } + if (sigmatch_table == NULL) { + SCLogError("Failed to allocate bigger sigmatch_table, falling back to static one"); + sigmatch_table = sigmatch_table_static; + DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC; + } + } +#endif memset(sigmatch_table, 0, DETECT_TBLSIZE * sizeof(SigTableElmt)); DetectSidRegister(); @@ -687,6 +722,18 @@ void SigTableSetup(void) DetectQuicCyuHashRegister(); DetectQuicCyuStringRegister(); +#ifdef ALPROTO_DYNAMIC_NB + for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) { + SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i); + if (app_layer_plugin == NULL) { + break; + } + if (app_layer_plugin->KeywordsRegister != NULL) { + app_layer_plugin->KeywordsRegister(); + } + } +#endif + DetectBypassRegister(); DetectConfigRegister(); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index e29b8220191f..eeb2dcb181c7 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -339,6 +339,7 @@ enum DetectKeywordId { extern int DETECT_TBLSIZE; extern int DETECT_TBLSIZE_IDX; int SigTableList(const char *keyword); +void SigTableCleanup(void); void SigTableSetup(void); void SigTableRegisterTests(void); diff --git a/src/detect-parse.c b/src/detect-parse.c index e6127fff956e..36e77a0f84c2 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -68,6 +68,7 @@ #include "string.h" #include "detect-parse.h" #include "detect-engine-iponly.h" +#include "detect-engine-file.h" #include "app-layer-detect-proto.h" #include "action-globals.h" @@ -76,31 +77,39 @@ /* Table with all filehandler registrations */ DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE_STATIC]; -void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg) +// file protocols with common file handling +DetectFileHandlerProtocol_t al_protocols[ALPROTO_MAX] = { + { .al_proto = ALPROTO_NFS, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, + { .al_proto = ALPROTO_SMB, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, + { .al_proto = ALPROTO_FTP, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, + { .al_proto = ALPROTO_FTPDATA, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, + { .al_proto = ALPROTO_HTTP1, + .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, + .to_client_progress = HTP_RESPONSE_BODY, + .to_server_progress = HTP_REQUEST_BODY }, + { .al_proto = ALPROTO_HTTP2, + .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, + .to_client_progress = HTTP2StateDataServer, + .to_server_progress = HTTP2StateDataClient }, + { .al_proto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER } +}; + +void DetectFileRegisterProto( + AppProto alproto, int direction, int to_client_progress, int to_server_progress) { - // file protocols with common file handling - typedef struct { - AppProto al_proto; - int direction; - int to_client_progress; - int to_server_progress; - } DetectFileHandlerProtocol_t; - static DetectFileHandlerProtocol_t al_protocols[] = { - { .al_proto = ALPROTO_NFS, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, - { .al_proto = ALPROTO_SMB, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, - { .al_proto = ALPROTO_FTP, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, - { .al_proto = ALPROTO_FTPDATA, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, - { .al_proto = ALPROTO_HTTP1, - .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, - .to_client_progress = HTP_RESPONSE_BODY, - .to_server_progress = HTP_REQUEST_BODY }, - { .al_proto = ALPROTO_HTTP2, - .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, - .to_client_progress = HTTP2StateDataServer, - .to_server_progress = HTTP2StateDataClient }, - { .al_proto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER } - }; + size_t i = 0; + while (al_protocols[i].al_proto != ALPROTO_UNKNOWN) { + i++; + } + al_protocols[i].al_proto = alproto; + al_protocols[i].direction = direction; + al_protocols[i].to_client_progress = to_client_progress; + al_protocols[i].to_server_progress = to_server_progress; + al_protocols[i + 1].al_proto = ALPROTO_UNKNOWN; +} +void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg) +{ for (size_t i = 0; i < ARRAY_SIZE(al_protocols); i++) { int direction = al_protocols[i].direction == 0 ? (int)(SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT) diff --git a/src/detect.h b/src/detect.h index edb84bbcc388..13aafaaa092e 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1546,6 +1546,7 @@ typedef struct DetectEngineMasterCtx_ { /* Table with all SigMatch registrations */ extern SigTableElmt *sigmatch_table; +extern SigTableElmt sigmatch_table_static[DETECT_TBLSIZE_STATIC]; /** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */ diff --git a/src/output.c b/src/output.c index 398604d554ec..3a64a6fe94b9 100644 --- a/src/output.c +++ b/src/output.c @@ -1346,3 +1346,11 @@ SimpleJsonAppLayerLogger *GetAppProtoSimpleJsonLogger(AppProto alproto) } return NULL; } + +void RegisterAppProtoAppLayerLogger(AppProto alproto, SimpleJsonAppLayerLogger *log) +{ + if (alproto < ALPROTO_MAX) { + simple_json_applayer_loggers[alproto].proto = alproto; + simple_json_applayer_loggers[alproto].LogTx = log; + } +} diff --git a/src/output.h b/src/output.h index 3daba06a33d3..83dc8ebaedf6 100644 --- a/src/output.h +++ b/src/output.h @@ -216,5 +216,6 @@ typedef struct SimpleJsonAppLayerLogger { } SimpleJsonAppLayerLogger; SimpleJsonAppLayerLogger *GetAppProtoSimpleJsonLogger(AppProto alproto); +void RegisterAppProtoAppLayerLogger(AppProto alproto, SimpleJsonAppLayerLogger *log); #endif /* ! __OUTPUT_H__ */ diff --git a/src/suricata-plugin.h b/src/suricata-plugin.h index 3296b51397b8..bb73c1ea35fd 100644 --- a/src/suricata-plugin.h +++ b/src/suricata-plugin.h @@ -74,4 +74,16 @@ typedef struct SCCapturePlugin_ { int SCPluginRegisterCapture(SCCapturePlugin *); +typedef struct SCAppLayerPlugin_ { + char *name; + char *logname; + char *confname; + void (*Register)(void); + bool (*Logger)(void *tx, void *jb); + void (*KeywordsRegister)(void); + uint32_t keywords_nb; +} SCAppLayerPlugin; + +int SCPluginRegisterAppLayer(SCAppLayerPlugin *); + #endif /* __SURICATA_PLUGIN_H */ diff --git a/src/suricata.c b/src/suricata.c index 7f979a7fbfcc..cb65d20275a6 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -385,6 +385,7 @@ static void GlobalsDestroy(SCInstance *suri) FeatureTrackingRelease(); SCProtoNameRelease(); TimeDeinit(); + SigTableCleanup(); TmqhCleanup(); TmModuleRunDeInit(); ParseSizeDeinit(); @@ -2675,6 +2676,9 @@ int PostConfLoadedSetup(SCInstance *suri) RunModeEngineIsIPS( suricata.run_mode, suricata.runmode_custom_mode, suricata.capture_plugin_name); +#ifdef HAVE_PLUGINS + SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); +#endif if (EngineModeIsUnknown()) { // if still uninitialized, set the default SCLogInfo("Setting engine mode to IDS mode by default"); @@ -2742,9 +2746,6 @@ int PostConfLoadedSetup(SCInstance *suri) FeatureTrackingRegister(); /* must occur prior to output mod registration */ RegisterAllModules(); -#ifdef HAVE_PLUGINS - SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); -#endif AppLayerHtpNeedFileInspection(); StorageFinalize(); diff --git a/src/util-plugin.c b/src/util-plugin.c index 3a08aa8876ad..182f3da9ff77 100644 --- a/src/util-plugin.c +++ b/src/util-plugin.c @@ -22,6 +22,7 @@ #include "output-eve-syslog.h" #include "util-plugin.h" #include "util-debug.h" +#include "app-layer-protos.h" #ifdef HAVE_PLUGINS @@ -211,4 +212,32 @@ SCCapturePlugin *SCPluginFindCaptureByName(const char *name) } return plugin; } + +#ifdef ALPROTO_DYNAMIC_NB +static SCAppLayerPlugin *app_layer_plugins[ALPROTO_DYNAMIC_NB]; +static size_t app_layer_plugins_nb = 0; +#endif + +int SCPluginRegisterAppLayer(SCAppLayerPlugin *plugin) +{ +#ifdef ALPROTO_DYNAMIC_NB + if (app_layer_plugins_nb < ALPROTO_DYNAMIC_NB) { + app_layer_plugins[app_layer_plugins_nb] = plugin; + RegisterAppProtoString((AppProto)(ALPROTO_MAX_STATIC + app_layer_plugins_nb), plugin->name); + app_layer_plugins_nb++; + return 0; + } +#endif + return 1; +} + +SCAppLayerPlugin *SCPluginFindAppLayerByIndex(size_t i) +{ +#ifdef ALPROTO_DYNAMIC_NB + if (i < ALPROTO_DYNAMIC_NB) { + return app_layer_plugins[i]; + } +#endif + return NULL; +} #endif diff --git a/src/util-plugin.h b/src/util-plugin.h index c958f2d79ccd..d187a25f1203 100644 --- a/src/util-plugin.h +++ b/src/util-plugin.h @@ -26,4 +26,6 @@ SCCapturePlugin *SCPluginFindCaptureByName(const char *name); bool RegisterPlugin(SCPlugin *, void *); +SCAppLayerPlugin *SCPluginFindAppLayerByIndex(size_t i); + #endif /* __UTIL_PLUGIN_H__ */