diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 9c54e07868ec..c7f658d596d1 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,7 +39,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Dumping github context for debugging - run: echo '${{ toJSON(github) }}' + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} - run: sudo apt update && sudo apt -y install jq curl - name: Parse repo and branch information env: diff --git a/doc/userguide/rules/differences-from-snort.rst b/doc/userguide/rules/differences-from-snort.rst index 8226e3a7e899..8387cf77a014 100644 --- a/doc/userguide/rules/differences-from-snort.rst +++ b/doc/userguide/rules/differences-from-snort.rst @@ -19,6 +19,7 @@ Automatic Protocol Detection - dns - http - imap (detection only by default; no parsing) + - pop3 (detection only by default; no parsing) - ftp - modbus (disabled by default; minimalist probe parser; can lead to false positives) - smb diff --git a/doc/userguide/rules/intro.rst b/doc/userguide/rules/intro.rst index 6b0ac46961d1..ab46a362e409 100644 --- a/doc/userguide/rules/intro.rst +++ b/doc/userguide/rules/intro.rst @@ -94,6 +94,7 @@ you can pick from. These are: * ssh * smtp * imap +* pop3 * modbus (disabled by default) * dnp3 (disabled by default) * enip (disabled by default) diff --git a/etc/schema.json b/etc/schema.json index 2d3d471ad9e1..cc7c6e6b0c62 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -3846,6 +3846,9 @@ "pgsql": { "$ref": "#/$defs/stats_applayer_error" }, + "pop3": { + "$ref": "#/$defs/stats_applayer_error" + }, "quic": { "$ref": "#/$defs/stats_applayer_error" }, @@ -3963,6 +3966,9 @@ "pgsql": { "type": "integer" }, + "pop3": { + "type": "integer" + }, "quic": { "type": "integer" }, @@ -4074,6 +4080,9 @@ "pgsql": { "type": "integer" }, + "pop3": { + "type": "integer" + }, "quic": { "type": "integer" }, diff --git a/rust/src/util.rs b/rust/src/util.rs index a0933689164e..d1d8de0dd01e 100644 --- a/rust/src/util.rs +++ b/rust/src/util.rs @@ -18,7 +18,103 @@ use std::ffi::CStr; use std::os::raw::c_char; +use nom7::bytes::complete::take_while1; +use nom7::character::complete::char; +use nom7::character::{is_alphabetic, is_alphanumeric}; +use nom7::combinator::verify; +use nom7::multi::many1_count; +use nom7::IResult; + #[no_mangle] pub unsafe extern "C" fn rs_check_utf8(val: *const c_char) -> bool { CStr::from_ptr(val).to_str().is_ok() } + +fn is_alphanumeric_or_hyphen(chr: u8) -> bool { + return is_alphanumeric(chr) || chr == b'-'; +} + +fn parse_domain_label(i: &[u8]) -> IResult<&[u8], ()> { + let (i, _) = verify(take_while1(is_alphanumeric_or_hyphen), |x: &[u8]| { + is_alphabetic(x[0]) && x[x.len() - 1] != b'-' + })(i)?; + return Ok((i, ())); +} + +fn parse_subdomain(input: &[u8]) -> IResult<&[u8], ()> { + let (input, _) = parse_domain_label(input)?; + let (input, _) = char('.')(input)?; + return Ok((input, ())); +} + +fn parse_domain(input: &[u8]) -> IResult<&[u8], ()> { + let (input, _) = many1_count(parse_subdomain)(input)?; + let (input, _) = parse_domain_label(input)?; + return Ok((input, ())); +} + +#[no_mangle] +pub unsafe extern "C" fn rs_validate_domain(input: *const u8, in_len: u32) -> u32 { + let islice = build_slice!(input, in_len as usize); + match parse_domain(islice) { + Ok((rem, _)) => { + return (islice.len() - rem.len()) as u32; + } + _ => { + return 0; + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_parse_domain() { + let buf0: &[u8] = "a-1.oisf.net more".as_bytes(); + let r0 = parse_domain(buf0); + match r0 { + Ok((rem, _)) => { + // And we should have 5 bytes left. + assert_eq!(rem.len(), 5); + } + _ => { + panic!("Result should have been ok."); + } + } + let buf1: &[u8] = "justatext".as_bytes(); + let r1 = parse_domain(buf1); + match r1 { + Ok((_, _)) => { + panic!("Result should not have been ok."); + } + _ => {} + } + let buf1: &[u8] = "1.com".as_bytes(); + let r1 = parse_domain(buf1); + match r1 { + Ok((_, _)) => { + panic!("Result should not have been ok."); + } + _ => {} + } + let buf1: &[u8] = "a-.com".as_bytes(); + let r1 = parse_domain(buf1); + match r1 { + Ok((_, _)) => { + panic!("Result should not have been ok."); + } + _ => {} + } + let buf1: &[u8] = "a(x)y.com".as_bytes(); + let r1 = parse_domain(buf1); + match r1 { + Ok((_, _)) => { + panic!("Result should not have been ok."); + } + _ => {} + } + } +} diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index 863fb920afd6..60ca2bc01c5c 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -287,6 +287,10 @@ static inline int PMGetProtoInspect(AppLayerProtoDetectThreadCtx *tctx, uint8_t pm_results_bf[(ALPROTO_MAX / 8) + 1]; memset(pm_results_bf, 0, sizeof(pm_results_bf)); + // Do not take pm_ctx->pp_max_len of all probing parsers, + // but only the the probing parsers which matched a pattern. + uint32_t pp_max_len = pm_ctx->mpm_ctx.maxdepth; + /* loop through unique pattern id's. Can't use search_cnt here, * as that contains all matches, tctx->pmq.pattern_id_array_cnt * contains only *unique* matches. */ @@ -296,6 +300,9 @@ static inline int PMGetProtoInspect(AppLayerProtoDetectThreadCtx *tctx, AppProto proto = AppLayerProtoDetectPMMatchSignature( s, tctx, f, flags, buf, buflen, searchlen, rflow); + if (s->pp_max_depth > pp_max_len) { + pp_max_len = s->pp_max_depth; + } /* store each unique proto once */ if (AppProtoIsValid(proto) && !(pm_results_bf[proto / 8] & (1 << (proto % 8))) ) @@ -306,7 +313,7 @@ static inline int PMGetProtoInspect(AppLayerProtoDetectThreadCtx *tctx, s = s->next; } } - if (pm_matches == 0 && buflen >= pm_ctx->pp_max_len) { + if (pm_matches == 0 && buflen >= pp_max_len) { pm_matches = -2; } PmqReset(&tctx->pmq); @@ -613,6 +620,10 @@ static AppProto AppLayerProtoDetectPPGetProto(Flow *f, const uint8_t *buf, uint3 mask = pp_port_dp->alproto_mask; else if (pp_port_sp) mask = pp_port_sp->alproto_mask; + else { + // only pe0 + mask = pe0->alproto_mask; + } if (alproto_masks[0] == mask) { FLOW_SET_PP_DONE(f, dir); @@ -874,6 +885,8 @@ static void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingPar printf(" alproto: ALPROTO_SSH\n"); else if (pp_pe->alproto == ALPROTO_IMAP) printf(" alproto: ALPROTO_IMAP\n"); + else if (pp_pe->alproto == ALPROTO_POP3) + printf(" alproto: ALPROTO_POP3\n"); else if (pp_pe->alproto == ALPROTO_JABBER) printf(" alproto: ALPROTO_JABBER\n"); else if (pp_pe->alproto == ALPROTO_SMB) @@ -957,6 +970,8 @@ static void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingPar printf(" alproto: ALPROTO_SSH\n"); else if (pp_pe->alproto == ALPROTO_IMAP) printf(" alproto: ALPROTO_IMAP\n"); + else if (pp_pe->alproto == ALPROTO_POP3) + printf(" alproto: ALPROTO_POP3\n"); else if (pp_pe->alproto == ALPROTO_JABBER) printf(" alproto: ALPROTO_JABBER\n"); else if (pp_pe->alproto == ALPROTO_SMB) diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index 4f7ed0b16baf..7b40f74622bc 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -949,6 +949,46 @@ static int FTPGetAlstateProgress(void *vtx, uint8_t direction) return FTP_STATE_FINISHED; } +static AppProto FTPUserProbingParser( + Flow *f, uint8_t direction, const uint8_t *input, uint32_t len, uint8_t *rdir) +{ + if (f->alproto_tc == ALPROTO_POP3) { + // POP traffic begins by same "USER" pattern as FTP + // So exclude port 110 for now, until Suricata has a POP parser + // recognizing the "+OK" pattern from server. + return ALPROTO_FAILED; + } + return ALPROTO_FTP; +} + +static AppProto FTPServerProbingParser( + Flow *f, uint8_t direction, const uint8_t *input, uint32_t len, uint8_t *rdir) +{ + // another check for minimum length + if (len < 5) { + return ALPROTO_UNKNOWN; + } + // begins by 220 + if (input[0] != '2' || input[1] != '2' || input[2] != '0') { + return ALPROTO_FAILED; + } + // followed by space or hypen + if (input[3] != ' ' && input[3] != '-') { + return ALPROTO_FAILED; + } + AppProto r = ALPROTO_UNKNOWN; + if (f->alproto_ts == ALPROTO_FTP || (f->todstbytecnt > 4 && f->alproto_ts == ALPROTO_UNKNOWN)) { + // only validates FTP if client side was FTP + // or if client side is unknown despite having received bytes + r = ALPROTO_FTP; + } + for (uint32_t i = 4; i < len; i++) { + if (input[i] == '\n') { + return r; + } + } + return ALPROTO_UNKNOWN; +} static int FTPRegisterPatternsForProtocolDetection(void) { @@ -960,8 +1000,8 @@ static int FTPRegisterPatternsForProtocolDetection(void) IPPROTO_TCP, ALPROTO_FTP, "FEAT", 4, 0, STREAM_TOSERVER) < 0) { return -1; } - if (AppLayerProtoDetectPMRegisterPatternCI( - IPPROTO_TCP, ALPROTO_FTP, "USER ", 5, 0, STREAM_TOSERVER) < 0) { + if (AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_FTP, "USER ", 5, 0, + STREAM_TOSERVER, FTPUserProbingParser, 5, 5) < 0) { return -1; } if (AppLayerProtoDetectPMRegisterPatternCI( @@ -972,7 +1012,13 @@ static int FTPRegisterPatternsForProtocolDetection(void) IPPROTO_TCP, ALPROTO_FTP, "PORT ", 5, 0, STREAM_TOSERVER) < 0) { return -1; } - + // Only check FTP on known ports as the banner has nothing special beyond + // the response code shared with SMTP. + if (!AppLayerProtoDetectPPParseConfPorts( + "tcp", IPPROTO_TCP, "ftp", ALPROTO_FTP, 0, 5, NULL, FTPServerProbingParser)) { + AppLayerProtoDetectPPRegister(IPPROTO_TCP, "21", ALPROTO_FTP, 0, 5, STREAM_TOCLIENT, NULL, + FTPServerProbingParser); + } return 0; } diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 66a7eec9149c..e96a47e68c23 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -1805,6 +1805,18 @@ void AppLayerParserRegisterProtocolParsers(void) "imap"); } + /** POP3 */ + AppLayerProtoDetectRegisterProtocol(ALPROTO_POP3, "pop3"); + if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "pop3")) { + if (AppLayerProtoDetectPMRegisterPatternCS( + IPPROTO_TCP, ALPROTO_POP3, "+OK ", 4, 0, STREAM_TOCLIENT) < 0) { + SCLogInfo("pop3 proto registration failure"); + exit(EXIT_FAILURE); + } + } else { + SCLogInfo("Protocol detection and parser disabled for pop3 protocol."); + } + ValidateParsers(); return; } diff --git a/src/app-layer-protos.c b/src/app-layer-protos.c index f944af7c438e..4ed51819c85c 100644 --- a/src/app-layer-protos.c +++ b/src/app-layer-protos.c @@ -51,6 +51,9 @@ const char *AppProtoToString(AppProto alproto) case ALPROTO_IMAP: proto_name = "imap"; break; + case ALPROTO_POP3: + proto_name = "pop3"; + break; case ALPROTO_JABBER: proto_name = "jabber"; break; @@ -169,6 +172,8 @@ AppProto StringToAppProto(const char *proto_name) return ALPROTO_SSH; if (strcmp(proto_name, "imap") == 0) return ALPROTO_IMAP; + if (strcmp(proto_name, "pop3") == 0) + return ALPROTO_POP3; if (strcmp(proto_name, "jabber") == 0) return ALPROTO_JABBER; if (strcmp(proto_name, "smb") == 0) diff --git a/src/app-layer-protos.h b/src/app-layer-protos.h index b0a5db1c8a01..ffc086bbcea5 100644 --- a/src/app-layer-protos.h +++ b/src/app-layer-protos.h @@ -60,6 +60,7 @@ enum AppProtoEnum { ALPROTO_RDP, ALPROTO_HTTP2, ALPROTO_BITTORRENT_DHT, + ALPROTO_POP3, // signature-only (ie not seen in flow) // HTTP for any version (ALPROTO_HTTP1 (version 1) or ALPROTO_HTTP2) diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 95e84b64c110..728e50a57b71 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -1612,6 +1612,50 @@ static int SMTPStateGetEventInfoById(int event_id, const char **event_name, return 0; } +static AppProto SMTPServerProbingParser( + Flow *f, uint8_t direction, const uint8_t *input, uint32_t len, uint8_t *rdir) +{ + // another check for minimum length + if (len < 5) { + return ALPROTO_UNKNOWN; + } + // begins by 220 + if (input[0] != '2' || input[1] != '2' || input[2] != '0') { + return ALPROTO_FAILED; + } + // followed by space or hypen + if (input[3] != ' ' && input[3] != '-') { + return ALPROTO_FAILED; + } + // If client side is SMTP, do not validate domain + // so that server banner can be parsed first. + if (f->alproto_ts == ALPROTO_SMTP) { + for (uint32_t i = 4; i < len; i++) { + if (input[i] == '\n') { + return ALPROTO_SMTP; + } + } + return ALPROTO_UNKNOWN; + } + AppProto r = ALPROTO_UNKNOWN; + if (f->todstbytecnt > 4 && f->alproto_ts == ALPROTO_UNKNOWN) { + // Only validates SMTP if client side is unknown + // despite having received bytes. + r = ALPROTO_SMTP; + } + uint32_t offset = rs_validate_domain(input + 4, len - 4); + if (offset == 0) { + return ALPROTO_FAILED; + } + for (uint32_t i = offset + 4; i < len; i++) { + if (input[i] == '\n') { + return r; + } + } + // This should not go forever because of engine limiting probing parsers. + return ALPROTO_UNKNOWN; +} + static int SMTPRegisterPatternsForProtocolDetection(void) { if (AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_SMTP, @@ -1629,6 +1673,19 @@ static int SMTPRegisterPatternsForProtocolDetection(void) { return -1; } + if (!AppLayerProtoDetectPPParseConfPorts( + "tcp", IPPROTO_TCP, "smtp", ALPROTO_SMTP, 0, 5, NULL, SMTPServerProbingParser)) { + AppLayerProtoDetectPPRegister(IPPROTO_TCP, "25", ALPROTO_SMTP, 0, 5, STREAM_TOCLIENT, NULL, + SMTPServerProbingParser); + } + if (AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMTP, "220 ", 4, 0, + STREAM_TOCLIENT, SMTPServerProbingParser, 5, 5) < 0) { + return -1; + } + if (AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMTP, "220-", 4, 0, + STREAM_TOCLIENT, SMTPServerProbingParser, 5, 5) < 0) { + return -1; + } return 0; } diff --git a/src/app-layer.c b/src/app-layer.c index 0e244caf2039..9748938d022a 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -779,6 +779,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, if (r != 1) { StreamTcpUpdateAppLayerProgress(ssn, direction, data_len); if (r < 0) { + if (f->alproto_tc == ALPROTO_UNKNOWN || f->alproto_ts == ALPROTO_UNKNOWN) { + AppLayerIncFlowCounter(tv, f); + } ExceptionPolicyApply( p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR); SCReturnInt(-1); diff --git a/suricata.yaml.in b/suricata.yaml.in index b337864264a4..3bae772ca7b2 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -953,6 +953,8 @@ app-layer: content-inspect-window: 4096 imap: enabled: detection-only + pop3: + enabled: detection-only smb: enabled: yes detection-ports: