Smtp server detection 1125 v13

.github/workflows/builds.yml

@@ -39,7 +39,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Dumping github context for debugging
-        run: echo '${{ toJSON(github) }}'
+        run: echo $JSON
+        env:
+          JSON: ${{ toJSON(github) }}
       - run: sudo apt update && sudo apt -y install jq curl
       - name: Parse repo and branch information
         env:

doc/userguide/rules/differences-from-snort.rst

@@ -19,6 +19,7 @@ Automatic Protocol Detection
    -  dns
    -  http
    -  imap (detection only by default; no parsing)
+   -  pop3 (detection only by default; no parsing)
    -  ftp
    -  modbus (disabled by default; minimalist probe parser; can lead to false positives)
    -  smb

doc/userguide/rules/intro.rst

@@ -94,6 +94,7 @@ you can pick from. These are:
 * ssh
 * smtp
 * imap
+* pop3
 * modbus (disabled by default)
 * dnp3 (disabled by default)
 * enip (disabled by default)

etc/schema.json

@@ -3846,6 +3846,9 @@
                                 "pgsql": {
                                     "$ref": "#/$defs/stats_applayer_error"
                                 },
+                                "pop3": {
+                                    "$ref": "#/$defs/stats_applayer_error"
+                                },
                                 "quic": {
                                     "$ref": "#/$defs/stats_applayer_error"
                                 },
@@ -3963,6 +3966,9 @@
                                 "pgsql": {
                                     "type": "integer"
                                 },
+                                "pop3": {
+                                    "type": "integer"
+                                },
                                 "quic": {
                                     "type": "integer"
                                 },
@@ -4074,6 +4080,9 @@
                                 "pgsql": {
                                     "type": "integer"
                                 },
+                                "pop3": {
+                                    "type": "integer"
+                                },
                                 "quic": {
                                     "type": "integer"
                                 },

rust/src/util.rs

@@ -18,7 +18,103 @@
 use std::ffi::CStr;
 use std::os::raw::c_char;
 
+use nom7::bytes::complete::take_while1;
+use nom7::character::complete::char;
+use nom7::character::{is_alphabetic, is_alphanumeric};
+use nom7::combinator::verify;
+use nom7::multi::many1_count;
+use nom7::IResult;
+
 #[no_mangle]
 pub unsafe extern "C" fn rs_check_utf8(val: *const c_char) -> bool {
     CStr::from_ptr(val).to_str().is_ok()
 }
+
+fn is_alphanumeric_or_hyphen(chr: u8) -> bool {
+    return is_alphanumeric(chr) || chr == b'-';
+}
+
+fn parse_domain_label(i: &[u8]) -> IResult<&[u8], ()> {
+    let (i, _) = verify(take_while1(is_alphanumeric_or_hyphen), |x: &[u8]| {
+        is_alphabetic(x[0]) && x[x.len() - 1] != b'-'
+    })(i)?;
+    return Ok((i, ()));
+}
+
+fn parse_subdomain(input: &[u8]) -> IResult<&[u8], ()> {
+    let (input, _) = parse_domain_label(input)?;
+    let (input, _) = char('.')(input)?;
+    return Ok((input, ()));
+}
+
+fn parse_domain(input: &[u8]) -> IResult<&[u8], ()> {
+    let (input, _) = many1_count(parse_subdomain)(input)?;
+    let (input, _) = parse_domain_label(input)?;
+    return Ok((input, ()));
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rs_validate_domain(input: *const u8, in_len: u32) -> u32 {
+    let islice = build_slice!(input, in_len as usize);
+    match parse_domain(islice) {
+        Ok((rem, _)) => {
+            return (islice.len() - rem.len()) as u32;
+        }
+        _ => {
+            return 0;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    #[test]
+    fn test_parse_domain() {
+        let buf0: &[u8] = "a-1.oisf.net more".as_bytes();
+        let r0 = parse_domain(buf0);
+        match r0 {
+            Ok((rem, _)) => {
+                // And we should have 5 bytes left.
+                assert_eq!(rem.len(), 5);
+            }
+            _ => {
+                panic!("Result should have been ok.");
+            }
+        }
+        let buf1: &[u8] = "justatext".as_bytes();
+        let r1 = parse_domain(buf1);
+        match r1 {
+            Ok((_, _)) => {
+                panic!("Result should not have been ok.");
+            }
+            _ => {}
+        }
+        let buf1: &[u8] = "1.com".as_bytes();
+        let r1 = parse_domain(buf1);
+        match r1 {
+            Ok((_, _)) => {
+                panic!("Result should not have been ok.");
+            }
+            _ => {}
+        }
+        let buf1: &[u8] = "a-.com".as_bytes();
+        let r1 = parse_domain(buf1);
+        match r1 {
+            Ok((_, _)) => {
+                panic!("Result should not have been ok.");
+            }
+            _ => {}
+        }
+        let buf1: &[u8] = "a(x)y.com".as_bytes();
+        let r1 = parse_domain(buf1);
+        match r1 {
+            Ok((_, _)) => {
+                panic!("Result should not have been ok.");
+            }
+            _ => {}
+        }
+    }
+}

src/app-layer-detect-proto.c

@@ -287,6 +287,10 @@ static inline int PMGetProtoInspect(AppLayerProtoDetectThreadCtx *tctx,
     uint8_t pm_results_bf[(ALPROTO_MAX / 8) + 1];
     memset(pm_results_bf, 0, sizeof(pm_results_bf));
 
+    // Do not take pm_ctx->pp_max_len of all probing parsers,
+    // but only the the probing parsers which matched a pattern.
+    uint32_t pp_max_len = pm_ctx->mpm_ctx.maxdepth;
+
     /* loop through unique pattern id's. Can't use search_cnt here,
      * as that contains all matches, tctx->pmq.pattern_id_array_cnt
      * contains only *unique* matches. */
@@ -296,6 +300,9 @@ static inline int PMGetProtoInspect(AppLayerProtoDetectThreadCtx *tctx,
             AppProto proto = AppLayerProtoDetectPMMatchSignature(
                     s, tctx, f, flags, buf, buflen, searchlen, rflow);
 
+            if (s->pp_max_depth > pp_max_len) {
+                pp_max_len = s->pp_max_depth;
+            }
             /* store each unique proto once */
             if (AppProtoIsValid(proto) &&
                 !(pm_results_bf[proto / 8] & (1 << (proto % 8))) )
@@ -306,7 +313,7 @@ static inline int PMGetProtoInspect(AppLayerProtoDetectThreadCtx *tctx,
             s = s->next;
         }
     }
-    if (pm_matches == 0 && buflen >= pm_ctx->pp_max_len) {
+    if (pm_matches == 0 && buflen >= pp_max_len) {
         pm_matches = -2;
     }
     PmqReset(&tctx->pmq);
@@ -613,6 +620,10 @@ static AppProto AppLayerProtoDetectPPGetProto(Flow *f, const uint8_t *buf, uint3
             mask = pp_port_dp->alproto_mask;
         else if (pp_port_sp)
             mask = pp_port_sp->alproto_mask;
+        else {
+            // only pe0
+            mask = pe0->alproto_mask;
+        }
 
         if (alproto_masks[0] == mask) {
             FLOW_SET_PP_DONE(f, dir);
@@ -874,6 +885,8 @@ static void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingPar
                         printf("            alproto: ALPROTO_SSH\n");
                     else if (pp_pe->alproto == ALPROTO_IMAP)
                         printf("            alproto: ALPROTO_IMAP\n");
+                    else if (pp_pe->alproto == ALPROTO_POP3)
+                        printf("            alproto: ALPROTO_POP3\n");
                     else if (pp_pe->alproto == ALPROTO_JABBER)
                         printf("            alproto: ALPROTO_JABBER\n");
                     else if (pp_pe->alproto == ALPROTO_SMB)
@@ -957,6 +970,8 @@ static void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingPar
                     printf("            alproto: ALPROTO_SSH\n");
                 else if (pp_pe->alproto == ALPROTO_IMAP)
                     printf("            alproto: ALPROTO_IMAP\n");
+                else if (pp_pe->alproto == ALPROTO_POP3)
+                    printf("            alproto: ALPROTO_POP3\n");
                 else if (pp_pe->alproto == ALPROTO_JABBER)
                     printf("            alproto: ALPROTO_JABBER\n");
                 else if (pp_pe->alproto == ALPROTO_SMB)

src/app-layer-ftp.c

@@ -949,6 +949,46 @@ static int FTPGetAlstateProgress(void *vtx, uint8_t direction)
     return FTP_STATE_FINISHED;
 }
 
+static AppProto FTPUserProbingParser(
+        Flow *f, uint8_t direction, const uint8_t *input, uint32_t len, uint8_t *rdir)
+{
+    if (f->alproto_tc == ALPROTO_POP3) {
+        // POP traffic begins by same "USER" pattern as FTP
+        // So exclude port 110 for now, until Suricata has a POP parser
+        // recognizing the "+OK" pattern from server.
+        return ALPROTO_FAILED;
+    }
+    return ALPROTO_FTP;
+}
+
+static AppProto FTPServerProbingParser(
+        Flow *f, uint8_t direction, const uint8_t *input, uint32_t len, uint8_t *rdir)
+{
+    // another check for minimum length
+    if (len < 5) {
+        return ALPROTO_UNKNOWN;
+    }
+    // begins by 220
+    if (input[0] != '2' || input[1] != '2' || input[2] != '0') {
+        return ALPROTO_FAILED;
+    }
+    // followed by space or hypen
+    if (input[3] != ' ' && input[3] != '-') {
+        return ALPROTO_FAILED;
+    }
+    AppProto r = ALPROTO_UNKNOWN;
+    if (f->alproto_ts == ALPROTO_FTP || (f->todstbytecnt > 4 && f->alproto_ts == ALPROTO_UNKNOWN)) {
+        // only validates FTP if client side was FTP
+        // or if client side is unknown despite having received bytes
+        r = ALPROTO_FTP;
+    }
+    for (uint32_t i = 4; i < len; i++) {
+        if (input[i] == '\n') {
+            return r;
+        }
+    }
+    return ALPROTO_UNKNOWN;
+}
 
 static int FTPRegisterPatternsForProtocolDetection(void)
 {
@@ -960,8 +1000,8 @@ static int FTPRegisterPatternsForProtocolDetection(void)
                 IPPROTO_TCP, ALPROTO_FTP, "FEAT", 4, 0, STREAM_TOSERVER) < 0) {
         return -1;
     }
-    if (AppLayerProtoDetectPMRegisterPatternCI(
-                IPPROTO_TCP, ALPROTO_FTP, "USER ", 5, 0, STREAM_TOSERVER) < 0) {
+    if (AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_FTP, "USER ", 5, 0,
+                STREAM_TOSERVER, FTPUserProbingParser, 5, 5) < 0) {
         return -1;
     }
     if (AppLayerProtoDetectPMRegisterPatternCI(
@@ -972,7 +1012,13 @@ static int FTPRegisterPatternsForProtocolDetection(void)
                 IPPROTO_TCP, ALPROTO_FTP, "PORT ", 5, 0, STREAM_TOSERVER) < 0) {
         return -1;
     }
-
+    // Only check FTP on known ports as the banner has nothing special beyond
+    // the response code shared with SMTP.
+    if (!AppLayerProtoDetectPPParseConfPorts(
+                "tcp", IPPROTO_TCP, "ftp", ALPROTO_FTP, 0, 5, NULL, FTPServerProbingParser)) {
+        AppLayerProtoDetectPPRegister(IPPROTO_TCP, "21", ALPROTO_FTP, 0, 5, STREAM_TOCLIENT, NULL,
+                FTPServerProbingParser);
+    }
     return 0;
 }
 

src/app-layer-parser.c

@@ -1805,6 +1805,18 @@ void AppLayerParserRegisterProtocolParsers(void)
                   "imap");
     }
 
+    /** POP3 */
+    AppLayerProtoDetectRegisterProtocol(ALPROTO_POP3, "pop3");
+    if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "pop3")) {
+        if (AppLayerProtoDetectPMRegisterPatternCS(
+                    IPPROTO_TCP, ALPROTO_POP3, "+OK ", 4, 0, STREAM_TOCLIENT) < 0) {
+            SCLogInfo("pop3 proto registration failure");
+            exit(EXIT_FAILURE);
+        }
+    } else {
+        SCLogInfo("Protocol detection and parser disabled for pop3 protocol.");
+    }
+
     ValidateParsers();
     return;
 }

src/app-layer-protos.c

@@ -51,6 +51,9 @@ const char *AppProtoToString(AppProto alproto)
         case ALPROTO_IMAP:
             proto_name = "imap";
             break;
+        case ALPROTO_POP3:
+            proto_name = "pop3";
+            break;
         case ALPROTO_JABBER:
             proto_name = "jabber";
             break;
@@ -169,6 +172,8 @@ AppProto StringToAppProto(const char *proto_name)
         return ALPROTO_SSH;
     if (strcmp(proto_name, "imap") == 0)
         return ALPROTO_IMAP;
+    if (strcmp(proto_name, "pop3") == 0)
+        return ALPROTO_POP3;
     if (strcmp(proto_name, "jabber") == 0)
         return ALPROTO_JABBER;
     if (strcmp(proto_name, "smb") == 0)

src/app-layer-protos.h

@@ -60,6 +60,7 @@ enum AppProtoEnum {
     ALPROTO_RDP,
     ALPROTO_HTTP2,
     ALPROTO_BITTORRENT_DHT,
+    ALPROTO_POP3,
 
     // signature-only (ie not seen in flow)
     // HTTP for any version (ALPROTO_HTTP1 (version 1) or ALPROTO_HTTP2)