Ish/6.0.x/Staging/v1

rust/src/dns/dns.rs

@@ -85,6 +85,7 @@ pub const DNS_RECORD_TYPE_TLSA        : u16 = 52;
 pub const DNS_RECORD_TYPE_HIP         : u16 = 55;
 pub const DNS_RECORD_TYPE_CDS         : u16 = 59;
 pub const DNS_RECORD_TYPE_CDNSKEY     : u16 = 60;
+pub const DNS_RECORD_TYPE_HTTPS       : u16 = 65;
 pub const DNS_RECORD_TYPE_SPF         : u16 = 99;  // Obsolete
 pub const DNS_RECORD_TYPE_TKEY        : u16 = 249;
 pub const DNS_RECORD_TYPE_TSIG        : u16 = 250;

rust/src/dns/log.rs

@@ -86,6 +86,7 @@ pub const LOG_URI        : u64 = BIT_U64!(59);
 
 pub const LOG_FORMAT_GROUPED  : u64 = BIT_U64!(60);
 pub const LOG_FORMAT_DETAILED : u64 = BIT_U64!(61);
+pub const LOG_HTTPS      : u64 = BIT_U64!(62);
 
 fn dns_log_rrtype_enabled(rtype: u16, flags: u64) -> bool
 {
@@ -250,6 +251,9 @@ fn dns_log_rrtype_enabled(rtype: u16, flags: u64) -> bool
         DNS_RECORD_TYPE_CDNSKEY => {
             return flags & LOG_CDNSKEY != 0;
         }
+        DNS_RECORD_TYPE_HTTPS => {
+            return flags & LOG_HTTPS != 0;
+        }
         DNS_RECORD_TYPE_SPF => {
             return flags & LOG_SPF != 0;
         }
@@ -324,6 +328,7 @@ pub fn dns_rrtype_string(rrtype: u16) -> String {
         DNS_RECORD_TYPE_HIP => "HIP",
         DNS_RECORD_TYPE_CDS => "CDS",
         DNS_RECORD_TYPE_CDNSKEY => "CDSNKEY",
+        DNS_RECORD_TYPE_HTTPS => "HTTPS",
         DNS_RECORD_TYPE_MAILA => "MAILA",
         DNS_RECORD_TYPE_URI => "URI",
         DNS_RECORD_TYPE_MB => "MB",

src/detect-engine-build.c

@@ -1967,7 +1967,7 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
     ThresholdHashAllocate(de_ctx);
 
     if (!DetectEngineMultiTenantEnabled()) {
-        VarNameStoreActivateStaging();
+        VarNameStoreActivate();
     }
     return 0;
 }

src/detect-engine.c

@@ -75,6 +75,7 @@
 #include "util-spm.h"
 #include "util-device.h"
 #include "util-var-name.h"
+#include "util-path.h"
 #include "util-profiling.h"
 #include "util-validate.h"
 
@@ -1986,6 +1987,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(enum DetectEngineType type, cons
     TAILQ_INIT(&de_ctx->sig_stat.failed_sigs);
     de_ctx->sigerror = NULL;
     de_ctx->type = type;
+    de_ctx->filemagic_thread_ctx_id = -1;
 
     if (type == DETECT_ENGINE_TYPE_DD_STUB || type == DETECT_ENGINE_TYPE_MT_STUB) {
         de_ctx->version = DetectEngineGetVersion();
@@ -2039,7 +2041,6 @@ static DetectEngineCtx *DetectEngineCtxInitReal(enum DetectEngineType type, cons
     }
 
     de_ctx->version = DetectEngineGetVersion();
-    VarNameStoreSetupStaging(de_ctx->version);
     SCLogDebug("dectx with version %u", de_ctx->version);
     return de_ctx;
 error:
@@ -2170,8 +2171,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     DetectPortCleanupList(de_ctx, de_ctx->udp_whitelist);
 
     DetectBufferTypeFreeDetectEngine(de_ctx);
-    /* freed our var name hash */
-    VarNameStoreFree(de_ctx->version);
 
     SCFree(de_ctx);
     //DetectAddressGroupPrintMemory();
@@ -3303,13 +3302,8 @@ static int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *fil
 
     snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id);
 
-#ifdef OS_WIN32
-    struct _stat st;
-    if(_stat(filename, &st) != 0) {
-#else
-    struct stat st;
-    if(stat(filename, &st) != 0) {
-#endif /* OS_WIN32 */
+    SCStat st;
+    if (SCStatFn(filename, &st) != 0) {
         SCLogError(SC_ERR_FOPEN, "failed to stat file %s", filename);
         goto error;
     }
@@ -3768,7 +3762,7 @@ int DetectEngineMultiTenantSetup(void)
             goto error;
         }
 
-        VarNameStoreActivateStaging();
+        VarNameStoreActivate();
 
     } else {
         SCLogDebug("multi-detect not enabled (multi tenancy)");

src/detect-filemagic.c

@@ -92,8 +92,6 @@ static int DetectEngineInspectFilemagic(
         const Signature *s,
         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
 
-static int g_magic_thread_ctx_id = -1;
-
 /**
  * \brief Registration function for keyword: filemagic
  */
@@ -251,10 +249,10 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, const ch
                 de_ctx, s, NULL, DETECT_FILE_MAGIC, g_file_magic_buffer_id, s->alproto) < 0)
         return -1;
 
-    if (g_magic_thread_ctx_id == -1) {
-        g_magic_thread_ctx_id = DetectRegisterThreadCtxFuncs(
+    if (de_ctx->filemagic_thread_ctx_id == -1) {
+        de_ctx->filemagic_thread_ctx_id = DetectRegisterThreadCtxFuncs(
                 de_ctx, "filemagic", DetectFilemagicThreadInit, NULL, DetectFilemagicThreadFree, 1);
-        if (g_magic_thread_ctx_id == -1)
+        if (de_ctx->filemagic_thread_ctx_id == -1)
             return -1;
     }
     return 0;
@@ -276,11 +274,10 @@ static int DetectFilemagicSetupSticky(DetectEngineCtx *de_ctx, Signature *s, con
     if (DetectBufferSetActiveList(s, g_file_magic_buffer_id) < 0)
         return -1;
 
-    if (g_magic_thread_ctx_id == -1) {
-        g_magic_thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "filemagic",
-                DetectFilemagicThreadInit, NULL,
-                DetectFilemagicThreadFree, 1);
-        if (g_magic_thread_ctx_id == -1)
+    if (de_ctx->filemagic_thread_ctx_id == -1) {
+        de_ctx->filemagic_thread_ctx_id = DetectRegisterThreadCtxFuncs(
+                de_ctx, "filemagic", DetectFilemagicThreadInit, NULL, DetectFilemagicThreadFree, 1);
+        if (de_ctx->filemagic_thread_ctx_id == -1)
             return -1;
     }
     return 0;
@@ -301,7 +298,8 @@ static InspectionBuffer *FilemagicGetDataCallback(DetectEngineThreadCtx *det_ctx
 
     if (cur_file->magic == NULL) {
         DetectFilemagicThreadData *tfilemagic =
-            (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(det_ctx, g_magic_thread_ctx_id);
+                (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(
+                        det_ctx, det_ctx->de_ctx->filemagic_thread_ctx_id);
         if (tfilemagic == NULL) {
             return NULL;
         }

src/detect-flowbits.c

@@ -109,7 +109,7 @@ static int FlowbitOrAddData(DetectEngineCtx *de_ctx, DetectFlowbitsData *cd, cha
     if (unlikely(cd->or_list == NULL))
         return -1;
     for (uint8_t j = 0; j < cd->or_list_size ; j++) {
-        cd->or_list[j] = VarNameStoreSetupAdd(strarr[j], VAR_TYPE_FLOW_BIT);
+        cd->or_list[j] = VarNameStoreRegister(strarr[j], VAR_TYPE_FLOW_BIT);
         de_ctx->max_fb_id = MAX(cd->or_list[j], de_ctx->max_fb_id);
     }
 
@@ -310,7 +310,7 @@ static int DetectFlowbitParse(
         }
         cd->cmd = cmd;
     } else {
-        cd->idx = VarNameStoreSetupAdd(name, VAR_TYPE_FLOW_BIT);
+        cd->idx = VarNameStoreRegister(name, VAR_TYPE_FLOW_BIT);
         de_ctx->max_fb_id = MAX(cd->idx, de_ctx->max_fb_id);
         cd->cmd = cmd;
         cd->or_list_size = 0;
@@ -386,8 +386,13 @@ void DetectFlowbitFree (DetectEngineCtx *de_ctx, void *ptr)
     DetectFlowbitsData *fd = (DetectFlowbitsData *)ptr;
     if (fd == NULL)
         return;
-    if (fd->or_list != NULL)
+    VarNameStoreUnregister(fd->idx, VAR_TYPE_FLOW_BIT);
+    if (fd->or_list != NULL) {
+        for (uint8_t i = 0; i < fd->or_list_size; i++) {
+            VarNameStoreUnregister(fd->or_list[i], VAR_TYPE_FLOW_BIT);
+        }
         SCFree(fd->or_list);
+    }
     SCFree(fd);
 }
 
@@ -590,7 +595,7 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
 
     /* walk array to see if all bits make sense */
     for (uint32_t i = 0; i < array_size; i++) {
-        char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
+        const char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
         if (varname == NULL)
             continue;
 
@@ -643,7 +648,6 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
                         "stateful rules that set flowbit %s", s->id, varname);
             }
         }
-        SCFree(varname);
     }
 
     if (rule_engine_analysis_set) {
@@ -673,7 +677,7 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
 
     jb_open_array(js, "flowbits");
     for (uint32_t x = 0; x < elements; x++) {
-        char *varname = VarNameStoreSetupLookup(x, VAR_TYPE_FLOW_BIT);
+        const char *varname = VarNameStoreSetupLookup(x, VAR_TYPE_FLOW_BIT);
         if (varname == NULL)
             continue;
 
@@ -733,7 +737,6 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
             }
             jb_close(js);
         }
-        SCFree(varname);
         jb_close(js);
     }
     jb_close(js); // array
@@ -911,8 +914,8 @@ static int FlowBitsTestSig04(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"isset option\"; flowbits:isset,fbt; content:\"GET \"; sid:1;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("fbt", VAR_TYPE_FLOW_BIT);
-    FAIL_IF(idx != 1);
+    idx = VarNameStoreRegister("fbt", VAR_TYPE_FLOW_BIT);
+    FAIL_IF(idx == 0);
 
     SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
@@ -994,7 +997,7 @@ static int FlowBitsTestSig06(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit set\"; flowbits:set,myflow; sid:10;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreRegister("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
@@ -1068,7 +1071,7 @@ static int FlowBitsTestSig07(void)
     s = s->next = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit unset\"; flowbits:unset,myflow2; sid:11;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreRegister("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
@@ -1144,7 +1147,7 @@ static int FlowBitsTestSig08(void)
     s = s->next  = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit unset\"; flowbits:toggle,myflow2; sid:11;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreRegister("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 

src/detect-flowint.c

@@ -329,7 +329,7 @@ static DetectFlowintData *DetectFlowintParse(DetectEngineCtx *de_ctx, const char
         SCLogError(SC_ERR_MEM_ALLOC, "malloc from strdup failed");
         goto error;
     }
-    sfd->idx = VarNameStoreSetupAdd(varname, VAR_TYPE_FLOW_INT);
+    sfd->idx = VarNameStoreRegister(varname, VAR_TYPE_FLOW_INT);
     SCLogDebug("sfd->name %s id %u", sfd->name, sfd->idx);
     sfd->modifier = modifier;
 
@@ -416,6 +416,7 @@ void DetectFlowintFree(DetectEngineCtx *de_ctx, void *tmp)
 {
     DetectFlowintData *sfd =(DetectFlowintData*) tmp;
     if (sfd != NULL) {
+        VarNameStoreUnregister(sfd->idx, VAR_TYPE_FLOW_INT);
         if (sfd->name != NULL)
             SCFree(sfd->name);
         if (sfd->targettype == FLOWINT_TARGET_VAR)

src/detect-flowvar.c

@@ -78,6 +78,9 @@ static void DetectFlowvarDataFree(DetectEngineCtx *de_ctx, void *ptr)
         SCReturn;
 
     DetectFlowvarData *fd = (DetectFlowvarData *)ptr;
+    /* leave unregistration to pcre keyword */
+    if (!fd->post_match)
+        VarNameStoreUnregister(fd->idx, VAR_TYPE_FLOW_VAR);
 
     if (fd->name)
         SCFree(fd->name);
@@ -172,7 +175,7 @@ static int DetectFlowvarSetup (DetectEngineCtx *de_ctx, Signature *s, const char
     fd->name = SCStrdup(varname);
     if (unlikely(fd->name == NULL))
         goto error;
-    fd->idx = VarNameStoreSetupAdd(varname, VAR_TYPE_FLOW_VAR);
+    fd->idx = VarNameStoreRegister(varname, VAR_TYPE_FLOW_VAR);
 
     /* Okay so far so good, lets get this into a SigMatch
      * and put it in the Signature. */
@@ -267,6 +270,7 @@ int DetectFlowvarPostMatchSetup(DetectEngineCtx *de_ctx, Signature *s, uint32_t
 
     /* we only need the idx */
     fv->idx = idx;
+    fv->post_match = true;
 
     sm = SigMatchAlloc();
     if (unlikely(sm == NULL))

src/detect-flowvar.h

@@ -28,8 +28,10 @@ typedef struct DetectFlowvarData_ {
     char *name;
     uint32_t idx;
     uint8_t *content;
-    uint8_t content_len;
-    uint8_t flags;
+    uint16_t content_len;
+    /** set to true if used in a post-match */
+    bool post_match;
+    uint32_t flags;
 } DetectFlowvarData;
 
 /* prototypes */

src/detect-hostbits.c

@@ -387,7 +387,7 @@ int DetectHostbitSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     if (unlikely(cd == NULL))
         goto error;
 
-    cd->idx = VarNameStoreSetupAdd(fb_name, VAR_TYPE_HOST_BIT);
+    cd->idx = VarNameStoreRegister(fb_name, VAR_TYPE_HOST_BIT);
     cd->cmd = fb_cmd;
     cd->tracker = hb_dir;
     cd->type = VAR_TYPE_HOST_BIT;
@@ -443,6 +443,7 @@ void DetectHostbitFree (DetectEngineCtx *de_ctx, void *ptr)
 
     if (fd == NULL)
         return;
+    VarNameStoreUnregister(fd->idx, VAR_TYPE_HOST_BIT);
 
     SCFree(fd);
 }
@@ -760,8 +761,8 @@ static int HostBitsTestSig04(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"isset option\"; hostbits:isset,fbt; content:\"GET \"; sid:1;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("fbt", VAR_TYPE_HOST_BIT);
-    FAIL_IF(idx != 1);
+    idx = VarNameStoreRegister("fbt", VAR_TYPE_HOST_BIT);
+    FAIL_IF(idx == 0);
 
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);