Skip to content

Output alert applayer v13.3 #9807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
catenacyber wants to merge 12 commits into from
Closed

Output alert applayer v13.3 #9807

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
da33595
output/alert: rewrite code for app-layer properties
catenacyber May 11, 2023
405d00c
output/ftp: have ftp properties in alerts
catenacyber Nov 16, 2023
604b682
output/tftp: have tftp properties in alerts
catenacyber Nov 16, 2023
172a9bc
output/krb5: have krb5 properties in alerts
catenacyber Nov 16, 2023
7c7aa8a
output/dnp3: restrict function scope to one file
catenacyber May 15, 2023
1acb710
output/dns: do not add empty app-layer metadata
catenacyber Jun 5, 2023
4391e05
output: generic simple tx json logger
catenacyber May 14, 2023
5636269
app-layer: do not require probing parser
catenacyber Nov 16, 2023
3ddee5e
detect: make number of keywords dynamic
catenacyber Nov 16, 2023
e6637ac
detect: helper to have pure rust keywords
catenacyber Nov 16, 2023
acd4ac0
app-layer: allow at compile-time more app-layer
catenacyber Nov 16, 2023
0143503
plugins: app-layer plugins
catenacyber Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/app-layer-parser.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@
#include "app-layer-rdp.h"
#include "app-layer-http2.h"

#ifdef ALPROTO_DYNAMIC_NB
#include "util-plugin.h"
#endif

struct AppLayerParserThreadCtx_ {
void *alproto_local_storage[FLOW_PROTO_MAX][ALPROTO_MAX];
};
Expand Down Expand Up @@ -1789,6 +1793,16 @@ void AppLayerParserRegisterProtocolParsers(void)
"imap");
}

#ifdef ALPROTO_DYNAMIC_NB
for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) {
SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i);
if (app_layer_plugin == NULL) {
break;
}
app_layer_plugin->Register();
}
#endif

ValidateParsers();
return;
}
Expand Down
2 changes: 1 addition & 1 deletion src/app-layer-protos.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ AppProto StringToAppProto(const char *proto_name)

// We could use a Multi Pattern Matcher
for (size_t i = 0; i < ARRAY_SIZE(AppProtoStrings); i++) {
if (strcmp(proto_name, AppProtoStrings[i].str) == 0)
if (AppProtoStrings[i].str != NULL && strcmp(proto_name, AppProtoStrings[i].str) == 0)
return AppProtoStrings[i].alproto;
}

Expand Down
11 changes: 11 additions & 0 deletions src/detect-engine-file.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,15 @@ uint8_t DetectFileInspectGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx
const struct DetectEngineAppInspectionEngine_ *engine, const Signature *s, Flow *f,
uint8_t flags, void *_alstate, void *tx, uint64_t tx_id);

// file protocols with common file handling
typedef struct {
AppProto al_proto;
int direction;
int to_client_progress;
int to_server_progress;
} DetectFileHandlerProtocol_t;

void DetectFileRegisterProto(
AppProto alproto, int direction, int to_client_progress, int to_server_progress);

#endif /* __DETECT_ENGINE_FILE_H__ */
49 changes: 48 additions & 1 deletion src/detect-engine-register.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,7 @@
#include "util-path.h"
#include "util-mpm-ac.h"
#include "runmodes.h"
#include "util-plugin.h"

int DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC;
int DETECT_TBLSIZE_IDX = DETECT_TBLSIZE_STATIC;
Expand Down Expand Up @@ -452,14 +453,48 @@ int SigTableList(const char *keyword)

static void DetectFileHandlerRegister(void)
{
for (int i = 0; i < DETECT_TBLSIZE; i++) {
for (int i = 0; i < DETECT_TBLSIZE_STATIC; i++) {
if (filehandler_table[i].name)
DetectFileRegisterFileProtocols(&filehandler_table[i]);
}
}

void SigTableCleanup(void)
{
if (sigmatch_table != sigmatch_table_static) {
SCFree(sigmatch_table);
sigmatch_table = sigmatch_table_static;
DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC;
}
}

void SigTableSetup(void)
{
#ifdef ALPROTO_DYNAMIC_NB
bool to_grow = false;
for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) {
SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i);
if (app_layer_plugin == NULL) {
break;
}
if (app_layer_plugin->keywords_nb > 0) {
DETECT_TBLSIZE += app_layer_plugin->keywords_nb;
to_grow = true;
}
}
if (to_grow) {
if (sigmatch_table == sigmatch_table_static) {
sigmatch_table = SCMalloc(DETECT_TBLSIZE * sizeof(SigTableElmt));
} else {
sigmatch_table = SCRealloc(sigmatch_table, DETECT_TBLSIZE * sizeof(SigTableElmt));
}
if (sigmatch_table == NULL) {
SCLogError("Failed to allocate bigger sigmatch_table, falling back to static one");
sigmatch_table = sigmatch_table_static;
DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC;
}
}
#endif
memset(sigmatch_table, 0, DETECT_TBLSIZE * sizeof(SigTableElmt));

DetectSidRegister();
Expand Down Expand Up @@ -687,6 +722,18 @@ void SigTableSetup(void)
DetectQuicCyuHashRegister();
DetectQuicCyuStringRegister();

#ifdef ALPROTO_DYNAMIC_NB
for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) {
SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i);
if (app_layer_plugin == NULL) {
break;
}
if (app_layer_plugin->KeywordsRegister != NULL) {
app_layer_plugin->KeywordsRegister();
}
}
#endif

DetectBypassRegister();
DetectConfigRegister();

Expand Down
1 change: 1 addition & 0 deletions src/detect-engine-register.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,7 @@ enum DetectKeywordId {
extern int DETECT_TBLSIZE;
extern int DETECT_TBLSIZE_IDX;
int SigTableList(const char *keyword);
void SigTableCleanup(void);
void SigTableSetup(void);
void SigTableRegisterTests(void);

Expand Down
55 changes: 32 additions & 23 deletions src/detect-parse.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
#include "string.h"
#include "detect-parse.h"
#include "detect-engine-iponly.h"
#include "detect-engine-file.h"
#include "app-layer-detect-proto.h"

#include "action-globals.h"
Expand All @@ -76,31 +77,39 @@
/* Table with all filehandler registrations */
DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE_STATIC];

void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg)
// file protocols with common file handling
DetectFileHandlerProtocol_t al_protocols[ALPROTO_MAX] = {
{ .al_proto = ALPROTO_NFS, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_SMB, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_FTP, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_FTPDATA, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_HTTP1,
.direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT,
.to_client_progress = HTP_RESPONSE_BODY,
.to_server_progress = HTP_REQUEST_BODY },
{ .al_proto = ALPROTO_HTTP2,
.direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT,
.to_client_progress = HTTP2StateDataServer,
.to_server_progress = HTTP2StateDataClient },
{ .al_proto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER }
};

void DetectFileRegisterProto(
AppProto alproto, int direction, int to_client_progress, int to_server_progress)
{
// file protocols with common file handling
typedef struct {
AppProto al_proto;
int direction;
int to_client_progress;
int to_server_progress;
} DetectFileHandlerProtocol_t;
static DetectFileHandlerProtocol_t al_protocols[] = {
{ .al_proto = ALPROTO_NFS, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_SMB, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_FTP, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_FTPDATA, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT },
{ .al_proto = ALPROTO_HTTP1,
.direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT,
.to_client_progress = HTP_RESPONSE_BODY,
.to_server_progress = HTP_REQUEST_BODY },
{ .al_proto = ALPROTO_HTTP2,
.direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT,
.to_client_progress = HTTP2StateDataServer,
.to_server_progress = HTTP2StateDataClient },
{ .al_proto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER }
};
size_t i = 0;
while (al_protocols[i].al_proto != ALPROTO_UNKNOWN) {
i++;
}
al_protocols[i].al_proto = alproto;
al_protocols[i].direction = direction;
al_protocols[i].to_client_progress = to_client_progress;
al_protocols[i].to_server_progress = to_server_progress;
al_protocols[i + 1].al_proto = ALPROTO_UNKNOWN;
}

void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg)
{
for (size_t i = 0; i < ARRAY_SIZE(al_protocols); i++) {
int direction = al_protocols[i].direction == 0
? (int)(SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)
Expand Down
1 change: 1 addition & 0 deletions src/detect.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1546,6 +1546,7 @@ typedef struct DetectEngineMasterCtx_ {

/* Table with all SigMatch registrations */
extern SigTableElmt *sigmatch_table;
extern SigTableElmt sigmatch_table_static[DETECT_TBLSIZE_STATIC];

/** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */

Expand Down
8 changes: 8 additions & 0 deletions src/output.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1346,3 +1346,11 @@ SimpleJsonAppLayerLogger *GetAppProtoSimpleJsonLogger(AppProto alproto)
}
return NULL;
}

void RegisterAppProtoAppLayerLogger(AppProto alproto, SimpleJsonAppLayerLogger *log)
{
if (alproto < ALPROTO_MAX) {
simple_json_applayer_loggers[alproto].proto = alproto;
simple_json_applayer_loggers[alproto].LogTx = log;
}
}
1 change: 1 addition & 0 deletions src/output.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,5 +216,6 @@ typedef struct SimpleJsonAppLayerLogger {
} SimpleJsonAppLayerLogger;

SimpleJsonAppLayerLogger *GetAppProtoSimpleJsonLogger(AppProto alproto);
void RegisterAppProtoAppLayerLogger(AppProto alproto, SimpleJsonAppLayerLogger *log);

#endif /* ! __OUTPUT_H__ */
12 changes: 12 additions & 0 deletions src/suricata-plugin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,4 +74,16 @@ typedef struct SCCapturePlugin_ {

int SCPluginRegisterCapture(SCCapturePlugin *);

typedef struct SCAppLayerPlugin_ {
char *name;
char *logname;
char *confname;
void (*Register)(void);
bool (*Logger)(void *tx, void *jb);
void (*KeywordsRegister)(void);
uint32_t keywords_nb;
} SCAppLayerPlugin;

int SCPluginRegisterAppLayer(SCAppLayerPlugin *);

#endif /* __SURICATA_PLUGIN_H */
7 changes: 4 additions & 3 deletions src/suricata.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,7 @@ static void GlobalsDestroy(SCInstance *suri)
FeatureTrackingRelease();
SCProtoNameRelease();
TimeDeinit();
SigTableCleanup();
TmqhCleanup();
TmModuleRunDeInit();
ParseSizeDeinit();
Expand Down Expand Up @@ -2675,6 +2676,9 @@ int PostConfLoadedSetup(SCInstance *suri)

RunModeEngineIsIPS(
suricata.run_mode, suricata.runmode_custom_mode, suricata.capture_plugin_name);
#ifdef HAVE_PLUGINS
SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args);
#endif

if (EngineModeIsUnknown()) { // if still uninitialized, set the default
SCLogInfo("Setting engine mode to IDS mode by default");
Expand Down Expand Up @@ -2742,9 +2746,6 @@ int PostConfLoadedSetup(SCInstance *suri)

FeatureTrackingRegister(); /* must occur prior to output mod registration */
RegisterAllModules();
#ifdef HAVE_PLUGINS
SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args);
#endif
AppLayerHtpNeedFileInspection();

StorageFinalize();
Expand Down
29 changes: 29 additions & 0 deletions src/util-plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "output-eve-syslog.h"
#include "util-plugin.h"
#include "util-debug.h"
#include "app-layer-protos.h"

#ifdef HAVE_PLUGINS

Expand Down Expand Up @@ -211,4 +212,32 @@ SCCapturePlugin *SCPluginFindCaptureByName(const char *name)
}
return plugin;
}

#ifdef ALPROTO_DYNAMIC_NB
static SCAppLayerPlugin *app_layer_plugins[ALPROTO_DYNAMIC_NB];
static size_t app_layer_plugins_nb = 0;
#endif

int SCPluginRegisterAppLayer(SCAppLayerPlugin *plugin)
Copy link
Copy Markdown
Member

@jasonish jasonish Nov 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer to not think about registering an app-layer as a plugin. Could app-layers not be registered the same being included or as plugins? For plugin support we just expose the API that Suricata would already be using.

Copy link
Copy Markdown
Contributor Author

@catenacyber catenacyber Nov 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not understand your sentences here... Could you rephrase ?

(Plus I need to push a plugin example)

Copy link
Copy Markdown
Member

@jasonish jasonish Nov 16, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why couldn't a non-plugin AppLayer not register itself through this API? In which case its no longer a plugin API, its just the API for registering app-layers. In short, putting Plugin in here seems a little artificial.

It was a mistake in the EVE file type API which I've already started to undo.

Copy link
Copy Markdown
Contributor Author

@catenacyber catenacyber Nov 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, so this is just about renaming SCPluginRegisterAppLayer to SCRegisterDynamicAppLayer, do I get it ?

Would we want to get rid of all the static ones ?

Copy link
Copy Markdown
Member

@jasonish jasonish Nov 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would we want to get rid of all the static ones ?

Personally I'd think so unless there is a performance hit?

I like to think of plugin support as just exposing the APIs we already use internally. Then there is nothing really different from a built-in or plugin, and we're constantly dog-fooding the APIs.

Copy link
Copy Markdown
Contributor Author

@catenacyber catenacyber Nov 23, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And so, should this be moved out of util-plugin.c to app-layer-register.c ?

{
#ifdef ALPROTO_DYNAMIC_NB
if (app_layer_plugins_nb < ALPROTO_DYNAMIC_NB) {
app_layer_plugins[app_layer_plugins_nb] = plugin;
RegisterAppProtoString((AppProto)(ALPROTO_MAX_STATIC + app_layer_plugins_nb), plugin->name);
app_layer_plugins_nb++;
return 0;
}
#endif
return 1;
}

SCAppLayerPlugin *SCPluginFindAppLayerByIndex(size_t i)
{
#ifdef ALPROTO_DYNAMIC_NB
if (i < ALPROTO_DYNAMIC_NB) {
return app_layer_plugins[i];
}
#endif
return NULL;
}
#endif
2 changes: 2 additions & 0 deletions src/util-plugin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,6 @@ SCCapturePlugin *SCPluginFindCaptureByName(const char *name);

bool RegisterPlugin(SCPlugin *, void *);

SCAppLayerPlugin *SCPluginFindAppLayerByIndex(size_t i);

#endif /* __UTIL_PLUGIN_H__ */