From 35722bae1c324e6cd982c14e69f1d84731044987 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Mon, 25 May 2026 10:25:15 +0300 Subject: [PATCH 1/9] Combine on_send and on_tcp_entail callbacks into a union MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `on_tcp_entail` is always called after `on_send`, so both callbacks can be combined into a single union, freeing an additional 8 bytes for callback private data. This private data will be used to store a pointer to the request, allowing us in the future to retrieve the request in `on_tcp_entail` from the skb head before it is enqueued into the socket write queue, in order to decide whether to drop or entail it. We decided to do so because the `TFW_SKB_CB` structure has very limited space, and there are no extra 8 bytes available to store a request pointer. We also cannot use the `opaque_data` pointer in `TFW_SKB_CB` to store a request pointer and implement a corresponding skb destructor, because if the server connection is destroyed, the destructor may be invoked from `ss_do_send -> ss_skb_queue_purge` or `ss_tx_action -> ss_skb_queue_purge`. In such cases, accessing the request structure may result in a use-after-free, since the request may already have been freed during server connection cleanup. This is a preparatory step toward fixing excessive memory usage caused by forwarded requests whose client connection has already been closed but remain in the server’s forwarding queue. These requests should be dropped before being placed into the server socket’s write queue. Part-of: release-server-queued-requests Issue: #2284 --- fw/http_frame.c | 16 +++++++++++----- fw/ss_skb.h | 33 ++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/fw/http_frame.c b/fw/http_frame.c index 1988bc583..415afd94d 100644 --- a/fw/http_frame.c +++ b/fw/http_frame.c @@ -294,6 +294,15 @@ tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head) return 0; } +static int +tfw_h2_on_send_ack(void *conn, struct sk_buff** skb_head) +{ + TFW_SKB_CB(*skb_head)->on_tcp_entail = tfw_h2_on_tcp_entail_ack; + tfw_h2_on_send_dflt(conn, skb_head); + + return 0; +} + /** * Prepare and send HTTP/2 frame to the client; @hdr must contain * the valid data to fill in the frame's header; @data may carry @@ -349,15 +358,12 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data, } else if (hdr->type == HTTP2_RST_STREAM) { TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_rst_stream; TFW_SKB_CB(msg.skb_head)->stream_id = hdr->stream_id; + } else if (hdr->type == HTTP2_SETTINGS && hdr->flags == HTTP2_F_ACK) { + TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_ack; } else { TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_dflt; } - if (hdr->type == HTTP2_SETTINGS && hdr->flags == HTTP2_F_ACK) { - TFW_SKB_CB(msg.skb_head)->on_tcp_entail = - tfw_h2_on_tcp_entail_ack; - } - if ((r = tfw_connection_send(conn, &msg))) goto err; /* diff --git a/fw/ss_skb.h b/fw/ss_skb.h index 5e78a4511..3cc495105 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -34,12 +34,14 @@ typedef struct tfw_client_mem_t TfwClientMem; /* * Tempesta FW sk_buff private data. - * @opaque_data - pointer to some private data (typically http response); + * @opaque_data - pointer to some private data (typically TfwClientMem); * @destructor - destructor of the opaque data, should be set if data is - * not NULL + * not NULL; * @on_send - callback to special handling this skb before sending; * @on_tcp_entail - callback to special handling this skb before pushing * to socket write queue; + * @on_send_data - pointer to some private data for `on_send` callback; + * @on_tcp_entail_data - pointer to some private data for `on_tcp_entail` callback; * @mem - memory used for this skb, used to account appropriate * client memory; * @stream_id - id of sender stream; @@ -48,13 +50,19 @@ typedef struct tfw_client_mem_t TfwClientMem; * @is_head - flag indicates that this is a head of skb list; */ struct tfw_skb_cb { - void *opaque_data; - void (*destructor)(struct sk_buff *); - on_send_cb_t on_send; - on_tcp_entail_t on_tcp_entail; - long int mem; - unsigned int stream_id; - bool is_head; + void *opaque_data; + void (*destructor)(struct sk_buff *); + union { + on_send_cb_t on_send; + on_tcp_entail_t on_tcp_entail; + }; + union { + void *on_send_data; + void *on_tcp_entail_data; + }; + long int mem; + unsigned int stream_id; + bool is_head; }; #define TFW_SKB_CB(skb) ((struct tfw_skb_cb *)&((skb)->cb[0])) @@ -105,6 +113,13 @@ ss_skb_on_send(void *conn, struct sk_buff **skb_head) on_send_cb_t on_send = TFW_SKB_CB(*skb_head)->on_send; int r = 0; + /* + * `on_send` pointer is located inside the union with `on_tcp_entail` + * callback, so we should zero it. For the same reason, we reset + * `on_send_data` pointer here. + */ + TFW_SKB_CB(*skb_head)->on_send = NULL; + TFW_SKB_CB(*skb_head)->on_send_data = NULL; if (on_send) r = on_send(conn, skb_head); if (!r && *skb_head) From bd6f800e465d3ec55b9222a117a78d5e49863c98 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Mon, 25 May 2026 13:46:24 +0300 Subject: [PATCH 2/9] Rework skb->cb callbacks and skb->destructor - Remove `opaque_data` from `skb->cb`, now use TfwClientMem instead. Use only default skb destructor to decrement client memory when skb is freed. - Save http2 response in `on_send_data`. Pass NULL (as a conn) in `on_send` callback, if appropriate socket is not active or conn is real NULL (previously we don't call `on_send` callback in such case). Destroy `on_send_data` directly in `on_send` callack if we can't send data (conn is NULL). This patch simplifies the code and makes it consistent with the previous changes to the `TFW_SKB_CB` structure, which already provides dedicated storage for data associated with the `on_send` and `on_tcp_entail` callbacks. Response objects are now stored in this dedicated storage. This allows us to apply the same approach to requests, allowing them to be accessed safely from the `on_send_data` and `on_tcp_entail_data` callbacks. Part-of: release-server-queued-requests Issue: #2284 --- fw/connection.h | 8 +----- fw/http.c | 61 +++++++++++++++++++------------------------ fw/http.h | 3 ++- fw/http_frame.c | 38 ++++++++++++++++++++------- fw/http_stream.c | 11 +------- fw/sock.c | 11 ++++---- fw/ss_skb.c | 23 ++++------------- fw/ss_skb.h | 67 +++++++++++++----------------------------------- fw/tls.c | 6 ++++- 9 files changed, 92 insertions(+), 136 deletions(-) diff --git a/fw/connection.h b/fw/connection.h index fca74a0a5..282480dad 100644 --- a/fw/connection.h +++ b/fw/connection.h @@ -438,16 +438,10 @@ tfw_connection_live(TfwConn *conn) #define tfw_srv_conn_live(c) tfw_connection_live((TfwConn *)(c)) -static inline void -tfw_connection_get_many(TfwConn *conn, int cnt) -{ - atomic_add(cnt, &conn->refcnt); -} - static inline void tfw_connection_get(TfwConn *conn) { - tfw_connection_get_many(conn, 1); + atomic_add(1, &conn->refcnt); } #define TFW_CONNETION_GET_IF(name, cond) \ diff --git a/fw/http.c b/fw/http.c index 7cf2f90f0..5d81b5092 100644 --- a/fw/http.c +++ b/fw/http.c @@ -1146,14 +1146,6 @@ tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status, return 0; } -static inline bool -tfw_http_resp_check_skb_head_owner(TfwHttpResp *resp) -{ - return (TFW_SKB_CB(resp->msg.skb_head)->opaque_data == - CLIENT_MEM_FROM_CONN(resp->req->conn)) - && ss_skb_has_dflt_destructor(resp->msg.skb_head); -} - void tfw_h2_resp_fwd(TfwHttpResp *resp) { @@ -1165,30 +1157,26 @@ tfw_h2_resp_fwd(TfwHttpResp *resp) * `tfw_h2_stream_init_for_xmit` should be called for * response before send it to the client. */ - if (WARN_ON(!TFW_SKB_CB(resp->msg.skb_head)->stream_id - || !tfw_http_resp_check_skb_head_owner(resp))) + if (WARN_ON(!TFW_SKB_CB(resp->msg.skb_head)->stream_id)) return; /* - * We need this extra get, because if send fails, connection - * will be put during freeing skbs of sending response (in - * skb destructor). + * Connection reference counter will be decremented and response will + * be freed in `on_send` callback, if it fails or after encoding headers + * in `tfw_h2_stream_xmit_process`. */ - tfw_connection_get_many(conn, 2); - __ss_skb_set_owner(resp->msg.skb_head, tfw_h2_stream_skb_destructor, - resp); + tfw_connection_get(conn); do_access_log(resp); if (tfw_cli_conn_send((TfwCliConn *)conn, (TfwMsg *)resp)) { T_DBG("%s: cannot send data to client via HTTP/2\n", __func__); TFW_INC_STAT_BH(serv.msgs_otherr); tfw_connection_close(conn, true); + tfw_http_resp_pair_free_and_put_conn(resp); } else { TFW_INC_STAT_BH(serv.msgs_forwarded); tfw_inc_global_hm_stats(status); } - - tfw_connection_put(conn); } /* @@ -4626,19 +4614,16 @@ tfw_http_resp_get_conn_flags(TfwHttpResp *resp) static int tfw_http_resp_set_empty_skb_head(TfwHttpResp *resp, TfwHttpMsgCleanup *cleanup) { - void *opaque_data = TFW_SKB_CB(resp->msg.skb_head)->opaque_data; + TfwClientMem *cli_mem = TFW_SKB_CB(resp->msg.skb_head)->cli_mem; TfwMsgIter *iter = &resp->iter; struct sk_buff *nskb; - if (WARN_ON(!tfw_http_resp_check_skb_head_owner(resp))) - return -EINVAL; - nskb = ss_skb_alloc(0); if (unlikely(!nskb)) return -ENOMEM; ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - opaque_data, nskb->truesize); + cli_mem, nskb->truesize); nskb->mark = resp->msg.skb_head->mark; cleanup->skb_head = resp->msg.skb_head; resp->msg.skb_head = NULL; @@ -5469,13 +5454,19 @@ tfw_h2_append_predefined_body(TfwHttpResp *resp, const TfwStr *body) ALLOW_ERROR_INJECTION(tfw_h2_append_predefined_body, ERRNO); int -tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head) +tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head, + void *on_send_data) { - TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); - TfwHttpResp *resp = TFW_SKB_CB(*skb_head)->opaque_data; - unsigned int stream_id = TFW_SKB_CB(*skb_head)->stream_id; + TfwHttpResp *resp = (TfwHttpResp *)on_send_data; + TfwH2Ctx *ctx; + unsigned int stream_id; TfwStream *stream; + if (unlikely(!conn)) + goto fail; + + ctx = tfw_h2_context_unsafe((TfwConn *)conn); + stream_id = TFW_SKB_CB(*skb_head)->stream_id; stream = tfw_h2_find_not_closed_stream(ctx, stream_id, false); /* * Very unlikely case. We check that stream is active, before @@ -5484,7 +5475,7 @@ tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head) * before ss_do_send was called. */ if (unlikely(!stream)) - return -EPIPE; + goto fail; /* * These pointers are zeroed in `tfw_h2_stream_init_for_xmit`. @@ -5492,14 +5483,10 @@ tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head) * that we can catch memory corruption. During client stream or * connection closing we check stream->xmit.resp in * `tfw_h2_stream_purge_all_and_free_response` and if it is - * not zero free it). If this function returns error we also - * free response in skb destructor, so if stream or connection - * is already closed here catch double free of response. + * not zero free it). In case of invalid response pointer we + * catch BUG later. */ BUG_ON(stream->xmit.skb_head || stream->xmit.resp); - - __ss_skb_set_owner(*skb_head, ss_skb_dflt_destructor, - CLIENT_MEM_FROM_CONN(resp->req->conn)); stream->xmit.resp = resp; if (test_bit(TFW_HTTP_B_CLOSE_ERROR_RESPONSE, stream->xmit.resp->flags)) @@ -5510,6 +5497,10 @@ tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head) tfw_h2_sched_activate_stream(&ctx->sched, stream); return 0; + +fail: + tfw_http_resp_pair_free_and_put_conn(resp); + return -EPIPE; } /** @@ -6627,7 +6618,7 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb, * For tls connections we already set `skb->owner` before * tls decryption. */ - if (!TFW_SKB_CB(skb)->opaque_data) { + if (!TFW_SKB_CB(skb)->cli_mem) { ss_skb_set_owner(skb, ss_skb_dflt_destructor, CLIENT_MEM_FROM_CONN(conn), skb->truesize); diff --git a/fw/http.h b/fw/http.h index c77100f83..1f7712336 100644 --- a/fw/http.h +++ b/fw/http.h @@ -814,6 +814,7 @@ int tfw_http_resp_copy_encodings(TfwHttpResp *resp, TfwStr* dst, void tfw_http_extract_request_authority(TfwHttpReq *req); bool tfw_http_mark_is_in_whitlist(unsigned int mark); char *tfw_http_resp_status_line(int status, size_t *len); -int tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head); +int tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head, + void *on_send_data); #endif /* __TFW_HTTP_H__ */ diff --git a/fw/http_frame.c b/fw/http_frame.c index 415afd94d..d3241b74c 100644 --- a/fw/http_frame.c +++ b/fw/http_frame.c @@ -237,10 +237,15 @@ tfw_h2_on_tcp_entail_ack(void *conn, struct sk_buff *skb_head) } static int -tfw_h2_on_send_goaway(void *conn, struct sk_buff **skb_head) +tfw_h2_on_send_goaway(void *conn, struct sk_buff **skb_head, + void *on_send_data) { - TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); + TfwH2Ctx *ctx; + + if (unlikely(!conn)) + return -EPIPE; + ctx = tfw_h2_context_unsafe((TfwConn *)conn); if (ctx->error && ctx->error->xmit.skb_head) { ss_skb_queue_splice(&ctx->error->xmit.skb_head, skb_head); } else if (ctx->cur_send_headers) { @@ -257,12 +262,18 @@ tfw_h2_on_send_goaway(void *conn, struct sk_buff **skb_head) } static int -tfw_h2_on_send_rst_stream(void *conn, struct sk_buff **skb_head) +tfw_h2_on_send_rst_stream(void *conn, struct sk_buff **skb_head, + void *on_send_data) { - TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); - unsigned int stream_id = TFW_SKB_CB(*skb_head)->stream_id; + TfwH2Ctx *ctx; + unsigned int stream_id; TfwStream *stream; + if (unlikely(!conn)) + return -EPIPE; + + ctx = tfw_h2_context_unsafe((TfwConn *)conn); + stream_id = TFW_SKB_CB(*skb_head)->stream_id; stream = tfw_h2_find_not_closed_stream(ctx, stream_id, false); /* @@ -282,10 +293,15 @@ tfw_h2_on_send_rst_stream(void *conn, struct sk_buff **skb_head) } static int -tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head) +tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head, + void *on_send_data) { - TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); + TfwH2Ctx *ctx; + + if (unlikely(!conn)) + return -EPIPE; + ctx = tfw_h2_context_unsafe((TfwConn *)conn); if (ctx->cur_send_headers) { ss_skb_queue_splice(&ctx->cur_send_headers->xmit.postponed, skb_head); @@ -295,10 +311,14 @@ tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head) } static int -tfw_h2_on_send_ack(void *conn, struct sk_buff** skb_head) +tfw_h2_on_send_ack(void *conn, struct sk_buff** skb_head, + void *on_send_data) { + if (unlikely(!conn)) + return -EPIPE; + TFW_SKB_CB(*skb_head)->on_tcp_entail = tfw_h2_on_tcp_entail_ack; - tfw_h2_on_send_dflt(conn, skb_head); + tfw_h2_on_send_dflt(conn, skb_head, on_send_data); return 0; } diff --git a/fw/http_stream.c b/fw/http_stream.c index 923ea20e0..e72480609 100644 --- a/fw/http_stream.c +++ b/fw/http_stream.c @@ -812,16 +812,6 @@ tfw_h2_delete_stream(TfwH2Ctx *ctx, TfwStream *stream) kmem_cache_free(stream_cache, stream); } -void -tfw_h2_stream_skb_destructor(struct sk_buff *skb) -{ - TfwHttpResp *resp = (TfwHttpResp *)TFW_SKB_CB(skb)->opaque_data; - - TFW_SKB_CB(skb)->opaque_data = CLIENT_MEM_FROM_CONN(resp->req->conn); - ss_skb_dflt_destructor(skb); - tfw_http_resp_pair_free_and_put_conn(resp); -} - int tfw_h2_stream_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state, unsigned long h_len, unsigned long b_len) @@ -839,6 +829,7 @@ tfw_h2_stream_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state, } TFW_SKB_CB(skb_head)->on_send = tfw_h2_on_send_resp; + TFW_SKB_CB(skb_head)->on_send_data = resp; TFW_SKB_CB(skb_head)->stream_id = stream->id; stream->xmit.resp = NULL; diff --git a/fw/sock.c b/fw/sock.c index d66ace4ea..4ae1a6d96 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -562,6 +562,7 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags) void *conn = sk->sk_user_data; unsigned char tls_type = flags & SS_F_ENCRYPT ? SS_SKB_F2TYPE(flags) : 0; + bool send_is_allowed = conn && ss_sock_active(sk); T_DBG3("[%d]: %s: sk=%pK queue_empty=%d send_head=%pK" " sk_state=%d\n", @@ -569,13 +570,9 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags) sk, tcp_write_queue_empty(sk), tcp_send_head(sk), sk->sk_state); - /* If the socket is inactive, there's no recourse. Drop the data. */ - if (unlikely(!conn || !ss_sock_active(sk))) - goto cleanup; - ss_skb_setup_head_of_list(*skb_head, (*skb_head)->mark, tls_type); - if (ss_skb_on_send(conn, skb_head)) + if (ss_skb_on_send(likely(send_is_allowed) ? conn : NULL, skb_head)) goto cleanup; if (flags & SS_F_CONN_CLOSE) @@ -677,7 +674,7 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) SKB_DATA_ALIGN(head_data + sizeof(struct skb_shared_info)); ss_skb_set_owner(twin_skb, ss_skb_dflt_destructor, - TFW_SKB_CB(skb)->opaque_data, + TFW_SKB_CB(skb)->cli_mem, copied_truesize); ss_skb_queue_tail(&sw.skb_head, twin_skb); skb = skb->next; @@ -1776,6 +1773,8 @@ ss_tx_action(void) } dead_sock: sock_put(sk); /* paired with push() calls */ + if (sw.skb_head) + ss_skb_on_send(NULL, &sw.skb_head); ss_skb_queue_purge(&sw.skb_head); } diff --git a/fw/ss_skb.c b/fw/ss_skb.c index d8cba68e3..c4f74606b 100644 --- a/fw/ss_skb.c +++ b/fw/ss_skb.c @@ -221,18 +221,8 @@ __extend_pgfrags(struct sk_buff *skb_head, struct sk_buff *skb, int from, int n) return -ENOMEM; if (!skb_tfw_is_in_socket_write_queue(skb)) { - /* - * If skb destructor is not default, that means - * that skb `opaque_data` is not a `client_mem`. - * In this case we can't call `ss_skb_set_owner` - * because this function use `opaque_data` as - * a `client_mem` structure and we catch memory - * corruption. - */ - BUG_ON(TFW_SKB_CB(skb)->destructor - && !ss_skb_has_dflt_destructor(skb)); ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - TFW_SKB_CB(skb)->opaque_data, + TFW_SKB_CB(skb)->cli_mem, nskb->truesize); } skb_shinfo(nskb)->flags = skb_shinfo(skb)->flags; @@ -1735,16 +1725,13 @@ int ss_skb_realloc_headroom(struct sk_buff *skb) { int delta = MAX_TCP_HEADER - skb_headroom(skb); - bool skb_has_owner = !!TFW_SKB_CB(skb)->opaque_data; + bool skb_has_owner = !!TFW_SKB_CB(skb)->cli_mem; unsigned int old_truesize; int r; if (likely(delta <= 0)) return 0; - if (WARN_ON(skb_has_owner && !ss_skb_has_dflt_destructor(skb))) - return -EINVAL; - if (skb_has_owner) old_truesize = skb->truesize; @@ -1763,7 +1750,7 @@ void ss_skb_dflt_destructor(struct sk_buff *skb) { TfwClientMem *cli_mem = - (TfwClientMem *)TFW_SKB_CB(skb)->opaque_data; + (TfwClientMem *)TFW_SKB_CB(skb)->cli_mem; /* * If skb is in socket write queue, skb->cb is used to store @@ -1792,7 +1779,7 @@ ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), return; WARN_ON(TFW_SKB_CB(skb)->mem != 0); - WARN_ON(TFW_SKB_CB(skb)->destructor || TFW_SKB_CB(skb)->opaque_data); + WARN_ON(TFW_SKB_CB(skb)->destructor || TFW_SKB_CB(skb)->cli_mem); __ss_skb_set_owner(skb, destructor, owner); ss_skb_adjust_client_mem(skb, mem); } @@ -1805,7 +1792,7 @@ ss_skb_adjust_client_mem(struct sk_buff *skb, int delta) if (skb_tfw_is_in_socket_write_queue(skb)) return; - cli_mem = (TfwClientMem *)TFW_SKB_CB(skb)->opaque_data; + cli_mem = (TfwClientMem *)TFW_SKB_CB(skb)->cli_mem; if (cli_mem) { TFW_SKB_CB(skb)->mem += delta; WARN_ON(TFW_SKB_CB(skb)->mem < 0); diff --git a/fw/ss_skb.h b/fw/ss_skb.h index 3cc495105..fbe80aa31 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -27,16 +27,18 @@ #include "str.h" -typedef int (*on_send_cb_t)(void *conn, struct sk_buff **skb_head); +typedef int (*on_send_cb_t)(void *conn, struct sk_buff **skb_head, + void *on_send_data); typedef void (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head); typedef void (*on_send_fail_cb_t)(void *conn, struct sk_buff *skb_head); typedef struct tfw_client_mem_t TfwClientMem; /* * Tempesta FW sk_buff private data. - * @opaque_data - pointer to some private data (typically TfwClientMem); - * @destructor - destructor of the opaque data, should be set if data is - * not NULL; + * @cli_mem - pointer to TfwClientMem structure for memory + * accountion; + * @destructor - destructor to adjust memory when skb is freed, should + * be set if `cli_mem` is not NULL; * @on_send - callback to special handling this skb before sending; * @on_tcp_entail - callback to special handling this skb before pushing * to socket write queue; @@ -50,7 +52,7 @@ typedef struct tfw_client_mem_t TfwClientMem; * @is_head - flag indicates that this is a head of skb list; */ struct tfw_skb_cb { - void *opaque_data; + TfwClientMem *cli_mem; void (*destructor)(struct sk_buff *); union { on_send_cb_t on_send; @@ -73,18 +75,12 @@ void ss_skb_adjust_client_mem(struct sk_buff *skb, int delta); void ss_skb_dflt_destructor(struct sk_buff *skb); void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); -static inline bool -ss_skb_has_dflt_destructor(struct sk_buff *skb) -{ - return TFW_SKB_CB(skb)->destructor == ss_skb_dflt_destructor; -} - static inline void __ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), - void *owner) + TfwClientMem *owner) { TFW_SKB_CB(skb)->destructor = destructor; - TFW_SKB_CB(skb)->opaque_data = owner; + TFW_SKB_CB(skb)->cli_mem = owner; } static inline bool @@ -111,6 +107,7 @@ static inline int ss_skb_on_send(void *conn, struct sk_buff **skb_head) { on_send_cb_t on_send = TFW_SKB_CB(*skb_head)->on_send; + void *on_send_data = TFW_SKB_CB(*skb_head)->on_send_data; int r = 0; /* @@ -121,8 +118,9 @@ ss_skb_on_send(void *conn, struct sk_buff **skb_head) TFW_SKB_CB(*skb_head)->on_send = NULL; TFW_SKB_CB(*skb_head)->on_send_data = NULL; if (on_send) - r = on_send(conn, skb_head); - if (!r && *skb_head) + r = on_send(conn, skb_head, on_send_data); + + if (!r && conn && *skb_head) ss_skb_on_send_dflt(conn, skb_head); return r; @@ -214,10 +212,10 @@ ss_skb_orphan(struct sk_buff *skb) if (destructor) { /* * The same BUG_ON is present in linux kernel in `skb_orphan`. - * `skb->opaque_data` will be used inside destructor, so if it + * `skb->cli_mem` will be used inside destructor, so if it * is NULL, we still catch BUG later. */ - BUG_ON(!TFW_SKB_CB(skb)->opaque_data); + BUG_ON(!TFW_SKB_CB(skb)->cli_mem); destructor(skb); __ss_skb_set_owner(skb, NULL, NULL); } else { @@ -225,7 +223,7 @@ ss_skb_orphan(struct sk_buff *skb) * The same BUG_ON is present in linux kernel in * `skb_orphan`. */ - BUG_ON(TFW_SKB_CB(skb)->opaque_data); + BUG_ON(TFW_SKB_CB(skb)->cli_mem); } } @@ -234,13 +232,6 @@ __ss_kfree_skb(struct sk_buff *skb) { if (!skb) return; - /* - * Not default destructor can be set only for the head of skb list, - * which belongs to some data structure and should be called in a - * special way from `ss_skb_queue_purge` to prevent memory corruption. - */ - BUG_ON(TFW_SKB_CB(skb)->destructor - && !ss_skb_has_dflt_destructor(skb)); ss_skb_orphan(skb); __kfree_skb(skb); } @@ -250,13 +241,7 @@ ss_kfree_skb(struct sk_buff *skb) { if (!skb) return; - /* - * Not default destructor can be set only for the head of skb list, - * which belongs to some data structure and should be called in a - * special way from `ss_skb_queue_purge` to prevent memory corruption. - */ - BUG_ON(TFW_SKB_CB(skb)->destructor - && !ss_skb_has_dflt_destructor(skb)); + ss_skb_orphan(skb); kfree_skb(skb); } @@ -368,26 +353,10 @@ ss_skb_dequeue(struct sk_buff **skb_head) static inline void ss_skb_queue_purge(struct sk_buff **skb_head) { - struct sk_buff *skb, *head; - - if (!(head = ss_skb_dequeue(skb_head))) - return; + struct sk_buff *skb; while ((skb = ss_skb_dequeue(skb_head)) != NULL) ss_kfree_skb(skb); - - /* - * We implement a special handling of deleting `head` of the list - * to prevent memory corruption during calling not default skb - * destructor. For example if `head` opaque data points to `response` - * such response can be freed from skb destructor. But `skb_head` - * can also belong to response, so we can't access it after calling - * skb_destructor - so first we should remove `head` from the list, - * destroy all other skbs in the list and after all free the `head` - * of the list. - */ - ss_skb_orphan(head); - kfree_skb(head); } static inline void diff --git a/fw/tls.c b/fw/tls.c index aa0ef03a2..f492a91eb 100644 --- a/fw/tls.c +++ b/fw/tls.c @@ -503,10 +503,14 @@ tfw_tls_close_msg_flags(TlsIOCtx *io) } static inline int -tfw_tls_on_send_alert(void *conn, struct sk_buff **skb_head) +tfw_tls_on_send_alert(void *conn, struct sk_buff **skb_head, + void *on_send_data) { TfwH2Ctx *ctx; + if (unlikely(!conn)) + return -EPIPE; + BUG_ON(TFW_CONN_PROTO((TfwConn *)conn) != TFW_FSM_H2); ctx = tfw_h2_context_safe((TfwConn *)conn); if (!ctx) From 40d9e1f0efad586ca432c7a9b07bb0a75b8e43d4 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Tue, 12 May 2026 19:21:31 +0300 Subject: [PATCH 3/9] Do not forward requests for already dropped client connections. - Rework `on_tcp_entail` callback invocation. Now it is called from `ss_skb_tcp_entail_list` not from `ss_skb_tcp_entail`. If this callback fails we drop all skbs from appropriate list. - Implement `tfw_http_req_on_tcp_entail` callback to check that client connection for sending request is not closed and drop skbs and request if connection is already closed. - Implement `ss_skb_copy_cb` function for correct copying of skb->cb (including all callbacks and destructors). Previously, requests that had already been placed into a server connection's `fwd_queue` were still forwarded to the server even after the originating client connection had been closed. Although the responses to such requests were no longer needed, the requests themselves were still fully transmitted to the backend. This patch changes this behavior by performing an additional client connection state check immediately before entailing a request's skbs to the server socket write queue for transmission. If the client connection has already been closed, the request is discarded together with all associated skbs instead of being sent to the server. The chosen approach has a theoretical disadvantage: if a large number of requests from a reset client connection coexist with many requests from active client connections and the TCP window is small, requests belonging to the inactive client connection may remain in the server connection queue for an extended period of time, consuming memory. An alternative approach would have been to drop the server connection and purge requests belonging to inactive client connections from the server send queue during reconnection. This approach was rejected because it would also impact active client connections by forcing their requests to be retransmitted. Since the server connection is also dropped upon receiving the first response byte for an inactive client connection (see bac89d1), the current implementation provides a practical and effective fix for issue #2284. Part-of: release-server-queued-requests Closes #2284 --- fw/http.c | 245 +++++++++++++++++++++++++++++++++++------------- fw/http2.c | 11 ++- fw/http_frame.c | 4 +- fw/sock.c | 15 ++- fw/ss_skb.c | 8 ++ fw/ss_skb.h | 30 +++--- 6 files changed, 230 insertions(+), 83 deletions(-) diff --git a/fw/http.c b/fw/http.c index 5d81b5092..b7ae38248 100644 --- a/fw/http.c +++ b/fw/http.c @@ -1326,15 +1326,6 @@ tfw_http_req_init_ss_flags(TfwSrvConn *srv_conn, TfwHttpReq *req) ((TfwMsg *)req)->ss_flags |= SS_F_KEEP_SKB; } -static inline void -tfw_http_resp_init_ss_flags(TfwHttpResp *resp) -{ - if (test_bit(TFW_HTTP_B_CONN_CLOSE, resp->req->flags)) - resp->msg.ss_flags |= SS_F_CONN_CLOSE; - if (test_bit(TFW_HTTP_B_CONN_CLOSE_FORCE, resp->req->flags)) - resp->msg.ss_flags |= __SS_F_FORCE; -} - /* * Reset the flag saying that @srv_conn has non-idempotent requests. */ @@ -1345,18 +1336,6 @@ tfw_http_conn_nip_reset(TfwSrvConn *srv_conn) clear_bit(TFW_CONN_B_HASNIP, &srv_conn->flags); } -/* - * Put @req on the list of non-idempotent requests in @srv_conn. - * Raise the flag saying that @srv_conn has non-idempotent requests. - */ -static inline void -tfw_http_req_nip_enlist(TfwSrvConn *srv_conn, TfwHttpReq *req) -{ - BUG_ON(!list_empty(&req->nip_list)); - list_add_tail(&req->nip_list, &srv_conn->nip_queue); - set_bit(TFW_CONN_B_HASNIP, &srv_conn->flags); -} - /* * Remove @req from the list of non-idempotent requests in @srv_conn. * If it is the last request on the list, then clear the flag saying @@ -1373,6 +1352,165 @@ tfw_http_req_nip_delist(TfwSrvConn *srv_conn, TfwHttpReq *req) } } +/** + * Remove @req from the server connection's forwarding queue. + * Caller must care about @srv_conn->last_msg_sent on it's own to keep the + * queue state consistent. + */ +static inline void +tfw_http_req_delist(TfwSrvConn *srv_conn, TfwHttpReq *req) +{ + tfw_http_req_nip_delist(srv_conn, req); + list_del_init(&req->fwd_list); + srv_conn->qsize--; +} + +/* + * Get the request that is previous to @msg. + */ +static inline TfwMsg * +__tfw_http_conn_msg_sent_prev(TfwSrvConn *srv_conn, TfwMsg *msg) +{ + TfwHttpReq *req_sent = (TfwHttpReq *)msg; + + /* + * There is list_is_last() function in the Linux kernel, + * but there is no list_is_first(). The condition below + * is an implementation of list_is_first(). + */ + return (srv_conn->fwd_queue.next == &req_sent->fwd_list) ? + NULL : (TfwMsg *)list_prev_entry(req_sent, fwd_list); +} + +static inline TfwMsg * +__tfw_http_conn_curr_msg_sent_prev(TfwSrvConn *srv_conn) +{ + if (unlikely(!srv_conn->curr_msg_sent)) + return NULL; + + return __tfw_http_conn_msg_sent_prev(srv_conn, + srv_conn->curr_msg_sent); +} + +static inline TfwMsg * +__tfw_http_conn_last_msg_sent_prev(TfwSrvConn *srv_conn) +{ + if (unlikely(!srv_conn->last_msg_sent)) + return NULL; + + return __tfw_http_conn_msg_sent_prev(srv_conn, + srv_conn->last_msg_sent); +} + +static inline bool +tfw_http_req_is_valid(TfwHttpReq *req) +{ + return ((TFW_MSG_H2(req) && req->stream) + || (!TFW_MSG_H2(req) + && !test_bit(TFW_HTTP_B_REQ_DROP, req->flags))); +} + +static int +tfw_http_on_tcp_entail_req(void *conn, struct sk_buff *skb) +{ + TfwSrvConn *srv_conn = (TfwSrvConn *)conn; + TfwHttpReq *req; + + /* + * We can safely access `req` pointer here. + * - request is never deleted on the client side if it was already + * sent to backend server. + * - on the server side request can be deleted during rescheduling + * (after server connection reestablishing) or during server + * connection released. In both these cases server connection + * queue was already purged (see tfw_srv_conn_release`), so + * this function will never called. + */ + req = (TfwHttpReq *)TFW_SKB_CB(skb)->on_tcp_entail_data; + + if (unlikely(!tfw_http_req_is_valid(req))) { + spin_lock_bh(&srv_conn->fwd_qlock); + /* + * Should be called before removing request from the + * server connection queue. + */ + if ((TfwMsg *)req == srv_conn->last_msg_sent) { + srv_conn->last_msg_sent = + __tfw_http_conn_last_msg_sent_prev(srv_conn); + } + if ((TfwMsg *)req == srv_conn->curr_msg_sent) { + srv_conn->curr_msg_sent = + __tfw_http_conn_curr_msg_sent_prev(srv_conn); + } + tfw_http_req_delist(srv_conn, req); + tfw_http_conn_msg_free((TfwHttpMsg *)req); + + spin_unlock_bh(&srv_conn->fwd_qlock); + + return -EINVAL; + } + + return 0; +} + +static int +tfw_http_on_send_req(void *conn, struct sk_buff **skb_head, void *on_send_data) +{ + TfwHttpReq *req; + + if (unlikely(!conn)) + return -EPIPE; + + /* + * We can safely access `req` pointer here. + * - request is never deleted on the client side if it was already + * sent to backend server. + * - on the server side request can be deleted during rescheduling + * (after server connection reestablishing) or during server + * connection released. In both these cases server connection + * queue was already purged (see tfw_srv_conn_release`), so + * this function will never called. + */ + req = (TfwHttpReq *)on_send_data; + + TFW_SKB_CB(*skb_head)->on_tcp_entail = tfw_http_on_tcp_entail_req; + TFW_SKB_CB(*skb_head)->on_tcp_entail_data = req; + + return 0; +} + +static inline void +tfw_http_req_init_for_send(TfwSrvConn *srv_conn, TfwHttpReq *req) +{ + tfw_http_req_init_ss_flags(srv_conn, req); + if (unlikely(test_bit(TFW_HTTP_B_HMONITOR, req->flags))) + return; + + TFW_SKB_CB(req->msg.skb_head)->on_send = tfw_http_on_send_req; + TFW_SKB_CB(req->msg.skb_head)->on_send_data = req; +} + +static inline void +tfw_http_resp_init_ss_flags(TfwHttpResp *resp) +{ + if (test_bit(TFW_HTTP_B_CONN_CLOSE, resp->req->flags)) + resp->msg.ss_flags |= SS_F_CONN_CLOSE; + if (test_bit(TFW_HTTP_B_CONN_CLOSE_FORCE, resp->req->flags)) + resp->msg.ss_flags |= __SS_F_FORCE; +} + +/* + * Put @req on the list of non-idempotent requests in @srv_conn. + * Raise the flag saying that @srv_conn has non-idempotent requests. + */ +static inline void +tfw_http_req_nip_enlist(TfwSrvConn *srv_conn, TfwHttpReq *req) +{ + BUG_ON(!list_empty(&req->nip_list)); + list_add_tail(&req->nip_list, &srv_conn->nip_queue); + set_bit(TFW_CONN_B_HASNIP, &srv_conn->flags); +} + /* * Remove idempotent requests from the list of non-idempotent requests * in @srv_conn. A non-idempotent request may become idempotent when @@ -1441,23 +1579,6 @@ tfw_http_conn_need_fwd(TfwSrvConn *srv_conn) && !tfw_http_conn_drained(srv_conn)); } -/* - * Get the request that is previous to @srv_conn->last_msg_sent. - */ -static inline TfwMsg * -__tfw_http_conn_msg_sent_prev(TfwSrvConn *srv_conn) -{ - TfwHttpReq *req_sent = (TfwHttpReq *)srv_conn->last_msg_sent; - - /* - * There is list_is_last() function in the Linux kernel, - * but there is no list_is_first(). The condition below - * is an implementation of list_is_first(). - */ - return (srv_conn->fwd_queue.next == &req_sent->fwd_list) ? - NULL : (TfwMsg *)list_prev_entry(req_sent, fwd_list); -} - /* * Reset server connection's @fwd_queue and move all requests * to @dst list. @@ -1484,19 +1605,6 @@ tfw_http_req_enlist(TfwSrvConn *srv_conn, TfwHttpReq *req) tfw_http_req_nip_enlist(srv_conn, req); } -/** - * Remove @req from the server connection's forwarding queue. - * Caller must care about @srv_conn->last_msg_sent on it's own to keep the - * queue state consistent. - */ -static inline void -tfw_http_req_delist(TfwSrvConn *srv_conn, TfwHttpReq *req) -{ - tfw_http_req_nip_delist(srv_conn, req); - list_del_init(&req->fwd_list); - srv_conn->qsize--; -} - /* * Common actions in case of an error while forwarding requests. * Erroneous requests are removed from the forwarding queue and placed @@ -1941,14 +2049,10 @@ tfw_http_req_zap_error(struct list_head *eq) list_for_each_entry_safe(req, tmp, eq, fwd_list) { list_del_init(&req->fwd_list); - if ((TFW_MSG_H2(req) && req->stream) - || (!TFW_MSG_H2(req) - && !test_bit(TFW_HTTP_B_REQ_DROP, req->flags))) - { + if (tfw_http_req_is_valid(req)) { tfw_http_send_err_resp(req, req->httperr.status, req->httperr.reason); - } - else { + } else { tfw_http_conn_msg_free((TfwHttpMsg *)req); } @@ -2063,7 +2167,6 @@ tfw_http_req_fwd_send(TfwSrvConn *srv_conn, TfwServer *srv, TfwHttpReq *req, int r; req->jtxtstamp = jiffies; - tfw_http_req_init_ss_flags(srv_conn, req); /* * We set TFW_CONN_B_UNSCHED on server connection. New requests must @@ -2087,6 +2190,8 @@ tfw_http_req_fwd_send(TfwSrvConn *srv_conn, TfwServer *srv, TfwHttpReq *req, return -EBADF; } + tfw_http_req_init_for_send(srv_conn, req); + if (!(r = tfw_connection_send((TfwConn *)srv_conn, (TfwMsg *)req))) return 0; @@ -2275,8 +2380,16 @@ tfw_http_conn_treatnip(TfwSrvConn *srv_conn, struct list_head *eq) && !(srv->sg->flags & TFW_SRV_RETRY_NIP)) { BUG_ON(list_empty(&req_sent->nip_list)); + /* + * This function is called during connection repair, + * so `srv_conn->curr_msg_sent` is set to NULL. + * If it is not NULL and we don't update it here + * (when we update `last_msg_sent`), we catch BUG_ON + * later during requests rescheduling. + */ + BUG_ON(srv_conn->curr_msg_sent); srv_conn->last_msg_sent = - __tfw_http_conn_msg_sent_prev(srv_conn); + __tfw_http_conn_last_msg_sent_prev(srv_conn); tfw_http_nip_req_resched_err(srv_conn, req_sent, eq); } } @@ -2608,6 +2721,12 @@ tfw_http_conn_shrink_fwdq(TfwSrvConn *srv_conn) return; } + /* + * This function is called during connection repairing, + * `curr_msg_sent` was set to NULL. If it is not NULL + * and we don't update it during updating `last_msg_sent` + * we catch BUG_ON. + */ BUG_ON(srv_conn->curr_msg_sent); /* @@ -2632,7 +2751,7 @@ tfw_http_conn_shrink_fwdq(TfwSrvConn *srv_conn) * reassign @srv_conn->last_msg_sent in case it is evicted. * @req is now the same as @srv_conn->last_msg_sent. */ - msg_sent_prev = __tfw_http_conn_msg_sent_prev(srv_conn); + msg_sent_prev = __tfw_http_conn_last_msg_sent_prev(srv_conn); if (tfw_http_req_evict_stale_req(srv_conn, srv, req, &eq)) srv_conn->last_msg_sent = msg_sent_prev; } @@ -7461,11 +7580,7 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb, /* `cli_conn` is equal to zero for health monitor requests. */ if (likely(cli_conn)) { - if (TFW_FSM_TYPE(cli_conn->proto.type) == TFW_FSM_H2) - conn_stop = !hmresp->req->stream; - else - conn_stop = test_bit(TFW_HTTP_B_REQ_DROP, - hmresp->req->flags); + conn_stop = !tfw_http_req_is_valid(hmresp->req); ss_skb_set_owner(skb, ss_skb_dflt_destructor, CLIENT_MEM_FROM_CONN(cli_conn), skb->truesize); diff --git a/fw/http2.c b/fw/http2.c index aedd48780..2ff04e9a8 100644 --- a/fw/http2.c +++ b/fw/http2.c @@ -721,7 +721,16 @@ tfw_h2_entail_stream_skb(struct sock *sk, TfwStream *stream, struct sk_buff *skb, *split; int r = 0; - BUG_ON(!TFW_SKB_CB(stream->xmit.skb_head)->is_head); + /* + * Currently we use this function only for sending HEADERS, DATA + * or TRAILER frames for http2 response. `on_tcp_entail` callback + * is not implemented for responses. For it's implementation + * we should rework this function. + */ + if (WARN_ON(!TFW_SKB_CB(stream->xmit.skb_head)->is_head + || TFW_SKB_CB(stream->xmit.skb_head)->on_tcp_entail)) + return -EINVAL; + while (*len) { skb = ss_skb_dequeue(&stream->xmit.skb_head); BUG_ON(!skb); diff --git a/fw/http_frame.c b/fw/http_frame.c index d3241b74c..d5cc8dd82 100644 --- a/fw/http_frame.c +++ b/fw/http_frame.c @@ -227,13 +227,15 @@ ctx_new_settings_flags[] = { [HTTP2_SETTINGS_MAX_HDR_LIST_SIZE] = 0x20 }; -static void +static int tfw_h2_on_tcp_entail_ack(void *conn, struct sk_buff *skb_head) { TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); if (test_bit(HTTP2_SETTINGS_NEED_TO_APPLY, ctx->settings_to_apply)) tfw_h2_apply_new_settings(ctx); + + return 0; } static int diff --git a/fw/sock.c b/fw/sock.c index 4ae1a6d96..0b1a2d8f2 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -467,7 +467,6 @@ ss_skb_tcp_entail(struct sock *sk, struct sk_buff *skb, unsigned int mark, { struct tcp_sock *tp = tcp_sk(sk); - ss_skb_on_tcp_entail(sk->sk_user_data, skb); T_DBG3("[%d]: %s: entail sk=%pK skb=%pK data_len=%u len=%u" " truesize=%u mark=%u tls_type=%x\n", smp_processor_id(), __func__, sk, skb, skb->data_len, @@ -493,6 +492,7 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, struct sk_buff *tail, *next, *to_destroy; unsigned char tls_type = 0; unsigned int mark = 0; + bool skip_list = false; int r; while ((*snd_wnd = tfw_tcp_calc_snd_wnd(sk, mss_now))) { @@ -508,16 +508,24 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, * belongs. */ if (TFW_SKB_CB(skb)->is_head) { + skip_list = false; tls_type = skb_tfw_tls_type(skb); mark = skb->mark; tail = tcp_write_queue_tail(sk); } + + if (!skip_list + && ss_skb_on_tcp_entail(sk->sk_user_data, skb)) + skip_list = true; + /* * Zero-sized SKBs may appear when the message headers (or any * other contents) are modified or deleted by Tempesta. Drop * these SKBs. + * If `on_tcp_entail` callback fails we also free all skbs from + * the list. */ - if (!skb->len) { + if (skip_list || !skb->len) { T_DBG3("[%d]: %s: drop skb=%pK data_len=%u len=%u\n", smp_processor_id(), __func__, skb, skb->data_len, skb->len); @@ -663,7 +671,7 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) twin_skb = __pskb_copy_fclone(skb, MAX_TCP_HEADER, GFP_ATOMIC, true); if (!twin_skb) { - T_WARN("Unable to copy an egress SKB.\n"); + T_WARN("Unable to copy an egress skb.\n"); r = -ENOMEM; goto err; } @@ -676,6 +684,7 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) ss_skb_set_owner(twin_skb, ss_skb_dflt_destructor, TFW_SKB_CB(skb)->cli_mem, copied_truesize); + ss_skb_copy_cb(twin_skb, skb); ss_skb_queue_tail(&sw.skb_head, twin_skb); skb = skb->next; } while (skb != *skb_head); diff --git a/fw/ss_skb.c b/fw/ss_skb.c index c4f74606b..c5404a75b 100644 --- a/fw/ss_skb.c +++ b/fw/ss_skb.c @@ -1799,3 +1799,11 @@ ss_skb_adjust_client_mem(struct sk_buff *skb, int delta) tfw_client_adjust_mem(cli_mem, delta); } } + +void +ss_skb_copy_cb(struct sk_buff *to, struct sk_buff *from) +{ + memcpy(&TFW_SKB_CB(to)->copy, &TFW_SKB_CB(from)->copy, + sizeof(TFW_SKB_CB(from)->copy)); + bzero_fast(&TFW_SKB_CB(from)->copy, sizeof(TFW_SKB_CB(from)->copy)); +} diff --git a/fw/ss_skb.h b/fw/ss_skb.h index fbe80aa31..4fd3e551c 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -29,7 +29,7 @@ typedef int (*on_send_cb_t)(void *conn, struct sk_buff **skb_head, void *on_send_data); -typedef void (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head); +typedef int (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head); typedef void (*on_send_fail_cb_t)(void *conn, struct sk_buff *skb_head); typedef struct tfw_client_mem_t TfwClientMem; @@ -54,17 +54,19 @@ typedef struct tfw_client_mem_t TfwClientMem; struct tfw_skb_cb { TfwClientMem *cli_mem; void (*destructor)(struct sk_buff *); - union { - on_send_cb_t on_send; - on_tcp_entail_t on_tcp_entail; - }; - union { - void *on_send_data; - void *on_tcp_entail_data; - }; long int mem; - unsigned int stream_id; - bool is_head; + struct_group (copy, + union { + on_send_cb_t on_send; + on_tcp_entail_t on_tcp_entail; + }; + union { + void *on_send_data; + void *on_tcp_entail_data; + }; + unsigned int stream_id; + bool is_head; + ); }; #define TFW_SKB_CB(skb) ((struct tfw_skb_cb *)&((skb)->cb[0])) @@ -74,6 +76,7 @@ void ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), void ss_skb_adjust_client_mem(struct sk_buff *skb, int delta); void ss_skb_dflt_destructor(struct sk_buff *skb); void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); +void ss_skb_copy_cb(struct sk_buff *to, struct sk_buff *from); static inline void __ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), @@ -126,13 +129,14 @@ ss_skb_on_send(void *conn, struct sk_buff **skb_head) return r; } -static inline void +static inline int ss_skb_on_tcp_entail(void *conn, struct sk_buff *skb_head) { on_tcp_entail_t on_tcp_entail = TFW_SKB_CB(skb_head)->on_tcp_entail; if (on_tcp_entail) - on_tcp_entail(conn, skb_head); + return on_tcp_entail(conn, skb_head); + return 0; } typedef int ss_skb_actor_t(void *conn, unsigned char *data, unsigned int len, From 96399806874ef3e0b73d40f1e880743e575ee3d0 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Wed, 13 May 2026 12:01:11 +0300 Subject: [PATCH 4/9] Simplify `ss_skb_tcp_entail_list` function Previously there was a sofisticated logic in `ss_skb_tcp_entail_list` function, which we are planning to use to handle ENOMEM in a special way and do not drop connection. Since we decide to don't do it, remove this logic at all. (Made here since we already change `ss_skb_tcp_entail_list` in this PR). --- fw/sock.c | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/fw/sock.c b/fw/sock.c index 0b1a2d8f2..305e66bb4 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -489,7 +489,6 @@ int ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, unsigned int mss_now, unsigned long *snd_wnd) { - struct sk_buff *tail, *next, *to_destroy; unsigned char tls_type = 0; unsigned int mark = 0; bool skip_list = false; @@ -511,7 +510,6 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, skip_list = false; tls_type = skb_tfw_tls_type(skb); mark = skb->mark; - tail = tcp_write_queue_tail(sk); } if (!skip_list @@ -535,8 +533,8 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, r = ss_skb_realloc_headroom(skb); if (unlikely(r)) { - ss_skb_queue_head(skb_head, skb); - goto restore_sk_write_queue; + ss_kfree_skb(skb); + return r; } ss_skb_tcp_entail(sk, skb, mark, tls_type); @@ -546,19 +544,6 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, ss_skb_setup_head_of_list(*skb_head, mark, tls_type); return 0; - -restore_sk_write_queue: - to_destroy = tail ? tail->next : tcp_send_head(sk); - if (to_destroy) { - tcp_for_write_queue_from_safe(to_destroy, next, sk) { - tcp_unlink_write_queue(to_destroy, sk); - tcp_wmem_free_skb(sk, to_destroy); - } - } - if (*skb_head && !TFW_SKB_CB(*skb_head)->is_head) - ss_skb_setup_head_of_list(*skb_head, mark, tls_type); - - return r; } /** From cf875b0a94a460b17340abf3f028b8290bc05013 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Fri, 29 May 2026 19:38:43 +0300 Subject: [PATCH 5/9] Fix according review - Remove unnecessary `skb_copy_cb` (`__pskb_copy_fclone` already do it). Instead of it zero all fields for the cloned skb, which are related to client memory accounting (since we set client memory for the new cloned skb manually). Zero all other fields for original skb, we need appropriate callbacks only for skbs, which we will send to the socket write queue. - Move `ss_skb_on_tcp_entail` under `if (TFW_SKB_CB(skb)->is_head)` to show that it should be called only to the head of the list. - Add comments. - introduce skb callback container. There are currently many cases where different callbacks are required for request/response and HTTP/2 frame sending. Since the space available in `skb->cb` is limited, introduce a callback container structure and store a pointer to it in `skb->cb` instead of embedding callback-specific data directly. In addition to improving code clarity and freeing space in skb->cb, this change allows an unlimited number of callbacks to be added in the future. --- fw/connection.h | 2 +- fw/http.c | 76 ++++++++++++++++------------------------- fw/http.h | 3 +- fw/http2.c | 3 +- fw/http_frame.c | 68 ++++++++++++++++-------------------- fw/http_stream.c | 4 +-- fw/sock.c | 46 +++++++++++++++++++------ fw/ss_skb.c | 8 ----- fw/ss_skb.h | 89 ++++++++++++++++++++++++------------------------ fw/tls.c | 18 +++++----- 10 files changed, 156 insertions(+), 161 deletions(-) diff --git a/fw/connection.h b/fw/connection.h index 282480dad..d74fad5d7 100644 --- a/fw/connection.h +++ b/fw/connection.h @@ -441,7 +441,7 @@ tfw_connection_live(TfwConn *conn) static inline void tfw_connection_get(TfwConn *conn) { - atomic_add(1, &conn->refcnt); + atomic_inc(&conn->refcnt); } #define TFW_CONNETION_GET_IF(name, cond) \ diff --git a/fw/http.c b/fw/http.c index b7ae38248..2dadc7386 100644 --- a/fw/http.c +++ b/fw/http.c @@ -515,6 +515,19 @@ typedef enum { TFW_ERROR_TYPE_BAD, } ErrorType; +static int tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head); +static void tfw_h2_on_send_resp_fail(struct sk_buff *skb_head); +static int tfw_http_on_tcp_entail_req(void *conn, struct sk_buff *skb); + +TfwSkbHooks tfw_h2_resp_skb_hooks = { + .on_send = tfw_h2_on_send_resp, + .on_send_fail = tfw_h2_on_send_resp_fail, +}; + +static TfwSkbHooks tfw_http_req_skb_hooks = { + .on_tcp_entail = tfw_http_on_tcp_entail_req, +}; + /* * Prepare current date in the format required for HTTP "Date:" * header field. See RFC 2616 section 3.3. @@ -1162,8 +1175,9 @@ tfw_h2_resp_fwd(TfwHttpResp *resp) /* * Connection reference counter will be decremented and response will - * be freed in `on_send` callback, if it fails or after encoding headers - * in `tfw_h2_stream_xmit_process`. + * be freed in `on_send_fail` callback, if send fails, because of + * inactive socket or after encoding headers in + * `tfw_h2_stream_xmit_process`. */ tfw_connection_get(conn); do_access_log(resp); @@ -1426,7 +1440,7 @@ tfw_http_on_tcp_entail_req(void *conn, struct sk_buff *skb) * queue was already purged (see tfw_srv_conn_release`), so * this function will never called. */ - req = (TfwHttpReq *)TFW_SKB_CB(skb)->on_tcp_entail_data; + req = (TfwHttpReq *)TFW_SKB_CB(skb)->data; if (unlikely(!tfw_http_req_is_valid(req))) { spin_lock_bh(&srv_conn->fwd_qlock); @@ -1453,32 +1467,6 @@ tfw_http_on_tcp_entail_req(void *conn, struct sk_buff *skb) return 0; } -static int -tfw_http_on_send_req(void *conn, struct sk_buff **skb_head, void *on_send_data) -{ - TfwHttpReq *req; - - if (unlikely(!conn)) - return -EPIPE; - - /* - * We can safely access `req` pointer here. - * - request is never deleted on the client side if it was already - * sent to backend server. - * - on the server side request can be deleted during rescheduling - * (after server connection reestablishing) or during server - * connection released. In both these cases server connection - * queue was already purged (see tfw_srv_conn_release`), so - * this function will never called. - */ - req = (TfwHttpReq *)on_send_data; - - TFW_SKB_CB(*skb_head)->on_tcp_entail = tfw_http_on_tcp_entail_req; - TFW_SKB_CB(*skb_head)->on_tcp_entail_data = req; - - return 0; -} - static inline void tfw_http_req_init_for_send(TfwSrvConn *srv_conn, TfwHttpReq *req) { @@ -1486,8 +1474,8 @@ tfw_http_req_init_for_send(TfwSrvConn *srv_conn, TfwHttpReq *req) if (unlikely(test_bit(TFW_HTTP_B_HMONITOR, req->flags))) return; - TFW_SKB_CB(req->msg.skb_head)->on_send = tfw_http_on_send_req; - TFW_SKB_CB(req->msg.skb_head)->on_send_data = req; + TFW_SKB_CB(req->msg.skb_head)->skb_hooks = &tfw_http_req_skb_hooks; + TFW_SKB_CB(req->msg.skb_head)->data = req; } static inline void @@ -5572,20 +5560,14 @@ tfw_h2_append_predefined_body(TfwHttpResp *resp, const TfwStr *body) } ALLOW_ERROR_INJECTION(tfw_h2_append_predefined_body, ERRNO); -int -tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head, - void *on_send_data) +static int +tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head) { - TfwHttpResp *resp = (TfwHttpResp *)on_send_data; - TfwH2Ctx *ctx; - unsigned int stream_id; + TfwHttpResp *resp = (TfwHttpResp *)TFW_SKB_CB(*skb_head)->data; + TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); + unsigned int stream_id = TFW_SKB_CB(*skb_head)->stream_id; TfwStream *stream; - if (unlikely(!conn)) - goto fail; - - ctx = tfw_h2_context_unsafe((TfwConn *)conn); - stream_id = TFW_SKB_CB(*skb_head)->stream_id; stream = tfw_h2_find_not_closed_stream(ctx, stream_id, false); /* * Very unlikely case. We check that stream is active, before @@ -5594,7 +5576,7 @@ tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head, * before ss_do_send was called. */ if (unlikely(!stream)) - goto fail; + return -EPIPE; /* * These pointers are zeroed in `tfw_h2_stream_init_for_xmit`. @@ -5616,10 +5598,12 @@ tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head, tfw_h2_sched_activate_stream(&ctx->sched, stream); return 0; +} -fail: - tfw_http_resp_pair_free_and_put_conn(resp); - return -EPIPE; +static void +tfw_h2_on_send_resp_fail(struct sk_buff *skb_head) +{ + tfw_http_resp_pair_free_and_put_conn(TFW_SKB_CB(skb_head)->data); } /** diff --git a/fw/http.h b/fw/http.h index 1f7712336..506d82770 100644 --- a/fw/http.h +++ b/fw/http.h @@ -762,6 +762,7 @@ extern unsigned int max_header_list_size; extern bool allow_empty_body_content_type; extern unsigned int ctrl_frame_rate_mul; extern unsigned int wnd_update_frame_rate_mul; +extern TfwSkbHooks tfw_h2_resp_skb_hooks; /* External HTTP functions. */ int tfw_http_msg_process(TfwConn *conn, struct sk_buff *skb, @@ -814,7 +815,5 @@ int tfw_http_resp_copy_encodings(TfwHttpResp *resp, TfwStr* dst, void tfw_http_extract_request_authority(TfwHttpReq *req); bool tfw_http_mark_is_in_whitlist(unsigned int mark); char *tfw_http_resp_status_line(int status, size_t *len); -int tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head, - void *on_send_data); #endif /* __TFW_HTTP_H__ */ diff --git a/fw/http2.c b/fw/http2.c index 2ff04e9a8..12380aad7 100644 --- a/fw/http2.c +++ b/fw/http2.c @@ -716,6 +716,7 @@ int tfw_h2_entail_stream_skb(struct sock *sk, TfwStream *stream, unsigned int *len, bool should_split) { + TfwSkbHooks *skb_hooks = TFW_SKB_CB(stream->xmit.skb_head)->skb_hooks; unsigned char tls_type = skb_tfw_tls_type(stream->xmit.skb_head); unsigned int mark = stream->xmit.skb_head->mark; struct sk_buff *skb, *split; @@ -728,7 +729,7 @@ tfw_h2_entail_stream_skb(struct sock *sk, TfwStream *stream, * we should rework this function. */ if (WARN_ON(!TFW_SKB_CB(stream->xmit.skb_head)->is_head - || TFW_SKB_CB(stream->xmit.skb_head)->on_tcp_entail)) + || (skb_hooks && skb_hooks->on_tcp_entail))) return -EINVAL; while (*len) { diff --git a/fw/http_frame.c b/fw/http_frame.c index d5cc8dd82..c67de8ace 100644 --- a/fw/http_frame.c +++ b/fw/http_frame.c @@ -228,7 +228,7 @@ ctx_new_settings_flags[] = { }; static int -tfw_h2_on_tcp_entail_ack(void *conn, struct sk_buff *skb_head) +tfw_h2_on_tcp_entail_setting_ack(void *conn, struct sk_buff *skb_head) { TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); @@ -239,15 +239,10 @@ tfw_h2_on_tcp_entail_ack(void *conn, struct sk_buff *skb_head) } static int -tfw_h2_on_send_goaway(void *conn, struct sk_buff **skb_head, - void *on_send_data) +tfw_h2_on_send_goaway(void *conn, struct sk_buff **skb_head) { - TfwH2Ctx *ctx; - - if (unlikely(!conn)) - return -EPIPE; + TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); - ctx = tfw_h2_context_unsafe((TfwConn *)conn); if (ctx->error && ctx->error->xmit.skb_head) { ss_skb_queue_splice(&ctx->error->xmit.skb_head, skb_head); } else if (ctx->cur_send_headers) { @@ -264,18 +259,12 @@ tfw_h2_on_send_goaway(void *conn, struct sk_buff **skb_head, } static int -tfw_h2_on_send_rst_stream(void *conn, struct sk_buff **skb_head, - void *on_send_data) +tfw_h2_on_send_rst_stream(void *conn, struct sk_buff **skb_head) { - TfwH2Ctx *ctx; - unsigned int stream_id; + TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); + unsigned int stream_id = TFW_SKB_CB(*skb_head)->stream_id; TfwStream *stream; - if (unlikely(!conn)) - return -EPIPE; - - ctx = tfw_h2_context_unsafe((TfwConn *)conn); - stream_id = TFW_SKB_CB(*skb_head)->stream_id; stream = tfw_h2_find_not_closed_stream(ctx, stream_id, false); /* @@ -295,15 +284,10 @@ tfw_h2_on_send_rst_stream(void *conn, struct sk_buff **skb_head, } static int -tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head, - void *on_send_data) +tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head) { - TfwH2Ctx *ctx; - - if (unlikely(!conn)) - return -EPIPE; + TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); - ctx = tfw_h2_context_unsafe((TfwConn *)conn); if (ctx->cur_send_headers) { ss_skb_queue_splice(&ctx->cur_send_headers->xmit.postponed, skb_head); @@ -312,18 +296,22 @@ tfw_h2_on_send_dflt(void *conn, struct sk_buff **skb_head, return 0; } -static int -tfw_h2_on_send_ack(void *conn, struct sk_buff** skb_head, - void *on_send_data) -{ - if (unlikely(!conn)) - return -EPIPE; +static TfwSkbHooks tfw_h2_goaway_frame_skb_hooks = { + .on_send = tfw_h2_on_send_goaway, +}; - TFW_SKB_CB(*skb_head)->on_tcp_entail = tfw_h2_on_tcp_entail_ack; - tfw_h2_on_send_dflt(conn, skb_head, on_send_data); +static TfwSkbHooks tfw_h2_rst_frame_skb_hooks = { + .on_send = tfw_h2_on_send_rst_stream, +}; - return 0; -} +static TfwSkbHooks tfw_h2_settings_ack_frame_skb_hooks = { + .on_send = tfw_h2_on_send_dflt, + .on_tcp_entail = tfw_h2_on_tcp_entail_setting_ack, +}; + +static TfwSkbHooks tfw_h2_frame_dflt_skb_hooks = { + .on_send = tfw_h2_on_send_dflt, +}; /** * Prepare and send HTTP/2 frame to the client; @hdr must contain @@ -376,14 +364,18 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data, } if (hdr->type == HTTP2_GOAWAY) { - TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_goaway; + TFW_SKB_CB(msg.skb_head)->skb_hooks = + &tfw_h2_goaway_frame_skb_hooks; } else if (hdr->type == HTTP2_RST_STREAM) { - TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_rst_stream; + TFW_SKB_CB(msg.skb_head)->skb_hooks = + &tfw_h2_rst_frame_skb_hooks; TFW_SKB_CB(msg.skb_head)->stream_id = hdr->stream_id; } else if (hdr->type == HTTP2_SETTINGS && hdr->flags == HTTP2_F_ACK) { - TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_ack; + TFW_SKB_CB(msg.skb_head)->skb_hooks = + &tfw_h2_settings_ack_frame_skb_hooks; } else { - TFW_SKB_CB(msg.skb_head)->on_send = tfw_h2_on_send_dflt; + TFW_SKB_CB(msg.skb_head)->skb_hooks = + &tfw_h2_frame_dflt_skb_hooks; } if ((r = tfw_connection_send(conn, &msg))) diff --git a/fw/http_stream.c b/fw/http_stream.c index e72480609..6828d8267 100644 --- a/fw/http_stream.c +++ b/fw/http_stream.c @@ -828,8 +828,8 @@ tfw_h2_stream_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state, return -EPIPE; } - TFW_SKB_CB(skb_head)->on_send = tfw_h2_on_send_resp; - TFW_SKB_CB(skb_head)->on_send_data = resp; + TFW_SKB_CB(skb_head)->skb_hooks = &tfw_h2_resp_skb_hooks; + TFW_SKB_CB(skb_head)->data = resp; TFW_SKB_CB(skb_head)->stream_id = stream->id; stream->xmit.resp = NULL; diff --git a/fw/sock.c b/fw/sock.c index 305e66bb4..d43a61a71 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -507,15 +507,18 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, * belongs. */ if (TFW_SKB_CB(skb)->is_head) { - skip_list = false; tls_type = skb_tfw_tls_type(skb); mark = skb->mark; + /* + * If `on_tcp_entail` callback is set and fails, we free + * all skbs from the list. This callback can be set only + * for the head of skb list and we don't set it again at + * the end of this function, if we don't send the whole + * skb list. + */ + skip_list = ss_skb_on_tcp_entail(sk->sk_user_data, skb); } - if (!skip_list - && ss_skb_on_tcp_entail(sk->sk_user_data, skb)) - skip_list = true; - /* * Zero-sized SKBs may appear when the message headers (or any * other contents) are modified or deleted by Tempesta. Drop @@ -540,8 +543,18 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, ss_skb_tcp_entail(sk, skb, mark, tls_type); } - if (*skb_head && !TFW_SKB_CB(*skb_head)->is_head) + if (*skb_head && !TFW_SKB_CB(*skb_head)->is_head) { + /* + * If `skip_list` is true we should destroy all skbs + * until we reach the next message (`is_head` should be + * set). We don't add skb to the socket write queue if + * `skip_list` is true, so the loop located above will + * not be breaked until the whole message will be deleted. + */ + if (WARN_ON(skip_list)) + return -EINVAL; ss_skb_setup_head_of_list(*skb_head, mark, tls_type); + } return 0; } @@ -555,7 +568,6 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags) void *conn = sk->sk_user_data; unsigned char tls_type = flags & SS_F_ENCRYPT ? SS_SKB_F2TYPE(flags) : 0; - bool send_is_allowed = conn && ss_sock_active(sk); T_DBG3("[%d]: %s: sk=%pK queue_empty=%d send_head=%pK" " sk_state=%d\n", @@ -563,9 +575,13 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags) sk, tcp_write_queue_empty(sk), tcp_send_head(sk), sk->sk_state); + /* If the socket is inactive, there's no recourse. Drop the data. */ + if (unlikely(!conn || !ss_sock_active(sk))) + goto cleanup; + ss_skb_setup_head_of_list(*skb_head, (*skb_head)->mark, tls_type); - if (ss_skb_on_send(likely(send_is_allowed) ? conn : NULL, skb_head)) + if (ss_skb_on_send(conn, skb_head)) goto cleanup; if (flags & SS_F_CONN_CLOSE) @@ -599,6 +615,7 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags) return; cleanup: + ss_skb_on_send_fail(*skb_head); ss_skb_queue_purge(skb_head); } @@ -660,7 +677,15 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) r = -ENOMEM; goto err; } - memset(twin_skb->cb, 0, sizeof(twin_skb->cb)); + /* + * We set owner for the new cloned skb and adjust it's + * memory manually, because only `copied_truesize` + * allocated during skb cloning. + */ + bzero_fast(&TFW_SKB_CB(twin_skb)->memory, + sizeof(TFW_SKB_CB(twin_skb)->memory)); + bzero_fast(&TFW_SKB_CB(skb)->copy, + sizeof(TFW_SKB_CB(skb)->copy)); head_data = MAX_TCP_HEADER + skb_headlen(twin_skb); copied_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)) + @@ -669,7 +694,6 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) ss_skb_set_owner(twin_skb, ss_skb_dflt_destructor, TFW_SKB_CB(skb)->cli_mem, copied_truesize); - ss_skb_copy_cb(twin_skb, skb); ss_skb_queue_tail(&sw.skb_head, twin_skb); skb = skb->next; } while (skb != *skb_head); @@ -1768,7 +1792,7 @@ ss_tx_action(void) dead_sock: sock_put(sk); /* paired with push() calls */ if (sw.skb_head) - ss_skb_on_send(NULL, &sw.skb_head); + ss_skb_on_send_fail(sw.skb_head); ss_skb_queue_purge(&sw.skb_head); } diff --git a/fw/ss_skb.c b/fw/ss_skb.c index c5404a75b..c4f74606b 100644 --- a/fw/ss_skb.c +++ b/fw/ss_skb.c @@ -1799,11 +1799,3 @@ ss_skb_adjust_client_mem(struct sk_buff *skb, int delta) tfw_client_adjust_mem(cli_mem, delta); } } - -void -ss_skb_copy_cb(struct sk_buff *to, struct sk_buff *from) -{ - memcpy(&TFW_SKB_CB(to)->copy, &TFW_SKB_CB(from)->copy, - sizeof(TFW_SKB_CB(from)->copy)); - bzero_fast(&TFW_SKB_CB(from)->copy, sizeof(TFW_SKB_CB(from)->copy)); -} diff --git a/fw/ss_skb.h b/fw/ss_skb.h index 4fd3e551c..9aceffd52 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -27,43 +27,46 @@ #include "str.h" -typedef int (*on_send_cb_t)(void *conn, struct sk_buff **skb_head, - void *on_send_data); -typedef int (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head); -typedef void (*on_send_fail_cb_t)(void *conn, struct sk_buff *skb_head); +typedef int (*on_send_cb_t)(void *conn, struct sk_buff **skb_head); +typedef void (*on_send_fail_cb_t)(struct sk_buff *skb_head); +typedef int (*on_tcp_entail_cb_t)(void *conn, struct sk_buff *skb_head); typedef struct tfw_client_mem_t TfwClientMem; +/* + * Tempesta FW sk_buff callbacks. + * + * @on_send - callback to special handling this skb before sending; + * @on_send_fail - callback to special handling this skb if send fails; + * @on_tcp_entail - callback to special handling this skb before pushing + * to socket write queue; + */ +typedef struct { + on_send_cb_t on_send; + on_send_fail_cb_t on_send_fail; + on_tcp_entail_cb_t on_tcp_entail; +} TfwSkbHooks; + /* * Tempesta FW sk_buff private data. * @cli_mem - pointer to TfwClientMem structure for memory * accountion; * @destructor - destructor to adjust memory when skb is freed, should - * be set if `cli_mem` is not NULL; - * @on_send - callback to special handling this skb before sending; - * @on_tcp_entail - callback to special handling this skb before pushing - * to socket write queue; - * @on_send_data - pointer to some private data for `on_send` callback; - * @on_tcp_entail_data - pointer to some private data for `on_tcp_entail` callback; * @mem - memory used for this skb, used to account appropriate * client memory; + * @skb_hooks - pointer to the callbacks structure; + * @data - pointer to some private data for callbacks; * @stream_id - id of sender stream; - * @tls_type - tls type of current skb, if it's data should be - * encrypted; * @is_head - flag indicates that this is a head of skb list; */ struct tfw_skb_cb { - TfwClientMem *cli_mem; - void (*destructor)(struct sk_buff *); - long int mem; + struct_group (memory, + TfwClientMem *cli_mem; + void (*destructor)(struct sk_buff *); + long int mem; + ); struct_group (copy, - union { - on_send_cb_t on_send; - on_tcp_entail_t on_tcp_entail; - }; - union { - void *on_send_data; - void *on_tcp_entail_data; - }; + TfwSkbHooks *skb_hooks; + void *data; unsigned int stream_id; bool is_head; ); @@ -76,7 +79,6 @@ void ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), void ss_skb_adjust_client_mem(struct sk_buff *skb, int delta); void ss_skb_dflt_destructor(struct sk_buff *skb); void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); -void ss_skb_copy_cb(struct sk_buff *to, struct sk_buff *from); static inline void __ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), @@ -106,39 +108,38 @@ ss_skb_setup_head_of_list(struct sk_buff *skb_head, unsigned int mark, TFW_SKB_CB(skb_head)->is_head = true; } +#define TFW_SKB_HOOK_CALL(skb_hooks, f, ...) \ + (skb_hooks && skb_hooks->f) ? skb_hooks->f(__VA_ARGS__) : 0 + static inline int ss_skb_on_send(void *conn, struct sk_buff **skb_head) { - on_send_cb_t on_send = TFW_SKB_CB(*skb_head)->on_send; - void *on_send_data = TFW_SKB_CB(*skb_head)->on_send_data; - int r = 0; - - /* - * `on_send` pointer is located inside the union with `on_tcp_entail` - * callback, so we should zero it. For the same reason, we reset - * `on_send_data` pointer here. - */ - TFW_SKB_CB(*skb_head)->on_send = NULL; - TFW_SKB_CB(*skb_head)->on_send_data = NULL; - if (on_send) - r = on_send(conn, skb_head, on_send_data); - - if (!r && conn && *skb_head) + int r; + + r = TFW_SKB_HOOK_CALL(TFW_SKB_CB(*skb_head)->skb_hooks, + on_send, conn, skb_head); + if (!r && *skb_head) ss_skb_on_send_dflt(conn, skb_head); return r; } +static inline void +ss_skb_on_send_fail(struct sk_buff *skb_head) +{ + return TFW_SKB_HOOK_CALL(TFW_SKB_CB(skb_head)->skb_hooks, + on_send_fail, skb_head); +} + static inline int ss_skb_on_tcp_entail(void *conn, struct sk_buff *skb_head) { - on_tcp_entail_t on_tcp_entail = TFW_SKB_CB(skb_head)->on_tcp_entail; - - if (on_tcp_entail) - return on_tcp_entail(conn, skb_head); - return 0; + return TFW_SKB_HOOK_CALL(TFW_SKB_CB(skb_head)->skb_hooks, + on_tcp_entail, conn, skb_head); } +#undef TFW_SKB_HOOK_CALL + typedef int ss_skb_actor_t(void *conn, unsigned char *data, unsigned int len, unsigned int *read); diff --git a/fw/tls.c b/fw/tls.c index f492a91eb..84bf60f54 100644 --- a/fw/tls.c +++ b/fw/tls.c @@ -48,6 +48,12 @@ static bool tfw_tls_allow_any_sni; /* Temporal value for reconfiguration stage. */ static bool allow_any_sni_reconfig; +static int tfw_tls_on_send_alert(void *conn, struct sk_buff **skb_head); + +static TfwSkbHooks tfw_tls_alert_skb_hooks = { + .on_send = tfw_tls_on_send_alert, +}; + static inline void tfw_tls_purge_io_ctx(TlsIOCtx *io) { @@ -502,15 +508,11 @@ tfw_tls_close_msg_flags(TlsIOCtx *io) return flags; } -static inline int -tfw_tls_on_send_alert(void *conn, struct sk_buff **skb_head, - void *on_send_data) +static int +tfw_tls_on_send_alert(void *conn, struct sk_buff **skb_head) { TfwH2Ctx *ctx; - if (unlikely(!conn)) - return -EPIPE; - BUG_ON(TFW_CONN_PROTO((TfwConn *)conn) != TFW_FSM_H2); ctx = tfw_h2_context_safe((TfwConn *)conn); if (!ctx) @@ -613,8 +615,8 @@ tfw_tls_send(TlsCtx *tls, struct sg_table *sgt) TFW_CONN_TYPE(((TfwConn *)conn)) |= Conn_Stop; flags |= tfw_tls_close_msg_flags(io); if (TFW_CONN_PROTO((TfwConn *)conn) == TFW_FSM_H2) { - TFW_SKB_CB(io->skb_list)->on_send = - tfw_tls_on_send_alert; + TFW_SKB_CB(io->skb_list)->skb_hooks = + &tfw_tls_alert_skb_hooks; } } From b778a3c743015e3ddbf021541ddda9f8b1fa71d2 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Fri, 29 May 2026 19:57:59 +0300 Subject: [PATCH 6/9] Remove `destructor` from `skb->cb`. Since now we use only default skb destructor for client memory adjusting, we can remove appropriate pointer from `skb->cb` and save 8 bytes. --- fw/http.c | 9 +++------ fw/http_msg.c | 8 +++----- fw/sock.c | 3 +-- fw/ss_skb.c | 15 +++++++-------- fw/ss_skb.h | 35 +++++------------------------------ fw/t/unit/test_http_msg.c | 3 +-- fw/tls.c | 11 ++++------- 7 files changed, 24 insertions(+), 60 deletions(-) diff --git a/fw/http.c b/fw/http.c index 2dadc7386..c666c8357 100644 --- a/fw/http.c +++ b/fw/http.c @@ -4729,8 +4729,7 @@ tfw_http_resp_set_empty_skb_head(TfwHttpResp *resp, TfwHttpMsgCleanup *cleanup) if (unlikely(!nskb)) return -ENOMEM; - ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - cli_mem, nskb->truesize); + ss_skb_set_owner(nskb, cli_mem, nskb->truesize); nskb->mark = resp->msg.skb_head->mark; cleanup->skb_head = resp->msg.skb_head; resp->msg.skb_head = NULL; @@ -6722,8 +6721,7 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb, * tls decryption. */ if (!TFW_SKB_CB(skb)->cli_mem) { - ss_skb_set_owner(skb, ss_skb_dflt_destructor, - CLIENT_MEM_FROM_CONN(conn), + ss_skb_set_owner(skb, CLIENT_MEM_FROM_CONN(conn), skb->truesize); } @@ -7565,8 +7563,7 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb, /* `cli_conn` is equal to zero for health monitor requests. */ if (likely(cli_conn)) { conn_stop = !tfw_http_req_is_valid(hmresp->req); - ss_skb_set_owner(skb, ss_skb_dflt_destructor, - CLIENT_MEM_FROM_CONN(cli_conn), + ss_skb_set_owner(skb, CLIENT_MEM_FROM_CONN(cli_conn), skb->truesize); r = frang_client_mem_limit(cli_conn, false); diff --git a/fw/http_msg.c b/fw/http_msg.c index 09440a53d..9f1f79dd1 100644 --- a/fw/http_msg.c +++ b/fw/http_msg.c @@ -938,8 +938,7 @@ tfw_http_msg_expand_data(TfwHttpMsg *hm, struct sk_buff **skb_head, if (!(it->skb = ss_skb_alloc(SKB_MAX_HEADER))) return -ENOMEM; - ss_skb_set_owner(it->skb, ss_skb_dflt_destructor, - tfw_http_msg_client_mem(hm), + ss_skb_set_owner(it->skb, tfw_http_msg_client_mem(hm), it->skb->truesize); ss_skb_queue_tail(skb_head, it->skb); it->frag = -1; @@ -1204,15 +1203,14 @@ __tfw_http_msg_expand_from_pool(TfwHttpMsg *hm, const TfwStr *str, if (unlikely(skb_room == 0 || nr_frags == MAX_SKB_FRAGS)) { struct sk_buff *nskb = ss_skb_alloc(0); + TfwClientMem *cli_mem = tfw_http_msg_client_mem(hm); bool body_was_moved = false; int frag; if (!nskb) return -ENOMEM; - ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - tfw_http_msg_client_mem(hm), - nskb->truesize); + ss_skb_set_owner(nskb, cli_mem, nskb->truesize); /* * TODO #2136: Remove this flag during reworking * this function. Try to process headers and diff --git a/fw/sock.c b/fw/sock.c index d43a61a71..b9e195204 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -691,8 +691,7 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) SKB_DATA_ALIGN(sizeof(struct sk_buff)) + SKB_DATA_ALIGN(head_data + sizeof(struct skb_shared_info)); - ss_skb_set_owner(twin_skb, ss_skb_dflt_destructor, - TFW_SKB_CB(skb)->cli_mem, + ss_skb_set_owner(twin_skb, TFW_SKB_CB(skb)->cli_mem, copied_truesize); ss_skb_queue_tail(&sw.skb_head, twin_skb); skb = skb->next; diff --git a/fw/ss_skb.c b/fw/ss_skb.c index c4f74606b..ce406635a 100644 --- a/fw/ss_skb.c +++ b/fw/ss_skb.c @@ -120,8 +120,7 @@ ss_skb_alloc_data(struct sk_buff **skb_head, TfwClientMem *owner, size_t len) skb = ss_skb_alloc_pages(n); if (!skb) return -ENOMEM; - ss_skb_set_owner(skb, ss_skb_dflt_destructor, - owner, skb->truesize); + ss_skb_set_owner(skb, owner, skb->truesize); ss_skb_queue_tail(skb_head, skb); } @@ -221,8 +220,7 @@ __extend_pgfrags(struct sk_buff *skb_head, struct sk_buff *skb, int from, int n) return -ENOMEM; if (!skb_tfw_is_in_socket_write_queue(skb)) { - ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - TFW_SKB_CB(skb)->cli_mem, + ss_skb_set_owner(nskb, TFW_SKB_CB(skb)->cli_mem, nskb->truesize); } skb_shinfo(nskb)->flags = skb_shinfo(skb)->flags; @@ -1772,15 +1770,16 @@ ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head) } void -ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), - TfwClientMem *owner, unsigned int mem) +ss_skb_set_owner(struct sk_buff *skb, TfwClientMem *owner, + unsigned int mem) { if (!owner || !tfw_client_mem_get(owner)) return; WARN_ON(TFW_SKB_CB(skb)->mem != 0); - WARN_ON(TFW_SKB_CB(skb)->destructor || TFW_SKB_CB(skb)->cli_mem); - __ss_skb_set_owner(skb, destructor, owner); + WARN_ON(TFW_SKB_CB(skb)->cli_mem); + + TFW_SKB_CB(skb)->cli_mem = owner; ss_skb_adjust_client_mem(skb, mem); } diff --git a/fw/ss_skb.h b/fw/ss_skb.h index 9aceffd52..ef4d41b9a 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -50,7 +50,6 @@ typedef struct { * Tempesta FW sk_buff private data. * @cli_mem - pointer to TfwClientMem structure for memory * accountion; - * @destructor - destructor to adjust memory when skb is freed, should * @mem - memory used for this skb, used to account appropriate * client memory; * @skb_hooks - pointer to the callbacks structure; @@ -61,7 +60,6 @@ typedef struct { struct tfw_skb_cb { struct_group (memory, TfwClientMem *cli_mem; - void (*destructor)(struct sk_buff *); long int mem; ); struct_group (copy, @@ -74,20 +72,12 @@ struct tfw_skb_cb { #define TFW_SKB_CB(skb) ((struct tfw_skb_cb *)&((skb)->cb[0])) -void ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), - TfwClientMem *owner, unsigned int delta); +void ss_skb_set_owner(struct sk_buff *skb, TfwClientMem *owner, + unsigned int delta); void ss_skb_adjust_client_mem(struct sk_buff *skb, int delta); void ss_skb_dflt_destructor(struct sk_buff *skb); void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); -static inline void -__ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), - TfwClientMem *owner) -{ - TFW_SKB_CB(skb)->destructor = destructor; - TFW_SKB_CB(skb)->cli_mem = owner; -} - static inline bool ss_skb_is_within_fragment(char *begin_fragment, char *position, char *end_fragment) @@ -208,27 +198,12 @@ ss_skb_queue_splice(struct sk_buff **skb_head, struct sk_buff **skb) static inline void ss_skb_orphan(struct sk_buff *skb) { - void (*destructor)(struct sk_buff *); - if (skb_tfw_is_in_socket_write_queue(skb)) return; - destructor = TFW_SKB_CB(skb)->destructor; - if (destructor) { - /* - * The same BUG_ON is present in linux kernel in `skb_orphan`. - * `skb->cli_mem` will be used inside destructor, so if it - * is NULL, we still catch BUG later. - */ - BUG_ON(!TFW_SKB_CB(skb)->cli_mem); - destructor(skb); - __ss_skb_set_owner(skb, NULL, NULL); - } else { - /* - * The same BUG_ON is present in linux kernel in - * `skb_orphan`. - */ - BUG_ON(TFW_SKB_CB(skb)->cli_mem); + if (TFW_SKB_CB(skb)->cli_mem) { + ss_skb_dflt_destructor(skb); + TFW_SKB_CB(skb)->cli_mem = NULL; } } diff --git a/fw/t/unit/test_http_msg.c b/fw/t/unit/test_http_msg.c index 6a9e5a3cc..5459be4a1 100644 --- a/fw/t/unit/test_http_msg.c +++ b/fw/t/unit/test_http_msg.c @@ -120,8 +120,7 @@ __test_resp_data_alloc(TfwStr *head_data, TfwStr *paged_data, if (!skb) return false; - ss_skb_set_owner(skb, ss_skb_dflt_destructor, - tfw_http_msg_client_mem(hmresp), + ss_skb_set_owner(skb, tfw_http_msg_client_mem(hmresp), skb->truesize); skb->next = skb->prev = skb; it = &resp->iter; diff --git a/fw/tls.c b/fw/tls.c index 84bf60f54..646333da9 100644 --- a/fw/tls.c +++ b/fw/tls.c @@ -92,8 +92,7 @@ tfw_tls_connection_recv(TfwConn *conn, struct sk_buff *skb) next_msg: spin_lock(&tls->lock); ss_skb_queue_tail(&tls->io_in.skb_list, skb); - ss_skb_set_owner(skb, ss_skb_dflt_destructor, - CLIENT_MEM_FROM_CONN(conn), skb->truesize); + ss_skb_set_owner(skb, CLIENT_MEM_FROM_CONN(conn), skb->truesize); /* Call TLS layer to place skb into a TLS record on top of skb_list. */ parsed = 0; @@ -543,6 +542,7 @@ tfw_tls_send(TlsCtx *tls, struct sg_table *sgt) int r, flags = 0; TfwTlsConn *conn = container_of(tls, TfwTlsConn, tls); TfwCliConn *cli_conn = &conn->cli_conn; + TfwClientMem *cli_mem = CLIENT_MEM_FROM_CONN(cli_conn); TlsIOCtx *io = &tls->io_out; TfwMsgIter it; TfwStr str = {}; @@ -572,8 +572,7 @@ tfw_tls_send(TlsCtx *tls, struct sg_table *sgt) str.len, sgt ? sgt->nents : 0, io->msglen, io->msgtype, conn, cli_conn->sk->sk_write_xmit, ttls_xfrm_ready(tls)); - if ((r = tfw_msg_iter_setup(&it, CLIENT_MEM_FROM_CONN(cli_conn), - &io->skb_list, str.len))) + if ((r = tfw_msg_iter_setup(&it, cli_mem, &io->skb_list, str.len))) goto out; if ((r = tfw_msg_iter_write(&it, &str))) goto out; @@ -591,9 +590,7 @@ tfw_tls_send(TlsCtx *tls, struct sg_table *sgt) r = -ENOMEM; goto out; } - ss_skb_set_owner(skb, ss_skb_dflt_destructor, - CLIENT_MEM_FROM_CONN(cli_conn), - skb->truesize); + ss_skb_set_owner(skb, cli_mem, skb->truesize); ss_skb_queue_tail(&io->skb_list, skb); i = 0; } From 0f5eb7bae415bd4d736ed50f88705b314b1c7bac Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Tue, 2 Jun 2026 10:11:10 +0300 Subject: [PATCH 7/9] Do not calculate `snd_wnd` on each loop iteration. During making frames we adjust skb length to recalculate `snd_wnd`. It seems when we send http1 data, it's better to work in the same way. (Made here since we already change `ss_skb_tcp_entail_list` in this PR). --- fw/connection.c | 9 ++++++--- fw/sock.c | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fw/connection.c b/fw/connection.c index d876d4b3a..31a373546 100644 --- a/fw/connection.c +++ b/fw/connection.c @@ -275,6 +275,7 @@ tfw_connection_fill_sk_write_queue(TfwConn *conn, unsigned int mss_now) */ tcp_slow_start_after_idle_check(sk); + snd_wnd = tfw_tcp_calc_snd_wnd(sk, mss_now); /* * First of all Tempesta FW entails skb from connection write queue * (all http1 data, control frames, tls alerts and so on for http2), @@ -301,9 +302,11 @@ tfw_connection_fill_sk_write_queue(TfwConn *conn, unsigned int mss_now) return r; } - r = tfw_h2_make_frames(sk, h2, mss_now, snd_wnd); - if (unlikely(r)) - return r; + if (snd_wnd) { + r = tfw_h2_make_frames(sk, h2, mss_now, snd_wnd); + if (unlikely(r)) + return r; + } if (unlikely(!conn->write_queue)) { /* diff --git a/fw/sock.c b/fw/sock.c index b9e195204..cdb5a524c 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -494,7 +494,7 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, bool skip_list = false; int r; - while ((*snd_wnd = tfw_tcp_calc_snd_wnd(sk, mss_now))) { + while (*snd_wnd) { struct sk_buff *skb = ss_skb_dequeue(skb_head); if (!skb) @@ -541,6 +541,7 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, } ss_skb_tcp_entail(sk, skb, mark, tls_type); + *snd_wnd = (*snd_wnd > skb->len ? *snd_wnd - skb->len : 0); } if (*skb_head && !TFW_SKB_CB(*skb_head)->is_head) { From d76cdb0edec1cb4536bf4dc7892848e7826841ba Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Tue, 2 Jun 2026 15:27:57 +0300 Subject: [PATCH 8/9] Rework HTTTP/2 -> HTTP/1 request transformation Use pool-based approach for HTTP/2 to HTTP/1 request transformation. - Reuse the same pool-based approach already used for HTTP/1 request forwarding, providing a unified implementation. - Eliminate extra header copying. Previously, headers were written into the linear skb area and then copied by `_pskb_copy_fclone`. Now headers are written directly into skb fragments, reducing memory usage and copy overhead. - do not allocate a new skb during http1 request/response preparing. We should remove all old skbs except skb_head. In `skb_head` we can remove all fragments and linear data and reuse it, instead of new skb allocation. This patch demonstrate a very good performance improvement for big requests without cache: 2284: finished in 50.04s, 330629.14 req/s, 202.22MB/s finished in 50.04s, 328837.60 req/s, 201.04MB/s finished in 50.04s, 329637.96 req/s, 201.61MB/s master: finished in 50.04s, 328963.94 req/s, 201.21MB/s finished in 50.06s, 329731.98 req/s, 201.58MB/s finished in 50.04s, 328985.10 req/s, 201.17MB/s 2284 2K: finished in 50.04s, 330025.50 req/s, 201.98MB/s finished in 50.04s, 329206.90 req/s, 201.32MB/s finished in 50.04s, 329906.58 req/s, 201.74MB/s master 2K: finished in 50.03s, 277741.36 req/s, 169.92MB/s finished in 50.03s, 294516.36 req/s, 180.08MB/s finished in 50.03s, 298020.96 req/s, 182.21MB/s 2284 4k: finished in 50.03s, 313275.10 req/s, 191.66MB/s finished in 50.03s, 313052.38 req/s, 191.45MB/s finished in 50.03s, 304237.44 req/s, 186.14MB/s master 4K: finished in 50.03s, 261815.08 req/s, 160.15MB/s finished in 50.03s, 273399.16 req/s, 167.18MB/s finished in 50.03s, 270743.96 req/s, 165.64MB/s This patch is also important in the context of fixing issue #2284. As described earlier, when a large number of requests from a reset client connection coexist with requests from active client connections and the TCP window is small, requests belonging to inactive client connections may remain queued on the server connection for an extended period of time, consuming memory. This patch significantly reduce this memory overhead. Previously, request headers were stored in the skb linear data and were copied during `_pskb_copy_fclone` (doubling memory consumption!) Request headers are now stored in skb fragments instead, avoiding the copy in `_pskb_copy_fclone` and reducing memory usage by approximately 50%. --- fw/http.c | 319 ++++++++++++++++++-------------------------------- fw/http_msg.c | 6 +- fw/http_msg.h | 1 + 3 files changed, 116 insertions(+), 210 deletions(-) diff --git a/fw/http.c b/fw/http.c index c666c8357..6294be5cf 100644 --- a/fw/http.c +++ b/fw/http.c @@ -4241,7 +4241,7 @@ tfw_h2_req_set_loc_hdrs(TfwHttpReq *req, TfwHdrMods *h_mods) * Fuse multiple cookie headers into one. * Works only with TFW_STR_DUP strings. */ static int -write_merged_cookie_headers(TfwStr *hdr, TfwMsgIter *it) +write_merged_cookie_headers(TfwStr *hdr, TfwHttpMsg *hm) { int r = 0; static const DEFINE_TFW_STR(h_cookie, "cookie" S_DLM); @@ -4265,38 +4265,96 @@ write_merged_cookie_headers(TfwStr *hdr, TfwMsgIter *it) hval.nchunks--; hval.len -= chunk->len; } - r = tfw_msg_iter_write(it, cookie_dlm); + r = tfw_http_msg_expand_from_pool(hm, cookie_dlm); if (unlikely(r)) return r; - r = tfw_msg_iter_write(it, &hval); + r = tfw_http_msg_expand_from_pool(hm, &hval); if (unlikely(r)) return r; cookie_dlm = &val_dlm; } - return tfw_msg_iter_write(it, &STR_CRLF); + return tfw_http_msg_expand_from_pool(hm, &STR_CRLF); } static int -__h2_write_method(TfwHttpReq *req, TfwMsgIter *it) +__h2_write_method(TfwHttpReq *req) { + TfwHttpMsg *hm = (TfwHttpMsg *)req; TfwHttpHdrTbl *ht = req->h_tbl; if (test_bit(TFW_HTTP_B_REQ_HEAD_TO_GET, req->flags)) { static const DEFINE_TFW_STR(meth_get, "GET"); - return tfw_msg_iter_write(it, &meth_get); + return tfw_http_msg_expand_from_pool(hm, &meth_get); } else { TfwStr meth = {}; __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], &meth); - return tfw_msg_iter_write(it, &meth); + return tfw_http_msg_expand_from_pool(hm, &meth); } } ALLOW_ERROR_INJECTION(__h2_write_method, ERRNO); +/* + * Prepare current http message skb_head for adding new headers. Remove all old + * fragments and head data. With this we don't need logic related to finding + * the right place for cutting headers. + */ +static void +tfw_http_msg_set_empty_skb_head(TfwHttpMsg *hm, TfwHttpMsgCleanup *cleanup) +{ + struct sk_buff *skb_free; + TfwMsgIter *it = &hm->iter; + + /* Clean the current request's skb_head if no body exists. */ + if (skb_headlen(it->skb)) { + ss_skb_put(it->skb, -skb_headlen(it->skb)); + it->skb->tail_lock = 1; + } + + tfw_http_msg_rm_all_frags(it->skb, cleanup); + skb_free = it->skb_head->next; + while (skb_free != it->skb_head) { + ss_skb_unlink(&it->skb_head, skb_free); + ss_skb_queue_tail(&cleanup->skb_head, skb_free); + skb_free = it->skb_head->next; + } +} + +static int +tfw_h1_req_cutoff_headers(TfwHttpReq *req, TfwHttpMsgCleanup *cleanup) +{ + TfwHttpMsg *hm = (TfwHttpMsg *)req; + int r = 0; + + if (TFW_STR_EMPTY(&req->body)) { + tfw_http_msg_set_empty_skb_head(hm, cleanup); + } else { + struct sk_buff *trailer; + size_t len = 0; + + /* Response for regular request */ + r = tfw_http_msg_cutoff_headers(hm, cleanup); + if (unlikely(r)) + return r; + + trailer = req->msg.skb_head; + do { + len += trailer->len; + trailer = trailer->next; + } while ((trailer != req->msg.skb_head) && (len != req->body.len)); + if (trailer != req->msg.skb_head) { + ss_skb_queue_split(req->msg.skb_head, trailer); + ss_skb_queue_append(&cleanup->skb_head, trailer); + } + } + + return r; +} + /** * Transform h2 request to http1.1 request before forward it to backend server. * Usually we prefer in-place header modifications avoid copying, but here @@ -4314,14 +4372,11 @@ ALLOW_ERROR_INJECTION(__h2_write_method, ERRNO); static int tfw_h2_adjust_req(TfwHttpReq *req) { + TfwHttpMsg *hm = (TfwHttpMsg *)req; int r; - TfwMsgParseIter *pit = &req->pit; - ssize_t h1_hdrs_sz; TfwHttpHdrTbl *ht = req->h_tbl; bool auth, host; - size_t pseudo_num; TfwStr tmp_host = {}, *host_val, *field, *end; - struct sk_buff *new_head = NULL, *old_head = NULL; TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, TFW_VHOST_HDRMOD_REQ); static const DEFINE_TFW_STR(sp, " "); @@ -4363,11 +4418,9 @@ tfw_h2_adjust_req(TfwHttpReq *req) + req->multipart_boundary_raw.len + SLEN(S_CRLF) }; int h_ct_replace = 0; - TfwStr h_cl = {0}; char cl_data[TFW_ULTOA_BUF_SIZ] = {0}; size_t cl_data_len = 0; size_t cl_len = 0; - TfwMsgIter it; /* * The Transfer-Encoding header field cannot be in the h2 request, because @@ -4379,7 +4432,8 @@ tfw_h2_adjust_req(TfwHttpReq *req) req->cleanup = tfw_pool_alloc(req->pool, sizeof(TfwHttpMsgCleanup)); if (unlikely(!req->cleanup)) return -ENOMEM; - memset(req->cleanup, 0, sizeof(TfwHttpMsgCleanup)); + req->cleanup->pages_sz = 0; + req->cleanup->skb_head = NULL; if (need_cl) { cl_data_len = tfw_ultoa(req->body.len, cl_data, TFW_ULTOA_BUF_SIZ); @@ -4408,67 +4462,6 @@ tfw_h2_adjust_req(TfwHttpReq *req) ht = req->h_tbl; auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); - pseudo_num = 3; /* Count authority as usual header for now. */ - /* - * Calculate http1.1 headers size. H2 request contains pseudo headers - * that are represented in different way in the http1.1 requests. - * pit->hdrs_cnt is aware of header duplicates. Redirection mark is - * ignored and not copied. - */ - h1_hdrs_sz = pit->hdrs_len - + (pit->hdrs_cnt - pseudo_num) * (SLEN(S_DLM) + SLEN(S_CRLF)); - /* First request line: remove pseudo headers names, all values are on - * the same line. - */ - h1_hdrs_sz += (long int)2 + SLEN(S_VERSION11) + SLEN(S_CRLF) - - ht->tbl[TFW_HTTP_HDR_H2_SCHEME].len - - SLEN(S_H2_METHOD) - - SLEN(S_H2_PATH) - + SLEN(S_CRLF) /* After headers */; - /* :authority pseudo header */ - if (auth) { - /* RFC 7540: - * An intermediary that converts an HTTP/2 request to HTTP/1.1 - * MUST create a Host header field if one is not present in a - * request by copying the value of the :authority pseudo-header - * field. - * AND - * Clients that generate HTTP/2 requests directly SHOULD use - * the :authority pseudo-header field instead of the Host - * header field. - */ - if (host) { - h1_hdrs_sz -= ht->tbl[TFW_HTTP_HDR_HOST].len - + SLEN(S_DLM) + SLEN(S_CRLF); - h1_hdrs_sz -= SLEN(S_H2_AUTH); - /* S_F_HOST already contains S_DLM */ - h1_hdrs_sz += SLEN(S_F_HOST) - SLEN(S_DLM); - } - else { - h1_hdrs_sz -= SLEN(S_H2_AUTH); - /* S_F_HOST already contains S_DLM */ - h1_hdrs_sz += SLEN(S_F_HOST) - SLEN(S_DLM); - } - } - - /* 'x-forwarded-for' header must be updated. */ - if (!TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR])) { - TfwStr *xff_hdr = &ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR]; - TfwStr *dup, *dup_end; - - TFW_STR_FOR_EACH_DUP(dup, xff_hdr, dup_end) { - h1_hdrs_sz -= dup->len + SLEN(S_DLM) + SLEN(S_CRLF); - } - } - h1_hdrs_sz += h_xff.len; - h1_hdrs_sz += h_via.len; - h1_hdrs_sz += cl_len; - - /* Adjust header size based on how many cookie headers there were in - * request. */ - if (TFW_STR_DUP(&ht->tbl[TFW_HTTP_HDR_COOKIE])) - h1_hdrs_sz -= (ht->tbl[TFW_HTTP_HDR_COOKIE].nchunks - 1) - * (SLEN("cookie") + SLEN(S_DLM)); /* * Conditional substitution/additions of 'content-type' header. This is @@ -4484,33 +4477,31 @@ tfw_h2_adjust_req(TfwHttpReq *req) || TFW_STR_EMPTY(h_ct_old))) return -EINVAL; - h1_hdrs_sz -= h_ct_old->len + SLEN(S_DLM) + SLEN(S_CRLF); - h1_hdrs_sz += h_ct.len; h_ct_replace = 1; } - if (WARN_ON_ONCE(h1_hdrs_sz < 0)) - return -EINVAL; - - r = tfw_msg_iter_setup(&it, tfw_http_msg_client_mem((TfwHttpMsg *)req), - &new_head, h1_hdrs_sz); + tfw_msg_transform_setup(&req->iter, req->msg.skb_head); + r = tfw_h1_req_cutoff_headers(req, req->cleanup); if (unlikely(r)) return r; /* First line. */ - r = __h2_write_method(req, &it); + r = __h2_write_method(req); if (unlikely(r)) - goto err; + return r; - r = tfw_msg_iter_write(&it, &sp); + r = tfw_http_msg_expand_from_pool(hm, &sp); if (unlikely(r)) - goto err; - r = tfw_msg_iter_write(&it, &req->uri_path); + return r; + + r = tfw_http_msg_expand_from_pool(hm, &req->uri_path); if (unlikely(r)) - goto err; - r = tfw_msg_iter_write(&it, &fl_end); /* start of Host: header */ + return r; + + r = tfw_http_msg_expand_from_pool(hm, &fl_end); if (unlikely(r)) - goto err; + return r; + if (h_mods && test_bit(TFW_HTTP_HDR_HOST, h_mods->spec_hdrs)) { host_val = &h_mods->hdrs[h_mods->host_off].hdr->chunks[1]; } @@ -4523,12 +4514,14 @@ tfw_h2_adjust_req(TfwHttpReq *req) __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], &tmp_host); host_val = &tmp_host; } - r = tfw_msg_iter_write(&it, host_val); + + r = tfw_http_msg_expand_from_pool(hm, host_val); if (unlikely(r)) - goto err; - r = tfw_msg_iter_write(&it, &STR_CRLF); + return r; + + r = tfw_http_msg_expand_from_pool(hm, &STR_CRLF); if (unlikely(r)) - goto err; + return r; /* Skip host header: it's already written. */ FOR_EACH_HDR_FIELD_FROM(field, end, req, TFW_HTTP_HDR_REGULAR) { @@ -4539,25 +4532,24 @@ tfw_h2_adjust_req(TfwHttpReq *req) case TFW_HTTP_HDR_HOST: continue; /* Already written. */ case TFW_HTTP_HDR_X_FORWARDED_FOR: - r = tfw_msg_iter_write(&it, &h_xff); + r = tfw_http_msg_expand_from_pool(hm, &h_xff); if (unlikely(r)) - goto err; + return r; continue; case TFW_HTTP_HDR_CONTENT_TYPE: if (h_ct_replace) { - r = tfw_msg_iter_write(&it, &h_ct); + r = tfw_http_msg_expand_from_pool(hm, &h_ct); if (unlikely(r)) - goto err; - continue; + return r; } break; case TFW_HTTP_HDR_COOKIE: if (!TFW_STR_DUP(field)) break; r = write_merged_cookie_headers( - &ht->tbl[TFW_HTTP_HDR_COOKIE], &it); + &ht->tbl[TFW_HTTP_HDR_COOKIE], hm); if (unlikely(r)) - goto err; + return r; continue; default: break; @@ -4570,7 +4562,7 @@ tfw_h2_adjust_req(TfwHttpReq *req) if (unlikely(TFW_STR_PLAIN(dup))) { r = -EINVAL; - goto err; + return r; } hval.chunks = dup->chunks; @@ -4580,35 +4572,33 @@ tfw_h2_adjust_req(TfwHttpReq *req) hval.nchunks++; hval.len += chunk->len; } - r = tfw_msg_iter_write(&it, &hval); + r = tfw_http_msg_expand_from_pool(hm, &hval); if (unlikely(r)) - goto err; - r = tfw_msg_iter_write(&it, &dlm); + return r; + r = tfw_http_msg_expand_from_pool(hm, &dlm); if (unlikely(r)) - goto err; + return r; hval.chunks += hval.nchunks; hval.nchunks = dup->nchunks - hval.nchunks; hval.len = dup->len - hval.len; - r = tfw_msg_iter_write(&it, &hval); + r = tfw_http_msg_expand_from_pool(hm, &hval); if (unlikely(r)) - goto err; + return r; - r = tfw_msg_iter_write(&it, &STR_CRLF); + r = tfw_http_msg_expand_from_pool(hm, &STR_CRLF); if (unlikely(r)) - goto err; + return r; } - if (unlikely(r)) - goto err; } - r = tfw_msg_iter_write(&it, &h_via); + r = tfw_http_msg_expand_from_pool(hm, &h_via); if (unlikely(r)) - goto err; + return r; if (need_cl) { - h_cl = (TfwStr) { + TfwStr h_cl = (TfwStr) { .chunks = (TfwStr []) { { .data = "Content-Length", .len = SLEN("Content-Length") }, { .data = S_DLM, .len = SLEN(S_DLM) }, @@ -4618,71 +4608,18 @@ tfw_h2_adjust_req(TfwHttpReq *req) .len = cl_len, .nchunks = 4 }; - r = tfw_msg_iter_write(&it, &h_cl); + r = tfw_http_msg_expand_from_pool(hm, &h_cl); if (unlikely(r)) - goto err; + return r; } /* Finally close headers. */ - r = tfw_msg_iter_write(&it, &STR_CRLF); + r = tfw_http_msg_expand_from_pool(hm, &STR_CRLF); if (unlikely(r)) - goto err; + return r; T_DBG3("%s: req [%p] converted to http1.1\n", __func__, req); - old_head = req->msg.skb_head; - req->cleanup->skb_head = old_head; - req->msg.skb_head = new_head; - - /* Http chains might add a mark for the message, keep it. */ - new_head->mark = old_head->mark; - - if (!TFW_STR_EMPTY(&req->body)) { - /* - * Request has a body. we have to detach it from the old - * skb_head and append to a new one. There might be trailing - * headers after the body, but we're already copied them before - * body. This is not a problem, but we have to drop the trailer - * part after the body to avoid sending the same headers twice. - * - * Body travels in a separate DATA frame thus it's always in - * it's own skb. - */ - struct sk_buff *b_skbs = old_head, *trailer; - size_t len = 0; - - do { - b_skbs = b_skbs->next; - if (WARN_ON_ONCE((b_skbs == old_head))) { - /* - * @new_head will be freed in "err" label. - * Prevent use after free. - */ - req->msg.skb_head = NULL; - goto err; - } - } while (b_skbs != req->body.skb); - - ss_skb_queue_split(old_head, b_skbs); - trailer = b_skbs; - do { - len += trailer->len; - trailer = trailer->next; - - } while ((trailer != b_skbs) && (len != req->body.len)); - ss_skb_queue_append(&req->msg.skb_head, b_skbs); - if (trailer != b_skbs) { - ss_skb_queue_split(req->msg.skb_head, trailer); - ss_skb_queue_append(&old_head, trailer); - } - } - - return 0; - -err: - ss_skb_queue_purge(&new_head); - T_DBG3("%s: req [%p] convertation to http1.1 has failed" - " with result (%d)\n", __func__, req, r); - return -EINVAL; + return r; } static unsigned long @@ -4712,51 +4649,19 @@ tfw_http_resp_get_conn_flags(TfwHttpResp *resp) return conn_flg; } -/* - * Prepare current response skb_head for cleaning and replace current skb_head - * with new empty skb. With this approach headers will be copied to new skb - * skipping body and logic related to finding the right place for cutting - * headers will be avoided. - */ -static int -tfw_http_resp_set_empty_skb_head(TfwHttpResp *resp, TfwHttpMsgCleanup *cleanup) -{ - TfwClientMem *cli_mem = TFW_SKB_CB(resp->msg.skb_head)->cli_mem; - TfwMsgIter *iter = &resp->iter; - struct sk_buff *nskb; - - nskb = ss_skb_alloc(0); - if (unlikely(!nskb)) - return -ENOMEM; - - ss_skb_set_owner(nskb, cli_mem, nskb->truesize); - nskb->mark = resp->msg.skb_head->mark; - cleanup->skb_head = resp->msg.skb_head; - resp->msg.skb_head = NULL; - ss_skb_queue_tail(&resp->msg.skb_head, nskb); - iter->skb_head = resp->msg.skb_head; - iter->skb = resp->msg.skb_head; - - return 0; -} - static int tfw_h1_resp_cutoff_headers(TfwHttpResp *resp, TfwHttpMsgCleanup *cleanup) { TfwHttpMsg *hm = (TfwHttpMsg *)resp; TfwHttpReq *req = resp->req; - int r; + int r = 0; /* Response for PURGE/HEAD request. */ if ((test_bit(TFW_HTTP_B_PURGE_GET, req->flags) || test_bit(TFW_HTTP_B_REQ_HEAD_TO_GET, req->flags)) && resp->body.len > 0) { - /* Clean current response skb_head if body exists. */ - r = tfw_http_resp_set_empty_skb_head(resp, cleanup); - if (unlikely(r)) - return r; - + tfw_http_msg_set_empty_skb_head(hm, cleanup); if (test_bit(TFW_HTTP_B_PURGE_GET, req->flags)) { /* * For a response to PURGE request we drop the body. diff --git a/fw/http_msg.c b/fw/http_msg.c index 9f1f79dd1..3139ae727 100644 --- a/fw/http_msg.c +++ b/fw/http_msg.c @@ -1320,8 +1320,8 @@ __tfw_http_msg_move_frags(struct sk_buff *skb, int frag_idx, (si->nr_frags) * sizeof(skb_frag_t)); } -static inline void -__tfw_http_msg_rm_all_frags(struct sk_buff *skb, TfwHttpMsgCleanup *cleanup) +void +tfw_http_msg_rm_all_frags(struct sk_buff *skb, TfwHttpMsgCleanup *cleanup) { int i, len; struct skb_shared_info *si = skb_shinfo(skb); @@ -1400,7 +1400,7 @@ tfw_http_msg_cutoff_headers(TfwHttpMsg *hm, TfwHttpMsgCleanup* cleanup) * fragments from skb where LF is located. */ if (!body) { - __tfw_http_msg_rm_all_frags(it->skb, cleanup); + tfw_http_msg_rm_all_frags(it->skb, cleanup); goto end; } else if (off != begin) { /* diff --git a/fw/http_msg.h b/fw/http_msg.h index 22ed7375f..a20eb637c 100644 --- a/fw/http_msg.h +++ b/fw/http_msg.h @@ -164,6 +164,7 @@ int tfw_h2_msg_expand_from_pool_lc(TfwHttpMsg *hm, const TfwStr *str, TfwHttpTransIter *mit); int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr); int __http_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr); +void tfw_http_msg_rm_all_frags(struct sk_buff *skb, TfwHttpMsgCleanup *cleanup); int tfw_http_msg_cutoff_headers(TfwHttpMsg *hm, TfwHttpMsgCleanup* cleanup); #define TFW_H2_MSG_HDR_ADD(hm, name, val, idx) \ From adfa33a0cf3bbb73e0f6902f6ca4dbb7d6e554d2 Mon Sep 17 00:00:00 2001 From: EvgeniiMekhanik Date: Mon, 15 Jun 2026 13:10:51 +0300 Subject: [PATCH 9/9] Fix according review Remove `ss_skb_dflt_destructor` - this function is currently used only in one place. Additionally, the function name does not accurately reflect its behavior. --- fw/ss_skb.c | 30 +++++++++++++++++------------- fw/ss_skb.h | 21 ++------------------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/fw/ss_skb.c b/fw/ss_skb.c index ce406635a..7ea19c804 100644 --- a/fw/ss_skb.c +++ b/fw/ss_skb.c @@ -1744,22 +1744,26 @@ ss_skb_realloc_headroom(struct sk_buff *skb) } ALLOW_ERROR_INJECTION(ss_skb_realloc_headroom, ERRNO); +/* + * Orphan skb from Tempesta-specific ownership. + * We use our own version (instead of using `skb_orphan`) to + * don't use `skb->sk` field inside Tempesta FW source code. + */ void -ss_skb_dflt_destructor(struct sk_buff *skb) +ss_skb_orphan(struct sk_buff *skb) { - TfwClientMem *cli_mem = - (TfwClientMem *)TFW_SKB_CB(skb)->cli_mem; + TfwClientMem *cli_mem; - /* - * If skb is in socket write queue, skb->cb is used to store - * `struct tcp_sock` data. Inside `ss_skb_adjust_client_mem` - * we also access skb->cb, so we corrupt `tcp_sock` data for - * skbs in socket write queue. - */ - BUG_ON(skb_tfw_is_in_socket_write_queue(skb)); - ss_skb_adjust_client_mem(skb, -TFW_SKB_CB(skb)->mem); - WARN_ON(TFW_SKB_CB(skb)->mem); - tfw_client_mem_put(cli_mem); + if (skb_tfw_is_in_socket_write_queue(skb)) + return; + + cli_mem = (TfwClientMem *)TFW_SKB_CB(skb)->cli_mem; + if (cli_mem) { + ss_skb_adjust_client_mem(skb, -TFW_SKB_CB(skb)->mem); + WARN_ON(TFW_SKB_CB(skb)->mem); + tfw_client_mem_put(cli_mem); + TFW_SKB_CB(skb)->cli_mem = NULL; + } } void diff --git a/fw/ss_skb.h b/fw/ss_skb.h index ef4d41b9a..062d9aa54 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -72,11 +72,11 @@ struct tfw_skb_cb { #define TFW_SKB_CB(skb) ((struct tfw_skb_cb *)&((skb)->cb[0])) +void ss_skb_orphan(struct sk_buff *skb); +void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); void ss_skb_set_owner(struct sk_buff *skb, TfwClientMem *owner, unsigned int delta); void ss_skb_adjust_client_mem(struct sk_buff *skb, int delta); -void ss_skb_dflt_destructor(struct sk_buff *skb); -void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); static inline bool ss_skb_is_within_fragment(char *begin_fragment, char *position, @@ -190,23 +190,6 @@ ss_skb_queue_splice(struct sk_buff **skb_head, struct sk_buff **skb) *skb = NULL; } -/* - * Orphan skb from Tempesta-specific ownership. - * We use our own version (instead of using `skb_orphan`) to - * don't use `skb->sk` field inside Tempesta FW source code. - */ -static inline void -ss_skb_orphan(struct sk_buff *skb) -{ - if (skb_tfw_is_in_socket_write_queue(skb)) - return; - - if (TFW_SKB_CB(skb)->cli_mem) { - ss_skb_dflt_destructor(skb); - TFW_SKB_CB(skb)->cli_mem = NULL; - } -} - static inline void __ss_kfree_skb(struct sk_buff *skb) {