diff --git a/lib/conntrack.c b/lib/conntrack.c index fee8c05da..7c4e80a6b 100644 --- a/lib/conntrack.c +++ b/lib/conntrack.c @@ -914,7 +914,6 @@ nat_inner_packet(struct dp_packet *pkt, struct conn_key *key, const char *inner_l4 = NULL; uint16_t orig_l3_ofs = pkt->l3_ofs; uint16_t orig_l4_ofs = pkt->l4_ofs; - uint64_t orig_ol_flags = *dp_packet_ol_flags_ptr(pkt); uint32_t orig_offloads = pkt->offloads; void *l3 = dp_packet_l3(pkt); @@ -936,8 +935,8 @@ nat_inner_packet(struct dp_packet *pkt, struct conn_key *key, pkt->l4_ofs += inner_l4 - (char *) l4; /* Drop any offloads to force below helpers to calculate checksums * if needed. */ - *dp_packet_ol_flags_ptr(pkt) &= ~DP_PACKET_OL_TX_ANY_CKSUM; dp_packet_ip_checksum_set_unknown(pkt); + dp_packet_l4_checksum_set_unknown(pkt); /* Reverse the key for inner packet. */ struct conn_key rev_key = *key; @@ -963,7 +962,6 @@ nat_inner_packet(struct dp_packet *pkt, struct conn_key *key, pkt->l3_ofs = orig_l3_ofs; pkt->l4_ofs = orig_l4_ofs; - *dp_packet_ol_flags_ptr(pkt) = orig_ol_flags; pkt->offloads = orig_offloads; } @@ -2262,8 +2260,7 @@ conn_key_extract(struct conntrack *ct, struct dp_packet *pkt, ovs_be16 dl_type, /* Validate the checksum only when hwol is not supported. */ if (extract_l4(&ctx->key, l4, dp_packet_l4_size(pkt), &ctx->icmp_related, l3, - !dp_packet_l4_checksum_good(pkt) && - !dp_packet_hwol_tx_l4_checksum(pkt), + dp_packet_l4_checksum_unknown(pkt), NULL)) { ctx->hash = conn_key_hash(&ctx->key, ct->hash_basis); return true; @@ -3702,8 +3699,8 @@ handle_ftp_ctl(struct conntrack *ct, const struct conn_lookup_ctx *ctx, adj_seqnum(&th->tcp_seq, ec->seq_skew); } - if (dp_packet_hwol_tx_l4_checksum(pkt)) { - dp_packet_ol_reset_l4_csum_good(pkt); + if (dp_packet_l4_checksum_valid(pkt)) { + dp_packet_l4_checksum_set_partial(pkt); } else { th->tcp_csum = 0; if (ctx->key.dl_type == htons(ETH_TYPE_IPV6)) { diff --git a/lib/dp-packet-gso.c b/lib/dp-packet-gso.c index f49a296cd..5698c3f65 100644 --- a/lib/dp-packet-gso.c +++ b/lib/dp-packet-gso.c @@ -160,6 +160,7 @@ dp_packet_gso(struct dp_packet *p, struct dp_packet_batch **batches) tnl_hdr = dp_packet_l4(seg); tnl_hdr->udp_len = htons(dp_packet_l4_size(seg)); + dp_packet_l4_checksum_set_partial(seg); } if (udp_tnl || gre_tnl) { @@ -198,8 +199,10 @@ dp_packet_gso(struct dp_packet *p, struct dp_packet_batch **batches) /* Update L4 header. */ if (udp_tnl || gre_tnl) { tcp_hdr = dp_packet_inner_l4(seg); + dp_packet_inner_l4_checksum_set_partial(seg); } else { tcp_hdr = dp_packet_l4(seg); + dp_packet_l4_checksum_set_partial(seg); } put_16aligned_be32(&tcp_hdr->tcp_seq, htonl(tcp_seq)); tcp_seq += seg_len; diff --git a/lib/dp-packet.c b/lib/dp-packet.c index adcd5e52f..40246ca42 100644 --- a/lib/dp-packet.c +++ b/lib/dp-packet.c @@ -557,8 +557,9 @@ void dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t flags) { if (!dp_packet_ip_checksum_partial(p) + && !dp_packet_l4_checksum_partial(p) && !dp_packet_inner_ip_checksum_partial(p) - && !dp_packet_hwol_tx_is_any_csum(p)) { + && !dp_packet_inner_l4_checksum_partial(p)) { /* Only checksumming needs actions. */ return; } @@ -569,27 +570,20 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t flags) dp_packet_ip_set_header_csum(p, false); } - if (dp_packet_hwol_tx_l4_checksum(p)) { - if (dp_packet_l4_checksum_good(p)) { - dp_packet_hwol_reset_tx_l4_csum(p); - return; - } - - if (dp_packet_hwol_l4_is_tcp(p) - && !(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) { - packet_tcp_complete_csum(p, false); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_tx_l4_csum(p); - } else if (dp_packet_hwol_l4_is_udp(p) - && !(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) { - packet_udp_complete_csum(p, false); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_tx_l4_csum(p); - } else if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM) - && dp_packet_hwol_l4_is_sctp(p)) { - packet_sctp_complete_csum(p, false); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_tx_l4_csum(p); + if (dp_packet_l4_checksum_partial(p)) { + if (dp_packet_l4_proto_tcp(p)) { + if (!(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) { + packet_tcp_complete_csum(p, false); + } + } else if (dp_packet_l4_proto_udp(p)) { + if (!(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) { + packet_udp_complete_csum(p, false); + } + } else { + ovs_assert(dp_packet_l4_proto_sctp(p)); + if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM)) { + packet_sctp_complete_csum(p, false); + } } } @@ -603,8 +597,8 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t flags) * support inner checksum offload and an outer UDP checksum is * required, then we can't offload inner checksum either. As that would * invalidate the outer checksum. */ - if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM) && - dp_packet_hwol_is_outer_udp_cksum(p)) { + if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM) + && dp_packet_l4_checksum_partial(p)) { flags &= ~(NETDEV_TX_OFFLOAD_TCP_CKSUM | NETDEV_TX_OFFLOAD_UDP_CKSUM | NETDEV_TX_OFFLOAD_SCTP_CKSUM | @@ -617,22 +611,20 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t flags) dp_packet_ip_set_header_csum(p, true); } - if (dp_packet_hwol_tx_l4_checksum(p)) { - if (dp_packet_hwol_l4_is_tcp(p) - && !(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) { - packet_tcp_complete_csum(p, true); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_tx_l4_csum(p); - } else if (dp_packet_hwol_l4_is_udp(p) - && !(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) { - packet_udp_complete_csum(p, true); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_tx_l4_csum(p); - } else if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM) - && dp_packet_hwol_l4_is_sctp(p)) { - packet_sctp_complete_csum(p, true); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_tx_l4_csum(p); + if (dp_packet_inner_l4_checksum_partial(p)) { + if (dp_packet_inner_l4_proto_tcp(p)) { + if (!(flags & NETDEV_TX_OFFLOAD_TCP_CKSUM)) { + packet_tcp_complete_csum(p, true); + } + } else if (dp_packet_inner_l4_proto_udp(p)) { + if (!(flags & NETDEV_TX_OFFLOAD_UDP_CKSUM)) { + packet_udp_complete_csum(p, true); + } + } else { + ovs_assert(dp_packet_inner_l4_proto_sctp(p)); + if (!(flags & NETDEV_TX_OFFLOAD_SCTP_CKSUM)) { + packet_sctp_complete_csum(p, true); + } } } @@ -641,13 +633,10 @@ dp_packet_ol_send_prepare(struct dp_packet *p, uint64_t flags) dp_packet_ip_set_header_csum(p, false); } - if (!dp_packet_hwol_is_outer_udp_cksum(p)) { - return; - } - - if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM)) { - packet_udp_complete_csum(p, false); - dp_packet_ol_set_l4_csum_good(p); - dp_packet_hwol_reset_outer_udp_csum(p); + if (dp_packet_l4_checksum_partial(p)) { + ovs_assert(dp_packet_l4_proto_udp(p)); + if (!(flags & NETDEV_TX_OFFLOAD_OUTER_UDP_CKSUM)) { + packet_udp_complete_csum(p, false); + } } } diff --git a/lib/dp-packet.h b/lib/dp-packet.h index 1ea3cf776..12aa2d0bf 100644 --- a/lib/dp-packet.h +++ b/lib/dp-packet.h @@ -60,30 +60,11 @@ enum { /* Value 0 is not used. */ /* TCP Segmentation Offload. */ DEF_OL_FLAG(DP_PACKET_OL_TX_TCP_SEG, RTE_MBUF_F_TX_TCP_SEG, 0x40), - /* Offload TCP checksum. */ - DEF_OL_FLAG(DP_PACKET_OL_TX_TCP_CKSUM, RTE_MBUF_F_TX_TCP_CKSUM, 0x200), - /* Offload UDP checksum. */ - DEF_OL_FLAG(DP_PACKET_OL_TX_UDP_CKSUM, RTE_MBUF_F_TX_UDP_CKSUM, 0x400), - /* Offload SCTP checksum. */ - DEF_OL_FLAG(DP_PACKET_OL_TX_SCTP_CKSUM, RTE_MBUF_F_TX_SCTP_CKSUM, 0x800), - /* Offload tunnel outer UDP checksum. */ - DEF_OL_FLAG(DP_PACKET_OL_TX_OUTER_UDP_CKSUM, - RTE_MBUF_F_TX_OUTER_UDP_CKSUM, 0x20000), /* Adding new field requires adding to DP_PACKET_OL_SUPPORTED_MASK. */ }; -#define DP_PACKET_OL_SUPPORTED_MASK (DP_PACKET_OL_TX_TCP_SEG | \ - DP_PACKET_OL_TX_TCP_CKSUM | \ - DP_PACKET_OL_TX_UDP_CKSUM | \ - DP_PACKET_OL_TX_SCTP_CKSUM | \ - DP_PACKET_OL_TX_OUTER_UDP_CKSUM) - -#define DP_PACKET_OL_TX_L4_MASK (DP_PACKET_OL_TX_TCP_CKSUM | \ - DP_PACKET_OL_TX_UDP_CKSUM | \ - DP_PACKET_OL_TX_SCTP_CKSUM) -#define DP_PACKET_OL_TX_ANY_CKSUM (DP_PACKET_OL_TX_L4_MASK | \ - DP_PACKET_OL_TX_OUTER_UDP_CKSUM) +#define DP_PACKET_OL_SUPPORTED_MASK DP_PACKET_OL_TX_TCP_SEG /* Bit masks for the 'offloads' member of the 'dp_packet' structure. */ enum OVS_PACKED_ENUM dp_packet_offload_mask { @@ -97,6 +78,10 @@ enum OVS_PACKED_ENUM dp_packet_offload_mask { /* Valid L4 checksum in the packet. */ DP_PACKET_OL_L4_CKSUM_GOOD = UINT16_C(1) << 8, + /* Protocol corresponding to above L4 checksums. */ + DP_PACKET_OL_L4_PROTO_TCP = UINT16_C(1) << 9, + DP_PACKET_OL_L4_PROTO_UDP = UINT16_C(1) << 10, + /* Bits for marking a packet as tunneled. */ DP_PACKET_OL_TUNNEL_GENEVE = UINT16_C(1) << 11, DP_PACKET_OL_TUNNEL_VXLAN = UINT16_C(1) << 12, @@ -108,6 +93,14 @@ enum OVS_PACKED_ENUM dp_packet_offload_mask { DP_PACKET_OL_IP_CKSUM_BAD << DP_PACKET_OL_SHIFT_COUNT, DP_PACKET_OL_INNER_IP_CKSUM_GOOD = DP_PACKET_OL_IP_CKSUM_GOOD << DP_PACKET_OL_SHIFT_COUNT, + DP_PACKET_OL_INNER_L4_CKSUM_BAD = + DP_PACKET_OL_L4_CKSUM_BAD << DP_PACKET_OL_SHIFT_COUNT, + DP_PACKET_OL_INNER_L4_CKSUM_GOOD = + DP_PACKET_OL_L4_CKSUM_GOOD << DP_PACKET_OL_SHIFT_COUNT, + DP_PACKET_OL_INNER_L4_PROTO_TCP = + DP_PACKET_OL_L4_PROTO_TCP << DP_PACKET_OL_SHIFT_COUNT, + DP_PACKET_OL_INNER_L4_PROTO_UDP = + DP_PACKET_OL_L4_PROTO_UDP << DP_PACKET_OL_SHIFT_COUNT, }; #ifdef DPDK_NETDEV @@ -125,9 +118,18 @@ BUILD_ASSERT_DECL(DP_PACKET_OL_L4_CKSUM_GOOD == RTE_MBUF_F_RX_L4_CKSUM_GOOD); #define DP_PACKET_OL_TUNNEL_MASK (DP_PACKET_OL_TUNNEL_GENEVE \ | DP_PACKET_OL_TUNNEL_VXLAN) +#define DP_PACKET_OL_L4_PROTO_MASK (DP_PACKET_OL_L4_PROTO_TCP \ + | DP_PACKET_OL_L4_PROTO_UDP) + #define DP_PACKET_OL_INNER_IP_CKSUM_MASK (DP_PACKET_OL_INNER_IP_CKSUM_GOOD \ | DP_PACKET_OL_INNER_IP_CKSUM_BAD) +#define DP_PACKET_OL_INNER_L4_CKSUM_MASK (DP_PACKET_OL_INNER_L4_CKSUM_GOOD \ + | DP_PACKET_OL_INNER_L4_CKSUM_BAD) + +#define DP_PACKET_OL_INNER_L4_PROTO_MASK (DP_PACKET_OL_INNER_L4_PROTO_TCP \ + | DP_PACKET_OL_INNER_L4_PROTO_UDP) + /* Buffer for holding packet data. A dp_packet is automatically reallocated * as necessary if it grows too large for the available memory. * By default the packet type is set to Ethernet (PT_ETH). @@ -1120,20 +1122,6 @@ dp_packet_tunnel(const struct dp_packet *b) return !!(b->offloads & DP_PACKET_OL_TUNNEL_MASK); } -/* Returns the L4 cksum offload bitmask. */ -static inline uint64_t -dp_packet_hwol_l4_mask(const struct dp_packet *b) -{ - return *dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK; -} - -/* Return true if the packet 'b' requested L4 checksum offload. */ -static inline bool -dp_packet_hwol_tx_l4_checksum(const struct dp_packet *b) -{ - return !!dp_packet_hwol_l4_mask(b); -} - /* Returns 'true' if packet 'b' is marked for TCP segmentation offloading. */ static inline bool dp_packet_hwol_is_tso(const struct dp_packet *b) @@ -1141,74 +1129,6 @@ dp_packet_hwol_is_tso(const struct dp_packet *b) return !!(*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_TCP_SEG); } -/* Returns 'true' if packet 'b' is marked for TCP checksum offloading. */ -static inline bool -dp_packet_hwol_l4_is_tcp(const struct dp_packet *b) -{ - return (*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK) == - DP_PACKET_OL_TX_TCP_CKSUM; -} - -/* Returns 'true' if packet 'b' is marked for UDP checksum offloading. */ -static inline bool -dp_packet_hwol_l4_is_udp(struct dp_packet *b) -{ - return (*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK) == - DP_PACKET_OL_TX_UDP_CKSUM; -} - -/* Returns 'true' if packet 'b' is marked for SCTP checksum offloading. */ -static inline bool -dp_packet_hwol_l4_is_sctp(struct dp_packet *b) -{ - return (*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_L4_MASK) == - DP_PACKET_OL_TX_SCTP_CKSUM; -} - -/* Returns 'true' if packet 'b' is marked for outer UDP checksum offload. */ -static inline bool -dp_packet_hwol_is_outer_udp_cksum(struct dp_packet *b) -{ - return !!(*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_OUTER_UDP_CKSUM); -} - -/* Returns 'true' if packet 'b' is marked for any checksum offload. */ -static inline bool -dp_packet_hwol_tx_is_any_csum(struct dp_packet *b) -{ - return !!(*dp_packet_ol_flags_ptr(b) & DP_PACKET_OL_TX_ANY_CKSUM); -} - -static inline void -dp_packet_hwol_reset_tx_l4_csum(struct dp_packet *p) -{ - *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_TX_L4_MASK; -} - -/* Mark packet 'b' for TCP checksum offloading. It implies that either - * the packet 'b' is marked for IPv4 or IPv6 checksum offloading. */ -static inline void -dp_packet_hwol_set_csum_tcp(struct dp_packet *b) -{ - *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_TCP_CKSUM; -} - -/* Mark packet 'b' for UDP checksum offloading. It implies that either - * the packet 'b' is marked for IPv4 or IPv6 checksum offloading. */ -static inline void -dp_packet_hwol_set_csum_udp(struct dp_packet *b) -{ - *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_UDP_CKSUM; -} - -/* Mark packet 'b' for SCTP checksum offloading. It implies that either - * the packet 'b' is marked for IPv4 or IPv6 checksum offloading. */ -static inline void -dp_packet_hwol_set_csum_sctp(struct dp_packet *b) -{ - *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_SCTP_CKSUM; -} - /* Mark packet 'b' for TCP segmentation offloading. It implies that * either the packet 'b' is marked for IPv4 or IPv6 checksum offloading * and also for TCP checksum offloading. */ @@ -1218,36 +1138,11 @@ dp_packet_hwol_set_tcp_seg(struct dp_packet *b) *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_TCP_SEG; } -static inline void -dp_packet_hwol_reset_outer_udp_csum(struct dp_packet *p) -{ - *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_TX_OUTER_UDP_CKSUM; -} - -/* Mark packet 'b' for csum offloading in outer UDP header. */ -static inline void -dp_packet_hwol_set_outer_udp_csum(struct dp_packet *b) -{ - *dp_packet_ol_flags_ptr(b) |= DP_PACKET_OL_TX_OUTER_UDP_CKSUM; -} - -/* Resets TCP Segmentation in packet 'p' and adjust flags to indicate - * L3 and L4 checksumming is now required. */ +/* Resets TCP Segmentation in packet 'p'. */ static inline void dp_packet_hwol_reset_tcp_seg(struct dp_packet *p) { - uint64_t ol_flags = *dp_packet_ol_flags_ptr(p) - | DP_PACKET_OL_TX_TCP_CKSUM; - - ol_flags &= ~DP_PACKET_OL_TX_TCP_SEG; - p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD; - - if (dp_packet_tunnel_geneve(p) - || dp_packet_tunnel_vxlan(p)) { - ol_flags |= DP_PACKET_OL_TX_OUTER_UDP_CKSUM; - } - - *dp_packet_ol_flags_ptr(p) = ol_flags; + *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_TX_TCP_SEG; } /* Marks packet 'p' with good IPv4 checksum. */ @@ -1362,26 +1257,82 @@ dp_packet_ip_set_header_csum(struct dp_packet *p, bool inner) } } +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_l4_proto_tcp(const struct dp_packet *b) +{ + return (b->offloads & DP_PACKET_OL_L4_PROTO_MASK) + == DP_PACKET_OL_L4_PROTO_TCP; +} + +static inline void +dp_packet_l4_proto_set_tcp(struct dp_packet *b) +{ + b->offloads &= ~DP_PACKET_OL_L4_PROTO_UDP; + b->offloads |= DP_PACKET_OL_L4_PROTO_TCP; +} + +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_l4_proto_udp(const struct dp_packet *b) +{ + return (b->offloads & DP_PACKET_OL_L4_PROTO_MASK) + == DP_PACKET_OL_L4_PROTO_UDP; +} + +static inline void +dp_packet_l4_proto_set_udp(struct dp_packet *b) +{ + b->offloads &= ~DP_PACKET_OL_L4_PROTO_TCP; + b->offloads |= DP_PACKET_OL_L4_PROTO_UDP; +} + +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_l4_proto_sctp(const struct dp_packet *b) +{ + return (b->offloads & DP_PACKET_OL_L4_PROTO_MASK) + == DP_PACKET_OL_L4_PROTO_MASK; +} + +static inline void +dp_packet_l4_proto_set_sctp(struct dp_packet *b) +{ + b->offloads |= DP_PACKET_OL_L4_PROTO_MASK; +} + /* Returns 'true' if the packet 'p' has good integrity and the * checksum in it is correct. */ -static inline bool +static inline bool OVS_WARN_UNUSED_RESULT dp_packet_l4_checksum_good(const struct dp_packet *p) { return (p->offloads & DP_PACKET_OL_L4_CKSUM_MASK) == DP_PACKET_OL_L4_CKSUM_GOOD; } -static inline bool +/* Marks packet 'p' with good L4 checksum. */ +static inline void +dp_packet_l4_checksum_set_good(struct dp_packet *p) +{ + p->offloads &= ~DP_PACKET_OL_L4_CKSUM_BAD; + p->offloads |= DP_PACKET_OL_L4_CKSUM_GOOD; +} + +static inline bool OVS_WARN_UNUSED_RESULT dp_packet_l4_checksum_bad(const struct dp_packet *p) { return (p->offloads & DP_PACKET_OL_L4_CKSUM_MASK) == DP_PACKET_OL_L4_CKSUM_BAD; } +static inline void +dp_packet_l4_checksum_set_bad(struct dp_packet *p) +{ + p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD; + p->offloads |= DP_PACKET_OL_L4_CKSUM_BAD; +} + /* Returns 'true' if the packet has good integrity though the * checksum in the packet 'p' is not complete. */ -static inline bool -dp_packet_ol_l4_csum_partial(const struct dp_packet *p) +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_l4_checksum_partial(const struct dp_packet *p) { return (p->offloads & DP_PACKET_OL_L4_CKSUM_MASK) == DP_PACKET_OL_L4_CKSUM_MASK; @@ -1390,33 +1341,81 @@ dp_packet_ol_l4_csum_partial(const struct dp_packet *p) /* Marks packet 'p' with good integrity though the checksum in the * packet is not complete. */ static inline void -dp_packet_ol_set_l4_csum_partial(struct dp_packet *p) +dp_packet_l4_checksum_set_partial(struct dp_packet *p) { p->offloads |= DP_PACKET_OL_L4_CKSUM_MASK; } -/* Marks packet 'p' with good L4 checksum. */ -static inline void -dp_packet_ol_set_l4_csum_good(struct dp_packet *p) +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_l4_checksum_unknown(const struct dp_packet *p) { - p->offloads &= ~DP_PACKET_OL_L4_CKSUM_BAD; - p->offloads |= DP_PACKET_OL_L4_CKSUM_GOOD; -} - -/* Marks packet 'p' with good L4 checksum as modified. */ -static inline void -dp_packet_ol_reset_l4_csum_good(struct dp_packet *p) -{ - if (!dp_packet_ol_l4_csum_partial(p)) { - p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD; - } + return !(p->offloads & DP_PACKET_OL_L4_CKSUM_MASK); } static inline void -dp_packet_ol_set_l4_csum_bad(struct dp_packet *p) +dp_packet_l4_checksum_set_unknown(struct dp_packet *p) { - p->offloads &= ~DP_PACKET_OL_L4_CKSUM_GOOD; - p->offloads |= DP_PACKET_OL_L4_CKSUM_BAD; + p->offloads &= ~DP_PACKET_OL_L4_CKSUM_MASK; +} + +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_l4_checksum_valid(const struct dp_packet *p) +{ + return !!(p->offloads & DP_PACKET_OL_L4_CKSUM_GOOD); +} + +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_inner_l4_proto_tcp(const struct dp_packet *p) +{ + return (p->offloads & DP_PACKET_OL_INNER_L4_PROTO_MASK) + == DP_PACKET_OL_INNER_L4_PROTO_TCP; +} + +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_inner_l4_proto_udp(const struct dp_packet *p) +{ + return (p->offloads & DP_PACKET_OL_INNER_L4_PROTO_MASK) + == DP_PACKET_OL_INNER_L4_PROTO_UDP; +} + +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_inner_l4_proto_sctp(const struct dp_packet *p) +{ + return (p->offloads & DP_PACKET_OL_INNER_L4_PROTO_MASK) + == DP_PACKET_OL_INNER_L4_PROTO_MASK; +} + +/* Returns 'true' if the inner L4 header has good integrity and the + * checksum in it is complete. */ +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_inner_l4_checksum_good(const struct dp_packet *p) +{ + return (p->offloads & DP_PACKET_OL_INNER_L4_CKSUM_MASK) + == DP_PACKET_OL_INNER_L4_CKSUM_GOOD; +} + +/* Marks packet 'p' as having a valid inner l4 header, but no checksum. */ +static inline void +dp_packet_inner_l4_checksum_set_good(struct dp_packet *p) +{ + p->offloads &= ~DP_PACKET_OL_INNER_L4_CKSUM_BAD; + p->offloads |= DP_PACKET_OL_INNER_L4_CKSUM_GOOD; +} + +/* Returns 'true' if the inner L4 header has good integrity but the + * checksum in it is incomplete. */ +static inline bool OVS_WARN_UNUSED_RESULT +dp_packet_inner_l4_checksum_partial(const struct dp_packet *p) +{ + return (p->offloads & DP_PACKET_OL_INNER_L4_CKSUM_MASK) + == DP_PACKET_OL_INNER_L4_CKSUM_MASK; +} + +/* Marks packet 'p' as having a valid inner l4 header, but no checksum. */ +static inline void +dp_packet_inner_l4_checksum_set_partial(struct dp_packet *p) +{ + p->offloads |= DP_PACKET_OL_INNER_L4_CKSUM_MASK; } static inline void diff --git a/lib/dpif-netdev-extract-avx512.c b/lib/dpif-netdev-extract-avx512.c index 3ad892d37..90bf53a7a 100644 --- a/lib/dpif-netdev-extract-avx512.c +++ b/lib/dpif-netdev-extract-avx512.c @@ -758,24 +758,6 @@ mfex_check_tcp_data_offset(const struct tcp_header *tcp) return ret; } -static void -mfex_tcp_set_hwol(struct dp_packet *pkt) -{ - if (dp_packet_l4_checksum_good(pkt) - || dp_packet_ol_l4_csum_partial(pkt)) { - dp_packet_hwol_set_csum_tcp(pkt); - } -} - -static void -mfex_udp_set_hwol(struct dp_packet *pkt) -{ - if (dp_packet_l4_checksum_good(pkt) - || dp_packet_ol_l4_csum_partial(pkt)) { - dp_packet_hwol_set_csum_udp(pkt); - } -} - /* Generic loop to process any mfex profile. This code is specialized into * multiple actual MFEX implementation functions. Its marked ALWAYS_INLINE * to ensure the compiler specializes each instance. The code is marked "hot" @@ -877,7 +859,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, const struct tcp_header *tcp = (void *)&pkt[38]; mfex_handle_tcp_flags(tcp, &blocks[7]); dp_packet_update_rss_hash_ipv4_tcp_udp(packet); - mfex_tcp_set_hwol(packet); + dp_packet_l4_proto_set_tcp(packet); } break; case PROFILE_ETH_VLAN_IPV4_UDP: { @@ -890,7 +872,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, continue; } dp_packet_update_rss_hash_ipv4_tcp_udp(packet); - mfex_udp_set_hwol(packet); + dp_packet_l4_proto_set_udp(packet); } break; case PROFILE_ETH_IPV4_TCP: { @@ -906,7 +888,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, continue; } dp_packet_update_rss_hash_ipv4_tcp_udp(packet); - mfex_tcp_set_hwol(packet); + dp_packet_l4_proto_set_tcp(packet); } break; case PROFILE_ETH_IPV4_UDP: { @@ -918,7 +900,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, continue; } dp_packet_update_rss_hash_ipv4_tcp_udp(packet); - mfex_udp_set_hwol(packet); + dp_packet_l4_proto_set_udp(packet); } break; case PROFILE_ETH_IPV6_UDP: { @@ -937,7 +919,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, /* Process UDP header. */ mfex_handle_ipv6_l4((void *)&pkt[54], &blocks[9]); dp_packet_update_rss_hash_ipv6_tcp_udp(packet); - mfex_udp_set_hwol(packet); + dp_packet_l4_proto_set_udp(packet); } break; case PROFILE_ETH_IPV6_TCP: { @@ -961,7 +943,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, } mfex_handle_tcp_flags(tcp, &blocks[9]); dp_packet_update_rss_hash_ipv6_tcp_udp(packet); - mfex_tcp_set_hwol(packet); + dp_packet_l4_proto_set_tcp(packet); } break; case PROFILE_ETH_VLAN_IPV6_TCP: { @@ -988,7 +970,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, } mfex_handle_tcp_flags(tcp, &blocks[10]); dp_packet_update_rss_hash_ipv6_tcp_udp(packet); - mfex_tcp_set_hwol(packet); + dp_packet_l4_proto_set_tcp(packet); } break; case PROFILE_ETH_VLAN_IPV6_UDP: { @@ -1010,7 +992,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, /* Process UDP header. */ mfex_handle_ipv6_l4((void *)&pkt[58], &blocks[10]); dp_packet_update_rss_hash_ipv6_tcp_udp(packet); - mfex_udp_set_hwol(packet); + dp_packet_l4_proto_set_udp(packet); } break; case PROFILE_ETH_IPV4_NVGRE: { @@ -1021,7 +1003,7 @@ mfex_avx512_process(struct dp_packet_batch *packets, continue; } dp_packet_update_rss_hash_ipv4(packet); - mfex_udp_set_hwol(packet); + dp_packet_l4_proto_set_udp(packet); } break; default: diff --git a/lib/flow.c b/lib/flow.c index 6c8ce2e1a..b522f7f11 100644 --- a/lib/flow.c +++ b/lib/flow.c @@ -810,7 +810,6 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst) uint8_t nw_frag, nw_tos, nw_ttl, nw_proto; uint8_t *ct_nw_proto_p = NULL; ovs_be16 ct_tp_src = 0, ct_tp_dst = 0; - bool tunneling; /* Metadata. */ if (flow_tnl_dst_is_set(&md->tunnel)) { @@ -864,8 +863,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst) /* Initialize packet's layer pointer and offsets. */ frame = data; - tunneling = dp_packet_tunnel(packet); - if (tunneling) { + if (dp_packet_tunnel(packet)) { /* Preserve inner offsets from previous circulation. */ dp_packet_reset_outer_offsets(packet); } else { @@ -1063,10 +1061,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst) } else if (dl_type == htons(ETH_TYPE_IPV6)) { dp_packet_update_rss_hash_ipv6_tcp_udp(packet); } - if (dp_packet_l4_checksum_good(packet) - || dp_packet_ol_l4_csum_partial(packet)) { - dp_packet_hwol_set_csum_tcp(packet); - } + dp_packet_l4_proto_set_tcp(packet); } } } else if (OVS_LIKELY(nw_proto == IPPROTO_UDP)) { @@ -1082,14 +1077,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst) } else if (dl_type == htons(ETH_TYPE_IPV6)) { dp_packet_update_rss_hash_ipv6_tcp_udp(packet); } - if (dp_packet_l4_checksum_good(packet) - || dp_packet_ol_l4_csum_partial(packet)) { - if (tunneling) { - dp_packet_hwol_set_outer_udp_csum(packet); - } else { - dp_packet_hwol_set_csum_udp(packet); - } - } + dp_packet_l4_proto_set_udp(packet); } } else if (OVS_LIKELY(nw_proto == IPPROTO_SCTP)) { if (OVS_LIKELY(size >= SCTP_HEADER_LEN)) { @@ -1099,10 +1087,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst) miniflow_push_be16(mf, tp_dst, sctp->sctp_dst); miniflow_push_be16(mf, ct_tp_src, ct_tp_src); miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst); - if (dp_packet_l4_checksum_good(packet) - || dp_packet_ol_l4_csum_partial(packet)) { - dp_packet_hwol_set_csum_sctp(packet); - } + dp_packet_l4_proto_set_sctp(packet); } } else if (OVS_LIKELY(nw_proto == IPPROTO_ICMP)) { if (OVS_LIKELY(size >= ICMP_HEADER_LEN)) { @@ -1284,21 +1269,12 @@ parse_tcp_flags(struct dp_packet *packet, if (nw_proto == IPPROTO_TCP && size >= TCP_HEADER_LEN) { const struct tcp_header *tcp = data; - if (dp_packet_l4_checksum_good(packet) - || dp_packet_ol_l4_csum_partial(packet)) { - dp_packet_hwol_set_csum_tcp(packet); - } + dp_packet_l4_proto_set_tcp(packet); return TCP_FLAGS(tcp->tcp_ctl); } else if (nw_proto == IPPROTO_UDP && size >= UDP_HEADER_LEN) { - if (dp_packet_l4_checksum_good(packet) - || dp_packet_ol_l4_csum_partial(packet)) { - dp_packet_hwol_set_csum_udp(packet); - } + dp_packet_l4_proto_set_udp(packet); } else if (nw_proto == IPPROTO_SCTP && size >= SCTP_HEADER_LEN) { - if (dp_packet_l4_checksum_good(packet) - || dp_packet_ol_l4_csum_partial(packet)) { - dp_packet_hwol_set_csum_sctp(packet); - } + dp_packet_l4_proto_set_sctp(packet); } } @@ -3245,7 +3221,7 @@ flow_compose_l4_csum(struct dp_packet *p, const struct flow *flow, tcp->tcp_csum = 0; tcp->tcp_csum = csum_finish(csum_continue(pseudo_hdr_csum, tcp, l4_len)); - dp_packet_ol_set_l4_csum_good(p); + dp_packet_l4_checksum_set_good(p); } else if (flow->nw_proto == IPPROTO_UDP) { struct udp_header *udp = dp_packet_l4(p); @@ -3255,7 +3231,7 @@ flow_compose_l4_csum(struct dp_packet *p, const struct flow *flow, if (!udp->udp_csum) { udp->udp_csum = htons(0xffff); } - dp_packet_ol_set_l4_csum_good(p); + dp_packet_l4_checksum_set_good(p); } else if (flow->nw_proto == IPPROTO_ICMP) { struct icmp_header *icmp = dp_packet_l4(p); diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index 45a4de9f2..af5ecd5b3 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -2652,19 +2652,22 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) { struct dp_packet *pkt = CONTAINER_OF(mbuf, struct dp_packet, mbuf); const struct ip_header *ip; + bool is_sctp; bool l3_csum; + bool l4_csum; + bool is_tcp; + bool is_udp; void *l2; void *l3; void *l4; - const uint64_t all_inner_requests = (RTE_MBUF_F_TX_L4_MASK - | RTE_MBUF_F_TX_TCP_SEG); - const uint64_t all_outer_requests = RTE_MBUF_F_TX_OUTER_UDP_CKSUM; - const uint64_t all_requests = all_inner_requests | all_outer_requests; + const uint64_t all_inner_requests = RTE_MBUF_F_TX_TCP_SEG; if (!dp_packet_ip_checksum_partial(pkt) && !dp_packet_inner_ip_checksum_partial(pkt) - && !(mbuf->ol_flags & all_requests)) { + && !dp_packet_l4_checksum_partial(pkt) + && !dp_packet_inner_l4_checksum_partial(pkt) + && !(mbuf->ol_flags & all_inner_requests)) { uint64_t unexpected = mbuf->ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK; if (OVS_UNLIKELY(unexpected)) { @@ -2679,9 +2682,10 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) if (dp_packet_tunnel(pkt) && (dp_packet_inner_ip_checksum_partial(pkt) + || dp_packet_inner_l4_checksum_partial(pkt) || (mbuf->ol_flags & all_inner_requests))) { if (dp_packet_ip_checksum_partial(pkt) - || (mbuf->ol_flags & all_outer_requests)) { + || dp_packet_l4_checksum_partial(pkt)) { mbuf->outer_l2_len = (char *) dp_packet_l3(pkt) - (char *) dp_packet_eth(pkt); mbuf->outer_l3_len = (char *) dp_packet_l4(pkt) - @@ -2700,6 +2704,11 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_IP_CKSUM; } + if (dp_packet_l4_checksum_partial(pkt)) { + ovs_assert(dp_packet_l4_proto_udp(pkt)); + mbuf->ol_flags |= RTE_MBUF_F_TX_OUTER_UDP_CKSUM; + } + ip = dp_packet_l3(pkt); mbuf->ol_flags |= IP_VER(ip->ip_ihl_ver) == 4 ? RTE_MBUF_F_TX_OUTER_IPV4 @@ -2710,6 +2719,10 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) l3 = dp_packet_inner_l3(pkt); l3_csum = dp_packet_inner_ip_checksum_partial(pkt); l4 = dp_packet_inner_l4(pkt); + l4_csum = dp_packet_inner_l4_checksum_partial(pkt); + is_tcp = dp_packet_inner_l4_proto_tcp(pkt); + is_udp = dp_packet_inner_l4_proto_udp(pkt); + is_sctp = dp_packet_inner_l4_proto_sctp(pkt); } else { mbuf->outer_l2_len = 0; mbuf->outer_l3_len = 0; @@ -2719,16 +2732,12 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) l3 = dp_packet_inner_l3(pkt); l3_csum = dp_packet_inner_ip_checksum_partial(pkt); l4 = dp_packet_inner_l4(pkt); + l4_csum = dp_packet_inner_l4_checksum_partial(pkt); + is_tcp = dp_packet_inner_l4_proto_tcp(pkt); + is_udp = dp_packet_inner_l4_proto_udp(pkt); + is_sctp = dp_packet_inner_l4_proto_sctp(pkt); } } else { - if (dp_packet_tunnel(pkt)) { - /* No inner offload is requested, fallback to non tunnel - * checksum offloads. */ - if (mbuf->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM) { - mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; - } - mbuf->ol_flags &= ~all_outer_requests; - } mbuf->outer_l2_len = 0; mbuf->outer_l3_len = 0; @@ -2736,6 +2745,10 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) l3 = dp_packet_l3(pkt); l3_csum = dp_packet_ip_checksum_partial(pkt); l4 = dp_packet_l4(pkt); + l4_csum = dp_packet_l4_checksum_partial(pkt); + is_tcp = dp_packet_l4_proto_tcp(pkt); + is_udp = dp_packet_l4_proto_udp(pkt); + is_sctp = dp_packet_l4_proto_sctp(pkt); } ovs_assert(l4); @@ -2748,6 +2761,17 @@ netdev_dpdk_prep_hwol_packet(struct netdev_dpdk *dev, struct rte_mbuf *mbuf) mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; } + if (l4_csum) { + if (is_tcp) { + mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; + } else if (is_udp) { + mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM; + } else { + ovs_assert(is_sctp); + mbuf->ol_flags |= RTE_MBUF_F_TX_SCTP_CKSUM; + } + } + mbuf->l2_len = (char *) l3 - (char *) l2; mbuf->l3_len = (char *) l4 - (char *) l3; diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c index 4e699b795..8427441ac 100644 --- a/lib/netdev-dummy.c +++ b/lib/netdev-dummy.c @@ -1201,8 +1201,6 @@ netdev_dummy_rxq_recv(struct netdev_rxq *rxq_, struct dp_packet_batch *batch, netdev->custom_stats[0].value++; netdev->custom_stats[1].value++; - *dp_packet_ol_flags_ptr(packet) &= ~DP_PACKET_OL_TX_ANY_CKSUM; - if (netdev->ol_ip_rx_csum_set_good) { dp_packet_ip_checksum_set_good(packet); } else if (netdev->ol_ip_rx_csum_set_bad) { @@ -1214,13 +1212,13 @@ netdev_dummy_rxq_recv(struct netdev_rxq *rxq_, struct dp_packet_batch *batch, } if (netdev->ol_l4_rx_csum_set_good) { - dp_packet_ol_set_l4_csum_good(packet); + dp_packet_l4_checksum_set_good(packet); } else if (netdev->ol_l4_rx_csum_set_bad) { - dp_packet_ol_set_l4_csum_bad(packet); + dp_packet_l4_checksum_set_bad(packet); } else if (netdev->ol_l4_rx_csum_set_partial) { - dp_packet_ol_set_l4_csum_partial(packet); + dp_packet_l4_checksum_set_partial(packet); } else { - dp_packet_ol_reset_l4_csum_good(packet); + dp_packet_l4_checksum_set_unknown(packet); } if (userspace_tso_enabled() && netdev->ol_tso_segsz) { @@ -1236,10 +1234,8 @@ netdev_dummy_rxq_recv(struct netdev_rxq *rxq_, struct dp_packet_batch *batch, ip_csum_good = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_GOOD); ip_csum_bad = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_BAD); - l4_csum_good = !!(*dp_packet_ol_flags_ptr(packet) - & DP_PACKET_OL_L4_CKSUM_GOOD); - l4_csum_bad = !!(*dp_packet_ol_flags_ptr(packet) - & DP_PACKET_OL_L4_CKSUM_BAD); + l4_csum_good = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_GOOD); + l4_csum_bad = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_BAD); VLOG_DBG("Rx: packet with csum IP %s, L4 %s, segsz %"PRIu16, ip_csum_good ? (ip_csum_bad ? "partial" : "good") : (ip_csum_bad ? "bad" : "unknown"), @@ -1352,31 +1348,20 @@ netdev_dummy_send(struct netdev *netdev, int qid, ip_csum_good = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_GOOD); ip_csum_bad = !!(packet->offloads & DP_PACKET_OL_IP_CKSUM_BAD); - l4_csum_good = !!(*dp_packet_ol_flags_ptr(packet) - & DP_PACKET_OL_L4_CKSUM_GOOD); - l4_csum_bad = !!(*dp_packet_ol_flags_ptr(packet) - & DP_PACKET_OL_L4_CKSUM_BAD); + l4_csum_good = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_GOOD); + l4_csum_bad = !!(packet->offloads & DP_PACKET_OL_L4_CKSUM_BAD); VLOG_DBG("Tx: packet with csum IP %s, L4 %s, segsz %"PRIu16 - ", Tx flags %s, %s", + ", Tx flags %s", ip_csum_good ? (ip_csum_bad ? "partial" : "good") : (ip_csum_bad ? "bad" : "unknown"), l4_csum_good ? (l4_csum_bad ? "partial" : "good") : (l4_csum_bad ? "bad" : "unknown"), dp_packet_get_tso_segsz(packet), - dp_packet_hwol_tx_l4_checksum(packet) ? "l4_csum" - : "none", dp_packet_hwol_is_tso(packet) ? "tso" : "none"); } - if (dp_packet_ip_checksum_partial(packet)) { - dp_packet_ol_send_prepare(packet, flags); - } - if (dp_packet_hwol_l4_is_tcp(packet) - && !dp_packet_l4_checksum_good(packet)) { - dp_packet_ol_send_prepare(packet, flags); - } - if (dp_packet_hwol_l4_is_udp(packet) - && !dp_packet_l4_checksum_good(packet)) { + if (dp_packet_ip_checksum_partial(packet) + || dp_packet_l4_checksum_partial(packet)) { dp_packet_ol_send_prepare(packet, flags); } diff --git a/lib/netdev-linux.c b/lib/netdev-linux.c index 98255290d..e4917310e 100644 --- a/lib/netdev-linux.c +++ b/lib/netdev-linux.c @@ -7066,19 +7066,16 @@ netdev_linux_parse_vnet_hdr(struct dp_packet *b) return EINVAL; } - /* Chicken/egg situation, report L4 checksum as partial so that - * L4 extraction works. */ - dp_packet_ol_set_l4_csum_partial(b); parse_tcp_flags(b, NULL, NULL, NULL); if (csum_start == b->l4_ofs && ((csum_offset == offsetof(struct tcp_header, tcp_csum) - && dp_packet_hwol_l4_is_tcp(b)) + && dp_packet_l4_proto_tcp(b)) || (csum_offset == offsetof(struct udp_header, udp_csum) - && dp_packet_hwol_l4_is_udp(b)) + && dp_packet_l4_proto_udp(b)) || (csum_offset == offsetof(struct sctp_header, sctp_csum) - && dp_packet_hwol_l4_is_sctp(b)))) { - /* Nothing to do, L4 csum is already marked as partial. */ + && dp_packet_l4_proto_sctp(b)))) { + dp_packet_l4_checksum_set_partial(b); } else { ovs_be16 *csum_l4; void *l4; @@ -7094,12 +7091,10 @@ netdev_linux_parse_vnet_hdr(struct dp_packet *b) l4 = dp_packet_at(b, csum_start, dp_packet_size(b) - csum_start); *csum_l4 = csum(l4, dp_packet_size(b) - csum_start); - if (dp_packet_hwol_l4_is_tcp(b) - || dp_packet_hwol_l4_is_udp(b) - || dp_packet_hwol_l4_is_sctp(b)) { - dp_packet_ol_set_l4_csum_good(b); - } else { - b->offloads &= ~DP_PACKET_OL_L4_CKSUM_MASK; + if (dp_packet_l4_proto_tcp(b) + || dp_packet_l4_proto_udp(b) + || dp_packet_l4_proto_sctp(b)) { + dp_packet_l4_checksum_set_good(b); } } } @@ -7170,8 +7165,9 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int mtu) vnet->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; } else { VLOG_ERR_RL(&rl, "Unknown gso_type for TSO packet. " - "Flags: %#"PRIx64, - (uint64_t) *dp_packet_ol_flags_ptr(b)); + "Flags: %#"PRIx64", Offloads: %"PRIu32, + (uint64_t) *dp_packet_ol_flags_ptr(b), + b->offloads); return EINVAL; } } else { @@ -7180,26 +7176,38 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int mtu) vnet->gso_type = VIRTIO_NET_HDR_GSO_NONE; } - bool l4_is_good = dp_packet_l4_checksum_good(b); - - if ((dp_packet_tunnel_vxlan(b) - || dp_packet_tunnel_geneve(b)) - && dp_packet_hwol_tx_l4_checksum(b)) { - /* This condition is needed because dp-packet doesn't currently track - * outer and inner checksum statuses seperately. In the case of these - * two tunnel types we can end up setting outer l4 as good but still - * need to complete the inner l4. */ - l4_is_good = !(dp_packet_hwol_l4_is_tcp(b) || - dp_packet_hwol_l4_is_udp(b)); - } - - if (l4_is_good) { + if (dp_packet_l4_checksum_good(b) + && (!dp_packet_tunnel(b) + || dp_packet_inner_l4_checksum_good(b))) { /* The packet has good L4 checksum. No need to validate again. */ vnet->csum_start = vnet->csum_offset = (OVS_FORCE __virtio16) 0; vnet->flags = VIRTIO_NET_HDR_F_DATA_VALID; - } else if (dp_packet_hwol_tx_l4_checksum(b)) { + } else if (dp_packet_l4_checksum_partial(b) + || dp_packet_inner_l4_checksum_partial(b)) { + const struct ip_header *ip_hdr; + void *l3_off; + void *l4_off; + bool is_sctp; + bool is_tcp; + bool is_udp; + + if (dp_packet_inner_l4_checksum_partial(b)) { + l3_off = dp_packet_inner_l3(b); + l4_off = dp_packet_inner_l4(b); + is_tcp = dp_packet_inner_l4_proto_tcp(b); + is_udp = dp_packet_inner_l4_proto_udp(b); + is_sctp = dp_packet_inner_l4_proto_sctp(b); + } else { + l3_off = dp_packet_l3(b); + l4_off = dp_packet_l4(b); + is_tcp = dp_packet_l4_proto_tcp(b); + is_udp = dp_packet_l4_proto_udp(b); + is_sctp = dp_packet_l4_proto_sctp(b); + } + ip_hdr = l3_off; + /* The csum calculation is offloaded. */ - if (dp_packet_hwol_l4_is_tcp(b)) { + if (is_tcp) { /* Virtual I/O Device (VIRTIO) Version 1.1 * 5.1.6.2 Packet Transmission * If the driver negotiated VIRTIO_NET_F_CSUM, it can skip @@ -7214,17 +7222,9 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int mtu) * the TCP pseudo header, so that replacing it by the ones * complement checksum of the TCP header and body will give * the correct result. */ - void *l3_off = dp_packet_inner_l3(b); - void *l4_off = dp_packet_inner_l4(b); - - if (!l3_off || !l4_off) { - l3_off = dp_packet_l3(b); - l4_off = dp_packet_l4(b); - } - - const struct ip_header *ip_hdr = l3_off; struct tcp_header *tcp_hdr = l4_off; ovs_be16 csum = 0; + if (IP_VER(ip_hdr->ip_ihl_ver) == 4) { csum = ~csum_finish(packet_csum_pseudoheader(ip_hdr)); } else if (IP_VER(ip_hdr->ip_ihl_ver) == 6) { @@ -7238,19 +7238,10 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int mtu) (char *) dp_packet_data(b)); vnet->csum_offset = (OVS_FORCE __virtio16) __builtin_offsetof( struct tcp_header, tcp_csum); - } else if (dp_packet_hwol_l4_is_udp(b)) { - /* Favour the inner packet when indicating checksum offsets. */ - void *l3_off = dp_packet_inner_l3(b); - void *l4_off = dp_packet_inner_l4(b); - - if (!l3_off || !l4_off) { - l3_off = dp_packet_l3(b); - l4_off = dp_packet_l4(b); - } - - const struct ip_header *ip_hdr = l3_off; + } else if (is_udp) { struct udp_header *udp_hdr = l4_off; ovs_be16 csum = 0; + if (IP_VER(ip_hdr->ip_ihl_ver) == 4) { csum = ~csum_finish(packet_csum_pseudoheader(ip_hdr)); } else if (IP_VER(ip_hdr->ip_ihl_ver) == 6) { @@ -7264,7 +7255,7 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int mtu) (char *) dp_packet_data(b));; vnet->csum_offset = (OVS_FORCE __virtio16) __builtin_offsetof( struct udp_header, udp_csum); - } else if (dp_packet_hwol_l4_is_sctp(b)) { + } else if (is_sctp) { /* The Linux kernel networking stack only supports csum_start * and csum_offset when SCTP GSO is enabled. See kernel's * skb_csum_hwoffload_help(). Currently there is no SCTP @@ -7272,11 +7263,12 @@ netdev_linux_prepend_vnet_hdr(struct dp_packet *b, int mtu) vnet->csum_start = vnet->csum_offset = (OVS_FORCE __virtio16) 0; vnet->flags = 0; } else { - /* This should only happen when DP_PACKET_OL_TX_L4_MASK includes - * a new flag that is not covered in above checks. */ + /* This should only happen when a new L4 proto + * is not covered in above checks. */ VLOG_WARN_RL(&rl, "Unsupported L4 checksum offload. " - "Flags: %"PRIu64, - (uint64_t)*dp_packet_ol_flags_ptr(b)); + "Flags: %"PRIu64", Offloads: %"PRIu32, + (uint64_t)*dp_packet_ol_flags_ptr(b), + b->offloads); vnet->csum_start = vnet->csum_offset = (OVS_FORCE __virtio16) 0; vnet->flags = 0; } diff --git a/lib/netdev-native-tnl.c b/lib/netdev-native-tnl.c index fde4fe8e8..e07008825 100644 --- a/lib/netdev-native-tnl.c +++ b/lib/netdev-native-tnl.c @@ -233,8 +233,7 @@ udp_extract_tnl_md(struct dp_packet *packet, struct flow_tnl *tnl, if (udp->udp_csum) { bool bad_csum = dp_packet_l4_checksum_bad(packet); - if (OVS_LIKELY(!bad_csum && !dp_packet_ol_l4_csum_partial(packet)) - && OVS_UNLIKELY(!dp_packet_l4_checksum_good(packet))) { + if (OVS_UNLIKELY(!bad_csum && dp_packet_l4_checksum_unknown(packet))) { uint32_t csum; COVERAGE_INC(native_tnl_l4csum_checked); if (netdev_tnl_is_header_ipv6(dp_packet_data(packet))) { @@ -308,16 +307,11 @@ netdev_tnl_push_udp_header(const struct netdev *netdev OVS_UNUSED, udp->udp_src = udp_src; udp->udp_len = htons(ip_tot_size); + dp_packet_l4_proto_set_udp(packet); if (udp->udp_csum) { - dp_packet_ol_reset_l4_csum_good(packet); - if (dp_packet_tunnel_geneve(packet) - || dp_packet_tunnel_vxlan(packet)) { - dp_packet_hwol_set_outer_udp_csum(packet); - } else { - dp_packet_hwol_set_csum_udp(packet); - } + dp_packet_l4_checksum_set_partial(packet); } else { - dp_packet_ol_set_l4_csum_good(packet); + dp_packet_l4_checksum_set_good(packet); } if (l3_ofs != UINT16_MAX) { @@ -880,8 +874,8 @@ netdev_gtpu_push_header(const struct netdev *netdev, udp->udp_src = udp_src; udp->udp_len = htons(ip_tot_size); /* Postpone checksum to the egress netdev. */ - dp_packet_hwol_set_csum_udp(packet); - dp_packet_ol_reset_l4_csum_good(packet); + dp_packet_l4_proto_set_udp(packet); + dp_packet_l4_checksum_set_partial(packet); gtpuh = ALIGNED_CAST(struct gtpuhdr *, udp + 1); diff --git a/lib/netdev.c b/lib/netdev.c index a0aac0dc8..ea147f41f 100644 --- a/lib/netdev.c +++ b/lib/netdev.c @@ -929,7 +929,7 @@ netdev_send(struct netdev *netdev, int qid, struct dp_packet_batch *batch, if (dp_packet_hwol_is_tso(packet) && (dp_packet_tunnel_vxlan(packet) || dp_packet_tunnel_geneve(packet)) - && dp_packet_hwol_is_outer_udp_cksum(packet)) { + && dp_packet_l4_checksum_partial(packet)) { return netdev_send_tso(netdev, qid, batch, concurrent_txq); } } diff --git a/lib/odp-execute-avx512.c b/lib/odp-execute-avx512.c index 9ed5dff35..13afe0c79 100644 --- a/lib/odp-execute-avx512.c +++ b/lib/odp-execute-avx512.c @@ -498,8 +498,8 @@ action_avx512_ipv4_set_addrs(struct dp_packet_batch *batch, if (nh->ip_proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN) { struct udp_header *uh = dp_packet_l4(packet); - if (dp_packet_hwol_l4_is_udp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else if (uh->udp_csum) { /* New UDP checksum. */ uint16_t old_udp_checksum = ~uh->udp_csum; @@ -514,8 +514,8 @@ action_avx512_ipv4_set_addrs(struct dp_packet_batch *batch, } } else if (nh->ip_proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) { - if (dp_packet_hwol_l4_is_tcp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { /* New TCP checksum. */ struct tcp_header *th = dp_packet_l4(packet); @@ -691,8 +691,8 @@ action_avx512_set_ipv6(struct dp_packet_batch *batch, const struct nlattr *a) if (proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN) { struct udp_header *uh = dp_packet_l4(packet); - if (dp_packet_hwol_l4_is_udp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else if (uh->udp_csum) { delta_checksum = avx512_ipv6_addr_csum_delta(v_packet, v_new_hdr, @@ -711,8 +711,8 @@ action_avx512_set_ipv6(struct dp_packet_batch *batch, const struct nlattr *a) } } else if (proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) { - if (dp_packet_hwol_l4_is_tcp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { delta_checksum = avx512_ipv6_addr_csum_delta(v_packet, v_new_hdr, diff --git a/lib/packets.c b/lib/packets.c index 46e2fb323..a0bb2ad48 100644 --- a/lib/packets.c +++ b/lib/packets.c @@ -1129,15 +1129,15 @@ packet_set_ipv4_addr(struct dp_packet *packet, pkt_metadata_init_conn(&packet->md); if (nh->ip_proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) { - if (dp_packet_hwol_l4_is_tcp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { struct tcp_header *th = dp_packet_l4(packet); th->tcp_csum = recalc_csum32(th->tcp_csum, old_addr, new_addr); } } else if (nh->ip_proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN ) { - if (dp_packet_hwol_l4_is_udp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { struct udp_header *uh = dp_packet_l4(packet); if (uh->udp_csum) { @@ -1250,16 +1250,16 @@ packet_update_csum128(struct dp_packet *packet, uint8_t proto, size_t l4_size = dp_packet_l4_size(packet); if (proto == IPPROTO_TCP && l4_size >= TCP_HEADER_LEN) { - if (dp_packet_hwol_l4_is_tcp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { struct tcp_header *th = dp_packet_l4(packet); th->tcp_csum = recalc_csum128(th->tcp_csum, addr, new_addr); } } else if (proto == IPPROTO_UDP && l4_size >= UDP_HEADER_LEN) { - if (dp_packet_hwol_l4_is_udp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { struct udp_header *uh = dp_packet_l4(packet); @@ -1403,8 +1403,8 @@ packet_set_tcp_port(struct dp_packet *packet, ovs_be16 src, ovs_be16 dst) struct tcp_header *th = dp_packet_l4(packet); ovs_be16 *csum = NULL; - if (dp_packet_hwol_l4_is_tcp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); } else { csum = &th->tcp_csum; } @@ -1422,8 +1422,8 @@ packet_set_udp_port(struct dp_packet *packet, ovs_be16 src, ovs_be16 dst) { struct udp_header *uh = dp_packet_l4(packet); - if (dp_packet_hwol_l4_is_udp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); packet_set_port(&uh->udp_src, src, NULL); packet_set_port(&uh->udp_dst, dst, NULL); } else { @@ -1448,8 +1448,8 @@ packet_set_sctp_port(struct dp_packet *packet, ovs_be16 src, ovs_be16 dst) { struct sctp_header *sh = dp_packet_l4(packet); - if (dp_packet_hwol_l4_is_sctp(packet)) { - dp_packet_ol_reset_l4_csum_good(packet); + if (dp_packet_l4_checksum_valid(packet)) { + dp_packet_l4_checksum_set_partial(packet); sh->sctp_src = src; sh->sctp_dst = dst; } else { @@ -2025,6 +2025,12 @@ packet_tcp_complete_csum(struct dp_packet *p, bool inner) tcp->tcp_csum = packet_csum_upperlayer6(ip6, tcp, ip6->ip6_nxt, tcp_sz); } + + if (inner) { + dp_packet_inner_l4_checksum_set_good(p); + } else { + dp_packet_l4_checksum_set_good(p); + } } /* Set UDP checksum field in packet 'p' with complete checksum. @@ -2051,7 +2057,7 @@ packet_udp_complete_csum(struct dp_packet *p, bool inner) /* Skip csum calculation if the udp_csum is zero. */ if (!udp->udp_csum) { - return; + goto out; } udp->udp_csum = 0; @@ -2070,6 +2076,13 @@ packet_udp_complete_csum(struct dp_packet *p, bool inner) if (!udp->udp_csum) { udp->udp_csum = htons(0xffff); } + +out: + if (inner) { + dp_packet_inner_l4_checksum_set_good(p); + } else { + dp_packet_l4_checksum_set_good(p); + } } /* Set SCTP checksum field in packet 'p' with complete checksum. @@ -2094,4 +2107,10 @@ packet_sctp_complete_csum(struct dp_packet *p, bool inner) put_16aligned_be32(&sh->sctp_csum, 0); csum = crc32c((void *) sh, tp_len); put_16aligned_be32(&sh->sctp_csum, csum); + + if (inner) { + dp_packet_inner_l4_checksum_set_good(p); + } else { + dp_packet_l4_checksum_set_good(p); + } }