diff --git a/.mailmap b/.mailmap index cdaa471715a..81499a3337c 100644 --- a/.mailmap +++ b/.mailmap @@ -497,6 +497,7 @@ Ganghui Zeng Gao Feng Gaoxiang Liu Gargi Sau +Garvit Varshney Gary Mussar Gaurav Singh Gautam Dawar diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c index b0a4b46743d..a60983c6b73 100644 --- a/app/test/test_cryptodev.c +++ b/app/test/test_cryptodev.c @@ -9717,7 +9717,6 @@ test_pdcp_proto_SGL(int i, int oop, int ret = TEST_SUCCESS; int to_trn = 0; int to_trn_tbl[16]; - int segs = 1; unsigned int trn_data = 0; struct rte_cryptodev_info dev_info; uint64_t feat_flags; @@ -9786,7 +9785,6 @@ test_pdcp_proto_SGL(int i, int oop, */ while (trn_data < input_vec_len) { - ++segs; to_trn = (input_vec_len - trn_data < fragsz) ? (input_vec_len - trn_data) : fragsz; @@ -9794,6 +9792,7 @@ test_pdcp_proto_SGL(int i, int oop, buf->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); buf = buf->next; + ut_params->ibuf->nb_segs++; memset(rte_pktmbuf_mtod(buf, uint8_t *), 0, rte_pktmbuf_tailroom(buf)); @@ -9803,6 +9802,7 @@ test_pdcp_proto_SGL(int i, int oop, buf_oop->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); buf_oop = buf_oop->next; + ut_params->obuf->nb_segs++; memset(rte_pktmbuf_mtod(buf_oop, uint8_t *), 0, rte_pktmbuf_tailroom(buf_oop)); TEST_ASSERT_NOT_NULL(ut_params->obuf, "Output buffer not initialized"); @@ -9817,16 +9817,12 @@ test_pdcp_proto_SGL(int i, int oop, trn_data += to_trn; } - ut_params->ibuf->nb_segs = segs; - - segs = 1; if (fragsz_oop && oop) { to_trn = 0; ecx = 0; trn_data = frag_size_oop; while (trn_data < output_vec_len) { - ++segs; to_trn = (output_vec_len - trn_data < frag_size_oop) ? @@ -9838,13 +9834,13 @@ test_pdcp_proto_SGL(int i, int oop, buf_oop->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); buf_oop = buf_oop->next; + ut_params->obuf->nb_segs++; memset(rte_pktmbuf_mtod(buf_oop, uint8_t *), 0, rte_pktmbuf_tailroom(buf_oop)); TEST_ASSERT_NOT_NULL(rte_pktmbuf_append(ut_params->obuf, to_trn), "Failed to append to mbuf"); trn_data += to_trn; } - ut_params->obuf->nb_segs = segs; } /* Setup Cipher Parameters */ @@ -15869,7 +15865,6 @@ test_AES_GMAC_authentication_SGL(const struct gmac_test_data *tdata, uint64_t feature_flags; unsigned int trn_data = 0; void *digest_mem = NULL; - uint32_t segs = 1; unsigned int to_trn = 0; struct rte_mbuf *buf = NULL; uint8_t *auth_tag, *plaintext; @@ -15930,12 +15925,12 @@ test_AES_GMAC_authentication_SGL(const struct gmac_test_data *tdata, */ while (trn_data < tdata->plaintext.len) { - ++segs; to_trn = (tdata->plaintext.len - trn_data < fragsz) ? (tdata->plaintext.len - trn_data) : fragsz; buf->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); buf = buf->next; + ut_params->ibuf->nb_segs++; memset(rte_pktmbuf_mtod(buf, uint8_t *), 0, rte_pktmbuf_tailroom(buf)); @@ -15953,7 +15948,6 @@ test_AES_GMAC_authentication_SGL(const struct gmac_test_data *tdata, TEST_ASSERT_NOT_NULL(digest_mem, "Failed to append digest data"); } } - ut_params->ibuf->nb_segs = segs; /* * Place digest at the end of the last buffer @@ -17106,7 +17100,6 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, int retval; int to_trn = 0; int to_trn_tbl[SGL_MAX_NO]; - int segs = 1; unsigned int trn_data = 0; uint8_t *plaintext, *ciphertext, *auth_tag; struct rte_cryptodev_info dev_info; @@ -17231,7 +17224,6 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, */ while (trn_data < tdata->plaintext.len) { - ++segs; to_trn = (tdata->plaintext.len - trn_data < fragsz) ? (tdata->plaintext.len - trn_data) : fragsz; @@ -17239,6 +17231,7 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, buf->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); buf = buf->next; + ut_params->ibuf->nb_segs++; memset(rte_pktmbuf_mtod(buf, uint8_t *), 0, rte_pktmbuf_tailroom(buf)); @@ -17248,6 +17241,7 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, buf_last_oop = buf_oop->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); buf_oop = buf_oop->next; + ut_params->obuf->nb_segs++; memset(rte_pktmbuf_mtod(buf_oop, uint8_t *), 0, rte_pktmbuf_tailroom(buf_oop)); TEST_ASSERT_NOT_NULL(rte_pktmbuf_append(ut_params->obuf, to_trn), "Failed to append to mbuf"); @@ -17277,9 +17271,6 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, uint64_t digest_phys = 0; - ut_params->ibuf->nb_segs = segs; - - segs = 1; if (fragsz_oop && oop) { to_trn = 0; ecx = 0; @@ -17295,7 +17286,6 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, trn_data = frag_size_oop; while (trn_data < tdata->plaintext.len) { - ++segs; to_trn = (tdata->plaintext.len - trn_data < frag_size_oop) ? @@ -17307,6 +17297,7 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, buf_last_oop = buf_oop->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); TEST_ASSERT_NOT_NULL(buf_oop->next, "Unexpected end of chain"); + ut_params->obuf->nb_segs++; buf_oop = buf_oop->next; memset(rte_pktmbuf_mtod(buf_oop, uint8_t *), 0, rte_pktmbuf_tailroom(buf_oop)); @@ -17320,8 +17311,6 @@ test_authenticated_encryption_SGL(const struct aead_test_data *tdata, TEST_ASSERT_NOT_NULL(digest_mem, "Failed to append auth tag"); } } - - ut_params->obuf->nb_segs = segs; } /* diff --git a/app/test/test_cryptodev_asym.c b/app/test/test_cryptodev_asym.c index 111f675c948..34c15e26bcd 100644 --- a/app/test/test_cryptodev_asym.c +++ b/app/test/test_cryptodev_asym.c @@ -139,6 +139,7 @@ queue_ops_rsa_sign_verify(void *sess) /* Negative test */ result_op->asym->rsa.sign.data[0] ^= 0xff; + result_op->asym->rsa.sign.length = RTE_DIM(rsa_n); if (rte_cryptodev_enqueue_burst(dev_id, 0, &result_op, 1) != 1) { RTE_LOG(ERR, USER1, "Error sending packet for verify\n"); status = TEST_FAILED; diff --git a/app/test/test_security_inline_proto.c b/app/test/test_security_inline_proto.c index 09d710eff2f..8b88fce3e99 100644 --- a/app/test/test_security_inline_proto.c +++ b/app/test/test_security_inline_proto.c @@ -542,6 +542,48 @@ init_mempools(unsigned int nb_mbuf) return 0; } +static int +create_ipsec_flow(uint16_t portid, void *ses, uint32_t spi) +{ + struct rte_flow_item_esp esp_spec; + struct rte_flow_action action[2]; + struct rte_flow_item pattern[2]; + struct rte_flow_attr attr = {0}; + struct rte_flow_error err; + struct rte_flow *flow; + int ret; + + esp_spec.hdr.spi = rte_cpu_to_be_32(spi); + + pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP; + pattern[0].spec = &esp_spec; + pattern[0].mask = &rte_flow_item_esp_mask; + pattern[0].last = NULL; + pattern[1].type = RTE_FLOW_ITEM_TYPE_END; + + action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; + action[0].conf = ses; + action[1].type = RTE_FLOW_ACTION_TYPE_END; + action[1].conf = NULL; + + attr.ingress = 1; + + ret = rte_flow_validate(portid, &attr, pattern, action, &err); + if (ret) { + printf("\nValidate ESP flow failed, ret = %d\n", ret); + return -1; + } + flow = rte_flow_create(portid, &attr, pattern, action, &err); + if (flow == NULL) { + printf("\nESP flow rule create failed\n"); + return -1; + } + + default_flow[portid] = flow; + + return 0; +} + static int create_default_flow(uint16_t portid) { @@ -1373,7 +1415,15 @@ test_ipsec_inline_proto_process(struct ipsec_test_data *td, } if (td->ipsec_xform.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { - ret = create_default_flow(port_id); + if (flags->inb_oop) { + ret = create_ipsec_flow(port_id, ses, td->ipsec_xform.spi); + if (ret) { + /* Check with default flow rule */ + printf("\nFailed to create ESP flow, try with default flow"); + ret = create_default_flow(port_id); + } + } else + ret = create_default_flow(port_id); if (ret) goto out; } diff --git a/config/rte_config.h b/config/rte_config.h index a2609fa403a..05a900e9ce1 100644 --- a/config/rte_config.h +++ b/config/rte_config.h @@ -72,7 +72,7 @@ #define RTE_MAX_MULTI_HOST_CTRLS 4 /* cryptodev defines */ -#define RTE_CRYPTO_MAX_DEVS 64 +#define RTE_CRYPTO_MAX_DEVS 128 #define RTE_CRYPTODEV_NAME_LEN 64 #define RTE_CRYPTO_CALLBACKS 1 diff --git a/doc/guides/cryptodevs/features/openssl.ini b/doc/guides/cryptodevs/features/openssl.ini index df6e7de3164..536557e9e09 100644 --- a/doc/guides/cryptodevs/features/openssl.ini +++ b/doc/guides/cryptodevs/features/openssl.ini @@ -24,6 +24,8 @@ AES CBC (256) = Y AES CTR (128) = Y AES CTR (192) = Y AES CTR (256) = Y +AES XTS (128) = Y +AES XTS (256) = Y 3DES CBC = Y 3DES CTR = Y DES DOCSIS BPI = Y @@ -43,6 +45,8 @@ SHA384 = Y SHA384 HMAC = Y SHA512 = Y SHA512 HMAC = Y +SHAKE_128 = Y +SHAKE_256 = Y AES GMAC = Y ; diff --git a/doc/guides/cryptodevs/openssl.rst b/doc/guides/cryptodevs/openssl.rst index d467069cace..921592ba2d9 100644 --- a/doc/guides/cryptodevs/openssl.rst +++ b/doc/guides/cryptodevs/openssl.rst @@ -22,6 +22,7 @@ Supported cipher algorithms: * ``RTE_CRYPTO_CIPHER_3DES_CBC`` * ``RTE_CRYPTO_CIPHER_AES_CBC`` * ``RTE_CRYPTO_CIPHER_AES_CTR`` +* ``RTE_CRYPTO_CIPHER_AES_XTS`` * ``RTE_CRYPTO_CIPHER_3DES_CTR`` * ``RTE_CRYPTO_CIPHER_DES_DOCSISBPI`` @@ -40,6 +41,8 @@ Supported authentication algorithms: * ``RTE_CRYPTO_AUTH_SHA256_HMAC`` * ``RTE_CRYPTO_AUTH_SHA384_HMAC`` * ``RTE_CRYPTO_AUTH_SHA512_HMAC`` +* ``RTE_CRYPTO_AUTH_SHAKE_128`` +* ``RTE_CRYPTO_AUTH_SHAKE_256`` Supported AEAD algorithms: diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst index 68d792e4cc8..0c2b85444e3 100644 --- a/doc/guides/cryptodevs/qat.rst +++ b/doc/guides/cryptodevs/qat.rst @@ -501,6 +501,32 @@ If you are running on a kernel which includes a driver for your device, see The actual crypto services enabled on the system depend on QAT driver capabilities and hardware slice configuration. +.. note:: + + With the introduction of QAT Generation 4, + the in-tree drivers are available within the mainline Linux kernel. + Out-of-tree (OOT) drivers for QAT Gen 4 are currently in sustaining mode, + meaning they will only receive maintenance updates without new feature development. + Future generations of QAT will be supported with in-tree drivers exclusively. + +.. note:: + + For **in-tree drivers**, when multiple QAT instances are available, + each instance is assigned a different crypto service by default (asym;sym or dc). + The available crypto queue pair request will return zero + if the corresponding service is not enabled. + + To verify the device configuration, run:: + + cat /sys/bus/pci/devices//qat/cfg_services + + For symmetric and asymmetric crypto services, ensure that `"asym;sym"` are enabled. + + For **out-of-tree drivers**, the configuration file for each instance can be found at:: + + /etc/_dev.conf + + Installation using kernel.org driver ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/guides/nics/index.rst b/doc/guides/nics/index.rst index b00ed998c53..cb818284fea 100644 --- a/doc/guides/nics/index.rst +++ b/doc/guides/nics/index.rst @@ -60,10 +60,11 @@ Network Interface Controller Drivers null octeon_ep octeontx - pcap_ring + pcap pfe qede r8169 + ring rnp sfc_efx softnic diff --git a/doc/guides/nics/pcap.rst b/doc/guides/nics/pcap.rst new file mode 100644 index 00000000000..fbfe854bb10 --- /dev/null +++ b/doc/guides/nics/pcap.rst @@ -0,0 +1,249 @@ +.. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2010-2015 Intel Corporation. + +Pcap Poll Mode Driver +===================== + +The pcap-based PMD (**librte_net_pcap**) reads and writes packets using the pcap library, +both from files on disk and from physical NIC devices using standard kernel drivers. + +For more information about the pcap library, see the +`libpcap documentation `_. + +.. note:: + + The pcap-based PMD requires the libpcap development files to be installed. + This applies to all supported operating systems: Linux, FreeBSD, and Windows. + + +Using the Driver from the EAL Command Line +------------------------------------------ + +DPDK allows pseudo-Ethernet devices, as the pcap driver, +to be created at application startup time during EAL initialization. + +To do so, pass the ``--vdev=net_pcap0`` parameter to the EAL. +This parameter accepts options to allocate and use pcap-based Ethernet +transparently by the application. +This can be used, for example, for testing on a virtual machine +where there are no Ethernet ports. + +The device name must start with the ``net_pcap`` prefix followed by numbers or letters. +The name must be unique for each device. +Each device can have multiple stream options and multiple devices can be used. +Multiple device definitions can be specified using multiple ``--vdev`` arguments. +Device name and stream options must be separated by commas as shown below: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,stream_opt0=..,stream_opt1=..' \ + --vdev='net_pcap1,stream_opt0=..' + +Device Streams +~~~~~~~~~~~~~~ + +Stream definitions can be combined as long as one of the following two rules is met: + +* A device is provided with two different streams - reception and transmission. +* A device is provided with one network interface name used for reading and writing packets. + +The stream types are: + +``rx_pcap`` + + Defines a reception stream based on a pcap file. + The driver reads each packet within the given pcap file + as if it was receiving it from the wire. + The value is a path to a valid pcap file:: + + rx_pcap=/path/to/file.pcap + +``tx_pcap`` + + Defines a transmission stream based on a pcap file. + The driver writes each received packet to the given pcap file. + The file is overwritten if it already exists and it is created if it does not. + The value is a path to a pcap file:: + + tx_pcap=/path/to/file.pcap + +``rx_iface`` + + Defines a reception stream based on a network interface name. + The driver reads packets from the given interface + using the kernel driver for that interface. + The driver captures both the incoming and outgoing packets on that interface. + The value is an interface name:: + + rx_iface=eth0 + +``rx_iface_in`` + + Defines a reception stream based on a network interface name. + The driver reads packets from the given interface + using the kernel driver for that interface. + The driver captures only the incoming packets on that interface. + The value is an interface name:: + + rx_iface_in=eth0 + +``tx_iface`` + + Defines a transmission stream based on a network interface name. + The driver sends packets to the given interface + using the kernel driver for that interface. + The value is an interface name:: + + tx_iface=eth0 + +``iface`` + + Defines a device mapping a network interface. + The driver both reads and writes packets from and to the given interface. + The value is an interface name:: + + iface=eth0 + +Multi-queue Support +~~~~~~~~~~~~~~~~~~~ + +The pcap PMD supports multiple receive and transmit queues. +The number of receive queues is determined +by the number of ``rx_pcap`` or ``rx_iface`` arguments provided. +Similarly, the number of transmit queues is determined +by the number of ``tx_pcap`` or ``tx_iface`` arguments. + +Using the same file for multiple queues is not supported +because the underlying pcap library +does not support concurrent access to a single file handle. + +Runtime Config Options +~~~~~~~~~~~~~~~~~~~~~~ + +* Use pcap interface physical MAC + + When the ``iface=`` configuration is set, + the selected interface's physical MAC address can be used. + This can be done with the ``phy_mac`` devarg, for example:: + + --vdev 'net_pcap0,iface=eth0,phy_mac=1' + +* Use the Rx pcap file to infinitely receive packets + + When the ``rx_pcap=`` configuration is set, + the selected pcap file can be used for basic performance testing. + This can be done with the ``infinite_rx`` devarg, for example:: + + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,infinite_rx=1' + + When this mode is used, it is recommended to drop all packets on transmit + by not providing a ``tx_pcap`` or ``tx_iface``. + + This option is device-wide, + so all queues on a device will either have this enabled or disabled. + This option should only be provided once per device. + +* Drop all packets on transmit + + To drop all packets on transmit for a device, + do not provide a ``tx_pcap`` or ``tx_iface``, for example:: + + --vdev 'net_pcap0,rx_pcap=file_rx.pcap' + + In this case, one Tx drop queue is created for each Rx queue on that device. + +* Receive no packets on Rx + + To run without receiving any packets on Rx, + do not provide a ``rx_pcap`` or ``rx_iface``, for example:: + + --vdev 'net_pcap0,tx_pcap=file_tx.pcap' + + In this case, one dummy Rx queue is created for each Tx queue argument passed. + +Examples of Usage +~~~~~~~~~~~~~~~~~ + +Read packets from one pcap file and write them to another: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_pcap=file_tx.pcap' \ + -- --port-topology=chained + +Read packets from a network interface and write them to a pcap file: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,rx_iface=eth0,tx_pcap=file_tx.pcap' \ + -- --port-topology=chained + +Read packets from a pcap file and write them to a network interface: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_iface=eth1' \ + -- --port-topology=chained + +Forward packets through 2 network interfaces: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,iface=eth0' --vdev='net_pcap1,iface=eth1' + +Enable 2 Tx queues on a network interface: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,rx_iface=eth1,tx_iface=eth1,tx_iface=eth1' \ + -- --txq 2 + +Read only incoming packets from a network interface +and write them back to the same network interface: + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,rx_iface_in=eth1,tx_iface=eth1' + +Using Pcap-based PMD with the testpmd Application +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One of the first things that testpmd does before starting to forward packets +is to flush the Rx streams by reading the first 512 packets on every Rx stream +and discarding them. +When using a pcap-based PMD, this behavior can be turned off +using the ``--no-flush-rx`` option: + +.. code-block:: console + + --no-flush-rx + +This option is also available in the runtime command line: + +.. code-block:: console + + set flush_rx on/off + +It is useful for the case where the ``rx_pcap`` is being used +and no packets are meant to be discarded. +Otherwise, the first 512 packets from the input pcap file +will be discarded by the Rx flushing operation. + +.. code-block:: console + + dpdk-testpmd -l 0-3 \ + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_pcap=file_tx.pcap' \ + -- --port-topology=chained --no-flush-rx + +.. note:: + + The network interface provided to the PMD should be up. + The PMD will return an error if the interface is down, + and the PMD itself won't change the status of the external network interface. diff --git a/doc/guides/nics/pcap_ring.rst b/doc/guides/nics/pcap_ring.rst deleted file mode 100644 index 6955e911301..00000000000 --- a/doc/guides/nics/pcap_ring.rst +++ /dev/null @@ -1,318 +0,0 @@ -.. SPDX-License-Identifier: BSD-3-Clause - Copyright(c) 2010-2015 Intel Corporation. - -Libpcap and Ring Based Poll Mode Drivers -======================================== - -In addition to Poll Mode Drivers (PMDs) for physical and virtual hardware, -the DPDK also includes pure-software PMDs, two of these drivers are: - -* A libpcap -based PMD (**librte_net_pcap**) that reads and writes packets using libpcap, - - both from files on disk, as well as from physical NIC devices using standard Linux kernel drivers. - -* A ring-based PMD (**librte_net_ring**) that allows a set of software FIFOs (that is, rte_ring) - to be accessed using the PMD APIs, as though they were physical NICs. - -.. note:: - - The libpcap -based PMD has an external dependency on the libpcap development files which must - be installed on the board. - -Using the Drivers from the EAL Command Line -------------------------------------------- - -For ease of use, the DPDK EAL also has been extended to allow pseudo-Ethernet devices, -using one or more of these drivers, -to be created at application startup time during EAL initialization. - -To do so, the --vdev= parameter must be passed to the EAL. -This takes take options to allow ring and pcap-based Ethernet to be allocated and used transparently by the application. -This can be used, for example, for testing on a virtual machine where there are no Ethernet ports. - -Libpcap-based PMD -~~~~~~~~~~~~~~~~~ - -Pcap-based devices can be created using the virtual device --vdev option. -The device name must start with the net_pcap prefix followed by numbers or letters. -The name is unique for each device. Each device can have multiple stream options and multiple devices can be used. -Multiple device definitions can be arranged using multiple --vdev. -Device name and stream options must be separated by commas as shown below: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,stream_opt0=..,stream_opt1=..' \ - --vdev='net_pcap1,stream_opt0=..' - -Device Streams -^^^^^^^^^^^^^^ - -Multiple ways of stream definitions can be assessed and combined as long as the following two rules are respected: - -* A device is provided with two different streams - reception and transmission. - -* A device is provided with one network interface name used for reading and writing packets. - -The different stream types are: - -* rx_pcap: Defines a reception stream based on a pcap file. - The driver reads each packet within the given pcap file as if it was receiving it from the wire. - The value is a path to a valid pcap file. - - rx_pcap=/path/to/file.pcap - -* tx_pcap: Defines a transmission stream based on a pcap file. - The driver writes each received packet to the given pcap file. - The value is a path to a pcap file. - The file is overwritten if it already exists and it is created if it does not. - - tx_pcap=/path/to/file.pcap - -* rx_iface: Defines a reception stream based on a network interface name. - The driver reads packets from the given interface using the Linux kernel driver for that interface. - The driver captures both the incoming and outgoing packets on that interface. - The value is an interface name. - - rx_iface=eth0 - -* rx_iface_in: Defines a reception stream based on a network interface name. - The driver reads packets from the given interface using the Linux kernel driver for that interface. - The driver captures only the incoming packets on that interface. - The value is an interface name. - - rx_iface_in=eth0 - -* tx_iface: Defines a transmission stream based on a network interface name. - The driver sends packets to the given interface using the Linux kernel driver for that interface. - The value is an interface name. - - tx_iface=eth0 - -* iface: Defines a device mapping a network interface. - The driver both reads and writes packets from and to the given interface. - The value is an interface name. - - iface=eth0 - -Runtime Config Options -^^^^^^^^^^^^^^^^^^^^^^ - -- Use PCAP interface physical MAC - - In case ``iface=`` configuration is set, user may want to use the selected interface's physical MAC - address. This can be done with a ``devarg`` ``phy_mac``, for example:: - - --vdev 'net_pcap0,iface=eth0,phy_mac=1' - -- Use the RX PCAP file to infinitely receive packets - - In case ``rx_pcap=`` configuration is set, user may want to use the selected PCAP file for rudimental - performance testing. This can be done with a ``devarg`` ``infinite_rx``, for example:: - - --vdev 'net_pcap0,rx_pcap=file_rx.pcap,infinite_rx=1' - - When this mode is used, it is recommended to drop all packets on transmit by not providing a tx_pcap or tx_iface. - - This option is device wide, so all queues on a device will either have this enabled or disabled. - This option should only be provided once per device. - -- Drop all packets on transmit - - The user may want to drop all packets on tx for a device. This can be done by not providing a tx_pcap or tx_iface, for example:: - - --vdev 'net_pcap0,rx_pcap=file_rx.pcap' - - In this case, one tx drop queue is created for each rxq on that device. - - - Receive no packets on Rx - - The user may want to run without receiving any packets on Rx. This can be done by not providing a rx_pcap or rx_iface, for example:: - - --vdev 'net_pcap0,tx_pcap=file_tx.pcap' - -In this case, one dummy rx queue is created for each tx queue argument passed - -Examples of Usage -^^^^^^^^^^^^^^^^^ - -Read packets from one pcap file and write them to another: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_pcap=file_tx.pcap' \ - -- --port-topology=chained - -Read packets from a network interface and write them to a pcap file: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,rx_iface=eth0,tx_pcap=file_tx.pcap' \ - -- --port-topology=chained - -Read packets from a pcap file and write them to a network interface: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_iface=eth1' \ - -- --port-topology=chained - -Forward packets through two network interfaces: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,iface=eth0' --vdev='net_pcap1,iface=eth1' - -Enable 2 tx queues on a network interface: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,rx_iface=eth1,tx_iface=eth1,tx_iface=eth1' \ - -- --txq 2 - -Read only incoming packets from a network interface and write them back to the same network interface: - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,rx_iface_in=eth1,tx_iface=eth1' - -Using libpcap-based PMD with the testpmd Application -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -One of the first things that testpmd does before starting to forward packets is to flush the RX streams -by reading the first 512 packets on every RX stream and discarding them. -When using a libpcap-based PMD this behavior can be turned off using the following command line option: - -.. code-block:: console - - --no-flush-rx - -It is also available in the runtime command line: - -.. code-block:: console - - set flush_rx on/off - -It is useful for the case where the rx_pcap is being used and no packets are meant to be discarded. -Otherwise, the first 512 packets from the input pcap file will be discarded by the RX flushing operation. - -.. code-block:: console - - .//app/dpdk-testpmd -l 0-3 \ - --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_pcap=file_tx.pcap' \ - -- --port-topology=chained --no-flush-rx - -.. note:: - - The network interface provided to the PMD should be up. The PMD will return - an error if interface is down, and the PMD itself won't change the status - of the external network interface. - - -Rings-based PMD -~~~~~~~~~~~~~~~ - -To run a DPDK application on a machine without any Ethernet devices, a pair of ring-based rte_ethdevs can be used as below. -The device names passed to the --vdev option must start with net_ring and take no additional parameters. -Multiple devices may be specified, separated by commas. - -.. code-block:: console - - ./dpdk-testpmd -l 1-3 --vdev=net_ring0 --vdev=net_ring1 -- -i - ... - Interactive-mode selected - Configuring Port 0 (socket 0) - Configuring Port 1 (socket 0) - Checking link statuses... - Port 0 Link Up - speed 10000 Mbps - full-duplex - Port 1 Link Up - speed 10000 Mbps - full-duplex - Done - - testpmd> start tx_first - io packet forwarding - CRC stripping disabled - packets/burst=16 - nb forwarding cores=1 - nb forwarding ports=2 - RX queues=1 - RX desc=128 - RX free threshold=0 - RX threshold registers: pthresh=8 hthresh=8 wthresh=4 - TX queues=1 - TX desc=512 - TX free threshold=0 - TX threshold registers: pthresh=36 hthresh=0 wthresh=0 - TX RS bit threshold=0 - TXQ flags=0x0 - - testpmd> stop - Telling cores to stop... - Waiting for lcores to finish... - -.. image:: img/forward_stats.* - -.. code-block:: console - - +++++++++++++++ Accumulated forward statistics for allports++++++++++ - RX-packets: 462384736 RX-dropped: 0 RX-total: 462384736 - TX-packets: 462384768 TX-dropped: 0 TX-total: 462384768 - +++++++++++++++++++++++++++++++++++++++++++++++++++++ - - Done. - - -Using the Poll Mode Driver from an Application -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Both drivers can provide similar APIs to allow the user to create a PMD, that is, -rte_ethdev structure, instances at run-time in the end-application, -for example, using rte_eth_from_rings() or rte_eth_from_pcaps() APIs. -For the rings-based PMD, this functionality could be used, for example, -to allow data exchange between cores using rings to be done in exactly the -same way as sending or receiving packets from an Ethernet device. -For the libpcap-based PMD, it allows an application to open one or more pcap files -and use these as a source of packet input to the application. - -Usage Examples -^^^^^^^^^^^^^^ - -To create two pseudo-Ethernet ports where all traffic sent to a port is looped back -for reception on the same port (error handling omitted for clarity): - -.. code-block:: c - - #define RING_SIZE 256 - #define NUM_RINGS 2 - #define SOCKET0 0 - - struct rte_ring *ring[NUM_RINGS]; - int port0, port1; - - ring[0] = rte_ring_create("R0", RING_SIZE, SOCKET0, RING_F_SP_ENQ|RING_F_SC_DEQ); - ring[1] = rte_ring_create("R1", RING_SIZE, SOCKET0, RING_F_SP_ENQ|RING_F_SC_DEQ); - - /* create two ethdev's */ - - port0 = rte_eth_from_rings("net_ring0", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); - port1 = rte_eth_from_rings("net_ring1", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); - - -To create two pseudo-Ethernet ports where the traffic is switched between them, -that is, traffic sent to port 0 is read back from port 1 and vice-versa, -the final two lines could be changed as below: - -.. code-block:: c - - port0 = rte_eth_from_rings("net_ring0", &ring[0], 1, &ring[1], 1, SOCKET0); - port1 = rte_eth_from_rings("net_ring1", &ring[1], 1, &ring[0], 1, SOCKET0); - -This type of configuration could be useful in a pipeline model, for example, -where one may want to have inter-core communication using pseudo Ethernet devices rather than raw rings, -for reasons of API consistency. - -Enqueuing and dequeuing items from an rte_ring using the rings-based PMD may be slower than using the native rings API. -This is because DPDK Ethernet drivers make use of function pointers to call the appropriate enqueue or dequeue functions, -while the rte_ring specific functions are direct function calls in the code and are often inlined by the compiler. - - Once an ethdev has been created, for either a ring or a pcap-based PMD, - it should be configured and started in the same way as a regular Ethernet device, that is, - by calling rte_eth_dev_configure() to set the number of receive and transmit queues, - then calling rte_eth_rx_queue_setup() / tx_queue_setup() for each of those queues and - finally calling rte_eth_dev_start() to allow transmission and reception of packets to begin. diff --git a/doc/guides/nics/ring.rst b/doc/guides/nics/ring.rst new file mode 100644 index 00000000000..a6b2458a7f5 --- /dev/null +++ b/doc/guides/nics/ring.rst @@ -0,0 +1,118 @@ +.. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2010-2015 Intel Corporation. + +Ring Based Poll Mode Driver +=========================== + +The ring-based PMD (``librte_net_ring``) allows software FIFOs (rte_ring) +to be accessed using the PMD API, as though they were physical NICs. + + +Using the Driver from the EAL Command Line +------------------------------------------ + +DPDK allows pseudo-Ethernet devices, as the ring driver, +to be created at application startup time during EAL initialization. + +To do so, pass the ``--vdev=net_ring0`` parameter to the EAL. +This parameter accepts options to allocate and use ring-based Ethernet +transparently by the application. +This can be used, for example, for testing on a virtual machine +where there are no Ethernet ports. + +The device names passed to the ``--vdev`` option must start with ``net_ring`` +and take no additional parameters. +Multiple devices may be specified using multiple ``--vdev`` arguments. + +.. code-block:: console + + ./dpdk-testpmd -l 1-3 --vdev=net_ring0 --vdev=net_ring1 -- -i + ... + Interactive-mode selected + Configuring Port 0 (socket 0) + Configuring Port 1 (socket 0) + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + + testpmd> start tx_first + io packet forwarding - CRC stripping disabled - packets/burst=16 + nb forwarding cores=1 - nb forwarding ports=2 + RX queues=1 - RX desc=128 - RX free threshold=0 + RX threshold registers: pthresh=8 hthresh=8 wthresh=4 + TX queues=1 - TX desc=512 - TX free threshold=0 + TX threshold registers: pthresh=36 hthresh=0 wthresh=0 + TX RS bit threshold=0 - TXQ flags=0x0 + + testpmd> stop + Telling cores to stop... + Waiting for lcores to finish... + +.. image:: img/forward_stats.* + +.. code-block:: console + + +++++++++++++++ Accumulated forward statistics for allports++++++++++ + RX-packets: 462384736 RX-dropped: 0 RX-total: 462384736 + TX-packets: 462384768 TX-dropped: 0 TX-total: 462384768 + +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + Done. + + +Using the Ring-based PMD from an Application +-------------------------------------------- + +The driver provides an API to create PMD (``rte_ethdev`` structure) instances +at run-time in the end-application using the function ``rte_eth_from_rings()``. +This functionality can be used to allow data exchange between cores using rings +in the same way as sending or receiving packets from an Ethernet device. + +Usage Examples +^^^^^^^^^^^^^^ + +To create two pseudo-Ethernet ports where all traffic sent to a port is looped back +for reception on the same port (error handling omitted for clarity): + +.. code-block:: c + + #define RING_SIZE 256 + #define NUM_RINGS 2 + #define SOCKET0 0 + + struct rte_ring *ring[NUM_RINGS]; + int port0, port1; + + ring[0] = rte_ring_create("R0", RING_SIZE, SOCKET0, RING_F_SP_ENQ|RING_F_SC_DEQ); + ring[1] = rte_ring_create("R1", RING_SIZE, SOCKET0, RING_F_SP_ENQ|RING_F_SC_DEQ); + + /* create two ethdev's */ + port0 = rte_eth_from_rings("net_ring0", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); + port1 = rte_eth_from_rings("net_ring1", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); + + +To create two pseudo-Ethernet ports where the traffic is switched between them +(traffic sent to port 0 is read back from port 1 and vice-versa), +the final two lines can be changed as follows: + +.. code-block:: c + + port0 = rte_eth_from_rings("net_ring0", &ring[0], 1, &ring[1], 1, SOCKET0); + port1 = rte_eth_from_rings("net_ring1", &ring[1], 1, &ring[0], 1, SOCKET0); + +This type of configuration is useful in a pipeline model where inter-core communication +using pseudo Ethernet devices is preferred over raw rings for API consistency. + +Enqueuing and dequeuing items from an ``rte_ring`` +using the ring-based PMD may be slower than using the native ring API. +DPDK Ethernet drivers use function pointers +to call the appropriate enqueue or dequeue functions, +while the ``rte_ring`` specific functions are direct function calls +and are often inlined by the compiler. + +Once an ethdev has been created for a ring-based PMD, +it should be configured and started in the same way as a regular Ethernet device: +call ``rte_eth_dev_configure()`` to set the number of receive and transmit queues, +then call ``rte_eth_rx_queue_setup()`` / ``tx_queue_setup()`` for each of those queues, +and finally call ``rte_eth_dev_start()`` to allow transmission and reception of packets to begin. diff --git a/doc/guides/rel_notes/release_26_03.rst b/doc/guides/rel_notes/release_26_03.rst index 5c2a4bb32ec..afdf1af06c2 100644 --- a/doc/guides/rel_notes/release_26_03.rst +++ b/doc/guides/rel_notes/release_26_03.rst @@ -82,6 +82,11 @@ New Features * NEA5, NIA5, NCA5: AES 256 confidentiality, integrity and AEAD modes. * NEA6, NIA6, NCA6: ZUC 256 confidentiality, integrity and AEAD modes. +* **Updated openssl crypto driver.** + + * Added support for AES-XTS cipher algorithm. + * Added support for SHAKE-128 and SHAKE-256 authentication algorithms. + Removed Items ------------- diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst b/doc/guides/sample_app_ug/ipsec_secgw.rst index 7319505fe9f..7c31c96b7c3 100644 --- a/doc/guides/sample_app_ug/ipsec_secgw.rst +++ b/doc/guides/sample_app_ug/ipsec_secgw.rst @@ -576,6 +576,7 @@ where each options means: * *aes-128-ctr*: AES-CTR 128-bit algorithm * *3des-cbc*: 3DES-CBC 192-bit algorithm * *des-cbc*: DES-CBC 64-bit algorithm + * *sm4-cbc*: SM4-CBC 128-bit algorithm * Syntax: *cipher_algo * @@ -605,6 +606,7 @@ where each options means: * *sha1-hmac*: HMAC SHA1 algorithm * *sha256-hmac*: HMAC SHA256 algorithm * *aes-xcbc-mac*: AES XCBC MAC algorithm + * *sm3-hmac*: HMAC SM3 algorithm ```` @@ -820,6 +822,13 @@ Example SA rules: src 1111:1111:1111:1111:1111:1111:1111:5555 \ dst 2222:2222:2222:2222:2222:2222:2222:5555 + sa out 30 cipher_algo sm4-cbc \ + cipher_key 01:23:45:67:89:ab:cd:ef:fe:dc:ba:98:76:54:32:10 \ + auth_algo sm3-hmac \ + auth_key 01:23:45:67:89:ab:cd:ef:fe:dc:ba:98:76:54:32:10:11:22:33:44 \ + mode ipv4-tunnel \ + src 172.16.1.5 dst 172.16.2.5 + sa in 105 aead_algo aes-128-gcm \ aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \ mode ipv4-tunnel src 172.16.2.5 dst 172.16.1.5 diff --git a/drivers/common/cnxk/roc_cpt.c b/drivers/common/cnxk/roc_cpt.c index 83e0c9896b0..0deb0b52d5b 100644 --- a/drivers/common/cnxk/roc_cpt.c +++ b/drivers/common/cnxk/roc_cpt.c @@ -1275,8 +1275,8 @@ roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr, uint8_t egrp; int i; - if (!plt_is_aligned(sa_cptr, 128)) { - plt_err("Context pointer should be 128B aligned"); + if (!plt_is_aligned(sa_cptr, ROC_CPTR_ALIGN)) { + plt_err("Context pointer should be %dB aligned", ROC_CPTR_ALIGN); return -EINVAL; } diff --git a/drivers/common/cnxk/roc_cpt.h b/drivers/common/cnxk/roc_cpt.h index 67956758be8..37873820e20 100644 --- a/drivers/common/cnxk/roc_cpt.h +++ b/drivers/common/cnxk/roc_cpt.h @@ -130,7 +130,12 @@ BITS_PER_LONG_LONG) /* ROC CPTR Cache */ +#if defined(ROC_PLATFORM_CN10K) || defined(ROC_PLATFORM_CN9K) +#define ROC_CPTR_CACHE_LINE_SZ 128 +#else #define ROC_CPTR_CACHE_LINE_SZ 256 +#endif + #define ROC_CPTR_ALIGN ROC_CPTR_CACHE_LINE_SZ #define ROC_CPT_CQ_ENTRY_SIZE_UNIT 32 diff --git a/drivers/common/mlx5/linux/meson.build b/drivers/common/mlx5/linux/meson.build index 3767e7a69b6..b2ef6598b3f 100644 --- a/drivers/common/mlx5/linux/meson.build +++ b/drivers/common/mlx5/linux/meson.build @@ -223,7 +223,11 @@ if libmtcr_ul_found endif foreach arg:has_sym_args - mlx5_config.set(arg[0], cc.has_header_symbol(arg[1], arg[2], dependencies: libs, args: cflags)) + file_prefix = '#pragma clang diagnostic ignored "-Wunused-value"' + cflags += [ + '-Wno-pedantic', + ] + mlx5_config.set(arg[0], cc.has_header_symbol(arg[1], arg[2], prefix : file_prefix, dependencies: libs, args: cflags)) endforeach foreach arg:has_member_args file_prefix = '#include <' + arg[1] + '>' diff --git a/drivers/compress/isal/meson.build b/drivers/compress/isal/meson.build index 4b3eaa2274e..5daf24d1135 100644 --- a/drivers/compress/isal/meson.build +++ b/drivers/compress/isal/meson.build @@ -3,8 +3,12 @@ dep = dependency('libisal', required: false, method: 'pkg-config') if not dep.found() - build = false + build = true reason = 'missing dependency, "libisal"' + isal_dep = cc.find_library('libisal', required: false) + if isal_dep.found() + ext_deps += isal_dep + endif endif deps += 'bus_vdev' diff --git a/drivers/compress/zlib/zlib_pmd.c b/drivers/compress/zlib/zlib_pmd.c index 1d7651dd7d4..ed3905484ca 100644 --- a/drivers/compress/zlib/zlib_pmd.c +++ b/drivers/compress/zlib/zlib_pmd.c @@ -48,12 +48,12 @@ process_zlib_deflate_chksum(struct rte_comp_op *op, return; } - dictionary_start = (uint32_t)(*dictionary); - dictionary_end = (uint32_t)(*(dictionary + dictionary_len - 4)); + dictionary_start = *(uint32_t *)dictionary; + dictionary_end = *(uint32_t *)(dictionary + dictionary_len - 4); sum = (dictionary_start & BOTTOM_NIBBLE_OF_BYTES_IN_DOUBLE_WORD) - + (dictionary_start & (TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD >> 4)) + + ((dictionary_start & TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD) >> 4) + (dictionary_end & BOTTOM_NIBBLE_OF_BYTES_IN_DOUBLE_WORD) - + (dictionary_end & (TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD >> 4)); + + ((dictionary_end & TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD) >> 4); op->output_chksum = ~(sum_bytes[0] + sum_bytes[1] + sum_bytes[2] + sum_bytes[3]) & BOTTOM_NIBBLE_OF_BYTE; @@ -98,12 +98,12 @@ process_zlib_inflate_chksum(struct rte_comp_op *op, return; } - dictionary_start = (uint32_t)(*dictionary); - dictionary_end = (uint32_t)(*(dictionary + dictionary_len - 4)); + dictionary_start = *(uint32_t *)dictionary; + dictionary_end = *(uint32_t *)(dictionary + dictionary_len - 4); sum = (dictionary_start & BOTTOM_NIBBLE_OF_BYTES_IN_DOUBLE_WORD) - + (dictionary_start & (TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD >> 4)) + + ((dictionary_start & TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD) >> 4) + (dictionary_end & BOTTOM_NIBBLE_OF_BYTES_IN_DOUBLE_WORD) - + (dictionary_end & (TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD >> 4)); + + ((dictionary_end & TOP_NIBBLE_OF_BYTE_IN_DOUBLE_WORD) >> 4); op->output_chksum = ~(sum_bytes[0] + sum_bytes[1] + sum_bytes[2] + sum_bytes[3]) & BOTTOM_NIBBLE_OF_BYTE; diff --git a/drivers/crypto/cnxk/cn20k_tls.c b/drivers/crypto/cnxk/cn20k_tls.c index 9f7acefc19f..8556a95ab6c 100644 --- a/drivers/crypto/cnxk/cn20k_tls.c +++ b/drivers/crypto/cnxk/cn20k_tls.c @@ -385,13 +385,20 @@ cn20k_tls_read_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, int ret = 0; tls = &sec_sess->tls_rec; - read_sa = &tls->read_sa; + + read_sa = rte_zmalloc("cn20k_tls", sizeof(struct roc_ie_ow_tls_read_sa), ROC_CPTR_ALIGN); + if (read_sa == NULL) { + plt_err("Couldn't allocate memory for READ SA"); + return -ENOMEM; + } + tls->read_sa = read_sa; /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */ sa_dptr = plt_zmalloc(sizeof(struct roc_ie_ow_tls_read_sa), 8); if (sa_dptr == NULL) { plt_err("Could not allocate memory for SA dptr"); - return -ENOMEM; + ret = -ENOMEM; + goto sa_cptr_free; } /* Translate security parameters to SA */ @@ -457,6 +464,11 @@ cn20k_tls_read_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, sa_dptr_free: plt_free(sa_dptr); +sa_cptr_free: + if (ret != 0) { + rte_free(read_sa); + read_sa = NULL; + } return ret; } @@ -706,13 +718,20 @@ cn20k_tls_write_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, int ret = 0; tls = &sec_sess->tls_rec; - write_sa = &tls->write_sa; + + write_sa = rte_zmalloc("cn20k_tls", sizeof(struct roc_ie_ow_tls_write_sa), ROC_CPTR_ALIGN); + if (write_sa == NULL) { + plt_err("Couldn't allocate memory for WRITE SA"); + return -ENOMEM; + } + tls->write_sa = write_sa; /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */ sa_dptr = plt_zmalloc(sizeof(struct roc_ie_ow_tls_write_sa), 8); if (sa_dptr == NULL) { plt_err("Could not allocate memory for SA dptr"); - return -ENOMEM; + ret = -ENOMEM; + goto sa_cptr_free; } /* Translate security parameters to SA */ @@ -781,6 +800,11 @@ cn20k_tls_write_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, sa_dptr_free: plt_free(sa_dptr); +sa_cptr_free: + if (ret != 0) { + rte_free(write_sa); + write_sa = NULL; + } return ret; } @@ -868,15 +892,18 @@ cn20k_sec_tls_session_destroy(struct cnxk_cpt_qp *qp, struct cn20k_sec_session * tls = &sess->tls_rec; + if (tls->sa_ptr == NULL) + return -EINVAL; + /* Trigger CTX flush to write dirty data back to DRAM */ - roc_cpt_lf_ctx_flush(lf, &tls->read_sa, false); + roc_cpt_lf_ctx_flush(lf, tls->read_sa, false); if (sess->tls_opt.is_write) { sa_dptr = plt_zmalloc(sizeof(struct roc_ie_ow_tls_write_sa), 8); if (sa_dptr != NULL) { tls_write_sa_init(sa_dptr); - ret = roc_cpt_ctx_write(lf, sa_dptr, &tls->write_sa, + ret = roc_cpt_ctx_write(lf, sa_dptr, tls->write_sa, sizeof(struct roc_ie_ow_tls_write_sa)); plt_free(sa_dptr); } @@ -889,14 +916,14 @@ cn20k_sec_tls_session_destroy(struct cnxk_cpt_qp *qp, struct cn20k_sec_session * rte_atomic_thread_fence(rte_memory_order_seq_cst); /* Trigger CTX reload to fetch new data from DRAM */ - roc_cpt_lf_ctx_reload(lf, &tls->write_sa); + roc_cpt_lf_ctx_reload(lf, tls->write_sa); } } else { sa_dptr = plt_zmalloc(sizeof(struct roc_ie_ow_tls_read_sa), 8); if (sa_dptr != NULL) { tls_read_sa_init(sa_dptr); - ret = roc_cpt_ctx_write(lf, sa_dptr, &tls->read_sa, + ret = roc_cpt_ctx_write(lf, sa_dptr, tls->read_sa, sizeof(struct roc_ie_ow_tls_read_sa)); plt_free(sa_dptr); } @@ -909,9 +936,11 @@ cn20k_sec_tls_session_destroy(struct cnxk_cpt_qp *qp, struct cn20k_sec_session * rte_atomic_thread_fence(rte_memory_order_seq_cst); /* Trigger CTX reload to fetch new data from DRAM */ - roc_cpt_lf_ctx_reload(lf, &tls->read_sa); + roc_cpt_lf_ctx_reload(lf, tls->read_sa); } } + rte_free(tls->sa_ptr); + return 0; } diff --git a/drivers/crypto/cnxk/cn20k_tls.h b/drivers/crypto/cnxk/cn20k_tls.h index 27124602a0f..5fed7495459 100644 --- a/drivers/crypto/cnxk/cn20k_tls.h +++ b/drivers/crypto/cnxk/cn20k_tls.h @@ -16,13 +16,18 @@ /* Forward declaration */ struct cn20k_sec_session; -struct __rte_aligned(ROC_ALIGN) cn20k_tls_record +struct __rte_aligned(ROC_CPTR_ALIGN) cn20k_tls_record { union { - /** Read SA */ - struct roc_ie_ow_tls_read_sa read_sa; - /** Write SA */ - struct roc_ie_ow_tls_write_sa write_sa; + void *sa_ptr; + struct { + union { + /** Read SA */ + struct roc_ie_ow_tls_read_sa *read_sa; + /** Write SA */ + struct roc_ie_ow_tls_write_sa *write_sa; + }; + }; }; }; diff --git a/drivers/crypto/cnxk/cn20k_tls_ops.h b/drivers/crypto/cnxk/cn20k_tls_ops.h index 9f70a1d42d4..e7a8ba34aec 100644 --- a/drivers/crypto/cnxk/cn20k_tls_ops.h +++ b/drivers/crypto/cnxk/cn20k_tls_ops.h @@ -38,7 +38,11 @@ process_tls_write(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn20k pad_len = (pad_bytes >> tls_opt.pad_shift) * tls_opt.enable_padding; #ifdef LA_IPSEC_DEBUG - write_sa = &sess->tls_rec.write_sa; + write_sa = sess->tls_rec.write_sa; + if (write_sa == NULL) { + return -EINVAL; + } + if (write_sa->w2.s.iv_at_cptr == ROC_IE_OW_TLS_IV_SRC_FROM_SA) { uint8_t *iv = PLT_PTR_ADD(write_sa->cipher_key, 32); diff --git a/drivers/crypto/cnxk/cnxk_ae.h b/drivers/crypto/cnxk/cnxk_ae.h index 912a2a94965..21a0c8068a9 100644 --- a/drivers/crypto/cnxk/cnxk_ae.h +++ b/drivers/crypto/cnxk/cnxk_ae.h @@ -1591,9 +1591,10 @@ cnxk_ae_dequeue_rsa_op(struct rte_crypto_op *cop, uint8_t *rptr, break; case RTE_CRYPTO_ASYM_OP_VERIFY: if (rsa_ctx->padding.type == RTE_CRYPTO_RSA_PADDING_NONE) { - rsa->sign.length = rsa_ctx->n.length; - if (memcmp(rptr, rsa->message.data, rsa->message.length)) - cop->status = RTE_CRYPTO_OP_STATUS_ERROR; + /* Application compares decrypted data with message for SW padding schemes + */ + rsa->cipher.length = rsa_ctx->n.length; + memcpy(rsa->cipher.data, rptr, rsa->cipher.length); } else { /* Get length of signed output */ rsa->sign.length = rte_cpu_to_be_16(*((uint16_t *)rptr)); diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 698548e6ead..995e375fb5a 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -4456,8 +4456,6 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) retcode); goto init_error; } - snprintf(cryptodev->data->name, sizeof(cryptodev->data->name), - "dpsec-%u", hw_id); internals->max_nb_queue_pairs = attr.num_tx_queues; cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs; @@ -4482,7 +4480,6 @@ cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, struct rte_dpaa2_device *dpaa2_dev) { struct rte_cryptodev *cryptodev; - char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; int retval; struct rte_cryptodev_pmd_init_params init_params = { .name = "", @@ -4493,10 +4490,7 @@ cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, /* setting default, will be updated in init. */ }; - snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d", - dpaa2_dev->object_id); - - cryptodev = rte_cryptodev_pmd_create(cryptodev_name, &dpaa2_dev->device, + cryptodev = rte_cryptodev_pmd_create(dpaa2_dev->device.name, &dpaa2_dev->device, &init_params); if (cryptodev == NULL) { DPAA2_SEC_ERR("failed to create cryptodev vdev"); @@ -4524,14 +4518,10 @@ cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, static int cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) { - char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct rte_cryptodev *cryptodev; int ret; - snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d", - dpaa2_dev->object_id); - - cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); + cryptodev = rte_cryptodev_pmd_get_named_dev(dpaa2_dev->device.name); if (cryptodev == NULL) return -ENODEV; diff --git a/drivers/crypto/openssl/openssl_pmd_private.h b/drivers/crypto/openssl/openssl_pmd_private.h index fe89e522e1b..d5a751600a4 100644 --- a/drivers/crypto/openssl/openssl_pmd_private.h +++ b/drivers/crypto/openssl/openssl_pmd_private.h @@ -118,7 +118,7 @@ struct __rte_cache_aligned openssl_session { /**< cipher algorithm */ struct { - uint8_t data[32]; + uint8_t data[64]; /**< key data */ size_t length; /**< key length in bytes */ diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index 4f171f48cc9..e5fa1a4eeb9 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -211,6 +211,18 @@ get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, res = -EINVAL; } break; + case RTE_CRYPTO_CIPHER_AES_XTS: + switch (keylen) { + case 32: + *algo = EVP_aes_128_xts(); + break; + case 64: + *algo = EVP_aes_256_xts(); + break; + default: + res = -EINVAL; + } + break; case RTE_CRYPTO_CIPHER_AES_CTR: switch (keylen) { case 16: @@ -270,6 +282,14 @@ get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, case RTE_CRYPTO_AUTH_SHA512_HMAC: *algo = EVP_sha512(); break; +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + case RTE_CRYPTO_AUTH_SHAKE_128: + *algo = EVP_shake128(); + break; + case RTE_CRYPTO_AUTH_SHAKE_256: + *algo = EVP_shake256(); + break; +#endif default: res = -EINVAL; break; @@ -493,6 +513,7 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: case RTE_CRYPTO_CIPHER_AES_CTR: + case RTE_CRYPTO_CIPHER_AES_XTS: sess->cipher.mode = OPENSSL_CIPHER_LIB; sess->cipher.algo = xform->cipher.algo; sess->cipher.ctx = EVP_CIPHER_CTX_new(); @@ -659,6 +680,10 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, case RTE_CRYPTO_AUTH_SHA256: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_SHA512: +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + case RTE_CRYPTO_AUTH_SHAKE_128: + case RTE_CRYPTO_AUTH_SHAKE_256: +#endif sess->auth.mode = OPENSSL_AUTH_AS_AUTH; if (get_auth_algo(xform->auth.algo, &sess->auth.auth.evp_algo) != 0) @@ -1397,7 +1422,7 @@ process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, static int process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, - int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) + int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo, int digest_length) { size_t dstlen; struct rte_mbuf *m; @@ -1437,8 +1462,24 @@ process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, } process_auth_final: - if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) + /* SHAKE algorithms are XOFs and require EVP_DigestFinalXOF */ + if (algo == EVP_shake128() || algo == EVP_shake256()) { +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + /* Set XOF output length before calling EVP_DigestFinalXOF */ + if (EVP_MD_CTX_ctrl(ctx, EVP_MD_CTRL_XOF_LEN, digest_length, NULL) <= 0) + goto process_auth_err; + if (EVP_DigestFinalXOF(ctx, dst, digest_length) <= 0) + goto process_auth_err; +#else + RTE_SET_USED(digest_length); + OPENSSL_LOG(ERR, "SHAKE algorithms require OpenSSL 3.0+"); goto process_auth_err; +#endif + } else { + if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) + goto process_auth_err; + } + return 0; process_auth_err: @@ -1995,7 +2036,7 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, ctx_a = get_local_auth_ctx(sess, qp); status = process_openssl_auth(mbuf_src, dst, op->sym->auth.data.offset, NULL, NULL, srclen, - ctx_a, sess->auth.auth.evp_algo); + ctx_a, sess->auth.auth.evp_algo, sess->auth.digest_length); break; case OPENSSL_AUTH_AS_HMAC: ctx_h = get_local_hmac_ctx(sess, qp); @@ -4008,12 +4049,14 @@ mldsa_sign_op_evp(struct rte_crypto_op *cop, case RTE_CRYPTO_AUTH_SHA3_512: check_md = EVP_sha3_512(); break; +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) case RTE_CRYPTO_AUTH_SHAKE_128: check_md = EVP_shake128(); break; case RTE_CRYPTO_AUTH_SHAKE_256: check_md = EVP_shake256(); break; +#endif default: break; } diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c index 5095e6cbea8..0f2b82ec00f 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c +++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -269,6 +269,70 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { }, } }, } }, + { /* AES XTS */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_XTS, + .block_size = 16, + .key_size = { + .min = 32, + .max = 64, + .increment = 32 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + { /* SHAKE_128 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHAKE_128, + .block_size = 168, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 1, + .max = 256, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHAKE_256 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHAKE_256, + .block_size = 136, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 1, + .max = 256, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, +#endif { /* AES CBC */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { diff --git a/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h b/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h index 67dc889b503..22ee0fe4fef 100644 --- a/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +++ b/drivers/crypto/qat/dev/qat_crypto_pmd_gens.h @@ -443,6 +443,21 @@ qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req, n_dst > QAT_SYM_SGL_MAX_NUMBER)) return -1; + /* For crypto API only: try to align the in-place buffers*/ + if (op != NULL && likely(n_dst == 0) && likely(!is_sgl)) { + rte_iova_t offset = src_vec[0].iova & RTE_CACHE_LINE_MASK; + if (offset) { + rte_iova_t buff_addr = rte_mbuf_iova_get(op->sym->m_src); + /* make sure src_data_start is still within the buffer */ + if (src_vec[0].iova - offset >= buff_addr) { + src_vec[0].iova -= offset; + src_vec[0].len += offset; + ofs->ofs.auth.head += offset; + ofs->ofs.cipher.head += offset; + } + } + } + if (likely(!is_sgl)) { src_data_start = src_vec[0].iova; tl_src = total_len_src = @@ -503,24 +518,6 @@ qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req, dst_data_start = src_data_start; } - /* For crypto API only try to align the in-place buffers*/ - if (op != NULL && likely(n_dst == 0)) { - uint16_t offset = src_data_start & RTE_CACHE_LINE_MASK; - if (offset) { - rte_iova_t buff_addr = rte_mbuf_iova_get(op->sym->m_src); - /* make sure src_data_start is still within the buffer */ - if (src_data_start - offset >= buff_addr) { - src_data_start -= offset; - dst_data_start = src_data_start; - ofs->ofs.auth.head += offset; - ofs->ofs.cipher.head += offset; - tl_src += offset; - total_len_src = tl_src; - total_len_dst = tl_src; - } - } - } - req->comn_mid.src_data_addr = src_data_start; req->comn_mid.dest_data_addr = dst_data_start; req->comn_mid.src_length = total_len_src; diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c index b72a5604c80..46c3ad3ec73 100644 --- a/examples/ipsec-secgw/esp.c +++ b/examples/ipsec-secgw/esp.c @@ -103,6 +103,7 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa, case RTE_CRYPTO_CIPHER_DES_CBC: case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_SM4_CBC: /* Copy IV at the end of crypto operation */ rte_memcpy(iv_ptr, iv, sa->iv_len); break; @@ -123,6 +124,7 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa, case RTE_CRYPTO_AUTH_SHA1_HMAC: case RTE_CRYPTO_AUTH_SHA256_HMAC: case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SM3_HMAC: sym_cop->auth.data.offset = ip_hdr_len; sym_cop->auth.data.length = sizeof(struct rte_esp_hdr) + sa->iv_len + payload_len; @@ -341,6 +343,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa, case RTE_CRYPTO_CIPHER_DES_CBC: case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_SM4_CBC: memset(iv, 0, sa->iv_len); break; case RTE_CRYPTO_CIPHER_AES_CTR: @@ -405,6 +408,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa, case RTE_CRYPTO_CIPHER_DES_CBC: case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_SM4_CBC: sym_cop->cipher.data.offset = ip_hdr_len + sizeof(struct rte_esp_hdr); sym_cop->cipher.data.length = pad_payload_len + sa->iv_len; @@ -436,6 +440,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa, case RTE_CRYPTO_AUTH_SHA1_HMAC: case RTE_CRYPTO_AUTH_SHA256_HMAC: case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SM3_HMAC: sym_cop->auth.data.offset = ip_hdr_len; sym_cop->auth.data.length = sizeof(struct rte_esp_hdr) + sa->iv_len + pad_payload_len; diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c index 313919b4b57..86aeb25a498 100644 --- a/examples/ipsec-secgw/sa.c +++ b/examples/ipsec-secgw/sa.c @@ -128,6 +128,13 @@ const struct supported_cipher_algo cipher_algos[] = { .iv_len = 8, .block_size = 8, .key_len = 8 + }, + { + .keyword = "sm4-cbc", + .algo = RTE_CRYPTO_CIPHER_SM4_CBC, + .iv_len = 16, + .block_size = 16, + .key_len = 16 } }; @@ -175,6 +182,12 @@ const struct supported_auth_algo auth_algos[] = { .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC, .digest_len = 12, .key_len = 16 + }, + { + .keyword = "sm3-hmac", + .algo = RTE_CRYPTO_AUTH_SM3_HMAC, + .digest_len = 12, + .key_len = 20 } }; @@ -502,7 +515,8 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens, return; if (algo->algo == RTE_CRYPTO_CIPHER_AES_CBC || - algo->algo == RTE_CRYPTO_CIPHER_3DES_CBC) + algo->algo == RTE_CRYPTO_CIPHER_3DES_CBC || + algo->algo == RTE_CRYPTO_CIPHER_SM4_CBC) rule->salt = (uint32_t)rte_rand(); if (algo->algo == RTE_CRYPTO_CIPHER_AES_CTR) { @@ -1319,6 +1333,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], case RTE_CRYPTO_CIPHER_DES_CBC: case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_SM4_CBC: iv_length = sa->iv_len; break; case RTE_CRYPTO_CIPHER_AES_CTR: diff --git a/lib/eal/common/eal_common_memory.c b/lib/eal/common/eal_common_memory.c index dccf9406c58..3f1cf760d24 100644 --- a/lib/eal/common/eal_common_memory.c +++ b/lib/eal/common/eal_common_memory.c @@ -269,6 +269,11 @@ eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags) EAL_LOG(DEBUG, "VA reserved for memseg list at %p, size %zx", addr, mem_sz); + if (eal_memseg_list_map_asan_shadow(msl) != 0) { + RTE_LOG(ERR, EAL, "Failed to map ASan shadow region for memseg list"); + return -1; + } + return 0; } @@ -1087,6 +1092,8 @@ rte_eal_memory_detach(void) EAL_LOG(ERR, "Could not unmap memory: %s", rte_strerror(rte_errno)); + eal_memseg_list_unmap_asan_shadow(msl); + /* * we are detaching the fbarray rather than destroying because * other processes might still reference this fbarray, and we diff --git a/lib/eal/common/eal_private.h b/lib/eal/common/eal_private.h index e032dd10c9d..277b9723366 100644 --- a/lib/eal/common/eal_private.h +++ b/lib/eal/common/eal_private.h @@ -335,6 +335,60 @@ eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags); void eal_memseg_list_populate(struct rte_memseg_list *msl, void *addr, int n_segs); +/** + * Map shared memory for MSL ASan shadow region. + * + * @param msl + * Memory segment list. + * @return + * 0 on success, (-1) on failure. + */ +#ifdef RTE_MALLOC_ASAN +int +eal_memseg_list_map_asan_shadow(struct rte_memseg_list *msl); +#else +static inline int +eal_memseg_list_map_asan_shadow(__rte_unused struct rte_memseg_list *msl) +{ + return 0; +} +#endif + +/** + * Unmap the MSL ASan shadow region. + * + * @param msl + * Memory segment list. + */ +#ifdef RTE_MALLOC_ASAN +void +eal_memseg_list_unmap_asan_shadow(struct rte_memseg_list *msl); +#else +static inline void +eal_memseg_list_unmap_asan_shadow(__rte_unused struct rte_memseg_list *msl) +{ +} +#endif + +/** + * Get the MSL ASan shadow shared memory object file descriptor. + * + * @param msl + * Index of the MSL. + * @return + * A file descriptor. + */ +#ifdef RTE_MALLOC_ASAN +int +eal_memseg_list_get_asan_shadow_fd(int msl_idx); +#else +static inline int +eal_memseg_list_get_asan_shadow_fd(__rte_unused int msl_idx) +{ + return -1; +} +#endif + /** * Distribute available memory between MSLs. * diff --git a/lib/eal/common/malloc_heap.c b/lib/eal/common/malloc_heap.c index 39240c261cc..4c951e30a0f 100644 --- a/lib/eal/common/malloc_heap.c +++ b/lib/eal/common/malloc_heap.c @@ -401,8 +401,8 @@ try_expand_heap_primary(struct malloc_heap *heap, uint64_t pg_sz, int n_segs; bool callback_triggered = false; - alloc_sz = RTE_ALIGN_CEIL(RTE_ALIGN_CEIL(elt_size, align) + - MALLOC_ELEM_OVERHEAD, pg_sz); + alloc_sz = RTE_ALIGN_CEIL(RTE_MAX(MALLOC_ELEM_HEADER_LEN, align) + + elt_size + MALLOC_ELEM_TRAILER_LEN, pg_sz); n_segs = alloc_sz / pg_sz; /* we can't know in advance how many pages we'll need, so we malloc */ diff --git a/lib/eal/common/malloc_mp.c b/lib/eal/common/malloc_mp.c index 9765277f5d0..1373da44c9e 100644 --- a/lib/eal/common/malloc_mp.c +++ b/lib/eal/common/malloc_mp.c @@ -251,8 +251,8 @@ handle_alloc_request(const struct malloc_mp_req *m, return -1; } - alloc_sz = RTE_ALIGN_CEIL(RTE_ALIGN_CEIL(ar->elt_size, ar->align) + - MALLOC_ELEM_OVERHEAD, ar->page_sz); + alloc_sz = RTE_ALIGN_CEIL(RTE_MAX(MALLOC_ELEM_HEADER_LEN, ar->align) + + ar->elt_size + MALLOC_ELEM_TRAILER_LEN, ar->page_sz); n_segs = alloc_sz / ar->page_sz; /* we can't know in advance how many pages we'll need, so we malloc */ diff --git a/lib/eal/linux/eal_memalloc.c b/lib/eal/linux/eal_memalloc.c index 4dee224ac51..63012783dcf 100644 --- a/lib/eal/linux/eal_memalloc.c +++ b/lib/eal/linux/eal_memalloc.c @@ -34,6 +34,7 @@ #include "eal_memalloc.h" #include "eal_memcfg.h" #include "eal_private.h" +#include "malloc_elem.h" const int anonymous_hugepages_supported = #ifdef MAP_HUGE_SHIFT @@ -494,6 +495,21 @@ resize_hugefile(int fd, uint64_t fa_offset, uint64_t page_sz, bool grow, grow, dirty); } +__rte_no_asan +static inline void +page_fault(void *addr) +{ + /* We need to trigger a write to the page to enforce page fault but we + * can't overwrite value that is already there, so read the old value + * and write it back. Kernel populates the page with zeroes initially. + * + * Disable ASan instrumentation here because if the segment is already + * allocated by another process and is marked as free in the shadow, + * accessing this address will cause an ASan error. + */ + *(volatile int *)addr = *(volatile int *)addr; +} + static int alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, struct hugepage_info *hi, unsigned int list_idx, @@ -593,12 +609,8 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, goto mapped; } - /* we need to trigger a write to the page to enforce page fault and - * ensure that page is accessible to us, but we can't overwrite value - * that is already there, so read the old value, and write itback. - * kernel populates the page with zeroes initially. - */ - *(volatile int *)addr = *(volatile int *)addr; + /* enforce page fault and ensure that page is accessible to us */ + page_fault(addr); iova = rte_mem_virt2iova(addr); if (iova == RTE_BAD_PHYS_ADDR) { @@ -634,6 +646,35 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, __func__); #endif +#ifdef RTE_MALLOC_ASAN + struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + int shadow_shm_fd = eal_memseg_list_get_asan_shadow_fd(list_idx); + + if (shadow_shm_fd != -1) { + void *shadow_base_addr, *shadow_addr; + off_t shadow_map_offset; + size_t shadow_sz; + + shadow_base_addr = ASAN_MEM_TO_SHADOW(mcfg->memsegs[list_idx].base_va); + shadow_addr = ASAN_MEM_TO_SHADOW(addr); + shadow_map_offset = (char *)shadow_addr - (char *)shadow_base_addr; + shadow_sz = alloc_sz >> ASAN_SHADOW_SCALE; + + va = mmap(shadow_addr, shadow_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, shadow_shm_fd, shadow_map_offset); + if (va == MAP_FAILED) { + RTE_LOG(DEBUG, EAL, "shadow mmap() failed: %s\n", + strerror(errno)); + goto mapped; + } + + if (va != shadow_addr) { + RTE_LOG(DEBUG, EAL, "wrong shadow mmap() address\n"); + munmap(addr, shadow_sz); + goto mapped; + } + } +#endif huge_recover_sigbus(); ms->addr = addr; diff --git a/lib/eal/linux/eal_memory.c b/lib/eal/linux/eal_memory.c index bf783e3c76c..efa79cd4e6a 100644 --- a/lib/eal/linux/eal_memory.c +++ b/lib/eal/linux/eal_memory.c @@ -40,6 +40,7 @@ #include "eal_filesystem.h" #include "eal_hugepages.h" #include "eal_options.h" +#include "malloc_elem.h" #define PFN_MASK_SIZE 8 @@ -1489,6 +1490,7 @@ eal_legacy_hugepage_init(void) if (msl->memseg_arr.count > 0) continue; /* this is an unused list, deallocate it */ + eal_memseg_list_unmap_asan_shadow(msl); mem_sz = msl->len; munmap(msl->base_va, mem_sz); msl->base_va = NULL; @@ -1943,6 +1945,10 @@ memseg_secondary_init(void) return 0; } +#ifdef RTE_MALLOC_ASAN +static int msl_asan_shadow_fd[RTE_MAX_MEMSEG_LISTS]; +#endif + int rte_eal_memseg_init(void) { @@ -1952,6 +1958,9 @@ rte_eal_memseg_init(void) #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES const struct internal_config *internal_conf = eal_get_internal_configuration(); +#endif +#ifdef RTE_MALLOC_ASAN + int msl_idx; #endif if (getrlimit(RLIMIT_NOFILE, &lim) == 0) { /* set limit to maximum */ @@ -1975,6 +1984,11 @@ rte_eal_memseg_init(void) EAL_LOG(WARNING, "Please use --legacy-mem option, or recompile with NUMA support."); } #endif +#ifdef RTE_MALLOC_ASAN + for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) { + msl_asan_shadow_fd[msl_idx] = -1; + } +#endif return rte_eal_process_type() == RTE_PROC_PRIMARY ? #ifndef RTE_ARCH_64 @@ -1984,3 +1998,89 @@ rte_eal_memseg_init(void) #endif memseg_secondary_init(); } + +#ifdef RTE_MALLOC_ASAN +int +eal_memseg_list_map_asan_shadow(struct rte_memseg_list *msl) +{ + const struct internal_config *internal_conf = + eal_get_internal_configuration(); + int msl_idx = msl - rte_eal_get_configuration()->mem_config->memsegs; + int shm_oflag; + char shm_path[PATH_MAX]; + int shm_fd; + + if (!msl->heap) + return 0; + + /* these options imply no secondary process support */ + if (internal_conf->hugepage_file.unlink_before_mapping || + internal_conf->no_shconf || internal_conf->no_hugetlbfs) { + RTE_ASSERT(rte_eal_process_type() != RTE_PROC_SECONDARY); + return 0; + } + + snprintf(shm_path, sizeof(shm_path), "/%s_%s_shadow", + eal_get_hugefile_prefix(), msl->memseg_arr.name); + + shm_oflag = O_RDWR; + if (internal_conf->process_type == RTE_PROC_PRIMARY) + shm_oflag |= O_CREAT | O_TRUNC; + + shm_fd = shm_open(shm_path, shm_oflag, 0600); + if (shm_fd == -1) { + RTE_LOG(DEBUG, EAL, "shadow shm_open() failed: %s\n", + strerror(errno)); + return -1; + } + + if (internal_conf->process_type == RTE_PROC_PRIMARY) { + if (ftruncate(shm_fd, msl->len >> ASAN_SHADOW_SCALE) == -1) { + RTE_LOG(DEBUG, EAL, "shadow ftruncate() failed: %s\n", + strerror(errno)); + close(shm_fd); + if (internal_conf->process_type == RTE_PROC_PRIMARY) + shm_unlink(shm_path); + return -1; + } + } + + msl_asan_shadow_fd[msl_idx] = shm_fd; + + return 0; +} + +void +eal_memseg_list_unmap_asan_shadow(struct rte_memseg_list *msl) +{ + const struct internal_config *internal_conf = + eal_get_internal_configuration(); + int msl_idx = msl - rte_eal_get_configuration()->mem_config->memsegs; + int *shm_fd = &msl_asan_shadow_fd[msl_idx]; + + if (*shm_fd == -1) + return; + + close(*shm_fd); + *shm_fd = -1; + + if (munmap(ASAN_MEM_TO_SHADOW(msl->base_va), + msl->len >> ASAN_SHADOW_SCALE) != 0) + RTE_LOG(ERR, EAL, "Could not unmap asan shadow memory: %s\n", + strerror(errno)); + if (internal_conf->process_type == RTE_PROC_PRIMARY) { + char shm_path[PATH_MAX]; + + snprintf(shm_path, sizeof(shm_path), "/%s_%s_shadow", + eal_get_hugefile_prefix(), + msl->memseg_arr.name); + shm_unlink(shm_path); + } +} + +int +eal_memseg_list_get_asan_shadow_fd(int msl_idx) +{ + return msl_asan_shadow_fd[msl_idx]; +} +#endif diff --git a/lib/eal/linux/meson.build b/lib/eal/linux/meson.build index e99ebed2569..1e8a48c8d32 100644 --- a/lib/eal/linux/meson.build +++ b/lib/eal/linux/meson.build @@ -23,3 +23,7 @@ deps += ['kvargs', 'telemetry'] if has_libnuma dpdk_conf.set10('RTE_EAL_NUMA_AWARE_HUGEPAGES', true) endif + +if dpdk_conf.has('RTE_MALLOC_ASAN') + ext_deps += cc.find_library('rt') +endif diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h index a66c2abbdbf..0d8e2d0236a 100644 --- a/lib/ethdev/rte_ethdev.h +++ b/lib/ethdev/rte_ethdev.h @@ -1578,7 +1578,7 @@ struct rte_eth_conf { }; /** - * Rx offload capabilities of a device. + * Rx offload capabilities/configuration of a device or queue. */ #define RTE_ETH_RX_OFFLOAD_VLAN_STRIP RTE_BIT64(0) #define RTE_ETH_RX_OFFLOAD_IPV4_CKSUM RTE_BIT64(1) @@ -1613,12 +1613,12 @@ struct rte_eth_conf { RTE_ETH_RX_OFFLOAD_QINQ_STRIP) /* - * If new Rx offload capabilities are defined, they also must be + * If new Rx offloads are defined, they also must be * mentioned in rte_rx_offload_names in rte_ethdev.c file. */ /** - * Tx offload capabilities of a device. + * Tx offload capabilities/configuration of a device or queue. */ #define RTE_ETH_TX_OFFLOAD_VLAN_INSERT RTE_BIT64(0) #define RTE_ETH_TX_OFFLOAD_IPV4_CKSUM RTE_BIT64(1) @@ -1639,39 +1639,39 @@ struct rte_eth_conf { * Tx queue without SW lock. */ #define RTE_ETH_TX_OFFLOAD_MT_LOCKFREE RTE_BIT64(14) -/** Device supports multi segment send. */ +/** Multi segment send. */ #define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15) /** - * Device supports optimization for fast release of mbufs. + * Optimization for fast release of mbufs. * When set application must guarantee that per-queue all mbufs come from the same mempool, - * are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). + * have refcnt=1, and are direct. * * @see rte_mbuf_raw_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16) #define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17) /** - * Device supports generic UDP tunneled packet TSO. + * Generic UDP tunneled packet TSO. * Application must set RTE_MBUF_F_TX_TUNNEL_UDP and other mbuf fields required * for tunnel TSO. */ #define RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO RTE_BIT64(18) /** - * Device supports generic IP tunneled packet TSO. + * Generic IP tunneled packet TSO. * Application must set RTE_MBUF_F_TX_TUNNEL_IP and other mbuf fields required * for tunnel TSO. */ #define RTE_ETH_TX_OFFLOAD_IP_TNL_TSO RTE_BIT64(19) -/** Device supports outer UDP checksum */ +/** Outer UDP checksum. Used for tunneling packet. */ #define RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM RTE_BIT64(20) /** - * Device sends on time read from RTE_MBUF_DYNFIELD_TIMESTAMP_NAME + * Send on time read from RTE_MBUF_DYNFIELD_TIMESTAMP_NAME * if RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME is set in ol_flags. * The mbuf field and flag are registered when the offload is configured. */ #define RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP RTE_BIT64(21) /* - * If new Tx offload capabilities are defined, they also must be + * If new Tx offloads are defined, they also must be * mentioned in rte_tx_offload_names in rte_ethdev.c file. */ diff --git a/lib/hash/rte_cmp_riscv.h b/lib/hash/rte_cmp_riscv.h new file mode 100644 index 00000000000..b2ae0ce2875 --- /dev/null +++ b/lib/hash/rte_cmp_riscv.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2015 Intel Corporation + */ + +#include + +/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */ +static inline int +rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) +{ + const uint8_t *p1 = (const uint8_t *)key1; + const uint8_t *p2 = (const uint8_t *)key2; + size_t offset = 0; + + while (offset < 16) { + size_t vl = __riscv_vsetvl_e8m1(16 - offset); + + vuint8m1_t v1 = __riscv_vle8_v_u8m1(p1 + offset, vl); + vuint8m1_t v2 = __riscv_vle8_v_u8m1(p2 + offset, vl); + + /* find != bytes */ + vbool8_t neq = __riscv_vmsne_vv_u8m1_b8(v1, v2, vl); + + /* if any byte mismatches ¡ú not equal */ + if (__riscv_vfirst_m_b8(neq, vl) >= 0) + return 1; + + offset += vl; + } + + /* all bytes equal */ + return 0; +} + +static inline int +rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k16_cmp_eq(key1, key2, key_len) || + rte_hash_k16_cmp_eq((const char *) key1 + 16, + (const char *) key2 + 16, key_len); +} + +static inline int +rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k16_cmp_eq(key1, key2, key_len) || + rte_hash_k16_cmp_eq((const char *) key1 + 16, + (const char *) key2 + 16, key_len) || + rte_hash_k16_cmp_eq((const char *) key1 + 32, + (const char *) key2 + 32, key_len); +} + +static inline int +rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k32_cmp_eq(key1, key2, key_len) || + rte_hash_k32_cmp_eq((const char *) key1 + 32, + (const char *) key2 + 32, key_len); +} + +static inline int +rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) || + rte_hash_k16_cmp_eq((const char *) key1 + 64, + (const char *) key2 + 64, key_len); +} + +static inline int +rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) || + rte_hash_k32_cmp_eq((const char *) key1 + 64, + (const char *) key2 + 64, key_len); +} + +static inline int +rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) || + rte_hash_k32_cmp_eq((const char *) key1 + 64, + (const char *) key2 + 64, key_len) || + rte_hash_k16_cmp_eq((const char *) key1 + 96, + (const char *) key2 + 96, key_len); +} + +static inline int +rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len) +{ + return rte_hash_k64_cmp_eq(key1, key2, key_len) || + rte_hash_k64_cmp_eq((const char *) key1 + 64, + (const char *) key2 + 64, key_len); +} diff --git a/lib/hash/rte_cuckoo_hash.c b/lib/hash/rte_cuckoo_hash.c index da12825c6ed..ff566731983 100644 --- a/lib/hash/rte_cuckoo_hash.c +++ b/lib/hash/rte_cuckoo_hash.c @@ -409,7 +409,7 @@ rte_hash_create(const struct rte_hash_parameters *params) * If x86 architecture is used, select appropriate compare function, * which may use x86 intrinsics, otherwise use memcmp */ -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) +#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) || defined(RTE_ARCH_RISCV) /* Select function to compare keys */ switch (params->key_len) { case 16: diff --git a/lib/hash/rte_cuckoo_hash.h b/lib/hash/rte_cuckoo_hash.h index cf5cd29144f..21388c69069 100644 --- a/lib/hash/rte_cuckoo_hash.h +++ b/lib/hash/rte_cuckoo_hash.h @@ -21,6 +21,10 @@ #include "rte_cmp_arm64.h" #endif +#if defined(RTE_ARCH_RISCV) +#include "rte_cmp_riscv.h" +#endif + /* Macro to enable/disable run-time checking of function parameters */ #if defined(RTE_LIBRTE_HASH_DEBUG) #define RETURN_IF_TRUE(cond, retval) do { \ @@ -34,7 +38,7 @@ #include #include -#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) +#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) || defined(RTE_ARCH_RISCV) /* * All different options to select a key compare function, * based on the key size and custom function. diff --git a/lib/meson.build b/lib/meson.build index 8f5cfd28a58..d28197e8b7a 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -154,12 +154,8 @@ foreach l:libraries build = false reason = 'not in enabled libraries build config' elif disable_libs.contains(l) - if always_enable.contains(l) - warning('Cannot disable mandatory library "@0@"'.format(l)) - else - build = false - reason = 'explicitly disabled via build config' - endif + build = false + reason = 'explicitly disabled via build config' endif if build diff --git a/lib/pdcp/pdcp_process.c b/lib/pdcp/pdcp_process.c index 9b9b881124b..f55ae3bec06 100644 --- a/lib/pdcp/pdcp_process.c +++ b/lib/pdcp/pdcp_process.c @@ -351,6 +351,8 @@ cop_prepare(const struct entity_priv *en_priv, struct rte_mbuf *mb, struct rte_c op->auth.data.length = (pkt_len - RTE_PDCP_MAC_I_LEN) << auth_shift; op->auth.digest.data = rte_pktmbuf_mtod_offset(mb, uint8_t *, (pkt_len - RTE_PDCP_MAC_I_LEN)); + op->auth.digest.phys_addr = rte_pktmbuf_iova_offset(mb, + (pkt_len - RTE_PDCP_MAC_I_LEN)); } __rte_crypto_sym_op_attach_sym_session(op, en_priv->crypto_sess);