+++ /dev/null
-From 772438d01bf57bc8939f53c3101a323fc774428f Mon Sep 17 00:00:00 2001
-From: Madalin Bucur <madalin.bucur@nxp.com>
-Date: Wed, 10 May 2017 16:30:12 +0300
-Subject: [PATCH] fsl_qbman: SDK DPAA 1.x QBMan drivers
-
-Signed-off-by: Roy Pledge <roy.pledge@nxp.com>
-Signed-off-by: Madalin Bucur <madalin.bucur@nxp.com>
----
- drivers/staging/fsl_qbman/Kconfig | 228 +
- drivers/staging/fsl_qbman/Makefile | 28 +
- drivers/staging/fsl_qbman/bman_config.c | 720 +++
- drivers/staging/fsl_qbman/bman_debugfs.c | 119 +
- drivers/staging/fsl_qbman/bman_driver.c | 559 +++
- drivers/staging/fsl_qbman/bman_high.c | 1145 +++++
- drivers/staging/fsl_qbman/bman_low.h | 565 +++
- drivers/staging/fsl_qbman/bman_private.h | 166 +
- drivers/staging/fsl_qbman/bman_test.c | 56 +
- drivers/staging/fsl_qbman/bman_test.h | 44 +
- drivers/staging/fsl_qbman/bman_test_high.c | 183 +
- drivers/staging/fsl_qbman/bman_test_thresh.c | 196 +
- drivers/staging/fsl_qbman/dpa_alloc.c | 706 +++
- drivers/staging/fsl_qbman/dpa_sys.h | 259 ++
- drivers/staging/fsl_qbman/dpa_sys_arm.h | 95 +
- drivers/staging/fsl_qbman/dpa_sys_arm64.h | 102 +
- drivers/staging/fsl_qbman/dpa_sys_ppc32.h | 70 +
- drivers/staging/fsl_qbman/dpa_sys_ppc64.h | 79 +
- drivers/staging/fsl_qbman/fsl_usdpaa.c | 1984 ++++++++
- drivers/staging/fsl_qbman/fsl_usdpaa_irq.c | 289 ++
- drivers/staging/fsl_qbman/qbman_driver.c | 88 +
- drivers/staging/fsl_qbman/qman_config.c | 1224 +++++
- drivers/staging/fsl_qbman/qman_debugfs.c | 1594 +++++++
- drivers/staging/fsl_qbman/qman_driver.c | 961 ++++
- drivers/staging/fsl_qbman/qman_high.c | 5669 +++++++++++++++++++++++
- drivers/staging/fsl_qbman/qman_low.h | 1427 ++++++
- drivers/staging/fsl_qbman/qman_private.h | 398 ++
- drivers/staging/fsl_qbman/qman_test.c | 57 +
- drivers/staging/fsl_qbman/qman_test.h | 45 +
- drivers/staging/fsl_qbman/qman_test_high.c | 216 +
- drivers/staging/fsl_qbman/qman_test_hotpotato.c | 502 ++
- drivers/staging/fsl_qbman/qman_utility.c | 129 +
- include/linux/fsl_bman.h | 532 +++
- include/linux/fsl_qman.h | 3888 ++++++++++++++++
- include/linux/fsl_usdpaa.h | 372 ++
- 35 files changed, 24695 insertions(+)
- create mode 100644 drivers/staging/fsl_qbman/Kconfig
- create mode 100644 drivers/staging/fsl_qbman/Makefile
- create mode 100644 drivers/staging/fsl_qbman/bman_config.c
- create mode 100644 drivers/staging/fsl_qbman/bman_debugfs.c
- create mode 100644 drivers/staging/fsl_qbman/bman_driver.c
- create mode 100644 drivers/staging/fsl_qbman/bman_high.c
- create mode 100644 drivers/staging/fsl_qbman/bman_low.h
- create mode 100644 drivers/staging/fsl_qbman/bman_private.h
- create mode 100644 drivers/staging/fsl_qbman/bman_test.c
- create mode 100644 drivers/staging/fsl_qbman/bman_test.h
- create mode 100644 drivers/staging/fsl_qbman/bman_test_high.c
- create mode 100644 drivers/staging/fsl_qbman/bman_test_thresh.c
- create mode 100644 drivers/staging/fsl_qbman/dpa_alloc.c
- create mode 100644 drivers/staging/fsl_qbman/dpa_sys.h
- create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm.h
- create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm64.h
- create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc32.h
- create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc64.h
- create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa.c
- create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa_irq.c
- create mode 100644 drivers/staging/fsl_qbman/qbman_driver.c
- create mode 100644 drivers/staging/fsl_qbman/qman_config.c
- create mode 100644 drivers/staging/fsl_qbman/qman_debugfs.c
- create mode 100644 drivers/staging/fsl_qbman/qman_driver.c
- create mode 100644 drivers/staging/fsl_qbman/qman_high.c
- create mode 100644 drivers/staging/fsl_qbman/qman_low.h
- create mode 100644 drivers/staging/fsl_qbman/qman_private.h
- create mode 100644 drivers/staging/fsl_qbman/qman_test.c
- create mode 100644 drivers/staging/fsl_qbman/qman_test.h
- create mode 100644 drivers/staging/fsl_qbman/qman_test_high.c
- create mode 100644 drivers/staging/fsl_qbman/qman_test_hotpotato.c
- create mode 100644 drivers/staging/fsl_qbman/qman_utility.c
- create mode 100644 include/linux/fsl_bman.h
- create mode 100644 include/linux/fsl_qman.h
- create mode 100644 include/linux/fsl_usdpaa.h
-
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/Kconfig
-@@ -0,0 +1,228 @@
-+config FSL_SDK_DPA
-+ bool "Freescale Datapath Queue and Buffer management"
-+ depends on !FSL_DPAA
-+ select FSL_QMAN_FQ_LOOKUP if PPC64
-+ select FSL_QMAN_FQ_LOOKUP if ARM64
-+
-+
-+menu "Freescale Datapath QMan/BMan options"
-+ depends on FSL_SDK_DPA
-+
-+config FSL_DPA_CHECKING
-+ bool "additional driver checking"
-+ default n
-+ ---help---
-+ Compiles in additional checks to sanity-check the drivers and any
-+ use of it by other code. Not recommended for performance.
-+
-+config FSL_DPA_CAN_WAIT
-+ bool
-+ default y
-+
-+config FSL_DPA_CAN_WAIT_SYNC
-+ bool
-+ default y
-+
-+config FSL_DPA_PIRQ_FAST
-+ bool
-+ default y
-+
-+config FSL_DPA_PIRQ_SLOW
-+ bool
-+ default y
-+
-+config FSL_DPA_PORTAL_SHARE
-+ bool
-+ default y
-+
-+config FSL_SDK_BMAN
-+ bool "Freescale Buffer Manager (BMan) support"
-+ default y
-+
-+if FSL_SDK_BMAN
-+
-+config FSL_BMAN_CONFIG
-+ bool "BMan device management"
-+ default y
-+ ---help---
-+ If this linux image is running natively, you need this option. If this
-+ linux image is running as a guest OS under the hypervisor, only one
-+ guest OS ("the control plane") needs this option.
-+
-+config FSL_BMAN_TEST
-+ tristate "BMan self-tests"
-+ default n
-+ ---help---
-+ This option compiles self-test code for BMan.
-+
-+config FSL_BMAN_TEST_HIGH
-+ bool "BMan high-level self-test"
-+ depends on FSL_BMAN_TEST
-+ default y
-+ ---help---
-+ This requires the presence of cpu-affine portals, and performs
-+ high-level API testing with them (whichever portal(s) are affine to
-+ the cpu(s) the test executes on).
-+
-+config FSL_BMAN_TEST_THRESH
-+ bool "BMan threshold test"
-+ depends on FSL_BMAN_TEST
-+ default y
-+ ---help---
-+ Multi-threaded (SMP) test of BMan pool depletion. A pool is seeded
-+ before multiple threads (one per cpu) create pool objects to track
-+ depletion state changes. The pool is then drained to empty by a
-+ "drainer" thread, and the other threads that they observe exactly
-+ the depletion state changes that are expected.
-+
-+config FSL_BMAN_DEBUGFS
-+ tristate "BMan debugfs interface"
-+ depends on DEBUG_FS
-+ default y
-+ ---help---
-+ This option compiles debugfs code for BMan.
-+
-+endif # FSL_SDK_BMAN
-+
-+config FSL_SDK_QMAN
-+ bool "Freescale Queue Manager (QMan) support"
-+ default y
-+
-+if FSL_SDK_QMAN
-+
-+config FSL_QMAN_POLL_LIMIT
-+ int
-+ default 32
-+
-+config FSL_QMAN_CONFIG
-+ bool "QMan device management"
-+ default y
-+ ---help---
-+ If this linux image is running natively, you need this option. If this
-+ linux image is running as a guest OS under the hypervisor, only one
-+ guest OS ("the control plane") needs this option.
-+
-+config FSL_QMAN_TEST
-+ tristate "QMan self-tests"
-+ default n
-+ ---help---
-+ This option compiles self-test code for QMan.
-+
-+config FSL_QMAN_TEST_STASH_POTATO
-+ bool "QMan 'hot potato' data-stashing self-test"
-+ depends on FSL_QMAN_TEST
-+ default y
-+ ---help---
-+ This performs a "hot potato" style test enqueuing/dequeuing a frame
-+ across a series of FQs scheduled to different portals (and cpus), with
-+ DQRR, data and context stashing always on.
-+
-+config FSL_QMAN_TEST_HIGH
-+ bool "QMan high-level self-test"
-+ depends on FSL_QMAN_TEST
-+ default y
-+ ---help---
-+ This requires the presence of cpu-affine portals, and performs
-+ high-level API testing with them (whichever portal(s) are affine to
-+ the cpu(s) the test executes on).
-+
-+config FSL_QMAN_DEBUGFS
-+ tristate "QMan debugfs interface"
-+ depends on DEBUG_FS
-+ default y
-+ ---help---
-+ This option compiles debugfs code for QMan.
-+
-+# H/w settings that can be hard-coded for now.
-+config FSL_QMAN_FQD_SZ
-+ int "size of Frame Queue Descriptor region"
-+ default 10
-+ ---help---
-+ This is the size of the FQD region defined as: PAGE_SIZE * (2^value)
-+ ex: 10 => PAGE_SIZE * (2^10)
-+ Note: Default device-trees now require minimum Kconfig setting of 10.
-+
-+config FSL_QMAN_PFDR_SZ
-+ int "size of the PFDR pool"
-+ default 13
-+ ---help---
-+ This is the size of the PFDR pool defined as: PAGE_SIZE * (2^value)
-+ ex: 13 => PAGE_SIZE * (2^13)
-+
-+# Corenet initiator settings. Stash request queues are 4-deep to match cores'
-+# ability to snart. Stash priority is 3, other priorities are 2.
-+config FSL_QMAN_CI_SCHED_CFG_SRCCIV
-+ int
-+ depends on FSL_QMAN_CONFIG
-+ default 4
-+config FSL_QMAN_CI_SCHED_CFG_SRQ_W
-+ int
-+ depends on FSL_QMAN_CONFIG
-+ default 3
-+config FSL_QMAN_CI_SCHED_CFG_RW_W
-+ int
-+ depends on FSL_QMAN_CONFIG
-+ default 2
-+config FSL_QMAN_CI_SCHED_CFG_BMAN_W
-+ int
-+ depends on FSL_QMAN_CONFIG
-+ default 2
-+
-+# portal interrupt settings
-+config FSL_QMAN_PIRQ_DQRR_ITHRESH
-+ int
-+ default 12
-+config FSL_QMAN_PIRQ_MR_ITHRESH
-+ int
-+ default 4
-+config FSL_QMAN_PIRQ_IPERIOD
-+ int
-+ default 100
-+
-+# 64 bit kernel support
-+config FSL_QMAN_FQ_LOOKUP
-+ bool
-+ default n
-+
-+config QMAN_CEETM_UPDATE_PERIOD
-+ int "Token update period for shaping, in nanoseconds"
-+ default 1000
-+ ---help---
-+ Traffic shaping works by performing token calculations (using
-+ credits) on shaper instances periodically. This update period
-+ sets the granularity for how often those token rate credit
-+ updates are performed, and thus determines the accuracy and
-+ range of traffic rates that can be configured by users. The
-+ reference manual recommends a 1 microsecond period as providing
-+ a good balance between granularity and range.
-+
-+ Unless you know what you are doing, leave this value at its default.
-+
-+config FSL_QMAN_INIT_TIMEOUT
-+ int "timeout for qman init stage, in seconds"
-+ default 10
-+ ---help---
-+ The timeout setting to quit the initialization loop for non-control
-+ partition in case the control partition fails to boot-up.
-+
-+endif # FSL_SDK_QMAN
-+
-+config FSL_USDPAA
-+ bool "Freescale USDPAA process driver"
-+ depends on FSL_SDK_DPA
-+ default y
-+ ---help---
-+ This driver provides user-space access to kernel-managed
-+ resource interfaces for USDPAA applications, on the assumption
-+ that each process will open this device once. Specifically, this
-+ device exposes functionality that would be awkward if exposed
-+ via the portal devices - ie. this device exposes functionality
-+ that is inherently process-wide rather than portal-specific.
-+ This device is necessary for obtaining access to DMA memory and
-+ for allocation of Qman and Bman resources. In short, if you wish
-+ to use USDPAA applications, you need this.
-+
-+ If unsure, say Y.
-+
-+
-+endmenu
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/Makefile
-@@ -0,0 +1,28 @@
-+subdir-ccflags-y := -Werror
-+
-+# Common
-+obj-$(CONFIG_FSL_SDK_DPA) += dpa_alloc.o
-+obj-$(CONFIG_FSL_SDK_DPA) += qbman_driver.o
-+
-+# Bman
-+obj-$(CONFIG_FSL_SDK_BMAN) += bman_high.o
-+obj-$(CONFIG_FSL_BMAN_CONFIG) += bman_config.o bman_driver.o
-+obj-$(CONFIG_FSL_BMAN_TEST) += bman_tester.o
-+obj-$(CONFIG_FSL_BMAN_DEBUGFS) += bman_debugfs_interface.o
-+bman_tester-y = bman_test.o
-+bman_tester-$(CONFIG_FSL_BMAN_TEST_HIGH) += bman_test_high.o
-+bman_tester-$(CONFIG_FSL_BMAN_TEST_THRESH) += bman_test_thresh.o
-+bman_debugfs_interface-y = bman_debugfs.o
-+
-+# Qman
-+obj-$(CONFIG_FSL_SDK_QMAN) += qman_high.o qman_utility.o
-+obj-$(CONFIG_FSL_QMAN_CONFIG) += qman_config.o qman_driver.o
-+obj-$(CONFIG_FSL_QMAN_TEST) += qman_tester.o
-+qman_tester-y = qman_test.o
-+qman_tester-$(CONFIG_FSL_QMAN_TEST_STASH_POTATO) += qman_test_hotpotato.o
-+qman_tester-$(CONFIG_FSL_QMAN_TEST_HIGH) += qman_test_high.o
-+obj-$(CONFIG_FSL_QMAN_DEBUGFS) += qman_debugfs_interface.o
-+qman_debugfs_interface-y = qman_debugfs.o
-+
-+# USDPAA
-+obj-$(CONFIG_FSL_USDPAA) += fsl_usdpaa.o fsl_usdpaa_irq.o
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_config.c
-@@ -0,0 +1,720 @@
-+/* Copyright (c) 2009-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include <asm/cacheflush.h>
-+#include "bman_private.h"
-+#include <linux/of_reserved_mem.h>
-+
-+/* Last updated for v00.79 of the BG */
-+
-+struct bman;
-+
-+/* Register offsets */
-+#define REG_POOL_SWDET(n) (0x0000 + ((n) * 0x04))
-+#define REG_POOL_HWDET(n) (0x0100 + ((n) * 0x04))
-+#define REG_POOL_SWDXT(n) (0x0200 + ((n) * 0x04))
-+#define REG_POOL_HWDXT(n) (0x0300 + ((n) * 0x04))
-+#define REG_POOL_CONTENT(n) (0x0600 + ((n) * 0x04))
-+#define REG_FBPR_FPC 0x0800
-+#define REG_STATE_IDLE 0x960
-+#define REG_STATE_STOP 0x964
-+#define REG_ECSR 0x0a00
-+#define REG_ECIR 0x0a04
-+#define REG_EADR 0x0a08
-+#define REG_EDATA(n) (0x0a10 + ((n) * 0x04))
-+#define REG_SBEC(n) (0x0a80 + ((n) * 0x04))
-+#define REG_IP_REV_1 0x0bf8
-+#define REG_IP_REV_2 0x0bfc
-+#define REG_FBPR_BARE 0x0c00
-+#define REG_FBPR_BAR 0x0c04
-+#define REG_FBPR_AR 0x0c10
-+#define REG_SRCIDR 0x0d04
-+#define REG_LIODNR 0x0d08
-+#define REG_ERR_ISR 0x0e00 /* + "enum bm_isr_reg" */
-+
-+/* Used by all error interrupt registers except 'inhibit' */
-+#define BM_EIRQ_IVCI 0x00000010 /* Invalid Command Verb */
-+#define BM_EIRQ_FLWI 0x00000008 /* FBPR Low Watermark */
-+#define BM_EIRQ_MBEI 0x00000004 /* Multi-bit ECC Error */
-+#define BM_EIRQ_SBEI 0x00000002 /* Single-bit ECC Error */
-+#define BM_EIRQ_BSCN 0x00000001 /* pool State Change Notification */
-+
-+/* BMAN_ECIR valid error bit */
-+#define PORTAL_ECSR_ERR (BM_EIRQ_IVCI)
-+
-+union bman_ecir {
-+ u32 ecir_raw;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved1:4;
-+ u32 portal_num:4;
-+ u32 __reserved2:12;
-+ u32 numb:4;
-+ u32 __reserved3:2;
-+ u32 pid:6;
-+#else
-+ u32 pid:6;
-+ u32 __reserved3:2;
-+ u32 numb:4;
-+ u32 __reserved2:12;
-+ u32 portal_num:4;
-+ u32 __reserved1:4;
-+#endif
-+ } __packed info;
-+};
-+
-+union bman_eadr {
-+ u32 eadr_raw;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved1:5;
-+ u32 memid:3;
-+ u32 __reserved2:14;
-+ u32 eadr:10;
-+#else
-+ u32 eadr:10;
-+ u32 __reserved2:14;
-+ u32 memid:3;
-+ u32 __reserved1:5;
-+#endif
-+ } __packed info;
-+};
-+
-+struct bman_hwerr_txt {
-+ u32 mask;
-+ const char *txt;
-+};
-+
-+#define BMAN_HWE_TXT(a, b) { .mask = BM_EIRQ_##a, .txt = b }
-+
-+static const struct bman_hwerr_txt bman_hwerr_txts[] = {
-+ BMAN_HWE_TXT(IVCI, "Invalid Command Verb"),
-+ BMAN_HWE_TXT(FLWI, "FBPR Low Watermark"),
-+ BMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"),
-+ BMAN_HWE_TXT(SBEI, "Single-bit ECC Error"),
-+ BMAN_HWE_TXT(BSCN, "Pool State Change Notification"),
-+};
-+#define BMAN_HWE_COUNT (sizeof(bman_hwerr_txts)/sizeof(struct bman_hwerr_txt))
-+
-+struct bman_error_info_mdata {
-+ u16 addr_mask;
-+ u16 bits;
-+ const char *txt;
-+};
-+
-+#define BMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c}
-+static const struct bman_error_info_mdata error_mdata[] = {
-+ BMAN_ERR_MDATA(0x03FF, 192, "Stockpile memory"),
-+ BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 1"),
-+ BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 2"),
-+};
-+#define BMAN_ERR_MDATA_COUNT \
-+ (sizeof(error_mdata)/sizeof(struct bman_error_info_mdata))
-+
-+/* Add this in Kconfig */
-+#define BMAN_ERRS_TO_UNENABLE (BM_EIRQ_FLWI)
-+
-+/**
-+ * bm_err_isr_<reg>_<verb> - Manipulate global interrupt registers
-+ * @v: for accessors that write values, this is the 32-bit value
-+ *
-+ * Manipulates BMAN_ERR_ISR, BMAN_ERR_IER, BMAN_ERR_ISDR, BMAN_ERR_IIR. All
-+ * manipulations except bm_err_isr_[un]inhibit() use 32-bit masks composed of
-+ * the BM_EIRQ_*** definitions. Note that "bm_err_isr_enable_write" means
-+ * "write the enable register" rather than "enable the write register"!
-+ */
-+#define bm_err_isr_status_read(bm) \
-+ __bm_err_isr_read(bm, bm_isr_status)
-+#define bm_err_isr_status_clear(bm, m) \
-+ __bm_err_isr_write(bm, bm_isr_status, m)
-+#define bm_err_isr_enable_read(bm) \
-+ __bm_err_isr_read(bm, bm_isr_enable)
-+#define bm_err_isr_enable_write(bm, v) \
-+ __bm_err_isr_write(bm, bm_isr_enable, v)
-+#define bm_err_isr_disable_read(bm) \
-+ __bm_err_isr_read(bm, bm_isr_disable)
-+#define bm_err_isr_disable_write(bm, v) \
-+ __bm_err_isr_write(bm, bm_isr_disable, v)
-+#define bm_err_isr_inhibit(bm) \
-+ __bm_err_isr_write(bm, bm_isr_inhibit, 1)
-+#define bm_err_isr_uninhibit(bm) \
-+ __bm_err_isr_write(bm, bm_isr_inhibit, 0)
-+
-+/*
-+ * TODO: unimplemented registers
-+ *
-+ * BMAN_POOLk_SDCNT, BMAN_POOLk_HDCNT, BMAN_FULT,
-+ * BMAN_VLDPL, BMAN_EECC, BMAN_SBET, BMAN_EINJ
-+ */
-+
-+/* Encapsulate "struct bman *" as a cast of the register space address. */
-+
-+static struct bman *bm_create(void *regs)
-+{
-+ return (struct bman *)regs;
-+}
-+
-+static inline u32 __bm_in(struct bman *bm, u32 offset)
-+{
-+ return in_be32((void *)bm + offset);
-+}
-+static inline void __bm_out(struct bman *bm, u32 offset, u32 val)
-+{
-+ out_be32((void *)bm + offset, val);
-+}
-+#define bm_in(reg) __bm_in(bm, REG_##reg)
-+#define bm_out(reg, val) __bm_out(bm, REG_##reg, val)
-+
-+static u32 __bm_err_isr_read(struct bman *bm, enum bm_isr_reg n)
-+{
-+ return __bm_in(bm, REG_ERR_ISR + (n << 2));
-+}
-+
-+static void __bm_err_isr_write(struct bman *bm, enum bm_isr_reg n, u32 val)
-+{
-+ __bm_out(bm, REG_ERR_ISR + (n << 2), val);
-+}
-+
-+static void bm_get_version(struct bman *bm, u16 *id, u8 *major, u8 *minor)
-+{
-+ u32 v = bm_in(IP_REV_1);
-+ *id = (v >> 16);
-+ *major = (v >> 8) & 0xff;
-+ *minor = v & 0xff;
-+}
-+
-+static u32 __generate_thresh(u32 val, int roundup)
-+{
-+ u32 e = 0; /* co-efficient, exponent */
-+ int oddbit = 0;
-+ while (val > 0xff) {
-+ oddbit = val & 1;
-+ val >>= 1;
-+ e++;
-+ if (roundup && oddbit)
-+ val++;
-+ }
-+ DPA_ASSERT(e < 0x10);
-+ return val | (e << 8);
-+}
-+
-+static void bm_set_pool(struct bman *bm, u8 pool, u32 swdet, u32 swdxt,
-+ u32 hwdet, u32 hwdxt)
-+{
-+ DPA_ASSERT(pool < bman_pool_max);
-+ bm_out(POOL_SWDET(pool), __generate_thresh(swdet, 0));
-+ bm_out(POOL_SWDXT(pool), __generate_thresh(swdxt, 1));
-+ bm_out(POOL_HWDET(pool), __generate_thresh(hwdet, 0));
-+ bm_out(POOL_HWDXT(pool), __generate_thresh(hwdxt, 1));
-+}
-+
-+static void bm_set_memory(struct bman *bm, u64 ba, int prio, u32 size)
-+{
-+ u32 exp = ilog2(size);
-+ /* choke if size isn't within range */
-+ DPA_ASSERT((size >= 4096) && (size <= 1073741824) &&
-+ is_power_of_2(size));
-+ /* choke if '[e]ba' has lower-alignment than 'size' */
-+ DPA_ASSERT(!(ba & (size - 1)));
-+ bm_out(FBPR_BARE, upper_32_bits(ba));
-+ bm_out(FBPR_BAR, lower_32_bits(ba));
-+ bm_out(FBPR_AR, (prio ? 0x40000000 : 0) | (exp - 1));
-+}
-+
-+/*****************/
-+/* Config driver */
-+/*****************/
-+
-+/* TODO: Kconfig these? */
-+#define DEFAULT_FBPR_SZ (PAGE_SIZE << 12)
-+
-+/* We support only one of these. */
-+static struct bman *bm;
-+static struct device_node *bm_node;
-+
-+/* And this state belongs to 'bm'. It is set during fsl_bman_init(), but used
-+ * during bman_init_ccsr(). */
-+static dma_addr_t fbpr_a;
-+static size_t fbpr_sz = DEFAULT_FBPR_SZ;
-+
-+static int bman_fbpr(struct reserved_mem *rmem)
-+{
-+ fbpr_a = rmem->base;
-+ fbpr_sz = rmem->size;
-+
-+ WARN_ON(!(fbpr_a && fbpr_sz));
-+
-+ return 0;
-+}
-+RESERVEDMEM_OF_DECLARE(bman_fbpr, "fsl,bman-fbpr", bman_fbpr);
-+
-+static int __init fsl_bman_init(struct device_node *node)
-+{
-+ struct resource res;
-+ u32 __iomem *regs;
-+ const char *s;
-+ int ret, standby = 0;
-+ u16 id;
-+ u8 major, minor;
-+
-+ ret = of_address_to_resource(node, 0, &res);
-+ if (ret) {
-+ pr_err("Can't get %s property 'reg'\n",
-+ node->full_name);
-+ return ret;
-+ }
-+ s = of_get_property(node, "fsl,hv-claimable", &ret);
-+ if (s && !strcmp(s, "standby"))
-+ standby = 1;
-+ /* Global configuration */
-+ regs = ioremap(res.start, res.end - res.start + 1);
-+ bm = bm_create(regs);
-+ BUG_ON(!bm);
-+ bm_node = node;
-+ bm_get_version(bm, &id, &major, &minor);
-+ pr_info("Bman ver:%04x,%02x,%02x\n", id, major, minor);
-+ if ((major == 1) && (minor == 0)) {
-+ bman_ip_rev = BMAN_REV10;
-+ bman_pool_max = 64;
-+ } else if ((major == 2) && (minor == 0)) {
-+ bman_ip_rev = BMAN_REV20;
-+ bman_pool_max = 8;
-+ } else if ((major == 2) && (minor == 1)) {
-+ bman_ip_rev = BMAN_REV21;
-+ bman_pool_max = 64;
-+ } else {
-+ pr_warn("unknown Bman version, default to rev1.0\n");
-+ }
-+
-+ if (standby) {
-+ pr_info(" -> in standby mode\n");
-+ return 0;
-+ }
-+ return 0;
-+}
-+
-+int bman_have_ccsr(void)
-+{
-+ return bm ? 1 : 0;
-+}
-+
-+int bm_pool_set(u32 bpid, const u32 *thresholds)
-+{
-+ if (!bm)
-+ return -ENODEV;
-+ bm_set_pool(bm, bpid, thresholds[0],
-+ thresholds[1], thresholds[2],
-+ thresholds[3]);
-+ return 0;
-+}
-+EXPORT_SYMBOL(bm_pool_set);
-+
-+__init int bman_init_early(void)
-+{
-+ struct device_node *dn;
-+ int ret;
-+
-+ for_each_compatible_node(dn, NULL, "fsl,bman") {
-+ if (bm)
-+ pr_err("%s: only one 'fsl,bman' allowed\n",
-+ dn->full_name);
-+ else {
-+ if (!of_device_is_available(dn))
-+ continue;
-+
-+ ret = fsl_bman_init(dn);
-+ BUG_ON(ret);
-+ }
-+ }
-+ return 0;
-+}
-+postcore_initcall_sync(bman_init_early);
-+
-+
-+static void log_edata_bits(u32 bit_count)
-+{
-+ u32 i, j, mask = 0xffffffff;
-+
-+ pr_warn("Bman ErrInt, EDATA:\n");
-+ i = bit_count/32;
-+ if (bit_count%32) {
-+ i++;
-+ mask = ~(mask << bit_count%32);
-+ }
-+ j = 16-i;
-+ pr_warn(" 0x%08x\n", bm_in(EDATA(j)) & mask);
-+ j++;
-+ for (; j < 16; j++)
-+ pr_warn(" 0x%08x\n", bm_in(EDATA(j)));
-+}
-+
-+static void log_additional_error_info(u32 isr_val, u32 ecsr_val)
-+{
-+ union bman_ecir ecir_val;
-+ union bman_eadr eadr_val;
-+
-+ ecir_val.ecir_raw = bm_in(ECIR);
-+ /* Is portal info valid */
-+ if (ecsr_val & PORTAL_ECSR_ERR) {
-+ pr_warn("Bman ErrInt: SWP id %d, numb %d, pid %d\n",
-+ ecir_val.info.portal_num, ecir_val.info.numb,
-+ ecir_val.info.pid);
-+ }
-+ if (ecsr_val & (BM_EIRQ_SBEI|BM_EIRQ_MBEI)) {
-+ eadr_val.eadr_raw = bm_in(EADR);
-+ pr_warn("Bman ErrInt: EADR Memory: %s, 0x%x\n",
-+ error_mdata[eadr_val.info.memid].txt,
-+ error_mdata[eadr_val.info.memid].addr_mask
-+ & eadr_val.info.eadr);
-+ log_edata_bits(error_mdata[eadr_val.info.memid].bits);
-+ }
-+}
-+
-+/* Bman interrupt handler */
-+static irqreturn_t bman_isr(int irq, void *ptr)
-+{
-+ u32 isr_val, ier_val, ecsr_val, isr_mask, i;
-+
-+ ier_val = bm_err_isr_enable_read(bm);
-+ isr_val = bm_err_isr_status_read(bm);
-+ ecsr_val = bm_in(ECSR);
-+ isr_mask = isr_val & ier_val;
-+
-+ if (!isr_mask)
-+ return IRQ_NONE;
-+ for (i = 0; i < BMAN_HWE_COUNT; i++) {
-+ if (bman_hwerr_txts[i].mask & isr_mask) {
-+ pr_warn("Bman ErrInt: %s\n", bman_hwerr_txts[i].txt);
-+ if (bman_hwerr_txts[i].mask & ecsr_val) {
-+ log_additional_error_info(isr_mask, ecsr_val);
-+ /* Re-arm error capture registers */
-+ bm_out(ECSR, ecsr_val);
-+ }
-+ if (bman_hwerr_txts[i].mask & BMAN_ERRS_TO_UNENABLE) {
-+ pr_devel("Bman un-enabling error 0x%x\n",
-+ bman_hwerr_txts[i].mask);
-+ ier_val &= ~bman_hwerr_txts[i].mask;
-+ bm_err_isr_enable_write(bm, ier_val);
-+ }
-+ }
-+ }
-+ bm_err_isr_status_clear(bm, isr_val);
-+ return IRQ_HANDLED;
-+}
-+
-+static int __bind_irq(void)
-+{
-+ int ret, err_irq;
-+
-+ err_irq = of_irq_to_resource(bm_node, 0, NULL);
-+ if (err_irq == 0) {
-+ pr_info("Can't get %s property '%s'\n", bm_node->full_name,
-+ "interrupts");
-+ return -ENODEV;
-+ }
-+ ret = request_irq(err_irq, bman_isr, IRQF_SHARED, "bman-err", bm_node);
-+ if (ret) {
-+ pr_err("request_irq() failed %d for '%s'\n", ret,
-+ bm_node->full_name);
-+ return -ENODEV;
-+ }
-+ /* Disable Buffer Pool State Change */
-+ bm_err_isr_disable_write(bm, BM_EIRQ_BSCN);
-+ /* Write-to-clear any stale bits, (eg. starvation being asserted prior
-+ * to resource allocation during driver init). */
-+ bm_err_isr_status_clear(bm, 0xffffffff);
-+ /* Enable Error Interrupts */
-+ bm_err_isr_enable_write(bm, 0xffffffff);
-+ return 0;
-+}
-+
-+int bman_init_ccsr(struct device_node *node)
-+{
-+ int ret;
-+ if (!bman_have_ccsr())
-+ return 0;
-+ if (node != bm_node)
-+ return -EINVAL;
-+ /* FBPR memory */
-+ bm_set_memory(bm, fbpr_a, 0, fbpr_sz);
-+ pr_info("bman-fbpr addr %pad size 0x%zx\n", &fbpr_a, fbpr_sz);
-+
-+ ret = __bind_irq();
-+ if (ret)
-+ return ret;
-+ return 0;
-+}
-+
-+u32 bm_pool_free_buffers(u32 bpid)
-+{
-+ return bm_in(POOL_CONTENT(bpid));
-+}
-+
-+#ifdef CONFIG_SYSFS
-+
-+#define DRV_NAME "fsl-bman"
-+#define SBEC_MAX_ID 1
-+#define SBEC_MIN_ID 0
-+
-+static ssize_t show_fbpr_fpc(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(FBPR_FPC));
-+};
-+
-+static ssize_t show_pool_count(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ u32 data;
-+ int i;
-+
-+ if (!sscanf(dev_attr->attr.name, "%d", &i) || (i >= bman_pool_max))
-+ return -EINVAL;
-+ data = bm_in(POOL_CONTENT(i));
-+ return snprintf(buf, PAGE_SIZE, "%d\n", data);
-+};
-+
-+static ssize_t show_err_isr(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "0x%08x\n", bm_in(ERR_ISR));
-+};
-+
-+static ssize_t show_sbec(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ int i;
-+
-+ if (!sscanf(dev_attr->attr.name, "sbec_%d", &i))
-+ return -EINVAL;
-+ if (i < SBEC_MIN_ID || i > SBEC_MAX_ID)
-+ return -EINVAL;
-+ return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(SBEC(i)));
-+};
-+
-+static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL);
-+static DEVICE_ATTR(fbpr_fpc, S_IRUSR, show_fbpr_fpc, NULL);
-+
-+/* Didn't use DEVICE_ATTR as 64 of this would be required.
-+ * Initialize them when needed. */
-+static char *name_attrs_pool_count; /* "xx" + null-terminator */
-+static struct device_attribute *dev_attr_buffer_pool_count;
-+
-+static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL);
-+
-+static struct attribute *bman_dev_attributes[] = {
-+ &dev_attr_fbpr_fpc.attr,
-+ &dev_attr_err_isr.attr,
-+ NULL
-+};
-+
-+static struct attribute *bman_dev_ecr_attributes[] = {
-+ &dev_attr_sbec_0.attr,
-+ &dev_attr_sbec_1.attr,
-+ NULL
-+};
-+
-+static struct attribute **bman_dev_pool_count_attributes;
-+
-+
-+/* root level */
-+static const struct attribute_group bman_dev_attr_grp = {
-+ .name = NULL,
-+ .attrs = bman_dev_attributes
-+};
-+static const struct attribute_group bman_dev_ecr_grp = {
-+ .name = "error_capture",
-+ .attrs = bman_dev_ecr_attributes
-+};
-+static struct attribute_group bman_dev_pool_countent_grp = {
-+ .name = "pool_count",
-+};
-+
-+static int of_fsl_bman_remove(struct platform_device *ofdev)
-+{
-+ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp);
-+ return 0;
-+};
-+
-+static int of_fsl_bman_probe(struct platform_device *ofdev)
-+{
-+ int ret, i;
-+
-+ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_attr_grp);
-+ if (ret)
-+ goto done;
-+ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_ecr_grp);
-+ if (ret)
-+ goto del_group_0;
-+
-+ name_attrs_pool_count = kmalloc(sizeof(char) * bman_pool_max * 3,
-+ GFP_KERNEL);
-+ if (!name_attrs_pool_count) {
-+ pr_err("Can't alloc name_attrs_pool_count\n");
-+ goto del_group_1;
-+ }
-+
-+ dev_attr_buffer_pool_count = kmalloc(sizeof(struct device_attribute) *
-+ bman_pool_max, GFP_KERNEL);
-+ if (!dev_attr_buffer_pool_count) {
-+ pr_err("Can't alloc dev_attr-buffer_pool_count\n");
-+ goto del_group_2;
-+ }
-+
-+ bman_dev_pool_count_attributes = kmalloc(sizeof(struct attribute *) *
-+ (bman_pool_max + 1), GFP_KERNEL);
-+ if (!bman_dev_pool_count_attributes) {
-+ pr_err("can't alloc bman_dev_pool_count_attributes\n");
-+ goto del_group_3;
-+ }
-+
-+ for (i = 0; i < bman_pool_max; i++) {
-+ ret = scnprintf((name_attrs_pool_count + i * 3), 3, "%d", i);
-+ if (!ret)
-+ goto del_group_4;
-+ dev_attr_buffer_pool_count[i].attr.name =
-+ (name_attrs_pool_count + i * 3);
-+ dev_attr_buffer_pool_count[i].attr.mode = S_IRUSR;
-+ dev_attr_buffer_pool_count[i].show = show_pool_count;
-+ bman_dev_pool_count_attributes[i] =
-+ &dev_attr_buffer_pool_count[i].attr;
-+ sysfs_attr_init(bman_dev_pool_count_attributes[i]);
-+ }
-+ bman_dev_pool_count_attributes[bman_pool_max] = NULL;
-+
-+ bman_dev_pool_countent_grp.attrs = bman_dev_pool_count_attributes;
-+
-+ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_pool_countent_grp);
-+ if (ret)
-+ goto del_group_4;
-+
-+ goto done;
-+
-+del_group_4:
-+ kfree(bman_dev_pool_count_attributes);
-+del_group_3:
-+ kfree(dev_attr_buffer_pool_count);
-+del_group_2:
-+ kfree(name_attrs_pool_count);
-+del_group_1:
-+ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_ecr_grp);
-+del_group_0:
-+ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp);
-+done:
-+ if (ret)
-+ dev_err(&ofdev->dev,
-+ "Cannot create dev attributes ret=%d\n", ret);
-+ return ret;
-+};
-+
-+static struct of_device_id of_fsl_bman_ids[] = {
-+ {
-+ .compatible = "fsl,bman",
-+ },
-+ {}
-+};
-+MODULE_DEVICE_TABLE(of, of_fsl_bman_ids);
-+
-+#ifdef CONFIG_SUSPEND
-+static u32 saved_isdr;
-+
-+static int bman_pm_suspend_noirq(struct device *dev)
-+{
-+ uint32_t idle_state;
-+
-+ suspend_unused_bportal();
-+ /* save isdr, disable all, clear isr */
-+ saved_isdr = bm_err_isr_disable_read(bm);
-+ bm_err_isr_disable_write(bm, 0xffffffff);
-+ bm_err_isr_status_clear(bm, 0xffffffff);
-+
-+ if (bman_ip_rev < BMAN_REV21) {
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Bman version doesn't have STATE_IDLE\n");
-+#endif
-+ return 0;
-+ }
-+ idle_state = bm_in(STATE_IDLE);
-+ if (!(idle_state & 0x1)) {
-+ pr_err("Bman not idle 0x%x aborting\n", idle_state);
-+ bm_err_isr_disable_write(bm, saved_isdr);
-+ resume_unused_bportal();
-+ return -EBUSY;
-+ }
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Bman suspend code, IDLE_STAT = 0x%x\n", idle_state);
-+#endif
-+ return 0;
-+}
-+
-+static int bman_pm_resume_noirq(struct device *dev)
-+{
-+ /* restore isdr */
-+ bm_err_isr_disable_write(bm, saved_isdr);
-+ resume_unused_bportal();
-+ return 0;
-+}
-+#else
-+#define bman_pm_suspend_noirq NULL
-+#define bman_pm_resume_noirq NULL
-+#endif
-+
-+static const struct dev_pm_ops bman_pm_ops = {
-+ .suspend_noirq = bman_pm_suspend_noirq,
-+ .resume_noirq = bman_pm_resume_noirq,
-+};
-+
-+static struct platform_driver of_fsl_bman_driver = {
-+ .driver = {
-+ .owner = THIS_MODULE,
-+ .name = DRV_NAME,
-+ .of_match_table = of_fsl_bman_ids,
-+ .pm = &bman_pm_ops,
-+ },
-+ .probe = of_fsl_bman_probe,
-+ .remove = of_fsl_bman_remove,
-+};
-+
-+static int bman_ctrl_init(void)
-+{
-+ return platform_driver_register(&of_fsl_bman_driver);
-+}
-+
-+static void bman_ctrl_exit(void)
-+{
-+ platform_driver_unregister(&of_fsl_bman_driver);
-+}
-+
-+module_init(bman_ctrl_init);
-+module_exit(bman_ctrl_exit);
-+
-+#endif /* CONFIG_SYSFS */
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_debugfs.c
-@@ -0,0 +1,119 @@
-+/* Copyright 2010-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+#include <linux/module.h>
-+#include <linux/fsl_bman.h>
-+#include <linux/debugfs.h>
-+#include <linux/seq_file.h>
-+#include <linux/uaccess.h>
-+
-+static struct dentry *dfs_root; /* debugfs root directory */
-+
-+/*******************************************************************************
-+ * Query Buffer Pool State
-+ ******************************************************************************/
-+static int query_bp_state_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct bm_pool_state state;
-+ int i, j;
-+ u32 mask;
-+
-+ memset(&state, 0, sizeof(struct bm_pool_state));
-+ ret = bman_query_pools(&state);
-+ if (ret) {
-+ seq_printf(file, "Error %d\n", ret);
-+ return 0;
-+ }
-+ seq_puts(file, "bp_id free_buffers_avail bp_depleted\n");
-+ for (i = 0; i < 2; i++) {
-+ mask = 0x80000000;
-+ for (j = 0; j < 32; j++) {
-+ seq_printf(file,
-+ " %-2u %-3s %-3s\n",
-+ (i*32)+j,
-+ (state.as.state.__state[i] & mask) ? "no" : "yes",
-+ (state.ds.state.__state[i] & mask) ? "yes" : "no");
-+ mask >>= 1;
-+ }
-+ }
-+ return 0;
-+}
-+
-+static int query_bp_state_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, query_bp_state_show, NULL);
-+}
-+
-+static const struct file_operations query_bp_state_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_bp_state_open,
-+ .read = seq_read,
-+ .release = single_release,
-+};
-+
-+static int __init bman_debugfs_module_init(void)
-+{
-+ int ret = 0;
-+ struct dentry *d;
-+
-+ dfs_root = debugfs_create_dir("bman", NULL);
-+
-+ if (dfs_root == NULL) {
-+ ret = -ENOMEM;
-+ pr_err("Cannot create bman debugfs dir\n");
-+ goto _return;
-+ }
-+ d = debugfs_create_file("query_bp_state",
-+ S_IRUGO,
-+ dfs_root,
-+ NULL,
-+ &query_bp_state_fops);
-+ if (d == NULL) {
-+ ret = -ENOMEM;
-+ pr_err("Cannot create query_bp_state\n");
-+ goto _return;
-+ }
-+ return 0;
-+
-+_return:
-+ debugfs_remove_recursive(dfs_root);
-+ return ret;
-+}
-+
-+static void __exit bman_debugfs_module_exit(void)
-+{
-+ debugfs_remove_recursive(dfs_root);
-+}
-+
-+
-+module_init(bman_debugfs_module_init);
-+module_exit(bman_debugfs_module_exit);
-+MODULE_LICENSE("Dual BSD/GPL");
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_driver.c
-@@ -0,0 +1,559 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+#include "bman_low.h"
-+#ifdef CONFIG_HOTPLUG_CPU
-+#include <linux/cpu.h>
-+#endif
-+/*
-+ * Global variables of the max portal/pool number this bman version supported
-+ */
-+u16 bman_ip_rev;
-+EXPORT_SYMBOL(bman_ip_rev);
-+u16 bman_pool_max;
-+EXPORT_SYMBOL(bman_pool_max);
-+static u16 bman_portal_max;
-+
-+/* After initialising cpus that own shared portal configs, we cache the
-+ * resulting portals (ie. not just the configs) in this array. Then we
-+ * initialise slave cpus that don't have their own portals, redirecting them to
-+ * portals from this cache in a round-robin assignment. */
-+static struct bman_portal *shared_portals[NR_CPUS];
-+static int num_shared_portals;
-+static int shared_portals_idx;
-+static LIST_HEAD(unused_pcfgs);
-+static DEFINE_SPINLOCK(unused_pcfgs_lock);
-+static void *affine_bportals[NR_CPUS];
-+
-+static int __init fsl_bpool_init(struct device_node *node)
-+{
-+ int ret;
-+ u32 *thresh, *bpid = (u32 *)of_get_property(node, "fsl,bpid", &ret);
-+ if (!bpid || (ret != 4)) {
-+ pr_err("Can't get %s property 'fsl,bpid'\n", node->full_name);
-+ return -ENODEV;
-+ }
-+ thresh = (u32 *)of_get_property(node, "fsl,bpool-thresholds", &ret);
-+ if (thresh) {
-+ if (ret != 16) {
-+ pr_err("Invalid %s property '%s'\n",
-+ node->full_name, "fsl,bpool-thresholds");
-+ return -ENODEV;
-+ }
-+ }
-+ if (thresh) {
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+ ret = bm_pool_set(be32_to_cpu(*bpid), thresh);
-+ if (ret)
-+ pr_err("No CCSR node for %s property '%s'\n",
-+ node->full_name, "fsl,bpool-thresholds");
-+ return ret;
-+#else
-+ pr_err("Ignoring %s property '%s', no CCSR support\n",
-+ node->full_name, "fsl,bpool-thresholds");
-+#endif
-+ }
-+ return 0;
-+}
-+
-+static int __init fsl_bpid_range_init(struct device_node *node)
-+{
-+ int ret;
-+ u32 *range = (u32 *)of_get_property(node, "fsl,bpid-range", &ret);
-+ if (!range) {
-+ pr_err("No 'fsl,bpid-range' property in node %s\n",
-+ node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err("'fsl,bpid-range' is not a 2-cell range in node %s\n",
-+ node->full_name);
-+ return -EINVAL;
-+ }
-+ bman_seed_bpid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ pr_info("Bman: BPID allocator includes range %d:%d\n",
-+ be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ return 0;
-+}
-+
-+static struct bm_portal_config * __init parse_pcfg(struct device_node *node)
-+{
-+ struct bm_portal_config *pcfg;
-+ const u32 *index;
-+ int irq, ret;
-+ resource_size_t len;
-+
-+ pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL);
-+ if (!pcfg) {
-+ pr_err("can't allocate portal config");
-+ return NULL;
-+ }
-+
-+ if (of_device_is_compatible(node, "fsl,bman-portal-1.0") ||
-+ of_device_is_compatible(node, "fsl,bman-portal-1.0.0")) {
-+ bman_ip_rev = BMAN_REV10;
-+ bman_pool_max = 64;
-+ bman_portal_max = 10;
-+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.0") ||
-+ of_device_is_compatible(node, "fsl,bman-portal-2.0.8")) {
-+ bman_ip_rev = BMAN_REV20;
-+ bman_pool_max = 8;
-+ bman_portal_max = 3;
-+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.0")) {
-+ bman_ip_rev = BMAN_REV21;
-+ bman_pool_max = 64;
-+ bman_portal_max = 50;
-+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.1")) {
-+ bman_ip_rev = BMAN_REV21;
-+ bman_pool_max = 64;
-+ bman_portal_max = 25;
-+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.2")) {
-+ bman_ip_rev = BMAN_REV21;
-+ bman_pool_max = 64;
-+ bman_portal_max = 18;
-+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.3")) {
-+ bman_ip_rev = BMAN_REV21;
-+ bman_pool_max = 64;
-+ bman_portal_max = 10;
-+ } else {
-+ pr_warn("unknown BMan version in portal node,"
-+ "default to rev1.0\n");
-+ bman_ip_rev = BMAN_REV10;
-+ bman_pool_max = 64;
-+ bman_portal_max = 10;
-+ }
-+
-+ ret = of_address_to_resource(node, DPA_PORTAL_CE,
-+ &pcfg->addr_phys[DPA_PORTAL_CE]);
-+ if (ret) {
-+ pr_err("Can't get %s property 'reg::CE'\n", node->full_name);
-+ goto err;
-+ }
-+ ret = of_address_to_resource(node, DPA_PORTAL_CI,
-+ &pcfg->addr_phys[DPA_PORTAL_CI]);
-+ if (ret) {
-+ pr_err("Can't get %s property 'reg::CI'\n", node->full_name);
-+ goto err;
-+ }
-+
-+ index = of_get_property(node, "cell-index", &ret);
-+ if (!index || (ret != 4)) {
-+ pr_err("Can't get %s property '%s'\n", node->full_name,
-+ "cell-index");
-+ goto err;
-+ }
-+ if (be32_to_cpu(*index) >= bman_portal_max) {
-+ pr_err("BMan portal cell index %d out of range, max %d\n",
-+ be32_to_cpu(*index), bman_portal_max);
-+ goto err;
-+ }
-+
-+ pcfg->public_cfg.cpu = -1;
-+
-+ irq = irq_of_parse_and_map(node, 0);
-+ if (irq == 0) {
-+ pr_err("Can't get %s property 'interrupts'\n", node->full_name);
-+ goto err;
-+ }
-+ pcfg->public_cfg.irq = irq;
-+ pcfg->public_cfg.index = be32_to_cpu(*index);
-+ bman_depletion_fill(&pcfg->public_cfg.mask);
-+
-+ len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]);
-+ if (len != (unsigned long)len)
-+ goto err;
-+
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns(
-+ pcfg->addr_phys[DPA_PORTAL_CE].start,
-+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]));
-+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap(
-+ pcfg->addr_phys[DPA_PORTAL_CI].start,
-+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]));
-+
-+#else
-+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot(
-+ pcfg->addr_phys[DPA_PORTAL_CE].start,
-+ (unsigned long)len,
-+ 0);
-+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot(
-+ pcfg->addr_phys[DPA_PORTAL_CI].start,
-+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]),
-+ _PAGE_GUARDED | _PAGE_NO_CACHE);
-+#endif
-+ /* disable bp depletion */
-+ __raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(0));
-+ __raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(1));
-+ return pcfg;
-+err:
-+ kfree(pcfg);
-+ return NULL;
-+}
-+
-+static struct bm_portal_config *get_pcfg(struct list_head *list)
-+{
-+ struct bm_portal_config *pcfg;
-+ if (list_empty(list))
-+ return NULL;
-+ pcfg = list_entry(list->prev, struct bm_portal_config, list);
-+ list_del(&pcfg->list);
-+ return pcfg;
-+}
-+
-+static struct bm_portal_config *get_pcfg_idx(struct list_head *list,
-+ uint32_t idx)
-+{
-+ struct bm_portal_config *pcfg;
-+ if (list_empty(list))
-+ return NULL;
-+ list_for_each_entry(pcfg, list, list) {
-+ if (pcfg->public_cfg.index == idx) {
-+ list_del(&pcfg->list);
-+ return pcfg;
-+ }
-+ }
-+ return NULL;
-+}
-+
-+struct bm_portal_config *bm_get_unused_portal(void)
-+{
-+ return bm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX);
-+}
-+
-+struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx)
-+{
-+ struct bm_portal_config *ret;
-+ spin_lock(&unused_pcfgs_lock);
-+ if (idx == QBMAN_ANY_PORTAL_IDX)
-+ ret = get_pcfg(&unused_pcfgs);
-+ else
-+ ret = get_pcfg_idx(&unused_pcfgs, idx);
-+ spin_unlock(&unused_pcfgs_lock);
-+ return ret;
-+}
-+
-+void bm_put_unused_portal(struct bm_portal_config *pcfg)
-+{
-+ spin_lock(&unused_pcfgs_lock);
-+ list_add(&pcfg->list, &unused_pcfgs);
-+ spin_unlock(&unused_pcfgs_lock);
-+}
-+
-+static struct bman_portal *init_pcfg(struct bm_portal_config *pcfg)
-+{
-+ struct bman_portal *p;
-+ p = bman_create_affine_portal(pcfg);
-+ if (p) {
-+#ifdef CONFIG_FSL_DPA_PIRQ_SLOW
-+ bman_p_irqsource_add(p, BM_PIRQ_RCRI | BM_PIRQ_BSCN);
-+#endif
-+ pr_info("Bman portal %sinitialised, cpu %d\n",
-+ pcfg->public_cfg.is_shared ? "(shared) " : "",
-+ pcfg->public_cfg.cpu);
-+ affine_bportals[pcfg->public_cfg.cpu] = p;
-+ } else
-+ pr_crit("Bman portal failure on cpu %d\n",
-+ pcfg->public_cfg.cpu);
-+ return p;
-+}
-+
-+static void init_slave(int cpu)
-+{
-+ struct bman_portal *p;
-+ p = bman_create_affine_slave(shared_portals[shared_portals_idx++], cpu);
-+ if (!p)
-+ pr_err("Bman slave portal failure on cpu %d\n", cpu);
-+ else
-+ pr_info("Bman portal %sinitialised, cpu %d\n", "(slave) ", cpu);
-+ if (shared_portals_idx >= num_shared_portals)
-+ shared_portals_idx = 0;
-+ affine_bportals[cpu] = p;
-+}
-+
-+/* Bootarg "bportals=[...]" has the same syntax as "qportals=", and so the
-+ * parsing is in dpa_sys.h. The syntax is a comma-separated list of indexes
-+ * and/or ranges of indexes, with each being optionally prefixed by "s" to
-+ * explicitly mark it or them for sharing.
-+ * Eg;
-+ * bportals=s0,1-3,s4
-+ * means that cpus 1,2,3 get "unshared" portals, cpus 0 and 4 get "shared"
-+ * portals, and any remaining cpus share the portals that are assigned to cpus 0
-+ * or 4, selected in a round-robin fashion. (In this example, cpu 5 would share
-+ * cpu 0's portal, cpu 6 would share cpu4's portal, and cpu 7 would share cpu
-+ * 0's portal.) */
-+static struct cpumask want_unshared __initdata; /* cpus requested without "s" */
-+static struct cpumask want_shared __initdata; /* cpus requested with "s" */
-+
-+static int __init parse_bportals(char *str)
-+{
-+ return parse_portals_bootarg(str, &want_shared, &want_unshared,
-+ "bportals");
-+}
-+__setup("bportals=", parse_bportals);
-+
-+static int bman_offline_cpu(unsigned int cpu)
-+{
-+ struct bman_portal *p;
-+ const struct bm_portal_config *pcfg;
-+ p = (struct bman_portal *)affine_bportals[cpu];
-+ if (p) {
-+ pcfg = bman_get_bm_portal_config(p);
-+ if (pcfg)
-+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0));
-+ }
-+ return 0;
-+}
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+static int bman_online_cpu(unsigned int cpu)
-+{
-+ struct bman_portal *p;
-+ const struct bm_portal_config *pcfg;
-+ p = (struct bman_portal *)affine_bportals[cpu];
-+ if (p) {
-+ pcfg = bman_get_bm_portal_config(p);
-+ if (pcfg)
-+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu));
-+ }
-+ return 0;
-+}
-+#endif /* CONFIG_HOTPLUG_CPU */
-+
-+/* Initialise the Bman driver. The meat of this function deals with portals. The
-+ * following describes the flow of portal-handling, the code "steps" refer to
-+ * this description;
-+ * 1. Portal configs are parsed from the device-tree into 'unused_pcfgs', with
-+ * ::cpu==-1. Regions and interrupts are mapped (but interrupts are not
-+ * bound).
-+ * 2. The "want_shared" and "want_unshared" lists (as filled by the
-+ * "bportals=[...]" bootarg) are processed, allocating portals and assigning
-+ * them to cpus, placing them in the relevant list and setting ::cpu as
-+ * appropriate. If no "bportals" bootarg was present, the defaut is to try to
-+ * assign portals to all online cpus at the time of driver initialisation.
-+ * Any failure to allocate portals (when parsing the "want" lists or when
-+ * using default behaviour) will be silently tolerated (the "fixup" logic in
-+ * step 3 will determine what happens in this case).
-+ * 3. Do fixups relative to cpu_online_mask(). If no portals are marked for
-+ * sharing and sharing is required (because not all cpus have been assigned
-+ * portals), then one portal will marked for sharing. Conversely if no
-+ * sharing is required, any portals marked for sharing will not be shared. It
-+ * may be that sharing occurs when it wasn't expected, if portal allocation
-+ * failed to honour all the requested assignments (including the default
-+ * assignments if no bootarg is present).
-+ * 4. Unshared portals are initialised on their respective cpus.
-+ * 5. Shared portals are initialised on their respective cpus.
-+ * 6. Each remaining cpu is initialised to slave to one of the shared portals,
-+ * which are selected in a round-robin fashion.
-+ * Any portal configs left unused are available for USDPAA allocation.
-+ */
-+__init int bman_init(void)
-+{
-+ struct cpumask slave_cpus;
-+ struct cpumask unshared_cpus = *cpu_none_mask;
-+ struct cpumask shared_cpus = *cpu_none_mask;
-+ LIST_HEAD(unshared_pcfgs);
-+ LIST_HEAD(shared_pcfgs);
-+ struct device_node *dn;
-+ struct bm_portal_config *pcfg;
-+ struct bman_portal *p;
-+ int cpu, ret;
-+ struct cpumask offline_cpus;
-+
-+ /* Initialise the Bman (CCSR) device */
-+ for_each_compatible_node(dn, NULL, "fsl,bman") {
-+ if (!bman_init_ccsr(dn))
-+ pr_info("Bman err interrupt handler present\n");
-+ else
-+ pr_err("Bman CCSR setup failed\n");
-+ }
-+ /* Initialise any declared buffer pools */
-+ for_each_compatible_node(dn, NULL, "fsl,bpool") {
-+ ret = fsl_bpool_init(dn);
-+ if (ret)
-+ return ret;
-+ }
-+ /* Step 1. See comments at the beginning of the file. */
-+ for_each_compatible_node(dn, NULL, "fsl,bman-portal") {
-+ if (!of_device_is_available(dn))
-+ continue;
-+ pcfg = parse_pcfg(dn);
-+ if (pcfg)
-+ list_add_tail(&pcfg->list, &unused_pcfgs);
-+ }
-+ /* Step 2. */
-+ for_each_possible_cpu(cpu) {
-+ if (cpumask_test_cpu(cpu, &want_shared)) {
-+ pcfg = get_pcfg(&unused_pcfgs);
-+ if (!pcfg)
-+ break;
-+ pcfg->public_cfg.cpu = cpu;
-+ list_add_tail(&pcfg->list, &shared_pcfgs);
-+ cpumask_set_cpu(cpu, &shared_cpus);
-+ }
-+ if (cpumask_test_cpu(cpu, &want_unshared)) {
-+ if (cpumask_test_cpu(cpu, &shared_cpus))
-+ continue;
-+ pcfg = get_pcfg(&unused_pcfgs);
-+ if (!pcfg)
-+ break;
-+ pcfg->public_cfg.cpu = cpu;
-+ list_add_tail(&pcfg->list, &unshared_pcfgs);
-+ cpumask_set_cpu(cpu, &unshared_cpus);
-+ }
-+ }
-+ if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) {
-+ /* Default, give an unshared portal to each online cpu */
-+ for_each_online_cpu(cpu) {
-+ pcfg = get_pcfg(&unused_pcfgs);
-+ if (!pcfg)
-+ break;
-+ pcfg->public_cfg.cpu = cpu;
-+ list_add_tail(&pcfg->list, &unshared_pcfgs);
-+ cpumask_set_cpu(cpu, &unshared_cpus);
-+ }
-+ }
-+ /* Step 3. */
-+ cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus);
-+ cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus);
-+ if (cpumask_empty(&slave_cpus)) {
-+ /* No sharing required */
-+ if (!list_empty(&shared_pcfgs)) {
-+ /* Migrate "shared" to "unshared" */
-+ cpumask_or(&unshared_cpus, &unshared_cpus,
-+ &shared_cpus);
-+ cpumask_clear(&shared_cpus);
-+ list_splice_tail(&shared_pcfgs, &unshared_pcfgs);
-+ INIT_LIST_HEAD(&shared_pcfgs);
-+ }
-+ } else {
-+ /* Sharing required */
-+ if (list_empty(&shared_pcfgs)) {
-+ /* Migrate one "unshared" to "shared" */
-+ pcfg = get_pcfg(&unshared_pcfgs);
-+ if (!pcfg) {
-+ pr_crit("No BMan portals available!\n");
-+ return 0;
-+ }
-+ cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus);
-+ cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus);
-+ list_add_tail(&pcfg->list, &shared_pcfgs);
-+ }
-+ }
-+ /* Step 4. */
-+ list_for_each_entry(pcfg, &unshared_pcfgs, list) {
-+ pcfg->public_cfg.is_shared = 0;
-+ p = init_pcfg(pcfg);
-+ if (!p) {
-+ pr_crit("Unable to initialize bman portal\n");
-+ return 0;
-+ }
-+ }
-+ /* Step 5. */
-+ list_for_each_entry(pcfg, &shared_pcfgs, list) {
-+ pcfg->public_cfg.is_shared = 1;
-+ p = init_pcfg(pcfg);
-+ if (p)
-+ shared_portals[num_shared_portals++] = p;
-+ }
-+ /* Step 6. */
-+ if (!cpumask_empty(&slave_cpus))
-+ for_each_cpu(cpu, &slave_cpus)
-+ init_slave(cpu);
-+ pr_info("Bman portals initialised\n");
-+ cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask);
-+ for_each_cpu(cpu, &offline_cpus)
-+ bman_offline_cpu(cpu);
-+#ifdef CONFIG_HOTPLUG_CPU
-+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
-+ "soc/qbman_portal:online",
-+ bman_online_cpu, bman_offline_cpu);
-+ if (ret < 0) {
-+ pr_err("bman: failed to register hotplug callbacks.\n");
-+ return 0;
-+ }
-+#endif
-+ return 0;
-+}
-+
-+__init int bman_resource_init(void)
-+{
-+ struct device_node *dn;
-+ int ret;
-+
-+ /* Initialise BPID allocation ranges */
-+ for_each_compatible_node(dn, NULL, "fsl,bpid-range") {
-+ ret = fsl_bpid_range_init(dn);
-+ if (ret)
-+ return ret;
-+ }
-+ return 0;
-+}
-+
-+#ifdef CONFIG_SUSPEND
-+void suspend_unused_bportal(void)
-+{
-+ struct bm_portal_config *pcfg;
-+
-+ if (list_empty(&unused_pcfgs))
-+ return;
-+
-+ list_for_each_entry(pcfg, &unused_pcfgs, list) {
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Need to save bportal %d\n", pcfg->public_cfg.index);
-+#endif
-+ /* save isdr, disable all via isdr, clear isr */
-+ pcfg->saved_isdr =
-+ __raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
-+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
-+ 0xe08);
-+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
-+ 0xe00);
-+ }
-+ return;
-+}
-+
-+void resume_unused_bportal(void)
-+{
-+ struct bm_portal_config *pcfg;
-+
-+ if (list_empty(&unused_pcfgs))
-+ return;
-+
-+ list_for_each_entry(pcfg, &unused_pcfgs, list) {
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Need to resume bportal %d\n", pcfg->public_cfg.index);
-+#endif
-+ /* restore isdr */
-+ __raw_writel(pcfg->saved_isdr,
-+ pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
-+ }
-+ return;
-+}
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_high.c
-@@ -0,0 +1,1145 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "bman_low.h"
-+
-+/* Compilation constants */
-+#define RCR_THRESH 2 /* reread h/w CI when running out of space */
-+#define IRQNAME "BMan portal %d"
-+#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */
-+
-+struct bman_portal {
-+ struct bm_portal p;
-+ /* 2-element array. pools[0] is mask, pools[1] is snapshot. */
-+ struct bman_depletion *pools;
-+ int thresh_set;
-+ unsigned long irq_sources;
-+ u32 slowpoll; /* only used when interrupts are off */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */
-+#endif
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ raw_spinlock_t sharing_lock; /* only used if is_shared */
-+ int is_shared;
-+ struct bman_portal *sharing_redirect;
-+#endif
-+ /* When the cpu-affine portal is activated, this is non-NULL */
-+ const struct bm_portal_config *config;
-+ /* This is needed for power management */
-+ struct platform_device *pdev;
-+ /* 64-entry hash-table of pool objects that are tracking depletion
-+ * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so
-+ * we're not fussy about cache-misses and so forth - whereas the above
-+ * members should all fit in one cacheline.
-+ * BTW, with 64 entries in the hash table and 64 buffer pools to track,
-+ * you'll never guess the hash-function ... */
-+ struct bman_pool *cb[64];
-+ char irqname[MAX_IRQNAME];
-+ /* Track if the portal was alloced by the driver */
-+ u8 alloced;
-+ /* power management data */
-+ u32 save_isdr;
-+};
-+
-+/* For an explanation of the locking, redirection, or affine-portal logic,
-+ * please consult the Qman driver for details. This is the same, only simpler
-+ * (no fiddly Qman-specific bits.) */
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+#define PORTAL_IRQ_LOCK(p, irqflags) \
-+ do { \
-+ if ((p)->is_shared) \
-+ raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
-+ else \
-+ local_irq_save(irqflags); \
-+ } while (0)
-+#define PORTAL_IRQ_UNLOCK(p, irqflags) \
-+ do { \
-+ if ((p)->is_shared) \
-+ raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
-+ irqflags); \
-+ else \
-+ local_irq_restore(irqflags); \
-+ } while (0)
-+#else
-+#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
-+#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
-+#endif
-+
-+static cpumask_t affine_mask;
-+static DEFINE_SPINLOCK(affine_mask_lock);
-+static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
-+static inline struct bman_portal *get_raw_affine_portal(void)
-+{
-+ return &get_cpu_var(bman_affine_portal);
-+}
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+static inline struct bman_portal *get_affine_portal(void)
-+{
-+ struct bman_portal *p = get_raw_affine_portal();
-+ if (p->sharing_redirect)
-+ return p->sharing_redirect;
-+ return p;
-+}
-+#else
-+#define get_affine_portal() get_raw_affine_portal()
-+#endif
-+static inline void put_affine_portal(void)
-+{
-+ put_cpu_var(bman_affine_portal);
-+}
-+static inline struct bman_portal *get_poll_portal(void)
-+{
-+ return &get_cpu_var(bman_affine_portal);
-+}
-+#define put_poll_portal()
-+
-+/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be
-+ * more than one such object per Bman buffer pool, eg. if different users of the
-+ * pool are operating via different portals. */
-+struct bman_pool {
-+ struct bman_pool_params params;
-+ /* Used for hash-table admin when using depletion notifications. */
-+ struct bman_portal *portal;
-+ struct bman_pool *next;
-+ /* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */
-+ struct bm_buffer *sp;
-+ unsigned int sp_fill;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ atomic_t in_use;
-+#endif
-+};
-+
-+/* (De)Registration of depletion notification callbacks */
-+static void depletion_link(struct bman_portal *portal, struct bman_pool *pool)
-+{
-+ __maybe_unused unsigned long irqflags;
-+ pool->portal = portal;
-+ PORTAL_IRQ_LOCK(portal, irqflags);
-+ pool->next = portal->cb[pool->params.bpid];
-+ portal->cb[pool->params.bpid] = pool;
-+ if (!pool->next)
-+ /* First object for that bpid on this portal, enable the BSCN
-+ * mask bit. */
-+ bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1);
-+ PORTAL_IRQ_UNLOCK(portal, irqflags);
-+}
-+static void depletion_unlink(struct bman_pool *pool)
-+{
-+ struct bman_pool *it, *last = NULL;
-+ struct bman_pool **base = &pool->portal->cb[pool->params.bpid];
-+ __maybe_unused unsigned long irqflags;
-+ PORTAL_IRQ_LOCK(pool->portal, irqflags);
-+ it = *base; /* <-- gotcha, don't do this prior to the irq_save */
-+ while (it != pool) {
-+ last = it;
-+ it = it->next;
-+ }
-+ if (!last)
-+ *base = pool->next;
-+ else
-+ last->next = pool->next;
-+ if (!last && !pool->next) {
-+ /* Last object for that bpid on this portal, disable the BSCN
-+ * mask bit. */
-+ bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0);
-+ /* And "forget" that we last saw this pool as depleted */
-+ bman_depletion_unset(&pool->portal->pools[1],
-+ pool->params.bpid);
-+ }
-+ PORTAL_IRQ_UNLOCK(pool->portal, irqflags);
-+}
-+
-+/* In the case that the application's core loop calls qman_poll() and
-+ * bman_poll(), we ought to balance how often we incur the overheads of the
-+ * slow-path poll. We'll use two decrementer sources. The idle decrementer
-+ * constant is used when the last slow-poll detected no work to do, and the busy
-+ * decrementer constant when the last slow-poll had work to do. */
-+#define SLOW_POLL_IDLE 1000
-+#define SLOW_POLL_BUSY 10
-+static u32 __poll_portal_slow(struct bman_portal *p, u32 is);
-+
-+/* Portal interrupt handler */
-+static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
-+{
-+ struct bman_portal *p = ptr;
-+ u32 clear = p->irq_sources;
-+ u32 is = bm_isr_status_read(&p->p) & p->irq_sources;
-+ clear |= __poll_portal_slow(p, is);
-+ bm_isr_status_clear(&p->p, clear);
-+ return IRQ_HANDLED;
-+}
-+
-+#ifdef CONFIG_SUSPEND
-+static int _bman_portal_suspend_noirq(struct device *dev)
-+{
-+ struct bman_portal *p = (struct bman_portal *)dev->platform_data;
-+#ifdef CONFIG_PM_DEBUG
-+ struct platform_device *pdev = to_platform_device(dev);
-+#endif
-+ p->save_isdr = bm_isr_disable_read(&p->p);
-+ bm_isr_disable_write(&p->p, 0xffffffff);
-+ bm_isr_status_clear(&p->p, 0xffffffff);
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Suspend for %s\n", pdev->name);
-+#endif
-+ return 0;
-+}
-+
-+static int _bman_portal_resume_noirq(struct device *dev)
-+{
-+ struct bman_portal *p = (struct bman_portal *)dev->platform_data;
-+
-+ /* restore isdr */
-+ bm_isr_disable_write(&p->p, p->save_isdr);
-+ return 0;
-+}
-+#else
-+#define _bman_portal_suspend_noirq NULL
-+#define _bman_portal_resume_noirq NULL
-+#endif
-+
-+struct dev_pm_domain bman_portal_device_pm_domain = {
-+ .ops = {
-+ USE_PLATFORM_PM_SLEEP_OPS
-+ .suspend_noirq = _bman_portal_suspend_noirq,
-+ .resume_noirq = _bman_portal_resume_noirq,
-+ }
-+};
-+
-+struct bman_portal *bman_create_portal(
-+ struct bman_portal *portal,
-+ const struct bm_portal_config *config)
-+{
-+ struct bm_portal *__p;
-+ const struct bman_depletion *pools = &config->public_cfg.mask;
-+ int ret;
-+ u8 bpid = 0;
-+ char buf[16];
-+
-+ if (!portal) {
-+ portal = kmalloc(sizeof(*portal), GFP_KERNEL);
-+ if (!portal)
-+ return portal;
-+ portal->alloced = 1;
-+ } else
-+ portal->alloced = 0;
-+
-+ __p = &portal->p;
-+
-+ /* prep the low-level portal struct with the mapped addresses from the
-+ * config, everything that follows depends on it and "config" is more
-+ * for (de)reference... */
-+ __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
-+ __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
-+ if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) {
-+ pr_err("Bman RCR initialisation failed\n");
-+ goto fail_rcr;
-+ }
-+ if (bm_mc_init(__p)) {
-+ pr_err("Bman MC initialisation failed\n");
-+ goto fail_mc;
-+ }
-+ if (bm_isr_init(__p)) {
-+ pr_err("Bman ISR initialisation failed\n");
-+ goto fail_isr;
-+ }
-+ portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL);
-+ if (!portal->pools)
-+ goto fail_pools;
-+ portal->pools[0] = *pools;
-+ bman_depletion_init(portal->pools + 1);
-+ while (bpid < bman_pool_max) {
-+ /* Default to all BPIDs disabled, we enable as required at
-+ * run-time. */
-+ bm_isr_bscn_mask(__p, bpid, 0);
-+ bpid++;
-+ }
-+ portal->slowpoll = 0;
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ portal->rcri_owned = NULL;
-+#endif
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ raw_spin_lock_init(&portal->sharing_lock);
-+ portal->is_shared = config->public_cfg.is_shared;
-+ portal->sharing_redirect = NULL;
-+#endif
-+ sprintf(buf, "bportal-%u", config->public_cfg.index);
-+ portal->pdev = platform_device_alloc(buf, -1);
-+ if (!portal->pdev)
-+ goto fail_devalloc;
-+ portal->pdev->dev.pm_domain = &bman_portal_device_pm_domain;
-+ portal->pdev->dev.platform_data = portal;
-+ ret = platform_device_add(portal->pdev);
-+ if (ret)
-+ goto fail_devadd;
-+ memset(&portal->cb, 0, sizeof(portal->cb));
-+ /* Write-to-clear any stale interrupt status bits */
-+ bm_isr_disable_write(__p, 0xffffffff);
-+ portal->irq_sources = 0;
-+ bm_isr_enable_write(__p, portal->irq_sources);
-+ bm_isr_status_clear(__p, 0xffffffff);
-+ snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
-+ if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
-+ portal)) {
-+ pr_err("request_irq() failed\n");
-+ goto fail_irq;
-+ }
-+ if ((config->public_cfg.cpu != -1) &&
-+ irq_can_set_affinity(config->public_cfg.irq) &&
-+ irq_set_affinity(config->public_cfg.irq,
-+ cpumask_of(config->public_cfg.cpu))) {
-+ pr_err("irq_set_affinity() failed %s\n", portal->irqname);
-+ goto fail_affinity;
-+ }
-+
-+ /* Need RCR to be empty before continuing */
-+ ret = bm_rcr_get_fill(__p);
-+ if (ret) {
-+ pr_err("Bman RCR unclean\n");
-+ goto fail_rcr_empty;
-+ }
-+ /* Success */
-+ portal->config = config;
-+
-+ bm_isr_disable_write(__p, 0);
-+ bm_isr_uninhibit(__p);
-+ return portal;
-+fail_rcr_empty:
-+fail_affinity:
-+ free_irq(config->public_cfg.irq, portal);
-+fail_irq:
-+ platform_device_del(portal->pdev);
-+fail_devadd:
-+ platform_device_put(portal->pdev);
-+fail_devalloc:
-+ kfree(portal->pools);
-+fail_pools:
-+ bm_isr_finish(__p);
-+fail_isr:
-+ bm_mc_finish(__p);
-+fail_mc:
-+ bm_rcr_finish(__p);
-+fail_rcr:
-+ if (portal->alloced)
-+ kfree(portal);
-+ return NULL;
-+}
-+
-+struct bman_portal *bman_create_affine_portal(
-+ const struct bm_portal_config *config)
-+{
-+ struct bman_portal *portal;
-+
-+ portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu);
-+ portal = bman_create_portal(portal, config);
-+ if (portal) {
-+ spin_lock(&affine_mask_lock);
-+ cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
-+ spin_unlock(&affine_mask_lock);
-+ }
-+ return portal;
-+}
-+
-+
-+struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect,
-+ int cpu)
-+{
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ struct bman_portal *p;
-+ p = &per_cpu(bman_affine_portal, cpu);
-+ BUG_ON(p->config);
-+ BUG_ON(p->is_shared);
-+ BUG_ON(!redirect->config->public_cfg.is_shared);
-+ p->irq_sources = 0;
-+ p->sharing_redirect = redirect;
-+ return p;
-+#else
-+ BUG();
-+ return NULL;
-+#endif
-+}
-+
-+void bman_destroy_portal(struct bman_portal *bm)
-+{
-+ const struct bm_portal_config *pcfg;
-+ pcfg = bm->config;
-+ bm_rcr_cce_update(&bm->p);
-+ bm_rcr_cce_update(&bm->p);
-+
-+ free_irq(pcfg->public_cfg.irq, bm);
-+
-+ kfree(bm->pools);
-+ bm_isr_finish(&bm->p);
-+ bm_mc_finish(&bm->p);
-+ bm_rcr_finish(&bm->p);
-+ bm->config = NULL;
-+ if (bm->alloced)
-+ kfree(bm);
-+}
-+
-+const struct bm_portal_config *bman_destroy_affine_portal(void)
-+{
-+ struct bman_portal *bm = get_raw_affine_portal();
-+ const struct bm_portal_config *pcfg;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (bm->sharing_redirect) {
-+ bm->sharing_redirect = NULL;
-+ put_affine_portal();
-+ return NULL;
-+ }
-+ bm->is_shared = 0;
-+#endif
-+ pcfg = bm->config;
-+ bman_destroy_portal(bm);
-+ spin_lock(&affine_mask_lock);
-+ cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask);
-+ spin_unlock(&affine_mask_lock);
-+ put_affine_portal();
-+ return pcfg;
-+}
-+
-+/* When release logic waits on available RCR space, we need a global waitqueue
-+ * in the case of "affine" use (as the waits wake on different cpus which means
-+ * different portals - so we can't wait on any per-portal waitqueue). */
-+static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
-+
-+static u32 __poll_portal_slow(struct bman_portal *p, u32 is)
-+{
-+ struct bman_depletion tmp;
-+ u32 ret = is;
-+
-+ /* There is a gotcha to be aware of. If we do the query before clearing
-+ * the status register, we may miss state changes that occur between the
-+ * two. If we write to clear the status register before the query, the
-+ * cache-enabled query command may overtake the status register write
-+ * unless we use a heavyweight sync (which we don't want). Instead, we
-+ * write-to-clear the status register then *read it back* before doing
-+ * the query, hence the odd while loop with the 'is' accumulation. */
-+ if (is & BM_PIRQ_BSCN) {
-+ struct bm_mc_result *mcr;
-+ __maybe_unused unsigned long irqflags;
-+ unsigned int i, j;
-+ u32 __is;
-+ bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
-+ while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) {
-+ is |= __is;
-+ bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
-+ }
-+ is &= ~BM_PIRQ_BSCN;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ bm_mc_start(&p->p);
-+ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
-+ while (!(mcr = bm_mc_result(&p->p)))
-+ cpu_relax();
-+ tmp = mcr->query.ds.state;
-+ tmp.__state[0] = be32_to_cpu(tmp.__state[0]);
-+ tmp.__state[1] = be32_to_cpu(tmp.__state[1]);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ for (i = 0; i < 2; i++) {
-+ int idx = i * 32;
-+ /* tmp is a mask of currently-depleted pools.
-+ * pools[0] is mask of those we care about.
-+ * pools[1] is our previous view (we only want to
-+ * be told about changes). */
-+ tmp.__state[i] &= p->pools[0].__state[i];
-+ if (tmp.__state[i] == p->pools[1].__state[i])
-+ /* fast-path, nothing to see, move along */
-+ continue;
-+ for (j = 0; j <= 31; j++, idx++) {
-+ struct bman_pool *pool = p->cb[idx];
-+ int b4 = bman_depletion_get(&p->pools[1], idx);
-+ int af = bman_depletion_get(&tmp, idx);
-+ if (b4 == af)
-+ continue;
-+ while (pool) {
-+ pool->params.cb(p, pool,
-+ pool->params.cb_ctx, af);
-+ pool = pool->next;
-+ }
-+ }
-+ }
-+ p->pools[1] = tmp;
-+ }
-+
-+ if (is & BM_PIRQ_RCRI) {
-+ __maybe_unused unsigned long irqflags;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ bm_rcr_cce_update(&p->p);
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ /* If waiting for sync, we only cancel the interrupt threshold
-+ * when the ring utilisation hits zero. */
-+ if (p->rcri_owned) {
-+ if (!bm_rcr_get_fill(&p->p)) {
-+ p->rcri_owned = NULL;
-+ bm_rcr_set_ithresh(&p->p, 0);
-+ }
-+ } else
-+#endif
-+ bm_rcr_set_ithresh(&p->p, 0);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ wake_up(&affine_queue);
-+ bm_isr_status_clear(&p->p, BM_PIRQ_RCRI);
-+ is &= ~BM_PIRQ_RCRI;
-+ }
-+
-+ /* There should be no status register bits left undefined */
-+ DPA_ASSERT(!is);
-+ return ret;
-+}
-+
-+const struct bman_portal_config *bman_get_portal_config(void)
-+{
-+ struct bman_portal *p = get_affine_portal();
-+ const struct bman_portal_config *ret = &p->config->public_cfg;
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_get_portal_config);
-+
-+u32 bman_irqsource_get(void)
-+{
-+ struct bman_portal *p = get_raw_affine_portal();
-+ u32 ret = p->irq_sources & BM_PIRQ_VISIBLE;
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_irqsource_get);
-+
-+int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits)
-+{
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (p->sharing_redirect)
-+ return -EINVAL;
-+ else
-+#endif
-+ {
-+ __maybe_unused unsigned long irqflags;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources);
-+ bm_isr_enable_write(&p->p, p->irq_sources);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(bman_p_irqsource_add);
-+
-+int bman_irqsource_add(__maybe_unused u32 bits)
-+{
-+ struct bman_portal *p = get_raw_affine_portal();
-+ int ret = 0;
-+ ret = bman_p_irqsource_add(p, bits);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_irqsource_add);
-+
-+int bman_irqsource_remove(u32 bits)
-+{
-+ struct bman_portal *p = get_raw_affine_portal();
-+ __maybe_unused unsigned long irqflags;
-+ u32 ier;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (p->sharing_redirect) {
-+ put_affine_portal();
-+ return -EINVAL;
-+ }
-+#endif
-+ /* Our interrupt handler only processes+clears status register bits that
-+ * are in p->irq_sources. As we're trimming that mask, if one of them
-+ * were to assert in the status register just before we remove it from
-+ * the enable register, there would be an interrupt-storm when we
-+ * release the IRQ lock. So we wait for the enable register update to
-+ * take effect in h/w (by reading it back) and then clear all other bits
-+ * in the status register. Ie. we clear them from ISR once it's certain
-+ * IER won't allow them to reassert. */
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ bits &= BM_PIRQ_VISIBLE;
-+ clear_bits(bits, &p->irq_sources);
-+ bm_isr_enable_write(&p->p, p->irq_sources);
-+ ier = bm_isr_enable_read(&p->p);
-+ /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
-+ * data-dependency, ie. to protect against re-ordering. */
-+ bm_isr_status_clear(&p->p, ~ier);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return 0;
-+}
-+EXPORT_SYMBOL(bman_irqsource_remove);
-+
-+const cpumask_t *bman_affine_cpus(void)
-+{
-+ return &affine_mask;
-+}
-+EXPORT_SYMBOL(bman_affine_cpus);
-+
-+u32 bman_poll_slow(void)
-+{
-+ struct bman_portal *p = get_poll_portal();
-+ u32 ret;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (unlikely(p->sharing_redirect))
-+ ret = (u32)-1;
-+ else
-+#endif
-+ {
-+ u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;
-+ ret = __poll_portal_slow(p, is);
-+ bm_isr_status_clear(&p->p, ret);
-+ }
-+ put_poll_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_poll_slow);
-+
-+/* Legacy wrapper */
-+void bman_poll(void)
-+{
-+ struct bman_portal *p = get_poll_portal();
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (unlikely(p->sharing_redirect))
-+ goto done;
-+#endif
-+ if (!(p->slowpoll--)) {
-+ u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;
-+ u32 active = __poll_portal_slow(p, is);
-+ if (active)
-+ p->slowpoll = SLOW_POLL_BUSY;
-+ else
-+ p->slowpoll = SLOW_POLL_IDLE;
-+ }
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+done:
-+#endif
-+ put_poll_portal();
-+}
-+EXPORT_SYMBOL(bman_poll);
-+
-+static const u32 zero_thresholds[4] = {0, 0, 0, 0};
-+
-+struct bman_pool *bman_new_pool(const struct bman_pool_params *params)
-+{
-+ struct bman_pool *pool = NULL;
-+ u32 bpid;
-+
-+ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) {
-+ int ret = bman_alloc_bpid(&bpid);
-+ if (ret)
-+ return NULL;
-+ } else {
-+ if (params->bpid >= bman_pool_max)
-+ return NULL;
-+ bpid = params->bpid;
-+ }
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+ if (params->flags & BMAN_POOL_FLAG_THRESH) {
-+ int ret = bm_pool_set(bpid, params->thresholds);
-+ if (ret)
-+ goto err;
-+ }
-+#else
-+ if (params->flags & BMAN_POOL_FLAG_THRESH)
-+ goto err;
-+#endif
-+ pool = kmalloc(sizeof(*pool), GFP_KERNEL);
-+ if (!pool)
-+ goto err;
-+ pool->sp = NULL;
-+ pool->sp_fill = 0;
-+ pool->params = *params;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ atomic_set(&pool->in_use, 1);
-+#endif
-+ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
-+ pool->params.bpid = bpid;
-+ if (params->flags & BMAN_POOL_FLAG_STOCKPILE) {
-+ pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ,
-+ GFP_KERNEL);
-+ if (!pool->sp)
-+ goto err;
-+ }
-+ if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) {
-+ struct bman_portal *p = get_affine_portal();
-+ if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) {
-+ pr_err("Depletion events disabled for bpid %d\n", bpid);
-+ goto err;
-+ }
-+ depletion_link(p, pool);
-+ put_affine_portal();
-+ }
-+ return pool;
-+err:
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+ if (params->flags & BMAN_POOL_FLAG_THRESH)
-+ bm_pool_set(bpid, zero_thresholds);
-+#endif
-+ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
-+ bman_release_bpid(bpid);
-+ if (pool) {
-+ kfree(pool->sp);
-+ kfree(pool);
-+ }
-+ return NULL;
-+}
-+EXPORT_SYMBOL(bman_new_pool);
-+
-+void bman_free_pool(struct bman_pool *pool)
-+{
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+ if (pool->params.flags & BMAN_POOL_FLAG_THRESH)
-+ bm_pool_set(pool->params.bpid, zero_thresholds);
-+#endif
-+ if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION)
-+ depletion_unlink(pool);
-+ if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) {
-+ if (pool->sp_fill)
-+ pr_err("Stockpile not flushed, has %u in bpid %u.\n",
-+ pool->sp_fill, pool->params.bpid);
-+ kfree(pool->sp);
-+ pool->sp = NULL;
-+ pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE;
-+ }
-+ if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
-+ bman_release_bpid(pool->params.bpid);
-+ kfree(pool);
-+}
-+EXPORT_SYMBOL(bman_free_pool);
-+
-+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool)
-+{
-+ return &pool->params;
-+}
-+EXPORT_SYMBOL(bman_get_params);
-+
-+static noinline void update_rcr_ci(struct bman_portal *p, u8 avail)
-+{
-+ if (avail)
-+ bm_rcr_cce_prefetch(&p->p);
-+ else
-+ bm_rcr_cce_update(&p->p);
-+}
-+
-+int bman_rcr_is_empty(void)
-+{
-+ __maybe_unused unsigned long irqflags;
-+ struct bman_portal *p = get_affine_portal();
-+ u8 avail;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ update_rcr_ci(p, 0);
-+ avail = bm_rcr_get_fill(&p->p);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return avail == 0;
-+}
-+EXPORT_SYMBOL(bman_rcr_is_empty);
-+
-+static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p,
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ __maybe_unused struct bman_pool *pool,
-+#endif
-+ __maybe_unused unsigned long *irqflags,
-+ __maybe_unused u32 flags)
-+{
-+ struct bm_rcr_entry *r;
-+ u8 avail;
-+
-+ *p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(*p, (*irqflags));
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
-+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
-+ if ((*p)->rcri_owned) {
-+ PORTAL_IRQ_UNLOCK(*p, (*irqflags));
-+ put_affine_portal();
-+ return NULL;
-+ }
-+ (*p)->rcri_owned = pool;
-+ }
-+#endif
-+ avail = bm_rcr_get_avail(&(*p)->p);
-+ if (avail < 2)
-+ update_rcr_ci(*p, avail);
-+ r = bm_rcr_start(&(*p)->p);
-+ if (unlikely(!r)) {
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
-+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC)))
-+ (*p)->rcri_owned = NULL;
-+#endif
-+ PORTAL_IRQ_UNLOCK(*p, (*irqflags));
-+ put_affine_portal();
-+ }
-+ return r;
-+}
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p,
-+ struct bman_pool *pool,
-+ __maybe_unused unsigned long *irqflags,
-+ u32 flags)
-+{
-+ struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags);
-+ if (!rcr)
-+ bm_rcr_set_ithresh(&(*p)->p, 1);
-+ return rcr;
-+}
-+
-+static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p,
-+ struct bman_pool *pool,
-+ __maybe_unused unsigned long *irqflags,
-+ u32 flags)
-+{
-+ struct bm_rcr_entry *rcr;
-+#ifndef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ pool = NULL;
-+#endif
-+ if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
-+ /* NB: return NULL if signal occurs before completion. Signal
-+ * can occur during return. Caller must check for signal */
-+ wait_event_interruptible(affine_queue,
-+ (rcr = __wait_rel_start(p, pool, irqflags, flags)));
-+ else
-+ wait_event(affine_queue,
-+ (rcr = __wait_rel_start(p, pool, irqflags, flags)));
-+ return rcr;
-+}
-+#endif
-+
-+static inline int __bman_release(struct bman_pool *pool,
-+ const struct bm_buffer *bufs, u8 num, u32 flags)
-+{
-+ struct bman_portal *p;
-+ struct bm_rcr_entry *r;
-+ __maybe_unused unsigned long irqflags;
-+ u32 i = num - 1;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & BMAN_RELEASE_FLAG_WAIT)
-+ r = wait_rel_start(&p, pool, &irqflags, flags);
-+ else
-+ r = try_rel_start(&p, pool, &irqflags, flags);
-+#else
-+ r = try_rel_start(&p, &irqflags, flags);
-+#endif
-+ if (!r)
-+ return -EBUSY;
-+ /* We can copy all but the first entry, as this can trigger badness
-+ * with the valid-bit. Use the overlay to mask the verb byte. */
-+ r->bufs[0].opaque =
-+ ((cpu_to_be64((bufs[0].opaque |
-+ ((u64)pool->params.bpid<<48))
-+ & 0x00ffffffffffffff)));
-+ if (i) {
-+ for (i = 1; i < num; i++)
-+ r->bufs[i].opaque =
-+ cpu_to_be64(bufs[i].opaque);
-+ }
-+
-+ bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
-+ (num & BM_RCR_VERB_BUFCOUNT_MASK));
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ /* if we wish to sync we need to set the threshold after h/w sees the
-+ * new ring entry. As we're mixing cache-enabled and cache-inhibited
-+ * accesses, this requires a heavy-weight sync. */
-+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
-+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
-+ hwsync();
-+ bm_rcr_set_ithresh(&p->p, 1);
-+ }
-+#endif
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
-+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
-+ if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->rcri_owned != pool));
-+ else
-+ wait_event(affine_queue, (p->rcri_owned != pool));
-+ }
-+#endif
-+ return 0;
-+}
-+
-+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
-+ u32 flags)
-+{
-+ int ret;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (!num || (num > 8))
-+ return -EINVAL;
-+ if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE)
-+ return -EINVAL;
-+#endif
-+ /* Without stockpile, this API is a pass-through to the h/w operation */
-+ if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
-+ return __bman_release(pool, bufs, num, flags);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (!atomic_dec_and_test(&pool->in_use)) {
-+ pr_crit("Parallel attempts to enter bman_released() detected.");
-+ panic("only one instance of bman_released/acquired allowed");
-+ }
-+#endif
-+ /* Two movements of buffers are possible, and can occur in either order.
-+ * A: moving buffers from the caller to the stockpile.
-+ * B: moving buffers from the stockpile to hardware.
-+ * Order 1: if there is already enough space in the stockpile for A
-+ * then we want to do A first, and only do B if we trigger the
-+ * stockpile-high threshold.
-+ * Order 2: if there is not enough space in the stockpile for A, then
-+ * we want to do B first, then do A if B had succeeded. However in this
-+ * case B is dependent on how many buffers the user needs to release,
-+ * not the stockpile-high threshold.
-+ * Due to the different handling of B between the two cases, putting A
-+ * and B in a while() loop would require quite obscure logic, so handle
-+ * the different sequences explicitly. */
-+ if ((pool->sp_fill + num) <= BMAN_STOCKPILE_SZ) {
-+ /* Order 1: do A */
-+ copy_words(pool->sp + pool->sp_fill, bufs,
-+ sizeof(struct bm_buffer) * num);
-+ pool->sp_fill += num;
-+ /* do B relative to STOCKPILE_HIGH */
-+ while (pool->sp_fill >= BMAN_STOCKPILE_HIGH) {
-+ ret = __bman_release(pool,
-+ pool->sp + (pool->sp_fill - 8), 8,
-+ flags);
-+ if (ret >= 0)
-+ pool->sp_fill -= 8;
-+ }
-+ } else {
-+ /* Order 2: do B relative to 'num' */
-+ do {
-+ ret = __bman_release(pool,
-+ pool->sp + (pool->sp_fill - 8), 8,
-+ flags);
-+ if (ret < 0)
-+ /* failure */
-+ goto release_done;
-+ pool->sp_fill -= 8;
-+ } while ((pool->sp_fill + num) > BMAN_STOCKPILE_SZ);
-+ /* do A */
-+ copy_words(pool->sp + pool->sp_fill, bufs,
-+ sizeof(struct bm_buffer) * num);
-+ pool->sp_fill += num;
-+ }
-+ /* success */
-+ ret = 0;
-+release_done:
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ atomic_inc(&pool->in_use);
-+#endif
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_release);
-+
-+static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs,
-+ u8 num)
-+{
-+ struct bman_portal *p = get_affine_portal();
-+ struct bm_mc_command *mcc;
-+ struct bm_mc_result *mcr;
-+ __maybe_unused unsigned long irqflags;
-+ int ret, i;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = bm_mc_start(&p->p);
-+ mcc->acquire.bpid = pool->params.bpid;
-+ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
-+ (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
-+ while (!(mcr = bm_mc_result(&p->p)))
-+ cpu_relax();
-+ ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
-+ if (bufs) {
-+ for (i = 0; i < num; i++)
-+ bufs[i].opaque =
-+ be64_to_cpu(mcr->acquire.bufs[i].opaque);
-+ }
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (ret != num)
-+ ret = -ENOMEM;
-+ return ret;
-+}
-+
-+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
-+ u32 flags)
-+{
-+ int ret;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (!num || (num > 8))
-+ return -EINVAL;
-+ if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE)
-+ return -EINVAL;
-+#endif
-+ /* Without stockpile, this API is a pass-through to the h/w operation */
-+ if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
-+ return __bman_acquire(pool, bufs, num);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (!atomic_dec_and_test(&pool->in_use)) {
-+ pr_crit("Parallel attempts to enter bman_acquire() detected.");
-+ panic("only one instance of bman_released/acquired allowed");
-+ }
-+#endif
-+ /* Two movements of buffers are possible, and can occur in either order.
-+ * A: moving buffers from stockpile to the caller.
-+ * B: moving buffers from hardware to the stockpile.
-+ * Order 1: if there are already enough buffers in the stockpile for A
-+ * then we want to do A first, and only do B if we trigger the
-+ * stockpile-low threshold.
-+ * Order 2: if there are not enough buffers in the stockpile for A,
-+ * then we want to do B first, then do A if B had succeeded. However in
-+ * this case B is dependent on how many buffers the user needs, not the
-+ * stockpile-low threshold.
-+ * Due to the different handling of B between the two cases, putting A
-+ * and B in a while() loop would require quite obscure logic, so handle
-+ * the different sequences explicitly. */
-+ if (num <= pool->sp_fill) {
-+ /* Order 1: do A */
-+ copy_words(bufs, pool->sp + (pool->sp_fill - num),
-+ sizeof(struct bm_buffer) * num);
-+ pool->sp_fill -= num;
-+ /* do B relative to STOCKPILE_LOW */
-+ while (pool->sp_fill <= BMAN_STOCKPILE_LOW) {
-+ ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8);
-+ if (ret < 0)
-+ ret = __bman_acquire(pool,
-+ pool->sp + pool->sp_fill, 1);
-+ if (ret < 0)
-+ break;
-+ pool->sp_fill += ret;
-+ }
-+ } else {
-+ /* Order 2: do B relative to 'num' */
-+ do {
-+ ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8);
-+ if (ret < 0)
-+ ret = __bman_acquire(pool,
-+ pool->sp + pool->sp_fill, 1);
-+ if (ret < 0)
-+ /* failure */
-+ goto acquire_done;
-+ pool->sp_fill += ret;
-+ } while (pool->sp_fill < num);
-+ /* do A */
-+ copy_words(bufs, pool->sp + (pool->sp_fill - num),
-+ sizeof(struct bm_buffer) * num);
-+ pool->sp_fill -= num;
-+ }
-+ /* success */
-+ ret = num;
-+acquire_done:
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ atomic_inc(&pool->in_use);
-+#endif
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_acquire);
-+
-+int bman_flush_stockpile(struct bman_pool *pool, u32 flags)
-+{
-+ u8 num;
-+ int ret;
-+
-+ while (pool->sp_fill) {
-+ num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill);
-+ ret = __bman_release(pool, pool->sp + (pool->sp_fill - num),
-+ num, flags);
-+ if (ret)
-+ return ret;
-+ pool->sp_fill -= num;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(bman_flush_stockpile);
-+
-+int bman_query_pools(struct bm_pool_state *state)
-+{
-+ struct bman_portal *p = get_affine_portal();
-+ struct bm_mc_result *mcr;
-+ __maybe_unused unsigned long irqflags;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ bm_mc_start(&p->p);
-+ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
-+ while (!(mcr = bm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY);
-+ *state = mcr->query;
-+ state->as.state.__state[0] = be32_to_cpu(state->as.state.__state[0]);
-+ state->as.state.__state[1] = be32_to_cpu(state->as.state.__state[1]);
-+ state->ds.state.__state[0] = be32_to_cpu(state->ds.state.__state[0]);
-+ state->ds.state.__state[1] = be32_to_cpu(state->ds.state.__state[1]);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return 0;
-+}
-+EXPORT_SYMBOL(bman_query_pools);
-+
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+u32 bman_query_free_buffers(struct bman_pool *pool)
-+{
-+ return bm_pool_free_buffers(pool->params.bpid);
-+}
-+EXPORT_SYMBOL(bman_query_free_buffers);
-+
-+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds)
-+{
-+ u32 bpid;
-+
-+ bpid = bman_get_params(pool)->bpid;
-+
-+ return bm_pool_set(bpid, thresholds);
-+}
-+EXPORT_SYMBOL(bman_update_pool_thresholds);
-+#endif
-+
-+int bman_shutdown_pool(u32 bpid)
-+{
-+ struct bman_portal *p = get_affine_portal();
-+ __maybe_unused unsigned long irqflags;
-+ int ret;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ ret = bm_shutdown_pool(&p->p, bpid);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(bman_shutdown_pool);
-+
-+const struct bm_portal_config *bman_get_bm_portal_config(
-+ struct bman_portal *portal)
-+{
-+ return portal->sharing_redirect ? NULL : portal->config;
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_low.h
-@@ -0,0 +1,565 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "bman_private.h"
-+
-+/***************************/
-+/* Portal register assists */
-+/***************************/
-+
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+
-+/* Cache-inhibited register offsets */
-+#define BM_REG_RCR_PI_CINH 0x0000
-+#define BM_REG_RCR_CI_CINH 0x0004
-+#define BM_REG_RCR_ITR 0x0008
-+#define BM_REG_CFG 0x0100
-+#define BM_REG_SCN(n) (0x0200 + ((n) << 2))
-+#define BM_REG_ISR 0x0e00
-+#define BM_REG_IIR 0x0e0c
-+
-+/* Cache-enabled register offsets */
-+#define BM_CL_CR 0x0000
-+#define BM_CL_RR0 0x0100
-+#define BM_CL_RR1 0x0140
-+#define BM_CL_RCR 0x1000
-+#define BM_CL_RCR_PI_CENA 0x3000
-+#define BM_CL_RCR_CI_CENA 0x3100
-+
-+#endif
-+
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+
-+/* Cache-inhibited register offsets */
-+#define BM_REG_RCR_PI_CINH 0x3000
-+#define BM_REG_RCR_CI_CINH 0x3100
-+#define BM_REG_RCR_ITR 0x3200
-+#define BM_REG_CFG 0x3300
-+#define BM_REG_SCN(n) (0x3400 + ((n) << 6))
-+#define BM_REG_ISR 0x3e00
-+#define BM_REG_IIR 0x3ec0
-+
-+/* Cache-enabled register offsets */
-+#define BM_CL_CR 0x0000
-+#define BM_CL_RR0 0x0100
-+#define BM_CL_RR1 0x0140
-+#define BM_CL_RCR 0x1000
-+#define BM_CL_RCR_PI_CENA 0x3000
-+#define BM_CL_RCR_CI_CENA 0x3100
-+
-+#endif
-+
-+/* BTW, the drivers (and h/w programming model) already obtain the required
-+ * synchronisation for portal accesses via lwsync(), hwsync(), and
-+ * data-dependencies. Use of barrier()s or other order-preserving primitives
-+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which
-+ * simply ensure that the compiler treats the portal registers as volatile (ie.
-+ * non-coherent). */
-+
-+/* Cache-inhibited register access. */
-+#define __bm_in(bm, o) be32_to_cpu(__raw_readl((bm)->addr_ci + (o)))
-+#define __bm_out(bm, o, val) __raw_writel(cpu_to_be32(val), \
-+ (bm)->addr_ci + (o));
-+#define bm_in(reg) __bm_in(&portal->addr, BM_REG_##reg)
-+#define bm_out(reg, val) __bm_out(&portal->addr, BM_REG_##reg, val)
-+
-+/* Cache-enabled (index) register access */
-+#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->addr_ce + (o))
-+#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->addr_ce + (o))
-+#define __bm_cl_in(bm, o) be32_to_cpu(__raw_readl((bm)->addr_ce + (o)))
-+#define __bm_cl_out(bm, o, val) \
-+ do { \
-+ u32 *__tmpclout = (bm)->addr_ce + (o); \
-+ __raw_writel(cpu_to_be32(val), __tmpclout); \
-+ dcbf(__tmpclout); \
-+ } while (0)
-+#define __bm_cl_invalidate(bm, o) dcbi((bm)->addr_ce + (o))
-+#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA)
-+#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA)
-+#define bm_cl_in(reg) __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA)
-+#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val)
-+#define bm_cl_invalidate(reg)\
-+ __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA)
-+
-+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
-+ * analysis, look at using the "extra" bit in the ring index registers to avoid
-+ * cyclic issues. */
-+static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last)
-+{
-+ /* 'first' is included, 'last' is excluded */
-+ if (first <= last)
-+ return last - first;
-+ return ringsize + last - first;
-+}
-+
-+/* Portal modes.
-+ * Enum types;
-+ * pmode == production mode
-+ * cmode == consumption mode,
-+ * Enum values use 3 letter codes. First letter matches the portal mode,
-+ * remaining two letters indicate;
-+ * ci == cache-inhibited portal register
-+ * ce == cache-enabled portal register
-+ * vb == in-band valid-bit (cache-enabled)
-+ */
-+enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */
-+ bm_rcr_pci = 0, /* PI index, cache-inhibited */
-+ bm_rcr_pce = 1, /* PI index, cache-enabled */
-+ bm_rcr_pvb = 2 /* valid-bit */
-+};
-+enum bm_rcr_cmode { /* s/w-only */
-+ bm_rcr_cci, /* CI index, cache-inhibited */
-+ bm_rcr_cce /* CI index, cache-enabled */
-+};
-+
-+
-+/* ------------------------- */
-+/* --- Portal structures --- */
-+
-+#define BM_RCR_SIZE 8
-+
-+struct bm_rcr {
-+ struct bm_rcr_entry *ring, *cursor;
-+ u8 ci, available, ithresh, vbit;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ u32 busy;
-+ enum bm_rcr_pmode pmode;
-+ enum bm_rcr_cmode cmode;
-+#endif
-+};
-+
-+struct bm_mc {
-+ struct bm_mc_command *cr;
-+ struct bm_mc_result *rr;
-+ u8 rridx, vbit;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ enum {
-+ /* Can only be _mc_start()ed */
-+ mc_idle,
-+ /* Can only be _mc_commit()ed or _mc_abort()ed */
-+ mc_user,
-+ /* Can only be _mc_retry()ed */
-+ mc_hw
-+ } state;
-+#endif
-+};
-+
-+struct bm_addr {
-+ void __iomem *addr_ce; /* cache-enabled */
-+ void __iomem *addr_ci; /* cache-inhibited */
-+};
-+
-+struct bm_portal {
-+ struct bm_addr addr;
-+ struct bm_rcr rcr;
-+ struct bm_mc mc;
-+ struct bm_portal_config config;
-+} ____cacheline_aligned;
-+
-+
-+/* --------------- */
-+/* --- RCR API --- */
-+
-+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
-+#define RCR_CARRYCLEAR(p) \
-+ (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6)))
-+
-+/* Bit-wise logic to convert a ring pointer to a ring index */
-+static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e)
-+{
-+ return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1);
-+}
-+
-+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
-+static inline void RCR_INC(struct bm_rcr *rcr)
-+{
-+ /* NB: this is odd-looking, but experiments show that it generates
-+ * fast code with essentially no branching overheads. We increment to
-+ * the next RCR pointer and handle overflow and 'vbit'. */
-+ struct bm_rcr_entry *partial = rcr->cursor + 1;
-+ rcr->cursor = RCR_CARRYCLEAR(partial);
-+ if (partial != rcr->cursor)
-+ rcr->vbit ^= BM_RCR_VERB_VBIT;
-+}
-+
-+static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
-+ __maybe_unused enum bm_rcr_cmode cmode)
-+{
-+ /* This use of 'register', as well as all other occurrences, is because
-+ * it has been observed to generate much faster code with gcc than is
-+ * otherwise the case. */
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ u32 cfg;
-+ u8 pi;
-+
-+ rcr->ring = portal->addr.addr_ce + BM_CL_RCR;
-+ rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
-+
-+ pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
-+ rcr->cursor = rcr->ring + pi;
-+ rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0;
-+ rcr->available = BM_RCR_SIZE - 1
-+ - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
-+ rcr->ithresh = bm_in(RCR_ITR);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ rcr->busy = 0;
-+ rcr->pmode = pmode;
-+ rcr->cmode = cmode;
-+#endif
-+ cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
-+ bm_out(CFG, cfg);
-+ return 0;
-+}
-+
-+static inline void bm_rcr_finish(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
-+ u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
-+ DPA_ASSERT(!rcr->busy);
-+ if (pi != RCR_PTR2IDX(rcr->cursor))
-+ pr_crit("losing uncommited RCR entries\n");
-+ if (ci != rcr->ci)
-+ pr_crit("missing existing RCR completions\n");
-+ if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
-+ pr_crit("RCR destroyed unquiesced\n");
-+}
-+
-+static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(!rcr->busy);
-+ if (!rcr->available)
-+ return NULL;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ rcr->busy = 1;
-+#endif
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(rcr->cursor);
-+#endif
-+ return rcr->cursor;
-+}
-+
-+static inline void bm_rcr_abort(struct bm_portal *portal)
-+{
-+ __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(rcr->busy);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ rcr->busy = 0;
-+#endif
-+}
-+
-+static inline struct bm_rcr_entry *bm_rcr_pend_and_next(
-+ struct bm_portal *portal, u8 myverb)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(rcr->busy);
-+ DPA_ASSERT(rcr->pmode != bm_rcr_pvb);
-+ if (rcr->available == 1)
-+ return NULL;
-+ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
-+ dcbf_64(rcr->cursor);
-+ RCR_INC(rcr);
-+ rcr->available--;
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(rcr->cursor);
-+#endif
-+ return rcr->cursor;
-+}
-+
-+static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(rcr->busy);
-+ DPA_ASSERT(rcr->pmode == bm_rcr_pci);
-+ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
-+ RCR_INC(rcr);
-+ rcr->available--;
-+ hwsync();
-+ bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ rcr->busy = 0;
-+#endif
-+}
-+
-+static inline void bm_rcr_pce_prefetch(struct bm_portal *portal)
-+{
-+ __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(rcr->pmode == bm_rcr_pce);
-+ bm_cl_invalidate(RCR_PI);
-+ bm_cl_touch_rw(RCR_PI);
-+}
-+
-+static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(rcr->busy);
-+ DPA_ASSERT(rcr->pmode == bm_rcr_pce);
-+ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
-+ RCR_INC(rcr);
-+ rcr->available--;
-+ lwsync();
-+ bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ rcr->busy = 0;
-+#endif
-+}
-+
-+static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ struct bm_rcr_entry *rcursor;
-+ DPA_ASSERT(rcr->busy);
-+ DPA_ASSERT(rcr->pmode == bm_rcr_pvb);
-+ lwsync();
-+ rcursor = rcr->cursor;
-+ rcursor->__dont_write_directly__verb = myverb | rcr->vbit;
-+ dcbf_64(rcursor);
-+ RCR_INC(rcr);
-+ rcr->available--;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ rcr->busy = 0;
-+#endif
-+}
-+
-+static inline u8 bm_rcr_cci_update(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ u8 diff, old_ci = rcr->ci;
-+ DPA_ASSERT(rcr->cmode == bm_rcr_cci);
-+ rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
-+ diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
-+ rcr->available += diff;
-+ return diff;
-+}
-+
-+static inline void bm_rcr_cce_prefetch(struct bm_portal *portal)
-+{
-+ __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
-+ DPA_ASSERT(rcr->cmode == bm_rcr_cce);
-+ bm_cl_touch_ro(RCR_CI);
-+}
-+
-+static inline u8 bm_rcr_cce_update(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ u8 diff, old_ci = rcr->ci;
-+ DPA_ASSERT(rcr->cmode == bm_rcr_cce);
-+ rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1);
-+ bm_cl_invalidate(RCR_CI);
-+ diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
-+ rcr->available += diff;
-+ return diff;
-+}
-+
-+static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ return rcr->ithresh;
-+}
-+
-+static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ rcr->ithresh = ithresh;
-+ bm_out(RCR_ITR, ithresh);
-+}
-+
-+static inline u8 bm_rcr_get_avail(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ return rcr->available;
-+}
-+
-+static inline u8 bm_rcr_get_fill(struct bm_portal *portal)
-+{
-+ register struct bm_rcr *rcr = &portal->rcr;
-+ return BM_RCR_SIZE - 1 - rcr->available;
-+}
-+
-+
-+/* ------------------------------ */
-+/* --- Management command API --- */
-+
-+static inline int bm_mc_init(struct bm_portal *portal)
-+{
-+ register struct bm_mc *mc = &portal->mc;
-+ mc->cr = portal->addr.addr_ce + BM_CL_CR;
-+ mc->rr = portal->addr.addr_ce + BM_CL_RR0;
-+ mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
-+ BM_MCC_VERB_VBIT) ? 0 : 1;
-+ mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = mc_idle;
-+#endif
-+ return 0;
-+}
-+
-+static inline void bm_mc_finish(struct bm_portal *portal)
-+{
-+ __maybe_unused register struct bm_mc *mc = &portal->mc;
-+ DPA_ASSERT(mc->state == mc_idle);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (mc->state != mc_idle)
-+ pr_crit("Losing incomplete MC command\n");
-+#endif
-+}
-+
-+static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
-+{
-+ register struct bm_mc *mc = &portal->mc;
-+ DPA_ASSERT(mc->state == mc_idle);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = mc_user;
-+#endif
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(mc->cr);
-+#endif
-+ return mc->cr;
-+}
-+
-+static inline void bm_mc_abort(struct bm_portal *portal)
-+{
-+ __maybe_unused register struct bm_mc *mc = &portal->mc;
-+ DPA_ASSERT(mc->state == mc_user);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = mc_idle;
-+#endif
-+}
-+
-+static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
-+{
-+ register struct bm_mc *mc = &portal->mc;
-+ struct bm_mc_result *rr = mc->rr + mc->rridx;
-+ DPA_ASSERT(mc->state == mc_user);
-+ lwsync();
-+ mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
-+ dcbf(mc->cr);
-+ dcbit_ro(rr);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = mc_hw;
-+#endif
-+}
-+
-+static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
-+{
-+ register struct bm_mc *mc = &portal->mc;
-+ struct bm_mc_result *rr = mc->rr + mc->rridx;
-+ DPA_ASSERT(mc->state == mc_hw);
-+ /* The inactive response register's verb byte always returns zero until
-+ * its command is submitted and completed. This includes the valid-bit,
-+ * in case you were wondering... */
-+ if (!__raw_readb(&rr->verb)) {
-+ dcbit_ro(rr);
-+ return NULL;
-+ }
-+ mc->rridx ^= 1;
-+ mc->vbit ^= BM_MCC_VERB_VBIT;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = mc_idle;
-+#endif
-+ return rr;
-+}
-+
-+
-+/* ------------------------------------- */
-+/* --- Portal interrupt register API --- */
-+
-+static inline int bm_isr_init(__always_unused struct bm_portal *portal)
-+{
-+ return 0;
-+}
-+
-+static inline void bm_isr_finish(__always_unused struct bm_portal *portal)
-+{
-+}
-+
-+#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32)
-+#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
-+static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid,
-+ int enable)
-+{
-+ u32 val;
-+ DPA_ASSERT(bpid < bman_pool_max);
-+ /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
-+ val = __bm_in(&portal->addr, SCN_REG(bpid));
-+ if (enable)
-+ val |= SCN_BIT(bpid);
-+ else
-+ val &= ~SCN_BIT(bpid);
-+ __bm_out(&portal->addr, SCN_REG(bpid), val);
-+}
-+
-+static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
-+{
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ return __bm_in(&portal->addr, BM_REG_ISR + (n << 6));
-+#else
-+ return __bm_in(&portal->addr, BM_REG_ISR + (n << 2));
-+#endif
-+}
-+
-+static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n,
-+ u32 val)
-+{
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val);
-+#else
-+ __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val);
-+#endif
-+}
-+
-+/* Buffer Pool Cleanup */
-+static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid)
-+{
-+ struct bm_mc_command *bm_cmd;
-+ struct bm_mc_result *bm_res;
-+
-+ int aq_count = 0;
-+ bool stop = false;
-+ while (!stop) {
-+ /* Acquire buffers until empty */
-+ bm_cmd = bm_mc_start(p);
-+ bm_cmd->acquire.bpid = bpid;
-+ bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE | 1);
-+ while (!(bm_res = bm_mc_result(p)))
-+ cpu_relax();
-+ if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
-+ /* Pool is empty */
-+ /* TBD : Should we do a few extra iterations in
-+ case some other some blocks keep buffers 'on deck',
-+ which may also be problematic */
-+ stop = true;
-+ } else
-+ ++aq_count;
-+ }
-+ return 0;
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_private.h
-@@ -0,0 +1,166 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "dpa_sys.h"
-+#include <linux/fsl_bman.h>
-+
-+/* Revision info (for errata and feature handling) */
-+#define BMAN_REV10 0x0100
-+#define BMAN_REV20 0x0200
-+#define BMAN_REV21 0x0201
-+#define QBMAN_ANY_PORTAL_IDX 0xffffffff
-+extern u16 bman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */
-+
-+/*
-+ * Global variables of the max portal/pool number this bman version supported
-+ */
-+extern u16 bman_pool_max;
-+
-+/* used by CCSR and portal interrupt code */
-+enum bm_isr_reg {
-+ bm_isr_status = 0,
-+ bm_isr_enable = 1,
-+ bm_isr_disable = 2,
-+ bm_isr_inhibit = 3
-+};
-+
-+struct bm_portal_config {
-+ /* Corenet portal addresses;
-+ * [0]==cache-enabled, [1]==cache-inhibited. */
-+ __iomem void *addr_virt[2];
-+ struct resource addr_phys[2];
-+ /* Allow these to be joined in lists */
-+ struct list_head list;
-+ /* User-visible portal configuration settings */
-+ struct bman_portal_config public_cfg;
-+ /* power management saved data */
-+ u32 saved_isdr;
-+};
-+
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+/* Hooks from bman_driver.c to bman_config.c */
-+int bman_init_ccsr(struct device_node *node);
-+#endif
-+
-+/* Hooks from bman_driver.c in to bman_high.c */
-+struct bman_portal *bman_create_portal(
-+ struct bman_portal *portal,
-+ const struct bm_portal_config *config);
-+struct bman_portal *bman_create_affine_portal(
-+ const struct bm_portal_config *config);
-+struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect,
-+ int cpu);
-+void bman_destroy_portal(struct bman_portal *bm);
-+
-+const struct bm_portal_config *bman_destroy_affine_portal(void);
-+
-+/* Hooks from fsl_usdpaa.c to bman_driver.c */
-+struct bm_portal_config *bm_get_unused_portal(void);
-+struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx);
-+void bm_put_unused_portal(struct bm_portal_config *pcfg);
-+void bm_set_liodns(struct bm_portal_config *pcfg);
-+
-+/* Pool logic in the portal driver, during initialisation, needs to know if
-+ * there's access to CCSR or not (if not, it'll cripple the pool allocator). */
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+int bman_have_ccsr(void);
-+#else
-+#define bman_have_ccsr() 0
-+#endif
-+
-+/* Stockpile build constants. The _LOW value: when bman_acquire() is called and
-+ * the stockpile fill-level is <= _LOW, an acquire is attempted from h/w but it
-+ * might fail (if the buffer pool is depleted). So this value provides some
-+ * "stagger" in that the bman_acquire() function will only fail if lots of bufs
-+ * are requested at once or if h/w has been tested a couple of times without
-+ * luck. The _HIGH value: when bman_release() is called and the stockpile
-+ * fill-level is >= _HIGH, a release is attempted to h/w but it might fail (if
-+ * the release ring is full). So this value provides some "stagger" so that
-+ * ring-access is retried a couple of times prior to the API returning a
-+ * failure. The following *must* be true;
-+ * BMAN_STOCKPILE_HIGH-BMAN_STOCKPILE_LOW > 8
-+ * (to avoid thrashing)
-+ * BMAN_STOCKPILE_SZ >= 16
-+ * (as the release logic expects to either send 8 buffers to hw prior to
-+ * adding the given buffers to the stockpile or add the buffers to the
-+ * stockpile before sending 8 to hw, as the API must be an all-or-nothing
-+ * success/fail.)
-+ */
-+#define BMAN_STOCKPILE_SZ 16u /* number of bufs in per-pool cache */
-+#define BMAN_STOCKPILE_LOW 2u /* when fill is <= this, acquire from hw */
-+#define BMAN_STOCKPILE_HIGH 14u /* when fill is >= this, release to hw */
-+
-+/*************************************************/
-+/* BMan s/w corenet portal, low-level i/face */
-+/*************************************************/
-+
-+/* Used by all portal interrupt registers except 'inhibit'
-+ * This mask contains all the "irqsource" bits visible to API users
-+ */
-+#define BM_PIRQ_VISIBLE (BM_PIRQ_RCRI | BM_PIRQ_BSCN)
-+
-+/* These are bm_<reg>_<verb>(). So for example, bm_disable_write() means "write
-+ * the disable register" rather than "disable the ability to write". */
-+#define bm_isr_status_read(bm) __bm_isr_read(bm, bm_isr_status)
-+#define bm_isr_status_clear(bm, m) __bm_isr_write(bm, bm_isr_status, m)
-+#define bm_isr_enable_read(bm) __bm_isr_read(bm, bm_isr_enable)
-+#define bm_isr_enable_write(bm, v) __bm_isr_write(bm, bm_isr_enable, v)
-+#define bm_isr_disable_read(bm) __bm_isr_read(bm, bm_isr_disable)
-+#define bm_isr_disable_write(bm, v) __bm_isr_write(bm, bm_isr_disable, v)
-+#define bm_isr_inhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 1)
-+#define bm_isr_uninhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 0)
-+
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+/* Set depletion thresholds associated with a buffer pool. Requires that the
-+ * operating system have access to Bman CCSR (ie. compiled in support and
-+ * run-time access courtesy of the device-tree). */
-+int bm_pool_set(u32 bpid, const u32 *thresholds);
-+#define BM_POOL_THRESH_SW_ENTER 0
-+#define BM_POOL_THRESH_SW_EXIT 1
-+#define BM_POOL_THRESH_HW_ENTER 2
-+#define BM_POOL_THRESH_HW_EXIT 3
-+
-+/* Read the free buffer count for a given buffer */
-+u32 bm_pool_free_buffers(u32 bpid);
-+
-+__init int bman_init(void);
-+__init int bman_resource_init(void);
-+
-+const struct bm_portal_config *bman_get_bm_portal_config(
-+ struct bman_portal *portal);
-+
-+/* power management */
-+#ifdef CONFIG_SUSPEND
-+void suspend_unused_bportal(void);
-+void resume_unused_bportal(void);
-+#endif
-+
-+#endif /* CONFIG_FSL_BMAN_CONFIG */
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_test.c
-@@ -0,0 +1,56 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "bman_test.h"
-+
-+MODULE_AUTHOR("Geoff Thorpe");
-+MODULE_LICENSE("Dual BSD/GPL");
-+MODULE_DESCRIPTION("Bman testing");
-+
-+static int test_init(void)
-+{
-+#ifdef CONFIG_FSL_BMAN_TEST_HIGH
-+ int loop = 1;
-+ while (loop--)
-+ bman_test_high();
-+#endif
-+#ifdef CONFIG_FSL_BMAN_TEST_THRESH
-+ bman_test_thresh();
-+#endif
-+ return 0;
-+}
-+
-+static void test_exit(void)
-+{
-+}
-+
-+module_init(test_init);
-+module_exit(test_exit);
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_test.h
-@@ -0,0 +1,44 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include <linux/kernel.h>
-+#include <linux/errno.h>
-+#include <linux/io.h>
-+#include <linux/slab.h>
-+#include <linux/module.h>
-+#include <linux/interrupt.h>
-+#include <linux/delay.h>
-+#include <linux/kthread.h>
-+
-+#include <linux/fsl_bman.h>
-+
-+void bman_test_high(void);
-+void bman_test_thresh(void);
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_test_high.c
-@@ -0,0 +1,183 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "bman_test.h"
-+#include "bman_private.h"
-+
-+/*************/
-+/* constants */
-+/*************/
-+
-+#define PORTAL_OPAQUE ((void *)0xf00dbeef)
-+#define POOL_OPAQUE ((void *)0xdeadabba)
-+#define NUM_BUFS 93
-+#define LOOPS 3
-+#define BMAN_TOKEN_MASK 0x00FFFFFFFFFFLLU
-+
-+/***************/
-+/* global vars */
-+/***************/
-+
-+static struct bman_pool *pool;
-+static int depleted;
-+static struct bm_buffer bufs_in[NUM_BUFS] ____cacheline_aligned;
-+static struct bm_buffer bufs_out[NUM_BUFS] ____cacheline_aligned;
-+static int bufs_received;
-+
-+/* Predeclare the callback so we can instantiate pool parameters */
-+static void depletion_cb(struct bman_portal *, struct bman_pool *, void *, int);
-+
-+/**********************/
-+/* internal functions */
-+/**********************/
-+
-+static void bufs_init(void)
-+{
-+ int i;
-+ for (i = 0; i < NUM_BUFS; i++)
-+ bm_buffer_set64(&bufs_in[i], 0xfedc01234567LLU * i);
-+ bufs_received = 0;
-+}
-+
-+static inline int bufs_cmp(const struct bm_buffer *a, const struct bm_buffer *b)
-+{
-+ if ((bman_ip_rev == BMAN_REV20) || (bman_ip_rev == BMAN_REV21)) {
-+
-+ /* On SoCs with Bman revison 2.0, Bman only respects the 40
-+ * LS-bits of buffer addresses, masking off the upper 8-bits on
-+ * release commands. The API provides for 48-bit addresses
-+ * because some SoCs support all 48-bits. When generating
-+ * garbage addresses for testing, we either need to zero the
-+ * upper 8-bits when releasing to Bman (otherwise we'll be
-+ * disappointed when the buffers we acquire back from Bman
-+ * don't match), or we need to mask the upper 8-bits off when
-+ * comparing. We do the latter.
-+ */
-+ if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK)
-+ < (bm_buffer_get64(b) & BMAN_TOKEN_MASK))
-+ return -1;
-+ if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK)
-+ > (bm_buffer_get64(b) & BMAN_TOKEN_MASK))
-+ return 1;
-+ } else {
-+ if (bm_buffer_get64(a) < bm_buffer_get64(b))
-+ return -1;
-+ if (bm_buffer_get64(a) > bm_buffer_get64(b))
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+static void bufs_confirm(void)
-+{
-+ int i, j;
-+ for (i = 0; i < NUM_BUFS; i++) {
-+ int matches = 0;
-+ for (j = 0; j < NUM_BUFS; j++)
-+ if (!bufs_cmp(&bufs_in[i], &bufs_out[j]))
-+ matches++;
-+ BUG_ON(matches != 1);
-+ }
-+}
-+
-+/********/
-+/* test */
-+/********/
-+
-+static void depletion_cb(struct bman_portal *__portal, struct bman_pool *__pool,
-+ void *pool_ctx, int __depleted)
-+{
-+ BUG_ON(__pool != pool);
-+ BUG_ON(pool_ctx != POOL_OPAQUE);
-+ depleted = __depleted;
-+}
-+
-+void bman_test_high(void)
-+{
-+ struct bman_pool_params pparams = {
-+ .flags = BMAN_POOL_FLAG_DEPLETION | BMAN_POOL_FLAG_DYNAMIC_BPID,
-+ .cb = depletion_cb,
-+ .cb_ctx = POOL_OPAQUE,
-+ };
-+ int i, loops = LOOPS;
-+ struct bm_buffer tmp_buf;
-+
-+ bufs_init();
-+
-+ pr_info("BMAN: --- starting high-level test ---\n");
-+
-+ pool = bman_new_pool(&pparams);
-+ BUG_ON(!pool);
-+
-+ /*******************/
-+ /* Release buffers */
-+ /*******************/
-+do_loop:
-+ i = 0;
-+ while (i < NUM_BUFS) {
-+ u32 flags = BMAN_RELEASE_FLAG_WAIT;
-+ int num = 8;
-+ if ((i + num) > NUM_BUFS)
-+ num = NUM_BUFS - i;
-+ if ((i + num) == NUM_BUFS)
-+ flags |= BMAN_RELEASE_FLAG_WAIT_SYNC;
-+ if (bman_release(pool, bufs_in + i, num, flags))
-+ panic("bman_release() failed\n");
-+ i += num;
-+ }
-+
-+ /*******************/
-+ /* Acquire buffers */
-+ /*******************/
-+ while (i > 0) {
-+ int tmp, num = 8;
-+ if (num > i)
-+ num = i;
-+ tmp = bman_acquire(pool, bufs_out + i - num, num, 0);
-+ BUG_ON(tmp != num);
-+ i -= num;
-+ }
-+
-+ i = bman_acquire(pool, &tmp_buf, 1, 0);
-+ BUG_ON(i > 0);
-+
-+ bufs_confirm();
-+
-+ if (--loops)
-+ goto do_loop;
-+
-+ /************/
-+ /* Clean up */
-+ /************/
-+ bman_free_pool(pool);
-+ pr_info("BMAN: --- finished high-level test ---\n");
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/bman_test_thresh.c
-@@ -0,0 +1,196 @@
-+/* Copyright 2010-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "bman_test.h"
-+
-+/* Test constants */
-+#define TEST_NUMBUFS 129728
-+#define TEST_EXIT 129536
-+#define TEST_ENTRY 129024
-+
-+struct affine_test_data {
-+ struct task_struct *t;
-+ int cpu;
-+ int expect_affinity;
-+ int drain;
-+ int num_enter;
-+ int num_exit;
-+ struct list_head node;
-+ struct completion wakethread;
-+ struct completion wakeparent;
-+};
-+
-+static void cb_depletion(struct bman_portal *portal,
-+ struct bman_pool *pool,
-+ void *opaque,
-+ int depleted)
-+{
-+ struct affine_test_data *data = opaque;
-+ int c = smp_processor_id();
-+ pr_info("cb_depletion: bpid=%d, depleted=%d, cpu=%d, original=%d\n",
-+ bman_get_params(pool)->bpid, !!depleted, c, data->cpu);
-+ /* We should be executing on the CPU of the thread that owns the pool if
-+ * and that CPU has an affine portal (ie. it isn't slaved). */
-+ BUG_ON((c != data->cpu) && data->expect_affinity);
-+ BUG_ON((c == data->cpu) && !data->expect_affinity);
-+ if (depleted)
-+ data->num_enter++;
-+ else
-+ data->num_exit++;
-+}
-+
-+/* Params used to set up a pool, this also dynamically allocates a BPID */
-+static const struct bman_pool_params params_nocb = {
-+ .flags = BMAN_POOL_FLAG_DYNAMIC_BPID | BMAN_POOL_FLAG_THRESH,
-+ .thresholds = { TEST_ENTRY, TEST_EXIT, 0, 0 }
-+};
-+
-+/* Params used to set up each cpu's pool with callbacks enabled */
-+static struct bman_pool_params params_cb = {
-+ .bpid = 0, /* will be replaced to match pool_nocb */
-+ .flags = BMAN_POOL_FLAG_DEPLETION,
-+ .cb = cb_depletion
-+};
-+
-+static struct bman_pool *pool_nocb;
-+static LIST_HEAD(threads);
-+
-+static int affine_test(void *__data)
-+{
-+ struct bman_pool *pool;
-+ struct affine_test_data *data = __data;
-+ struct bman_pool_params my_params = params_cb;
-+
-+ pr_info("thread %d: starting\n", data->cpu);
-+ /* create the pool */
-+ my_params.cb_ctx = data;
-+ pool = bman_new_pool(&my_params);
-+ BUG_ON(!pool);
-+ complete(&data->wakeparent);
-+ wait_for_completion(&data->wakethread);
-+ init_completion(&data->wakethread);
-+
-+ /* if we're the drainer, we get signalled for that */
-+ if (data->drain) {
-+ struct bm_buffer buf;
-+ int ret;
-+ pr_info("thread %d: draining...\n", data->cpu);
-+ do {
-+ ret = bman_acquire(pool, &buf, 1, 0);
-+ } while (ret > 0);
-+ pr_info("thread %d: draining done.\n", data->cpu);
-+ complete(&data->wakeparent);
-+ wait_for_completion(&data->wakethread);
-+ init_completion(&data->wakethread);
-+ }
-+
-+ /* cleanup */
-+ bman_free_pool(pool);
-+ while (!kthread_should_stop())
-+ cpu_relax();
-+ pr_info("thread %d: exiting\n", data->cpu);
-+ return 0;
-+}
-+
-+static struct affine_test_data *start_affine_test(int cpu, int drain)
-+{
-+ struct affine_test_data *data = kmalloc(sizeof(*data), GFP_KERNEL);
-+
-+ if (!data)
-+ return NULL;
-+ data->cpu = cpu;
-+ data->expect_affinity = cpumask_test_cpu(cpu, bman_affine_cpus());
-+ data->drain = drain;
-+ data->num_enter = 0;
-+ data->num_exit = 0;
-+ init_completion(&data->wakethread);
-+ init_completion(&data->wakeparent);
-+ list_add_tail(&data->node, &threads);
-+ data->t = kthread_create(affine_test, data, "threshtest%d", cpu);
-+ BUG_ON(IS_ERR(data->t));
-+ kthread_bind(data->t, cpu);
-+ wake_up_process(data->t);
-+ return data;
-+}
-+
-+void bman_test_thresh(void)
-+{
-+ int loop = TEST_NUMBUFS;
-+ int ret, num_cpus = 0;
-+ struct affine_test_data *data, *drainer = NULL;
-+
-+ pr_info("bman_test_thresh: start\n");
-+
-+ /* allocate a BPID and seed it */
-+ pool_nocb = bman_new_pool(¶ms_nocb);
-+ BUG_ON(!pool_nocb);
-+ while (loop--) {
-+ struct bm_buffer buf;
-+ bm_buffer_set64(&buf, 0x0badbeef + loop);
-+ ret = bman_release(pool_nocb, &buf, 1,
-+ BMAN_RELEASE_FLAG_WAIT);
-+ BUG_ON(ret);
-+ }
-+ while (!bman_rcr_is_empty())
-+ cpu_relax();
-+ pr_info("bman_test_thresh: buffers are in\n");
-+
-+ /* create threads and wait for them to create pools */
-+ params_cb.bpid = bman_get_params(pool_nocb)->bpid;
-+ for_each_cpu(loop, cpu_online_mask) {
-+ data = start_affine_test(loop, drainer ? 0 : 1);
-+ BUG_ON(!data);
-+ if (!drainer)
-+ drainer = data;
-+ num_cpus++;
-+ wait_for_completion(&data->wakeparent);
-+ }
-+
-+ /* signal the drainer to start draining */
-+ complete(&drainer->wakethread);
-+ wait_for_completion(&drainer->wakeparent);
-+ init_completion(&drainer->wakeparent);
-+
-+ /* tear down */
-+ list_for_each_entry_safe(data, drainer, &threads, node) {
-+ complete(&data->wakethread);
-+ ret = kthread_stop(data->t);
-+ BUG_ON(ret);
-+ list_del(&data->node);
-+ /* check that we get the expected callbacks (and no others) */
-+ BUG_ON(data->num_enter != 1);
-+ BUG_ON(data->num_exit != 0);
-+ kfree(data);
-+ }
-+ bman_free_pool(pool_nocb);
-+
-+ pr_info("bman_test_thresh: done\n");
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/dpa_alloc.c
-@@ -0,0 +1,706 @@
-+/* Copyright 2009-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "dpa_sys.h"
-+#include <linux/fsl_qman.h>
-+#include <linux/fsl_bman.h>
-+
-+/* Qman and Bman APIs are front-ends to the common code; */
-+
-+static DECLARE_DPA_ALLOC(bpalloc); /* BPID allocator */
-+static DECLARE_DPA_ALLOC(fqalloc); /* FQID allocator */
-+static DECLARE_DPA_ALLOC(qpalloc); /* pool-channel allocator */
-+static DECLARE_DPA_ALLOC(cgralloc); /* CGR ID allocator */
-+static DECLARE_DPA_ALLOC(ceetm0_challoc); /* CEETM Channel ID allocator */
-+static DECLARE_DPA_ALLOC(ceetm0_lfqidalloc); /* CEETM LFQID allocator */
-+static DECLARE_DPA_ALLOC(ceetm1_challoc); /* CEETM Channel ID allocator */
-+static DECLARE_DPA_ALLOC(ceetm1_lfqidalloc); /* CEETM LFQID allocator */
-+
-+/* This is a sort-of-conditional dpa_alloc_free() routine. Eg. when releasing
-+ * FQIDs (probably from user-space), it can filter out those that aren't in the
-+ * OOS state (better to leak a h/w resource than to crash). This function
-+ * returns the number of invalid IDs that were not released. */
-+static u32 release_id_range(struct dpa_alloc *alloc, u32 id, u32 count,
-+ int (*is_valid)(u32 id))
-+{
-+ int valid_mode = 0;
-+ u32 loop = id, total_invalid = 0;
-+ while (loop < (id + count)) {
-+ int isvalid = is_valid ? is_valid(loop) : 1;
-+ if (!valid_mode) {
-+ /* We're looking for a valid ID to terminate an invalid
-+ * range */
-+ if (isvalid) {
-+ /* We finished a range of invalid IDs, a valid
-+ * range is now underway */
-+ valid_mode = 1;
-+ count -= (loop - id);
-+ id = loop;
-+ } else
-+ total_invalid++;
-+ } else {
-+ /* We're looking for an invalid ID to terminate a
-+ * valid range */
-+ if (!isvalid) {
-+ /* Release the range of valid IDs, an unvalid
-+ * range is now underway */
-+ if (loop > id)
-+ dpa_alloc_free(alloc, id, loop - id);
-+ valid_mode = 0;
-+ }
-+ }
-+ loop++;
-+ }
-+ /* Release any unterminated range of valid IDs */
-+ if (valid_mode && count)
-+ dpa_alloc_free(alloc, id, count);
-+ return total_invalid;
-+}
-+
-+/* BPID allocator front-end */
-+
-+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial)
-+{
-+ return dpa_alloc_new(&bpalloc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(bman_alloc_bpid_range);
-+
-+static int bp_cleanup(u32 bpid)
-+{
-+ return bman_shutdown_pool(bpid) == 0;
-+}
-+void bman_release_bpid_range(u32 bpid, u32 count)
-+{
-+ u32 total_invalid = release_id_range(&bpalloc, bpid, count, bp_cleanup);
-+ if (total_invalid)
-+ pr_err("BPID range [%d..%d] (%d) had %d leaks\n",
-+ bpid, bpid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(bman_release_bpid_range);
-+
-+void bman_seed_bpid_range(u32 bpid, u32 count)
-+{
-+ dpa_alloc_seed(&bpalloc, bpid, count);
-+}
-+EXPORT_SYMBOL(bman_seed_bpid_range);
-+
-+int bman_reserve_bpid_range(u32 bpid, u32 count)
-+{
-+ return dpa_alloc_reserve(&bpalloc, bpid, count);
-+}
-+EXPORT_SYMBOL(bman_reserve_bpid_range);
-+
-+
-+/* FQID allocator front-end */
-+
-+int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial)
-+{
-+ return dpa_alloc_new(&fqalloc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_fqid_range);
-+
-+static int fq_cleanup(u32 fqid)
-+{
-+ return qman_shutdown_fq(fqid) == 0;
-+}
-+void qman_release_fqid_range(u32 fqid, u32 count)
-+{
-+ u32 total_invalid = release_id_range(&fqalloc, fqid, count, fq_cleanup);
-+ if (total_invalid)
-+ pr_err("FQID range [%d..%d] (%d) had %d leaks\n",
-+ fqid, fqid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(qman_release_fqid_range);
-+
-+int qman_reserve_fqid_range(u32 fqid, u32 count)
-+{
-+ return dpa_alloc_reserve(&fqalloc, fqid, count);
-+}
-+EXPORT_SYMBOL(qman_reserve_fqid_range);
-+
-+void qman_seed_fqid_range(u32 fqid, u32 count)
-+{
-+ dpa_alloc_seed(&fqalloc, fqid, count);
-+}
-+EXPORT_SYMBOL(qman_seed_fqid_range);
-+
-+/* Pool-channel allocator front-end */
-+
-+int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial)
-+{
-+ return dpa_alloc_new(&qpalloc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_pool_range);
-+
-+static int qpool_cleanup(u32 qp)
-+{
-+ /* We query all FQDs starting from
-+ * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
-+ * whose destination channel is the pool-channel being released.
-+ * When a non-OOS FQD is found we attempt to clean it up */
-+ struct qman_fq fq = {
-+ .fqid = 1
-+ };
-+ int err;
-+ do {
-+ struct qm_mcr_queryfq_np np;
-+ err = qman_query_fq_np(&fq, &np);
-+ if (err)
-+ /* FQID range exceeded, found no problems */
-+ return 1;
-+ if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
-+ struct qm_fqd fqd;
-+ err = qman_query_fq(&fq, &fqd);
-+ BUG_ON(err);
-+ if (fqd.dest.channel == qp) {
-+ /* The channel is the FQ's target, clean it */
-+ if (qman_shutdown_fq(fq.fqid) != 0)
-+ /* Couldn't shut down the FQ
-+ so the pool must be leaked */
-+ return 0;
-+ }
-+ }
-+ /* Move to the next FQID */
-+ fq.fqid++;
-+ } while (1);
-+}
-+void qman_release_pool_range(u32 qp, u32 count)
-+{
-+ u32 total_invalid = release_id_range(&qpalloc, qp,
-+ count, qpool_cleanup);
-+ if (total_invalid) {
-+ /* Pool channels are almost always used individually */
-+ if (count == 1)
-+ pr_err("Pool channel 0x%x had %d leaks\n",
-+ qp, total_invalid);
-+ else
-+ pr_err("Pool channels [%d..%d] (%d) had %d leaks\n",
-+ qp, qp + count - 1, count, total_invalid);
-+ }
-+}
-+EXPORT_SYMBOL(qman_release_pool_range);
-+
-+
-+void qman_seed_pool_range(u32 poolid, u32 count)
-+{
-+ dpa_alloc_seed(&qpalloc, poolid, count);
-+
-+}
-+EXPORT_SYMBOL(qman_seed_pool_range);
-+
-+int qman_reserve_pool_range(u32 poolid, u32 count)
-+{
-+ return dpa_alloc_reserve(&qpalloc, poolid, count);
-+}
-+EXPORT_SYMBOL(qman_reserve_pool_range);
-+
-+
-+/* CGR ID allocator front-end */
-+
-+int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial)
-+{
-+ return dpa_alloc_new(&cgralloc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_cgrid_range);
-+
-+static int cqr_cleanup(u32 cgrid)
-+{
-+ /* We query all FQDs starting from
-+ * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
-+ * whose CGR is the CGR being released.
-+ */
-+ struct qman_fq fq = {
-+ .fqid = 1
-+ };
-+ int err;
-+ do {
-+ struct qm_mcr_queryfq_np np;
-+ err = qman_query_fq_np(&fq, &np);
-+ if (err)
-+ /* FQID range exceeded, found no problems */
-+ return 1;
-+ if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
-+ struct qm_fqd fqd;
-+ err = qman_query_fq(&fq, &fqd);
-+ BUG_ON(err);
-+ if ((fqd.fq_ctrl & QM_FQCTRL_CGE) &&
-+ (fqd.cgid == cgrid)) {
-+ pr_err("CRGID 0x%x is being used by FQID 0x%x,"
-+ " CGR will be leaked\n",
-+ cgrid, fq.fqid);
-+ return 1;
-+ }
-+ }
-+ /* Move to the next FQID */
-+ fq.fqid++;
-+ } while (1);
-+}
-+
-+void qman_release_cgrid_range(u32 cgrid, u32 count)
-+{
-+ u32 total_invalid = release_id_range(&cgralloc, cgrid,
-+ count, cqr_cleanup);
-+ if (total_invalid)
-+ pr_err("CGRID range [%d..%d] (%d) had %d leaks\n",
-+ cgrid, cgrid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(qman_release_cgrid_range);
-+
-+void qman_seed_cgrid_range(u32 cgrid, u32 count)
-+{
-+ dpa_alloc_seed(&cgralloc, cgrid, count);
-+
-+}
-+EXPORT_SYMBOL(qman_seed_cgrid_range);
-+
-+/* CEETM CHANNEL ID allocator front-end */
-+int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align,
-+ int partial)
-+{
-+ return dpa_alloc_new(&ceetm0_challoc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_ceetm0_channel_range);
-+
-+int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align,
-+ int partial)
-+{
-+ return dpa_alloc_new(&ceetm1_challoc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_ceetm1_channel_range);
-+
-+void qman_release_ceetm0_channel_range(u32 channelid, u32 count)
-+{
-+ u32 total_invalid;
-+
-+ total_invalid = release_id_range(&ceetm0_challoc, channelid, count,
-+ NULL);
-+ if (total_invalid)
-+ pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n",
-+ channelid, channelid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(qman_release_ceetm0_channel_range);
-+
-+void qman_seed_ceetm0_channel_range(u32 channelid, u32 count)
-+{
-+ dpa_alloc_seed(&ceetm0_challoc, channelid, count);
-+
-+}
-+EXPORT_SYMBOL(qman_seed_ceetm0_channel_range);
-+
-+void qman_release_ceetm1_channel_range(u32 channelid, u32 count)
-+{
-+ u32 total_invalid;
-+ total_invalid = release_id_range(&ceetm1_challoc, channelid, count,
-+ NULL);
-+ if (total_invalid)
-+ pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n",
-+ channelid, channelid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(qman_release_ceetm1_channel_range);
-+
-+void qman_seed_ceetm1_channel_range(u32 channelid, u32 count)
-+{
-+ dpa_alloc_seed(&ceetm1_challoc, channelid, count);
-+
-+}
-+EXPORT_SYMBOL(qman_seed_ceetm1_channel_range);
-+
-+/* CEETM LFQID allocator front-end */
-+int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align,
-+ int partial)
-+{
-+ return dpa_alloc_new(&ceetm0_lfqidalloc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_ceetm0_lfqid_range);
-+
-+int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align,
-+ int partial)
-+{
-+ return dpa_alloc_new(&ceetm1_lfqidalloc, result, count, align, partial);
-+}
-+EXPORT_SYMBOL(qman_alloc_ceetm1_lfqid_range);
-+
-+void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count)
-+{
-+ u32 total_invalid;
-+
-+ total_invalid = release_id_range(&ceetm0_lfqidalloc, lfqid, count,
-+ NULL);
-+ if (total_invalid)
-+ pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n",
-+ lfqid, lfqid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(qman_release_ceetm0_lfqid_range);
-+
-+void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count)
-+{
-+ dpa_alloc_seed(&ceetm0_lfqidalloc, lfqid, count);
-+
-+}
-+EXPORT_SYMBOL(qman_seed_ceetm0_lfqid_range);
-+
-+void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count)
-+{
-+ u32 total_invalid;
-+
-+ total_invalid = release_id_range(&ceetm1_lfqidalloc, lfqid, count,
-+ NULL);
-+ if (total_invalid)
-+ pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n",
-+ lfqid, lfqid + count - 1, count, total_invalid);
-+}
-+EXPORT_SYMBOL(qman_release_ceetm1_lfqid_range);
-+
-+void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count)
-+{
-+ dpa_alloc_seed(&ceetm1_lfqidalloc, lfqid, count);
-+
-+}
-+EXPORT_SYMBOL(qman_seed_ceetm1_lfqid_range);
-+
-+
-+/* Everything else is the common backend to all the allocators */
-+
-+/* The allocator is a (possibly-empty) list of these; */
-+struct alloc_node {
-+ struct list_head list;
-+ u32 base;
-+ u32 num;
-+ /* refcount and is_alloced are only set
-+ when the node is in the used list */
-+ unsigned int refcount;
-+ int is_alloced;
-+};
-+
-+/* #define DPA_ALLOC_DEBUG */
-+
-+#ifdef DPA_ALLOC_DEBUG
-+#define DPRINT pr_info
-+static void DUMP(struct dpa_alloc *alloc)
-+{
-+ int off = 0;
-+ char buf[256];
-+ struct alloc_node *p;
-+ pr_info("Free Nodes\n");
-+ list_for_each_entry(p, &alloc->free, list) {
-+ if (off < 255)
-+ off += snprintf(buf + off, 255-off, "{%d,%d}",
-+ p->base, p->base + p->num - 1);
-+ }
-+ pr_info("%s\n", buf);
-+
-+ off = 0;
-+ pr_info("Used Nodes\n");
-+ list_for_each_entry(p, &alloc->used, list) {
-+ if (off < 255)
-+ off += snprintf(buf + off, 255-off, "{%d,%d}",
-+ p->base, p->base + p->num - 1);
-+ }
-+ pr_info("%s\n", buf);
-+
-+
-+
-+}
-+#else
-+#define DPRINT(x...)
-+#define DUMP(a)
-+#endif
-+
-+int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align,
-+ int partial)
-+{
-+ struct alloc_node *i = NULL, *next_best = NULL, *used_node = NULL;
-+ u32 base, next_best_base = 0, num = 0, next_best_num = 0;
-+ struct alloc_node *margin_left, *margin_right;
-+
-+ *result = (u32)-1;
-+ DPRINT("alloc_range(%d,%d,%d)\n", count, align, partial);
-+ DUMP(alloc);
-+ /* If 'align' is 0, it should behave as though it was 1 */
-+ if (!align)
-+ align = 1;
-+ margin_left = kmalloc(sizeof(*margin_left), GFP_KERNEL);
-+ if (!margin_left)
-+ goto err;
-+ margin_right = kmalloc(sizeof(*margin_right), GFP_KERNEL);
-+ if (!margin_right) {
-+ kfree(margin_left);
-+ goto err;
-+ }
-+ spin_lock_irq(&alloc->lock);
-+ list_for_each_entry(i, &alloc->free, list) {
-+ base = (i->base + align - 1) / align;
-+ base *= align;
-+ if ((base - i->base) >= i->num)
-+ /* alignment is impossible, regardless of count */
-+ continue;
-+ num = i->num - (base - i->base);
-+ if (num >= count) {
-+ /* this one will do nicely */
-+ num = count;
-+ goto done;
-+ }
-+ if (num > next_best_num) {
-+ next_best = i;
-+ next_best_base = base;
-+ next_best_num = num;
-+ }
-+ }
-+ if (partial && next_best) {
-+ i = next_best;
-+ base = next_best_base;
-+ num = next_best_num;
-+ } else
-+ i = NULL;
-+done:
-+ if (i) {
-+ if (base != i->base) {
-+ margin_left->base = i->base;
-+ margin_left->num = base - i->base;
-+ list_add_tail(&margin_left->list, &i->list);
-+ } else
-+ kfree(margin_left);
-+ if ((base + num) < (i->base + i->num)) {
-+ margin_right->base = base + num;
-+ margin_right->num = (i->base + i->num) -
-+ (base + num);
-+ list_add(&margin_right->list, &i->list);
-+ } else
-+ kfree(margin_right);
-+ list_del(&i->list);
-+ kfree(i);
-+ *result = base;
-+ } else {
-+ spin_unlock_irq(&alloc->lock);
-+ kfree(margin_left);
-+ kfree(margin_right);
-+ }
-+
-+err:
-+ DPRINT("returning %d\n", i ? num : -ENOMEM);
-+ DUMP(alloc);
-+ if (!i)
-+ return -ENOMEM;
-+
-+ /* Add the allocation to the used list with a refcount of 1 */
-+ used_node = kmalloc(sizeof(*used_node), GFP_KERNEL);
-+ if (!used_node) {
-+ spin_unlock_irq(&alloc->lock);
-+ return -ENOMEM;
-+ }
-+ used_node->base = *result;
-+ used_node->num = num;
-+ used_node->refcount = 1;
-+ used_node->is_alloced = 1;
-+ list_add_tail(&used_node->list, &alloc->used);
-+ spin_unlock_irq(&alloc->lock);
-+ return (int)num;
-+}
-+
-+/* Allocate the list node using GFP_ATOMIC, because we *really* want to avoid
-+ * forcing error-handling on to users in the deallocation path. */
-+static void _dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count)
-+{
-+ struct alloc_node *i, *node = kmalloc(sizeof(*node), GFP_ATOMIC);
-+ BUG_ON(!node);
-+ DPRINT("release_range(%d,%d)\n", base_id, count);
-+ DUMP(alloc);
-+ BUG_ON(!count);
-+ spin_lock_irq(&alloc->lock);
-+
-+
-+ node->base = base_id;
-+ node->num = count;
-+ list_for_each_entry(i, &alloc->free, list) {
-+ if (i->base >= node->base) {
-+ /* BUG_ON(any overlapping) */
-+ BUG_ON(i->base < (node->base + node->num));
-+ list_add_tail(&node->list, &i->list);
-+ goto done;
-+ }
-+ }
-+ list_add_tail(&node->list, &alloc->free);
-+done:
-+ /* Merge to the left */
-+ i = list_entry(node->list.prev, struct alloc_node, list);
-+ if (node->list.prev != &alloc->free) {
-+ BUG_ON((i->base + i->num) > node->base);
-+ if ((i->base + i->num) == node->base) {
-+ node->base = i->base;
-+ node->num += i->num;
-+ list_del(&i->list);
-+ kfree(i);
-+ }
-+ }
-+ /* Merge to the right */
-+ i = list_entry(node->list.next, struct alloc_node, list);
-+ if (node->list.next != &alloc->free) {
-+ BUG_ON((node->base + node->num) > i->base);
-+ if ((node->base + node->num) == i->base) {
-+ node->num += i->num;
-+ list_del(&i->list);
-+ kfree(i);
-+ }
-+ }
-+ spin_unlock_irq(&alloc->lock);
-+ DUMP(alloc);
-+}
-+
-+
-+void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count)
-+{
-+ struct alloc_node *i = NULL;
-+ spin_lock_irq(&alloc->lock);
-+
-+ /* First find the node in the used list and decrement its ref count */
-+ list_for_each_entry(i, &alloc->used, list) {
-+ if (i->base == base_id && i->num == count) {
-+ --i->refcount;
-+ if (i->refcount == 0) {
-+ list_del(&i->list);
-+ spin_unlock_irq(&alloc->lock);
-+ if (i->is_alloced)
-+ _dpa_alloc_free(alloc, base_id, count);
-+ kfree(i);
-+ return;
-+ }
-+ spin_unlock_irq(&alloc->lock);
-+ return;
-+ }
-+ }
-+ /* Couldn't find the allocation */
-+ pr_err("Attempt to free ID 0x%x COUNT %d that wasn't alloc'd or reserved\n",
-+ base_id, count);
-+ spin_unlock_irq(&alloc->lock);
-+}
-+
-+void dpa_alloc_seed(struct dpa_alloc *alloc, u32 base_id, u32 count)
-+{
-+ /* Same as free but no previous allocation checking is needed */
-+ _dpa_alloc_free(alloc, base_id, count);
-+}
-+
-+
-+int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base, u32 num)
-+{
-+ struct alloc_node *i = NULL, *used_node;
-+
-+ DPRINT("alloc_reserve(%d,%d)\n", base, num);
-+ DUMP(alloc);
-+
-+ spin_lock_irq(&alloc->lock);
-+
-+ /* Check for the node in the used list.
-+ If found, increase it's refcount */
-+ list_for_each_entry(i, &alloc->used, list) {
-+ if ((i->base == base) && (i->num == num)) {
-+ ++i->refcount;
-+ spin_unlock_irq(&alloc->lock);
-+ return 0;
-+ }
-+ if ((base >= i->base) && (base < (i->base + i->num))) {
-+ /* This is an attempt to reserve a region that was
-+ already reserved or alloced with a different
-+ base or num */
-+ pr_err("Cannot reserve %d - %d, it overlaps with"
-+ " existing reservation from %d - %d\n",
-+ base, base + num - 1, i->base,
-+ i->base + i->num - 1);
-+ spin_unlock_irq(&alloc->lock);
-+ return -1;
-+ }
-+ }
-+ /* Check to make sure this ID isn't in the free list */
-+ list_for_each_entry(i, &alloc->free, list) {
-+ if ((base >= i->base) && (base < (i->base + i->num))) {
-+ /* yep, the reservation is within this node */
-+ pr_err("Cannot reserve %d - %d, it overlaps with"
-+ " free range %d - %d and must be alloced\n",
-+ base, base + num - 1,
-+ i->base, i->base + i->num - 1);
-+ spin_unlock_irq(&alloc->lock);
-+ return -1;
-+ }
-+ }
-+ /* Add the allocation to the used list with a refcount of 1 */
-+ used_node = kmalloc(sizeof(*used_node), GFP_KERNEL);
-+ if (!used_node) {
-+ spin_unlock_irq(&alloc->lock);
-+ return -ENOMEM;
-+
-+ }
-+ used_node->base = base;
-+ used_node->num = num;
-+ used_node->refcount = 1;
-+ used_node->is_alloced = 0;
-+ list_add_tail(&used_node->list, &alloc->used);
-+ spin_unlock_irq(&alloc->lock);
-+ return 0;
-+}
-+
-+
-+int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count)
-+{
-+ struct alloc_node *i = NULL;
-+ DPRINT("alloc_pop()\n");
-+ DUMP(alloc);
-+ spin_lock_irq(&alloc->lock);
-+ if (!list_empty(&alloc->free)) {
-+ i = list_entry(alloc->free.next, struct alloc_node, list);
-+ list_del(&i->list);
-+ }
-+ spin_unlock_irq(&alloc->lock);
-+ DPRINT("returning %d\n", i ? 0 : -ENOMEM);
-+ DUMP(alloc);
-+ if (!i)
-+ return -ENOMEM;
-+ *result = i->base;
-+ *count = i->num;
-+ kfree(i);
-+ return 0;
-+}
-+
-+int dpa_alloc_check(struct dpa_alloc *list_head, u32 item)
-+{
-+ struct alloc_node *i = NULL;
-+ int res = 0;
-+ DPRINT("alloc_check()\n");
-+ spin_lock_irq(&list_head->lock);
-+
-+ list_for_each_entry(i, &list_head->free, list) {
-+ if ((item >= i->base) && (item < (i->base + i->num))) {
-+ res = 1;
-+ break;
-+ }
-+ }
-+ spin_unlock_irq(&list_head->lock);
-+ return res;
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/dpa_sys.h
-@@ -0,0 +1,259 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef DPA_SYS_H
-+#define DPA_SYS_H
-+
-+#include <linux/kernel.h>
-+#include <linux/errno.h>
-+#include <linux/io.h>
-+#include <linux/dma-mapping.h>
-+#include <linux/bootmem.h>
-+#include <linux/slab.h>
-+#include <linux/module.h>
-+#include <linux/init.h>
-+#include <linux/interrupt.h>
-+#include <linux/delay.h>
-+#include <linux/of_platform.h>
-+#include <linux/of_address.h>
-+#include <linux/of_irq.h>
-+#include <linux/kthread.h>
-+#include <linux/memblock.h>
-+#include <linux/completion.h>
-+#include <linux/log2.h>
-+#include <linux/types.h>
-+#include <linux/ioctl.h>
-+#include <linux/miscdevice.h>
-+#include <linux/uaccess.h>
-+#include <linux/debugfs.h>
-+#include <linux/seq_file.h>
-+#include <linux/device.h>
-+#include <linux/uio_driver.h>
-+#include <linux/smp.h>
-+#include <linux/fsl_hypervisor.h>
-+#include <linux/vmalloc.h>
-+#include <linux/ctype.h>
-+#include <linux/math64.h>
-+#include <linux/bitops.h>
-+
-+#include <linux/fsl_usdpaa.h>
-+
-+/* When copying aligned words or shorts, try to avoid memcpy() */
-+#define CONFIG_TRY_BETTER_MEMCPY
-+
-+/* For 2-element tables related to cache-inhibited and cache-enabled mappings */
-+#define DPA_PORTAL_CE 0
-+#define DPA_PORTAL_CI 1
-+
-+/***********************/
-+/* Misc inline assists */
-+/***********************/
-+
-+#if defined CONFIG_PPC32
-+#include "dpa_sys_ppc32.h"
-+#elif defined CONFIG_PPC64
-+#include "dpa_sys_ppc64.h"
-+#elif defined CONFIG_ARM
-+#include "dpa_sys_arm.h"
-+#elif defined CONFIG_ARM64
-+#include "dpa_sys_arm64.h"
-+#endif
-+
-+
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+#define DPA_ASSERT(x) \
-+ do { \
-+ if (!(x)) { \
-+ pr_crit("ASSERT: (%s:%d) %s\n", __FILE__, __LINE__, \
-+ __stringify_1(x)); \
-+ dump_stack(); \
-+ panic("assertion failure"); \
-+ } \
-+ } while (0)
-+#else
-+#define DPA_ASSERT(x)
-+#endif
-+
-+/* memcpy() stuff - when you know alignments in advance */
-+#ifdef CONFIG_TRY_BETTER_MEMCPY
-+static inline void copy_words(void *dest, const void *src, size_t sz)
-+{
-+ u32 *__dest = dest;
-+ const u32 *__src = src;
-+ size_t __sz = sz >> 2;
-+ BUG_ON((unsigned long)dest & 0x3);
-+ BUG_ON((unsigned long)src & 0x3);
-+ BUG_ON(sz & 0x3);
-+ while (__sz--)
-+ *(__dest++) = *(__src++);
-+}
-+static inline void copy_shorts(void *dest, const void *src, size_t sz)
-+{
-+ u16 *__dest = dest;
-+ const u16 *__src = src;
-+ size_t __sz = sz >> 1;
-+ BUG_ON((unsigned long)dest & 0x1);
-+ BUG_ON((unsigned long)src & 0x1);
-+ BUG_ON(sz & 0x1);
-+ while (__sz--)
-+ *(__dest++) = *(__src++);
-+}
-+static inline void copy_bytes(void *dest, const void *src, size_t sz)
-+{
-+ u8 *__dest = dest;
-+ const u8 *__src = src;
-+ while (sz--)
-+ *(__dest++) = *(__src++);
-+}
-+#else
-+#define copy_words memcpy
-+#define copy_shorts memcpy
-+#define copy_bytes memcpy
-+#endif
-+
-+/************/
-+/* RB-trees */
-+/************/
-+
-+/* We encapsulate RB-trees so that its easier to use non-linux forms in
-+ * non-linux systems. This also encapsulates the extra plumbing that linux code
-+ * usually provides when using RB-trees. This encapsulation assumes that the
-+ * data type held by the tree is u32. */
-+
-+struct dpa_rbtree {
-+ struct rb_root root;
-+};
-+#define DPA_RBTREE { .root = RB_ROOT }
-+
-+static inline void dpa_rbtree_init(struct dpa_rbtree *tree)
-+{
-+ tree->root = RB_ROOT;
-+}
-+
-+#define IMPLEMENT_DPA_RBTREE(name, type, node_field, val_field) \
-+static inline int name##_push(struct dpa_rbtree *tree, type *obj) \
-+{ \
-+ struct rb_node *parent = NULL, **p = &tree->root.rb_node; \
-+ while (*p) { \
-+ u32 item; \
-+ parent = *p; \
-+ item = rb_entry(parent, type, node_field)->val_field; \
-+ if (obj->val_field < item) \
-+ p = &parent->rb_left; \
-+ else if (obj->val_field > item) \
-+ p = &parent->rb_right; \
-+ else \
-+ return -EBUSY; \
-+ } \
-+ rb_link_node(&obj->node_field, parent, p); \
-+ rb_insert_color(&obj->node_field, &tree->root); \
-+ return 0; \
-+} \
-+static inline void name##_del(struct dpa_rbtree *tree, type *obj) \
-+{ \
-+ rb_erase(&obj->node_field, &tree->root); \
-+} \
-+static inline type *name##_find(struct dpa_rbtree *tree, u32 val) \
-+{ \
-+ type *ret; \
-+ struct rb_node *p = tree->root.rb_node; \
-+ while (p) { \
-+ ret = rb_entry(p, type, node_field); \
-+ if (val < ret->val_field) \
-+ p = p->rb_left; \
-+ else if (val > ret->val_field) \
-+ p = p->rb_right; \
-+ else \
-+ return ret; \
-+ } \
-+ return NULL; \
-+}
-+
-+/************/
-+/* Bootargs */
-+/************/
-+
-+/* Qman has "qportals=" and Bman has "bportals=", they use the same syntax
-+ * though; a comma-separated list of items, each item being a cpu index and/or a
-+ * range of cpu indices, and each item optionally be prefixed by "s" to indicate
-+ * that the portal associated with that cpu should be shared. See bman_driver.c
-+ * for more specifics. */
-+static int __parse_portals_cpu(const char **s, unsigned int *cpu)
-+{
-+ *cpu = 0;
-+ if (!isdigit(**s))
-+ return -EINVAL;
-+ while (isdigit(**s))
-+ *cpu = *cpu * 10 + (*((*s)++) - '0');
-+ return 0;
-+}
-+static inline int parse_portals_bootarg(char *str, struct cpumask *want_shared,
-+ struct cpumask *want_unshared,
-+ const char *argname)
-+{
-+ const char *s = str;
-+ unsigned int shared, cpu1, cpu2, loop;
-+
-+keep_going:
-+ if (*s == 's') {
-+ shared = 1;
-+ s++;
-+ } else
-+ shared = 0;
-+ if (__parse_portals_cpu(&s, &cpu1))
-+ goto err;
-+ if (*s == '-') {
-+ s++;
-+ if (__parse_portals_cpu(&s, &cpu2))
-+ goto err;
-+ if (cpu2 < cpu1)
-+ goto err;
-+ } else
-+ cpu2 = cpu1;
-+ for (loop = cpu1; loop <= cpu2; loop++)
-+ cpumask_set_cpu(loop, shared ? want_shared : want_unshared);
-+ if (*s == ',') {
-+ s++;
-+ goto keep_going;
-+ } else if ((*s == '\0') || isspace(*s))
-+ return 0;
-+err:
-+ pr_crit("Malformed %s argument: %s, offset: %lu\n", argname, str,
-+ (unsigned long)s - (unsigned long)str);
-+ return -EINVAL;
-+}
-+#ifdef CONFIG_FSL_USDPAA
-+/* Hooks from fsl_usdpaa_irq.c to fsl_usdpaa.c */
-+int usdpaa_get_portal_config(struct file *filp, void *cinh,
-+ enum usdpaa_portal_type ptype, unsigned int *irq,
-+ void **iir_reg);
-+#endif
-+#endif /* DPA_SYS_H */
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/dpa_sys_arm.h
-@@ -0,0 +1,95 @@
-+/* Copyright 2016 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef DPA_SYS_ARM_H
-+#define DPA_SYS_ARM_H
-+
-+#include <asm/cacheflush.h>
-+#include <asm/barrier.h>
-+
-+/* Implementation of ARM specific routines */
-+
-+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
-+ * barriers and that dcb*() won't fall victim to compiler or execution
-+ * reordering with respect to other code/instructions that manipulate the same
-+ * cacheline. */
-+#define hwsync() { asm volatile("dmb st" : : : "memory"); }
-+#define lwsync() { asm volatile("dmb st" : : : "memory"); }
-+#define dcbf(p) { asm volatile("mcr p15, 0, %0, c7, c10, 1" : : "r" (p) : "memory"); }
-+#define dcbt_ro(p) { asm volatile("pld [%0, #64];": : "r" (p)); }
-+#define dcbt_rw(p) { asm volatile("pldw [%0, #64];": : "r" (p)); }
-+#define dcbi(p) { asm volatile("mcr p15, 0, %0, c7, c6, 1" : : "r" (p) : "memory"); }
-+
-+#define dcbz_64(p) { memset(p, 0, sizeof(*p)); }
-+
-+#define dcbf_64(p) \
-+ do { \
-+ dcbf((u32)p); \
-+ } while (0)
-+/* Commonly used combo */
-+#define dcbit_ro(p) \
-+ do { \
-+ dcbi((u32)p); \
-+ dcbt_ro((u32)p); \
-+ } while (0)
-+
-+static inline u64 mfatb(void)
-+{
-+ return get_cycles();
-+}
-+
-+static inline u32 in_be32(volatile void *addr)
-+{
-+ return be32_to_cpu(*((volatile u32 *) addr));
-+}
-+
-+static inline void out_be32(void *addr, u32 val)
-+{
-+ *((u32 *) addr) = cpu_to_be32(val);
-+}
-+
-+
-+static inline void set_bits(unsigned long mask, volatile unsigned long *p)
-+{
-+ *p |= mask;
-+}
-+static inline void clear_bits(unsigned long mask, volatile unsigned long *p)
-+{
-+ *p &= ~mask;
-+}
-+
-+static inline void flush_dcache_range(unsigned long start, unsigned long stop)
-+{
-+ __cpuc_flush_dcache_area((void *) start, stop - start);
-+}
-+
-+#define hard_smp_processor_id() raw_smp_processor_id()
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/dpa_sys_arm64.h
-@@ -0,0 +1,102 @@
-+/* Copyright 2014 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef DPA_SYS_ARM64_H
-+#define DPA_SYS_ARM64_H
-+
-+#include <asm/cacheflush.h>
-+#include <asm/barrier.h>
-+
-+/* Implementation of ARM 64 bit specific routines */
-+
-+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
-+ * barriers and that dcb*() won't fall victim to compiler or execution
-+ * reordering with respect to other code/instructions that manipulate the same
-+ * cacheline. */
-+#define hwsync() { asm volatile("dmb st" : : : "memory"); }
-+#define lwsync() { asm volatile("dmb st" : : : "memory"); }
-+#define dcbf(p) { asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); }
-+#define dcbt_ro(p) { asm volatile("prfm pldl1keep, [%0, #0]" : : "r" (p)); }
-+#define dcbt_rw(p) { asm volatile("prfm pstl1keep, [%0, #0]" : : "r" (p)); }
-+#define dcbi(p) { asm volatile("dc ivac, %0" : : "r"(p) : "memory"); }
-+#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); }
-+
-+#define dcbz_64(p) \
-+ do { \
-+ dcbz(p); \
-+ } while (0)
-+
-+#define dcbf_64(p) \
-+ do { \
-+ dcbf(p); \
-+ } while (0)
-+/* Commonly used combo */
-+#define dcbit_ro(p) \
-+ do { \
-+ dcbi(p); \
-+ dcbt_ro(p); \
-+ } while (0)
-+
-+static inline u64 mfatb(void)
-+{
-+ return get_cycles();
-+}
-+
-+static inline u32 in_be32(volatile void *addr)
-+{
-+ return be32_to_cpu(*((volatile u32 *) addr));
-+}
-+
-+static inline void out_be32(void *addr, u32 val)
-+{
-+ *((u32 *) addr) = cpu_to_be32(val);
-+}
-+
-+
-+static inline void set_bits(unsigned long mask, volatile unsigned long *p)
-+{
-+ *p |= mask;
-+}
-+static inline void clear_bits(unsigned long mask, volatile unsigned long *p)
-+{
-+ *p &= ~mask;
-+}
-+
-+static inline void flush_dcache_range(unsigned long start, unsigned long stop)
-+{
-+ __flush_dcache_area((void *) start, stop - start);
-+}
-+
-+#define hard_smp_processor_id() raw_smp_processor_id()
-+
-+
-+
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/dpa_sys_ppc32.h
-@@ -0,0 +1,70 @@
-+/* Copyright 2014 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef DPA_SYS_PPC32_H
-+#define DPA_SYS_PPC32_H
-+
-+/* Implementation of PowerPC 32 bit specific routines */
-+
-+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
-+ * barriers and that dcb*() won't fall victim to compiler or execution
-+ * reordering with respect to other code/instructions that manipulate the same
-+ * cacheline. */
-+#define hwsync() __asm__ __volatile__ ("sync" : : : "memory")
-+#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory")
-+#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory")
-+#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p))
-+#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p))
-+#define dcbi(p) dcbf(p)
-+
-+#define dcbzl(p) __asm__ __volatile__ ("dcbzl 0,%0" : : "r" (p))
-+#define dcbz_64(p) dcbzl(p)
-+#define dcbf_64(p) dcbf(p)
-+
-+/* Commonly used combo */
-+#define dcbit_ro(p) \
-+ do { \
-+ dcbi(p); \
-+ dcbt_ro(p); \
-+ } while (0)
-+
-+static inline u64 mfatb(void)
-+{
-+ u32 hi, lo, chk;
-+ do {
-+ hi = mfspr(SPRN_ATBU);
-+ lo = mfspr(SPRN_ATBL);
-+ chk = mfspr(SPRN_ATBU);
-+ } while (unlikely(hi != chk));
-+ return ((u64)hi << 32) | (u64)lo;
-+}
-+
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/dpa_sys_ppc64.h
-@@ -0,0 +1,79 @@
-+/* Copyright 2014 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef DPA_SYS_PPC64_H
-+#define DPA_SYS_PPC64_H
-+
-+/* Implementation of PowerPC 64 bit specific routines */
-+
-+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
-+ * barriers and that dcb*() won't fall victim to compiler or execution
-+ * reordering with respect to other code/instructions that manipulate the same
-+ * cacheline. */
-+#define hwsync() __asm__ __volatile__ ("sync" : : : "memory")
-+#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory")
-+#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory")
-+#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p))
-+#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p))
-+#define dcbi(p) dcbf(p)
-+
-+#define dcbz(p) __asm__ __volatile__ ("dcbz 0,%0" : : "r" (p))
-+#define dcbz_64(p) \
-+ do { \
-+ dcbz((void*)p + 32); \
-+ dcbz(p); \
-+ } while (0)
-+#define dcbf_64(p) \
-+ do { \
-+ dcbf((void*)p + 32); \
-+ dcbf(p); \
-+ } while (0)
-+/* Commonly used combo */
-+#define dcbit_ro(p) \
-+ do { \
-+ dcbi(p); \
-+ dcbi((void*)p + 32); \
-+ dcbt_ro(p); \
-+ dcbt_ro((void*)p + 32); \
-+ } while (0)
-+
-+static inline u64 mfatb(void)
-+{
-+ u32 hi, lo, chk;
-+ do {
-+ hi = mfspr(SPRN_ATBU);
-+ lo = mfspr(SPRN_ATBL);
-+ chk = mfspr(SPRN_ATBU);
-+ } while (unlikely(hi != chk));
-+ return ((u64)hi << 32) | (u64)lo;
-+}
-+
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/fsl_usdpaa.c
-@@ -0,0 +1,1984 @@
-+/* Copyright (C) 2008-2012 Freescale Semiconductor, Inc.
-+ * Authors: Andy Fleming <afleming@freescale.com>
-+ * Timur Tabi <timur@freescale.com>
-+ * Geoff Thorpe <Geoff.Thorpe@freescale.com>
-+ *
-+ * This file is licensed under the terms of the GNU General Public License
-+ * version 2. This program is licensed "as is" without any warranty of any
-+ * kind, whether express or implied.
-+ */
-+
-+
-+#include <linux/miscdevice.h>
-+#include <linux/fs.h>
-+#include <linux/cdev.h>
-+#include <linux/mm.h>
-+#include <linux/of.h>
-+#include <linux/memblock.h>
-+#include <linux/slab.h>
-+#include <linux/mman.h>
-+#include <linux/of_reserved_mem.h>
-+
-+#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64))
-+#include <mm/mmu_decl.h>
-+#endif
-+
-+#include "dpa_sys.h"
-+#include <linux/fsl_usdpaa.h>
-+#include "bman_low.h"
-+#include "qman_low.h"
-+
-+/* Physical address range of the memory reservation, exported for mm/mem.c */
-+static u64 phys_start;
-+static u64 phys_size;
-+static u64 arg_phys_size;
-+
-+/* PFN versions of the above */
-+static unsigned long pfn_start;
-+static unsigned long pfn_size;
-+
-+/* Memory reservations are manipulated under this spinlock (which is why 'refs'
-+ * isn't atomic_t). */
-+static DEFINE_SPINLOCK(mem_lock);
-+
-+/* The range of TLB1 indices */
-+static unsigned int first_tlb;
-+static unsigned int num_tlb = 1;
-+static unsigned int current_tlb; /* loops around for fault handling */
-+
-+/* Memory reservation is represented as a list of 'mem_fragment's, some of which
-+ * may be mapped. Unmapped fragments are always merged where possible. */
-+static LIST_HEAD(mem_list);
-+
-+struct mem_mapping;
-+
-+/* Memory fragments are in 'mem_list'. */
-+struct mem_fragment {
-+ u64 base;
-+ u64 len;
-+ unsigned long pfn_base; /* PFN version of 'base' */
-+ unsigned long pfn_len; /* PFN version of 'len' */
-+ unsigned int refs; /* zero if unmapped */
-+ u64 root_len; /* Size of the orignal fragment */
-+ unsigned long root_pfn; /* PFN of the orignal fragment */
-+ struct list_head list;
-+ /* if mapped, flags+name captured at creation time */
-+ u32 flags;
-+ char name[USDPAA_DMA_NAME_MAX];
-+ u64 map_len;
-+ /* support multi-process locks per-memory-fragment. */
-+ int has_locking;
-+ wait_queue_head_t wq;
-+ struct mem_mapping *owner;
-+};
-+
-+/* Mappings of memory fragments in 'struct ctx'. These are created from
-+ * ioctl(USDPAA_IOCTL_DMA_MAP), though the actual mapping then happens via a
-+ * mmap(). */
-+struct mem_mapping {
-+ struct mem_fragment *root_frag;
-+ u32 frag_count;
-+ u64 total_size;
-+ struct list_head list;
-+ int refs;
-+ void *virt_addr;
-+};
-+
-+struct portal_mapping {
-+ struct usdpaa_ioctl_portal_map user;
-+ union {
-+ struct qm_portal_config *qportal;
-+ struct bm_portal_config *bportal;
-+ };
-+ /* Declare space for the portals in case the process
-+ exits unexpectedly and needs to be cleaned by the kernel */
-+ union {
-+ struct qm_portal qman_portal_low;
-+ struct bm_portal bman_portal_low;
-+ };
-+ struct list_head list;
-+ struct resource *phys;
-+ struct iommu_domain *iommu_domain;
-+};
-+
-+/* Track the DPAA resources the process is using */
-+struct active_resource {
-+ struct list_head list;
-+ u32 id;
-+ u32 num;
-+ unsigned int refcount;
-+};
-+
-+/* Per-FD state (which should also be per-process but we don't enforce that) */
-+struct ctx {
-+ /* Lock to protect the context */
-+ spinlock_t lock;
-+ /* Allocated resources get put here for accounting */
-+ struct list_head resources[usdpaa_id_max];
-+ /* list of DMA maps */
-+ struct list_head maps;
-+ /* list of portal maps */
-+ struct list_head portals;
-+};
-+
-+/* Different resource classes */
-+static const struct alloc_backend {
-+ enum usdpaa_id_type id_type;
-+ int (*alloc)(u32 *, u32, u32, int);
-+ void (*release)(u32 base, unsigned int count);
-+ int (*reserve)(u32 base, unsigned int count);
-+ const char *acronym;
-+} alloc_backends[] = {
-+ {
-+ .id_type = usdpaa_id_fqid,
-+ .alloc = qman_alloc_fqid_range,
-+ .release = qman_release_fqid_range,
-+ .reserve = qman_reserve_fqid_range,
-+ .acronym = "FQID"
-+ },
-+ {
-+ .id_type = usdpaa_id_bpid,
-+ .alloc = bman_alloc_bpid_range,
-+ .release = bman_release_bpid_range,
-+ .reserve = bman_reserve_bpid_range,
-+ .acronym = "BPID"
-+ },
-+ {
-+ .id_type = usdpaa_id_qpool,
-+ .alloc = qman_alloc_pool_range,
-+ .release = qman_release_pool_range,
-+ .reserve = qman_reserve_pool_range,
-+ .acronym = "QPOOL"
-+ },
-+ {
-+ .id_type = usdpaa_id_cgrid,
-+ .alloc = qman_alloc_cgrid_range,
-+ .release = qman_release_cgrid_range,
-+ .acronym = "CGRID"
-+ },
-+ {
-+ .id_type = usdpaa_id_ceetm0_lfqid,
-+ .alloc = qman_alloc_ceetm0_lfqid_range,
-+ .release = qman_release_ceetm0_lfqid_range,
-+ .acronym = "CEETM0_LFQID"
-+ },
-+ {
-+ .id_type = usdpaa_id_ceetm0_channelid,
-+ .alloc = qman_alloc_ceetm0_channel_range,
-+ .release = qman_release_ceetm0_channel_range,
-+ .acronym = "CEETM0_LFQID"
-+ },
-+ {
-+ .id_type = usdpaa_id_ceetm1_lfqid,
-+ .alloc = qman_alloc_ceetm1_lfqid_range,
-+ .release = qman_release_ceetm1_lfqid_range,
-+ .acronym = "CEETM1_LFQID"
-+ },
-+ {
-+ .id_type = usdpaa_id_ceetm1_channelid,
-+ .alloc = qman_alloc_ceetm1_channel_range,
-+ .release = qman_release_ceetm1_channel_range,
-+ .acronym = "CEETM1_LFQID"
-+ },
-+ {
-+ /* This terminates the array */
-+ .id_type = usdpaa_id_max
-+ }
-+};
-+
-+/* Determines the largest acceptable page size for a given size
-+ The sizes are determined by what the TLB1 acceptable page sizes are */
-+static u32 largest_page_size(u32 size)
-+{
-+ int shift = 30; /* Start at 1G size */
-+ if (size < 4096)
-+ return 0;
-+ do {
-+ if (size >= (1<<shift))
-+ return 1<<shift;
-+ shift -= 2;
-+ } while (shift >= 12); /* Up to 4k */
-+ return 0;
-+}
-+
-+/* Determine if value is power of 4 */
-+static inline bool is_power_of_4(u64 x)
-+{
-+ if (x == 0 || ((x & (x - 1)) != 0))
-+ return false;
-+ return !!(x & 0x5555555555555555ull);
-+}
-+
-+/* Helper for ioctl_dma_map() when we have a larger fragment than we need. This
-+ * splits the fragment into 4 and returns the upper-most. (The caller can loop
-+ * until it has a suitable fragment size.) */
-+static struct mem_fragment *split_frag(struct mem_fragment *frag)
-+{
-+ struct mem_fragment *x[3];
-+
-+ x[0] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC);
-+ x[1] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC);
-+ x[2] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC);
-+ if (!x[0] || !x[1] || !x[2]) {
-+ kfree(x[0]);
-+ kfree(x[1]);
-+ kfree(x[2]);
-+ return NULL;
-+ }
-+ BUG_ON(frag->refs);
-+ frag->len >>= 2;
-+ frag->pfn_len >>= 2;
-+ x[0]->base = frag->base + frag->len;
-+ x[1]->base = x[0]->base + frag->len;
-+ x[2]->base = x[1]->base + frag->len;
-+ x[0]->len = x[1]->len = x[2]->len = frag->len;
-+ x[0]->pfn_base = frag->pfn_base + frag->pfn_len;
-+ x[1]->pfn_base = x[0]->pfn_base + frag->pfn_len;
-+ x[2]->pfn_base = x[1]->pfn_base + frag->pfn_len;
-+ x[0]->pfn_len = x[1]->pfn_len = x[2]->pfn_len = frag->pfn_len;
-+ x[0]->refs = x[1]->refs = x[2]->refs = 0;
-+ x[0]->root_len = x[1]->root_len = x[2]->root_len = frag->root_len;
-+ x[0]->root_pfn = x[1]->root_pfn = x[2]->root_pfn = frag->root_pfn;
-+ x[0]->name[0] = x[1]->name[0] = x[2]->name[0] = 0;
-+ list_add_tail(&x[0]->list, &frag->list);
-+ list_add_tail(&x[1]->list, &x[0]->list);
-+ list_add_tail(&x[2]->list, &x[1]->list);
-+ return x[2];
-+}
-+
-+static __maybe_unused void dump_frags(void)
-+{
-+ struct mem_fragment *frag;
-+ int i = 0;
-+ list_for_each_entry(frag, &mem_list, list) {
-+ pr_info("FRAG %d: base 0x%llx pfn_base 0x%lx len 0x%llx root_len 0x%llx root_pfn 0x%lx refs %d name %s\n",
-+ i, frag->base, frag->pfn_base,
-+ frag->len, frag->root_len, frag->root_pfn,
-+ frag->refs, frag->name);
-+ ++i;
-+ }
-+}
-+
-+/* Walk the list of fragments and adjoin neighbouring segments if possible */
-+static void compress_frags(void)
-+{
-+ /* Walk the fragment list and combine fragments */
-+ struct mem_fragment *frag, *nxtfrag;
-+ u64 len = 0;
-+
-+ int i, numfrags;
-+
-+
-+ frag = list_entry(mem_list.next, struct mem_fragment, list);
-+
-+ while (&frag->list != &mem_list) {
-+ /* Must combine consecutive fragemenst with
-+ same root_pfn such that they are power of 4 */
-+ if (frag->refs != 0) {
-+ frag = list_entry(frag->list.next,
-+ struct mem_fragment, list);
-+ continue; /* Not this window */
-+ }
-+ len = frag->len;
-+ numfrags = 0;
-+ nxtfrag = list_entry(frag->list.next,
-+ struct mem_fragment, list);
-+ while (true) {
-+ if (&nxtfrag->list == &mem_list) {
-+ numfrags = 0;
-+ break; /* End of list */
-+ }
-+ if (nxtfrag->refs) {
-+ numfrags = 0;
-+ break; /* In use still */
-+ }
-+ if (nxtfrag->root_pfn != frag->root_pfn) {
-+ numfrags = 0;
-+ break; /* Crosses root fragment boundary */
-+ }
-+ len += nxtfrag->len;
-+ numfrags++;
-+ if (is_power_of_4(len)) {
-+ /* These fragments can be combined */
-+ break;
-+ }
-+ nxtfrag = list_entry(nxtfrag->list.next,
-+ struct mem_fragment, list);
-+ }
-+ if (numfrags == 0) {
-+ frag = list_entry(frag->list.next,
-+ struct mem_fragment, list);
-+ continue; /* try the next window */
-+ }
-+ for (i = 0; i < numfrags; i++) {
-+ struct mem_fragment *todel =
-+ list_entry(nxtfrag->list.prev,
-+ struct mem_fragment, list);
-+ nxtfrag->len += todel->len;
-+ nxtfrag->pfn_len += todel->pfn_len;
-+ list_del(&todel->list);
-+ }
-+ /* Re evaluate the list, things may merge now */
-+ frag = list_entry(mem_list.next, struct mem_fragment, list);
-+ }
-+}
-+
-+/* Hook from arch/powerpc/mm/mem.c */
-+int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size)
-+{
-+ struct mem_fragment *frag;
-+ int idx = -1;
-+ if ((pfn < pfn_start) || (pfn >= (pfn_start + pfn_size)))
-+ return -1;
-+ /* It's in-range, we need to find the fragment */
-+ spin_lock(&mem_lock);
-+ list_for_each_entry(frag, &mem_list, list) {
-+ if ((pfn >= frag->pfn_base) && (pfn < (frag->pfn_base +
-+ frag->pfn_len))) {
-+ *phys_addr = frag->base;
-+ *size = frag->len;
-+ idx = current_tlb++;
-+ if (current_tlb >= (first_tlb + num_tlb))
-+ current_tlb = first_tlb;
-+ break;
-+ }
-+ }
-+ spin_unlock(&mem_lock);
-+ return idx;
-+}
-+
-+static int usdpaa_open(struct inode *inode, struct file *filp)
-+{
-+ const struct alloc_backend *backend = &alloc_backends[0];
-+ struct ctx *ctx = kmalloc(sizeof(struct ctx), GFP_KERNEL);
-+ if (!ctx)
-+ return -ENOMEM;
-+ filp->private_data = ctx;
-+
-+ while (backend->id_type != usdpaa_id_max) {
-+ INIT_LIST_HEAD(&ctx->resources[backend->id_type]);
-+ backend++;
-+ }
-+
-+ INIT_LIST_HEAD(&ctx->maps);
-+ INIT_LIST_HEAD(&ctx->portals);
-+ spin_lock_init(&ctx->lock);
-+
-+ //filp->f_mapping->backing_dev_info = &directly_mappable_cdev_bdi;
-+
-+ return 0;
-+}
-+
-+#define DQRR_MAXFILL 15
-+
-+/* Reset a QMan portal to its default state */
-+static int init_qm_portal(struct qm_portal_config *config,
-+ struct qm_portal *portal)
-+{
-+ const struct qm_dqrr_entry *dqrr = NULL;
-+ int i;
-+
-+ portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
-+ portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
-+
-+ /* Make sure interrupts are inhibited */
-+ qm_out(IIR, 1);
-+
-+ /* Initialize the DQRR. This will stop any dequeue
-+ commands that are in progress */
-+ if (qm_dqrr_init(portal, config, qm_dqrr_dpush, qm_dqrr_pvb,
-+ qm_dqrr_cdc, DQRR_MAXFILL)) {
-+ pr_err("qm_dqrr_init() failed when trying to"
-+ " recover portal, portal will be leaked\n");
-+ return 1;
-+ }
-+
-+ /* Discard any entries on the DQRR */
-+ /* If we consume the ring twice something is wrong */
-+ for (i = 0; i < DQRR_MAXFILL * 2; i++) {
-+ qm_dqrr_pvb_update(portal);
-+ dqrr = qm_dqrr_current(portal);
-+ if (!dqrr)
-+ break;
-+ qm_dqrr_cdc_consume_1ptr(portal, dqrr, 0);
-+ qm_dqrr_pvb_update(portal);
-+ qm_dqrr_next(portal);
-+ }
-+ /* Initialize the EQCR */
-+ if (qm_eqcr_init(portal, qm_eqcr_pvb,
-+ qm_eqcr_get_ci_stashing(portal), 1)) {
-+ pr_err("Qman EQCR initialisation failed\n");
-+ return 1;
-+ }
-+ /* initialize the MR */
-+ if (qm_mr_init(portal, qm_mr_pvb, qm_mr_cci)) {
-+ pr_err("Qman MR initialisation failed\n");
-+ return 1;
-+ }
-+ qm_mr_pvb_update(portal);
-+ while (qm_mr_current(portal)) {
-+ qm_mr_next(portal);
-+ qm_mr_cci_consume_to_current(portal);
-+ qm_mr_pvb_update(portal);
-+ }
-+
-+ if (qm_mc_init(portal)) {
-+ pr_err("Qman MC initialisation failed\n");
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+static int init_bm_portal(struct bm_portal_config *config,
-+ struct bm_portal *portal)
-+{
-+ portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
-+ portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
-+
-+ if (bm_rcr_init(portal, bm_rcr_pvb, bm_rcr_cce)) {
-+ pr_err("Bman RCR initialisation failed\n");
-+ return 1;
-+ }
-+ if (bm_mc_init(portal)) {
-+ pr_err("Bman MC initialisation failed\n");
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+/* Function that will scan all FQ's in the system. For each FQ that is not
-+ OOS it will call the check_channel helper to determine if the FQ should
-+ be torn down. If the check_channel helper returns true the FQ will be
-+ transitioned to the OOS state */
-+static int qm_check_and_destroy_fqs(struct qm_portal *portal, void *ctx,
-+ bool (*check_channel)(void*, u32))
-+{
-+ u32 fq_id = 0;
-+ while (1) {
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ u8 state;
-+ u32 channel;
-+
-+ /* Determine the channel for the FQID */
-+ mcc = qm_mc_start(portal);
-+ mcc->queryfq.fqid = fq_id;
-+ qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ);
-+ while (!(mcr = qm_mc_result(portal)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK)
-+ == QM_MCR_VERB_QUERYFQ);
-+ if (mcr->result != QM_MCR_RESULT_OK)
-+ break; /* End of valid FQIDs */
-+
-+ channel = mcr->queryfq.fqd.dest.channel;
-+ /* Determine the state of the FQID */
-+ mcc = qm_mc_start(portal);
-+ mcc->queryfq_np.fqid = fq_id;
-+ qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ_NP);
-+ while (!(mcr = qm_mc_result(portal)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK)
-+ == QM_MCR_VERB_QUERYFQ_NP);
-+ state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
-+ if (state == QM_MCR_NP_STATE_OOS)
-+ /* Already OOS, no need to do anymore checks */
-+ goto next;
-+
-+ if (check_channel(ctx, channel))
-+ qm_shutdown_fq(&portal, 1, fq_id);
-+ next:
-+ ++fq_id;
-+ }
-+ return 0;
-+}
-+
-+static bool check_channel_device(void *_ctx, u32 channel)
-+{
-+ struct ctx *ctx = _ctx;
-+ struct portal_mapping *portal, *tmpportal;
-+ struct active_resource *res;
-+
-+ /* See if the FQ is destined for one of the portals we're cleaning up */
-+ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) {
-+ if (portal->user.type == usdpaa_portal_qman) {
-+ if (portal->qportal->public_cfg.channel == channel) {
-+ /* This FQs destination is a portal
-+ we're cleaning, send a retire */
-+ return true;
-+ }
-+ }
-+ }
-+
-+ /* Check the pool channels that will be released as well */
-+ list_for_each_entry(res, &ctx->resources[usdpaa_id_qpool], list) {
-+ if ((res->id >= channel) &&
-+ ((res->id + res->num - 1) <= channel))
-+ return true;
-+ }
-+ return false;
-+}
-+
-+static bool check_portal_channel(void *ctx, u32 channel)
-+{
-+ u32 portal_channel = *(u32 *)ctx;
-+ if (portal_channel == channel) {
-+ /* This FQs destination is a portal
-+ we're cleaning, send a retire */
-+ return true;
-+ }
-+ return false;
-+}
-+
-+
-+
-+
-+static int usdpaa_release(struct inode *inode, struct file *filp)
-+{
-+ struct ctx *ctx = filp->private_data;
-+ struct mem_mapping *map, *tmpmap;
-+ struct portal_mapping *portal, *tmpportal;
-+ const struct alloc_backend *backend = &alloc_backends[0];
-+ struct active_resource *res;
-+ struct qm_portal *qm_cleanup_portal = NULL;
-+ struct bm_portal *bm_cleanup_portal = NULL;
-+ struct qm_portal_config *qm_alloced_portal = NULL;
-+ struct bm_portal_config *bm_alloced_portal = NULL;
-+
-+ struct qm_portal *portal_array[qman_portal_max];
-+ int portal_count = 0;
-+
-+ /* Ensure the release operation cannot be migrated to another
-+ CPU as CPU specific variables may be needed during cleanup */
-+#ifdef CONFIG_PREEMPT_RT_FULL
-+ migrate_disable();
-+#endif
-+ /* The following logic is used to recover resources that were not
-+ correctly released by the process that is closing the FD.
-+ Step 1: syncronize the HW with the qm_portal/bm_portal structures
-+ in the kernel
-+ */
-+
-+ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) {
-+ /* Try to recover any portals that weren't shut down */
-+ if (portal->user.type == usdpaa_portal_qman) {
-+ portal_array[portal_count] = &portal->qman_portal_low;
-+ ++portal_count;
-+ init_qm_portal(portal->qportal,
-+ &portal->qman_portal_low);
-+ if (!qm_cleanup_portal) {
-+ qm_cleanup_portal = &portal->qman_portal_low;
-+ } else {
-+ /* Clean FQs on the dedicated channel */
-+ u32 chan = portal->qportal->public_cfg.channel;
-+ qm_check_and_destroy_fqs(
-+ &portal->qman_portal_low, &chan,
-+ check_portal_channel);
-+ }
-+ } else {
-+ /* BMAN */
-+ init_bm_portal(portal->bportal,
-+ &portal->bman_portal_low);
-+ if (!bm_cleanup_portal)
-+ bm_cleanup_portal = &portal->bman_portal_low;
-+ }
-+ }
-+ /* If no portal was found, allocate one for cleanup */
-+ if (!qm_cleanup_portal) {
-+ qm_alloced_portal = qm_get_unused_portal();
-+ if (!qm_alloced_portal) {
-+ pr_crit("No QMan portal avalaible for cleanup\n");
-+#ifdef CONFIG_PREEMPT_RT_FULL
-+ migrate_enable();
-+#endif
-+ return -1;
-+ }
-+ qm_cleanup_portal = kmalloc(sizeof(struct qm_portal),
-+ GFP_KERNEL);
-+ if (!qm_cleanup_portal) {
-+#ifdef CONFIG_PREEMPT_RT_FULL
-+ migrate_enable();
-+#endif
-+ return -ENOMEM;
-+ }
-+ init_qm_portal(qm_alloced_portal, qm_cleanup_portal);
-+ portal_array[portal_count] = qm_cleanup_portal;
-+ ++portal_count;
-+ }
-+ if (!bm_cleanup_portal) {
-+ bm_alloced_portal = bm_get_unused_portal();
-+ if (!bm_alloced_portal) {
-+ pr_crit("No BMan portal avalaible for cleanup\n");
-+#ifdef CONFIG_PREEMPT_RT_FULL
-+ migrate_enable();
-+#endif
-+ return -1;
-+ }
-+ bm_cleanup_portal = kmalloc(sizeof(struct bm_portal),
-+ GFP_KERNEL);
-+ if (!bm_cleanup_portal) {
-+#ifdef CONFIG_PREEMPT_RT_FULL
-+ migrate_enable();
-+#endif
-+ return -ENOMEM;
-+ }
-+ init_bm_portal(bm_alloced_portal, bm_cleanup_portal);
-+ }
-+
-+ /* OOS the FQs associated with this process */
-+ qm_check_and_destroy_fqs(qm_cleanup_portal, ctx, check_channel_device);
-+
-+ while (backend->id_type != usdpaa_id_max) {
-+ int leaks = 0;
-+ list_for_each_entry(res, &ctx->resources[backend->id_type],
-+ list) {
-+ if (backend->id_type == usdpaa_id_fqid) {
-+ int i = 0;
-+ for (; i < res->num; i++) {
-+ /* Clean FQs with the cleanup portal */
-+ qm_shutdown_fq(portal_array,
-+ portal_count,
-+ res->id + i);
-+ }
-+ }
-+ leaks += res->num;
-+ backend->release(res->id, res->num);
-+ }
-+ if (leaks)
-+ pr_crit("USDPAA process leaking %d %s%s\n", leaks,
-+ backend->acronym, (leaks > 1) ? "s" : "");
-+ backend++;
-+ }
-+ /* Release any DMA regions */
-+ spin_lock(&mem_lock);
-+ list_for_each_entry_safe(map, tmpmap, &ctx->maps, list) {
-+ struct mem_fragment *current_frag = map->root_frag;
-+ int i;
-+ if (map->root_frag->has_locking &&
-+ (map->root_frag->owner == map)) {
-+ map->root_frag->owner = NULL;
-+ wake_up(&map->root_frag->wq);
-+ }
-+ /* Check each fragment and merge if the ref count is 0 */
-+ for (i = 0; i < map->frag_count; i++) {
-+ --current_frag->refs;
-+ current_frag = list_entry(current_frag->list.prev,
-+ struct mem_fragment, list);
-+ }
-+
-+ compress_frags();
-+ list_del(&map->list);
-+ kfree(map);
-+ }
-+ spin_unlock(&mem_lock);
-+
-+ /* Return portals */
-+ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) {
-+ if (portal->user.type == usdpaa_portal_qman) {
-+ /* Give the portal back to the allocator */
-+ init_qm_portal(portal->qportal,
-+ &portal->qman_portal_low);
-+ qm_put_unused_portal(portal->qportal);
-+ } else {
-+ init_bm_portal(portal->bportal,
-+ &portal->bman_portal_low);
-+ bm_put_unused_portal(portal->bportal);
-+ }
-+ list_del(&portal->list);
-+ kfree(portal);
-+ }
-+ if (qm_alloced_portal) {
-+ qm_put_unused_portal(qm_alloced_portal);
-+ kfree(qm_cleanup_portal);
-+ }
-+ if (bm_alloced_portal) {
-+ bm_put_unused_portal(bm_alloced_portal);
-+ kfree(bm_cleanup_portal);
-+ }
-+
-+ kfree(ctx);
-+#ifdef CONFIG_PREEMPT_RT_FULL
-+ migrate_enable();
-+#endif
-+ return 0;
-+}
-+
-+static int check_mmap_dma(struct ctx *ctx, struct vm_area_struct *vma,
-+ int *match, unsigned long *pfn)
-+{
-+ struct mem_mapping *map;
-+
-+ list_for_each_entry(map, &ctx->maps, list) {
-+ int i;
-+ struct mem_fragment *frag = map->root_frag;
-+
-+ for (i = 0; i < map->frag_count; i++) {
-+ if (frag->pfn_base == vma->vm_pgoff) {
-+ *match = 1;
-+ *pfn = frag->pfn_base;
-+ return 0;
-+ }
-+ frag = list_entry(frag->list.next, struct mem_fragment,
-+ list);
-+ }
-+ }
-+ *match = 0;
-+ return 0;
-+}
-+
-+static int check_mmap_resource(struct resource *res, struct vm_area_struct *vma,
-+ int *match, unsigned long *pfn)
-+{
-+ *pfn = res->start >> PAGE_SHIFT;
-+ if (*pfn == vma->vm_pgoff) {
-+ *match = 1;
-+ if ((vma->vm_end - vma->vm_start) != resource_size(res))
-+ return -EINVAL;
-+ } else
-+ *match = 0;
-+ return 0;
-+}
-+
-+static int check_mmap_portal(struct ctx *ctx, struct vm_area_struct *vma,
-+ int *match, unsigned long *pfn)
-+{
-+ struct portal_mapping *portal;
-+ int ret;
-+
-+ list_for_each_entry(portal, &ctx->portals, list) {
-+ ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CE], vma,
-+ match, pfn);
-+ if (*match) {
-+ vma->vm_page_prot =
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ pgprot_cached_ns(vma->vm_page_prot);
-+#else
-+ pgprot_cached_noncoherent(vma->vm_page_prot);
-+#endif
-+ return ret;
-+ }
-+ ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CI], vma,
-+ match, pfn);
-+ if (*match) {
-+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-+ return ret;
-+ }
-+ }
-+ *match = 0;
-+ return 0;
-+}
-+
-+static int usdpaa_mmap(struct file *filp, struct vm_area_struct *vma)
-+{
-+ struct ctx *ctx = filp->private_data;
-+ unsigned long pfn = 0;
-+ int match, ret;
-+
-+ spin_lock(&mem_lock);
-+ ret = check_mmap_dma(ctx, vma, &match, &pfn);
-+ if (!match)
-+ ret = check_mmap_portal(ctx, vma, &match, &pfn);
-+ spin_unlock(&mem_lock);
-+ if (!match)
-+ return -EINVAL;
-+ if (!ret)
-+ ret = remap_pfn_range(vma, vma->vm_start, pfn,
-+ vma->vm_end - vma->vm_start,
-+ vma->vm_page_prot);
-+ return ret;
-+}
-+
-+/* Return the nearest rounded-up address >= 'addr' that is 'sz'-aligned. 'sz'
-+ * must be a power of 2, but both 'addr' and 'sz' can be expressions. */
-+#define USDPAA_MEM_ROUNDUP(addr, sz) \
-+ ({ \
-+ unsigned long foo_align = (sz) - 1; \
-+ ((addr) + foo_align) & ~foo_align; \
-+ })
-+/* Searching for a size-aligned virtual address range starting from 'addr' */
-+static unsigned long usdpaa_get_unmapped_area(struct file *file,
-+ unsigned long addr,
-+ unsigned long len,
-+ unsigned long pgoff,
-+ unsigned long flags)
-+{
-+ struct vm_area_struct *vma;
-+
-+ if (len % PAGE_SIZE)
-+ return -EINVAL;
-+ if (!len)
-+ return -EINVAL;
-+
-+ /* Need to align the address to the largest pagesize of the mapping
-+ * because the MMU requires the virtual address to have the same
-+ * alignment as the physical address */
-+ addr = USDPAA_MEM_ROUNDUP(addr, largest_page_size(len));
-+ vma = find_vma(current->mm, addr);
-+ /* Keep searching until we reach the end of currently-used virtual
-+ * address-space or we find a big enough gap. */
-+ while (vma) {
-+ if ((addr + len) < vma->vm_start)
-+ return addr;
-+
-+ addr = USDPAA_MEM_ROUNDUP(vma->vm_end, largest_page_size(len));
-+ vma = vma->vm_next;
-+ }
-+ if ((TASK_SIZE - len) < addr)
-+ return -ENOMEM;
-+ return addr;
-+}
-+
-+static long ioctl_id_alloc(struct ctx *ctx, void __user *arg)
-+{
-+ struct usdpaa_ioctl_id_alloc i;
-+ const struct alloc_backend *backend;
-+ struct active_resource *res;
-+ int ret = copy_from_user(&i, arg, sizeof(i));
-+ if (ret)
-+ return ret;
-+ if ((i.id_type >= usdpaa_id_max) || !i.num)
-+ return -EINVAL;
-+ backend = &alloc_backends[i.id_type];
-+ /* Allocate the required resource type */
-+ ret = backend->alloc(&i.base, i.num, i.align, i.partial);
-+ if (ret < 0)
-+ return ret;
-+ i.num = ret;
-+ /* Copy the result to user-space */
-+ ret = copy_to_user(arg, &i, sizeof(i));
-+ if (ret) {
-+ backend->release(i.base, i.num);
-+ return ret;
-+ }
-+ /* Assign the allocated range to the FD accounting */
-+ res = kmalloc(sizeof(*res), GFP_KERNEL);
-+ if (!res) {
-+ backend->release(i.base, i.num);
-+ return -ENOMEM;
-+ }
-+ spin_lock(&ctx->lock);
-+ res->id = i.base;
-+ res->num = i.num;
-+ res->refcount = 1;
-+ list_add(&res->list, &ctx->resources[i.id_type]);
-+ spin_unlock(&ctx->lock);
-+ return 0;
-+}
-+
-+static long ioctl_id_release(struct ctx *ctx, void __user *arg)
-+{
-+ struct usdpaa_ioctl_id_release i;
-+ const struct alloc_backend *backend;
-+ struct active_resource *tmp, *pos;
-+
-+ int ret = copy_from_user(&i, arg, sizeof(i));
-+ if (ret)
-+ return ret;
-+ if ((i.id_type >= usdpaa_id_max) || !i.num)
-+ return -EINVAL;
-+ backend = &alloc_backends[i.id_type];
-+ /* Pull the range out of the FD accounting - the range is valid iff this
-+ * succeeds. */
-+ spin_lock(&ctx->lock);
-+ list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) {
-+ if (pos->id == i.base && pos->num == i.num) {
-+ pos->refcount--;
-+ if (pos->refcount) {
-+ spin_unlock(&ctx->lock);
-+ return 0; /* Still being used */
-+ }
-+ list_del(&pos->list);
-+ kfree(pos);
-+ spin_unlock(&ctx->lock);
-+ goto found;
-+ }
-+ }
-+ /* Failed to find the resource */
-+ spin_unlock(&ctx->lock);
-+ pr_err("Couldn't find resource type %d base 0x%x num %d\n",
-+ i.id_type, i.base, i.num);
-+ return -EINVAL;
-+found:
-+ /* Release the resource to the backend */
-+ backend->release(i.base, i.num);
-+ return 0;
-+}
-+
-+static long ioctl_id_reserve(struct ctx *ctx, void __user *arg)
-+{
-+ struct usdpaa_ioctl_id_reserve i;
-+ const struct alloc_backend *backend;
-+ struct active_resource *tmp, *pos;
-+
-+ int ret = copy_from_user(&i, arg, sizeof(i));
-+ if (ret)
-+ return ret;
-+ if ((i.id_type >= usdpaa_id_max) || !i.num)
-+ return -EINVAL;
-+ backend = &alloc_backends[i.id_type];
-+ if (!backend->reserve)
-+ return -EINVAL;
-+ /* Pull the range out of the FD accounting - the range is valid iff this
-+ * succeeds. */
-+ spin_lock(&ctx->lock);
-+ list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) {
-+ if (pos->id == i.base && pos->num == i.num) {
-+ pos->refcount++;
-+ spin_unlock(&ctx->lock);
-+ return 0;
-+ }
-+ }
-+
-+ /* Failed to find the resource */
-+ spin_unlock(&ctx->lock);
-+
-+ /* Reserve the resource in the backend */
-+ ret = backend->reserve(i.base, i.num);
-+ if (ret)
-+ return ret;
-+ /* Assign the reserved range to the FD accounting */
-+ pos = kmalloc(sizeof(*pos), GFP_KERNEL);
-+ if (!pos) {
-+ backend->release(i.base, i.num);
-+ return -ENOMEM;
-+ }
-+ spin_lock(&ctx->lock);
-+ pos->id = i.base;
-+ pos->num = i.num;
-+ pos->refcount = 1;
-+ list_add(&pos->list, &ctx->resources[i.id_type]);
-+ spin_unlock(&ctx->lock);
-+ return 0;
-+}
-+
-+static long ioctl_dma_map(struct file *fp, struct ctx *ctx,
-+ struct usdpaa_ioctl_dma_map *i)
-+{
-+ struct mem_fragment *frag, *start_frag, *next_frag;
-+ struct mem_mapping *map, *tmp;
-+ int ret = 0;
-+ u32 largest_page, so_far = 0;
-+ int frag_count = 0;
-+ unsigned long next_addr = PAGE_SIZE, populate;
-+
-+ /* error checking to ensure values copied from user space are valid */
-+ if (i->len % PAGE_SIZE)
-+ return -EINVAL;
-+
-+ map = kmalloc(sizeof(*map), GFP_KERNEL);
-+ if (!map)
-+ return -ENOMEM;
-+
-+ spin_lock(&mem_lock);
-+ if (i->flags & USDPAA_DMA_FLAG_SHARE) {
-+ list_for_each_entry(frag, &mem_list, list) {
-+ if (frag->refs && (frag->flags &
-+ USDPAA_DMA_FLAG_SHARE) &&
-+ !strncmp(i->name, frag->name,
-+ USDPAA_DMA_NAME_MAX)) {
-+ /* Matching entry */
-+ if ((i->flags & USDPAA_DMA_FLAG_CREATE) &&
-+ !(i->flags & USDPAA_DMA_FLAG_LAZY)) {
-+ ret = -EBUSY;
-+ goto out;
-+ }
-+
-+ /* Check to ensure size matches record */
-+ if (i->len != frag->map_len && i->len) {
-+ pr_err("ioctl_dma_map() Size requested does not match %s and is none zero\n",
-+ frag->name);
-+ return -EINVAL;
-+ }
-+
-+ /* Check if this has already been mapped
-+ to this process */
-+ list_for_each_entry(tmp, &ctx->maps, list)
-+ if (tmp->root_frag == frag) {
-+ /* Already mapped, just need to
-+ inc ref count */
-+ tmp->refs++;
-+ kfree(map);
-+ i->did_create = 0;
-+ i->len = tmp->total_size;
-+ i->phys_addr = frag->base;
-+ i->ptr = tmp->virt_addr;
-+ spin_unlock(&mem_lock);
-+ return 0;
-+ }
-+ /* Matching entry - just need to map */
-+ i->has_locking = frag->has_locking;
-+ i->did_create = 0;
-+ i->len = frag->map_len;
-+ start_frag = frag;
-+ goto do_map;
-+ }
-+ }
-+ /* No matching entry */
-+ if (!(i->flags & USDPAA_DMA_FLAG_CREATE)) {
-+ pr_err("ioctl_dma_map() No matching entry\n");
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+ }
-+ /* New fragment required, size must be provided. */
-+ if (!i->len) {
-+ ret = -EINVAL;
-+ goto out;
-+ }
-+
-+ /* Find one of more contiguous fragments that satisfy the total length
-+ trying to minimize the number of fragments
-+ compute the largest page size that the allocation could use */
-+ largest_page = largest_page_size(i->len);
-+ start_frag = NULL;
-+ while (largest_page &&
-+ largest_page <= largest_page_size(phys_size) &&
-+ start_frag == NULL) {
-+ /* Search the list for a frag of that size */
-+ list_for_each_entry(frag, &mem_list, list) {
-+ if (!frag->refs && (frag->len == largest_page)) {
-+ /* See if the next x fragments are free
-+ and can accomidate the size */
-+ u32 found_size = largest_page;
-+ next_frag = list_entry(frag->list.prev,
-+ struct mem_fragment,
-+ list);
-+ /* If the fragement is too small check
-+ if the neighbours cab support it */
-+ while (found_size < i->len) {
-+ if (&mem_list == &next_frag->list)
-+ break; /* End of list */
-+ if (next_frag->refs != 0 ||
-+ next_frag->len == 0)
-+ break; /* not enough space */
-+ found_size += next_frag->len;
-+ next_frag = list_entry(
-+ next_frag->list.prev,
-+ struct mem_fragment,
-+ list);
-+ }
-+ if (found_size >= i->len) {
-+ /* Success! there is enough contigous
-+ free space */
-+ start_frag = frag;
-+ break;
-+ }
-+ }
-+ } /* next frag loop */
-+ /* Couldn't statisfy the request with this
-+ largest page size, try a smaller one */
-+ largest_page <<= 2;
-+ }
-+ if (start_frag == NULL) {
-+ /* Couldn't find proper amount of space */
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+ i->did_create = 1;
-+do_map:
-+ /* Verify there is sufficient space to do the mapping */
-+ down_write(¤t->mm->mmap_sem);
-+ next_addr = usdpaa_get_unmapped_area(fp, next_addr, i->len, 0, 0);
-+ up_write(¤t->mm->mmap_sem);
-+
-+ if (next_addr & ~PAGE_MASK) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+
-+ /* We may need to divide the final fragment to accomidate the mapping */
-+ next_frag = start_frag;
-+ while (so_far != i->len) {
-+ BUG_ON(next_frag->len == 0);
-+ while ((next_frag->len + so_far) > i->len) {
-+ /* Split frag until they match */
-+ split_frag(next_frag);
-+ }
-+ so_far += next_frag->len;
-+ next_frag->refs++;
-+ ++frag_count;
-+ next_frag = list_entry(next_frag->list.prev,
-+ struct mem_fragment, list);
-+ }
-+ if (i->did_create) {
-+ size_t name_len = 0;
-+ start_frag->flags = i->flags;
-+ strncpy(start_frag->name, i->name, USDPAA_DMA_NAME_MAX);
-+ name_len = strnlen(start_frag->name, USDPAA_DMA_NAME_MAX);
-+ if (name_len >= USDPAA_DMA_NAME_MAX) {
-+ ret = -EFAULT;
-+ goto out;
-+ }
-+ start_frag->map_len = i->len;
-+ start_frag->has_locking = i->has_locking;
-+ init_waitqueue_head(&start_frag->wq);
-+ start_frag->owner = NULL;
-+ }
-+
-+ /* Setup the map entry */
-+ map->root_frag = start_frag;
-+ map->total_size = i->len;
-+ map->frag_count = frag_count;
-+ map->refs = 1;
-+ list_add(&map->list, &ctx->maps);
-+ i->phys_addr = start_frag->base;
-+out:
-+ spin_unlock(&mem_lock);
-+
-+ if (!ret) {
-+ unsigned long longret;
-+ down_write(¤t->mm->mmap_sem);
-+ longret = do_mmap_pgoff(fp, next_addr, map->total_size,
-+ PROT_READ |
-+ (i->flags &
-+ USDPAA_DMA_FLAG_RDONLY ? 0
-+ : PROT_WRITE),
-+ MAP_SHARED,
-+ start_frag->pfn_base,
-+ &populate,
-+ NULL);
-+ up_write(¤t->mm->mmap_sem);
-+ if (longret & ~PAGE_MASK) {
-+ ret = (int)longret;
-+ } else {
-+ i->ptr = (void *)longret;
-+ map->virt_addr = i->ptr;
-+ }
-+ } else
-+ kfree(map);
-+ return ret;
-+}
-+
-+static long ioctl_dma_unmap(struct ctx *ctx, void __user *arg)
-+{
-+ struct mem_mapping *map;
-+ struct vm_area_struct *vma;
-+ int ret, i;
-+ struct mem_fragment *current_frag;
-+ size_t sz;
-+ unsigned long base;
-+ unsigned long vaddr;
-+
-+ down_write(¤t->mm->mmap_sem);
-+ vma = find_vma(current->mm, (unsigned long)arg);
-+ if (!vma || (vma->vm_start > (unsigned long)arg)) {
-+ up_write(¤t->mm->mmap_sem);
-+ return -EFAULT;
-+ }
-+ spin_lock(&mem_lock);
-+ list_for_each_entry(map, &ctx->maps, list) {
-+ if (map->root_frag->pfn_base == vma->vm_pgoff) {
-+ /* Drop the map lock if we hold it */
-+ if (map->root_frag->has_locking &&
-+ (map->root_frag->owner == map)) {
-+ map->root_frag->owner = NULL;
-+ wake_up(&map->root_frag->wq);
-+ }
-+ goto map_match;
-+ }
-+ }
-+ /* Failed to find a matching mapping for this process */
-+ ret = -EFAULT;
-+ spin_unlock(&mem_lock);
-+ goto out;
-+map_match:
-+ map->refs--;
-+ if (map->refs != 0) {
-+ /* Another call the dma_map is referencing this */
-+ ret = 0;
-+ spin_unlock(&mem_lock);
-+ goto out;
-+ }
-+
-+ current_frag = map->root_frag;
-+ vaddr = (unsigned long) map->virt_addr;
-+ for (i = 0; i < map->frag_count; i++) {
-+ DPA_ASSERT(current_frag->refs > 0);
-+ --current_frag->refs;
-+#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64))
-+ /*
-+ * Make sure we invalidate the TLB entry for
-+ * this fragment, otherwise a remap of a different
-+ * page to this vaddr would give acces to an
-+ * incorrect piece of memory
-+ */
-+ cleartlbcam(vaddr, mfspr(SPRN_PID));
-+#endif
-+ vaddr += current_frag->len;
-+ current_frag = list_entry(current_frag->list.prev,
-+ struct mem_fragment, list);
-+ }
-+ map->root_frag->name[0] = 0;
-+ list_del(&map->list);
-+ compress_frags();
-+ spin_unlock(&mem_lock);
-+
-+ base = vma->vm_start;
-+ sz = vma->vm_end - vma->vm_start;
-+ do_munmap(current->mm, base, sz, NULL);
-+ ret = 0;
-+ out:
-+ up_write(¤t->mm->mmap_sem);
-+ return ret;
-+}
-+
-+static long ioctl_dma_stats(struct ctx *ctx, void __user *arg)
-+{
-+ struct mem_fragment *frag;
-+ struct usdpaa_ioctl_dma_used result;
-+
-+ result.free_bytes = 0;
-+ result.total_bytes = phys_size;
-+
-+ list_for_each_entry(frag, &mem_list, list) {
-+ if (frag->refs == 0)
-+ result.free_bytes += frag->len;
-+ }
-+
-+ return copy_to_user(arg, &result, sizeof(result)); }
-+
-+static int test_lock(struct mem_mapping *map)
-+{
-+ int ret = 0;
-+ spin_lock(&mem_lock);
-+ if (!map->root_frag->owner) {
-+ map->root_frag->owner = map;
-+ ret = 1;
-+ }
-+ spin_unlock(&mem_lock);
-+ return ret;
-+}
-+
-+static long ioctl_dma_lock(struct ctx *ctx, void __user *arg)
-+{
-+ struct mem_mapping *map;
-+ struct vm_area_struct *vma;
-+
-+ down_read(¤t->mm->mmap_sem);
-+ vma = find_vma(current->mm, (unsigned long)arg);
-+ if (!vma || (vma->vm_start > (unsigned long)arg)) {
-+ up_read(¤t->mm->mmap_sem);
-+ return -EFAULT;
-+ }
-+ spin_lock(&mem_lock);
-+ list_for_each_entry(map, &ctx->maps, list) {
-+ if (map->root_frag->pfn_base == vma->vm_pgoff)
-+ goto map_match;
-+ }
-+ map = NULL;
-+map_match:
-+ spin_unlock(&mem_lock);
-+ up_read(¤t->mm->mmap_sem);
-+
-+ if (!map)
-+ return -EFAULT;
-+ if (!map->root_frag->has_locking)
-+ return -ENODEV;
-+ return wait_event_interruptible(map->root_frag->wq, test_lock(map));
-+}
-+
-+static long ioctl_dma_unlock(struct ctx *ctx, void __user *arg)
-+{
-+ struct mem_mapping *map;
-+ struct vm_area_struct *vma;
-+ int ret;
-+
-+ down_read(¤t->mm->mmap_sem);
-+ vma = find_vma(current->mm, (unsigned long)arg);
-+ if (!vma || (vma->vm_start > (unsigned long)arg))
-+ ret = -EFAULT;
-+ else {
-+ spin_lock(&mem_lock);
-+ list_for_each_entry(map, &ctx->maps, list) {
-+ if (map->root_frag->pfn_base == vma->vm_pgoff) {
-+ if (!map->root_frag->has_locking)
-+ ret = -ENODEV;
-+ else if (map->root_frag->owner == map) {
-+ map->root_frag->owner = NULL;
-+ wake_up(&map->root_frag->wq);
-+ ret = 0;
-+ } else
-+ ret = -EBUSY;
-+ goto map_match;
-+ }
-+ }
-+ ret = -EINVAL;
-+map_match:
-+ spin_unlock(&mem_lock);
-+ }
-+ up_read(¤t->mm->mmap_sem);
-+ return ret;
-+}
-+
-+static int portal_mmap(struct file *fp, struct resource *res, void **ptr)
-+{
-+ unsigned long longret = 0, populate;
-+ resource_size_t len;
-+
-+ down_write(¤t->mm->mmap_sem);
-+ len = resource_size(res);
-+ if (len != (unsigned long)len)
-+ return -EINVAL;
-+ longret = do_mmap_pgoff(fp, PAGE_SIZE, (unsigned long)len,
-+ PROT_READ | PROT_WRITE, MAP_SHARED,
-+ res->start >> PAGE_SHIFT, &populate, NULL);
-+ up_write(¤t->mm->mmap_sem);
-+
-+ if (longret & ~PAGE_MASK)
-+ return (int)longret;
-+
-+ *ptr = (void *) longret;
-+ return 0;
-+}
-+
-+static void portal_munmap(struct resource *res, void *ptr)
-+{
-+ down_write(¤t->mm->mmap_sem);
-+ do_munmap(current->mm, (unsigned long)ptr, resource_size(res), NULL);
-+ up_write(¤t->mm->mmap_sem);
-+}
-+
-+static long ioctl_portal_map(struct file *fp, struct ctx *ctx,
-+ struct usdpaa_ioctl_portal_map *arg)
-+{
-+ struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
-+ int ret;
-+
-+ if (!mapping)
-+ return -ENOMEM;
-+
-+ mapping->user = *arg;
-+ mapping->iommu_domain = NULL;
-+
-+ if (mapping->user.type == usdpaa_portal_qman) {
-+ mapping->qportal =
-+ qm_get_unused_portal_idx(mapping->user.index);
-+ if (!mapping->qportal) {
-+ ret = -ENODEV;
-+ goto err_get_portal;
-+ }
-+ mapping->phys = &mapping->qportal->addr_phys[0];
-+ mapping->user.channel = mapping->qportal->public_cfg.channel;
-+ mapping->user.pools = mapping->qportal->public_cfg.pools;
-+ mapping->user.index = mapping->qportal->public_cfg.index;
-+ } else if (mapping->user.type == usdpaa_portal_bman) {
-+ mapping->bportal =
-+ bm_get_unused_portal_idx(mapping->user.index);
-+ if (!mapping->bportal) {
-+ ret = -ENODEV;
-+ goto err_get_portal;
-+ }
-+ mapping->phys = &mapping->bportal->addr_phys[0];
-+ mapping->user.index = mapping->bportal->public_cfg.index;
-+ } else {
-+ ret = -EINVAL;
-+ goto err_copy_from_user;
-+ }
-+ /* Need to put pcfg in ctx's list before the mmaps because the mmap
-+ * handlers look it up. */
-+ spin_lock(&mem_lock);
-+ list_add(&mapping->list, &ctx->portals);
-+ spin_unlock(&mem_lock);
-+ ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CE],
-+ &mapping->user.addr.cena);
-+ if (ret)
-+ goto err_mmap_cena;
-+ ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CI],
-+ &mapping->user.addr.cinh);
-+ if (ret)
-+ goto err_mmap_cinh;
-+ *arg = mapping->user;
-+ return ret;
-+
-+err_mmap_cinh:
-+ portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena);
-+err_mmap_cena:
-+ if ((mapping->user.type == usdpaa_portal_qman) && mapping->qportal)
-+ qm_put_unused_portal(mapping->qportal);
-+ else if ((mapping->user.type == usdpaa_portal_bman) && mapping->bportal)
-+ bm_put_unused_portal(mapping->bportal);
-+ spin_lock(&mem_lock);
-+ list_del(&mapping->list);
-+ spin_unlock(&mem_lock);
-+err_get_portal:
-+err_copy_from_user:
-+ kfree(mapping);
-+ return ret;
-+}
-+
-+static long ioctl_portal_unmap(struct ctx *ctx, struct usdpaa_portal_map *i)
-+{
-+ struct portal_mapping *mapping;
-+ struct vm_area_struct *vma;
-+ unsigned long pfn;
-+ u32 channel;
-+
-+ /* Get the PFN corresponding to one of the virt addresses */
-+ down_read(¤t->mm->mmap_sem);
-+ vma = find_vma(current->mm, (unsigned long)i->cinh);
-+ if (!vma || (vma->vm_start > (unsigned long)i->cinh)) {
-+ up_read(¤t->mm->mmap_sem);
-+ return -EFAULT;
-+ }
-+ pfn = vma->vm_pgoff;
-+ up_read(¤t->mm->mmap_sem);
-+
-+ /* Find the corresponding portal */
-+ spin_lock(&mem_lock);
-+ list_for_each_entry(mapping, &ctx->portals, list) {
-+ if (pfn == (mapping->phys[DPA_PORTAL_CI].start >> PAGE_SHIFT))
-+ goto found;
-+ }
-+ mapping = NULL;
-+found:
-+ if (mapping)
-+ list_del(&mapping->list);
-+ spin_unlock(&mem_lock);
-+ if (!mapping)
-+ return -ENODEV;
-+ portal_munmap(&mapping->phys[DPA_PORTAL_CI], mapping->user.addr.cinh);
-+ portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena);
-+ if (mapping->user.type == usdpaa_portal_qman) {
-+ init_qm_portal(mapping->qportal,
-+ &mapping->qman_portal_low);
-+
-+ /* Tear down any FQs this portal is referencing */
-+ channel = mapping->qportal->public_cfg.channel;
-+ qm_check_and_destroy_fqs(&mapping->qman_portal_low,
-+ &channel,
-+ check_portal_channel);
-+ qm_put_unused_portal(mapping->qportal);
-+ } else if (mapping->user.type == usdpaa_portal_bman) {
-+ init_bm_portal(mapping->bportal,
-+ &mapping->bman_portal_low);
-+ bm_put_unused_portal(mapping->bportal);
-+ }
-+ kfree(mapping);
-+ return 0;
-+}
-+
-+static void portal_config_pamu(struct qm_portal_config *pcfg, uint8_t sdest,
-+ uint32_t cpu, uint32_t cache, uint32_t window)
-+{
-+#ifdef CONFIG_FSL_PAMU
-+ int ret;
-+ int window_count = 1;
-+ struct iommu_domain_geometry geom_attr;
-+ struct pamu_stash_attribute stash_attr;
-+
-+ pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type);
-+ if (!pcfg->iommu_domain) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed",
-+ __func__);
-+ goto _no_iommu;
-+ }
-+ geom_attr.aperture_start = 0;
-+ geom_attr.aperture_end =
-+ ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1;
-+ geom_attr.force_aperture = true;
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY,
-+ &geom_attr);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS,
-+ &window_count);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ stash_attr.cpu = cpu;
-+ stash_attr.cache = cache;
-+ /* set stash information for the window */
-+ stash_attr.window = 0;
-+
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain,
-+ DOMAIN_ATTR_FSL_PAMU_STASH,
-+ &stash_attr);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36,
-+ IOMMU_READ | IOMMU_WRITE);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain,
-+ DOMAIN_ATTR_FSL_PAMU_ENABLE,
-+ &window_count);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_detach_device;
-+ }
-+_no_iommu:
-+#endif
-+
-+#ifdef CONFIG_FSL_QMAN_CONFIG
-+ if (qman_set_sdest(pcfg->public_cfg.channel, sdest))
-+#endif
-+ pr_warn("Failed to set QMan portal's stash request queue\n");
-+
-+ return;
-+
-+#ifdef CONFIG_FSL_PAMU
-+_iommu_detach_device:
-+ iommu_detach_device(pcfg->iommu_domain, NULL);
-+_iommu_domain_free:
-+ iommu_domain_free(pcfg->iommu_domain);
-+#endif
-+}
-+
-+static long ioctl_allocate_raw_portal(struct file *fp, struct ctx *ctx,
-+ struct usdpaa_ioctl_raw_portal *arg)
-+{
-+ struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
-+ int ret;
-+
-+ if (!mapping)
-+ return -ENOMEM;
-+
-+ mapping->user.type = arg->type;
-+ mapping->iommu_domain = NULL;
-+ if (arg->type == usdpaa_portal_qman) {
-+ mapping->qportal = qm_get_unused_portal_idx(arg->index);
-+ if (!mapping->qportal) {
-+ ret = -ENODEV;
-+ goto err;
-+ }
-+ mapping->phys = &mapping->qportal->addr_phys[0];
-+ arg->index = mapping->qportal->public_cfg.index;
-+ arg->cinh = mapping->qportal->addr_phys[DPA_PORTAL_CI].start;
-+ arg->cena = mapping->qportal->addr_phys[DPA_PORTAL_CE].start;
-+ if (arg->enable_stash) {
-+ /* Setup the PAMU with the supplied parameters */
-+ portal_config_pamu(mapping->qportal, arg->sdest,
-+ arg->cpu, arg->cache, arg->window);
-+ }
-+ } else if (mapping->user.type == usdpaa_portal_bman) {
-+ mapping->bportal =
-+ bm_get_unused_portal_idx(arg->index);
-+ if (!mapping->bportal) {
-+ ret = -ENODEV;
-+ goto err;
-+ }
-+ mapping->phys = &mapping->bportal->addr_phys[0];
-+ arg->index = mapping->bportal->public_cfg.index;
-+ arg->cinh = mapping->bportal->addr_phys[DPA_PORTAL_CI].start;
-+ arg->cena = mapping->bportal->addr_phys[DPA_PORTAL_CE].start;
-+ } else {
-+ ret = -EINVAL;
-+ goto err;
-+ }
-+ /* Need to put pcfg in ctx's list before the mmaps because the mmap
-+ * handlers look it up. */
-+ spin_lock(&mem_lock);
-+ list_add(&mapping->list, &ctx->portals);
-+ spin_unlock(&mem_lock);
-+ return 0;
-+err:
-+ kfree(mapping);
-+ return ret;
-+}
-+
-+static long ioctl_free_raw_portal(struct file *fp, struct ctx *ctx,
-+ struct usdpaa_ioctl_raw_portal *arg)
-+{
-+ struct portal_mapping *mapping;
-+ u32 channel;
-+
-+ /* Find the corresponding portal */
-+ spin_lock(&mem_lock);
-+ list_for_each_entry(mapping, &ctx->portals, list) {
-+ if (mapping->phys[DPA_PORTAL_CI].start == arg->cinh)
-+ goto found;
-+ }
-+ mapping = NULL;
-+found:
-+ if (mapping)
-+ list_del(&mapping->list);
-+ spin_unlock(&mem_lock);
-+ if (!mapping)
-+ return -ENODEV;
-+ if (mapping->user.type == usdpaa_portal_qman) {
-+ init_qm_portal(mapping->qportal,
-+ &mapping->qman_portal_low);
-+
-+ /* Tear down any FQs this portal is referencing */
-+ channel = mapping->qportal->public_cfg.channel;
-+ qm_check_and_destroy_fqs(&mapping->qman_portal_low,
-+ &channel,
-+ check_portal_channel);
-+ qm_put_unused_portal(mapping->qportal);
-+ } else if (mapping->user.type == usdpaa_portal_bman) {
-+ init_bm_portal(mapping->bportal,
-+ &mapping->bman_portal_low);
-+ bm_put_unused_portal(mapping->bportal);
-+ }
-+ kfree(mapping);
-+ return 0;
-+}
-+
-+static long usdpaa_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
-+{
-+ struct ctx *ctx = fp->private_data;
-+ void __user *a = (void __user *)arg;
-+ switch (cmd) {
-+ case USDPAA_IOCTL_ID_ALLOC:
-+ return ioctl_id_alloc(ctx, a);
-+ case USDPAA_IOCTL_ID_RELEASE:
-+ return ioctl_id_release(ctx, a);
-+ case USDPAA_IOCTL_ID_RESERVE:
-+ return ioctl_id_reserve(ctx, a);
-+ case USDPAA_IOCTL_DMA_MAP:
-+ {
-+ struct usdpaa_ioctl_dma_map input;
-+ int ret;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ ret = ioctl_dma_map(fp, ctx, &input);
-+ if (copy_to_user(a, &input, sizeof(input)))
-+ return -EFAULT;
-+ return ret;
-+ }
-+ case USDPAA_IOCTL_DMA_UNMAP:
-+ return ioctl_dma_unmap(ctx, a);
-+ case USDPAA_IOCTL_DMA_LOCK:
-+ return ioctl_dma_lock(ctx, a);
-+ case USDPAA_IOCTL_DMA_UNLOCK:
-+ return ioctl_dma_unlock(ctx, a);
-+ case USDPAA_IOCTL_PORTAL_MAP:
-+ {
-+ struct usdpaa_ioctl_portal_map input;
-+ int ret;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ ret = ioctl_portal_map(fp, ctx, &input);
-+ if (copy_to_user(a, &input, sizeof(input)))
-+ return -EFAULT;
-+ return ret;
-+ }
-+ case USDPAA_IOCTL_PORTAL_UNMAP:
-+ {
-+ struct usdpaa_portal_map input;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ return ioctl_portal_unmap(ctx, &input);
-+ }
-+ case USDPAA_IOCTL_DMA_USED:
-+ return ioctl_dma_stats(ctx, a);
-+ case USDPAA_IOCTL_ALLOC_RAW_PORTAL:
-+ {
-+ struct usdpaa_ioctl_raw_portal input;
-+ int ret;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ ret = ioctl_allocate_raw_portal(fp, ctx, &input);
-+ if (copy_to_user(a, &input, sizeof(input)))
-+ return -EFAULT;
-+ return ret;
-+ }
-+ case USDPAA_IOCTL_FREE_RAW_PORTAL:
-+ {
-+ struct usdpaa_ioctl_raw_portal input;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ return ioctl_free_raw_portal(fp, ctx, &input);
-+ }
-+ }
-+ return -EINVAL;
-+}
-+
-+static long usdpaa_ioctl_compat(struct file *fp, unsigned int cmd,
-+ unsigned long arg)
-+{
-+#ifdef CONFIG_COMPAT
-+ struct ctx *ctx = fp->private_data;
-+ void __user *a = (void __user *)arg;
-+#endif
-+ switch (cmd) {
-+#ifdef CONFIG_COMPAT
-+ case USDPAA_IOCTL_DMA_MAP_COMPAT:
-+ {
-+ int ret;
-+ struct usdpaa_ioctl_dma_map_compat input;
-+ struct usdpaa_ioctl_dma_map converted;
-+
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+
-+ converted.ptr = compat_ptr(input.ptr);
-+ converted.phys_addr = input.phys_addr;
-+ converted.len = input.len;
-+ converted.flags = input.flags;
-+ strncpy(converted.name, input.name, USDPAA_DMA_NAME_MAX);
-+ converted.has_locking = input.has_locking;
-+ converted.did_create = input.did_create;
-+
-+ ret = ioctl_dma_map(fp, ctx, &converted);
-+ input.ptr = ptr_to_compat(converted.ptr);
-+ input.phys_addr = converted.phys_addr;
-+ input.len = converted.len;
-+ input.flags = converted.flags;
-+ strncpy(input.name, converted.name, USDPAA_DMA_NAME_MAX);
-+ input.has_locking = converted.has_locking;
-+ input.did_create = converted.did_create;
-+ if (copy_to_user(a, &input, sizeof(input)))
-+ return -EFAULT;
-+ return ret;
-+ }
-+ case USDPAA_IOCTL_PORTAL_MAP_COMPAT:
-+ {
-+ int ret;
-+ struct compat_usdpaa_ioctl_portal_map input;
-+ struct usdpaa_ioctl_portal_map converted;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ converted.type = input.type;
-+ converted.index = input.index;
-+ ret = ioctl_portal_map(fp, ctx, &converted);
-+ input.addr.cinh = ptr_to_compat(converted.addr.cinh);
-+ input.addr.cena = ptr_to_compat(converted.addr.cena);
-+ input.channel = converted.channel;
-+ input.pools = converted.pools;
-+ input.index = converted.index;
-+ if (copy_to_user(a, &input, sizeof(input)))
-+ return -EFAULT;
-+ return ret;
-+ }
-+ case USDPAA_IOCTL_PORTAL_UNMAP_COMPAT:
-+ {
-+ struct usdpaa_portal_map_compat input;
-+ struct usdpaa_portal_map converted;
-+
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ converted.cinh = compat_ptr(input.cinh);
-+ converted.cena = compat_ptr(input.cena);
-+ return ioctl_portal_unmap(ctx, &converted);
-+ }
-+ case USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT:
-+ {
-+ int ret;
-+ struct usdpaa_ioctl_raw_portal converted;
-+ struct compat_ioctl_raw_portal input;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ converted.type = input.type;
-+ converted.index = input.index;
-+ converted.enable_stash = input.enable_stash;
-+ converted.cpu = input.cpu;
-+ converted.cache = input.cache;
-+ converted.window = input.window;
-+ converted.sdest = input.sdest;
-+ ret = ioctl_allocate_raw_portal(fp, ctx, &converted);
-+
-+ input.cinh = converted.cinh;
-+ input.cena = converted.cena;
-+ input.index = converted.index;
-+
-+ if (copy_to_user(a, &input, sizeof(input)))
-+ return -EFAULT;
-+ return ret;
-+ }
-+ case USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT:
-+ {
-+ struct usdpaa_ioctl_raw_portal converted;
-+ struct compat_ioctl_raw_portal input;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ converted.type = input.type;
-+ converted.index = input.index;
-+ converted.cinh = input.cinh;
-+ converted.cena = input.cena;
-+ return ioctl_free_raw_portal(fp, ctx, &converted);
-+ }
-+#endif
-+ default:
-+ return usdpaa_ioctl(fp, cmd, arg);
-+ }
-+ return -EINVAL;
-+}
-+
-+int usdpaa_get_portal_config(struct file *filp, void *cinh,
-+ enum usdpaa_portal_type ptype, unsigned int *irq,
-+ void **iir_reg)
-+{
-+ /* Walk the list of portals for filp and return the config
-+ for the portal that matches the hint */
-+ struct ctx *context;
-+ struct portal_mapping *portal;
-+
-+ /* First sanitize the filp */
-+ if (filp->f_op->open != usdpaa_open)
-+ return -ENODEV;
-+ context = filp->private_data;
-+ spin_lock(&context->lock);
-+ list_for_each_entry(portal, &context->portals, list) {
-+ if (portal->user.type == ptype &&
-+ portal->user.addr.cinh == cinh) {
-+ if (ptype == usdpaa_portal_qman) {
-+ *irq = portal->qportal->public_cfg.irq;
-+ *iir_reg = portal->qportal->addr_virt[1] +
-+ QM_REG_IIR;
-+ } else {
-+ *irq = portal->bportal->public_cfg.irq;
-+ *iir_reg = portal->bportal->addr_virt[1] +
-+ BM_REG_IIR;
-+ }
-+ spin_unlock(&context->lock);
-+ return 0;
-+ }
-+ }
-+ spin_unlock(&context->lock);
-+ return -EINVAL;
-+}
-+
-+static const struct file_operations usdpaa_fops = {
-+ .open = usdpaa_open,
-+ .release = usdpaa_release,
-+ .mmap = usdpaa_mmap,
-+ .get_unmapped_area = usdpaa_get_unmapped_area,
-+ .unlocked_ioctl = usdpaa_ioctl,
-+ .compat_ioctl = usdpaa_ioctl_compat
-+};
-+
-+static struct miscdevice usdpaa_miscdev = {
-+ .name = "fsl-usdpaa",
-+ .fops = &usdpaa_fops,
-+ .minor = MISC_DYNAMIC_MINOR,
-+};
-+
-+/* Early-boot memory allocation. The boot-arg "usdpaa_mem=<x>" is used to
-+ * indicate how much memory (if any) to allocate during early boot. If the
-+ * format "usdpaa_mem=<x>,<y>" is used, then <y> will be interpreted as the
-+ * number of TLB1 entries to reserve (default is 1). If there are more mappings
-+ * than there are TLB1 entries, fault-handling will occur. */
-+
-+static __init int usdpaa_mem(char *arg)
-+{
-+ pr_warn("uspdaa_mem argument is depracated\n");
-+ arg_phys_size = memparse(arg, &arg);
-+ num_tlb = 1;
-+ if (*arg == ',') {
-+ unsigned long ul;
-+ int err = kstrtoul(arg + 1, 0, &ul);
-+ if (err < 0) {
-+ num_tlb = 1;
-+ pr_warn("ERROR, usdpaa_mem arg is invalid\n");
-+ } else
-+ num_tlb = (unsigned int)ul;
-+ }
-+ return 0;
-+}
-+early_param("usdpaa_mem", usdpaa_mem);
-+
-+static int usdpaa_mem_init(struct reserved_mem *rmem)
-+{
-+ phys_start = rmem->base;
-+ phys_size = rmem->size;
-+
-+ WARN_ON(!(phys_start && phys_size));
-+
-+ return 0;
-+}
-+RESERVEDMEM_OF_DECLARE(usdpaa_mem_init, "fsl,usdpaa-mem", usdpaa_mem_init);
-+
-+__init int fsl_usdpaa_init_early(void)
-+{
-+ if (!phys_size || !phys_start) {
-+ pr_info("No USDPAA memory, no 'fsl,usdpaa-mem' in device-tree\n");
-+ return 0;
-+ }
-+ if (phys_size % PAGE_SIZE) {
-+ pr_err("'fsl,usdpaa-mem' size must be a multiple of page size\n");
-+ phys_size = 0;
-+ return 0;
-+ }
-+ if (arg_phys_size && phys_size != arg_phys_size) {
-+ pr_err("'usdpaa_mem argument size (0x%llx) does not match device tree size (0x%llx)\n",
-+ arg_phys_size, phys_size);
-+ phys_size = 0;
-+ return 0;
-+ }
-+ pfn_start = phys_start >> PAGE_SHIFT;
-+ pfn_size = phys_size >> PAGE_SHIFT;
-+#ifdef CONFIG_PPC
-+ first_tlb = current_tlb = tlbcam_index;
-+ tlbcam_index += num_tlb;
-+#endif
-+ pr_info("USDPAA region at %llx:%llx(%lx:%lx), %d TLB1 entries)\n",
-+ phys_start, phys_size, pfn_start, pfn_size, num_tlb);
-+ return 0;
-+}
-+subsys_initcall(fsl_usdpaa_init_early);
-+
-+
-+static int __init usdpaa_init(void)
-+{
-+ struct mem_fragment *frag;
-+ int ret;
-+ u64 tmp_size = phys_size;
-+ u64 tmp_start = phys_start;
-+ u64 tmp_pfn_size = pfn_size;
-+ u64 tmp_pfn_start = pfn_start;
-+
-+ pr_info("Freescale USDPAA process driver\n");
-+ if (!phys_start) {
-+ pr_warn("fsl-usdpaa: no region found\n");
-+ return 0;
-+ }
-+
-+ while (tmp_size != 0) {
-+ u32 frag_size = largest_page_size(tmp_size);
-+ frag = kmalloc(sizeof(*frag), GFP_KERNEL);
-+ if (!frag) {
-+ pr_err("Failed to setup USDPAA memory accounting\n");
-+ return -ENOMEM;
-+ }
-+ frag->base = tmp_start;
-+ frag->len = frag->root_len = frag_size;
-+ frag->root_pfn = tmp_pfn_start;
-+ frag->pfn_base = tmp_pfn_start;
-+ frag->pfn_len = frag_size / PAGE_SIZE;
-+ frag->refs = 0;
-+ init_waitqueue_head(&frag->wq);
-+ frag->owner = NULL;
-+ list_add(&frag->list, &mem_list);
-+
-+ /* Adjust for this frag */
-+ tmp_start += frag_size;
-+ tmp_size -= frag_size;
-+ tmp_pfn_start += frag_size / PAGE_SIZE;
-+ tmp_pfn_size -= frag_size / PAGE_SIZE;
-+ }
-+ ret = misc_register(&usdpaa_miscdev);
-+ if (ret)
-+ pr_err("fsl-usdpaa: failed to register misc device\n");
-+ return ret;
-+}
-+
-+static void __exit usdpaa_exit(void)
-+{
-+ misc_deregister(&usdpaa_miscdev);
-+}
-+
-+module_init(usdpaa_init);
-+module_exit(usdpaa_exit);
-+
-+MODULE_LICENSE("GPL");
-+MODULE_AUTHOR("Freescale Semiconductor");
-+MODULE_DESCRIPTION("Freescale USDPAA process driver");
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/fsl_usdpaa_irq.c
-@@ -0,0 +1,289 @@
-+/* Copyright (c) 2013 Freescale Semiconductor, Inc.
-+ * All rights reserved.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+/* define a device that allows USPDAA processes to open a file
-+ descriptor and specify which IRQ it wants to montior using an ioctl()
-+ When an IRQ is received, the device becomes readable so that a process
-+ can use read() or select() type calls to monitor for IRQs */
-+
-+#include <linux/miscdevice.h>
-+#include <linux/fs.h>
-+#include <linux/cdev.h>
-+#include <linux/slab.h>
-+#include <linux/interrupt.h>
-+#include <linux/poll.h>
-+#include <linux/uaccess.h>
-+#include <linux/fsl_usdpaa.h>
-+#include <linux/module.h>
-+#include <linux/fdtable.h>
-+#include <linux/file.h>
-+
-+#include "qman_low.h"
-+#include "bman_low.h"
-+
-+struct usdpaa_irq_ctx {
-+ int irq_set; /* Set to true once the irq is set via ioctl */
-+ unsigned int irq_num;
-+ u32 last_irq_count; /* Last value returned from read */
-+ u32 irq_count; /* Number of irqs since last read */
-+ wait_queue_head_t wait_queue; /* Waiting processes */
-+ spinlock_t lock;
-+ void *inhibit_addr; /* inhibit register address */
-+ struct file *usdpaa_filp;
-+ char irq_name[128];
-+};
-+
-+static int usdpaa_irq_open(struct inode *inode, struct file *filp)
-+{
-+ struct usdpaa_irq_ctx *ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
-+ if (!ctx)
-+ return -ENOMEM;
-+ ctx->irq_set = 0;
-+ ctx->irq_count = 0;
-+ ctx->last_irq_count = 0;
-+ init_waitqueue_head(&ctx->wait_queue);
-+ spin_lock_init(&ctx->lock);
-+ filp->private_data = ctx;
-+ return 0;
-+}
-+
-+static int usdpaa_irq_release(struct inode *inode, struct file *filp)
-+{
-+ struct usdpaa_irq_ctx *ctx = filp->private_data;
-+ if (ctx->irq_set) {
-+ /* Inhibit the IRQ */
-+ out_be32(ctx->inhibit_addr, 0x1);
-+ irq_set_affinity_hint(ctx->irq_num, NULL);
-+ free_irq(ctx->irq_num, ctx);
-+ ctx->irq_set = 0;
-+ fput(ctx->usdpaa_filp);
-+ }
-+ kfree(filp->private_data);
-+ return 0;
-+}
-+
-+static irqreturn_t usdpaa_irq_handler(int irq, void *_ctx)
-+{
-+ unsigned long flags;
-+ struct usdpaa_irq_ctx *ctx = _ctx;
-+ spin_lock_irqsave(&ctx->lock, flags);
-+ ++ctx->irq_count;
-+ spin_unlock_irqrestore(&ctx->lock, flags);
-+ wake_up_all(&ctx->wait_queue);
-+ /* Set the inhibit register. This will be reenabled
-+ once the USDPAA code handles the IRQ */
-+ out_be32(ctx->inhibit_addr, 0x1);
-+ pr_info("Inhibit at %p count %d", ctx->inhibit_addr, ctx->irq_count);
-+ return IRQ_HANDLED;
-+}
-+
-+static int map_irq(struct file *fp, struct usdpaa_ioctl_irq_map *irq_map)
-+{
-+ struct usdpaa_irq_ctx *ctx = fp->private_data;
-+ int ret;
-+
-+ if (ctx->irq_set) {
-+ pr_debug("Setting USDPAA IRQ when it was already set!\n");
-+ return -EBUSY;
-+ }
-+
-+ ctx->usdpaa_filp = fget(irq_map->fd);
-+ if (!ctx->usdpaa_filp) {
-+ pr_debug("USDPAA fget(%d) returned NULL\n", irq_map->fd);
-+ return -EINVAL;
-+ }
-+
-+ ret = usdpaa_get_portal_config(ctx->usdpaa_filp, irq_map->portal_cinh,
-+ irq_map->type, &ctx->irq_num,
-+ &ctx->inhibit_addr);
-+ if (ret) {
-+ pr_debug("USDPAA IRQ couldn't identify portal\n");
-+ fput(ctx->usdpaa_filp);
-+ return ret;
-+ }
-+
-+ ctx->irq_set = 1;
-+
-+ snprintf(ctx->irq_name, sizeof(ctx->irq_name),
-+ "usdpaa_irq %d", ctx->irq_num);
-+
-+ ret = request_irq(ctx->irq_num, usdpaa_irq_handler, 0,
-+ ctx->irq_name, ctx);
-+ if (ret) {
-+ pr_err("USDPAA request_irq(%d) failed, ret= %d\n",
-+ ctx->irq_num, ret);
-+ ctx->irq_set = 0;
-+ fput(ctx->usdpaa_filp);
-+ return ret;
-+ }
-+ ret = irq_set_affinity(ctx->irq_num, ¤t->cpus_allowed);
-+ if (ret)
-+ pr_err("USDPAA irq_set_affinity() failed, ret= %d\n", ret);
-+
-+ ret = irq_set_affinity_hint(ctx->irq_num, ¤t->cpus_allowed);
-+ if (ret)
-+ pr_err("USDPAA irq_set_affinity_hint() failed, ret= %d\n", ret);
-+
-+ return 0;
-+}
-+
-+static long usdpaa_irq_ioctl(struct file *fp, unsigned int cmd,
-+ unsigned long arg)
-+{
-+ int ret;
-+ struct usdpaa_ioctl_irq_map irq_map;
-+
-+ if (cmd != USDPAA_IOCTL_PORTAL_IRQ_MAP) {
-+ pr_debug("USDPAA IRQ unknown command 0x%x\n", cmd);
-+ return -EINVAL;
-+ }
-+
-+ ret = copy_from_user(&irq_map, (void __user *)arg,
-+ sizeof(irq_map));
-+ if (ret)
-+ return ret;
-+ return map_irq(fp, &irq_map);
-+}
-+
-+static ssize_t usdpaa_irq_read(struct file *filp, char __user *buff,
-+ size_t count, loff_t *offp)
-+{
-+ struct usdpaa_irq_ctx *ctx = filp->private_data;
-+ int ret;
-+
-+ if (!ctx->irq_set) {
-+ pr_debug("Reading USDPAA IRQ before it was set\n");
-+ return -EINVAL;
-+ }
-+
-+ if (count < sizeof(ctx->irq_count)) {
-+ pr_debug("USDPAA IRQ Read too small\n");
-+ return -EINVAL;
-+ }
-+ if (ctx->irq_count == ctx->last_irq_count) {
-+ if (filp->f_flags & O_NONBLOCK)
-+ return -EAGAIN;
-+
-+ ret = wait_event_interruptible(ctx->wait_queue,
-+ ctx->irq_count != ctx->last_irq_count);
-+ if (ret == -ERESTARTSYS)
-+ return ret;
-+ }
-+
-+ ctx->last_irq_count = ctx->irq_count;
-+
-+ if (copy_to_user(buff, &ctx->last_irq_count,
-+ sizeof(ctx->last_irq_count)))
-+ return -EFAULT;
-+ return sizeof(ctx->irq_count);
-+}
-+
-+static unsigned int usdpaa_irq_poll(struct file *filp, poll_table *wait)
-+{
-+ struct usdpaa_irq_ctx *ctx = filp->private_data;
-+ unsigned int ret = 0;
-+ unsigned long flags;
-+
-+ if (!ctx->irq_set)
-+ return POLLHUP;
-+
-+ poll_wait(filp, &ctx->wait_queue, wait);
-+
-+ spin_lock_irqsave(&ctx->lock, flags);
-+ if (ctx->irq_count != ctx->last_irq_count)
-+ ret |= POLLIN | POLLRDNORM;
-+ spin_unlock_irqrestore(&ctx->lock, flags);
-+ return ret;
-+}
-+
-+static long usdpaa_irq_ioctl_compat(struct file *fp, unsigned int cmd,
-+ unsigned long arg)
-+{
-+#ifdef CONFIG_COMPAT
-+ void __user *a = (void __user *)arg;
-+#endif
-+ switch (cmd) {
-+#ifdef CONFIG_COMPAT
-+ case USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT:
-+ {
-+ struct compat_ioctl_irq_map input;
-+ struct usdpaa_ioctl_irq_map converted;
-+ if (copy_from_user(&input, a, sizeof(input)))
-+ return -EFAULT;
-+ converted.type = input.type;
-+ converted.fd = input.fd;
-+ converted.portal_cinh = compat_ptr(input.portal_cinh);
-+ return map_irq(fp, &converted);
-+ }
-+#endif
-+ default:
-+ return usdpaa_irq_ioctl(fp, cmd, arg);
-+ }
-+}
-+
-+static const struct file_operations usdpaa_irq_fops = {
-+ .open = usdpaa_irq_open,
-+ .release = usdpaa_irq_release,
-+ .unlocked_ioctl = usdpaa_irq_ioctl,
-+ .compat_ioctl = usdpaa_irq_ioctl_compat,
-+ .read = usdpaa_irq_read,
-+ .poll = usdpaa_irq_poll
-+};
-+
-+static struct miscdevice usdpaa_miscdev = {
-+ .name = "fsl-usdpaa-irq",
-+ .fops = &usdpaa_irq_fops,
-+ .minor = MISC_DYNAMIC_MINOR,
-+};
-+
-+static int __init usdpaa_irq_init(void)
-+{
-+ int ret;
-+
-+ pr_info("Freescale USDPAA process IRQ driver\n");
-+ ret = misc_register(&usdpaa_miscdev);
-+ if (ret)
-+ pr_err("fsl-usdpaa-irq: failed to register misc device\n");
-+ return ret;
-+}
-+
-+static void __exit usdpaa_irq_exit(void)
-+{
-+ misc_deregister(&usdpaa_miscdev);
-+}
-+
-+module_init(usdpaa_irq_init);
-+module_exit(usdpaa_irq_exit);
-+
-+MODULE_LICENSE("GPL");
-+MODULE_AUTHOR("Freescale Semiconductor");
-+MODULE_DESCRIPTION("Freescale USDPAA process IRQ driver");
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qbman_driver.c
-@@ -0,0 +1,88 @@
-+/* Copyright 2013 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include <linux/time.h>
-+#include "qman_private.h"
-+#include "bman_private.h"
-+__init void qman_init_early(void);
-+__init void bman_init_early(void);
-+
-+static __init int qbman_init(void)
-+{
-+ struct device_node *dn;
-+ u32 is_portal_available;
-+
-+ bman_init();
-+ qman_init();
-+
-+ is_portal_available = 0;
-+ for_each_compatible_node(dn, NULL, "fsl,qman-portal") {
-+ if (!of_device_is_available(dn))
-+ continue;
-+ else
-+ is_portal_available = 1;
-+ }
-+
-+ if (!qman_have_ccsr() && is_portal_available) {
-+ struct qman_fq fq = {
-+ .fqid = 1
-+ };
-+ struct qm_mcr_queryfq_np np;
-+ int err, retry = CONFIG_FSL_QMAN_INIT_TIMEOUT;
-+ struct timespec nowts, diffts, startts = current_kernel_time();
-+ /* Loop while querying given fqid succeeds or time out */
-+ while (1) {
-+ err = qman_query_fq_np(&fq, &np);
-+ if (!err) {
-+ /* success, control-plane has configured QMan */
-+ break;
-+ } else if (err != -ERANGE) {
-+ pr_err("QMan: I/O error, continuing anyway\n");
-+ break;
-+ }
-+ nowts = current_kernel_time();
-+ diffts = timespec_sub(nowts, startts);
-+ if (diffts.tv_sec > 0) {
-+ if (!retry--) {
-+ pr_err("QMan: time out, control-plane"
-+ " dead?\n");
-+ break;
-+ }
-+ pr_warn("QMan: polling for the control-plane"
-+ " (%d)\n", retry);
-+ }
-+ }
-+ }
-+ bman_resource_init();
-+ qman_resource_init();
-+ return 0;
-+}
-+subsys_initcall(qbman_init);
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_config.c
-@@ -0,0 +1,1224 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include <asm/cacheflush.h>
-+#include "qman_private.h"
-+#include <linux/highmem.h>
-+#include <linux/of_reserved_mem.h>
-+
-+/* Last updated for v00.800 of the BG */
-+
-+/* Register offsets */
-+#define REG_QCSP_LIO_CFG(n) (0x0000 + ((n) * 0x10))
-+#define REG_QCSP_IO_CFG(n) (0x0004 + ((n) * 0x10))
-+#define REG_QCSP_DD_CFG(n) (0x000c + ((n) * 0x10))
-+#define REG_DD_CFG 0x0200
-+#define REG_DCP_CFG(n) (0x0300 + ((n) * 0x10))
-+#define REG_DCP_DD_CFG(n) (0x0304 + ((n) * 0x10))
-+#define REG_DCP_DLM_AVG(n) (0x030c + ((n) * 0x10))
-+#define REG_PFDR_FPC 0x0400
-+#define REG_PFDR_FP_HEAD 0x0404
-+#define REG_PFDR_FP_TAIL 0x0408
-+#define REG_PFDR_FP_LWIT 0x0410
-+#define REG_PFDR_CFG 0x0414
-+#define REG_SFDR_CFG 0x0500
-+#define REG_SFDR_IN_USE 0x0504
-+#define REG_WQ_CS_CFG(n) (0x0600 + ((n) * 0x04))
-+#define REG_WQ_DEF_ENC_WQID 0x0630
-+#define REG_WQ_SC_DD_CFG(n) (0x640 + ((n) * 0x04))
-+#define REG_WQ_PC_DD_CFG(n) (0x680 + ((n) * 0x04))
-+#define REG_WQ_DC0_DD_CFG(n) (0x6c0 + ((n) * 0x04))
-+#define REG_WQ_DC1_DD_CFG(n) (0x700 + ((n) * 0x04))
-+#define REG_WQ_DCn_DD_CFG(n) (0x6c0 + ((n) * 0x40)) /* n=2,3 */
-+#define REG_CM_CFG 0x0800
-+#define REG_ECSR 0x0a00
-+#define REG_ECIR 0x0a04
-+#define REG_EADR 0x0a08
-+#define REG_ECIR2 0x0a0c
-+#define REG_EDATA(n) (0x0a10 + ((n) * 0x04))
-+#define REG_SBEC(n) (0x0a80 + ((n) * 0x04))
-+#define REG_MCR 0x0b00
-+#define REG_MCP(n) (0x0b04 + ((n) * 0x04))
-+#define REG_MISC_CFG 0x0be0
-+#define REG_HID_CFG 0x0bf0
-+#define REG_IDLE_STAT 0x0bf4
-+#define REG_IP_REV_1 0x0bf8
-+#define REG_IP_REV_2 0x0bfc
-+#define REG_FQD_BARE 0x0c00
-+#define REG_PFDR_BARE 0x0c20
-+#define REG_offset_BAR 0x0004 /* relative to REG_[FQD|PFDR]_BARE */
-+#define REG_offset_AR 0x0010 /* relative to REG_[FQD|PFDR]_BARE */
-+#define REG_QCSP_BARE 0x0c80
-+#define REG_QCSP_BAR 0x0c84
-+#define REG_CI_SCHED_CFG 0x0d00
-+#define REG_SRCIDR 0x0d04
-+#define REG_LIODNR 0x0d08
-+#define REG_CI_RLM_AVG 0x0d14
-+#define REG_ERR_ISR 0x0e00 /* + "enum qm_isr_reg" */
-+#define REG_REV3_QCSP_LIO_CFG(n) (0x1000 + ((n) * 0x10))
-+#define REG_REV3_QCSP_IO_CFG(n) (0x1004 + ((n) * 0x10))
-+#define REG_REV3_QCSP_DD_CFG(n) (0x100c + ((n) * 0x10))
-+#define REG_CEETM_CFG_IDX 0x900
-+#define REG_CEETM_CFG_PRES 0x904
-+#define REG_CEETM_XSFDR_IN_USE 0x908
-+
-+/* Assists for QMAN_MCR */
-+#define MCR_INIT_PFDR 0x01000000
-+#define MCR_get_rslt(v) (u8)((v) >> 24)
-+#define MCR_rslt_idle(r) (!rslt || (rslt >= 0xf0))
-+#define MCR_rslt_ok(r) (rslt == 0xf0)
-+#define MCR_rslt_eaccess(r) (rslt == 0xf8)
-+#define MCR_rslt_inval(r) (rslt == 0xff)
-+
-+struct qman;
-+
-+/* Follows WQ_CS_CFG0-5 */
-+enum qm_wq_class {
-+ qm_wq_portal = 0,
-+ qm_wq_pool = 1,
-+ qm_wq_fman0 = 2,
-+ qm_wq_fman1 = 3,
-+ qm_wq_caam = 4,
-+ qm_wq_pme = 5,
-+ qm_wq_first = qm_wq_portal,
-+ qm_wq_last = qm_wq_pme
-+};
-+
-+/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */
-+enum qm_memory {
-+ qm_memory_fqd,
-+ qm_memory_pfdr
-+};
-+
-+/* Used by all error interrupt registers except 'inhibit' */
-+#define QM_EIRQ_CIDE 0x20000000 /* Corenet Initiator Data Error */
-+#define QM_EIRQ_CTDE 0x10000000 /* Corenet Target Data Error */
-+#define QM_EIRQ_CITT 0x08000000 /* Corenet Invalid Target Transaction */
-+#define QM_EIRQ_PLWI 0x04000000 /* PFDR Low Watermark */
-+#define QM_EIRQ_MBEI 0x02000000 /* Multi-bit ECC Error */
-+#define QM_EIRQ_SBEI 0x01000000 /* Single-bit ECC Error */
-+#define QM_EIRQ_PEBI 0x00800000 /* PFDR Enqueues Blocked Interrupt */
-+#define QM_EIRQ_IFSI 0x00020000 /* Invalid FQ Flow Control State */
-+#define QM_EIRQ_ICVI 0x00010000 /* Invalid Command Verb */
-+#define QM_EIRQ_IDDI 0x00000800 /* Invalid Dequeue (Direct-connect) */
-+#define QM_EIRQ_IDFI 0x00000400 /* Invalid Dequeue FQ */
-+#define QM_EIRQ_IDSI 0x00000200 /* Invalid Dequeue Source */
-+#define QM_EIRQ_IDQI 0x00000100 /* Invalid Dequeue Queue */
-+#define QM_EIRQ_IECE 0x00000010 /* Invalid Enqueue Configuration */
-+#define QM_EIRQ_IEOI 0x00000008 /* Invalid Enqueue Overflow */
-+#define QM_EIRQ_IESI 0x00000004 /* Invalid Enqueue State */
-+#define QM_EIRQ_IECI 0x00000002 /* Invalid Enqueue Channel */
-+#define QM_EIRQ_IEQI 0x00000001 /* Invalid Enqueue Queue */
-+
-+/* QMAN_ECIR valid error bit */
-+#define PORTAL_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \
-+ QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \
-+ QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI)
-+#define FQID_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \
-+ QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \
-+ QM_EIRQ_IFSI)
-+
-+union qman_ecir {
-+ u32 ecir_raw;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved:2;
-+ u32 portal_type:1;
-+ u32 portal_num:5;
-+ u32 fqid:24;
-+#else
-+ u32 fqid:24;
-+ u32 portal_num:5;
-+ u32 portal_type:1;
-+ u32 __reserved:2;
-+#endif
-+ } __packed info;
-+};
-+
-+union qman_ecir2 {
-+ u32 ecir2_raw;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 portal_type:1;
-+ u32 __reserved:21;
-+ u32 portal_num:10;
-+#else
-+ u32 portal_num:10;
-+ u32 __reserved:21;
-+ u32 portal_type:1;
-+#endif
-+ } __packed info;
-+};
-+
-+union qman_eadr {
-+ u32 eadr_raw;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved1:4;
-+ u32 memid:4;
-+ u32 __reserved2:12;
-+ u32 eadr:12;
-+#else
-+ u32 eadr:12;
-+ u32 __reserved2:12;
-+ u32 memid:4;
-+ u32 __reserved1:4;
-+#endif
-+ } __packed info;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved1:3;
-+ u32 memid:5;
-+ u32 __reserved:8;
-+ u32 eadr:16;
-+#else
-+ u32 eadr:16;
-+ u32 __reserved:8;
-+ u32 memid:5;
-+ u32 __reserved1:3;
-+#endif
-+ } __packed info_rev3;
-+};
-+
-+struct qman_hwerr_txt {
-+ u32 mask;
-+ const char *txt;
-+};
-+
-+#define QMAN_HWE_TXT(a, b) { .mask = QM_EIRQ_##a, .txt = b }
-+
-+static const struct qman_hwerr_txt qman_hwerr_txts[] = {
-+ QMAN_HWE_TXT(CIDE, "Corenet Initiator Data Error"),
-+ QMAN_HWE_TXT(CTDE, "Corenet Target Data Error"),
-+ QMAN_HWE_TXT(CITT, "Corenet Invalid Target Transaction"),
-+ QMAN_HWE_TXT(PLWI, "PFDR Low Watermark"),
-+ QMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"),
-+ QMAN_HWE_TXT(SBEI, "Single-bit ECC Error"),
-+ QMAN_HWE_TXT(PEBI, "PFDR Enqueues Blocked Interrupt"),
-+ QMAN_HWE_TXT(ICVI, "Invalid Command Verb"),
-+ QMAN_HWE_TXT(IFSI, "Invalid Flow Control State"),
-+ QMAN_HWE_TXT(IDDI, "Invalid Dequeue (Direct-connect)"),
-+ QMAN_HWE_TXT(IDFI, "Invalid Dequeue FQ"),
-+ QMAN_HWE_TXT(IDSI, "Invalid Dequeue Source"),
-+ QMAN_HWE_TXT(IDQI, "Invalid Dequeue Queue"),
-+ QMAN_HWE_TXT(IECE, "Invalid Enqueue Configuration"),
-+ QMAN_HWE_TXT(IEOI, "Invalid Enqueue Overflow"),
-+ QMAN_HWE_TXT(IESI, "Invalid Enqueue State"),
-+ QMAN_HWE_TXT(IECI, "Invalid Enqueue Channel"),
-+ QMAN_HWE_TXT(IEQI, "Invalid Enqueue Queue")
-+};
-+#define QMAN_HWE_COUNT (sizeof(qman_hwerr_txts)/sizeof(struct qman_hwerr_txt))
-+
-+struct qman_error_info_mdata {
-+ u16 addr_mask;
-+ u16 bits;
-+ const char *txt;
-+};
-+
-+#define QMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c}
-+static const struct qman_error_info_mdata error_mdata[] = {
-+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 0"),
-+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 1"),
-+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 2"),
-+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 3"),
-+ QMAN_ERR_MDATA(0x0FFF, 512, "FQD cache memory"),
-+ QMAN_ERR_MDATA(0x07FF, 128, "SFDR memory"),
-+ QMAN_ERR_MDATA(0x01FF, 72, "WQ context memory"),
-+ QMAN_ERR_MDATA(0x00FF, 240, "CGR memory"),
-+ QMAN_ERR_MDATA(0x00FF, 302, "Internal Order Restoration List memory"),
-+ QMAN_ERR_MDATA(0x01FF, 256, "SW portal ring memory"),
-+ QMAN_ERR_MDATA(0x07FF, 181, "CEETM class queue descriptor memory"),
-+ QMAN_ERR_MDATA(0x0FFF, 140, "CEETM extended SFDR memory"),
-+ QMAN_ERR_MDATA(0x0FFF, 25, "CEETM logical FQ mapping memory"),
-+ QMAN_ERR_MDATA(0x0FFF, 96, "CEETM dequeue context memory"),
-+ QMAN_ERR_MDATA(0x07FF, 396, "CEETM ccgr memory"),
-+ QMAN_ERR_MDATA(0x00FF, 146, "CEETM CQ channel shaping memory"),
-+ QMAN_ERR_MDATA(0x007F, 256, "CEETM CQ channel scheduling memory"),
-+ QMAN_ERR_MDATA(0x01FF, 88, "CEETM dequeue statistics memory"),
-+};
-+#define QMAN_ERR_MDATA_COUNT \
-+ (sizeof(error_mdata)/sizeof(struct qman_error_info_mdata))
-+
-+/* Add this in Kconfig */
-+#define QMAN_ERRS_TO_UNENABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI)
-+
-+/**
-+ * qm_err_isr_<reg>_<verb> - Manipulate global interrupt registers
-+ * @v: for accessors that write values, this is the 32-bit value
-+ *
-+ * Manipulates QMAN_ERR_ISR, QMAN_ERR_IER, QMAN_ERR_ISDR, QMAN_ERR_IIR. All
-+ * manipulations except qm_err_isr_[un]inhibit() use 32-bit masks composed of
-+ * the QM_EIRQ_*** definitions. Note that "qm_err_isr_enable_write" means
-+ * "write the enable register" rather than "enable the write register"!
-+ */
-+#define qm_err_isr_status_read(qm) \
-+ __qm_err_isr_read(qm, qm_isr_status)
-+#define qm_err_isr_status_clear(qm, m) \
-+ __qm_err_isr_write(qm, qm_isr_status, m)
-+#define qm_err_isr_enable_read(qm) \
-+ __qm_err_isr_read(qm, qm_isr_enable)
-+#define qm_err_isr_enable_write(qm, v) \
-+ __qm_err_isr_write(qm, qm_isr_enable, v)
-+#define qm_err_isr_disable_read(qm) \
-+ __qm_err_isr_read(qm, qm_isr_disable)
-+#define qm_err_isr_disable_write(qm, v) \
-+ __qm_err_isr_write(qm, qm_isr_disable, v)
-+#define qm_err_isr_inhibit(qm) \
-+ __qm_err_isr_write(qm, qm_isr_inhibit, 1)
-+#define qm_err_isr_uninhibit(qm) \
-+ __qm_err_isr_write(qm, qm_isr_inhibit, 0)
-+
-+/*
-+ * TODO: unimplemented registers
-+ *
-+ * Keeping a list here of Qman registers I have not yet covered;
-+ * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR,
-+ * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG,
-+ * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12
-+ */
-+
-+/* Encapsulate "struct qman *" as a cast of the register space address. */
-+
-+static struct qman *qm_create(void *regs)
-+{
-+ return (struct qman *)regs;
-+}
-+
-+static inline u32 __qm_in(struct qman *qm, u32 offset)
-+{
-+ return in_be32((void *)qm + offset);
-+}
-+static inline void __qm_out(struct qman *qm, u32 offset, u32 val)
-+{
-+ out_be32((void *)qm + offset, val);
-+}
-+#define qm_in(reg) __qm_in(qm, REG_##reg)
-+#define qm_out(reg, val) __qm_out(qm, REG_##reg, val)
-+
-+static u32 __qm_err_isr_read(struct qman *qm, enum qm_isr_reg n)
-+{
-+ return __qm_in(qm, REG_ERR_ISR + (n << 2));
-+}
-+
-+static void __qm_err_isr_write(struct qman *qm, enum qm_isr_reg n, u32 val)
-+{
-+ __qm_out(qm, REG_ERR_ISR + (n << 2), val);
-+}
-+
-+static void qm_set_dc(struct qman *qm, enum qm_dc_portal portal,
-+ int ed, u8 sernd)
-+{
-+ DPA_ASSERT(!ed || (portal == qm_dc_portal_fman0) ||
-+ (portal == qm_dc_portal_fman1));
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-+ qm_out(DCP_CFG(portal), (ed ? 0x1000 : 0) | (sernd & 0x3ff));
-+ else
-+ qm_out(DCP_CFG(portal), (ed ? 0x100 : 0) | (sernd & 0x1f));
-+}
-+
-+static void qm_set_wq_scheduling(struct qman *qm, enum qm_wq_class wq_class,
-+ u8 cs_elev, u8 csw2, u8 csw3, u8 csw4, u8 csw5,
-+ u8 csw6, u8 csw7)
-+{
-+ qm_out(WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) |
-+ ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) |
-+ ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) |
-+ ((csw6 & 0x7) << 4) | (csw7 & 0x7));
-+}
-+
-+static void qm_set_hid(struct qman *qm)
-+{
-+ qm_out(HID_CFG, 0);
-+}
-+
-+static void qm_set_corenet_initiator(struct qman *qm)
-+{
-+ qm_out(CI_SCHED_CFG,
-+ 0x80000000 | /* write srcciv enable */
-+ (CONFIG_FSL_QMAN_CI_SCHED_CFG_SRCCIV << 24) |
-+ (CONFIG_FSL_QMAN_CI_SCHED_CFG_SRQ_W << 8) |
-+ (CONFIG_FSL_QMAN_CI_SCHED_CFG_RW_W << 4) |
-+ CONFIG_FSL_QMAN_CI_SCHED_CFG_BMAN_W);
-+}
-+
-+static void qm_get_version(struct qman *qm, u16 *id, u8 *major, u8 *minor,
-+ u8 *cfg)
-+{
-+ u32 v = qm_in(IP_REV_1);
-+ u32 v2 = qm_in(IP_REV_2);
-+ *id = (v >> 16);
-+ *major = (v >> 8) & 0xff;
-+ *minor = v & 0xff;
-+ *cfg = v2 & 0xff;
-+}
-+
-+static void qm_set_memory(struct qman *qm, enum qm_memory memory, u64 ba,
-+ int enable, int prio, int stash, u32 size)
-+{
-+ u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE;
-+ u32 exp = ilog2(size);
-+ /* choke if size isn't within range */
-+ DPA_ASSERT((size >= 4096) && (size <= 1073741824) &&
-+ is_power_of_2(size));
-+ /* choke if 'ba' has lower-alignment than 'size' */
-+ DPA_ASSERT(!(ba & (size - 1)));
-+ __qm_out(qm, offset, upper_32_bits(ba));
-+ __qm_out(qm, offset + REG_offset_BAR, lower_32_bits(ba));
-+ __qm_out(qm, offset + REG_offset_AR,
-+ (enable ? 0x80000000 : 0) |
-+ (prio ? 0x40000000 : 0) |
-+ (stash ? 0x20000000 : 0) |
-+ (exp - 1));
-+}
-+
-+static void qm_set_pfdr_threshold(struct qman *qm, u32 th, u8 k)
-+{
-+ qm_out(PFDR_FP_LWIT, th & 0xffffff);
-+ qm_out(PFDR_CFG, k);
-+}
-+
-+static void qm_set_sfdr_threshold(struct qman *qm, u16 th)
-+{
-+ qm_out(SFDR_CFG, th & 0x3ff);
-+}
-+
-+static int qm_init_pfdr(struct qman *qm, u32 pfdr_start, u32 num)
-+{
-+ u8 rslt = MCR_get_rslt(qm_in(MCR));
-+
-+ DPA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num);
-+ /* Make sure the command interface is 'idle' */
-+ if (!MCR_rslt_idle(rslt))
-+ panic("QMAN_MCR isn't idle");
-+
-+ /* Write the MCR command params then the verb */
-+ qm_out(MCP(0), pfdr_start);
-+ /* TODO: remove this - it's a workaround for a model bug that is
-+ * corrected in more recent versions. We use the workaround until
-+ * everyone has upgraded. */
-+ qm_out(MCP(1), (pfdr_start + num - 16));
-+ lwsync();
-+ qm_out(MCR, MCR_INIT_PFDR);
-+ /* Poll for the result */
-+ do {
-+ rslt = MCR_get_rslt(qm_in(MCR));
-+ } while (!MCR_rslt_idle(rslt));
-+ if (MCR_rslt_ok(rslt))
-+ return 0;
-+ if (MCR_rslt_eaccess(rslt))
-+ return -EACCES;
-+ if (MCR_rslt_inval(rslt))
-+ return -EINVAL;
-+ pr_crit("Unexpected result from MCR_INIT_PFDR: %02x\n", rslt);
-+ return -ENOSYS;
-+}
-+
-+/*****************/
-+/* Config driver */
-+/*****************/
-+
-+#define DEFAULT_FQD_SZ (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ)
-+#define DEFAULT_PFDR_SZ (PAGE_SIZE << CONFIG_FSL_QMAN_PFDR_SZ)
-+
-+/* We support only one of these */
-+static struct qman *qm;
-+static struct device_node *qm_node;
-+
-+/* And this state belongs to 'qm'. It is set during fsl_qman_init(), but used
-+ * during qman_init_ccsr(). */
-+static dma_addr_t fqd_a, pfdr_a;
-+static size_t fqd_sz = DEFAULT_FQD_SZ, pfdr_sz = DEFAULT_PFDR_SZ;
-+
-+static int qman_fqd(struct reserved_mem *rmem)
-+{
-+ fqd_a = rmem->base;
-+ fqd_sz = rmem->size;
-+
-+ WARN_ON(!(fqd_a && fqd_sz));
-+
-+ return 0;
-+}
-+RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd);
-+
-+static int qman_pfdr(struct reserved_mem *rmem)
-+{
-+ pfdr_a = rmem->base;
-+ pfdr_sz = rmem->size;
-+
-+ WARN_ON(!(pfdr_a && pfdr_sz));
-+
-+ return 0;
-+}
-+RESERVEDMEM_OF_DECLARE(qman_fbpr, "fsl,qman-pfdr", qman_pfdr);
-+
-+size_t get_qman_fqd_size()
-+{
-+ return fqd_sz;
-+}
-+
-+/* Parse the <name> property to extract the memory location and size and
-+ * memblock_reserve() it. If it isn't supplied, memblock_alloc() the default
-+ * size. Also flush this memory range from data cache so that QMAN originated
-+ * transactions for this memory region could be marked non-coherent.
-+ */
-+static __init int parse_mem_property(struct device_node *node, const char *name,
-+ dma_addr_t *addr, size_t *sz, int zero)
-+{
-+ int ret;
-+
-+ /* If using a "zero-pma", don't try to zero it, even if you asked */
-+ if (zero && of_find_property(node, "zero-pma", &ret)) {
-+ pr_info(" it's a 'zero-pma', not zeroing from s/w\n");
-+ zero = 0;
-+ }
-+
-+ if (zero) {
-+ /* map as cacheable, non-guarded */
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ void __iomem *tmpp = ioremap_cache(*addr, *sz);
-+#else
-+ void __iomem *tmpp = ioremap(*addr, *sz);
-+#endif
-+
-+ if (!tmpp)
-+ return -ENOMEM;
-+ memset_io(tmpp, 0, *sz);
-+ flush_dcache_range((unsigned long)tmpp,
-+ (unsigned long)tmpp + *sz);
-+ iounmap(tmpp);
-+ }
-+
-+ return 0;
-+}
-+
-+/* TODO:
-+ * - there is obviously no handling of errors,
-+ * - the calls to qm_set_memory() hard-code the priority and CPC-stashing for
-+ * both memory resources to zero.
-+ */
-+static int __init fsl_qman_init(struct device_node *node)
-+{
-+ struct resource res;
-+ resource_size_t len;
-+ u32 __iomem *regs;
-+ const char *s;
-+ int ret, standby = 0;
-+ u16 id;
-+ u8 major, minor, cfg;
-+ ret = of_address_to_resource(node, 0, &res);
-+ if (ret) {
-+ pr_err("Can't get %s property '%s'\n", node->full_name, "reg");
-+ return ret;
-+ }
-+ s = of_get_property(node, "fsl,hv-claimable", &ret);
-+ if (s && !strcmp(s, "standby"))
-+ standby = 1;
-+ if (!standby) {
-+ ret = parse_mem_property(node, "fsl,qman-fqd",
-+ &fqd_a, &fqd_sz, 1);
-+ pr_info("qman-fqd addr %pad size 0x%zx\n", &fqd_a, fqd_sz);
-+ BUG_ON(ret);
-+ ret = parse_mem_property(node, "fsl,qman-pfdr",
-+ &pfdr_a, &pfdr_sz, 0);
-+ pr_info("qman-pfdr addr %pad size 0x%zx\n", &pfdr_a, pfdr_sz);
-+ BUG_ON(ret);
-+ }
-+ /* Global configuration */
-+ len = resource_size(&res);
-+ if (len != (unsigned long)len)
-+ return -EINVAL;
-+ regs = ioremap(res.start, (unsigned long)len);
-+ qm = qm_create(regs);
-+ qm_node = node;
-+ qm_get_version(qm, &id, &major, &minor, &cfg);
-+ pr_info("Qman ver:%04x,%02x,%02x,%02x\n", id, major, minor, cfg);
-+ if (!qman_ip_rev) {
-+ if ((major == 1) && (minor == 0)) {
-+ pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n");
-+ iounmap(regs);
-+ return -ENODEV;
-+ } else if ((major == 1) && (minor == 1))
-+ qman_ip_rev = QMAN_REV11;
-+ else if ((major == 1) && (minor == 2))
-+ qman_ip_rev = QMAN_REV12;
-+ else if ((major == 2) && (minor == 0))
-+ qman_ip_rev = QMAN_REV20;
-+ else if ((major == 3) && (minor == 0))
-+ qman_ip_rev = QMAN_REV30;
-+ else if ((major == 3) && (minor == 1))
-+ qman_ip_rev = QMAN_REV31;
-+ else if ((major == 3) && (minor == 2))
-+ qman_ip_rev = QMAN_REV32;
-+ else {
-+ pr_warn("unknown Qman version, default to rev1.1\n");
-+ qman_ip_rev = QMAN_REV11;
-+ }
-+ qman_ip_cfg = cfg;
-+ }
-+
-+ if (standby) {
-+ pr_info(" -> in standby mode\n");
-+ return 0;
-+ }
-+ return 0;
-+}
-+
-+int qman_have_ccsr(void)
-+{
-+ return qm ? 1 : 0;
-+}
-+
-+__init int qman_init_early(void)
-+{
-+ struct device_node *dn;
-+ int ret;
-+
-+ for_each_compatible_node(dn, NULL, "fsl,qman") {
-+ if (qm)
-+ pr_err("%s: only one 'fsl,qman' allowed\n",
-+ dn->full_name);
-+ else {
-+ if (!of_device_is_available(dn))
-+ continue;
-+
-+ ret = fsl_qman_init(dn);
-+ BUG_ON(ret);
-+ }
-+ }
-+ return 0;
-+}
-+postcore_initcall_sync(qman_init_early);
-+
-+static void log_edata_bits(u32 bit_count)
-+{
-+ u32 i, j, mask = 0xffffffff;
-+
-+ pr_warn("Qman ErrInt, EDATA:\n");
-+ i = bit_count/32;
-+ if (bit_count%32) {
-+ i++;
-+ mask = ~(mask << bit_count%32);
-+ }
-+ j = 16-i;
-+ pr_warn(" 0x%08x\n", qm_in(EDATA(j)) & mask);
-+ j++;
-+ for (; j < 16; j++)
-+ pr_warn(" 0x%08x\n", qm_in(EDATA(j)));
-+}
-+
-+static void log_additional_error_info(u32 isr_val, u32 ecsr_val)
-+{
-+ union qman_ecir ecir_val;
-+ union qman_eadr eadr_val;
-+
-+ ecir_val.ecir_raw = qm_in(ECIR);
-+ /* Is portal info valid */
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
-+ union qman_ecir2 ecir2_val;
-+ ecir2_val.ecir2_raw = qm_in(ECIR2);
-+ if (ecsr_val & PORTAL_ECSR_ERR) {
-+ pr_warn("Qman ErrInt: %s id %d\n",
-+ (ecir2_val.info.portal_type) ?
-+ "DCP" : "SWP", ecir2_val.info.portal_num);
-+ }
-+ if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE)) {
-+ pr_warn("Qman ErrInt: ecir.fqid 0x%x\n",
-+ ecir_val.info.fqid);
-+ }
-+ if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
-+ eadr_val.eadr_raw = qm_in(EADR);
-+ pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n",
-+ error_mdata[eadr_val.info_rev3.memid].txt,
-+ error_mdata[eadr_val.info_rev3.memid].addr_mask
-+ & eadr_val.info_rev3.eadr);
-+ log_edata_bits(
-+ error_mdata[eadr_val.info_rev3.memid].bits);
-+ }
-+ } else {
-+ if (ecsr_val & PORTAL_ECSR_ERR) {
-+ pr_warn("Qman ErrInt: %s id %d\n",
-+ (ecir_val.info.portal_type) ?
-+ "DCP" : "SWP", ecir_val.info.portal_num);
-+ }
-+ if (ecsr_val & FQID_ECSR_ERR) {
-+ pr_warn("Qman ErrInt: ecir.fqid 0x%x\n",
-+ ecir_val.info.fqid);
-+ }
-+ if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
-+ eadr_val.eadr_raw = qm_in(EADR);
-+ pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n",
-+ error_mdata[eadr_val.info.memid].txt,
-+ error_mdata[eadr_val.info.memid].addr_mask
-+ & eadr_val.info.eadr);
-+ log_edata_bits(error_mdata[eadr_val.info.memid].bits);
-+ }
-+ }
-+}
-+
-+/* Qman interrupt handler */
-+static irqreturn_t qman_isr(int irq, void *ptr)
-+{
-+ u32 isr_val, ier_val, ecsr_val, isr_mask, i;
-+
-+ ier_val = qm_err_isr_enable_read(qm);
-+ isr_val = qm_err_isr_status_read(qm);
-+ ecsr_val = qm_in(ECSR);
-+ isr_mask = isr_val & ier_val;
-+
-+ if (!isr_mask)
-+ return IRQ_NONE;
-+ for (i = 0; i < QMAN_HWE_COUNT; i++) {
-+ if (qman_hwerr_txts[i].mask & isr_mask) {
-+ pr_warn("Qman ErrInt: %s\n", qman_hwerr_txts[i].txt);
-+ if (qman_hwerr_txts[i].mask & ecsr_val) {
-+ log_additional_error_info(isr_mask, ecsr_val);
-+ /* Re-arm error capture registers */
-+ qm_out(ECSR, ecsr_val);
-+ }
-+ if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_UNENABLE) {
-+ pr_devel("Qman un-enabling error 0x%x\n",
-+ qman_hwerr_txts[i].mask);
-+ ier_val &= ~qman_hwerr_txts[i].mask;
-+ qm_err_isr_enable_write(qm, ier_val);
-+ }
-+ }
-+ }
-+ qm_err_isr_status_clear(qm, isr_val);
-+ return IRQ_HANDLED;
-+}
-+
-+static int __bind_irq(void)
-+{
-+ int ret, err_irq;
-+
-+ err_irq = of_irq_to_resource(qm_node, 0, NULL);
-+ if (err_irq == 0) {
-+ pr_info("Can't get %s property '%s'\n", qm_node->full_name,
-+ "interrupts");
-+ return -ENODEV;
-+ }
-+ ret = request_irq(err_irq, qman_isr, IRQF_SHARED, "qman-err", qm_node);
-+ if (ret) {
-+ pr_err("request_irq() failed %d for '%s'\n", ret,
-+ qm_node->full_name);
-+ return -ENODEV;
-+ }
-+ /* Write-to-clear any stale bits, (eg. starvation being asserted prior
-+ * to resource allocation during driver init). */
-+ qm_err_isr_status_clear(qm, 0xffffffff);
-+ /* Enable Error Interrupts */
-+ qm_err_isr_enable_write(qm, 0xffffffff);
-+ return 0;
-+}
-+
-+int qman_init_ccsr(struct device_node *node)
-+{
-+ int ret;
-+ if (!qman_have_ccsr())
-+ return 0;
-+ if (node != qm_node)
-+ return -EINVAL;
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ /* TEMP for LS1043 : should be done in uboot */
-+ qm_out(QCSP_BARE, 0x5);
-+ qm_out(QCSP_BAR, 0x0);
-+#endif
-+ /* FQD memory */
-+ qm_set_memory(qm, qm_memory_fqd, fqd_a, 1, 0, 0, fqd_sz);
-+ /* PFDR memory */
-+ qm_set_memory(qm, qm_memory_pfdr, pfdr_a, 1, 0, 0, pfdr_sz);
-+ qm_init_pfdr(qm, 8, pfdr_sz / 64 - 8);
-+ /* thresholds */
-+ qm_set_pfdr_threshold(qm, 512, 64);
-+ qm_set_sfdr_threshold(qm, 128);
-+ /* clear stale PEBI bit from interrupt status register */
-+ qm_err_isr_status_clear(qm, QM_EIRQ_PEBI);
-+ /* corenet initiator settings */
-+ qm_set_corenet_initiator(qm);
-+ /* HID settings */
-+ qm_set_hid(qm);
-+ /* Set scheduling weights to defaults */
-+ for (ret = qm_wq_first; ret <= qm_wq_last; ret++)
-+ qm_set_wq_scheduling(qm, ret, 0, 0, 0, 0, 0, 0, 0);
-+ /* We are not prepared to accept ERNs for hardware enqueues */
-+ qm_set_dc(qm, qm_dc_portal_fman0, 1, 0);
-+ qm_set_dc(qm, qm_dc_portal_fman1, 1, 0);
-+ /* Initialise Error Interrupt Handler */
-+ ret = __bind_irq();
-+ if (ret)
-+ return ret;
-+ return 0;
-+}
-+
-+#define LIO_CFG_LIODN_MASK 0x0fff0000
-+void qman_liodn_fixup(u16 channel)
-+{
-+ static int done;
-+ static u32 liodn_offset;
-+ u32 before, after;
-+ int idx = channel - QM_CHANNEL_SWPORTAL0;
-+
-+ if (!qman_have_ccsr())
-+ return;
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-+ before = qm_in(REV3_QCSP_LIO_CFG(idx));
-+ else
-+ before = qm_in(QCSP_LIO_CFG(idx));
-+ if (!done) {
-+ liodn_offset = before & LIO_CFG_LIODN_MASK;
-+ done = 1;
-+ return;
-+ }
-+ after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset;
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-+ qm_out(REV3_QCSP_LIO_CFG(idx), after);
-+ else
-+ qm_out(QCSP_LIO_CFG(idx), after);
-+}
-+
-+#define IO_CFG_SDEST_MASK 0x00ff0000
-+int qman_set_sdest(u16 channel, unsigned int cpu_idx)
-+{
-+ int idx = channel - QM_CHANNEL_SWPORTAL0;
-+ u32 before, after;
-+
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+ if ((qman_ip_rev & 0xFF00) == QMAN_REV31) {
-+ /* LS1043A - only one L2 cache */
-+ cpu_idx = 0;
-+ }
-+
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
-+ before = qm_in(REV3_QCSP_IO_CFG(idx));
-+ /* Each pair of vcpu share the same SRQ(SDEST) */
-+ cpu_idx /= 2;
-+ after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
-+ qm_out(REV3_QCSP_IO_CFG(idx), after);
-+ } else {
-+ before = qm_in(QCSP_IO_CFG(idx));
-+ after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
-+ qm_out(QCSP_IO_CFG(idx), after);
-+ }
-+ return 0;
-+}
-+
-+#define MISC_CFG_WPM_MASK 0x00000002
-+int qm_set_wpm(int wpm)
-+{
-+ u32 before;
-+ u32 after;
-+
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+
-+ before = qm_in(MISC_CFG);
-+ after = (before & (~MISC_CFG_WPM_MASK)) | (wpm << 1);
-+ qm_out(MISC_CFG, after);
-+ return 0;
-+}
-+
-+int qm_get_wpm(int *wpm)
-+{
-+ u32 before;
-+
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+
-+ before = qm_in(MISC_CFG);
-+ *wpm = (before & MISC_CFG_WPM_MASK) >> 1;
-+ return 0;
-+}
-+
-+/* CEETM_CFG_PRES register has PRES field which is calculated by:
-+ * PRES = (2^22 / credit update reference period) * QMan clock period
-+ * = (2^22 * 10^9)/ CONFIG_QMAN_CEETM_UPDATE_PERIOD) / qman_clk
-+ */
-+
-+int qman_ceetm_set_prescaler(enum qm_dc_portal portal)
-+{
-+ u64 temp;
-+ u16 pres;
-+
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+
-+ temp = 0x400000 * 100;
-+ do_div(temp, CONFIG_QMAN_CEETM_UPDATE_PERIOD);
-+ temp *= 10000000;
-+ do_div(temp, qman_clk);
-+ pres = (u16) temp;
-+ qm_out(CEETM_CFG_IDX, portal);
-+ qm_out(CEETM_CFG_PRES, pres);
-+ return 0;
-+}
-+
-+int qman_ceetm_get_prescaler(u16 *pres)
-+{
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+ *pres = (u16)qm_in(CEETM_CFG_PRES);
-+ return 0;
-+}
-+
-+#define DCP_CFG_CEETME_MASK 0xFFFF0000
-+#define QM_SP_ENABLE_CEETM(n) (0x80000000 >> (n))
-+int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal)
-+{
-+ u32 dcp_cfg;
-+
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+
-+ dcp_cfg = qm_in(DCP_CFG(portal));
-+ dcp_cfg |= QM_SP_ENABLE_CEETM(sub_portal);
-+ qm_out(DCP_CFG(portal), dcp_cfg);
-+ return 0;
-+}
-+
-+int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal)
-+{
-+ u32 dcp_cfg;
-+
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+ dcp_cfg = qm_in(DCP_CFG(portal));
-+ dcp_cfg &= ~(QM_SP_ENABLE_CEETM(sub_portal));
-+ qm_out(DCP_CFG(portal), dcp_cfg);
-+ return 0;
-+}
-+
-+int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num)
-+{
-+ if (!qman_have_ccsr())
-+ return -ENODEV;
-+ *num = qm_in(CEETM_XSFDR_IN_USE);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_get_xsfdr);
-+
-+#ifdef CONFIG_SYSFS
-+
-+#define DRV_NAME "fsl-qman"
-+#define DCP_MAX_ID 3
-+#define DCP_MIN_ID 0
-+
-+static ssize_t show_pfdr_fpc(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_FPC));
-+};
-+
-+static ssize_t show_dlm_avg(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ u32 data;
-+ int i;
-+
-+ if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i))
-+ return -EINVAL;
-+ if (i < DCP_MIN_ID || i > DCP_MAX_ID)
-+ return -EINVAL;
-+ data = qm_in(DCP_DLM_AVG(i));
-+ return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8,
-+ (data & 0x000000ff)*390625);
-+};
-+
-+static ssize_t set_dlm_avg(struct device *dev,
-+ struct device_attribute *dev_attr, const char *buf, size_t count)
-+{
-+ unsigned long val;
-+ int i;
-+
-+ if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i))
-+ return -EINVAL;
-+ if (i < DCP_MIN_ID || i > DCP_MAX_ID)
-+ return -EINVAL;
-+ if (kstrtoul(buf, 0, &val)) {
-+ dev_dbg(dev, "invalid input %s\n", buf);
-+ return -EINVAL;
-+ }
-+ qm_out(DCP_DLM_AVG(i), val);
-+ return count;
-+};
-+
-+static ssize_t show_pfdr_cfg(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_CFG));
-+};
-+
-+static ssize_t set_pfdr_cfg(struct device *dev,
-+ struct device_attribute *dev_attr, const char *buf, size_t count)
-+{
-+ unsigned long val;
-+
-+ if (kstrtoul(buf, 0, &val)) {
-+ dev_dbg(dev, "invalid input %s\n", buf);
-+ return -EINVAL;
-+ }
-+ qm_out(PFDR_CFG, val);
-+ return count;
-+};
-+
-+static ssize_t show_sfdr_in_use(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SFDR_IN_USE));
-+};
-+
-+static ssize_t show_idle_stat(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(IDLE_STAT));
-+};
-+
-+static ssize_t show_ci_rlm_avg(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ u32 data = qm_in(CI_RLM_AVG);
-+ return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8,
-+ (data & 0x000000ff)*390625);
-+};
-+
-+static ssize_t set_ci_rlm_avg(struct device *dev,
-+ struct device_attribute *dev_attr, const char *buf, size_t count)
-+{
-+ unsigned long val;
-+
-+ if (kstrtoul(buf, 0, &val)) {
-+ dev_dbg(dev, "invalid input %s\n", buf);
-+ return -EINVAL;
-+ }
-+ qm_out(CI_RLM_AVG, val);
-+ return count;
-+};
-+
-+static ssize_t show_err_isr(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ return snprintf(buf, PAGE_SIZE, "0x%08x\n", qm_in(ERR_ISR));
-+};
-+
-+#define SBEC_MAX_ID 14
-+#define SBEC_MIN_ID 0
-+
-+static ssize_t show_sbec(struct device *dev,
-+ struct device_attribute *dev_attr, char *buf)
-+{
-+ int i;
-+
-+ if (!sscanf(dev_attr->attr.name, "sbec_%d", &i))
-+ return -EINVAL;
-+ if (i < SBEC_MIN_ID || i > SBEC_MAX_ID)
-+ return -EINVAL;
-+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SBEC(i)));
-+};
-+
-+static DEVICE_ATTR(pfdr_fpc, S_IRUSR, show_pfdr_fpc, NULL);
-+static DEVICE_ATTR(pfdr_cfg, S_IRUSR, show_pfdr_cfg, set_pfdr_cfg);
-+static DEVICE_ATTR(idle_stat, S_IRUSR, show_idle_stat, NULL);
-+static DEVICE_ATTR(ci_rlm_avg, (S_IRUSR|S_IWUSR),
-+ show_ci_rlm_avg, set_ci_rlm_avg);
-+static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL);
-+static DEVICE_ATTR(sfdr_in_use, S_IRUSR, show_sfdr_in_use, NULL);
-+
-+static DEVICE_ATTR(dcp0_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-+static DEVICE_ATTR(dcp1_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-+static DEVICE_ATTR(dcp2_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-+static DEVICE_ATTR(dcp3_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-+
-+static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_2, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_3, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_4, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_5, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_6, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_7, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_8, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_9, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_10, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_11, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_12, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_13, S_IRUSR, show_sbec, NULL);
-+static DEVICE_ATTR(sbec_14, S_IRUSR, show_sbec, NULL);
-+
-+static struct attribute *qman_dev_attributes[] = {
-+ &dev_attr_pfdr_fpc.attr,
-+ &dev_attr_pfdr_cfg.attr,
-+ &dev_attr_idle_stat.attr,
-+ &dev_attr_ci_rlm_avg.attr,
-+ &dev_attr_err_isr.attr,
-+ &dev_attr_dcp0_dlm_avg.attr,
-+ &dev_attr_dcp1_dlm_avg.attr,
-+ &dev_attr_dcp2_dlm_avg.attr,
-+ &dev_attr_dcp3_dlm_avg.attr,
-+ /* sfdr_in_use will be added if necessary */
-+ NULL
-+};
-+
-+static struct attribute *qman_dev_ecr_attributes[] = {
-+ &dev_attr_sbec_0.attr,
-+ &dev_attr_sbec_1.attr,
-+ &dev_attr_sbec_2.attr,
-+ &dev_attr_sbec_3.attr,
-+ &dev_attr_sbec_4.attr,
-+ &dev_attr_sbec_5.attr,
-+ &dev_attr_sbec_6.attr,
-+ &dev_attr_sbec_7.attr,
-+ &dev_attr_sbec_8.attr,
-+ &dev_attr_sbec_9.attr,
-+ &dev_attr_sbec_10.attr,
-+ &dev_attr_sbec_11.attr,
-+ &dev_attr_sbec_12.attr,
-+ &dev_attr_sbec_13.attr,
-+ &dev_attr_sbec_14.attr,
-+ NULL
-+};
-+
-+/* root level */
-+static const struct attribute_group qman_dev_attr_grp = {
-+ .name = NULL,
-+ .attrs = qman_dev_attributes
-+};
-+static const struct attribute_group qman_dev_ecr_grp = {
-+ .name = "error_capture",
-+ .attrs = qman_dev_ecr_attributes
-+};
-+
-+static int of_fsl_qman_remove(struct platform_device *ofdev)
-+{
-+ sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
-+ return 0;
-+};
-+
-+static int of_fsl_qman_probe(struct platform_device *ofdev)
-+{
-+ int ret;
-+
-+ ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
-+ if (ret)
-+ goto done;
-+ ret = sysfs_add_file_to_group(&ofdev->dev.kobj,
-+ &dev_attr_sfdr_in_use.attr, qman_dev_attr_grp.name);
-+ if (ret)
-+ goto del_group_0;
-+ ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_ecr_grp);
-+ if (ret)
-+ goto del_group_0;
-+
-+ goto done;
-+
-+del_group_0:
-+ sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
-+done:
-+ if (ret)
-+ dev_err(&ofdev->dev,
-+ "Cannot create dev attributes ret=%d\n", ret);
-+ return ret;
-+};
-+
-+static struct of_device_id of_fsl_qman_ids[] = {
-+ {
-+ .compatible = "fsl,qman",
-+ },
-+ {}
-+};
-+MODULE_DEVICE_TABLE(of, of_fsl_qman_ids);
-+
-+#ifdef CONFIG_SUSPEND
-+
-+static u32 saved_isdr;
-+static int qman_pm_suspend_noirq(struct device *dev)
-+{
-+ uint32_t idle_state;
-+
-+ suspend_unused_qportal();
-+ /* save isdr, disable all, clear isr */
-+ saved_isdr = qm_err_isr_disable_read(qm);
-+ qm_err_isr_disable_write(qm, 0xffffffff);
-+ qm_err_isr_status_clear(qm, 0xffffffff);
-+ idle_state = qm_in(IDLE_STAT);
-+ if (!(idle_state & 0x1)) {
-+ pr_err("Qman not idle 0x%x aborting\n", idle_state);
-+ qm_err_isr_disable_write(qm, saved_isdr);
-+ resume_unused_qportal();
-+ return -EBUSY;
-+ }
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Qman suspend code, IDLE_STAT = 0x%x\n", idle_state);
-+#endif
-+ return 0;
-+}
-+
-+static int qman_pm_resume_noirq(struct device *dev)
-+{
-+ /* restore isdr */
-+ qm_err_isr_disable_write(qm, saved_isdr);
-+ resume_unused_qportal();
-+ return 0;
-+}
-+#else
-+#define qman_pm_suspend_noirq NULL
-+#define qman_pm_resume_noirq NULL
-+#endif
-+
-+static const struct dev_pm_ops qman_pm_ops = {
-+ .suspend_noirq = qman_pm_suspend_noirq,
-+ .resume_noirq = qman_pm_resume_noirq,
-+};
-+
-+static struct platform_driver of_fsl_qman_driver = {
-+ .driver = {
-+ .owner = THIS_MODULE,
-+ .name = DRV_NAME,
-+ .of_match_table = of_fsl_qman_ids,
-+ .pm = &qman_pm_ops,
-+ },
-+ .probe = of_fsl_qman_probe,
-+ .remove = of_fsl_qman_remove,
-+};
-+
-+static int qman_ctrl_init(void)
-+{
-+ return platform_driver_register(&of_fsl_qman_driver);
-+}
-+
-+static void qman_ctrl_exit(void)
-+{
-+ platform_driver_unregister(&of_fsl_qman_driver);
-+}
-+
-+module_init(qman_ctrl_init);
-+module_exit(qman_ctrl_exit);
-+
-+#endif /* CONFIG_SYSFS */
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_debugfs.c
-@@ -0,0 +1,1594 @@
-+/* Copyright 2010-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+#include "qman_private.h"
-+
-+#define MAX_FQID (0x00ffffff)
-+#define QM_FQD_BLOCK_SIZE 64
-+#define QM_FQD_AR (0xC10)
-+
-+static u32 fqid_max;
-+static u64 qman_ccsr_start;
-+static u64 qman_ccsr_size;
-+
-+static const char * const state_txt[] = {
-+ "Out of Service",
-+ "Retired",
-+ "Tentatively Scheduled",
-+ "Truly Scheduled",
-+ "Parked",
-+ "Active, Active Held or Held Suspended",
-+ "Unknown State 6",
-+ "Unknown State 7",
-+ NULL,
-+};
-+
-+static const u8 fqd_states[] = {
-+ QM_MCR_NP_STATE_OOS, QM_MCR_NP_STATE_RETIRED, QM_MCR_NP_STATE_TEN_SCHED,
-+ QM_MCR_NP_STATE_TRU_SCHED, QM_MCR_NP_STATE_PARKED,
-+ QM_MCR_NP_STATE_ACTIVE};
-+
-+struct mask_to_text {
-+ u16 mask;
-+ const char *txt;
-+};
-+
-+struct mask_filter_s {
-+ u16 mask;
-+ u8 filter;
-+};
-+
-+static const struct mask_filter_s mask_filter[] = {
-+ {QM_FQCTRL_PREFERINCACHE, 0},
-+ {QM_FQCTRL_PREFERINCACHE, 1},
-+ {QM_FQCTRL_HOLDACTIVE, 0},
-+ {QM_FQCTRL_HOLDACTIVE, 1},
-+ {QM_FQCTRL_AVOIDBLOCK, 0},
-+ {QM_FQCTRL_AVOIDBLOCK, 1},
-+ {QM_FQCTRL_FORCESFDR, 0},
-+ {QM_FQCTRL_FORCESFDR, 1},
-+ {QM_FQCTRL_CPCSTASH, 0},
-+ {QM_FQCTRL_CPCSTASH, 1},
-+ {QM_FQCTRL_CTXASTASHING, 0},
-+ {QM_FQCTRL_CTXASTASHING, 1},
-+ {QM_FQCTRL_ORP, 0},
-+ {QM_FQCTRL_ORP, 1},
-+ {QM_FQCTRL_TDE, 0},
-+ {QM_FQCTRL_TDE, 1},
-+ {QM_FQCTRL_CGE, 0},
-+ {QM_FQCTRL_CGE, 1}
-+};
-+
-+static const struct mask_to_text fq_ctrl_text_list[] = {
-+ {
-+ .mask = QM_FQCTRL_PREFERINCACHE,
-+ .txt = "Prefer in cache",
-+ },
-+ {
-+ .mask = QM_FQCTRL_HOLDACTIVE,
-+ .txt = "Hold active in portal",
-+ },
-+ {
-+ .mask = QM_FQCTRL_AVOIDBLOCK,
-+ .txt = "Avoid Blocking",
-+ },
-+ {
-+ .mask = QM_FQCTRL_FORCESFDR,
-+ .txt = "High-priority SFDRs",
-+ },
-+ {
-+ .mask = QM_FQCTRL_CPCSTASH,
-+ .txt = "CPC Stash Enable",
-+ },
-+ {
-+ .mask = QM_FQCTRL_CTXASTASHING,
-+ .txt = "Context-A stashing",
-+ },
-+ {
-+ .mask = QM_FQCTRL_ORP,
-+ .txt = "ORP Enable",
-+ },
-+ {
-+ .mask = QM_FQCTRL_TDE,
-+ .txt = "Tail-Drop Enable",
-+ },
-+ {
-+ .mask = QM_FQCTRL_CGE,
-+ .txt = "Congestion Group Enable",
-+ },
-+ {
-+ .mask = 0,
-+ .txt = NULL,
-+ }
-+};
-+
-+static const char *get_fqd_ctrl_text(u16 mask)
-+{
-+ int i = 0;
-+
-+ while (fq_ctrl_text_list[i].txt != NULL) {
-+ if (fq_ctrl_text_list[i].mask == mask)
-+ return fq_ctrl_text_list[i].txt;
-+ i++;
-+ }
-+ return NULL;
-+}
-+
-+static const struct mask_to_text stashing_text_list[] = {
-+ {
-+ .mask = QM_STASHING_EXCL_CTX,
-+ .txt = "FQ Ctx Stash"
-+ },
-+ {
-+ .mask = QM_STASHING_EXCL_DATA,
-+ .txt = "Frame Data Stash",
-+ },
-+ {
-+ .mask = QM_STASHING_EXCL_ANNOTATION,
-+ .txt = "Frame Annotation Stash",
-+ },
-+ {
-+ .mask = 0,
-+ .txt = NULL,
-+ },
-+};
-+
-+static int user_input_convert(const char __user *user_buf, size_t count,
-+ unsigned long *val)
-+{
-+ char buf[12];
-+
-+ if (count > sizeof(buf) - 1)
-+ return -EINVAL;
-+ if (copy_from_user(buf, user_buf, count))
-+ return -EFAULT;
-+ buf[count] = '\0';
-+ if (kstrtoul(buf, 0, val))
-+ return -EINVAL;
-+ return 0;
-+}
-+
-+struct line_buffer_fq {
-+ u32 buf[8];
-+ u32 buf_cnt;
-+ int line_cnt;
-+};
-+
-+static void add_to_line_buffer(struct line_buffer_fq *line_buf, u32 fqid,
-+ struct seq_file *file)
-+{
-+ line_buf->buf[line_buf->buf_cnt] = fqid;
-+ line_buf->buf_cnt++;
-+ if (line_buf->buf_cnt == 8) {
-+ /* Buffer is full, flush it */
-+ if (line_buf->line_cnt != 0)
-+ seq_puts(file, ",\n");
-+ seq_printf(file, "0x%06x,0x%06x,0x%06x,0x%06x,0x%06x,"
-+ "0x%06x,0x%06x,0x%06x",
-+ line_buf->buf[0], line_buf->buf[1], line_buf->buf[2],
-+ line_buf->buf[3], line_buf->buf[4], line_buf->buf[5],
-+ line_buf->buf[6], line_buf->buf[7]);
-+ line_buf->buf_cnt = 0;
-+ line_buf->line_cnt++;
-+ }
-+}
-+
-+static void flush_line_buffer(struct line_buffer_fq *line_buf,
-+ struct seq_file *file)
-+{
-+ if (line_buf->buf_cnt) {
-+ int y = 0;
-+ if (line_buf->line_cnt != 0)
-+ seq_puts(file, ",\n");
-+ while (y != line_buf->buf_cnt) {
-+ if (y+1 == line_buf->buf_cnt)
-+ seq_printf(file, "0x%06x", line_buf->buf[y]);
-+ else
-+ seq_printf(file, "0x%06x,", line_buf->buf[y]);
-+ y++;
-+ }
-+ line_buf->line_cnt++;
-+ }
-+ if (line_buf->line_cnt)
-+ seq_putc(file, '\n');
-+}
-+
-+static struct dentry *dfs_root; /* debugfs root directory */
-+
-+/*******************************************************************************
-+ * Query Frame Queue Non Programmable Fields
-+ ******************************************************************************/
-+struct query_fq_np_fields_data_s {
-+ u32 fqid;
-+};
-+static struct query_fq_np_fields_data_s query_fq_np_fields_data = {
-+ .fqid = 1,
-+};
-+
-+static int query_fq_np_fields_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_queryfq_np np;
-+ struct qman_fq fq;
-+
-+ fq.fqid = query_fq_np_fields_data.fqid;
-+ ret = qman_query_fq_np(&fq, &np);
-+ if (ret)
-+ return ret;
-+ /* Print state */
-+ seq_printf(file, "Query FQ Non Programmable Fields Result fqid 0x%x\n",
-+ fq.fqid);
-+ seq_printf(file, " force eligible pending: %s\n",
-+ (np.state & QM_MCR_NP_STATE_FE) ? "yes" : "no");
-+ seq_printf(file, " retirement pending: %s\n",
-+ (np.state & QM_MCR_NP_STATE_R) ? "yes" : "no");
-+ seq_printf(file, " state: %s\n",
-+ state_txt[np.state & QM_MCR_NP_STATE_MASK]);
-+ seq_printf(file, " fq_link: 0x%x\n", np.fqd_link);
-+ seq_printf(file, " odp_seq: %u\n", np.odp_seq);
-+ seq_printf(file, " orp_nesn: %u\n", np.orp_nesn);
-+ seq_printf(file, " orp_ea_hseq: %u\n", np.orp_ea_hseq);
-+ seq_printf(file, " orp_ea_tseq: %u\n", np.orp_ea_tseq);
-+ seq_printf(file, " orp_ea_hptr: 0x%x\n", np.orp_ea_hptr);
-+ seq_printf(file, " orp_ea_tptr: 0x%x\n", np.orp_ea_tptr);
-+ seq_printf(file, " pfdr_hptr: 0x%x\n", np.pfdr_hptr);
-+ seq_printf(file, " pfdr_tptr: 0x%x\n", np.pfdr_tptr);
-+ seq_printf(file, " is: ics_surp contains a %s\n",
-+ (np.is) ? "deficit" : "surplus");
-+ seq_printf(file, " ics_surp: %u\n", np.ics_surp);
-+ seq_printf(file, " byte_cnt: %u\n", np.byte_cnt);
-+ seq_printf(file, " frm_cnt: %u\n", np.frm_cnt);
-+ seq_printf(file, " ra1_sfdr: 0x%x\n", np.ra1_sfdr);
-+ seq_printf(file, " ra2_sfdr: 0x%x\n", np.ra2_sfdr);
-+ seq_printf(file, " od1_sfdr: 0x%x\n", np.od1_sfdr);
-+ seq_printf(file, " od2_sfdr: 0x%x\n", np.od2_sfdr);
-+ seq_printf(file, " od3_sfdr: 0x%x\n", np.od3_sfdr);
-+ return 0;
-+}
-+
-+static int query_fq_np_fields_open(struct inode *inode,
-+ struct file *file)
-+{
-+ return single_open(file, query_fq_np_fields_show, NULL);
-+}
-+
-+static ssize_t query_fq_np_fields_write(struct file *f,
-+ const char __user *buf, size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > MAX_FQID)
-+ return -EINVAL;
-+ query_fq_np_fields_data.fqid = (u32)val;
-+ return count;
-+}
-+
-+static const struct file_operations query_fq_np_fields_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_fq_np_fields_open,
-+ .read = seq_read,
-+ .write = query_fq_np_fields_write,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * Frame Queue Programmable Fields
-+ ******************************************************************************/
-+struct query_fq_fields_data_s {
-+ u32 fqid;
-+};
-+
-+static struct query_fq_fields_data_s query_fq_fields_data = {
-+ .fqid = 1,
-+};
-+
-+static int query_fq_fields_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_fqd fqd;
-+ struct qman_fq fq;
-+ int i = 0;
-+
-+ memset(&fqd, 0, sizeof(struct qm_fqd));
-+ fq.fqid = query_fq_fields_data.fqid;
-+ ret = qman_query_fq(&fq, &fqd);
-+ if (ret)
-+ return ret;
-+ seq_printf(file, "Query FQ Programmable Fields Result fqid 0x%x\n",
-+ fq.fqid);
-+ seq_printf(file, " orprws: %u\n", fqd.orprws);
-+ seq_printf(file, " oa: %u\n", fqd.oa);
-+ seq_printf(file, " olws: %u\n", fqd.olws);
-+
-+ seq_printf(file, " cgid: %u\n", fqd.cgid);
-+
-+ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) == 0)
-+ seq_puts(file, " fq_ctrl: None\n");
-+ else {
-+ i = 0;
-+ seq_puts(file, " fq_ctrl:\n");
-+ while (fq_ctrl_text_list[i].txt != NULL) {
-+ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) &
-+ fq_ctrl_text_list[i].mask)
-+ seq_printf(file, " %s\n",
-+ fq_ctrl_text_list[i].txt);
-+ i++;
-+ }
-+ }
-+ seq_printf(file, " dest_channel: %u\n", fqd.dest.channel);
-+ seq_printf(file, " dest_wq: %u\n", fqd.dest.wq);
-+ seq_printf(file, " ics_cred: %u\n", fqd.ics_cred);
-+ seq_printf(file, " td_mant: %u\n", fqd.td.mant);
-+ seq_printf(file, " td_exp: %u\n", fqd.td.exp);
-+
-+ seq_printf(file, " ctx_b: 0x%x\n", fqd.context_b);
-+
-+ seq_printf(file, " ctx_a: 0x%llx\n", qm_fqd_stashing_get64(&fqd));
-+ /* Any stashing configured */
-+ if ((fqd.context_a.stashing.exclusive & 0x7) == 0)
-+ seq_puts(file, " ctx_a_stash_exclusive: None\n");
-+ else {
-+ seq_puts(file, " ctx_a_stash_exclusive:\n");
-+ i = 0;
-+ while (stashing_text_list[i].txt != NULL) {
-+ if ((fqd.fq_ctrl & 0x7) & stashing_text_list[i].mask)
-+ seq_printf(file, " %s\n",
-+ stashing_text_list[i].txt);
-+ i++;
-+ }
-+ }
-+ seq_printf(file, " ctx_a_stash_annotation_cl: %u\n",
-+ fqd.context_a.stashing.annotation_cl);
-+ seq_printf(file, " ctx_a_stash_data_cl: %u\n",
-+ fqd.context_a.stashing.data_cl);
-+ seq_printf(file, " ctx_a_stash_context_cl: %u\n",
-+ fqd.context_a.stashing.context_cl);
-+ return 0;
-+}
-+
-+static int query_fq_fields_open(struct inode *inode,
-+ struct file *file)
-+{
-+ return single_open(file, query_fq_fields_show, NULL);
-+}
-+
-+static ssize_t query_fq_fields_write(struct file *f,
-+ const char __user *buf, size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > MAX_FQID)
-+ return -EINVAL;
-+ query_fq_fields_data.fqid = (u32)val;
-+ return count;
-+}
-+
-+static const struct file_operations query_fq_fields_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_fq_fields_open,
-+ .read = seq_read,
-+ .write = query_fq_fields_write,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * Query WQ lengths
-+ ******************************************************************************/
-+struct query_wq_lengths_data_s {
-+ union {
-+ u16 channel_wq; /* ignores wq (3 lsbits) */
-+ struct {
-+ u16 id:13; /* qm_channel */
-+ u16 __reserved:3;
-+ } __packed channel;
-+ };
-+};
-+static struct query_wq_lengths_data_s query_wq_lengths_data;
-+static int query_wq_lengths_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_querywq wq;
-+ int i;
-+
-+ memset(&wq, 0, sizeof(struct qm_mcr_querywq));
-+ wq.channel.id = query_wq_lengths_data.channel.id;
-+ ret = qman_query_wq(0, &wq);
-+ if (ret)
-+ return ret;
-+ seq_printf(file, "Query Result For Channel: 0x%x\n", wq.channel.id);
-+ for (i = 0; i < 8; i++)
-+ /* mask out upper 4 bits since they are not part of length */
-+ seq_printf(file, " wq%d_len : %u\n", i, wq.wq_len[i] & 0x0fff);
-+ return 0;
-+}
-+
-+static int query_wq_lengths_open(struct inode *inode,
-+ struct file *file)
-+{
-+ return single_open(file, query_wq_lengths_show, NULL);
-+}
-+
-+static ssize_t query_wq_lengths_write(struct file *f,
-+ const char __user *buf, size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > 0xfff8)
-+ return -EINVAL;
-+ query_wq_lengths_data.channel.id = (u16)val;
-+ return count;
-+}
-+
-+static const struct file_operations query_wq_lengths_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_wq_lengths_open,
-+ .read = seq_read,
-+ .write = query_wq_lengths_write,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * Query CGR
-+ ******************************************************************************/
-+struct query_cgr_s {
-+ u8 cgid;
-+};
-+static struct query_cgr_s query_cgr_data;
-+
-+static int query_cgr_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_querycgr cgrd;
-+ struct qman_cgr cgr;
-+ int i, j;
-+ u32 mask;
-+
-+ memset(&cgr, 0, sizeof(cgr));
-+ memset(&cgrd, 0, sizeof(cgrd));
-+ cgr.cgrid = query_cgr_data.cgid;
-+ ret = qman_query_cgr(&cgr, &cgrd);
-+ if (ret)
-+ return ret;
-+ seq_printf(file, "Query CGR id 0x%x\n", cgr.cgrid);
-+ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ cgrd.cgr.wr_parm_g.MA, cgrd.cgr.wr_parm_g.Mn,
-+ cgrd.cgr.wr_parm_g.SA, cgrd.cgr.wr_parm_g.Sn,
-+ cgrd.cgr.wr_parm_g.Pn);
-+
-+ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ cgrd.cgr.wr_parm_y.MA, cgrd.cgr.wr_parm_y.Mn,
-+ cgrd.cgr.wr_parm_y.SA, cgrd.cgr.wr_parm_y.Sn,
-+ cgrd.cgr.wr_parm_y.Pn);
-+
-+ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ cgrd.cgr.wr_parm_r.MA, cgrd.cgr.wr_parm_r.Mn,
-+ cgrd.cgr.wr_parm_r.SA, cgrd.cgr.wr_parm_r.Sn,
-+ cgrd.cgr.wr_parm_r.Pn);
-+
-+ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n",
-+ cgrd.cgr.wr_en_g, cgrd.cgr.wr_en_y, cgrd.cgr.wr_en_r);
-+
-+ seq_printf(file, " cscn_en: %u\n", cgrd.cgr.cscn_en);
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
-+ seq_puts(file, " cscn_targ_dcp:\n");
-+ mask = 0x80000000;
-+ for (i = 0; i < 32; i++) {
-+ if (cgrd.cgr.cscn_targ & mask)
-+ seq_printf(file, " send CSCN to dcp %u\n",
-+ (31 - i));
-+ mask >>= 1;
-+ }
-+
-+ seq_puts(file, " cscn_targ_swp:\n");
-+ for (i = 0; i < 4; i++) {
-+ mask = 0x80000000;
-+ for (j = 0; j < 32; j++) {
-+ if (cgrd.cscn_targ_swp[i] & mask)
-+ seq_printf(file, " send CSCN to swp"
-+ " %u\n", (127 - (i * 32) - j));
-+ mask >>= 1;
-+ }
-+ }
-+ } else {
-+ seq_printf(file, " cscn_targ: %u\n", cgrd.cgr.cscn_targ);
-+ }
-+ seq_printf(file, " cstd_en: %u\n", cgrd.cgr.cstd_en);
-+ seq_printf(file, " cs: %u\n", cgrd.cgr.cs);
-+
-+ seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n",
-+ cgrd.cgr.cs_thres.TA, cgrd.cgr.cs_thres.Tn);
-+
-+ seq_printf(file, " mode: %s\n",
-+ (cgrd.cgr.mode & QMAN_CGR_MODE_FRAME) ?
-+ "frame count" : "byte count");
-+ seq_printf(file, " i_bcnt: %llu\n", qm_mcr_querycgr_i_get64(&cgrd));
-+ seq_printf(file, " a_bcnt: %llu\n", qm_mcr_querycgr_a_get64(&cgrd));
-+
-+ return 0;
-+}
-+
-+static int query_cgr_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, query_cgr_show, NULL);
-+}
-+
-+static ssize_t query_cgr_write(struct file *f, const char __user *buf,
-+ size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > 0xff)
-+ return -EINVAL;
-+ query_cgr_data.cgid = (u8)val;
-+ return count;
-+}
-+
-+static const struct file_operations query_cgr_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_cgr_open,
-+ .read = seq_read,
-+ .write = query_cgr_write,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * Test Write CGR
-+ ******************************************************************************/
-+struct test_write_cgr_s {
-+ u64 i_bcnt;
-+ u8 cgid;
-+};
-+static struct test_write_cgr_s test_write_cgr_data;
-+
-+static int testwrite_cgr_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_cgrtestwrite result;
-+ struct qman_cgr cgr;
-+ u64 i_bcnt;
-+
-+ memset(&cgr, 0, sizeof(struct qman_cgr));
-+ memset(&result, 0, sizeof(struct qm_mcr_cgrtestwrite));
-+ cgr.cgrid = test_write_cgr_data.cgid;
-+ i_bcnt = test_write_cgr_data.i_bcnt;
-+ ret = qman_testwrite_cgr(&cgr, i_bcnt, &result);
-+ if (ret)
-+ return ret;
-+ seq_printf(file, "CGR Test Write CGR id 0x%x\n", cgr.cgrid);
-+ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ result.cgr.wr_parm_g.MA, result.cgr.wr_parm_g.Mn,
-+ result.cgr.wr_parm_g.SA, result.cgr.wr_parm_g.Sn,
-+ result.cgr.wr_parm_g.Pn);
-+ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ result.cgr.wr_parm_y.MA, result.cgr.wr_parm_y.Mn,
-+ result.cgr.wr_parm_y.SA, result.cgr.wr_parm_y.Sn,
-+ result.cgr.wr_parm_y.Pn);
-+ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ result.cgr.wr_parm_r.MA, result.cgr.wr_parm_r.Mn,
-+ result.cgr.wr_parm_r.SA, result.cgr.wr_parm_r.Sn,
-+ result.cgr.wr_parm_r.Pn);
-+ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n",
-+ result.cgr.wr_en_g, result.cgr.wr_en_y, result.cgr.wr_en_r);
-+ seq_printf(file, " cscn_en: %u\n", result.cgr.cscn_en);
-+ seq_printf(file, " cscn_targ: %u\n", result.cgr.cscn_targ);
-+ seq_printf(file, " cstd_en: %u\n", result.cgr.cstd_en);
-+ seq_printf(file, " cs: %u\n", result.cgr.cs);
-+ seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n",
-+ result.cgr.cs_thres.TA, result.cgr.cs_thres.Tn);
-+
-+ /* Add Mode for Si 2 */
-+ seq_printf(file, " mode: %s\n",
-+ (result.cgr.mode & QMAN_CGR_MODE_FRAME) ?
-+ "frame count" : "byte count");
-+
-+ seq_printf(file, " i_bcnt: %llu\n",
-+ qm_mcr_cgrtestwrite_i_get64(&result));
-+ seq_printf(file, " a_bcnt: %llu\n",
-+ qm_mcr_cgrtestwrite_a_get64(&result));
-+ seq_printf(file, " wr_prob_g: %u\n", result.wr_prob_g);
-+ seq_printf(file, " wr_prob_y: %u\n", result.wr_prob_y);
-+ seq_printf(file, " wr_prob_r: %u\n", result.wr_prob_r);
-+ return 0;
-+}
-+
-+static int testwrite_cgr_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, testwrite_cgr_show, NULL);
-+}
-+
-+static const struct file_operations testwrite_cgr_fops = {
-+ .owner = THIS_MODULE,
-+ .open = testwrite_cgr_open,
-+ .read = seq_read,
-+ .release = single_release,
-+};
-+
-+
-+static int testwrite_cgr_ibcnt_show(struct seq_file *file, void *offset)
-+{
-+ seq_printf(file, "i_bcnt: %llu\n", test_write_cgr_data.i_bcnt);
-+ return 0;
-+}
-+static int testwrite_cgr_ibcnt_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, testwrite_cgr_ibcnt_show, NULL);
-+}
-+
-+static ssize_t testwrite_cgr_ibcnt_write(struct file *f, const char __user *buf,
-+ size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ test_write_cgr_data.i_bcnt = val;
-+ return count;
-+}
-+
-+static const struct file_operations teswrite_cgr_ibcnt_fops = {
-+ .owner = THIS_MODULE,
-+ .open = testwrite_cgr_ibcnt_open,
-+ .read = seq_read,
-+ .write = testwrite_cgr_ibcnt_write,
-+ .release = single_release,
-+};
-+
-+static int testwrite_cgr_cgrid_show(struct seq_file *file, void *offset)
-+{
-+ seq_printf(file, "cgrid: %u\n", (u32)test_write_cgr_data.cgid);
-+ return 0;
-+}
-+static int testwrite_cgr_cgrid_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, testwrite_cgr_cgrid_show, NULL);
-+}
-+
-+static ssize_t testwrite_cgr_cgrid_write(struct file *f, const char __user *buf,
-+ size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > 0xff)
-+ return -EINVAL;
-+ test_write_cgr_data.cgid = (u8)val;
-+ return count;
-+}
-+
-+static const struct file_operations teswrite_cgr_cgrid_fops = {
-+ .owner = THIS_MODULE,
-+ .open = testwrite_cgr_cgrid_open,
-+ .read = seq_read,
-+ .write = testwrite_cgr_cgrid_write,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * Query Congestion State
-+ ******************************************************************************/
-+static int query_congestion_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_querycongestion cs;
-+ int i, j, in_cong = 0;
-+ u32 mask;
-+
-+ memset(&cs, 0, sizeof(struct qm_mcr_querycongestion));
-+ ret = qman_query_congestion(&cs);
-+ if (ret)
-+ return ret;
-+ seq_puts(file, "Query Congestion Result\n");
-+ for (i = 0; i < 8; i++) {
-+ mask = 0x80000000;
-+ for (j = 0; j < 32; j++) {
-+ if (cs.state.__state[i] & mask) {
-+ in_cong = 1;
-+ seq_printf(file, " cg %u: %s\n", (i*32)+j,
-+ "in congestion");
-+ }
-+ mask >>= 1;
-+ }
-+ }
-+ if (!in_cong)
-+ seq_puts(file, " All congestion groups not congested.\n");
-+ return 0;
-+}
-+
-+static int query_congestion_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, query_congestion_show, NULL);
-+}
-+
-+static const struct file_operations query_congestion_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_congestion_open,
-+ .read = seq_read,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * Query CCGR
-+ ******************************************************************************/
-+struct query_ccgr_s {
-+ u32 ccgid;
-+};
-+static struct query_ccgr_s query_ccgr_data;
-+
-+static int query_ccgr_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_ceetm_ccgr_query ccgr_query;
-+ struct qm_mcc_ceetm_ccgr_query query_opts;
-+ int i, j;
-+ u32 mask;
-+
-+ memset(&ccgr_query, 0, sizeof(struct qm_mcr_ceetm_ccgr_query));
-+ memset(&query_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_query));
-+
-+ if ((qman_ip_rev & 0xFF00) < QMAN_REV30)
-+ return -EINVAL;
-+
-+ seq_printf(file, "Query CCGID %x\n", query_ccgr_data.ccgid);
-+ query_opts.dcpid = ((query_ccgr_data.ccgid & 0xFF000000) >> 24);
-+ query_opts.ccgrid = query_ccgr_data.ccgid & 0x000001FF;
-+ ret = qman_ceetm_query_ccgr(&query_opts, &ccgr_query);
-+ if (ret)
-+ return ret;
-+ seq_printf(file, "Query CCGR id %x in DCP %d\n", query_opts.ccgrid,
-+ query_opts.dcpid);
-+ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ ccgr_query.cm_query.wr_parm_g.MA,
-+ ccgr_query.cm_query.wr_parm_g.Mn,
-+ ccgr_query.cm_query.wr_parm_g.SA,
-+ ccgr_query.cm_query.wr_parm_g.Sn,
-+ ccgr_query.cm_query.wr_parm_g.Pn);
-+
-+ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ ccgr_query.cm_query.wr_parm_y.MA,
-+ ccgr_query.cm_query.wr_parm_y.Mn,
-+ ccgr_query.cm_query.wr_parm_y.SA,
-+ ccgr_query.cm_query.wr_parm_y.Sn,
-+ ccgr_query.cm_query.wr_parm_y.Pn);
-+
-+ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
-+ ccgr_query.cm_query.wr_parm_r.MA,
-+ ccgr_query.cm_query.wr_parm_r.Mn,
-+ ccgr_query.cm_query.wr_parm_r.SA,
-+ ccgr_query.cm_query.wr_parm_r.Sn,
-+ ccgr_query.cm_query.wr_parm_r.Pn);
-+
-+ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n",
-+ ccgr_query.cm_query.ctl_wr_en_g,
-+ ccgr_query.cm_query.ctl_wr_en_y,
-+ ccgr_query.cm_query.ctl_wr_en_r);
-+
-+ seq_printf(file, " cscn_en: %u\n", ccgr_query.cm_query.ctl_cscn_en);
-+ seq_puts(file, " cscn_targ_dcp:\n");
-+ mask = 0x80000000;
-+ for (i = 0; i < 32; i++) {
-+ if (ccgr_query.cm_query.cscn_targ_dcp & mask)
-+ seq_printf(file, " send CSCN to dcp %u\n", (31 - i));
-+ mask >>= 1;
-+ }
-+
-+ seq_puts(file, " cscn_targ_swp:\n");
-+ for (i = 0; i < 4; i++) {
-+ mask = 0x80000000;
-+ for (j = 0; j < 32; j++) {
-+ if (ccgr_query.cm_query.cscn_targ_swp[i] & mask)
-+ seq_printf(file, " send CSCN to swp"
-+ "%u\n", (127 - (i * 32) - j));
-+ mask >>= 1;
-+ }
-+ }
-+
-+ seq_printf(file, " td_en: %u\n", ccgr_query.cm_query.ctl_td_en);
-+
-+ seq_printf(file, " cs_thresh_in_TA: %u, cs_thresh_in_Tn: %u\n",
-+ ccgr_query.cm_query.cs_thres.TA,
-+ ccgr_query.cm_query.cs_thres.Tn);
-+
-+ seq_printf(file, " cs_thresh_out_TA: %u, cs_thresh_out_Tn: %u\n",
-+ ccgr_query.cm_query.cs_thres_x.TA,
-+ ccgr_query.cm_query.cs_thres_x.Tn);
-+
-+ seq_printf(file, " td_thresh_TA: %u, td_thresh_Tn: %u\n",
-+ ccgr_query.cm_query.td_thres.TA,
-+ ccgr_query.cm_query.td_thres.Tn);
-+
-+ seq_printf(file, " mode: %s\n",
-+ (ccgr_query.cm_query.ctl_mode &
-+ QMAN_CGR_MODE_FRAME) ?
-+ "frame count" : "byte count");
-+ seq_printf(file, " i_cnt: %llu\n", (u64)ccgr_query.cm_query.i_cnt);
-+ seq_printf(file, " a_cnt: %llu\n", (u64)ccgr_query.cm_query.a_cnt);
-+
-+ return 0;
-+}
-+
-+static int query_ccgr_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, query_ccgr_show, NULL);
-+}
-+
-+static ssize_t query_ccgr_write(struct file *f, const char __user *buf,
-+ size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ query_ccgr_data.ccgid = val;
-+ return count;
-+}
-+
-+static const struct file_operations query_ccgr_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_ccgr_open,
-+ .read = seq_read,
-+ .write = query_ccgr_write,
-+ .release = single_release,
-+};
-+/*******************************************************************************
-+ * QMan register
-+ ******************************************************************************/
-+struct qman_register_s {
-+ u32 val;
-+};
-+static struct qman_register_s qman_register_data;
-+
-+static void init_ccsrmempeek(void)
-+{
-+ struct device_node *dn;
-+ const u32 *regaddr_p;
-+
-+ dn = of_find_compatible_node(NULL, NULL, "fsl,qman");
-+ if (!dn) {
-+ pr_info("No fsl,qman node\n");
-+ return;
-+ }
-+ regaddr_p = of_get_address(dn, 0, &qman_ccsr_size, NULL);
-+ if (!regaddr_p) {
-+ of_node_put(dn);
-+ return;
-+ }
-+ qman_ccsr_start = of_translate_address(dn, regaddr_p);
-+ of_node_put(dn);
-+}
-+/* This function provides access to QMan ccsr memory map */
-+static int qman_ccsrmempeek(u32 *val, u32 offset)
-+{
-+ void __iomem *addr;
-+ u64 phys_addr;
-+
-+ if (!qman_ccsr_start)
-+ return -EINVAL;
-+
-+ if (offset > (qman_ccsr_size - sizeof(u32)))
-+ return -EINVAL;
-+
-+ phys_addr = qman_ccsr_start + offset;
-+ addr = ioremap(phys_addr, sizeof(u32));
-+ if (!addr) {
-+ pr_err("ccsrmempeek, ioremap failed\n");
-+ return -EINVAL;
-+ }
-+ *val = in_be32(addr);
-+ iounmap(addr);
-+ return 0;
-+}
-+
-+static int qman_ccsrmempeek_show(struct seq_file *file, void *offset)
-+{
-+ u32 b;
-+
-+ qman_ccsrmempeek(&b, qman_register_data.val);
-+ seq_printf(file, "QMan register offset = 0x%x\n",
-+ qman_register_data.val);
-+ seq_printf(file, "value = 0x%08x\n", b);
-+
-+ return 0;
-+}
-+
-+static int qman_ccsrmempeek_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, qman_ccsrmempeek_show, NULL);
-+}
-+
-+static ssize_t qman_ccsrmempeek_write(struct file *f, const char __user *buf,
-+ size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ /* multiple of 4 */
-+ if (val > (qman_ccsr_size - sizeof(u32))) {
-+ pr_info("Input 0x%lx > 0x%llx\n",
-+ val, (qman_ccsr_size - sizeof(u32)));
-+ return -EINVAL;
-+ }
-+ if (val & 0x3) {
-+ pr_info("Input 0x%lx not multiple of 4\n", val);
-+ return -EINVAL;
-+ }
-+ qman_register_data.val = val;
-+ return count;
-+}
-+
-+static const struct file_operations qman_ccsrmempeek_fops = {
-+ .owner = THIS_MODULE,
-+ .open = qman_ccsrmempeek_open,
-+ .read = seq_read,
-+ .write = qman_ccsrmempeek_write,
-+};
-+
-+/*******************************************************************************
-+ * QMan state
-+ ******************************************************************************/
-+static int qman_fqd_state_show(struct seq_file *file, void *offset)
-+{
-+ struct qm_mcr_queryfq_np np;
-+ struct qman_fq fq;
-+ struct line_buffer_fq line_buf;
-+ int ret, i;
-+ u8 *state = file->private;
-+ u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)];
-+
-+ memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt));
-+ memset(&line_buf, 0, sizeof(line_buf));
-+
-+ seq_printf(file, "List of fq ids in state: %s\n", state_txt[*state]);
-+
-+ for (i = 1; i < fqid_max; i++) {
-+ fq.fqid = i;
-+ ret = qman_query_fq_np(&fq, &np);
-+ if (ret)
-+ return ret;
-+ if (*state == (np.state & QM_MCR_NP_STATE_MASK))
-+ add_to_line_buffer(&line_buf, fq.fqid, file);
-+ /* Keep a summary count of all states */
-+ if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states))
-+ qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++;
-+ }
-+ flush_line_buffer(&line_buf, file);
-+
-+ for (i = 0; i < ARRAY_SIZE(fqd_states); i++) {
-+ seq_printf(file, "%s count = %u\n", state_txt[i],
-+ qm_fq_state_cnt[i]);
-+ }
-+ return 0;
-+}
-+
-+static int qman_fqd_state_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, qman_fqd_state_show, inode->i_private);
-+}
-+
-+static const struct file_operations qman_fqd_state_fops = {
-+ .owner = THIS_MODULE,
-+ .open = qman_fqd_state_open,
-+ .read = seq_read,
-+};
-+
-+static int qman_fqd_ctrl_show(struct seq_file *file, void *offset)
-+{
-+ struct qm_fqd fqd;
-+ struct qman_fq fq;
-+ u32 fq_en_cnt = 0, fq_di_cnt = 0;
-+ int ret, i;
-+ struct mask_filter_s *data = file->private;
-+ const char *ctrl_txt = get_fqd_ctrl_text(data->mask);
-+ struct line_buffer_fq line_buf;
-+
-+ memset(&line_buf, 0, sizeof(line_buf));
-+ seq_printf(file, "List of fq ids with: %s :%s\n",
-+ ctrl_txt, (data->filter) ? "enabled" : "disabled");
-+ for (i = 1; i < fqid_max; i++) {
-+ fq.fqid = i;
-+ memset(&fqd, 0, sizeof(struct qm_fqd));
-+ ret = qman_query_fq(&fq, &fqd);
-+ if (ret)
-+ return ret;
-+ if (data->filter) {
-+ if (fqd.fq_ctrl & data->mask)
-+ add_to_line_buffer(&line_buf, fq.fqid, file);
-+ } else {
-+ if (!(fqd.fq_ctrl & data->mask))
-+ add_to_line_buffer(&line_buf, fq.fqid, file);
-+ }
-+ if (fqd.fq_ctrl & data->mask)
-+ fq_en_cnt++;
-+ else
-+ fq_di_cnt++;
-+ }
-+ flush_line_buffer(&line_buf, file);
-+
-+ seq_printf(file, "Total FQD with: %s : enabled = %u\n",
-+ ctrl_txt, fq_en_cnt);
-+ seq_printf(file, "Total FQD with: %s : disabled = %u\n",
-+ ctrl_txt, fq_di_cnt);
-+ return 0;
-+}
-+
-+/*******************************************************************************
-+ * QMan ctrl CGE, TDE, ORP, CTX, CPC, SFDR, BLOCK, HOLD, CACHE
-+ ******************************************************************************/
-+static int qman_fqd_ctrl_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, qman_fqd_ctrl_show, inode->i_private);
-+}
-+
-+static const struct file_operations qman_fqd_ctrl_fops = {
-+ .owner = THIS_MODULE,
-+ .open = qman_fqd_ctrl_open,
-+ .read = seq_read,
-+};
-+
-+/*******************************************************************************
-+ * QMan ctrl summary
-+ ******************************************************************************/
-+/*******************************************************************************
-+ * QMan summary state
-+ ******************************************************************************/
-+static int qman_fqd_non_prog_summary_show(struct seq_file *file, void *offset)
-+{
-+ struct qm_mcr_queryfq_np np;
-+ struct qman_fq fq;
-+ int ret, i;
-+ u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)];
-+
-+ memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt));
-+
-+ for (i = 1; i < fqid_max; i++) {
-+ fq.fqid = i;
-+ ret = qman_query_fq_np(&fq, &np);
-+ if (ret)
-+ return ret;
-+ /* Keep a summary count of all states */
-+ if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states))
-+ qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++;
-+ }
-+
-+ for (i = 0; i < ARRAY_SIZE(fqd_states); i++) {
-+ seq_printf(file, "%s count = %u\n", state_txt[i],
-+ qm_fq_state_cnt[i]);
-+ }
-+ return 0;
-+}
-+
-+static int qman_fqd_prog_summary_show(struct seq_file *file, void *offset)
-+{
-+ struct qm_fqd fqd;
-+ struct qman_fq fq;
-+ int ret, i , j;
-+ u32 qm_prog_cnt[ARRAY_SIZE(mask_filter)/2];
-+
-+ memset(qm_prog_cnt, 0, sizeof(qm_prog_cnt));
-+
-+ for (i = 1; i < fqid_max; i++) {
-+ memset(&fqd, 0, sizeof(struct qm_fqd));
-+ fq.fqid = i;
-+ ret = qman_query_fq(&fq, &fqd);
-+ if (ret)
-+ return ret;
-+ /* Keep a summary count of all states */
-+ for (j = 0; j < ARRAY_SIZE(mask_filter); j += 2)
-+ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) &
-+ mask_filter[j].mask)
-+ qm_prog_cnt[j/2]++;
-+ }
-+ for (i = 0; i < ARRAY_SIZE(mask_filter) / 2; i++) {
-+ seq_printf(file, "%s count = %u\n",
-+ get_fqd_ctrl_text(mask_filter[i*2].mask),
-+ qm_prog_cnt[i]);
-+ }
-+ return 0;
-+}
-+
-+static int qman_fqd_summary_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+
-+ /* Display summary of non programmable fields */
-+ ret = qman_fqd_non_prog_summary_show(file, offset);
-+ if (ret)
-+ return ret;
-+ seq_puts(file, "-----------------------------------------\n");
-+ /* Display programmable fields */
-+ ret = qman_fqd_prog_summary_show(file, offset);
-+ if (ret)
-+ return ret;
-+ return 0;
-+}
-+
-+static int qman_fqd_summary_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, qman_fqd_summary_show, NULL);
-+}
-+
-+static const struct file_operations qman_fqd_summary_fops = {
-+ .owner = THIS_MODULE,
-+ .open = qman_fqd_summary_open,
-+ .read = seq_read,
-+};
-+
-+/*******************************************************************************
-+ * QMan destination work queue
-+ ******************************************************************************/
-+struct qman_dest_wq_s {
-+ u16 wq_id;
-+};
-+static struct qman_dest_wq_s qman_dest_wq_data = {
-+ .wq_id = 0,
-+};
-+
-+static int qman_fqd_dest_wq_show(struct seq_file *file, void *offset)
-+{
-+ struct qm_fqd fqd;
-+ struct qman_fq fq;
-+ int ret, i;
-+ u16 *wq, wq_id = qman_dest_wq_data.wq_id;
-+ struct line_buffer_fq line_buf;
-+
-+ memset(&line_buf, 0, sizeof(line_buf));
-+ /* use vmalloc : need to allocate large memory region and don't
-+ * require the memory to be physically contiguous. */
-+ wq = vzalloc(sizeof(u16) * (0xFFFF+1));
-+ if (!wq)
-+ return -ENOMEM;
-+
-+ seq_printf(file, "List of fq ids with destination work queue id"
-+ " = 0x%x\n", wq_id);
-+
-+ for (i = 1; i < fqid_max; i++) {
-+ fq.fqid = i;
-+ memset(&fqd, 0, sizeof(struct qm_fqd));
-+ ret = qman_query_fq(&fq, &fqd);
-+ if (ret) {
-+ vfree(wq);
-+ return ret;
-+ }
-+ if (wq_id == fqd.dest_wq)
-+ add_to_line_buffer(&line_buf, fq.fqid, file);
-+ wq[fqd.dest_wq]++;
-+ }
-+ flush_line_buffer(&line_buf, file);
-+
-+ seq_puts(file, "Summary of all FQD destination work queue values\n");
-+ for (i = 0; i < 0xFFFF; i++) {
-+ if (wq[i])
-+ seq_printf(file, "Channel: 0x%x WQ: 0x%x WQ_ID: 0x%x, "
-+ "count = %u\n", i >> 3, i & 0x3, i, wq[i]);
-+ }
-+ vfree(wq);
-+ return 0;
-+}
-+
-+static ssize_t qman_fqd_dest_wq_write(struct file *f, const char __user *buf,
-+ size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > 0xFFFF)
-+ return -EINVAL;
-+ qman_dest_wq_data.wq_id = val;
-+ return count;
-+}
-+
-+static int qman_fqd_dest_wq_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, qman_fqd_dest_wq_show, NULL);
-+}
-+
-+static const struct file_operations qman_fqd_dest_wq_fops = {
-+ .owner = THIS_MODULE,
-+ .open = qman_fqd_dest_wq_open,
-+ .read = seq_read,
-+ .write = qman_fqd_dest_wq_write,
-+};
-+
-+/*******************************************************************************
-+ * QMan Intra-Class Scheduling Credit
-+ ******************************************************************************/
-+static int qman_fqd_cred_show(struct seq_file *file, void *offset)
-+{
-+ struct qm_fqd fqd;
-+ struct qman_fq fq;
-+ int ret, i;
-+ u32 fq_cnt = 0;
-+ struct line_buffer_fq line_buf;
-+
-+ memset(&line_buf, 0, sizeof(line_buf));
-+ seq_puts(file, "List of fq ids with Intra-Class Scheduling Credit > 0"
-+ "\n");
-+
-+ for (i = 1; i < fqid_max; i++) {
-+ fq.fqid = i;
-+ memset(&fqd, 0, sizeof(struct qm_fqd));
-+ ret = qman_query_fq(&fq, &fqd);
-+ if (ret)
-+ return ret;
-+ if (fqd.ics_cred > 0) {
-+ add_to_line_buffer(&line_buf, fq.fqid, file);
-+ fq_cnt++;
-+ }
-+ }
-+ flush_line_buffer(&line_buf, file);
-+
-+ seq_printf(file, "Total FQD with ics_cred > 0 = %d\n", fq_cnt);
-+ return 0;
-+}
-+
-+static int qman_fqd_cred_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, qman_fqd_cred_show, NULL);
-+}
-+
-+static const struct file_operations qman_fqd_cred_fops = {
-+ .owner = THIS_MODULE,
-+ .open = qman_fqd_cred_open,
-+ .read = seq_read,
-+};
-+
-+/*******************************************************************************
-+ * Class Queue Fields
-+ ******************************************************************************/
-+struct query_cq_fields_data_s {
-+ u32 cqid;
-+};
-+
-+static struct query_cq_fields_data_s query_cq_fields_data = {
-+ .cqid = 1,
-+};
-+
-+static int query_cq_fields_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ struct qm_mcr_ceetm_cq_query query_result;
-+ unsigned int cqid;
-+ unsigned int portal;
-+
-+ if ((qman_ip_rev & 0xFF00) < QMAN_REV30)
-+ return -EINVAL;
-+
-+ cqid = query_cq_fields_data.cqid & 0x00FFFFFF;
-+ portal = query_cq_fields_data.cqid >> 24;
-+ if (portal > qm_dc_portal_fman1)
-+ return -EINVAL;
-+
-+ ret = qman_ceetm_query_cq(cqid, portal, &query_result);
-+ if (ret)
-+ return ret;
-+ seq_printf(file, "Query CQ Fields Result cqid 0x%x on DCP %d\n",
-+ cqid, portal);
-+ seq_printf(file, " ccgid: %u\n", query_result.ccgid);
-+ seq_printf(file, " state: %u\n", query_result.state);
-+ seq_printf(file, " pfdr_hptr: %u\n", query_result.pfdr_hptr);
-+ seq_printf(file, " pfdr_tptr: %u\n", query_result.pfdr_tptr);
-+ seq_printf(file, " od1_xsfdr: %u\n", query_result.od1_xsfdr);
-+ seq_printf(file, " od2_xsfdr: %u\n", query_result.od2_xsfdr);
-+ seq_printf(file, " od3_xsfdr: %u\n", query_result.od3_xsfdr);
-+ seq_printf(file, " od4_xsfdr: %u\n", query_result.od4_xsfdr);
-+ seq_printf(file, " od5_xsfdr: %u\n", query_result.od5_xsfdr);
-+ seq_printf(file, " od6_xsfdr: %u\n", query_result.od6_xsfdr);
-+ seq_printf(file, " ra1_xsfdr: %u\n", query_result.ra1_xsfdr);
-+ seq_printf(file, " ra2_xsfdr: %u\n", query_result.ra2_xsfdr);
-+ seq_printf(file, " frame_count: %u\n", query_result.frm_cnt);
-+
-+ return 0;
-+}
-+
-+static int query_cq_fields_open(struct inode *inode,
-+ struct file *file)
-+{
-+ return single_open(file, query_cq_fields_show, NULL);
-+}
-+
-+static ssize_t query_cq_fields_write(struct file *f,
-+ const char __user *buf, size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ query_cq_fields_data.cqid = (u32)val;
-+ return count;
-+}
-+
-+static const struct file_operations query_cq_fields_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_cq_fields_open,
-+ .read = seq_read,
-+ .write = query_cq_fields_write,
-+ .release = single_release,
-+};
-+
-+/*******************************************************************************
-+ * READ CEETM_XSFDR_IN_USE
-+ ******************************************************************************/
-+struct query_ceetm_xsfdr_data_s {
-+ enum qm_dc_portal dcp_portal;
-+};
-+
-+static struct query_ceetm_xsfdr_data_s query_ceetm_xsfdr_data;
-+
-+static int query_ceetm_xsfdr_show(struct seq_file *file, void *offset)
-+{
-+ int ret;
-+ unsigned int xsfdr_in_use;
-+ enum qm_dc_portal portal;
-+
-+
-+ if (qman_ip_rev < QMAN_REV31)
-+ return -EINVAL;
-+
-+ portal = query_ceetm_xsfdr_data.dcp_portal;
-+ ret = qman_ceetm_get_xsfdr(portal, &xsfdr_in_use);
-+ if (ret) {
-+ seq_printf(file, "Read CEETM_XSFDR_IN_USE on DCP %d failed\n",
-+ portal);
-+ return ret;
-+ }
-+
-+ seq_printf(file, "DCP%d: CEETM_XSFDR_IN_USE number is %u\n", portal,
-+ (xsfdr_in_use & 0x1FFF));
-+ return 0;
-+}
-+
-+static int query_ceetm_xsfdr_open(struct inode *inode,
-+ struct file *file)
-+{
-+ return single_open(file, query_ceetm_xsfdr_show, NULL);
-+}
-+
-+static ssize_t query_ceetm_xsfdr_write(struct file *f,
-+ const char __user *buf, size_t count, loff_t *off)
-+{
-+ int ret;
-+ unsigned long val;
-+
-+ ret = user_input_convert(buf, count, &val);
-+ if (ret)
-+ return ret;
-+ if (val > qm_dc_portal_fman1)
-+ return -EINVAL;
-+ query_ceetm_xsfdr_data.dcp_portal = (u32)val;
-+ return count;
-+}
-+
-+static const struct file_operations query_ceetm_xsfdr_fops = {
-+ .owner = THIS_MODULE,
-+ .open = query_ceetm_xsfdr_open,
-+ .read = seq_read,
-+ .write = query_ceetm_xsfdr_write,
-+ .release = single_release,
-+};
-+
-+/* helper macros used in qman_debugfs_module_init */
-+#define QMAN_DBGFS_ENTRY(name, mode, parent, data, fops) \
-+ do { \
-+ d = debugfs_create_file(name, \
-+ mode, parent, \
-+ data, \
-+ fops); \
-+ if (d == NULL) { \
-+ ret = -ENOMEM; \
-+ goto _return; \
-+ } \
-+ } while (0)
-+
-+/* dfs_root as parent */
-+#define QMAN_DBGFS_ENTRY_ROOT(name, mode, data, fops) \
-+ QMAN_DBGFS_ENTRY(name, mode, dfs_root, data, fops)
-+
-+/* fqd_root as parent */
-+#define QMAN_DBGFS_ENTRY_FQDROOT(name, mode, data, fops) \
-+ QMAN_DBGFS_ENTRY(name, mode, fqd_root, data, fops)
-+
-+/* fqd state */
-+#define QMAN_DBGFS_ENTRY_FQDSTATE(name, index) \
-+ QMAN_DBGFS_ENTRY_FQDROOT(name, S_IRUGO, \
-+ (void *)&mask_filter[index], &qman_fqd_ctrl_fops)
-+
-+static int __init qman_debugfs_module_init(void)
-+{
-+ int ret = 0;
-+ struct dentry *d, *fqd_root;
-+ u32 reg;
-+
-+ fqid_max = 0;
-+ init_ccsrmempeek();
-+ if (qman_ccsr_start) {
-+ if (!qman_ccsrmempeek(®, QM_FQD_AR)) {
-+ /* extract the size of the FQD window */
-+ reg = reg & 0x3f;
-+ /* calculate valid frame queue descriptor range */
-+ fqid_max = (1 << (reg + 1)) / QM_FQD_BLOCK_SIZE;
-+ }
-+ }
-+ dfs_root = debugfs_create_dir("qman", NULL);
-+ fqd_root = debugfs_create_dir("fqd", dfs_root);
-+ if (dfs_root == NULL || fqd_root == NULL) {
-+ ret = -ENOMEM;
-+ pr_err("Cannot create qman/fqd debugfs dir\n");
-+ goto _return;
-+ }
-+ if (fqid_max) {
-+ QMAN_DBGFS_ENTRY_ROOT("ccsrmempeek", S_IRUGO | S_IWUGO,
-+ NULL, &qman_ccsrmempeek_fops);
-+ }
-+ QMAN_DBGFS_ENTRY_ROOT("query_fq_np_fields", S_IRUGO | S_IWUGO,
-+ &query_fq_np_fields_data, &query_fq_np_fields_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("query_fq_fields", S_IRUGO | S_IWUGO,
-+ &query_fq_fields_data, &query_fq_fields_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("query_wq_lengths", S_IRUGO | S_IWUGO,
-+ &query_wq_lengths_data, &query_wq_lengths_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("query_cgr", S_IRUGO | S_IWUGO,
-+ &query_cgr_data, &query_cgr_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("query_congestion", S_IRUGO,
-+ NULL, &query_congestion_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr", S_IRUGO,
-+ NULL, &testwrite_cgr_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_cgrid", S_IRUGO | S_IWUGO,
-+ NULL, &teswrite_cgr_cgrid_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_ibcnt", S_IRUGO | S_IWUGO,
-+ NULL, &teswrite_cgr_ibcnt_fops);
-+
-+ QMAN_DBGFS_ENTRY_ROOT("query_ceetm_ccgr", S_IRUGO | S_IWUGO,
-+ &query_ccgr_data, &query_ccgr_fops);
-+ /* Create files with fqd_root as parent */
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("stateoos", S_IRUGO,
-+ (void *)&fqd_states[QM_MCR_NP_STATE_OOS], &qman_fqd_state_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("state_retired", S_IRUGO,
-+ (void *)&fqd_states[QM_MCR_NP_STATE_RETIRED],
-+ &qman_fqd_state_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("state_tentatively_sched", S_IRUGO,
-+ (void *)&fqd_states[QM_MCR_NP_STATE_TEN_SCHED],
-+ &qman_fqd_state_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("state_truly_sched", S_IRUGO,
-+ (void *)&fqd_states[QM_MCR_NP_STATE_TRU_SCHED],
-+ &qman_fqd_state_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("state_parked", S_IRUGO,
-+ (void *)&fqd_states[QM_MCR_NP_STATE_PARKED],
-+ &qman_fqd_state_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("state_active", S_IRUGO,
-+ (void *)&fqd_states[QM_MCR_NP_STATE_ACTIVE],
-+ &qman_fqd_state_fops);
-+ QMAN_DBGFS_ENTRY_ROOT("query_cq_fields", S_IRUGO | S_IWUGO,
-+ &query_cq_fields_data, &query_cq_fields_fops);
-+ QMAN_DBGFS_ENTRY_ROOT("query_ceetm_xsfdr_in_use", S_IRUGO | S_IWUGO,
-+ &query_ceetm_xsfdr_data, &query_ceetm_xsfdr_fops);
-+
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("cge_enable", 17);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("cge_disable", 16);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("tde_enable", 15);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("tde_disable", 14);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("orp_enable", 13);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("orp_disable", 12);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_enable", 11);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_disable", 10);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("cpc_enable", 9);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("cpc_disable", 8);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_enable", 7);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_disable", 6);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_enable", 5);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_disable", 4);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_enable", 3);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_disable", 2);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_enable", 1);
-+
-+ QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_disable", 0);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("summary", S_IRUGO,
-+ NULL, &qman_fqd_summary_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("wq", S_IRUGO | S_IWUGO,
-+ NULL, &qman_fqd_dest_wq_fops);
-+
-+ QMAN_DBGFS_ENTRY_FQDROOT("cred", S_IRUGO,
-+ NULL, &qman_fqd_cred_fops);
-+
-+ return 0;
-+
-+_return:
-+ debugfs_remove_recursive(dfs_root);
-+ return ret;
-+}
-+
-+static void __exit qman_debugfs_module_exit(void)
-+{
-+ debugfs_remove_recursive(dfs_root);
-+}
-+
-+module_init(qman_debugfs_module_init);
-+module_exit(qman_debugfs_module_exit);
-+MODULE_LICENSE("Dual BSD/GPL");
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_driver.c
-@@ -0,0 +1,961 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "qman_private.h"
-+
-+#include <asm/smp.h> /* hard_smp_processor_id() if !CONFIG_SMP */
-+#ifdef CONFIG_HOTPLUG_CPU
-+#include <linux/cpu.h>
-+#endif
-+
-+/* Global variable containing revision id (even on non-control plane systems
-+ * where CCSR isn't available) */
-+u16 qman_ip_rev;
-+EXPORT_SYMBOL(qman_ip_rev);
-+u8 qman_ip_cfg;
-+EXPORT_SYMBOL(qman_ip_cfg);
-+u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1;
-+EXPORT_SYMBOL(qm_channel_pool1);
-+u16 qm_channel_caam = QMAN_CHANNEL_CAAM;
-+EXPORT_SYMBOL(qm_channel_caam);
-+u16 qm_channel_pme = QMAN_CHANNEL_PME;
-+EXPORT_SYMBOL(qm_channel_pme);
-+u16 qm_channel_dce = QMAN_CHANNEL_DCE;
-+EXPORT_SYMBOL(qm_channel_dce);
-+u16 qman_portal_max;
-+EXPORT_SYMBOL(qman_portal_max);
-+
-+u32 qman_clk;
-+struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX];
-+/* the qman ceetm instances on the given SoC */
-+u8 num_ceetms;
-+
-+/* For these variables, and the portal-initialisation logic, the
-+ * comments in bman_driver.c apply here so won't be repeated. */
-+static struct qman_portal *shared_portals[NR_CPUS];
-+static int num_shared_portals;
-+static int shared_portals_idx;
-+static LIST_HEAD(unused_pcfgs);
-+static DEFINE_SPINLOCK(unused_pcfgs_lock);
-+
-+/* A SDQCR mask comprising all the available/visible pool channels */
-+static u32 pools_sdqcr;
-+
-+#define STR_ERR_NOPROP "No '%s' property in node %s\n"
-+#define STR_ERR_CELL "'%s' is not a %d-cell range in node %s\n"
-+#define STR_FQID_RANGE "fsl,fqid-range"
-+#define STR_POOL_CHAN_RANGE "fsl,pool-channel-range"
-+#define STR_CGRID_RANGE "fsl,cgrid-range"
-+
-+/* A "fsl,fqid-range" node; release the given range to the allocator */
-+static __init int fsl_fqid_range_init(struct device_node *node)
-+{
-+ int ret;
-+ const u32 *range = of_get_property(node, STR_FQID_RANGE, &ret);
-+ if (!range) {
-+ pr_err(STR_ERR_NOPROP, STR_FQID_RANGE, node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err(STR_ERR_CELL, STR_FQID_RANGE, 2, node->full_name);
-+ return -EINVAL;
-+ }
-+ qman_seed_fqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ pr_info("Qman: FQID allocator includes range %d:%d\n",
-+ be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ return 0;
-+}
-+
-+/* A "fsl,pool-channel-range" node; add to the SDQCR mask only */
-+static __init int fsl_pool_channel_range_sdqcr(struct device_node *node)
-+{
-+ int ret;
-+ const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret);
-+ if (!chanid) {
-+ pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name);
-+ return -EINVAL;
-+ }
-+ for (ret = 0; ret < be32_to_cpu(chanid[1]); ret++)
-+ pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(be32_to_cpu(chanid[0]) + ret);
-+ return 0;
-+}
-+
-+/* A "fsl,pool-channel-range" node; release the given range to the allocator */
-+static __init int fsl_pool_channel_range_init(struct device_node *node)
-+{
-+ int ret;
-+ const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret);
-+ if (!chanid) {
-+ pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name);
-+ return -EINVAL;
-+ }
-+ qman_seed_pool_range(be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1]));
-+ pr_info("Qman: pool channel allocator includes range %d:%d\n",
-+ be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1]));
-+ return 0;
-+}
-+
-+/* A "fsl,cgrid-range" node; release the given range to the allocator */
-+static __init int fsl_cgrid_range_init(struct device_node *node)
-+{
-+ struct qman_cgr cgr;
-+ int ret, errors = 0;
-+ const u32 *range = of_get_property(node, STR_CGRID_RANGE, &ret);
-+ if (!range) {
-+ pr_err(STR_ERR_NOPROP, STR_CGRID_RANGE, node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err(STR_ERR_CELL, STR_CGRID_RANGE, 2, node->full_name);
-+ return -EINVAL;
-+ }
-+ qman_seed_cgrid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ pr_info("Qman: CGRID allocator includes range %d:%d\n",
-+ be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ for (cgr.cgrid = 0; cgr.cgrid < __CGR_NUM; cgr.cgrid++) {
-+ ret = qman_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL);
-+ if (ret)
-+ errors++;
-+ }
-+ if (errors)
-+ pr_err("Warning: %d error%s while initialising CGRs %d:%d\n",
-+ errors, (errors > 1) ? "s" : "", range[0], range[1]);
-+ return 0;
-+}
-+
-+static __init int fsl_ceetm_init(struct device_node *node)
-+{
-+ enum qm_dc_portal dcp_portal;
-+ struct qm_ceetm_sp *sp;
-+ struct qm_ceetm_lni *lni;
-+ int ret, i;
-+ const u32 *range;
-+
-+ /* Find LFQID range */
-+ range = of_get_property(node, "fsl,ceetm-lfqid-range", &ret);
-+ if (!range) {
-+ pr_err("No fsl,ceetm-lfqid-range in node %s\n",
-+ node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err("fsl,ceetm-lfqid-range is not a 2-cell range in node"
-+ " %s\n", node->full_name);
-+ return -EINVAL;
-+ }
-+
-+ dcp_portal = (be32_to_cpu(range[0]) & 0x0F0000) >> 16;
-+ if (dcp_portal > qm_dc_portal_fman1) {
-+ pr_err("The DCP portal %d doesn't support CEETM\n", dcp_portal);
-+ return -EINVAL;
-+ }
-+
-+ if (dcp_portal == qm_dc_portal_fman0)
-+ qman_seed_ceetm0_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ if (dcp_portal == qm_dc_portal_fman1)
-+ qman_seed_ceetm1_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ pr_debug("Qman: The lfqid allocator of CEETM %d includes range"
-+ " 0x%x:0x%x\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+
-+ qman_ceetms[dcp_portal].idx = dcp_portal;
-+ INIT_LIST_HEAD(&qman_ceetms[dcp_portal].sub_portals);
-+ INIT_LIST_HEAD(&qman_ceetms[dcp_portal].lnis);
-+
-+ /* Find Sub-portal range */
-+ range = of_get_property(node, "fsl,ceetm-sp-range", &ret);
-+ if (!range) {
-+ pr_err("No fsl,ceetm-sp-range in node %s\n", node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err("fsl,ceetm-sp-range is not a 2-cell range in node %s\n",
-+ node->full_name);
-+ return -EINVAL;
-+ }
-+
-+ for (i = 0; i < be32_to_cpu(range[1]); i++) {
-+ sp = kzalloc(sizeof(*sp), GFP_KERNEL);
-+ if (!sp) {
-+ pr_err("Can't alloc memory for sub-portal %d\n",
-+ range[0] + i);
-+ return -ENOMEM;
-+ }
-+ sp->idx = be32_to_cpu(range[0]) + i;
-+ sp->dcp_idx = dcp_portal;
-+ sp->is_claimed = 0;
-+ list_add_tail(&sp->node, &qman_ceetms[dcp_portal].sub_portals);
-+ sp++;
-+ }
-+ pr_debug("Qman: Reserve sub-portal %d:%d for CEETM %d\n",
-+ be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal);
-+ qman_ceetms[dcp_portal].sp_range[0] = be32_to_cpu(range[0]);
-+ qman_ceetms[dcp_portal].sp_range[1] = be32_to_cpu(range[1]);
-+
-+ /* Find LNI range */
-+ range = of_get_property(node, "fsl,ceetm-lni-range", &ret);
-+ if (!range) {
-+ pr_err("No fsl,ceetm-lni-range in node %s\n", node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err("fsl,ceetm-lni-range is not a 2-cell range in node %s\n",
-+ node->full_name);
-+ return -EINVAL;
-+ }
-+
-+ for (i = 0; i < be32_to_cpu(range[1]); i++) {
-+ lni = kzalloc(sizeof(*lni), GFP_KERNEL);
-+ if (!lni) {
-+ pr_err("Can't alloc memory for LNI %d\n",
-+ range[0] + i);
-+ return -ENOMEM;
-+ }
-+ lni->idx = be32_to_cpu(range[0]) + i;
-+ lni->dcp_idx = dcp_portal;
-+ lni->is_claimed = 0;
-+ INIT_LIST_HEAD(&lni->channels);
-+ list_add_tail(&lni->node, &qman_ceetms[dcp_portal].lnis);
-+ lni++;
-+ }
-+ pr_debug("Qman: Reserve LNI %d:%d for CEETM %d\n",
-+ be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal);
-+ qman_ceetms[dcp_portal].lni_range[0] = be32_to_cpu(range[0]);
-+ qman_ceetms[dcp_portal].lni_range[1] = be32_to_cpu(range[1]);
-+
-+ /* Find CEETM channel range */
-+ range = of_get_property(node, "fsl,ceetm-channel-range", &ret);
-+ if (!range) {
-+ pr_err("No fsl,ceetm-channel-range in node %s\n",
-+ node->full_name);
-+ return -EINVAL;
-+ }
-+ if (ret != 8) {
-+ pr_err("fsl,ceetm-channel-range is not a 2-cell range in node"
-+ "%s\n", node->full_name);
-+ return -EINVAL;
-+ }
-+
-+ if (dcp_portal == qm_dc_portal_fman0)
-+ qman_seed_ceetm0_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ if (dcp_portal == qm_dc_portal_fman1)
-+ qman_seed_ceetm1_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+ pr_debug("Qman: The channel allocator of CEETM %d includes"
-+ " range %d:%d\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1]));
-+
-+ /* Set CEETM PRES register */
-+ ret = qman_ceetm_set_prescaler(dcp_portal);
-+ if (ret)
-+ return ret;
-+ return 0;
-+}
-+
-+static void qman_get_ip_revision(struct device_node *dn)
-+{
-+ u16 ip_rev = 0;
-+ u8 ip_cfg = QMAN_REV_CFG_0;
-+ for_each_compatible_node(dn, NULL, "fsl,qman-portal") {
-+ if (!of_device_is_available(dn))
-+ continue;
-+ if (of_device_is_compatible(dn, "fsl,qman-portal-1.0") ||
-+ of_device_is_compatible(dn, "fsl,qman-portal-1.0.0")) {
-+ pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n");
-+ BUG_ON(1);
-+ } else if (of_device_is_compatible(dn, "fsl,qman-portal-1.1") ||
-+ of_device_is_compatible(dn, "fsl,qman-portal-1.1.0")) {
-+ ip_rev = QMAN_REV11;
-+ qman_portal_max = 10;
-+ } else if (of_device_is_compatible(dn, "fsl,qman-portal-1.2") ||
-+ of_device_is_compatible(dn, "fsl,qman-portal-1.2.0")) {
-+ ip_rev = QMAN_REV12;
-+ qman_portal_max = 10;
-+ } else if (of_device_is_compatible(dn, "fsl,qman-portal-2.0") ||
-+ of_device_is_compatible(dn, "fsl,qman-portal-2.0.0")) {
-+ ip_rev = QMAN_REV20;
-+ qman_portal_max = 3;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.0.0")) {
-+ ip_rev = QMAN_REV30;
-+ qman_portal_max = 50;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.0.1")) {
-+ ip_rev = QMAN_REV30;
-+ qman_portal_max = 25;
-+ ip_cfg = QMAN_REV_CFG_1;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.1.0")) {
-+ ip_rev = QMAN_REV31;
-+ qman_portal_max = 50;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.1.1")) {
-+ ip_rev = QMAN_REV31;
-+ qman_portal_max = 25;
-+ ip_cfg = QMAN_REV_CFG_1;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.1.2")) {
-+ ip_rev = QMAN_REV31;
-+ qman_portal_max = 18;
-+ ip_cfg = QMAN_REV_CFG_2;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.1.3")) {
-+ ip_rev = QMAN_REV31;
-+ qman_portal_max = 10;
-+ ip_cfg = QMAN_REV_CFG_3;
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.2.0")) {
-+ ip_rev = QMAN_REV32;
-+ qman_portal_max = 10;
-+ ip_cfg = QMAN_REV_CFG_3; // TODO: Verify for ls1043
-+ } else if (of_device_is_compatible(dn,
-+ "fsl,qman-portal-3.2.1")) {
-+ ip_rev = QMAN_REV32;
-+ qman_portal_max = 10;
-+ ip_cfg = QMAN_REV_CFG_3;
-+ } else {
-+ pr_warn("unknown QMan version in portal node,"
-+ "default to rev1.1\n");
-+ ip_rev = QMAN_REV11;
-+ qman_portal_max = 10;
-+ }
-+
-+ if (!qman_ip_rev) {
-+ if (ip_rev) {
-+ qman_ip_rev = ip_rev;
-+ qman_ip_cfg = ip_cfg;
-+ } else {
-+ pr_warn("unknown Qman version,"
-+ " default to rev1.1\n");
-+ qman_ip_rev = QMAN_REV11;
-+ qman_ip_cfg = QMAN_REV_CFG_0;
-+ }
-+ } else if (ip_rev && (qman_ip_rev != ip_rev))
-+ pr_warn("Revision=0x%04x, but portal '%s' has"
-+ " 0x%04x\n",
-+ qman_ip_rev, dn->full_name, ip_rev);
-+ if (qman_ip_rev == ip_rev)
-+ break;
-+ }
-+}
-+
-+/* Parse a portal node, perform generic mapping duties and return the config. It
-+ * is not known at this stage for what purpose (or even if) the portal will be
-+ * used. */
-+static struct qm_portal_config * __init parse_pcfg(struct device_node *node)
-+{
-+ struct qm_portal_config *pcfg;
-+ const u32 *index_p;
-+ u32 index, channel;
-+ int irq, ret;
-+ resource_size_t len;
-+
-+ pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL);
-+ if (!pcfg) {
-+ pr_err("can't allocate portal config");
-+ return NULL;
-+ }
-+
-+ /*
-+ * This is a *horrible hack*, but the IOMMU/PAMU driver needs a
-+ * 'struct device' in order to get the PAMU stashing setup and the QMan
-+ * portal [driver] won't function at all without ring stashing
-+ *
-+ * Making the QMan portal driver nice and proper is part of the
-+ * upstreaming effort
-+ */
-+ pcfg->dev.bus = &platform_bus_type;
-+ pcfg->dev.of_node = node;
-+#ifdef CONFIG_FSL_PAMU
-+ pcfg->dev.archdata.iommu_domain = NULL;
-+#endif
-+
-+ ret = of_address_to_resource(node, DPA_PORTAL_CE,
-+ &pcfg->addr_phys[DPA_PORTAL_CE]);
-+ if (ret) {
-+ pr_err("Can't get %s property '%s'\n", node->full_name,
-+ "reg::CE");
-+ goto err;
-+ }
-+ ret = of_address_to_resource(node, DPA_PORTAL_CI,
-+ &pcfg->addr_phys[DPA_PORTAL_CI]);
-+ if (ret) {
-+ pr_err("Can't get %s property '%s'\n", node->full_name,
-+ "reg::CI");
-+ goto err;
-+ }
-+ index_p = of_get_property(node, "cell-index", &ret);
-+ if (!index_p || (ret != 4)) {
-+ pr_err("Can't get %s property '%s'\n", node->full_name,
-+ "cell-index");
-+ goto err;
-+ }
-+ index = be32_to_cpu(*index_p);
-+ if (index >= qman_portal_max) {
-+ pr_err("QMan portal index %d is beyond max (%d)\n",
-+ index, qman_portal_max);
-+ goto err;
-+ }
-+
-+ channel = index + QM_CHANNEL_SWPORTAL0;
-+ pcfg->public_cfg.channel = channel;
-+ pcfg->public_cfg.cpu = -1;
-+ irq = irq_of_parse_and_map(node, 0);
-+ if (irq == 0) {
-+ pr_err("Can't get %s property '%s'\n", node->full_name,
-+ "interrupts");
-+ goto err;
-+ }
-+ pcfg->public_cfg.irq = irq;
-+ pcfg->public_cfg.index = index;
-+#ifdef CONFIG_FSL_QMAN_CONFIG
-+ /* We need the same LIODN offset for all portals */
-+ qman_liodn_fixup(pcfg->public_cfg.channel);
-+#endif
-+
-+ len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]);
-+ if (len != (unsigned long)len)
-+ goto err;
-+
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns(
-+ pcfg->addr_phys[DPA_PORTAL_CE].start,
-+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]));
-+
-+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap(
-+ pcfg->addr_phys[DPA_PORTAL_CI].start,
-+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]));
-+#else
-+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot(
-+ pcfg->addr_phys[DPA_PORTAL_CE].start,
-+ (unsigned long)len,
-+ 0);
-+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot(
-+ pcfg->addr_phys[DPA_PORTAL_CI].start,
-+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]),
-+ _PAGE_GUARDED | _PAGE_NO_CACHE);
-+#endif
-+ return pcfg;
-+err:
-+ kfree(pcfg);
-+ return NULL;
-+}
-+
-+static struct qm_portal_config *get_pcfg(struct list_head *list)
-+{
-+ struct qm_portal_config *pcfg;
-+ if (list_empty(list))
-+ return NULL;
-+ pcfg = list_entry(list->prev, struct qm_portal_config, list);
-+ list_del(&pcfg->list);
-+ return pcfg;
-+}
-+
-+static struct qm_portal_config *get_pcfg_idx(struct list_head *list, u32 idx)
-+{
-+ struct qm_portal_config *pcfg;
-+ if (list_empty(list))
-+ return NULL;
-+ list_for_each_entry(pcfg, list, list) {
-+ if (pcfg->public_cfg.index == idx) {
-+ list_del(&pcfg->list);
-+ return pcfg;
-+ }
-+ }
-+ return NULL;
-+}
-+
-+static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu)
-+{
-+#ifdef CONFIG_FSL_PAMU
-+ int ret;
-+ int window_count = 1;
-+ struct iommu_domain_geometry geom_attr;
-+ struct pamu_stash_attribute stash_attr;
-+
-+ pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type);
-+ if (!pcfg->iommu_domain) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed",
-+ __func__);
-+ goto _no_iommu;
-+ }
-+ geom_attr.aperture_start = 0;
-+ geom_attr.aperture_end =
-+ ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1;
-+ geom_attr.force_aperture = true;
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY,
-+ &geom_attr);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS,
-+ &window_count);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ stash_attr.cpu = cpu;
-+ stash_attr.cache = PAMU_ATTR_CACHE_L1;
-+ /* set stash information for the window */
-+ stash_attr.window = 0;
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain,
-+ DOMAIN_ATTR_FSL_PAMU_STASH,
-+ &stash_attr);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36,
-+ IOMMU_READ | IOMMU_WRITE);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d",
-+ __func__, ret);
-+ goto _iommu_domain_free;
-+ }
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain,
-+ DOMAIN_ATTR_FSL_PAMU_ENABLE,
-+ &window_count);
-+ if (ret < 0) {
-+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
-+ __func__, ret);
-+ goto _iommu_detach_device;
-+ }
-+
-+_no_iommu:
-+#endif
-+#ifdef CONFIG_FSL_QMAN_CONFIG
-+ if (qman_set_sdest(pcfg->public_cfg.channel, cpu))
-+#endif
-+ pr_warn("Failed to set QMan portal's stash request queue\n");
-+
-+ return;
-+
-+#ifdef CONFIG_FSL_PAMU
-+_iommu_detach_device:
-+ iommu_detach_device(pcfg->iommu_domain, NULL);
-+_iommu_domain_free:
-+ iommu_domain_free(pcfg->iommu_domain);
-+#endif
-+}
-+
-+struct qm_portal_config *qm_get_unused_portal_idx(u32 idx)
-+{
-+ struct qm_portal_config *ret;
-+ spin_lock(&unused_pcfgs_lock);
-+ if (idx == QBMAN_ANY_PORTAL_IDX)
-+ ret = get_pcfg(&unused_pcfgs);
-+ else
-+ ret = get_pcfg_idx(&unused_pcfgs, idx);
-+ spin_unlock(&unused_pcfgs_lock);
-+ /* Bind stashing LIODNs to the CPU we are currently executing on, and
-+ * set the portal to use the stashing request queue corresonding to the
-+ * cpu as well. The user-space driver assumption is that the pthread has
-+ * to already be affine to one cpu only before opening a portal. If that
-+ * check is circumvented, the only risk is a performance degradation -
-+ * stashing will go to whatever cpu they happened to be running on when
-+ * opening the device file, and if that isn't the cpu they subsequently
-+ * bind to and do their polling on, tough. */
-+ if (ret)
-+ portal_set_cpu(ret, hard_smp_processor_id());
-+ return ret;
-+}
-+
-+struct qm_portal_config *qm_get_unused_portal(void)
-+{
-+ return qm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX);
-+}
-+
-+void qm_put_unused_portal(struct qm_portal_config *pcfg)
-+{
-+ spin_lock(&unused_pcfgs_lock);
-+ list_add(&pcfg->list, &unused_pcfgs);
-+ spin_unlock(&unused_pcfgs_lock);
-+}
-+
-+static struct qman_portal *init_pcfg(struct qm_portal_config *pcfg)
-+{
-+ struct qman_portal *p;
-+
-+ pcfg->iommu_domain = NULL;
-+ portal_set_cpu(pcfg, pcfg->public_cfg.cpu);
-+ p = qman_create_affine_portal(pcfg, NULL);
-+ if (p) {
-+ u32 irq_sources = 0;
-+ /* Determine what should be interrupt-vs-poll driven */
-+#ifdef CONFIG_FSL_DPA_PIRQ_SLOW
-+ irq_sources |= QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI |
-+ QM_PIRQ_CSCI | QM_PIRQ_CCSCI;
-+#endif
-+#ifdef CONFIG_FSL_DPA_PIRQ_FAST
-+ irq_sources |= QM_PIRQ_DQRI;
-+#endif
-+ qman_p_irqsource_add(p, irq_sources);
-+ pr_info("Qman portal %sinitialised, cpu %d\n",
-+ pcfg->public_cfg.is_shared ? "(shared) " : "",
-+ pcfg->public_cfg.cpu);
-+ } else
-+ pr_crit("Qman portal failure on cpu %d\n",
-+ pcfg->public_cfg.cpu);
-+ return p;
-+}
-+
-+static void init_slave(int cpu)
-+{
-+ struct qman_portal *p;
-+ struct cpumask oldmask = current->cpus_allowed;
-+ set_cpus_allowed_ptr(current, get_cpu_mask(cpu));
-+ p = qman_create_affine_slave(shared_portals[shared_portals_idx++], cpu);
-+ if (!p)
-+ pr_err("Qman slave portal failure on cpu %d\n", cpu);
-+ else
-+ pr_info("Qman portal %sinitialised, cpu %d\n", "(slave) ", cpu);
-+ set_cpus_allowed_ptr(current, &oldmask);
-+ if (shared_portals_idx >= num_shared_portals)
-+ shared_portals_idx = 0;
-+}
-+
-+static struct cpumask want_unshared __initdata;
-+static struct cpumask want_shared __initdata;
-+
-+static int __init parse_qportals(char *str)
-+{
-+ return parse_portals_bootarg(str, &want_shared, &want_unshared,
-+ "qportals");
-+}
-+__setup("qportals=", parse_qportals);
-+
-+static void qman_portal_update_sdest(const struct qm_portal_config *pcfg,
-+ unsigned int cpu)
-+{
-+#ifdef CONFIG_FSL_PAMU
-+ struct pamu_stash_attribute stash_attr;
-+ int ret;
-+
-+ if (pcfg->iommu_domain) {
-+ stash_attr.cpu = cpu;
-+ stash_attr.cache = PAMU_ATTR_CACHE_L1;
-+ /* set stash information for the window */
-+ stash_attr.window = 0;
-+ ret = iommu_domain_set_attr(pcfg->iommu_domain,
-+ DOMAIN_ATTR_FSL_PAMU_STASH, &stash_attr);
-+ if (ret < 0) {
-+ pr_err("Failed to update pamu stash setting\n");
-+ return;
-+ }
-+ }
-+#endif
-+#ifdef CONFIG_FSL_QMAN_CONFIG
-+ if (qman_set_sdest(pcfg->public_cfg.channel, cpu))
-+ pr_warn("Failed to update portal's stash request queue\n");
-+#endif
-+}
-+
-+static int qman_offline_cpu(unsigned int cpu)
-+{
-+ struct qman_portal *p;
-+ const struct qm_portal_config *pcfg;
-+ p = (struct qman_portal *)affine_portals[cpu];
-+ if (p) {
-+ pcfg = qman_get_qm_portal_config(p);
-+ if (pcfg) {
-+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0));
-+ qman_portal_update_sdest(pcfg, 0);
-+ }
-+ }
-+ return 0;
-+}
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+static int qman_online_cpu(unsigned int cpu)
-+{
-+ struct qman_portal *p;
-+ const struct qm_portal_config *pcfg;
-+ p = (struct qman_portal *)affine_portals[cpu];
-+ if (p) {
-+ pcfg = qman_get_qm_portal_config(p);
-+ if (pcfg) {
-+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu));
-+ qman_portal_update_sdest(pcfg, cpu);
-+ }
-+ }
-+ return 0;
-+}
-+
-+#endif /* CONFIG_HOTPLUG_CPU */
-+
-+__init int qman_init(void)
-+{
-+ struct cpumask slave_cpus;
-+ struct cpumask unshared_cpus = *cpu_none_mask;
-+ struct cpumask shared_cpus = *cpu_none_mask;
-+ LIST_HEAD(unshared_pcfgs);
-+ LIST_HEAD(shared_pcfgs);
-+ struct device_node *dn;
-+ struct qm_portal_config *pcfg;
-+ struct qman_portal *p;
-+ int cpu, ret;
-+ const u32 *clk;
-+ struct cpumask offline_cpus;
-+
-+ /* Initialise the Qman (CCSR) device */
-+ for_each_compatible_node(dn, NULL, "fsl,qman") {
-+ if (!qman_init_ccsr(dn))
-+ pr_info("Qman err interrupt handler present\n");
-+ else
-+ pr_err("Qman CCSR setup failed\n");
-+
-+ clk = of_get_property(dn, "clock-frequency", NULL);
-+ if (!clk)
-+ pr_warn("Can't find Qman clock frequency\n");
-+ else
-+ qman_clk = be32_to_cpu(*clk);
-+ }
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ /* Setup lookup table for FQ demux */
-+ ret = qman_setup_fq_lookup_table(get_qman_fqd_size()/64);
-+ if (ret)
-+ return ret;
-+#endif
-+
-+ /* Get qman ip revision */
-+ qman_get_ip_revision(dn);
-+ if ((qman_ip_rev & 0xff00) >= QMAN_REV30) {
-+ qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3;
-+ qm_channel_caam = QMAN_CHANNEL_CAAM_REV3;
-+ qm_channel_pme = QMAN_CHANNEL_PME_REV3;
-+ }
-+
-+ if ((qman_ip_rev == QMAN_REV31) && (qman_ip_cfg == QMAN_REV_CFG_2))
-+ qm_channel_dce = QMAN_CHANNEL_DCE_QMANREV312;
-+
-+ /*
-+ * Parse the ceetm node to get how many ceetm instances are supported
-+ * on the current silicon. num_ceetms must be confirmed before portals
-+ * are intiailized.
-+ */
-+ num_ceetms = 0;
-+ for_each_compatible_node(dn, NULL, "fsl,qman-ceetm")
-+ num_ceetms++;
-+
-+ /* Parse pool channels into the SDQCR mask. (Must happen before portals
-+ * are initialised.) */
-+ for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") {
-+ ret = fsl_pool_channel_range_sdqcr(dn);
-+ if (ret)
-+ return ret;
-+ }
-+
-+ memset(affine_portals, 0, sizeof(void *) * num_possible_cpus());
-+ /* Initialise portals. See bman_driver.c for comments */
-+ for_each_compatible_node(dn, NULL, "fsl,qman-portal") {
-+ if (!of_device_is_available(dn))
-+ continue;
-+ pcfg = parse_pcfg(dn);
-+ if (pcfg) {
-+ pcfg->public_cfg.pools = pools_sdqcr;
-+ list_add_tail(&pcfg->list, &unused_pcfgs);
-+ }
-+ }
-+ for_each_possible_cpu(cpu) {
-+ if (cpumask_test_cpu(cpu, &want_shared)) {
-+ pcfg = get_pcfg(&unused_pcfgs);
-+ if (!pcfg)
-+ break;
-+ pcfg->public_cfg.cpu = cpu;
-+ list_add_tail(&pcfg->list, &shared_pcfgs);
-+ cpumask_set_cpu(cpu, &shared_cpus);
-+ }
-+ if (cpumask_test_cpu(cpu, &want_unshared)) {
-+ if (cpumask_test_cpu(cpu, &shared_cpus))
-+ continue;
-+ pcfg = get_pcfg(&unused_pcfgs);
-+ if (!pcfg)
-+ break;
-+ pcfg->public_cfg.cpu = cpu;
-+ list_add_tail(&pcfg->list, &unshared_pcfgs);
-+ cpumask_set_cpu(cpu, &unshared_cpus);
-+ }
-+ }
-+ if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) {
-+ for_each_online_cpu(cpu) {
-+ pcfg = get_pcfg(&unused_pcfgs);
-+ if (!pcfg)
-+ break;
-+ pcfg->public_cfg.cpu = cpu;
-+ list_add_tail(&pcfg->list, &unshared_pcfgs);
-+ cpumask_set_cpu(cpu, &unshared_cpus);
-+ }
-+ }
-+ cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus);
-+ cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus);
-+ if (cpumask_empty(&slave_cpus)) {
-+ if (!list_empty(&shared_pcfgs)) {
-+ cpumask_or(&unshared_cpus, &unshared_cpus,
-+ &shared_cpus);
-+ cpumask_clear(&shared_cpus);
-+ list_splice_tail(&shared_pcfgs, &unshared_pcfgs);
-+ INIT_LIST_HEAD(&shared_pcfgs);
-+ }
-+ } else {
-+ if (list_empty(&shared_pcfgs)) {
-+ pcfg = get_pcfg(&unshared_pcfgs);
-+ if (!pcfg) {
-+ pr_crit("No QMan portals available!\n");
-+ return 0;
-+ }
-+ cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus);
-+ cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus);
-+ list_add_tail(&pcfg->list, &shared_pcfgs);
-+ }
-+ }
-+ list_for_each_entry(pcfg, &unshared_pcfgs, list) {
-+ pcfg->public_cfg.is_shared = 0;
-+ p = init_pcfg(pcfg);
-+ if (!p) {
-+ pr_crit("Unable to configure portals\n");
-+ return 0;
-+ }
-+ }
-+ list_for_each_entry(pcfg, &shared_pcfgs, list) {
-+ pcfg->public_cfg.is_shared = 1;
-+ p = init_pcfg(pcfg);
-+ if (p)
-+ shared_portals[num_shared_portals++] = p;
-+ }
-+ if (!cpumask_empty(&slave_cpus))
-+ for_each_cpu(cpu, &slave_cpus)
-+ init_slave(cpu);
-+ pr_info("Qman portals initialised\n");
-+ cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask);
-+ for_each_cpu(cpu, &offline_cpus)
-+ qman_offline_cpu(cpu);
-+#ifdef CONFIG_HOTPLUG_CPU
-+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
-+ "soc/qman_portal:online",
-+ qman_online_cpu, qman_offline_cpu);
-+ if (ret < 0) {
-+ pr_err("qman: failed to register hotplug callbacks.\n");
-+ return ret;
-+ }
-+#endif
-+ return 0;
-+}
-+
-+__init int qman_resource_init(void)
-+{
-+ struct device_node *dn;
-+ int ret;
-+
-+ /* Initialise FQID allocation ranges */
-+ for_each_compatible_node(dn, NULL, "fsl,fqid-range") {
-+ ret = fsl_fqid_range_init(dn);
-+ if (ret)
-+ return ret;
-+ }
-+ /* Initialise CGRID allocation ranges */
-+ for_each_compatible_node(dn, NULL, "fsl,cgrid-range") {
-+ ret = fsl_cgrid_range_init(dn);
-+ if (ret)
-+ return ret;
-+ }
-+ /* Parse pool channels into the allocator. (Must happen after portals
-+ * are initialised.) */
-+ for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") {
-+ ret = fsl_pool_channel_range_init(dn);
-+ if (ret)
-+ return ret;
-+ }
-+
-+ /* Parse CEETM */
-+ for_each_compatible_node(dn, NULL, "fsl,qman-ceetm") {
-+ ret = fsl_ceetm_init(dn);
-+ if (ret)
-+ return ret;
-+ }
-+ return 0;
-+}
-+
-+#ifdef CONFIG_SUSPEND
-+void suspend_unused_qportal(void)
-+{
-+ struct qm_portal_config *pcfg;
-+
-+ if (list_empty(&unused_pcfgs))
-+ return;
-+
-+ list_for_each_entry(pcfg, &unused_pcfgs, list) {
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Need to save qportal %d\n", pcfg->public_cfg.index);
-+#endif
-+ /* save isdr, disable all via isdr, clear isr */
-+ pcfg->saved_isdr =
-+ __raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
-+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
-+ 0xe08);
-+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
-+ 0xe00);
-+ }
-+ return;
-+}
-+
-+void resume_unused_qportal(void)
-+{
-+ struct qm_portal_config *pcfg;
-+
-+ if (list_empty(&unused_pcfgs))
-+ return;
-+
-+ list_for_each_entry(pcfg, &unused_pcfgs, list) {
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Need to resume qportal %d\n", pcfg->public_cfg.index);
-+#endif
-+ /* restore isdr */
-+ __raw_writel(pcfg->saved_isdr,
-+ pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
-+ }
-+ return;
-+}
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_high.c
-@@ -0,0 +1,5669 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "qman_low.h"
-+
-+/* Compilation constants */
-+#define DQRR_MAXFILL 15
-+#define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */
-+#define IRQNAME "QMan portal %d"
-+#define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */
-+
-+/* Divide 'n' by 'd', rounding down if 'r' is negative, rounding up if it's
-+ * positive, and rounding to the closest value if it's zero. NB, this macro
-+ * implicitly upgrades parameters to unsigned 64-bit, so feed it with types
-+ * that are compatible with this. NB, these arguments should not be expressions
-+ * unless it is safe for them to be evaluated multiple times. Eg. do not pass
-+ * in "some_value++" as a parameter to the macro! */
-+#define ROUNDING(n, d, r) \
-+ (((r) < 0) ? div64_u64((n), (d)) : \
-+ (((r) > 0) ? div64_u64(((n) + (d) - 1), (d)) : \
-+ div64_u64(((n) + ((d) / 2)), (d))))
-+
-+/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
-+ * inter-processor locking only. Note, FQLOCK() is always called either under a
-+ * local_irq_save() or from interrupt context - hence there's no need for irq
-+ * protection (and indeed, attempting to nest irq-protection doesn't work, as
-+ * the "irq en/disable" machinery isn't recursive...). */
-+#define FQLOCK(fq) \
-+ do { \
-+ struct qman_fq *__fq478 = (fq); \
-+ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
-+ spin_lock(&__fq478->fqlock); \
-+ } while (0)
-+#define FQUNLOCK(fq) \
-+ do { \
-+ struct qman_fq *__fq478 = (fq); \
-+ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
-+ spin_unlock(&__fq478->fqlock); \
-+ } while (0)
-+
-+static inline void fq_set(struct qman_fq *fq, u32 mask)
-+{
-+ set_bits(mask, &fq->flags);
-+}
-+static inline void fq_clear(struct qman_fq *fq, u32 mask)
-+{
-+ clear_bits(mask, &fq->flags);
-+}
-+static inline int fq_isset(struct qman_fq *fq, u32 mask)
-+{
-+ return fq->flags & mask;
-+}
-+static inline int fq_isclear(struct qman_fq *fq, u32 mask)
-+{
-+ return !(fq->flags & mask);
-+}
-+
-+struct qman_portal {
-+ struct qm_portal p;
-+ unsigned long bits; /* PORTAL_BITS_*** - dynamic, strictly internal */
-+ unsigned long irq_sources;
-+ u32 use_eqcr_ci_stashing;
-+ u32 slowpoll; /* only used when interrupts are off */
-+ struct qman_fq *vdqcr_owned; /* only 1 volatile dequeue at a time */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ struct qman_fq *eqci_owned; /* only 1 enqueue WAIT_SYNC at a time */
-+#endif
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ raw_spinlock_t sharing_lock; /* only used if is_shared */
-+ int is_shared;
-+ struct qman_portal *sharing_redirect;
-+#endif
-+ u32 sdqcr;
-+ int dqrr_disable_ref;
-+ /* A portal-specific handler for DCP ERNs. If this is NULL, the global
-+ * handler is called instead. */
-+ qman_cb_dc_ern cb_dc_ern;
-+ /* When the cpu-affine portal is activated, this is non-NULL */
-+ const struct qm_portal_config *config;
-+ /* This is needed for providing a non-NULL device to dma_map_***() */
-+ struct platform_device *pdev;
-+ struct dpa_rbtree retire_table;
-+ char irqname[MAX_IRQNAME];
-+ /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
-+ struct qman_cgrs *cgrs;
-+ /* linked-list of CSCN handlers. */
-+ struct list_head cgr_cbs;
-+ /* list lock */
-+ spinlock_t cgr_lock;
-+ /* 2-element array. ccgrs[0] is mask, ccgrs[1] is snapshot. */
-+ struct qman_ccgrs *ccgrs[QMAN_CEETM_MAX];
-+ /* 256-element array, each is a linked-list of CCSCN handlers. */
-+ struct list_head ccgr_cbs[QMAN_CEETM_MAX];
-+ /* list lock */
-+ spinlock_t ccgr_lock;
-+ /* track if memory was allocated by the driver */
-+ u8 alloced;
-+ /* power management data */
-+ u32 save_isdr;
-+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-+ /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
-+ * do byte swaps of DQRR read only memory. First entry must be aligned
-+ * to 2 ** 10 to ensure DQRR index calculations based shadow copy
-+ * address (6 bits for address shift + 4 bits for the DQRR size).
-+ */
-+ struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] __aligned(1024);
-+#endif
-+};
-+
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+#define PORTAL_IRQ_LOCK(p, irqflags) \
-+ do { \
-+ if ((p)->is_shared) \
-+ raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
-+ else \
-+ local_irq_save(irqflags); \
-+ } while (0)
-+#define PORTAL_IRQ_UNLOCK(p, irqflags) \
-+ do { \
-+ if ((p)->is_shared) \
-+ raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
-+ irqflags); \
-+ else \
-+ local_irq_restore(irqflags); \
-+ } while (0)
-+#else
-+#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
-+#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
-+#endif
-+
-+/* Global handler for DCP ERNs. Used when the portal receiving the message does
-+ * not have a portal-specific handler. */
-+static qman_cb_dc_ern cb_dc_ern;
-+
-+static cpumask_t affine_mask;
-+static DEFINE_SPINLOCK(affine_mask_lock);
-+static u16 affine_channels[NR_CPUS];
-+static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal);
-+void *affine_portals[NR_CPUS];
-+
-+/* "raw" gets the cpu-local struct whether it's a redirect or not. */
-+static inline struct qman_portal *get_raw_affine_portal(void)
-+{
-+ return &get_cpu_var(qman_affine_portal);
-+}
-+/* For ops that can redirect, this obtains the portal to use */
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+static inline struct qman_portal *get_affine_portal(void)
-+{
-+ struct qman_portal *p = get_raw_affine_portal();
-+ if (p->sharing_redirect)
-+ return p->sharing_redirect;
-+ return p;
-+}
-+#else
-+#define get_affine_portal() get_raw_affine_portal()
-+#endif
-+/* For every "get", there must be a "put" */
-+static inline void put_affine_portal(void)
-+{
-+ put_cpu_var(qman_affine_portal);
-+}
-+/* Exception: poll functions assume the caller is cpu-affine and in no risk of
-+ * re-entrance, which are the two reasons we usually use the get/put_cpu_var()
-+ * semantic - ie. to disable pre-emption. Some use-cases expect the execution
-+ * context to remain as non-atomic during poll-triggered callbacks as it was
-+ * when the poll API was first called (eg. NAPI), so we go out of our way in
-+ * this case to not disable pre-emption. */
-+static inline struct qman_portal *get_poll_portal(void)
-+{
-+ return &get_cpu_var(qman_affine_portal);
-+}
-+#define put_poll_portal()
-+
-+/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
-+ * retirement notifications (the fact they are sometimes h/w-consumed means that
-+ * contextB isn't always a s/w demux - and as we can't know which case it is
-+ * when looking at the notification, we have to use the slow lookup for all of
-+ * them). NB, it's possible to have multiple FQ objects refer to the same FQID
-+ * (though at most one of them should be the consumer), so this table isn't for
-+ * all FQs - FQs are added when retirement commands are issued, and removed when
-+ * they complete, which also massively reduces the size of this table. */
-+IMPLEMENT_DPA_RBTREE(fqtree, struct qman_fq, node, fqid);
-+
-+/* This is what everything can wait on, even if it migrates to a different cpu
-+ * to the one whose affine portal it is waiting on. */
-+static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
-+
-+static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
-+{
-+ int ret = fqtree_push(&p->retire_table, fq);
-+ if (ret)
-+ pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
-+ return ret;
-+}
-+
-+static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
-+{
-+ fqtree_del(&p->retire_table, fq);
-+}
-+
-+static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
-+{
-+ return fqtree_find(&p->retire_table, fqid);
-+}
-+
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+static void **qman_fq_lookup_table;
-+static size_t qman_fq_lookup_table_size;
-+
-+int qman_setup_fq_lookup_table(size_t num_entries)
-+{
-+ num_entries++;
-+ /* Allocate 1 more entry since the first entry is not used */
-+ qman_fq_lookup_table = vzalloc((num_entries * sizeof(void *)));
-+ if (!qman_fq_lookup_table) {
-+ pr_err("QMan: Could not allocate fq lookup table\n");
-+ return -ENOMEM;
-+ }
-+ qman_fq_lookup_table_size = num_entries;
-+ pr_info("QMan: Allocated lookup table at %p, entry count %lu\n",
-+ qman_fq_lookup_table,
-+ (unsigned long)qman_fq_lookup_table_size);
-+ return 0;
-+}
-+
-+/* global structure that maintains fq object mapping */
-+static DEFINE_SPINLOCK(fq_hash_table_lock);
-+
-+static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
-+{
-+ u32 i;
-+
-+ spin_lock(&fq_hash_table_lock);
-+ /* Can't use index zero because this has special meaning
-+ * in context_b field. */
-+ for (i = 1; i < qman_fq_lookup_table_size; i++) {
-+ if (qman_fq_lookup_table[i] == NULL) {
-+ *entry = i;
-+ qman_fq_lookup_table[i] = fq;
-+ spin_unlock(&fq_hash_table_lock);
-+ return 0;
-+ }
-+ }
-+ spin_unlock(&fq_hash_table_lock);
-+ return -ENOMEM;
-+}
-+
-+static void clear_fq_table_entry(u32 entry)
-+{
-+ spin_lock(&fq_hash_table_lock);
-+ BUG_ON(entry >= qman_fq_lookup_table_size);
-+ qman_fq_lookup_table[entry] = NULL;
-+ spin_unlock(&fq_hash_table_lock);
-+}
-+
-+static inline struct qman_fq *get_fq_table_entry(u32 entry)
-+{
-+ BUG_ON(entry >= qman_fq_lookup_table_size);
-+ return qman_fq_lookup_table[entry];
-+}
-+#endif
-+
-+static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
-+{
-+ /* Byteswap the FQD to HW format */
-+ fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
-+ fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
-+ fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
-+ fqd->context_b = cpu_to_be32(fqd->context_b);
-+ fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
-+}
-+
-+static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
-+{
-+ /* Byteswap the FQD to CPU format */
-+ fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
-+ fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
-+ fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
-+ fqd->context_b = be32_to_cpu(fqd->context_b);
-+ fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
-+}
-+
-+/* Swap a 40 bit address */
-+static inline u64 cpu_to_be40(u64 in)
-+{
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ return in;
-+#else
-+ u64 out = 0;
-+ u8 *p = (u8 *) &out;
-+ p[0] = in >> 32;
-+ p[1] = in >> 24;
-+ p[2] = in >> 16;
-+ p[3] = in >> 8;
-+ p[4] = in >> 0;
-+ return out;
-+#endif
-+}
-+static inline u64 be40_to_cpu(u64 in)
-+{
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ return in;
-+#else
-+ u64 out = 0;
-+ u8 *pout = (u8 *) &out;
-+ u8 *pin = (u8 *) ∈
-+ pout[0] = pin[4];
-+ pout[1] = pin[3];
-+ pout[2] = pin[2];
-+ pout[3] = pin[1];
-+ pout[4] = pin[0];
-+ return out;
-+#endif
-+}
-+
-+/* Swap a 24 bit value */
-+static inline u32 cpu_to_be24(u32 in)
-+{
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ return in;
-+#else
-+ u32 out = 0;
-+ u8 *p = (u8 *) &out;
-+ p[0] = in >> 16;
-+ p[1] = in >> 8;
-+ p[2] = in >> 0;
-+ return out;
-+#endif
-+}
-+
-+static inline u32 be24_to_cpu(u32 in)
-+{
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ return in;
-+#else
-+ u32 out = 0;
-+ u8 *pout = (u8 *) &out;
-+ u8 *pin = (u8 *) ∈
-+ pout[0] = pin[2];
-+ pout[1] = pin[1];
-+ pout[2] = pin[0];
-+ return out;
-+#endif
-+}
-+
-+static inline u64 be48_to_cpu(u64 in)
-+{
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ return in;
-+#else
-+ u64 out = 0;
-+ u8 *pout = (u8 *) &out;
-+ u8 *pin = (u8 *) ∈
-+
-+ pout[0] = pin[5];
-+ pout[1] = pin[4];
-+ pout[2] = pin[3];
-+ pout[3] = pin[2];
-+ pout[4] = pin[1];
-+ pout[5] = pin[0];
-+ return out;
-+#endif
-+}
-+static inline void cpu_to_hw_fd(struct qm_fd *fd)
-+{
-+ fd->opaque_addr = cpu_to_be64(fd->opaque_addr);
-+ fd->status = cpu_to_be32(fd->status);
-+ fd->opaque = cpu_to_be32(fd->opaque);
-+}
-+
-+static inline void hw_fd_to_cpu(struct qm_fd *fd)
-+{
-+ fd->opaque_addr = be64_to_cpu(fd->opaque_addr);
-+ fd->status = be32_to_cpu(fd->status);
-+ fd->opaque = be32_to_cpu(fd->opaque);
-+}
-+
-+static inline void hw_cq_query_to_cpu(struct qm_mcr_ceetm_cq_query *cq_query)
-+{
-+ cq_query->ccgid = be16_to_cpu(cq_query->ccgid);
-+ cq_query->state = be16_to_cpu(cq_query->state);
-+ cq_query->pfdr_hptr = be24_to_cpu(cq_query->pfdr_hptr);
-+ cq_query->pfdr_tptr = be24_to_cpu(cq_query->pfdr_tptr);
-+ cq_query->od1_xsfdr = be16_to_cpu(cq_query->od1_xsfdr);
-+ cq_query->od2_xsfdr = be16_to_cpu(cq_query->od2_xsfdr);
-+ cq_query->od3_xsfdr = be16_to_cpu(cq_query->od3_xsfdr);
-+ cq_query->od4_xsfdr = be16_to_cpu(cq_query->od4_xsfdr);
-+ cq_query->od5_xsfdr = be16_to_cpu(cq_query->od5_xsfdr);
-+ cq_query->od6_xsfdr = be16_to_cpu(cq_query->od6_xsfdr);
-+ cq_query->ra1_xsfdr = be16_to_cpu(cq_query->ra1_xsfdr);
-+ cq_query->ra2_xsfdr = be16_to_cpu(cq_query->ra2_xsfdr);
-+ cq_query->frm_cnt = be24_to_cpu(cq_query->frm_cnt);
-+}
-+
-+static inline void hw_ccgr_query_to_cpu(struct qm_mcr_ceetm_ccgr_query *ccgr_q)
-+{
-+ int i;
-+
-+ ccgr_q->cm_query.cs_thres.hword =
-+ be16_to_cpu(ccgr_q->cm_query.cs_thres.hword);
-+ ccgr_q->cm_query.cs_thres_x.hword =
-+ be16_to_cpu(ccgr_q->cm_query.cs_thres_x.hword);
-+ ccgr_q->cm_query.td_thres.hword =
-+ be16_to_cpu(ccgr_q->cm_query.td_thres.hword);
-+ ccgr_q->cm_query.wr_parm_g.word =
-+ be32_to_cpu(ccgr_q->cm_query.wr_parm_g.word);
-+ ccgr_q->cm_query.wr_parm_y.word =
-+ be32_to_cpu(ccgr_q->cm_query.wr_parm_y.word);
-+ ccgr_q->cm_query.wr_parm_r.word =
-+ be32_to_cpu(ccgr_q->cm_query.wr_parm_r.word);
-+ ccgr_q->cm_query.cscn_targ_dcp =
-+ be16_to_cpu(ccgr_q->cm_query.cscn_targ_dcp);
-+ ccgr_q->cm_query.i_cnt = be40_to_cpu(ccgr_q->cm_query.i_cnt);
-+ ccgr_q->cm_query.a_cnt = be40_to_cpu(ccgr_q->cm_query.a_cnt);
-+ for (i = 0; i < ARRAY_SIZE(ccgr_q->cm_query.cscn_targ_swp); i++)
-+ ccgr_q->cm_query.cscn_targ_swp[i] =
-+ be32_to_cpu(ccgr_q->cm_query.cscn_targ_swp[i]);
-+}
-+
-+/* In the case that slow- and fast-path handling are both done by qman_poll()
-+ * (ie. because there is no interrupt handling), we ought to balance how often
-+ * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
-+ * sources, so we call the fast poll 'n' times before calling the slow poll
-+ * once. The idle decrementer constant is used when the last slow-poll detected
-+ * no work to do, and the busy decrementer constant when the last slow-poll had
-+ * work to do. */
-+#define SLOW_POLL_IDLE 1000
-+#define SLOW_POLL_BUSY 10
-+static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
-+static inline unsigned int __poll_portal_fast(struct qman_portal *p,
-+ unsigned int poll_limit);
-+
-+/* Portal interrupt handler */
-+static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
-+{
-+ struct qman_portal *p = ptr;
-+ /*
-+ * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
-+ * it could race against a Query Congestion State command also given
-+ * as part of the handling of this interrupt source. We mustn't
-+ * clear it a second time in this top-level function.
-+ */
-+ u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
-+ ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
-+ u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
-+ /* DQRR-handling if it's interrupt-driven */
-+ if (is & QM_PIRQ_DQRI)
-+ __poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT);
-+ /* Handling of anything else that's interrupt-driven */
-+ clear |= __poll_portal_slow(p, is);
-+ qm_isr_status_clear(&p->p, clear);
-+ return IRQ_HANDLED;
-+}
-+
-+/* This inner version is used privately by qman_create_affine_portal(), as well
-+ * as by the exported qman_stop_dequeues(). */
-+static inline void qman_stop_dequeues_ex(struct qman_portal *p)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ if (!(p->dqrr_disable_ref++))
-+ qm_dqrr_set_maxfill(&p->p, 0);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+}
-+
-+static int drain_mr_fqrni(struct qm_portal *p)
-+{
-+ const struct qm_mr_entry *msg;
-+loop:
-+ msg = qm_mr_current(p);
-+ if (!msg) {
-+ /* if MR was full and h/w had other FQRNI entries to produce, we
-+ * need to allow it time to produce those entries once the
-+ * existing entries are consumed. A worst-case situation
-+ * (fully-loaded system) means h/w sequencers may have to do 3-4
-+ * other things before servicing the portal's MR pump, each of
-+ * which (if slow) may take ~50 qman cycles (which is ~200
-+ * processor cycles). So rounding up and then multiplying this
-+ * worst-case estimate by a factor of 10, just to be
-+ * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
-+ * one entry at a time, so h/w has an opportunity to produce new
-+ * entries well before the ring has been fully consumed, so
-+ * we're being *really* paranoid here. */
-+ u64 now, then = mfatb();
-+ do {
-+ now = mfatb();
-+ } while ((then + 10000) > now);
-+ msg = qm_mr_current(p);
-+ if (!msg)
-+ return 0;
-+ }
-+ if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
-+ /* We aren't draining anything but FQRNIs */
-+ pr_err("QMan found verb 0x%x in MR\n", msg->verb);
-+ return -1;
-+ }
-+ qm_mr_next(p);
-+ qm_mr_cci_consume(p, 1);
-+ goto loop;
-+}
-+
-+#ifdef CONFIG_SUSPEND
-+static int _qman_portal_suspend_noirq(struct device *dev)
-+{
-+ struct qman_portal *p = (struct qman_portal *)dev->platform_data;
-+#ifdef CONFIG_PM_DEBUG
-+ struct platform_device *pdev = to_platform_device(dev);
-+#endif
-+
-+ p->save_isdr = qm_isr_disable_read(&p->p);
-+ qm_isr_disable_write(&p->p, 0xffffffff);
-+ qm_isr_status_clear(&p->p, 0xffffffff);
-+#ifdef CONFIG_PM_DEBUG
-+ pr_info("Suspend for %s\n", pdev->name);
-+#endif
-+ return 0;
-+}
-+
-+static int _qman_portal_resume_noirq(struct device *dev)
-+{
-+ struct qman_portal *p = (struct qman_portal *)dev->platform_data;
-+
-+ /* restore isdr */
-+ qm_isr_disable_write(&p->p, p->save_isdr);
-+ return 0;
-+}
-+#else
-+#define _qman_portal_suspend_noirq NULL
-+#define _qman_portal_resume_noirq NULL
-+#endif
-+
-+struct dev_pm_domain qman_portal_device_pm_domain = {
-+ .ops = {
-+ USE_PLATFORM_PM_SLEEP_OPS
-+ .suspend_noirq = _qman_portal_suspend_noirq,
-+ .resume_noirq = _qman_portal_resume_noirq,
-+ }
-+};
-+
-+struct qman_portal *qman_create_portal(
-+ struct qman_portal *portal,
-+ const struct qm_portal_config *config,
-+ const struct qman_cgrs *cgrs)
-+{
-+ struct qm_portal *__p;
-+ char buf[16];
-+ int ret;
-+ u32 isdr;
-+
-+ if (!portal) {
-+ portal = kmalloc(sizeof(*portal), GFP_KERNEL);
-+ if (!portal)
-+ return portal;
-+ portal->alloced = 1;
-+ } else
-+ portal->alloced = 0;
-+
-+ __p = &portal->p;
-+
-+#if (defined CONFIG_PPC || defined CONFIG_PPC64) && defined CONFIG_FSL_PAMU
-+ /* PAMU is required for stashing */
-+ portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ?
-+ 1 : 0);
-+#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ portal->use_eqcr_ci_stashing = 1;
-+#else
-+ portal->use_eqcr_ci_stashing = 0;
-+#endif
-+
-+ /* prep the low-level portal struct with the mapped addresses from the
-+ * config, everything that follows depends on it and "config" is more
-+ * for (de)reference... */
-+ __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
-+ __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
-+ /*
-+ * If CI-stashing is used, the current defaults use a threshold of 3,
-+ * and stash with high-than-DQRR priority.
-+ */
-+ if (qm_eqcr_init(__p, qm_eqcr_pvb,
-+ portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
-+ pr_err("Qman EQCR initialisation failed\n");
-+ goto fail_eqcr;
-+ }
-+ if (qm_dqrr_init(__p, config, qm_dqrr_dpush, qm_dqrr_pvb,
-+ qm_dqrr_cdc, DQRR_MAXFILL)) {
-+ pr_err("Qman DQRR initialisation failed\n");
-+ goto fail_dqrr;
-+ }
-+ if (qm_mr_init(__p, qm_mr_pvb, qm_mr_cci)) {
-+ pr_err("Qman MR initialisation failed\n");
-+ goto fail_mr;
-+ }
-+ if (qm_mc_init(__p)) {
-+ pr_err("Qman MC initialisation failed\n");
-+ goto fail_mc;
-+ }
-+ if (qm_isr_init(__p)) {
-+ pr_err("Qman ISR initialisation failed\n");
-+ goto fail_isr;
-+ }
-+ /* static interrupt-gating controls */
-+ qm_dqrr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_DQRR_ITHRESH);
-+ qm_mr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_MR_ITHRESH);
-+ qm_isr_set_iperiod(__p, CONFIG_FSL_QMAN_PIRQ_IPERIOD);
-+ portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
-+ if (!portal->cgrs)
-+ goto fail_cgrs;
-+ /* initial snapshot is no-depletion */
-+ qman_cgrs_init(&portal->cgrs[1]);
-+ if (cgrs)
-+ portal->cgrs[0] = *cgrs;
-+ else
-+ /* if the given mask is NULL, assume all CGRs can be seen */
-+ qman_cgrs_fill(&portal->cgrs[0]);
-+ INIT_LIST_HEAD(&portal->cgr_cbs);
-+ spin_lock_init(&portal->cgr_lock);
-+ if (num_ceetms) {
-+ for (ret = 0; ret < num_ceetms; ret++) {
-+ portal->ccgrs[ret] = kmalloc(2 *
-+ sizeof(struct qman_ccgrs), GFP_KERNEL);
-+ if (!portal->ccgrs[ret])
-+ goto fail_ccgrs;
-+ qman_ccgrs_init(&portal->ccgrs[ret][1]);
-+ qman_ccgrs_fill(&portal->ccgrs[ret][0]);
-+ INIT_LIST_HEAD(&portal->ccgr_cbs[ret]);
-+ }
-+ }
-+ spin_lock_init(&portal->ccgr_lock);
-+ portal->bits = 0;
-+ portal->slowpoll = 0;
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ portal->eqci_owned = NULL;
-+#endif
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ raw_spin_lock_init(&portal->sharing_lock);
-+ portal->is_shared = config->public_cfg.is_shared;
-+ portal->sharing_redirect = NULL;
-+#endif
-+ portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
-+ QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
-+ QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
-+ portal->dqrr_disable_ref = 0;
-+ portal->cb_dc_ern = NULL;
-+ sprintf(buf, "qportal-%d", config->public_cfg.channel);
-+ portal->pdev = platform_device_alloc(buf, -1);
-+ if (!portal->pdev) {
-+ pr_err("qman_portal - platform_device_alloc() failed\n");
-+ goto fail_devalloc;
-+ }
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ portal->pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40);
-+ portal->pdev->dev.dma_mask = &portal->pdev->dev.coherent_dma_mask;
-+#else
-+ if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40))) {
-+ pr_err("qman_portal - dma_set_mask() failed\n");
-+ goto fail_devadd;
-+ }
-+#endif
-+ portal->pdev->dev.pm_domain = &qman_portal_device_pm_domain;
-+ portal->pdev->dev.platform_data = portal;
-+ ret = platform_device_add(portal->pdev);
-+ if (ret) {
-+ pr_err("qman_portal - platform_device_add() failed\n");
-+ goto fail_devadd;
-+ }
-+ dpa_rbtree_init(&portal->retire_table);
-+ isdr = 0xffffffff;
-+ qm_isr_disable_write(__p, isdr);
-+ portal->irq_sources = 0;
-+ qm_isr_enable_write(__p, portal->irq_sources);
-+ qm_isr_status_clear(__p, 0xffffffff);
-+ snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
-+ if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
-+ portal)) {
-+ pr_err("request_irq() failed\n");
-+ goto fail_irq;
-+ }
-+ if ((config->public_cfg.cpu != -1) &&
-+ irq_can_set_affinity(config->public_cfg.irq) &&
-+ irq_set_affinity(config->public_cfg.irq,
-+ cpumask_of(config->public_cfg.cpu))) {
-+ pr_err("irq_set_affinity() failed\n");
-+ goto fail_affinity;
-+ }
-+
-+ /* Need EQCR to be empty before continuing */
-+ isdr ^= QM_PIRQ_EQCI;
-+ qm_isr_disable_write(__p, isdr);
-+ ret = qm_eqcr_get_fill(__p);
-+ if (ret) {
-+ pr_err("Qman EQCR unclean\n");
-+ goto fail_eqcr_empty;
-+ }
-+ isdr ^= (QM_PIRQ_DQRI | QM_PIRQ_MRI);
-+ qm_isr_disable_write(__p, isdr);
-+ if (qm_dqrr_current(__p) != NULL) {
-+ pr_err("Qman DQRR unclean\n");
-+ qm_dqrr_cdc_consume_n(__p, 0xffff);
-+ }
-+ if (qm_mr_current(__p) != NULL) {
-+ /* special handling, drain just in case it's a few FQRNIs */
-+ if (drain_mr_fqrni(__p)) {
-+ const struct qm_mr_entry *e = qm_mr_current(__p);
-+ /*
-+ * Message ring cannot be empty no need to check
-+ * qm_mr_current returned successfully
-+ */
-+ pr_err("Qman MR unclean, MR VERB 0x%x, rc 0x%x\n, addr 0x%x",
-+ e->verb, e->ern.rc, e->ern.fd.addr_lo);
-+ goto fail_dqrr_mr_empty;
-+ }
-+ }
-+ /* Success */
-+ portal->config = config;
-+ qm_isr_disable_write(__p, 0);
-+ qm_isr_uninhibit(__p);
-+ /* Write a sane SDQCR */
-+ qm_dqrr_sdqcr_set(__p, portal->sdqcr);
-+ return portal;
-+fail_dqrr_mr_empty:
-+fail_eqcr_empty:
-+fail_affinity:
-+ free_irq(config->public_cfg.irq, portal);
-+fail_irq:
-+ platform_device_del(portal->pdev);
-+fail_devadd:
-+ platform_device_put(portal->pdev);
-+fail_devalloc:
-+ if (num_ceetms)
-+ for (ret = 0; ret < num_ceetms; ret++)
-+ kfree(portal->ccgrs[ret]);
-+fail_ccgrs:
-+ kfree(portal->cgrs);
-+fail_cgrs:
-+ qm_isr_finish(__p);
-+fail_isr:
-+ qm_mc_finish(__p);
-+fail_mc:
-+ qm_mr_finish(__p);
-+fail_mr:
-+ qm_dqrr_finish(__p);
-+fail_dqrr:
-+ qm_eqcr_finish(__p);
-+fail_eqcr:
-+ if (portal->alloced)
-+ kfree(portal);
-+ return NULL;
-+}
-+
-+struct qman_portal *qman_create_affine_portal(
-+ const struct qm_portal_config *config,
-+ const struct qman_cgrs *cgrs)
-+{
-+ struct qman_portal *res;
-+ struct qman_portal *portal;
-+
-+ portal = &per_cpu(qman_affine_portal, config->public_cfg.cpu);
-+ res = qman_create_portal(portal, config, cgrs);
-+ if (res) {
-+ spin_lock(&affine_mask_lock);
-+ cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
-+ affine_channels[config->public_cfg.cpu] =
-+ config->public_cfg.channel;
-+ affine_portals[config->public_cfg.cpu] = portal;
-+ spin_unlock(&affine_mask_lock);
-+ }
-+ return res;
-+}
-+
-+/* These checks are BUG_ON()s because the driver is already supposed to avoid
-+ * these cases. */
-+struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect,
-+ int cpu)
-+{
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ struct qman_portal *p;
-+ p = &per_cpu(qman_affine_portal, cpu);
-+ /* Check that we don't already have our own portal */
-+ BUG_ON(p->config);
-+ /* Check that we aren't already slaving to another portal */
-+ BUG_ON(p->is_shared);
-+ /* Check that 'redirect' is prepared to have us */
-+ BUG_ON(!redirect->config->public_cfg.is_shared);
-+ /* These are the only elements to initialise when redirecting */
-+ p->irq_sources = 0;
-+ p->sharing_redirect = redirect;
-+ affine_portals[cpu] = p;
-+ return p;
-+#else
-+ BUG();
-+ return NULL;
-+#endif
-+}
-+
-+void qman_destroy_portal(struct qman_portal *qm)
-+{
-+ const struct qm_portal_config *pcfg;
-+ int i;
-+
-+ /* Stop dequeues on the portal */
-+ qm_dqrr_sdqcr_set(&qm->p, 0);
-+
-+ /* NB we do this to "quiesce" EQCR. If we add enqueue-completions or
-+ * something related to QM_PIRQ_EQCI, this may need fixing.
-+ * Also, due to the prefetching model used for CI updates in the enqueue
-+ * path, this update will only invalidate the CI cacheline *after*
-+ * working on it, so we need to call this twice to ensure a full update
-+ * irrespective of where the enqueue processing was at when the teardown
-+ * began. */
-+ qm_eqcr_cce_update(&qm->p);
-+ qm_eqcr_cce_update(&qm->p);
-+ pcfg = qm->config;
-+
-+ free_irq(pcfg->public_cfg.irq, qm);
-+
-+ kfree(qm->cgrs);
-+ if (num_ceetms)
-+ for (i = 0; i < num_ceetms; i++)
-+ kfree(qm->ccgrs[i]);
-+ qm_isr_finish(&qm->p);
-+ qm_mc_finish(&qm->p);
-+ qm_mr_finish(&qm->p);
-+ qm_dqrr_finish(&qm->p);
-+ qm_eqcr_finish(&qm->p);
-+
-+ platform_device_del(qm->pdev);
-+ platform_device_put(qm->pdev);
-+
-+ qm->config = NULL;
-+ if (qm->alloced)
-+ kfree(qm);
-+}
-+
-+const struct qm_portal_config *qman_destroy_affine_portal(void)
-+{
-+ /* We don't want to redirect if we're a slave, use "raw" */
-+ struct qman_portal *qm = get_raw_affine_portal();
-+ const struct qm_portal_config *pcfg;
-+ int cpu;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (qm->sharing_redirect) {
-+ qm->sharing_redirect = NULL;
-+ put_affine_portal();
-+ return NULL;
-+ }
-+ qm->is_shared = 0;
-+#endif
-+ pcfg = qm->config;
-+ cpu = pcfg->public_cfg.cpu;
-+
-+ qman_destroy_portal(qm);
-+
-+ spin_lock(&affine_mask_lock);
-+ cpumask_clear_cpu(cpu, &affine_mask);
-+ spin_unlock(&affine_mask_lock);
-+ put_affine_portal();
-+ return pcfg;
-+}
-+
-+const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal *p)
-+{
-+ return &p->config->public_cfg;
-+}
-+EXPORT_SYMBOL(qman_p_get_portal_config);
-+
-+const struct qman_portal_config *qman_get_portal_config(void)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ const struct qman_portal_config *ret = qman_p_get_portal_config(p);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_get_portal_config);
-+
-+/* Inline helper to reduce nesting in __poll_portal_slow() */
-+static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_mr_entry *msg, u8 verb)
-+{
-+ FQLOCK(fq);
-+ switch (verb) {
-+ case QM_MR_VERB_FQRL:
-+ DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
-+ fq_clear(fq, QMAN_FQ_STATE_ORL);
-+ table_del_fq(p, fq);
-+ break;
-+ case QM_MR_VERB_FQRN:
-+ DPA_ASSERT((fq->state == qman_fq_state_parked) ||
-+ (fq->state == qman_fq_state_sched));
-+ DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
-+ fq_clear(fq, QMAN_FQ_STATE_CHANGING);
-+ if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
-+ fq_set(fq, QMAN_FQ_STATE_NE);
-+ if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
-+ fq_set(fq, QMAN_FQ_STATE_ORL);
-+ else
-+ table_del_fq(p, fq);
-+ fq->state = qman_fq_state_retired;
-+ break;
-+ case QM_MR_VERB_FQPN:
-+ DPA_ASSERT(fq->state == qman_fq_state_sched);
-+ DPA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
-+ fq->state = qman_fq_state_parked;
-+ }
-+ FQUNLOCK(fq);
-+}
-+
-+static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
-+{
-+ const struct qm_mr_entry *msg;
-+ struct qm_mr_entry swapped_msg;
-+ int k;
-+
-+ if (is & QM_PIRQ_CSCI) {
-+ struct qman_cgrs rr, c;
-+ struct qm_mc_result *mcr;
-+ struct qman_cgr *cgr;
-+ unsigned long irqflags __maybe_unused;
-+
-+ spin_lock_irqsave(&p->cgr_lock, irqflags);
-+ /*
-+ * The CSCI bit must be cleared _before_ issuing the
-+ * Query Congestion State command, to ensure that a long
-+ * CGR State Change callback cannot miss an intervening
-+ * state change.
-+ */
-+ qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
-+ qm_mc_start(&p->p);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ for (k = 0; k < 8; k++)
-+ mcr->querycongestion.state.__state[k] = be32_to_cpu(
-+ mcr->querycongestion.state.__state[k]);
-+ /* mask out the ones I'm not interested in */
-+ qman_cgrs_and(&rr, (const struct qman_cgrs *)
-+ &mcr->querycongestion.state, &p->cgrs[0]);
-+ /* check previous snapshot for delta, enter/exit congestion */
-+ qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
-+ /* update snapshot */
-+ qman_cgrs_cp(&p->cgrs[1], &rr);
-+ /* Invoke callback */
-+ list_for_each_entry(cgr, &p->cgr_cbs, node)
-+ if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
-+ cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
-+ spin_unlock_irqrestore(&p->cgr_lock, irqflags);
-+ }
-+ if (is & QM_PIRQ_CCSCI) {
-+ struct qman_ccgrs rr, c, congestion_result;
-+ struct qm_mc_result *mcr;
-+ struct qm_mc_command *mcc;
-+ struct qm_ceetm_ccg *ccg;
-+ unsigned long irqflags __maybe_unused;
-+ int i, j;
-+
-+ spin_lock_irqsave(&p->ccgr_lock, irqflags);
-+ /*
-+ * The CCSCI bit must be cleared _before_ issuing the
-+ * Query Congestion State command, to ensure that a long
-+ * CCGR State Change callback cannot miss an intervening
-+ * state change.
-+ */
-+ qm_isr_status_clear(&p->p, QM_PIRQ_CCSCI);
-+
-+ for (i = 0; i < num_ceetms; i++) {
-+ for (j = 0; j < 2; j++) {
-+ mcc = qm_mc_start(&p->p);
-+ mcc->ccgr_query.ccgrid = cpu_to_be16(
-+ CEETM_QUERY_CONGESTION_STATE | j);
-+ mcc->ccgr_query.dcpid = i;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ for (k = 0; k < 8; k++)
-+ mcr->ccgr_query.congestion_state.state.
-+ __state[k] = be32_to_cpu(
-+ mcr->ccgr_query.
-+ congestion_state.state.
-+ __state[k]);
-+ congestion_result.q[j] =
-+ mcr->ccgr_query.congestion_state.state;
-+ }
-+ /* mask out the ones I'm not interested in */
-+ qman_ccgrs_and(&rr, &congestion_result,
-+ &p->ccgrs[i][0]);
-+ /*
-+ * check previous snapshot for delta, enter/exit
-+ * congestion.
-+ */
-+ qman_ccgrs_xor(&c, &rr, &p->ccgrs[i][1]);
-+ /* update snapshot */
-+ qman_ccgrs_cp(&p->ccgrs[i][1], &rr);
-+ /* Invoke callback */
-+ list_for_each_entry(ccg, &p->ccgr_cbs[i], cb_node)
-+ if (ccg->cb && qman_ccgrs_get(&c,
-+ (ccg->parent->idx << 4) | ccg->idx))
-+ ccg->cb(ccg, ccg->cb_ctx,
-+ qman_ccgrs_get(&rr,
-+ (ccg->parent->idx << 4)
-+ | ccg->idx));
-+ }
-+ spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
-+ }
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (is & QM_PIRQ_EQCI) {
-+ unsigned long irqflags;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ p->eqci_owned = NULL;
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ wake_up(&affine_queue);
-+ }
-+#endif
-+
-+ if (is & QM_PIRQ_EQRI) {
-+ unsigned long irqflags __maybe_unused;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ qm_eqcr_cce_update(&p->p);
-+ qm_eqcr_set_ithresh(&p->p, 0);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ wake_up(&affine_queue);
-+ }
-+
-+ if (is & QM_PIRQ_MRI) {
-+ struct qman_fq *fq;
-+ u8 verb, num = 0;
-+mr_loop:
-+ qm_mr_pvb_update(&p->p);
-+ msg = qm_mr_current(&p->p);
-+ if (!msg)
-+ goto mr_done;
-+ swapped_msg = *msg;
-+ hw_fd_to_cpu(&swapped_msg.ern.fd);
-+ verb = msg->verb & QM_MR_VERB_TYPE_MASK;
-+ /* The message is a software ERN iff the 0x20 bit is set */
-+ if (verb & 0x20) {
-+ switch (verb) {
-+ case QM_MR_VERB_FQRNI:
-+ /* nada, we drop FQRNIs on the floor */
-+ break;
-+ case QM_MR_VERB_FQRN:
-+ case QM_MR_VERB_FQRL:
-+ /* Lookup in the retirement table */
-+ fq = table_find_fq(p, be32_to_cpu(msg->fq.fqid));
-+ BUG_ON(!fq);
-+ fq_state_change(p, fq, &swapped_msg, verb);
-+ if (fq->cb.fqs)
-+ fq->cb.fqs(p, fq, &swapped_msg);
-+ break;
-+ case QM_MR_VERB_FQPN:
-+ /* Parked */
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ fq = get_fq_table_entry(
-+ be32_to_cpu(msg->fq.contextB));
-+#else
-+ fq = (void *)(uintptr_t)
-+ be32_to_cpu(msg->fq.contextB);
-+#endif
-+ fq_state_change(p, fq, msg, verb);
-+ if (fq->cb.fqs)
-+ fq->cb.fqs(p, fq, &swapped_msg);
-+ break;
-+ case QM_MR_VERB_DC_ERN:
-+ /* DCP ERN */
-+ if (p->cb_dc_ern)
-+ p->cb_dc_ern(p, msg);
-+ else if (cb_dc_ern)
-+ cb_dc_ern(p, msg);
-+ else {
-+ static int warn_once;
-+ if (!warn_once) {
-+ pr_crit("Leaking DCP ERNs!\n");
-+ warn_once = 1;
-+ }
-+ }
-+ break;
-+ default:
-+ pr_crit("Invalid MR verb 0x%02x\n", verb);
-+ }
-+ } else {
-+ /* Its a software ERN */
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
-+#else
-+ fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
-+#endif
-+ fq->cb.ern(p, fq, &swapped_msg);
-+ }
-+ num++;
-+ qm_mr_next(&p->p);
-+ goto mr_loop;
-+mr_done:
-+ qm_mr_cci_consume(&p->p, num);
-+ }
-+ /*
-+ * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
-+ * processing. If that interrupt source has meanwhile been re-asserted,
-+ * we mustn't clear it here (or in the top-level interrupt handler).
-+ */
-+ return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
-+}
-+
-+/* remove some slowish-path stuff from the "fast path" and make sure it isn't
-+ * inlined. */
-+static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
-+{
-+ p->vdqcr_owned = NULL;
-+ FQLOCK(fq);
-+ fq_clear(fq, QMAN_FQ_STATE_VDQCR);
-+ FQUNLOCK(fq);
-+ wake_up(&affine_queue);
-+}
-+
-+/* Copy a DQRR entry ensuring reads reach QBMan in order */
-+static inline void safe_copy_dqrr(struct qm_dqrr_entry *dst,
-+ const struct qm_dqrr_entry *src)
-+{
-+ int i = 0;
-+ const u64 *s64 = (u64*)src;
-+ u64 *d64 = (u64*)dst;
-+
-+ /* DQRR only has 32 bytes of valid data so only need to
-+ * copy 4 - 64 bit values */
-+ *d64 = *s64;
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ {
-+ u32 res, zero = 0;
-+ /* Create a dependancy after copying first bytes ensures no wrap
-+ transaction generated to QBMan */
-+ /* Logical AND the value pointed to by s64 with 0x0 and
-+ store the result in res */
-+ asm volatile("and %[result], %[in1], %[in2]"
-+ : [result] "=r" (res)
-+ : [in1] "r" (zero), [in2] "r" (*s64)
-+ : "memory");
-+ /* Add res to s64 - this creates a dependancy on the result of
-+ reading the value of s64 before the next read. The side
-+ effect of this is that the core must stall until the first
-+ aligned read is complete therefore preventing a WRAP
-+ transaction to be seen by the QBMan */
-+ asm volatile("add %[result], %[in1], %[in2]"
-+ : [result] "=r" (s64)
-+ : [in1] "r" (res), [in2] "r" (s64)
-+ : "memory");
-+ }
-+#endif
-+ /* Copy the last 3 64 bit parts */
-+ d64++; s64++;
-+ for (;i<3; i++)
-+ *d64++ = *s64++;
-+}
-+
-+/* Look: no locks, no irq_save()s, no preempt_disable()s! :-) The only states
-+ * that would conflict with other things if they ran at the same time on the
-+ * same cpu are;
-+ *
-+ * (i) setting/clearing vdqcr_owned, and
-+ * (ii) clearing the NE (Not Empty) flag.
-+ *
-+ * Both are safe. Because;
-+ *
-+ * (i) this clearing can only occur after qman_volatile_dequeue() has set the
-+ * vdqcr_owned field (which it does before setting VDQCR), and
-+ * qman_volatile_dequeue() blocks interrupts and preemption while this is
-+ * done so that we can't interfere.
-+ * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
-+ * with (i) that API prevents us from interfering until it's safe.
-+ *
-+ * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
-+ * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
-+ * advantage comes from this function not having to "lock" anything at all.
-+ *
-+ * Note also that the callbacks are invoked at points which are safe against the
-+ * above potential conflicts, but that this function itself is not re-entrant
-+ * (this is because the function tracks one end of each FIFO in the portal and
-+ * we do *not* want to lock that). So the consequence is that it is safe for
-+ * user callbacks to call into any Qman API *except* qman_poll() (as that's the
-+ * sole API that could be invoking the callback through this function).
-+ */
-+static inline unsigned int __poll_portal_fast(struct qman_portal *p,
-+ unsigned int poll_limit)
-+{
-+ const struct qm_dqrr_entry *dq;
-+ struct qman_fq *fq;
-+ enum qman_cb_dqrr_result res;
-+ unsigned int limit = 0;
-+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-+ struct qm_dqrr_entry *shadow;
-+ const struct qm_dqrr_entry *orig_dq;
-+#endif
-+loop:
-+ qm_dqrr_pvb_update(&p->p);
-+ dq = qm_dqrr_current(&p->p);
-+ if (!dq)
-+ goto done;
-+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-+ /* If running on an LE system the fields of the
-+ dequeue entry must be swapped. Because the
-+ QMan HW will ignore writes the DQRR entry is
-+ copied and the index stored within the copy */
-+ shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
-+ /* Use safe copy here to avoid WRAP transaction */
-+ safe_copy_dqrr(shadow, dq);
-+ orig_dq = dq;
-+ dq = shadow;
-+ shadow->fqid = be32_to_cpu(shadow->fqid);
-+ shadow->contextB = be32_to_cpu(shadow->contextB);
-+ shadow->seqnum = be16_to_cpu(shadow->seqnum);
-+ hw_fd_to_cpu(&shadow->fd);
-+#endif
-+ if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
-+ /* VDQCR: don't trust contextB as the FQ may have been
-+ * configured for h/w consumption and we're draining it
-+ * post-retirement. */
-+ fq = p->vdqcr_owned;
-+ /* We only set QMAN_FQ_STATE_NE when retiring, so we only need
-+ * to check for clearing it when doing volatile dequeues. It's
-+ * one less thing to check in the critical path (SDQCR). */
-+ if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
-+ fq_clear(fq, QMAN_FQ_STATE_NE);
-+ /* this is duplicated from the SDQCR code, but we have stuff to
-+ * do before *and* after this callback, and we don't want
-+ * multiple if()s in the critical path (SDQCR). */
-+ res = fq->cb.dqrr(p, fq, dq);
-+ if (res == qman_cb_dqrr_stop)
-+ goto done;
-+ /* Check for VDQCR completion */
-+ if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
-+ clear_vdqcr(p, fq);
-+ } else {
-+ /* SDQCR: contextB points to the FQ */
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ fq = get_fq_table_entry(dq->contextB);
-+#else
-+ fq = (void *)(uintptr_t)dq->contextB;
-+#endif
-+ /* Now let the callback do its stuff */
-+ res = fq->cb.dqrr(p, fq, dq);
-+
-+ /* The callback can request that we exit without consuming this
-+ * entry nor advancing; */
-+ if (res == qman_cb_dqrr_stop)
-+ goto done;
-+ }
-+ /* Interpret 'dq' from a driver perspective. */
-+ /* Parking isn't possible unless HELDACTIVE was set. NB,
-+ * FORCEELIGIBLE implies HELDACTIVE, so we only need to
-+ * check for HELDACTIVE to cover both. */
-+ DPA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
-+ (res != qman_cb_dqrr_park));
-+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-+ if (res != qman_cb_dqrr_defer)
-+ qm_dqrr_cdc_consume_1ptr(&p->p, orig_dq,
-+ (res == qman_cb_dqrr_park));
-+#else
-+ /* Defer just means "skip it, I'll consume it myself later on" */
-+ if (res != qman_cb_dqrr_defer)
-+ qm_dqrr_cdc_consume_1ptr(&p->p, dq, (res == qman_cb_dqrr_park));
-+#endif
-+ /* Move forward */
-+ qm_dqrr_next(&p->p);
-+ /* Entry processed and consumed, increment our counter. The callback can
-+ * request that we exit after consuming the entry, and we also exit if
-+ * we reach our processing limit, so loop back only if neither of these
-+ * conditions is met. */
-+ if ((++limit < poll_limit) && (res != qman_cb_dqrr_consume_stop))
-+ goto loop;
-+done:
-+ return limit;
-+}
-+
-+u32 qman_irqsource_get(void)
-+{
-+ /* "irqsource" and "poll" APIs mustn't redirect when sharing, they
-+ * should shut the user out if they are not the primary CPU hosting the
-+ * portal. That's why we use the "raw" interface. */
-+ struct qman_portal *p = get_raw_affine_portal();
-+ u32 ret = p->irq_sources & QM_PIRQ_VISIBLE;
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_irqsource_get);
-+
-+int qman_p_irqsource_add(struct qman_portal *p, u32 bits __maybe_unused)
-+{
-+ __maybe_unused unsigned long irqflags;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (p->sharing_redirect)
-+ return -EINVAL;
-+ else
-+#endif
-+ {
-+ bits = bits & QM_PIRQ_VISIBLE;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ /* Clear any previously remaining interrupt conditions in
-+ * QCSP_ISR. This prevents raising a false interrupt when
-+ * interrupt conditions are enabled in QCSP_IER.
-+ */
-+ qm_isr_status_clear(&p->p, bits);
-+ set_bits(bits, &p->irq_sources);
-+ qm_isr_enable_write(&p->p, p->irq_sources);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_p_irqsource_add);
-+
-+int qman_irqsource_add(u32 bits __maybe_unused)
-+{
-+ struct qman_portal *p = get_raw_affine_portal();
-+ int ret;
-+ ret = qman_p_irqsource_add(p, bits);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_irqsource_add);
-+
-+int qman_p_irqsource_remove(struct qman_portal *p, u32 bits)
-+{
-+ __maybe_unused unsigned long irqflags;
-+ u32 ier;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (p->sharing_redirect) {
-+ put_affine_portal();
-+ return -EINVAL;
-+ }
-+#endif
-+ /* Our interrupt handler only processes+clears status register bits that
-+ * are in p->irq_sources. As we're trimming that mask, if one of them
-+ * were to assert in the status register just before we remove it from
-+ * the enable register, there would be an interrupt-storm when we
-+ * release the IRQ lock. So we wait for the enable register update to
-+ * take effect in h/w (by reading it back) and then clear all other bits
-+ * in the status register. Ie. we clear them from ISR once it's certain
-+ * IER won't allow them to reassert. */
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ bits &= QM_PIRQ_VISIBLE;
-+ clear_bits(bits, &p->irq_sources);
-+ qm_isr_enable_write(&p->p, p->irq_sources);
-+
-+ ier = qm_isr_enable_read(&p->p);
-+ /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
-+ * data-dependency, ie. to protect against re-ordering. */
-+ qm_isr_status_clear(&p->p, ~ier);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_p_irqsource_remove);
-+
-+int qman_irqsource_remove(u32 bits)
-+{
-+ struct qman_portal *p = get_raw_affine_portal();
-+ int ret;
-+ ret = qman_p_irqsource_remove(p, bits);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_irqsource_remove);
-+
-+const cpumask_t *qman_affine_cpus(void)
-+{
-+ return &affine_mask;
-+}
-+EXPORT_SYMBOL(qman_affine_cpus);
-+
-+u16 qman_affine_channel(int cpu)
-+{
-+ if (cpu < 0) {
-+ struct qman_portal *portal = get_raw_affine_portal();
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ BUG_ON(portal->sharing_redirect);
-+#endif
-+ cpu = portal->config->public_cfg.cpu;
-+ put_affine_portal();
-+ }
-+ BUG_ON(!cpumask_test_cpu(cpu, &affine_mask));
-+ return affine_channels[cpu];
-+}
-+EXPORT_SYMBOL(qman_affine_channel);
-+
-+void *qman_get_affine_portal(int cpu)
-+{
-+ return affine_portals[cpu];
-+}
-+EXPORT_SYMBOL(qman_get_affine_portal);
-+
-+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit)
-+{
-+ int ret;
-+
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (unlikely(p->sharing_redirect))
-+ ret = -EINVAL;
-+ else
-+#endif
-+ {
-+ BUG_ON(p->irq_sources & QM_PIRQ_DQRI);
-+ ret = __poll_portal_fast(p, limit);
-+ }
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_p_poll_dqrr);
-+
-+int qman_poll_dqrr(unsigned int limit)
-+{
-+ struct qman_portal *p = get_poll_portal();
-+ int ret;
-+ ret = qman_p_poll_dqrr(p, limit);
-+ put_poll_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_poll_dqrr);
-+
-+u32 qman_p_poll_slow(struct qman_portal *p)
-+{
-+ u32 ret;
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (unlikely(p->sharing_redirect))
-+ ret = (u32)-1;
-+ else
-+#endif
-+ {
-+ u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
-+ ret = __poll_portal_slow(p, is);
-+ qm_isr_status_clear(&p->p, ret);
-+ }
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_p_poll_slow);
-+
-+u32 qman_poll_slow(void)
-+{
-+ struct qman_portal *p = get_poll_portal();
-+ u32 ret;
-+ ret = qman_p_poll_slow(p);
-+ put_poll_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_poll_slow);
-+
-+/* Legacy wrapper */
-+void qman_p_poll(struct qman_portal *p)
-+{
-+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
-+ if (unlikely(p->sharing_redirect))
-+ return;
-+#endif
-+ if ((~p->irq_sources) & QM_PIRQ_SLOW) {
-+ if (!(p->slowpoll--)) {
-+ u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
-+ u32 active = __poll_portal_slow(p, is);
-+ if (active) {
-+ qm_isr_status_clear(&p->p, active);
-+ p->slowpoll = SLOW_POLL_BUSY;
-+ } else
-+ p->slowpoll = SLOW_POLL_IDLE;
-+ }
-+ }
-+ if ((~p->irq_sources) & QM_PIRQ_DQRI)
-+ __poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT);
-+}
-+EXPORT_SYMBOL(qman_p_poll);
-+
-+void qman_poll(void)
-+{
-+ struct qman_portal *p = get_poll_portal();
-+ qman_p_poll(p);
-+ put_poll_portal();
-+}
-+EXPORT_SYMBOL(qman_poll);
-+
-+void qman_p_stop_dequeues(struct qman_portal *p)
-+{
-+ qman_stop_dequeues_ex(p);
-+}
-+EXPORT_SYMBOL(qman_p_stop_dequeues);
-+
-+void qman_stop_dequeues(void)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ qman_p_stop_dequeues(p);
-+ put_affine_portal();
-+}
-+EXPORT_SYMBOL(qman_stop_dequeues);
-+
-+void qman_p_start_dequeues(struct qman_portal *p)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ DPA_ASSERT(p->dqrr_disable_ref > 0);
-+ if (!(--p->dqrr_disable_ref))
-+ qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+}
-+EXPORT_SYMBOL(qman_p_start_dequeues);
-+
-+void qman_start_dequeues(void)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ qman_p_start_dequeues(p);
-+ put_affine_portal();
-+}
-+EXPORT_SYMBOL(qman_start_dequeues);
-+
-+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ pools &= p->config->public_cfg.pools;
-+ p->sdqcr |= pools;
-+ qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+}
-+EXPORT_SYMBOL(qman_p_static_dequeue_add);
-+
-+void qman_static_dequeue_add(u32 pools)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ qman_p_static_dequeue_add(p, pools);
-+ put_affine_portal();
-+}
-+EXPORT_SYMBOL(qman_static_dequeue_add);
-+
-+void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ pools &= p->config->public_cfg.pools;
-+ p->sdqcr &= ~pools;
-+ qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+}
-+EXPORT_SYMBOL(qman_p_static_dequeue_del);
-+
-+void qman_static_dequeue_del(u32 pools)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ qman_p_static_dequeue_del(p, pools);
-+ put_affine_portal();
-+}
-+EXPORT_SYMBOL(qman_static_dequeue_del);
-+
-+u32 qman_p_static_dequeue_get(struct qman_portal *p)
-+{
-+ return p->sdqcr;
-+}
-+EXPORT_SYMBOL(qman_p_static_dequeue_get);
-+
-+u32 qman_static_dequeue_get(void)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ u32 ret = qman_p_static_dequeue_get(p);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_static_dequeue_get);
-+
-+void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq,
-+ int park_request)
-+{
-+ qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
-+}
-+EXPORT_SYMBOL(qman_p_dca);
-+
-+void qman_dca(struct qm_dqrr_entry *dq, int park_request)
-+{
-+ struct qman_portal *p = get_affine_portal();
-+ qman_p_dca(p, dq, park_request);
-+ put_affine_portal();
-+}
-+EXPORT_SYMBOL(qman_dca);
-+
-+/*******************/
-+/* Frame queue API */
-+/*******************/
-+
-+static const char *mcr_result_str(u8 result)
-+{
-+ switch (result) {
-+ case QM_MCR_RESULT_NULL:
-+ return "QM_MCR_RESULT_NULL";
-+ case QM_MCR_RESULT_OK:
-+ return "QM_MCR_RESULT_OK";
-+ case QM_MCR_RESULT_ERR_FQID:
-+ return "QM_MCR_RESULT_ERR_FQID";
-+ case QM_MCR_RESULT_ERR_FQSTATE:
-+ return "QM_MCR_RESULT_ERR_FQSTATE";
-+ case QM_MCR_RESULT_ERR_NOTEMPTY:
-+ return "QM_MCR_RESULT_ERR_NOTEMPTY";
-+ case QM_MCR_RESULT_PENDING:
-+ return "QM_MCR_RESULT_PENDING";
-+ case QM_MCR_RESULT_ERR_BADCOMMAND:
-+ return "QM_MCR_RESULT_ERR_BADCOMMAND";
-+ }
-+ return "<unknown MCR result>";
-+}
-+
-+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
-+{
-+ struct qm_fqd fqd;
-+ struct qm_mcr_queryfq_np np;
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+
-+ if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
-+ int ret = qman_alloc_fqid(&fqid);
-+ if (ret)
-+ return ret;
-+ }
-+ spin_lock_init(&fq->fqlock);
-+ fq->fqid = fqid;
-+ fq->flags = flags;
-+ fq->state = qman_fq_state_oos;
-+ fq->cgr_groupid = 0;
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ if (unlikely(find_empty_fq_table_entry(&fq->key, fq)))
-+ return -ENOMEM;
-+#endif
-+ if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
-+ return 0;
-+ /* Everything else is AS_IS support */
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = qm_mc_start(&p->p);
-+ mcc->queryfq.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
-+ if (mcr->result != QM_MCR_RESULT_OK) {
-+ pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
-+ goto err;
-+ }
-+ fqd = mcr->queryfq.fqd;
-+ hw_fqd_to_cpu(&fqd);
-+ mcc = qm_mc_start(&p->p);
-+ mcc->queryfq_np.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
-+ if (mcr->result != QM_MCR_RESULT_OK) {
-+ pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
-+ goto err;
-+ }
-+ np = mcr->queryfq_np;
-+ /* Phew, have queryfq and queryfq_np results, stitch together
-+ * the FQ object from those. */
-+ fq->cgr_groupid = fqd.cgid;
-+ switch (np.state & QM_MCR_NP_STATE_MASK) {
-+ case QM_MCR_NP_STATE_OOS:
-+ break;
-+ case QM_MCR_NP_STATE_RETIRED:
-+ fq->state = qman_fq_state_retired;
-+ if (np.frm_cnt)
-+ fq_set(fq, QMAN_FQ_STATE_NE);
-+ break;
-+ case QM_MCR_NP_STATE_TEN_SCHED:
-+ case QM_MCR_NP_STATE_TRU_SCHED:
-+ case QM_MCR_NP_STATE_ACTIVE:
-+ fq->state = qman_fq_state_sched;
-+ if (np.state & QM_MCR_NP_STATE_R)
-+ fq_set(fq, QMAN_FQ_STATE_CHANGING);
-+ break;
-+ case QM_MCR_NP_STATE_PARKED:
-+ fq->state = qman_fq_state_parked;
-+ break;
-+ default:
-+ DPA_ASSERT(NULL == "invalid FQ state");
-+ }
-+ if (fqd.fq_ctrl & QM_FQCTRL_CGE)
-+ fq->state |= QMAN_FQ_STATE_CGR_EN;
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return 0;
-+err:
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
-+ qman_release_fqid(fqid);
-+ return -EIO;
-+}
-+EXPORT_SYMBOL(qman_create_fq);
-+
-+void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
-+{
-+
-+ /* We don't need to lock the FQ as it is a pre-condition that the FQ be
-+ * quiesced. Instead, run some checks. */
-+ switch (fq->state) {
-+ case qman_fq_state_parked:
-+ DPA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
-+ case qman_fq_state_oos:
-+ if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
-+ qman_release_fqid(fq->fqid);
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ clear_fq_table_entry(fq->key);
-+#endif
-+ return;
-+ default:
-+ break;
-+ }
-+ DPA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
-+}
-+EXPORT_SYMBOL(qman_destroy_fq);
-+
-+u32 qman_fq_fqid(struct qman_fq *fq)
-+{
-+ return fq->fqid;
-+}
-+EXPORT_SYMBOL(qman_fq_fqid);
-+
-+void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
-+{
-+ if (state)
-+ *state = fq->state;
-+ if (flags)
-+ *flags = fq->flags;
-+}
-+EXPORT_SYMBOL(qman_fq_state);
-+
-+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
-+ QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
-+
-+ if ((fq->state != qman_fq_state_oos) &&
-+ (fq->state != qman_fq_state_parked))
-+ return -EINVAL;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
-+ return -EINVAL;
-+#endif
-+ if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
-+ /* And can't be set at the same time as TDTHRESH */
-+ if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
-+ return -EINVAL;
-+ }
-+ /* Issue an INITFQ_[PARKED|SCHED] management command */
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ FQLOCK(fq);
-+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
-+ ((fq->state != qman_fq_state_oos) &&
-+ (fq->state != qman_fq_state_parked)))) {
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return -EBUSY;
-+ }
-+ mcc = qm_mc_start(&p->p);
-+ if (opts)
-+ mcc->initfq = *opts;
-+ mcc->initfq.fqid = cpu_to_be32(fq->fqid);
-+ mcc->initfq.count = 0;
-+
-+ /* If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a
-+ * demux pointer. Otherwise, the caller-provided value is allowed to
-+ * stand, don't overwrite it. */
-+ if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
-+ dma_addr_t phys_fq;
-+ mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ mcc->initfq.fqd.context_b = fq->key;
-+#else
-+ mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
-+#endif
-+ /* and the physical address - NB, if the user wasn't trying to
-+ * set CONTEXTA, clear the stashing settings. */
-+ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
-+ mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
-+ memset(&mcc->initfq.fqd.context_a, 0,
-+ sizeof(mcc->initfq.fqd.context_a));
-+ } else {
-+ phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq),
-+ DMA_TO_DEVICE);
-+ qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
-+ }
-+ }
-+ if (flags & QMAN_INITFQ_FLAG_LOCAL) {
-+ mcc->initfq.fqd.dest.channel = p->config->public_cfg.channel;
-+ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
-+ mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
-+ mcc->initfq.fqd.dest.wq = 4;
-+ }
-+ }
-+ mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
-+ cpu_to_hw_fqd(&mcc->initfq.fqd);
-+ qm_mc_commit(&p->p, myverb);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return -EIO;
-+ }
-+ if (opts) {
-+ if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
-+ if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
-+ fq_set(fq, QMAN_FQ_STATE_CGR_EN);
-+ else
-+ fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
-+ }
-+ if (opts->we_mask & QM_INITFQ_WE_CGID)
-+ fq->cgr_groupid = opts->fqd.cgid;
-+ }
-+ fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
-+ qman_fq_state_sched : qman_fq_state_parked;
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_init_fq);
-+
-+int qman_schedule_fq(struct qman_fq *fq)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ int ret = 0;
-+ u8 res;
-+
-+ if (fq->state != qman_fq_state_parked)
-+ return -EINVAL;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
-+ return -EINVAL;
-+#endif
-+ /* Issue a ALTERFQ_SCHED management command */
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ FQLOCK(fq);
-+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
-+ (fq->state != qman_fq_state_parked))) {
-+ ret = -EBUSY;
-+ goto out;
-+ }
-+ mcc = qm_mc_start(&p->p);
-+ mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ ret = -EIO;
-+ goto out;
-+ }
-+ fq->state = qman_fq_state_sched;
-+out:
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_schedule_fq);
-+
-+int qman_retire_fq(struct qman_fq *fq, u32 *flags)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ int rval;
-+ u8 res;
-+
-+ if ((fq->state != qman_fq_state_parked) &&
-+ (fq->state != qman_fq_state_sched))
-+ return -EINVAL;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
-+ return -EINVAL;
-+#endif
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ FQLOCK(fq);
-+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
-+ (fq->state == qman_fq_state_retired) ||
-+ (fq->state == qman_fq_state_oos))) {
-+ rval = -EBUSY;
-+ goto out;
-+ }
-+ rval = table_push_fq(p, fq);
-+ if (rval)
-+ goto out;
-+ mcc = qm_mc_start(&p->p);
-+ mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
-+ res = mcr->result;
-+ /* "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
-+ * and defer the flags until FQRNI or FQRN (respectively) show up. But
-+ * "Friendly" is to process OK immediately, and not set CHANGING. We do
-+ * friendly, otherwise the caller doesn't necessarily have a fully
-+ * "retired" FQ on return even if the retirement was immediate. However
-+ * this does mean some code duplication between here and
-+ * fq_state_change(). */
-+ if (likely(res == QM_MCR_RESULT_OK)) {
-+ rval = 0;
-+ /* Process 'fq' right away, we'll ignore FQRNI */
-+ if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
-+ fq_set(fq, QMAN_FQ_STATE_NE);
-+ if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
-+ fq_set(fq, QMAN_FQ_STATE_ORL);
-+ else
-+ table_del_fq(p, fq);
-+ if (flags)
-+ *flags = fq->flags;
-+ fq->state = qman_fq_state_retired;
-+ if (fq->cb.fqs) {
-+ /* Another issue with supporting "immediate" retirement
-+ * is that we're forced to drop FQRNIs, because by the
-+ * time they're seen it may already be "too late" (the
-+ * fq may have been OOS'd and free()'d already). But if
-+ * the upper layer wants a callback whether it's
-+ * immediate or not, we have to fake a "MR" entry to
-+ * look like an FQRNI... */
-+ struct qm_mr_entry msg;
-+ msg.verb = QM_MR_VERB_FQRNI;
-+ msg.fq.fqs = mcr->alterfq.fqs;
-+ msg.fq.fqid = fq->fqid;
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ msg.fq.contextB = fq->key;
-+#else
-+ msg.fq.contextB = (u32)(uintptr_t)fq;
-+#endif
-+ fq->cb.fqs(p, fq, &msg);
-+ }
-+ } else if (res == QM_MCR_RESULT_PENDING) {
-+ rval = 1;
-+ fq_set(fq, QMAN_FQ_STATE_CHANGING);
-+ } else {
-+ rval = -EIO;
-+ table_del_fq(p, fq);
-+ }
-+out:
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return rval;
-+}
-+EXPORT_SYMBOL(qman_retire_fq);
-+
-+int qman_oos_fq(struct qman_fq *fq)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ int ret = 0;
-+ u8 res;
-+
-+ if (fq->state != qman_fq_state_retired)
-+ return -EINVAL;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
-+ return -EINVAL;
-+#endif
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ FQLOCK(fq);
-+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
-+ (fq->state != qman_fq_state_retired))) {
-+ ret = -EBUSY;
-+ goto out;
-+ }
-+ mcc = qm_mc_start(&p->p);
-+ mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ ret = -EIO;
-+ goto out;
-+ }
-+ fq->state = qman_fq_state_oos;
-+out:
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_oos_fq);
-+
-+int qman_fq_flow_control(struct qman_fq *fq, int xon)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ int ret = 0;
-+ u8 res;
-+ u8 myverb;
-+
-+ if ((fq->state == qman_fq_state_oos) ||
-+ (fq->state == qman_fq_state_retired) ||
-+ (fq->state == qman_fq_state_parked))
-+ return -EINVAL;
-+
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
-+ return -EINVAL;
-+#endif
-+ /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ FQLOCK(fq);
-+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
-+ (fq->state == qman_fq_state_parked) ||
-+ (fq->state == qman_fq_state_oos) ||
-+ (fq->state == qman_fq_state_retired))) {
-+ ret = -EBUSY;
-+ goto out;
-+ }
-+ mcc = qm_mc_start(&p->p);
-+ mcc->alterfq.fqid = fq->fqid;
-+ mcc->alterfq.count = 0;
-+ myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
-+
-+ qm_mc_commit(&p->p, myverb);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
-+
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ ret = -EIO;
-+ goto out;
-+ }
-+out:
-+ FQUNLOCK(fq);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_fq_flow_control);
-+
-+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = qm_mc_start(&p->p);
-+ mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK)
-+ *fqd = mcr->queryfq.fqd;
-+ hw_fqd_to_cpu(fqd);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK)
-+ return -EIO;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_query_fq);
-+
-+int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = qm_mc_start(&p->p);
-+ mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK) {
-+ *np = mcr->queryfq_np;
-+ np->fqd_link = be24_to_cpu(np->fqd_link);
-+ np->odp_seq = be16_to_cpu(np->odp_seq);
-+ np->orp_nesn = be16_to_cpu(np->orp_nesn);
-+ np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq);
-+ np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq);
-+ np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
-+ np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
-+ np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
-+ np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
-+ np->ics_surp = be16_to_cpu(np->ics_surp);
-+ np->byte_cnt = be32_to_cpu(np->byte_cnt);
-+ np->frm_cnt = be24_to_cpu(np->frm_cnt);
-+ np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
-+ np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
-+ np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
-+ np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
-+ np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
-+ }
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res == QM_MCR_RESULT_ERR_FQID)
-+ return -ERANGE;
-+ else if (res != QM_MCR_RESULT_OK)
-+ return -EIO;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_query_fq_np);
-+
-+int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res, myverb;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
-+ QM_MCR_VERB_QUERYWQ;
-+ mcc = qm_mc_start(&p->p);
-+ mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
-+ qm_mc_commit(&p->p, myverb);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK) {
-+ int i, array_len;
-+ wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
-+ array_len = ARRAY_SIZE(mcr->querywq.wq_len);
-+ for (i = 0; i < array_len; i++)
-+ wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
-+ }
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_query_wq);
-+
-+int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
-+ struct qm_mcr_cgrtestwrite *result)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = qm_mc_start(&p->p);
-+ mcc->cgrtestwrite.cgid = cgr->cgrid;
-+ mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
-+ mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
-+ qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK)
-+ *result = mcr->cgrtestwrite;
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_testwrite_cgr);
-+
-+int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+ int i;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = qm_mc_start(&p->p);
-+ mcc->querycgr.cgid = cgr->cgrid;
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK)
-+ *cgrd = mcr->querycgr;
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
-+ return -EIO;
-+ }
-+ cgrd->cgr.wr_parm_g.word =
-+ be32_to_cpu(cgrd->cgr.wr_parm_g.word);
-+ cgrd->cgr.wr_parm_y.word =
-+ be32_to_cpu(cgrd->cgr.wr_parm_y.word);
-+ cgrd->cgr.wr_parm_r.word =
-+ be32_to_cpu(cgrd->cgr.wr_parm_r.word);
-+ cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ);
-+ cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
-+ for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
-+ be32_to_cpus(&cgrd->cscn_targ_swp[i]);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_query_cgr);
-+
-+int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
-+{
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+ int i;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ qm_mc_start(&p->p);
-+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_MCC_VERB_QUERYCONGESTION);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK)
-+ memcpy_fromio(congestion, &mcr->querycongestion,
-+ sizeof(*congestion));
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
-+ return -EIO;
-+ }
-+
-+ for (i = 0; i < ARRAY_SIZE(congestion->state.__state); i++)
-+ be32_to_cpus(&congestion->state.__state[i]);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_query_congestion);
-+
-+/* internal function used as a wait_event() expression */
-+static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ int ret = -EBUSY;
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ if (!p->vdqcr_owned) {
-+ FQLOCK(fq);
-+ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
-+ goto escape;
-+ fq_set(fq, QMAN_FQ_STATE_VDQCR);
-+ FQUNLOCK(fq);
-+ p->vdqcr_owned = fq;
-+ ret = 0;
-+ }
-+escape:
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ if (!ret)
-+ qm_dqrr_vdqcr_set(&p->p, vdqcr);
-+ return ret;
-+}
-+
-+static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr)
-+{
-+ int ret;
-+ *p = get_affine_portal();
-+ ret = set_p_vdqcr(*p, fq, vdqcr);
-+ put_affine_portal();
-+ return ret;
-+}
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+static int wait_p_vdqcr_start(struct qman_portal *p, struct qman_fq *fq,
-+ u32 vdqcr, u32 flags)
-+{
-+ int ret = 0;
-+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
-+ ret = wait_event_interruptible(affine_queue,
-+ !(ret = set_p_vdqcr(p, fq, vdqcr)));
-+ else
-+ wait_event(affine_queue, !(ret = set_p_vdqcr(p, fq, vdqcr)));
-+ return ret;
-+}
-+
-+static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq,
-+ u32 vdqcr, u32 flags)
-+{
-+ int ret = 0;
-+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
-+ ret = wait_event_interruptible(affine_queue,
-+ !(ret = set_vdqcr(p, fq, vdqcr)));
-+ else
-+ wait_event(affine_queue, !(ret = set_vdqcr(p, fq, vdqcr)));
-+ return ret;
-+}
-+#endif
-+
-+int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq,
-+ u32 flags __maybe_unused, u32 vdqcr)
-+{
-+ int ret;
-+
-+ if ((fq->state != qman_fq_state_parked) &&
-+ (fq->state != qman_fq_state_retired))
-+ return -EINVAL;
-+ if (vdqcr & QM_VDQCR_FQID_MASK)
-+ return -EINVAL;
-+ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
-+ return -EBUSY;
-+ vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_VOLATILE_FLAG_WAIT)
-+ ret = wait_p_vdqcr_start(p, fq, vdqcr, flags);
-+ else
-+#endif
-+ ret = set_p_vdqcr(p, fq, vdqcr);
-+ if (ret)
-+ return ret;
-+ /* VDQCR is set */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_VOLATILE_FLAG_FINISH) {
-+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
-+ /* NB: don't propagate any error - the caller wouldn't
-+ * know whether the VDQCR was issued or not. A signal
-+ * could arrive after returning anyway, so the caller
-+ * can check signal_pending() if that's an issue. */
-+ wait_event_interruptible(affine_queue,
-+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR));
-+ else
-+ wait_event(affine_queue,
-+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_p_volatile_dequeue);
-+
-+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
-+ u32 vdqcr)
-+{
-+ struct qman_portal *p;
-+ int ret;
-+
-+ if ((fq->state != qman_fq_state_parked) &&
-+ (fq->state != qman_fq_state_retired))
-+ return -EINVAL;
-+ if (vdqcr & QM_VDQCR_FQID_MASK)
-+ return -EINVAL;
-+ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
-+ return -EBUSY;
-+ vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_VOLATILE_FLAG_WAIT)
-+ ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
-+ else
-+#endif
-+ ret = set_vdqcr(&p, fq, vdqcr);
-+ if (ret)
-+ return ret;
-+ /* VDQCR is set */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_VOLATILE_FLAG_FINISH) {
-+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
-+ /* NB: don't propagate any error - the caller wouldn't
-+ * know whether the VDQCR was issued or not. A signal
-+ * could arrive after returning anyway, so the caller
-+ * can check signal_pending() if that's an issue. */
-+ wait_event_interruptible(affine_queue,
-+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR));
-+ else
-+ wait_event(affine_queue,
-+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_volatile_dequeue);
-+
-+static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
-+{
-+ if (avail)
-+ qm_eqcr_cce_prefetch(&p->p);
-+ else
-+ qm_eqcr_cce_update(&p->p);
-+}
-+
-+int qman_eqcr_is_empty(void)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ struct qman_portal *p = get_affine_portal();
-+ u8 avail;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ update_eqcr_ci(p, 0);
-+ avail = qm_eqcr_get_fill(&p->p);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return avail == 0;
-+}
-+EXPORT_SYMBOL(qman_eqcr_is_empty);
-+
-+void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
-+{
-+ if (affine) {
-+ unsigned long irqflags __maybe_unused;
-+ struct qman_portal *p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ p->cb_dc_ern = handler;
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ } else
-+ cb_dc_ern = handler;
-+}
-+EXPORT_SYMBOL(qman_set_dc_ern);
-+
-+static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
-+ unsigned long *irqflags __maybe_unused,
-+ struct qman_fq *fq,
-+ const struct qm_fd *fd,
-+ u32 flags)
-+{
-+ struct qm_eqcr_entry *eq;
-+ u8 avail;
-+ PORTAL_IRQ_LOCK(p, (*irqflags));
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (p->eqci_owned) {
-+ PORTAL_IRQ_UNLOCK(p, (*irqflags));
-+ return NULL;
-+ }
-+ p->eqci_owned = fq;
-+ }
-+#endif
-+ if (p->use_eqcr_ci_stashing) {
-+ /*
-+ * The stashing case is easy, only update if we need to in
-+ * order to try and liberate ring entries.
-+ */
-+ eq = qm_eqcr_start_stash(&p->p);
-+ } else {
-+ /*
-+ * The non-stashing case is harder, need to prefetch ahead of
-+ * time.
-+ */
-+ avail = qm_eqcr_get_avail(&p->p);
-+ if (avail < 2)
-+ update_eqcr_ci(p, avail);
-+ eq = qm_eqcr_start_no_stash(&p->p);
-+ }
-+
-+ if (unlikely(!eq)) {
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC)))
-+ p->eqci_owned = NULL;
-+#endif
-+ PORTAL_IRQ_UNLOCK(p, (*irqflags));
-+ return NULL;
-+ }
-+ if (flags & QMAN_ENQUEUE_FLAG_DCA)
-+ eq->dca = QM_EQCR_DCA_ENABLE |
-+ ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
-+ QM_EQCR_DCA_PARK : 0) |
-+ ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
-+ eq->fqid = cpu_to_be32(fq->fqid);
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ eq->tag = cpu_to_be32(fq->key);
-+#else
-+ eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
-+#endif
-+ eq->fd = *fd;
-+ cpu_to_hw_fd(&eq->fd);
-+ return eq;
-+}
-+
-+static inline struct qm_eqcr_entry *try_eq_start(struct qman_portal **p,
-+ unsigned long *irqflags __maybe_unused,
-+ struct qman_fq *fq,
-+ const struct qm_fd *fd,
-+ u32 flags)
-+{
-+ struct qm_eqcr_entry *eq;
-+ *p = get_affine_portal();
-+ eq = try_p_eq_start(*p, irqflags, fq, fd, flags);
-+ if (!eq)
-+ put_affine_portal();
-+ return eq;
-+}
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+static noinline struct qm_eqcr_entry *__wait_eq_start(struct qman_portal **p,
-+ unsigned long *irqflags __maybe_unused,
-+ struct qman_fq *fq,
-+ const struct qm_fd *fd,
-+ u32 flags)
-+{
-+ struct qm_eqcr_entry *eq = try_eq_start(p, irqflags, fq, fd, flags);
-+ if (!eq)
-+ qm_eqcr_set_ithresh(&(*p)->p, EQCR_ITHRESH);
-+ return eq;
-+}
-+static noinline struct qm_eqcr_entry *wait_eq_start(struct qman_portal **p,
-+ unsigned long *irqflags __maybe_unused,
-+ struct qman_fq *fq,
-+ const struct qm_fd *fd,
-+ u32 flags)
-+{
-+ struct qm_eqcr_entry *eq;
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return NULL if signal occurs before completion. Signal
-+ * can occur during return. Caller must check for signal */
-+ wait_event_interruptible(affine_queue,
-+ (eq = __wait_eq_start(p, irqflags, fq, fd, flags)));
-+ else
-+ wait_event(affine_queue,
-+ (eq = __wait_eq_start(p, irqflags, fq, fd, flags)));
-+ return eq;
-+}
-+static noinline struct qm_eqcr_entry *__wait_p_eq_start(struct qman_portal *p,
-+ unsigned long *irqflags __maybe_unused,
-+ struct qman_fq *fq,
-+ const struct qm_fd *fd,
-+ u32 flags)
-+{
-+ struct qm_eqcr_entry *eq = try_p_eq_start(p, irqflags, fq, fd, flags);
-+ if (!eq)
-+ qm_eqcr_set_ithresh(&p->p, EQCR_ITHRESH);
-+ return eq;
-+}
-+static noinline struct qm_eqcr_entry *wait_p_eq_start(struct qman_portal *p,
-+ unsigned long *irqflags __maybe_unused,
-+ struct qman_fq *fq,
-+ const struct qm_fd *fd,
-+ u32 flags)
-+{
-+ struct qm_eqcr_entry *eq;
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return NULL if signal occurs before completion. Signal
-+ * can occur during return. Caller must check for signal */
-+ wait_event_interruptible(affine_queue,
-+ (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags)));
-+ else
-+ wait_event(affine_queue,
-+ (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags)));
-+ return eq;
-+}
-+#endif
-+
-+int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_fd *fd, u32 flags)
-+{
-+ struct qm_eqcr_entry *eq;
-+ unsigned long irqflags __maybe_unused;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT)
-+ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
-+ else
-+#endif
-+ eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
-+ if (!eq)
-+ return -EBUSY;
-+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
-+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
-+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
-+ /* Factor the below out, it's used from qman_enqueue_orp() too */
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->eqci_owned != fq));
-+ else
-+ wait_event(affine_queue, (p->eqci_owned != fq));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_p_enqueue);
-+
-+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
-+{
-+ struct qman_portal *p;
-+ struct qm_eqcr_entry *eq;
-+ unsigned long irqflags __maybe_unused;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT)
-+ eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
-+ else
-+#endif
-+ eq = try_eq_start(&p, &irqflags, fq, fd, flags);
-+ if (!eq)
-+ return -EBUSY;
-+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
-+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
-+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
-+ /* Factor the below out, it's used from qman_enqueue_orp() too */
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->eqci_owned != fq));
-+ else
-+ wait_event(affine_queue, (p->eqci_owned != fq));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_enqueue);
-+
-+int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_fd *fd, u32 flags,
-+ struct qman_fq *orp, u16 orp_seqnum)
-+{
-+ struct qm_eqcr_entry *eq;
-+ unsigned long irqflags __maybe_unused;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT)
-+ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
-+ else
-+#endif
-+ eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
-+ if (!eq)
-+ return -EBUSY;
-+ /* Process ORP-specifics here */
-+ if (flags & QMAN_ENQUEUE_FLAG_NLIS)
-+ orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
-+ else {
-+ orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
-+ if (flags & QMAN_ENQUEUE_FLAG_NESN)
-+ orp_seqnum |= QM_EQCR_SEQNUM_NESN;
-+ else
-+ /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
-+ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
-+ }
-+ eq->seqnum = cpu_to_be16(orp_seqnum);
-+ eq->orp = cpu_to_be32(orp->fqid);
-+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
-+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
-+ ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
-+ 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
-+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->eqci_owned != fq));
-+ else
-+ wait_event(affine_queue, (p->eqci_owned != fq));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_p_enqueue_orp);
-+
-+int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
-+ struct qman_fq *orp, u16 orp_seqnum)
-+{
-+ struct qman_portal *p;
-+ struct qm_eqcr_entry *eq;
-+ unsigned long irqflags __maybe_unused;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT)
-+ eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
-+ else
-+#endif
-+ eq = try_eq_start(&p, &irqflags, fq, fd, flags);
-+ if (!eq)
-+ return -EBUSY;
-+ /* Process ORP-specifics here */
-+ if (flags & QMAN_ENQUEUE_FLAG_NLIS)
-+ orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
-+ else {
-+ orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
-+ if (flags & QMAN_ENQUEUE_FLAG_NESN)
-+ orp_seqnum |= QM_EQCR_SEQNUM_NESN;
-+ else
-+ /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
-+ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
-+ }
-+ eq->seqnum = cpu_to_be16(orp_seqnum);
-+ eq->orp = cpu_to_be32(orp->fqid);
-+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
-+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
-+ ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
-+ 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
-+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->eqci_owned != fq));
-+ else
-+ wait_event(affine_queue, (p->eqci_owned != fq));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_enqueue_orp);
-+
-+int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_fd *fd, u32 flags,
-+ qman_cb_precommit cb, void *cb_arg)
-+{
-+ struct qm_eqcr_entry *eq;
-+ unsigned long irqflags __maybe_unused;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT)
-+ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
-+ else
-+#endif
-+ eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
-+ if (!eq)
-+ return -EBUSY;
-+ /* invoke user supplied callback function before writing commit verb */
-+ if (cb(cb_arg)) {
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ return -EINVAL;
-+ }
-+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
-+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
-+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
-+ /* Factor the below out, it's used from qman_enqueue_orp() too */
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->eqci_owned != fq));
-+ else
-+ wait_event(affine_queue, (p->eqci_owned != fq));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_p_enqueue_precommit);
-+
-+int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd,
-+ u32 flags, qman_cb_precommit cb, void *cb_arg)
-+{
-+ struct qman_portal *p;
-+ struct qm_eqcr_entry *eq;
-+ unsigned long irqflags __maybe_unused;
-+
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT)
-+ eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
-+ else
-+#endif
-+ eq = try_eq_start(&p, &irqflags, fq, fd, flags);
-+ if (!eq)
-+ return -EBUSY;
-+ /* invoke user supplied callback function before writing commit verb */
-+ if (cb(cb_arg)) {
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return -EINVAL;
-+ }
-+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
-+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
-+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
-+ /* Factor the below out, it's used from qman_enqueue_orp() too */
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
-+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
-+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
-+ /* NB: return success even if signal occurs before
-+ * condition is true. pvb_commit guarantees success */
-+ wait_event_interruptible(affine_queue,
-+ (p->eqci_owned != fq));
-+ else
-+ wait_event(affine_queue, (p->eqci_owned != fq));
-+ }
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_enqueue_precommit);
-+
-+int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
-+ struct qm_mcc_initcgr *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+ u8 verb = QM_MCC_VERB_MODIFYCGR;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ mcc = qm_mc_start(&p->p);
-+ if (opts)
-+ mcc->initcgr = *opts;
-+ mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
-+ mcc->initcgr.cgr.wr_parm_g.word =
-+ cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
-+ mcc->initcgr.cgr.wr_parm_y.word =
-+ cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
-+ mcc->initcgr.cgr.wr_parm_r.word =
-+ cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
-+ mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
-+ mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
-+
-+ mcc->initcgr.cgid = cgr->cgrid;
-+ if (flags & QMAN_CGR_FLAG_USE_INIT)
-+ verb = QM_MCC_VERB_INITCGR;
-+ qm_mc_commit(&p->p, verb);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
-+ res = mcr->result;
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
-+}
-+EXPORT_SYMBOL(qman_modify_cgr);
-+
-+#define TARG_MASK(n) (0x80000000 >> (n->config->public_cfg.channel - \
-+ QM_CHANNEL_SWPORTAL0))
-+#define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
-+#define PORTAL_IDX(n) (n->config->public_cfg.channel - QM_CHANNEL_SWPORTAL0)
-+
-+static u8 qman_cgr_cpus[__CGR_NUM];
-+
-+int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
-+ struct qm_mcc_initcgr *opts)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ struct qm_mcr_querycgr cgr_state;
-+ struct qm_mcc_initcgr local_opts;
-+ int ret;
-+ struct qman_portal *p;
-+
-+ /* We have to check that the provided CGRID is within the limits of the
-+ * data-structures, for obvious reasons. However we'll let h/w take
-+ * care of determining whether it's within the limits of what exists on
-+ * the SoC. */
-+ if (cgr->cgrid >= __CGR_NUM)
-+ return -EINVAL;
-+
-+ preempt_disable();
-+ p = get_affine_portal();
-+ qman_cgr_cpus[cgr->cgrid] = smp_processor_id();
-+ preempt_enable();
-+
-+ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
-+ cgr->chan = p->config->public_cfg.channel;
-+ spin_lock_irqsave(&p->cgr_lock, irqflags);
-+
-+ /* if no opts specified, just add it to the list */
-+ if (!opts)
-+ goto add_list;
-+
-+ ret = qman_query_cgr(cgr, &cgr_state);
-+ if (ret)
-+ goto release_lock;
-+ if (opts)
-+ local_opts = *opts;
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-+ local_opts.cgr.cscn_targ_upd_ctrl =
-+ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
-+ else
-+ /* Overwrite TARG */
-+ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
-+ TARG_MASK(p);
-+ local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
-+
-+ /* send init if flags indicate so */
-+ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
-+ ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
-+ else
-+ ret = qman_modify_cgr(cgr, 0, &local_opts);
-+ if (ret)
-+ goto release_lock;
-+add_list:
-+ list_add(&cgr->node, &p->cgr_cbs);
-+
-+ /* Determine if newly added object requires its callback to be called */
-+ ret = qman_query_cgr(cgr, &cgr_state);
-+ if (ret) {
-+ /* we can't go back, so proceed and return success, but screen
-+ * and wail to the log file */
-+ pr_crit("CGR HW state partially modified\n");
-+ ret = 0;
-+ goto release_lock;
-+ }
-+ if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
-+ cgr->cgrid))
-+ cgr->cb(p, cgr, 1);
-+release_lock:
-+ spin_unlock_irqrestore(&p->cgr_lock, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_create_cgr);
-+
-+int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
-+ struct qm_mcc_initcgr *opts)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ struct qm_mcc_initcgr local_opts;
-+ struct qm_mcr_querycgr cgr_state;
-+ int ret;
-+
-+ if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
-+ pr_warn("This QMan version doesn't support to send CSCN to DCP portal\n");
-+ return -EINVAL;
-+ }
-+ /* We have to check that the provided CGRID is within the limits of the
-+ * data-structures, for obvious reasons. However we'll let h/w take
-+ * care of determining whether it's within the limits of what exists on
-+ * the SoC.
-+ */
-+ if (cgr->cgrid >= __CGR_NUM)
-+ return -EINVAL;
-+
-+ ret = qman_query_cgr(cgr, &cgr_state);
-+ if (ret)
-+ return ret;
-+
-+ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
-+ if (opts)
-+ local_opts = *opts;
-+
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-+ local_opts.cgr.cscn_targ_upd_ctrl =
-+ QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
-+ QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
-+ else
-+ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
-+ TARG_DCP_MASK(dcp_portal);
-+ local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
-+
-+ /* send init if flags indicate so */
-+ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
-+ ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
-+ &local_opts);
-+ else
-+ ret = qman_modify_cgr(cgr, 0, &local_opts);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_create_cgr_to_dcp);
-+
-+int qman_delete_cgr(struct qman_cgr *cgr)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ struct qm_mcr_querycgr cgr_state;
-+ struct qm_mcc_initcgr local_opts;
-+ int ret = 0;
-+ struct qman_cgr *i;
-+ struct qman_portal *p = get_affine_portal();
-+
-+ if (cgr->chan != p->config->public_cfg.channel) {
-+ pr_crit("Attempting to delete cgr from different portal "
-+ "than it was create: create 0x%x, delete 0x%x\n",
-+ cgr->chan, p->config->public_cfg.channel);
-+ ret = -EINVAL;
-+ goto put_portal;
-+ }
-+ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
-+ spin_lock_irqsave(&p->cgr_lock, irqflags);
-+ list_del(&cgr->node);
-+ /*
-+ * If there are no other CGR objects for this CGRID in the list, update
-+ * CSCN_TARG accordingly
-+ */
-+ list_for_each_entry(i, &p->cgr_cbs, node)
-+ if ((i->cgrid == cgr->cgrid) && i->cb)
-+ goto release_lock;
-+ ret = qman_query_cgr(cgr, &cgr_state);
-+ if (ret) {
-+ /* add back to the list */
-+ list_add(&cgr->node, &p->cgr_cbs);
-+ goto release_lock;
-+ }
-+ /* Overwrite TARG */
-+ local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
-+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-+ local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
-+ else
-+ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
-+ ~(TARG_MASK(p));
-+ ret = qman_modify_cgr(cgr, 0, &local_opts);
-+ if (ret)
-+ /* add back to the list */
-+ list_add(&cgr->node, &p->cgr_cbs);
-+release_lock:
-+ spin_unlock_irqrestore(&p->cgr_lock, irqflags);
-+put_portal:
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_delete_cgr);
-+
-+struct cgr_comp {
-+ struct qman_cgr *cgr;
-+ struct completion completion;
-+};
-+
-+static int qman_delete_cgr_thread(void *p)
-+{
-+ struct cgr_comp *cgr_comp = (struct cgr_comp *)p;
-+ int res;
-+
-+ res = qman_delete_cgr((struct qman_cgr *)cgr_comp->cgr);
-+ complete(&cgr_comp->completion);
-+
-+ return res;
-+}
-+
-+void qman_delete_cgr_safe(struct qman_cgr *cgr)
-+{
-+ struct task_struct *thread;
-+ struct cgr_comp cgr_comp;
-+
-+ preempt_disable();
-+ if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) {
-+ init_completion(&cgr_comp.completion);
-+ cgr_comp.cgr = cgr;
-+ thread = kthread_create(qman_delete_cgr_thread, &cgr_comp,
-+ "cgr_del");
-+
-+ if (likely(!IS_ERR(thread))) {
-+ kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]);
-+ wake_up_process(thread);
-+ wait_for_completion(&cgr_comp.completion);
-+ preempt_enable();
-+ return;
-+ }
-+ }
-+ qman_delete_cgr(cgr);
-+ preempt_enable();
-+}
-+EXPORT_SYMBOL(qman_delete_cgr_safe);
-+
-+int qm_get_clock(u64 *clock_hz)
-+{
-+ if (!qman_clk) {
-+ pr_warn("Qman clock speed is unknown\n");
-+ return -EINVAL;
-+ }
-+ *clock_hz = (u64)qman_clk;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qm_get_clock);
-+
-+int qm_set_clock(u64 clock_hz)
-+{
-+ if (qman_clk)
-+ return -1;
-+ qman_clk = (u32)clock_hz;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qm_set_clock);
-+
-+/* CEETM management command */
-+static int qman_ceetm_configure_lfqmt(struct qm_mcc_ceetm_lfqmt_config *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->lfqmt_config = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_CONFIG);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_LFQMT_CONFIG);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CONFIGURE LFQMT failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+
-+int qman_ceetm_query_lfqmt(int lfqid,
-+ struct qm_mcr_ceetm_lfqmt_query *lfqmt_query)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->lfqmt_query.lfqid = lfqid;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_QUERY);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_LFQMT_QUERY);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK)
-+ *lfqmt_query = mcr->lfqmt_query;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: QUERY LFQMT failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_query_lfqmt);
-+
-+static int qman_ceetm_configure_cq(struct qm_mcc_ceetm_cq_config *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->cq_config = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_CONFIG);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ res = mcr->result;
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_CONFIG);
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CONFIGURE CQ failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+
-+int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid,
-+ struct qm_mcr_ceetm_cq_query *cq_query)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->cq_query.cqid = cpu_to_be16(cqid);
-+ mcc->cq_query.dcpid = dcpid;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_QUERY);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_QUERY);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK) {
-+ *cq_query = mcr->cq_query;
-+ hw_cq_query_to_cpu(cq_query);
-+ }
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: QUERY CQ failed\n");
-+ return -EIO;
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_query_cq);
-+
-+static int qman_ceetm_configure_dct(struct qm_mcc_ceetm_dct_config *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->dct_config = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_CONFIG);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_CONFIG);
-+ res = mcr->result;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CONFIGURE DCT failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+
-+static int qman_ceetm_query_dct(struct qm_mcc_ceetm_dct_query *opts,
-+ struct qm_mcr_ceetm_dct_query *dct_query)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p = get_affine_portal();
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->dct_query = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_QUERY);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_QUERY);
-+ res = mcr->result;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: QUERY DCT failed\n");
-+ return -EIO;
-+ }
-+
-+ *dct_query = mcr->dct_query;
-+ return 0;
-+}
-+
-+static int qman_ceetm_configure_class_scheduler(
-+ struct qm_mcc_ceetm_class_scheduler_config *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->csch_config = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG);
-+ res = mcr->result;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CONFIGURE CLASS SCHEDULER failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+
-+static int qman_ceetm_query_class_scheduler(struct qm_ceetm_channel *channel,
-+ struct qm_mcr_ceetm_class_scheduler_query *query)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->csch_query.cqcid = cpu_to_be16(channel->idx);
-+ mcc->csch_query.dcpid = channel->dcp_idx;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_QUERY);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_CLASS_SCHEDULER_QUERY);
-+ res = mcr->result;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: QUERY CLASS SCHEDULER failed\n");
-+ return -EIO;
-+ }
-+ *query = mcr->csch_query;
-+ return 0;
-+}
-+
-+static int qman_ceetm_configure_mapping_shaper_tcfc(
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->mst_config = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG);
-+ res = mcr->result;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CONFIGURE CHANNEL MAPPING failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+
-+static int qman_ceetm_query_mapping_shaper_tcfc(
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query *opts,
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query *response)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->mst_query = *opts;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY);
-+ res = mcr->result;
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: QUERY CHANNEL MAPPING failed\n");
-+ return -EIO;
-+ }
-+
-+ *response = mcr->mst_query;
-+ return 0;
-+}
-+
-+static int qman_ceetm_configure_ccgr(struct qm_mcc_ceetm_ccgr_config *opts)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->ccgr_config = *opts;
-+
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_CONFIG);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_CONFIG);
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CONFIGURE CCGR failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+
-+int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query,
-+ struct qm_mcr_ceetm_ccgr_query *response)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->ccgr_query.ccgrid = cpu_to_be16(ccgr_query->ccgrid);
-+ mcc->ccgr_query.dcpid = ccgr_query->dcpid;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
-+
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_QUERY);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK) {
-+ *response = mcr->ccgr_query;
-+ hw_ccgr_query_to_cpu(response);
-+ }
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: QUERY CCGR failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_query_ccgr);
-+
-+static int qman_ceetm_cq_peek_pop_xsfdrread(struct qm_ceetm_cq *cq,
-+ u8 command_type, u16 xsfdr,
-+ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread *cq_ppxr)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ switch (command_type) {
-+ case 0:
-+ case 1:
-+ mcc->cq_ppxr.cqid = (cq->parent->idx << 4) | cq->idx;
-+ break;
-+ case 2:
-+ mcc->cq_ppxr.xsfdr = xsfdr;
-+ break;
-+ default:
-+ break;
-+ }
-+ mcc->cq_ppxr.ct = command_type;
-+ mcc->cq_ppxr.dcpid = cq->parent->dcp_idx;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD);
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD);
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: CQ PEEK/POP/XSFDR READ failed\n");
-+ return -EIO;
-+ }
-+ *cq_ppxr = mcr->cq_ppxr;
-+ return 0;
-+}
-+
-+static int qman_ceetm_query_statistics(u16 cid,
-+ enum qm_dc_portal dcp_idx,
-+ u16 command_type,
-+ struct qm_mcr_ceetm_statistics_query *query_result)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->stats_query_write.cid = cid;
-+ mcc->stats_query_write.dcpid = dcp_idx;
-+ mcc->stats_query_write.ct = command_type;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
-+
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: STATISTICS QUERY failed\n");
-+ return -EIO;
-+ }
-+ *query_result = mcr->stats_query;
-+ return 0;
-+}
-+
-+int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx,
-+ u16 command_type, u64 frame_count,
-+ u64 byte_count)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ mcc->stats_query_write.cid = cid;
-+ mcc->stats_query_write.dcpid = dcp_idx;
-+ mcc->stats_query_write.ct = command_type;
-+ mcc->stats_query_write.frm_cnt = frame_count;
-+ mcc->stats_query_write.byte_cnt = byte_count;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
-+
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
-+
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+
-+ res = mcr->result;
-+ if (res != QM_MCR_RESULT_OK) {
-+ pr_err("CEETM: STATISTICS WRITE failed\n");
-+ return -EIO;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_query_write_statistics);
-+
-+int qman_ceetm_bps2tokenrate(u64 bps, struct qm_ceetm_rate *token_rate,
-+ int rounding)
-+{
-+ u16 pres;
-+ u64 temp;
-+ u64 qman_freq;
-+ int ret;
-+
-+ /* Read PRES from CEET_CFG_PRES register */
-+ ret = qman_ceetm_get_prescaler(&pres);
-+ if (ret)
-+ return -EINVAL;
-+
-+ ret = qm_get_clock(&qman_freq);
-+ if (ret)
-+ return -EINVAL;
-+
-+ /* token-rate = bytes-per-second * update-reference-period
-+ *
-+ * Where token-rate is N/8192 for a integer N, and
-+ * update-reference-period is (2^22)/(PRES*QHz), where PRES
-+ * is the prescalar value and QHz is the QMan clock frequency.
-+ * So:
-+ *
-+ * token-rate = (byte-per-second*2^22)/PRES*QHZ)
-+ *
-+ * Converting to bits-per-second gives;
-+ *
-+ * token-rate = (bps*2^19) / (PRES*QHZ)
-+ * N = (bps*2^32) / (PRES*QHz)
-+ *
-+ * And to avoid 64-bit overflow if 'bps' is larger than 4Gbps
-+ * (yet minimise rounding error if 'bps' is small), we reorganise
-+ * the formula to use two 16-bit shifts rather than 1 32-bit shift.
-+ * N = (((bps*2^16)/PRES)*2^16)/QHz
-+ */
-+ temp = ROUNDING((bps << 16), pres, rounding);
-+ temp = ROUNDING((temp << 16), qman_freq, rounding);
-+ token_rate->whole = temp >> 13;
-+ token_rate->fraction = temp & (((u64)1 << 13) - 1);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_bps2tokenrate);
-+
-+int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, u64 *bps,
-+ int rounding)
-+{
-+ u16 pres;
-+ u64 temp;
-+ u64 qman_freq;
-+ int ret;
-+
-+ /* Read PRES from CEET_CFG_PRES register */
-+ ret = qman_ceetm_get_prescaler(&pres);
-+ if (ret)
-+ return -EINVAL;
-+
-+ ret = qm_get_clock(&qman_freq);
-+ if (ret)
-+ return -EINVAL;
-+
-+ /* bytes-per-second = token-rate / update-reference-period
-+ *
-+ * where "token-rate" is N/8192 for an integer N, and
-+ * "update-reference-period" is (2^22)/(PRES*QHz), where PRES is
-+ * the prescalar value and QHz is the QMan clock frequency. So;
-+ *
-+ * bytes-per-second = (N/8192) / (4194304/PRES*QHz)
-+ * = N*PRES*QHz / (4194304*8192)
-+ * = N*PRES*QHz / (2^35)
-+ *
-+ * Converting to bits-per-second gives;
-+ *
-+ * bps = N*PRES*QHZ / (2^32)
-+ *
-+ * Note, the numerator has a maximum width of 72 bits! So to
-+ * avoid 64-bit overflow errors, we calculate PRES*QHZ (maximum
-+ * width 48 bits) divided by 2^9 (reducing to maximum 39 bits), before
-+ * multiplying by N (goes to maximum of 63 bits).
-+ *
-+ * temp = PRES*QHZ / (2^16)
-+ * kbps = temp*N / (2^16)
-+ */
-+ temp = ROUNDING(qman_freq * pres, (u64)1 << 16 , rounding);
-+ temp *= ((token_rate->whole << 13) + token_rate->fraction);
-+ *bps = ROUNDING(temp, (u64)(1) << 16, rounding);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_tokenrate2bps);
-+
-+int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, enum qm_dc_portal dcp_idx,
-+ unsigned int sp_idx)
-+{
-+ struct qm_ceetm_sp *p;
-+
-+ DPA_ASSERT((dcp_idx == qm_dc_portal_fman0) ||
-+ (dcp_idx == qm_dc_portal_fman1));
-+
-+ if ((sp_idx < qman_ceetms[dcp_idx].sp_range[0]) ||
-+ (sp_idx >= (qman_ceetms[dcp_idx].sp_range[0] +
-+ qman_ceetms[dcp_idx].sp_range[1]))) {
-+ pr_err("Sub-portal index doesn't exist\n");
-+ return -EINVAL;
-+ }
-+
-+ list_for_each_entry(p, &qman_ceetms[dcp_idx].sub_portals, node) {
-+ if ((p->idx == sp_idx) && (p->is_claimed == 0)) {
-+ p->is_claimed = 1;
-+ *sp = p;
-+ return 0;
-+ }
-+ }
-+ pr_err("The sub-portal#%d is not available!\n", sp_idx);
-+ return -ENODEV;
-+}
-+EXPORT_SYMBOL(qman_ceetm_sp_claim);
-+
-+int qman_ceetm_sp_release(struct qm_ceetm_sp *sp)
-+{
-+ struct qm_ceetm_sp *p;
-+
-+ if (sp->lni && sp->lni->is_claimed == 1) {
-+ pr_err("The dependency of sub-portal has not been released!\n");
-+ return -EBUSY;
-+ }
-+
-+ list_for_each_entry(p, &qman_ceetms[sp->dcp_idx].sub_portals, node) {
-+ if (p->idx == sp->idx) {
-+ p->is_claimed = 0;
-+ p->lni = NULL;
-+ }
-+ }
-+ /* Disable CEETM mode of this sub-portal */
-+ qman_sp_disable_ceetm_mode(sp->dcp_idx, sp->idx);
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_sp_release);
-+
-+int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, enum qm_dc_portal dcp_idx,
-+ unsigned int lni_idx)
-+{
-+ struct qm_ceetm_lni *p;
-+
-+ if ((lni_idx < qman_ceetms[dcp_idx].lni_range[0]) ||
-+ (lni_idx >= (qman_ceetms[dcp_idx].lni_range[0] +
-+ qman_ceetms[dcp_idx].lni_range[1]))) {
-+ pr_err("The lni index is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ list_for_each_entry(p, &qman_ceetms[dcp_idx].lnis, node) {
-+ if ((p->idx == lni_idx) && (p->is_claimed == 0)) {
-+ *lni = p;
-+ p->is_claimed = 1;
-+ return 0;
-+ }
-+ }
-+
-+ pr_err("The LNI#%d is not available!\n", lni_idx);
-+ return -EINVAL;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_claim);
-+
-+int qman_ceetm_lni_release(struct qm_ceetm_lni *lni)
-+{
-+ struct qm_ceetm_lni *p;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ if (!list_empty(&lni->channels)) {
-+ pr_err("The LNI dependencies are not released!\n");
-+ return -EBUSY;
-+ }
-+
-+ list_for_each_entry(p, &qman_ceetms[lni->dcp_idx].lnis, node) {
-+ if (p->idx == lni->idx) {
-+ p->shaper_enable = 0;
-+ p->shaper_couple = 0;
-+ p->cr_token_rate.whole = 0;
-+ p->cr_token_rate.fraction = 0;
-+ p->er_token_rate.whole = 0;
-+ p->er_token_rate.fraction = 0;
-+ p->cr_token_bucket_limit = 0;
-+ p->er_token_bucket_limit = 0;
-+ p->is_claimed = 0;
-+ }
-+ }
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ memset(&config_opts.shaper_config, 0,
-+ sizeof(config_opts.shaper_config));
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_release);
-+
-+int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, struct qm_ceetm_lni *lni)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx);
-+ config_opts.dcpid = sp->dcp_idx;
-+ config_opts.sp_mapping.map_lni_id = lni->idx;
-+ sp->lni = lni;
-+
-+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts))
-+ return -EINVAL;
-+
-+ /* Enable CEETM mode for this sub-portal */
-+ return qman_sp_enable_ceetm_mode(sp->dcp_idx, sp->idx);
-+}
-+EXPORT_SYMBOL(qman_ceetm_sp_set_lni);
-+
-+int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx);
-+ query_opts.dcpid = sp->dcp_idx;
-+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
-+ pr_err("Can't get SP <-> LNI mapping\n");
-+ return -EINVAL;
-+ }
-+ *lni_idx = query_result.sp_mapping_query.map_lni_id;
-+ sp->lni->idx = query_result.sp_mapping_query.map_lni_id;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_sp_get_lni);
-+
-+int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled,
-+ int oal)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ if (lni->shaper_enable) {
-+ pr_err("The shaper has already been enabled\n");
-+ return -EINVAL;
-+ }
-+ lni->shaper_enable = 1;
-+ lni->shaper_couple = coupled;
-+ lni->oal = oal;
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ config_opts.shaper_config.cpl = coupled;
-+ config_opts.shaper_config.oal = oal;
-+ config_opts.shaper_config.crtcr = cpu_to_be24((lni->cr_token_rate.whole
-+ << 13) | lni->cr_token_rate.fraction);
-+ config_opts.shaper_config.ertcr = cpu_to_be24((lni->er_token_rate.whole
-+ << 13) | lni->er_token_rate.fraction);
-+ config_opts.shaper_config.crtbl =
-+ cpu_to_be16(lni->cr_token_bucket_limit);
-+ config_opts.shaper_config.ertbl =
-+ cpu_to_be16(lni->er_token_bucket_limit);
-+ config_opts.shaper_config.mps = 60;
-+
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_enable_shaper);
-+
-+int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ if (!lni->shaper_enable) {
-+ pr_err("The shaper has been disabled\n");
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ config_opts.shaper_config.cpl = lni->shaper_couple;
-+ config_opts.shaper_config.oal = lni->oal;
-+ config_opts.shaper_config.crtbl =
-+ cpu_to_be16(lni->cr_token_bucket_limit);
-+ config_opts.shaper_config.ertbl =
-+ cpu_to_be16(lni->er_token_bucket_limit);
-+ /* Set CR/ER rate with all 1's to configure an infinite rate, thus
-+ * disable the shaping.
-+ */
-+ config_opts.shaper_config.crtcr = 0xFFFFFF;
-+ config_opts.shaper_config.ertcr = 0xFFFFFF;
-+ config_opts.shaper_config.mps = 60;
-+ lni->shaper_enable = 0;
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_disable_shaper);
-+
-+int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni)
-+{
-+ return lni->shaper_enable;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_is_shaper_enabled);
-+
-+int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ lni->cr_token_rate.whole = token_rate->whole;
-+ lni->cr_token_rate.fraction = token_rate->fraction;
-+ lni->cr_token_bucket_limit = token_limit;
-+ if (!lni->shaper_enable)
-+ return 0;
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ query_opts.dcpid = lni->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts,
-+ &query_result);
-+ if (ret) {
-+ pr_err("Fail to get current LNI shaper setting\n");
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole << 13)
-+ | (token_rate->fraction));
-+ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit);
-+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
-+ config_opts.shaper_config.oal = query_result.shaper_query.oal;
-+ config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr;
-+ config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl;
-+ config_opts.shaper_config.mps = query_result.shaper_query.mps;
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate);
-+
-+int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 bps,
-+ u16 token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
-+ if (ret) {
-+ pr_err("Can not convert bps to token rate\n");
-+ return -EINVAL;
-+ }
-+
-+ return qman_ceetm_lni_set_commit_rate(lni, &token_rate, token_limit);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate_bps);
-+
-+int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ query_opts.dcpid = lni->dcp_idx;
-+
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret) {
-+ pr_err("The LNI CR rate or limit is not set\n");
-+ return -EINVAL;
-+ }
-+ token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13;
-+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) &
-+ 0x1FFF;
-+ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate);
-+
-+int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 *bps, u16 *token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_lni_get_commit_rate(lni, &token_rate, token_limit);
-+ if (ret) {
-+ pr_err("The LNI CR rate or limit is not available\n");
-+ return -EINVAL;
-+ }
-+
-+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate_bps);
-+
-+int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ lni->er_token_rate.whole = token_rate->whole;
-+ lni->er_token_rate.fraction = token_rate->fraction;
-+ lni->er_token_bucket_limit = token_limit;
-+ if (!lni->shaper_enable)
-+ return 0;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ query_opts.dcpid = lni->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts,
-+ &query_result);
-+ if (ret) {
-+ pr_err("Fail to get current LNI shaper setting\n");
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ config_opts.shaper_config.ertcr = cpu_to_be24(
-+ (token_rate->whole << 13) | (token_rate->fraction));
-+ config_opts.shaper_config.ertbl = cpu_to_be16(token_limit);
-+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
-+ config_opts.shaper_config.oal = query_result.shaper_query.oal;
-+ config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr;
-+ config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl;
-+ config_opts.shaper_config.mps = query_result.shaper_query.mps;
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate);
-+
-+int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 bps,
-+ u16 token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
-+ if (ret) {
-+ pr_err("Can not convert bps to token rate\n");
-+ return -EINVAL;
-+ }
-+ return qman_ceetm_lni_set_excess_rate(lni, &token_rate, token_limit);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate_bps);
-+
-+int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
-+ query_opts.dcpid = lni->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret) {
-+ pr_err("The LNI ER rate or limit is not set\n");
-+ return -EINVAL;
-+ }
-+ token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13;
-+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) &
-+ 0x1FFF;
-+ *token_limit = be16_to_cpu(query_result.shaper_query.ertbl);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate);
-+
-+int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 *bps, u16 *token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_lni_get_excess_rate(lni, &token_rate, token_limit);
-+ if (ret) {
-+ pr_err("The LNI ER rate or limit is not available\n");
-+ return -EINVAL;
-+ }
-+
-+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate_bps);
-+
-+#define QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(n) ((15 - n) * 4)
-+#define QMAN_CEETM_LNITCFCC_ENABLE 0x8
-+int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni,
-+ unsigned int cq_level,
-+ int traffic_class)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ u64 lnitcfcc;
-+
-+ if ((cq_level > 15) | (traffic_class > 7)) {
-+ pr_err("The CQ or traffic class id is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx);
-+ query_opts.dcpid = lni->dcp_idx;
-+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
-+ pr_err("Fail to query tcfcc\n");
-+ return -EINVAL;
-+ }
-+
-+ lnitcfcc = be64_to_cpu(query_result.tcfc_query.lnitcfcc);
-+ if (traffic_class == -1) {
-+ /* disable tcfc for this CQ */
-+ lnitcfcc &= ~((u64)QMAN_CEETM_LNITCFCC_ENABLE <<
-+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
-+ } else {
-+ lnitcfcc &= ~((u64)0xF <<
-+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
-+ lnitcfcc |= ((u64)(QMAN_CEETM_LNITCFCC_ENABLE |
-+ traffic_class)) <<
-+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level);
-+ }
-+ config_opts.tcfc_config.lnitcfcc = cpu_to_be64(lnitcfcc);
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_set_tcfcc);
-+
-+#define QMAN_CEETM_LNITCFCC_TC_MASK 0x7
-+int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, unsigned int cq_level,
-+ int *traffic_class)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+ u8 lnitcfcc;
-+
-+ if (cq_level > 15) {
-+ pr_err("the CQ level is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx);
-+ query_opts.dcpid = lni->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret)
-+ return ret;
-+ lnitcfcc = (u8)be64_to_cpu((query_result.tcfc_query.lnitcfcc) >>
-+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
-+ if (lnitcfcc & QMAN_CEETM_LNITCFCC_ENABLE)
-+ *traffic_class = lnitcfcc & QMAN_CEETM_LNITCFCC_TC_MASK;
-+ else
-+ *traffic_class = -1;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lni_get_tcfcc);
-+
-+int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel,
-+ struct qm_ceetm_lni *lni)
-+{
-+ struct qm_ceetm_channel *p;
-+ u32 channel_idx;
-+ int ret = 0;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ if (lni->dcp_idx == qm_dc_portal_fman0) {
-+ ret = qman_alloc_ceetm0_channel(&channel_idx);
-+ } else if (lni->dcp_idx == qm_dc_portal_fman1) {
-+ ret = qman_alloc_ceetm1_channel(&channel_idx);
-+ } else {
-+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
-+ lni->dcp_idx);
-+ return -EINVAL;
-+ }
-+
-+ if (ret) {
-+ pr_err("The is no channel available for LNI#%d\n", lni->idx);
-+ return -ENODEV;
-+ }
-+
-+ p = kzalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p)
-+ return -ENOMEM;
-+ p->idx = channel_idx;
-+ p->dcp_idx = lni->dcp_idx;
-+ p->lni_idx = lni->idx;
-+ list_add_tail(&p->node, &lni->channels);
-+ INIT_LIST_HEAD(&p->class_queues);
-+ INIT_LIST_HEAD(&p->ccgs);
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
-+ channel_idx);
-+ config_opts.dcpid = lni->dcp_idx;
-+ config_opts.channel_mapping.map_lni_id = lni->idx;
-+ config_opts.channel_mapping.map_shaped = 0;
-+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
-+ pr_err("Can't map channel#%d for LNI#%d\n",
-+ channel_idx, lni->idx);
-+ return -EINVAL;
-+ }
-+ *channel = p;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_claim);
-+
-+int qman_ceetm_channel_release(struct qm_ceetm_channel *channel)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+ if (!list_empty(&channel->class_queues)) {
-+ pr_err("CEETM channel#%d has class queue unreleased!\n",
-+ channel->idx);
-+ return -EBUSY;
-+ }
-+ if (!list_empty(&channel->ccgs)) {
-+ pr_err("CEETM channel#%d has ccg unreleased!\n",
-+ channel->idx);
-+ return -EBUSY;
-+ }
-+
-+ /* channel->dcp_idx corresponds to known fman validation */
-+ if ((channel->dcp_idx != qm_dc_portal_fman0) &&
-+ (channel->dcp_idx != qm_dc_portal_fman1)) {
-+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
-+ channel->dcp_idx);
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ memset(&config_opts.shaper_config, 0,
-+ sizeof(config_opts.shaper_config));
-+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
-+ pr_err("Can't reset channel shapping parameters\n");
-+ return -EINVAL;
-+ }
-+
-+ if (channel->dcp_idx == qm_dc_portal_fman0) {
-+ qman_release_ceetm0_channelid(channel->idx);
-+ } else if (channel->dcp_idx == qm_dc_portal_fman1) {
-+ qman_release_ceetm1_channelid(channel->idx);
-+ } else {
-+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
-+ channel->dcp_idx);
-+ return -EINVAL;
-+ }
-+ list_del(&channel->node);
-+ kfree(channel);
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_release);
-+
-+int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel,
-+ int coupled)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ if (channel->shaper_enable == 1) {
-+ pr_err("This channel shaper has been enabled!\n");
-+ return -EINVAL;
-+ }
-+
-+ channel->shaper_enable = 1;
-+ channel->shaper_couple = coupled;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+
-+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
-+ pr_err("Can't query channel mapping\n");
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
-+ channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ config_opts.channel_mapping.map_lni_id =
-+ query_result.channel_mapping_query.map_lni_id;
-+ config_opts.channel_mapping.map_shaped = 1;
-+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
-+ pr_err("Can't enable shaper for channel #%d\n", channel->idx);
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ config_opts.shaper_config.cpl = coupled;
-+ config_opts.shaper_config.crtcr =
-+ cpu_to_be24((channel->cr_token_rate.whole
-+ << 13) |
-+ channel->cr_token_rate.fraction);
-+ config_opts.shaper_config.ertcr =
-+ cpu_to_be24(channel->er_token_rate.whole
-+ << 13 |
-+ channel->er_token_rate.fraction);
-+ config_opts.shaper_config.crtbl =
-+ cpu_to_be16(channel->cr_token_bucket_limit);
-+ config_opts.shaper_config.ertbl =
-+ cpu_to_be16(channel->er_token_bucket_limit);
-+
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_enable_shaper);
-+
-+int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+
-+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
-+ pr_err("Can't query channel mapping\n");
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
-+ channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ config_opts.channel_mapping.map_shaped = 0;
-+ config_opts.channel_mapping.map_lni_id =
-+ query_result.channel_mapping_query.map_lni_id;
-+
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_disable_shaper);
-+
-+int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+
-+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
-+ pr_err("Can't query channel mapping\n");
-+ return -EINVAL;
-+ }
-+
-+ return query_result.channel_mapping_query.map_shaped;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_is_shaper_enabled);
-+
-+int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret) {
-+ pr_err("Fail to get the current channel shaper setting\n");
-+ return -EINVAL;
-+ }
-+
-+ channel->cr_token_rate.whole = token_rate->whole;
-+ channel->cr_token_rate.fraction = token_rate->fraction;
-+ channel->cr_token_bucket_limit = token_limit;
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole
-+ << 13) | (token_rate->fraction));
-+ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit);
-+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
-+ config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr;
-+ config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl;
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate);
-+
-+int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 bps, u16 token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
-+ if (ret) {
-+ pr_err("Can not convert bps to token rate\n");
-+ return -EINVAL;
-+ }
-+ return qman_ceetm_channel_set_commit_rate(channel, &token_rate,
-+ token_limit);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate_bps);
-+
-+int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret | !query_result.shaper_query.crtcr |
-+ !query_result.shaper_query.crtbl) {
-+ pr_err("The channel commit rate or limit is not set\n");
-+ return -EINVAL;
-+ }
-+ token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13;
-+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) &
-+ 0x1FFF;
-+ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate);
-+
-+int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 *bps, u16 *token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_channel_get_commit_rate(channel, &token_rate,
-+ token_limit);
-+ if (ret) {
-+ pr_err("The channel CR rate or limit is not available\n");
-+ return -EINVAL;
-+ }
-+
-+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate_bps);
-+
-+int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret) {
-+ pr_err("Fail to get the current channel shaper setting\n");
-+ return -EINVAL;
-+ }
-+
-+ channel->er_token_rate.whole = token_rate->whole;
-+ channel->er_token_rate.fraction = token_rate->fraction;
-+ channel->er_token_bucket_limit = token_limit;
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ config_opts.shaper_config.ertcr = cpu_to_be24(
-+ (token_rate->whole << 13) | (token_rate->fraction));
-+ config_opts.shaper_config.ertbl = cpu_to_be16(token_limit);
-+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
-+ config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr;
-+ config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl;
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate);
-+
-+int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 bps, u16 token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
-+ if (ret) {
-+ pr_err("Can not convert bps to token rate\n");
-+ return -EINVAL;
-+ }
-+ return qman_ceetm_channel_set_excess_rate(channel, &token_rate,
-+ token_limit);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate_bps);
-+
-+int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret | !query_result.shaper_query.ertcr |
-+ !query_result.shaper_query.ertbl) {
-+ pr_err("The channel excess rate or limit is not set\n");
-+ return -EINVAL;
-+ }
-+ token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13;
-+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) &
-+ 0x1FFF;
-+ *token_limit = be16_to_cpu(query_result.shaper_query.ertbl);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate);
-+
-+int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 *bps, u16 *token_limit)
-+{
-+ struct qm_ceetm_rate token_rate;
-+ int ret;
-+
-+ ret = qman_ceetm_channel_get_excess_rate(channel, &token_rate,
-+ token_limit);
-+ if (ret) {
-+ pr_err("The channel ER rate or limit is not available\n");
-+ return -EINVAL;
-+ }
-+
-+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate_bps);
-+
-+int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel,
-+ u16 token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
-+
-+ if (channel->shaper_enable) {
-+ pr_err("This channel is a shaped one\n");
-+ return -EINVAL;
-+ }
-+
-+ channel->cr_token_bucket_limit = token_limit;
-+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit);
-+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_weight);
-+
-+int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel,
-+ u16 *token_limit)
-+{
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
-+ int ret;
-+
-+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
-+ channel->idx);
-+ query_opts.dcpid = channel->dcp_idx;
-+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
-+ if (ret | !query_result.shaper_query.crtbl) {
-+ pr_err("This unshaped channel's uFQ wight is unavailable\n");
-+ return -EINVAL;
-+ }
-+ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_get_weight);
-+
-+int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, int group_b,
-+ unsigned int prio_a, unsigned int prio_b)
-+{
-+ struct qm_mcc_ceetm_class_scheduler_config config_opts;
-+ struct qm_mcr_ceetm_class_scheduler_query query_result;
-+ int i;
-+
-+ if (prio_a > 7) {
-+ pr_err("The priority of group A is out of range\n");
-+ return -EINVAL;
-+ }
-+ if (group_b && (prio_b > 7)) {
-+ pr_err("The priority of group B is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ if (qman_ceetm_query_class_scheduler(channel, &query_result)) {
-+ pr_err("Can't query channel#%d's scheduler!\n", channel->idx);
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cqcid = cpu_to_be16(channel->idx);
-+ config_opts.dcpid = channel->dcp_idx;
-+ config_opts.gpc_combine_flag = !group_b;
-+ config_opts.gpc_prio_a = prio_a;
-+ config_opts.gpc_prio_b = prio_b;
-+
-+ for (i = 0; i < 8; i++)
-+ config_opts.w[i] = query_result.w[i];
-+ config_opts.crem = query_result.crem;
-+ config_opts.erem = query_result.erem;
-+
-+ return qman_ceetm_configure_class_scheduler(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_group);
-+
-+int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, int *group_b,
-+ unsigned int *prio_a, unsigned int *prio_b)
-+{
-+ struct qm_mcr_ceetm_class_scheduler_query query_result;
-+
-+ if (qman_ceetm_query_class_scheduler(channel, &query_result)) {
-+ pr_err("Can't query channel#%d's scheduler!\n", channel->idx);
-+ return -EINVAL;
-+ }
-+ *group_b = !query_result.gpc_combine_flag;
-+ *prio_a = query_result.gpc_prio_a;
-+ *prio_b = query_result.gpc_prio_b;
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_get_group);
-+
-+#define GROUP_A_ELIGIBILITY_SET (1 << 8)
-+#define GROUP_B_ELIGIBILITY_SET (1 << 9)
-+#define CQ_ELIGIBILITY_SET(n) (1 << (7 - n))
-+int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel
-+ *channel, int group_b, int cre)
-+{
-+ struct qm_mcc_ceetm_class_scheduler_config csch_config;
-+ struct qm_mcr_ceetm_class_scheduler_query csch_query;
-+ int i;
-+
-+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
-+ pr_err("Cannot get the channel %d scheduler setting.\n",
-+ channel->idx);
-+ return -EINVAL;
-+ }
-+ csch_config.cqcid = cpu_to_be16(channel->idx);
-+ csch_config.dcpid = channel->dcp_idx;
-+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
-+ csch_config.gpc_prio_a = csch_query.gpc_prio_a;
-+ csch_config.gpc_prio_b = csch_query.gpc_prio_b;
-+
-+ for (i = 0; i < 8; i++)
-+ csch_config.w[i] = csch_query.w[i];
-+ csch_config.erem = csch_query.erem;
-+ if (group_b)
-+ csch_config.crem = (be16_to_cpu(csch_query.crem)
-+ & ~GROUP_B_ELIGIBILITY_SET)
-+ | (cre ? GROUP_B_ELIGIBILITY_SET : 0);
-+ else
-+ csch_config.crem = (be16_to_cpu(csch_query.crem)
-+ & ~GROUP_A_ELIGIBILITY_SET)
-+ | (cre ? GROUP_A_ELIGIBILITY_SET : 0);
-+
-+ csch_config.crem = cpu_to_be16(csch_config.crem);
-+
-+ if (qman_ceetm_configure_class_scheduler(&csch_config)) {
-+ pr_err("Cannot config channel %d's scheduler with "
-+ "group_%c's cr eligibility\n", channel->idx,
-+ group_b ? 'b' : 'a');
-+ return -EINVAL;
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_group_cr_eligibility);
-+
-+int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel
-+ *channel, int group_b, int ere)
-+{
-+ struct qm_mcc_ceetm_class_scheduler_config csch_config;
-+ struct qm_mcr_ceetm_class_scheduler_query csch_query;
-+ int i;
-+
-+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
-+ pr_err("Cannot get the channel %d scheduler setting.\n",
-+ channel->idx);
-+ return -EINVAL;
-+ }
-+ csch_config.cqcid = cpu_to_be16(channel->idx);
-+ csch_config.dcpid = channel->dcp_idx;
-+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
-+ csch_config.gpc_prio_a = csch_query.gpc_prio_a;
-+ csch_config.gpc_prio_b = csch_query.gpc_prio_b;
-+
-+ for (i = 0; i < 8; i++)
-+ csch_config.w[i] = csch_query.w[i];
-+ csch_config.crem = csch_query.crem;
-+ if (group_b)
-+ csch_config.erem = (be16_to_cpu(csch_query.erem)
-+ & ~GROUP_B_ELIGIBILITY_SET)
-+ | (ere ? GROUP_B_ELIGIBILITY_SET : 0);
-+ else
-+ csch_config.erem = (be16_to_cpu(csch_query.erem)
-+ & ~GROUP_A_ELIGIBILITY_SET)
-+ | (ere ? GROUP_A_ELIGIBILITY_SET : 0);
-+
-+ csch_config.erem = cpu_to_be16(csch_config.erem);
-+
-+ if (qman_ceetm_configure_class_scheduler(&csch_config)) {
-+ pr_err("Cannot config channel %d's scheduler with "
-+ "group_%c's er eligibility\n", channel->idx,
-+ group_b ? 'b' : 'a');
-+ return -EINVAL;
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_group_er_eligibility);
-+
-+int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel,
-+ unsigned int idx, int cre)
-+{
-+ struct qm_mcc_ceetm_class_scheduler_config csch_config;
-+ struct qm_mcr_ceetm_class_scheduler_query csch_query;
-+ int i;
-+
-+ if (idx > 7) {
-+ pr_err("CQ index is out of range\n");
-+ return -EINVAL;
-+ }
-+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
-+ pr_err("Cannot get the channel %d scheduler setting.\n",
-+ channel->idx);
-+ return -EINVAL;
-+ }
-+ csch_config.cqcid = cpu_to_be16(channel->idx);
-+ csch_config.dcpid = channel->dcp_idx;
-+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
-+ csch_config.gpc_prio_a = csch_query.gpc_prio_a;
-+ csch_config.gpc_prio_b = csch_query.gpc_prio_b;
-+ for (i = 0; i < 8; i++)
-+ csch_config.w[i] = csch_query.w[i];
-+ csch_config.erem = csch_query.erem;
-+ csch_config.crem = (be16_to_cpu(csch_query.crem)
-+ & ~CQ_ELIGIBILITY_SET(idx)) |
-+ (cre ? CQ_ELIGIBILITY_SET(idx) : 0);
-+ csch_config.crem = cpu_to_be16(csch_config.crem);
-+ if (qman_ceetm_configure_class_scheduler(&csch_config)) {
-+ pr_err("Cannot config channel scheduler to set "
-+ "cr eligibility mask for CQ#%d\n", idx);
-+ return -EINVAL;
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_cq_cr_eligibility);
-+
-+int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel,
-+ unsigned int idx, int ere)
-+{
-+ struct qm_mcc_ceetm_class_scheduler_config csch_config;
-+ struct qm_mcr_ceetm_class_scheduler_query csch_query;
-+ int i;
-+
-+ if (idx > 7) {
-+ pr_err("CQ index is out of range\n");
-+ return -EINVAL;
-+ }
-+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
-+ pr_err("Cannot get the channel %d scheduler setting.\n",
-+ channel->idx);
-+ return -EINVAL;
-+ }
-+ csch_config.cqcid = cpu_to_be16(channel->idx);
-+ csch_config.dcpid = channel->dcp_idx;
-+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
-+ csch_config.gpc_prio_a = csch_query.gpc_prio_a;
-+ csch_config.gpc_prio_b = csch_query.gpc_prio_b;
-+ for (i = 0; i < 8; i++)
-+ csch_config.w[i] = csch_query.w[i];
-+ csch_config.crem = csch_query.crem;
-+ csch_config.erem = (be16_to_cpu(csch_query.erem)
-+ & ~CQ_ELIGIBILITY_SET(idx)) |
-+ (ere ? CQ_ELIGIBILITY_SET(idx) : 0);
-+ csch_config.erem = cpu_to_be16(csch_config.erem);
-+ if (qman_ceetm_configure_class_scheduler(&csch_config)) {
-+ pr_err("Cannot config channel scheduler to set "
-+ "er eligibility mask for CQ#%d\n", idx);
-+ return -EINVAL;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_channel_set_cq_er_eligibility);
-+
-+int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq,
-+ struct qm_ceetm_channel *channel, unsigned int idx,
-+ struct qm_ceetm_ccg *ccg)
-+{
-+ struct qm_ceetm_cq *p;
-+ struct qm_mcc_ceetm_cq_config cq_config;
-+
-+ if (idx > 7) {
-+ pr_err("The independent class queue id is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ list_for_each_entry(p, &channel->class_queues, node) {
-+ if (p->idx == idx) {
-+ pr_err("The CQ#%d has been claimed!\n", idx);
-+ return -EINVAL;
-+ }
-+ }
-+
-+ p = kmalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p) {
-+ pr_err("Can't allocate memory for CQ#%d!\n", idx);
-+ return -ENOMEM;
-+ }
-+
-+ list_add_tail(&p->node, &channel->class_queues);
-+ p->idx = idx;
-+ p->is_claimed = 1;
-+ p->parent = channel;
-+ INIT_LIST_HEAD(&p->bound_lfqids);
-+
-+ if (ccg) {
-+ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx);
-+ cq_config.dcpid = channel->dcp_idx;
-+ cq_config.ccgid = cpu_to_be16(ccg->idx);
-+ if (qman_ceetm_configure_cq(&cq_config)) {
-+ pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
-+ idx, ccg->idx);
-+ list_del(&p->node);
-+ kfree(p);
-+ return -EINVAL;
-+ }
-+ }
-+
-+ *cq = p;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cq_claim);
-+
-+int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq,
-+ struct qm_ceetm_channel *channel, unsigned int idx,
-+ struct qm_ceetm_ccg *ccg)
-+{
-+ struct qm_ceetm_cq *p;
-+ struct qm_mcc_ceetm_cq_config cq_config;
-+
-+ if ((idx < 8) || (idx > 15)) {
-+ pr_err("This grouped class queue id is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ list_for_each_entry(p, &channel->class_queues, node) {
-+ if (p->idx == idx) {
-+ pr_err("The CQ#%d has been claimed!\n", idx);
-+ return -EINVAL;
-+ }
-+ }
-+
-+ p = kmalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p) {
-+ pr_err("Can't allocate memory for CQ#%d!\n", idx);
-+ return -ENOMEM;
-+ }
-+
-+ list_add_tail(&p->node, &channel->class_queues);
-+ p->idx = idx;
-+ p->is_claimed = 1;
-+ p->parent = channel;
-+ INIT_LIST_HEAD(&p->bound_lfqids);
-+
-+ if (ccg) {
-+ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx);
-+ cq_config.dcpid = channel->dcp_idx;
-+ cq_config.ccgid = cpu_to_be16(ccg->idx);
-+ if (qman_ceetm_configure_cq(&cq_config)) {
-+ pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
-+ idx, ccg->idx);
-+ list_del(&p->node);
-+ kfree(p);
-+ return -EINVAL;
-+ }
-+ }
-+ *cq = p;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cq_claim_A);
-+
-+int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq,
-+ struct qm_ceetm_channel *channel, unsigned int idx,
-+ struct qm_ceetm_ccg *ccg)
-+{
-+ struct qm_ceetm_cq *p;
-+ struct qm_mcc_ceetm_cq_config cq_config;
-+
-+ if ((idx < 12) || (idx > 15)) {
-+ pr_err("This grouped class queue id is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ list_for_each_entry(p, &channel->class_queues, node) {
-+ if (p->idx == idx) {
-+ pr_err("The CQ#%d has been claimed!\n", idx);
-+ return -EINVAL;
-+ }
-+ }
-+
-+ p = kmalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p) {
-+ pr_err("Can't allocate memory for CQ#%d!\n", idx);
-+ return -ENOMEM;
-+ }
-+
-+ list_add_tail(&p->node, &channel->class_queues);
-+ p->idx = idx;
-+ p->is_claimed = 1;
-+ p->parent = channel;
-+ INIT_LIST_HEAD(&p->bound_lfqids);
-+
-+ if (ccg) {
-+ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx);
-+ cq_config.dcpid = channel->dcp_idx;
-+ cq_config.ccgid = cpu_to_be16(ccg->idx);
-+ if (qman_ceetm_configure_cq(&cq_config)) {
-+ pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
-+ idx, ccg->idx);
-+ list_del(&p->node);
-+ kfree(p);
-+ return -EINVAL;
-+ }
-+ }
-+ *cq = p;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cq_claim_B);
-+
-+int qman_ceetm_cq_release(struct qm_ceetm_cq *cq)
-+{
-+ if (!list_empty(&cq->bound_lfqids)) {
-+ pr_err("The CQ#%d has unreleased LFQID\n", cq->idx);
-+ return -EBUSY;
-+ }
-+ list_del(&cq->node);
-+ qman_ceetm_drain_cq(cq);
-+ kfree(cq);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cq_release);
-+
-+int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq,
-+ struct qm_ceetm_weight_code *weight_code)
-+{
-+ struct qm_mcc_ceetm_class_scheduler_config config_opts;
-+ struct qm_mcr_ceetm_class_scheduler_query query_result;
-+ int i;
-+
-+ if (cq->idx < 8) {
-+ pr_err("Can not set weight for ungrouped class queue\n");
-+ return -EINVAL;
-+ }
-+
-+ if (qman_ceetm_query_class_scheduler(cq->parent, &query_result)) {
-+ pr_err("Can't query channel#%d's scheduler!\n",
-+ cq->parent->idx);
-+ return -EINVAL;
-+ }
-+
-+ config_opts.cqcid = cpu_to_be16(cq->parent->idx);
-+ config_opts.dcpid = cq->parent->dcp_idx;
-+ config_opts.crem = query_result.crem;
-+ config_opts.erem = query_result.erem;
-+ config_opts.gpc_combine_flag = query_result.gpc_combine_flag;
-+ config_opts.gpc_prio_a = query_result.gpc_prio_a;
-+ config_opts.gpc_prio_b = query_result.gpc_prio_b;
-+
-+ for (i = 0; i < 8; i++)
-+ config_opts.w[i] = query_result.w[i];
-+ config_opts.w[cq->idx - 8] = ((weight_code->y << 3) |
-+ (weight_code->x & 0x7));
-+ return qman_ceetm_configure_class_scheduler(&config_opts);
-+}
-+EXPORT_SYMBOL(qman_ceetm_set_queue_weight);
-+
-+int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq,
-+ struct qm_ceetm_weight_code *weight_code)
-+{
-+ struct qm_mcr_ceetm_class_scheduler_query query_result;
-+
-+ if (cq->idx < 8) {
-+ pr_err("Can not get weight for ungrouped class queue\n");
-+ return -EINVAL;
-+ }
-+
-+ if (qman_ceetm_query_class_scheduler(cq->parent,
-+ &query_result)) {
-+ pr_err("Can't get the weight code for CQ#%d!\n", cq->idx);
-+ return -EINVAL;
-+ }
-+ weight_code->y = query_result.w[cq->idx - 8] >> 3;
-+ weight_code->x = query_result.w[cq->idx - 8] & 0x7;
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_get_queue_weight);
-+
-+/* The WBFS code is represent as {x,y}, the effect wieght can be calculated as:
-+ * effective weight = 2^x / (1 - (y/64))
-+ * = 2^(x+6) / (64 - y)
-+ */
-+static void reduce_fraction(u32 *n, u32 *d)
-+{
-+ u32 factor = 2;
-+ u32 lesser = (*n < *d) ? *n : *d;
-+ /* If factor exceeds the square-root of the lesser of *n and *d,
-+ * then there's no point continuing. Proof: if there was a factor
-+ * bigger than the square root, that would imply there exists
-+ * another factor smaller than the square-root with which it
-+ * multiplies to give 'lesser' - but that's a contradiction
-+ * because the other factor would have already been found and
-+ * divided out.
-+ */
-+ while ((factor * factor) <= lesser) {
-+ /* If 'factor' is a factor of *n and *d, divide them both
-+ * by 'factor' as many times as possible.
-+ */
-+ while (!(*n % factor) && !(*d % factor)) {
-+ *n /= factor;
-+ *d /= factor;
-+ lesser /= factor;
-+ }
-+ if (factor == 2)
-+ factor = 3;
-+ else
-+ factor += 2;
-+ }
-+}
-+
-+int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code,
-+ u32 *numerator,
-+ u32 *denominator)
-+{
-+ *numerator = (u32) 1 << (weight_code->x + 6);
-+ *denominator = 64 - weight_code->y;
-+ reduce_fraction(numerator, denominator);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_wbfs2ratio);
-+
-+/* For a given x, the weight is between 2^x (inclusive) and 2^(x+1) (exclusive).
-+ * So find 'x' by range, and then estimate 'y' using:
-+ * 64 - y = 2^(x + 6) / weight
-+ * = 2^(x + 6) / (n/d)
-+ * = d * 2^(x+6) / n
-+ * y = 64 - (d * 2^(x+6) / n)
-+ */
-+int qman_ceetm_ratio2wbfs(u32 numerator,
-+ u32 denominator,
-+ struct qm_ceetm_weight_code *weight_code,
-+ int rounding)
-+{
-+ unsigned int y, x = 0;
-+ /* search incrementing 'x' until:
-+ * weight < 2^(x+1)
-+ * n/d < 2^(x+1)
-+ * n < d * 2^(x+1)
-+ */
-+ while ((x < 8) && (numerator >= (denominator << (x + 1))))
-+ x++;
-+ if (x >= 8)
-+ return -ERANGE;
-+ /* because of the subtraction, use '-rounding' */
-+ y = 64 - ROUNDING(denominator << (x + 6), numerator, -rounding);
-+ if (y >= 32)
-+ return -ERANGE;
-+ weight_code->x = x;
-+ weight_code->y = y;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_ratio2wbfs);
-+
-+int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio)
-+{
-+ struct qm_ceetm_weight_code weight_code;
-+
-+ if (qman_ceetm_ratio2wbfs(ratio, 100, &weight_code, 0)) {
-+ pr_err("Cannot get wbfs code for cq %x\n", cq->idx);
-+ return -EINVAL;
-+ }
-+ return qman_ceetm_set_queue_weight(cq, &weight_code);
-+}
-+EXPORT_SYMBOL(qman_ceetm_set_queue_weight_in_ratio);
-+
-+int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio)
-+{
-+ struct qm_ceetm_weight_code weight_code;
-+ u32 n, d;
-+
-+ if (qman_ceetm_get_queue_weight(cq, &weight_code)) {
-+ pr_err("Cannot query the weight code for cq%x\n", cq->idx);
-+ return -EINVAL;
-+ }
-+
-+ if (qman_ceetm_wbfs2ratio(&weight_code, &n, &d)) {
-+ pr_err("Cannot get the ratio with wbfs code\n");
-+ return -EINVAL;
-+ }
-+
-+ *ratio = (n * 100) / d;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_get_queue_weight_in_ratio);
-+
-+int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags,
-+ u64 *frame_count, u64 *byte_count)
-+{
-+ struct qm_mcr_ceetm_statistics_query result;
-+ u16 cid, command_type;
-+ enum qm_dc_portal dcp_idx;
-+ int ret;
-+
-+ cid = cpu_to_be16((cq->parent->idx << 4) | cq->idx);
-+ dcp_idx = cq->parent->dcp_idx;
-+ if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER)
-+ command_type = CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS;
-+ else
-+ command_type = CEETM_QUERY_DEQUEUE_STATISTICS;
-+
-+ ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result);
-+ if (ret) {
-+ pr_err("Can't query the statistics of CQ#%d!\n", cq->idx);
-+ return -EINVAL;
-+ }
-+
-+ *frame_count = be40_to_cpu(result.frm_cnt);
-+ *byte_count = be48_to_cpu(result.byte_cnt);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cq_get_dequeue_statistics);
-+
-+int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq)
-+{
-+ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread ppxr;
-+ int ret;
-+
-+ do {
-+ ret = qman_ceetm_cq_peek_pop_xsfdrread(cq, 1, 0, &ppxr);
-+ if (ret) {
-+ pr_err("Failed to pop frame from CQ\n");
-+ return -EINVAL;
-+ }
-+ } while (!(ppxr.stat & 0x2));
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_drain_cq);
-+
-+#define CEETM_LFQMT_LFQID_MSB 0xF00000
-+#define CEETM_LFQMT_LFQID_LSB 0x000FFF
-+int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq,
-+ struct qm_ceetm_cq *cq)
-+{
-+ struct qm_ceetm_lfq *p;
-+ u32 lfqid;
-+ int ret = 0;
-+ struct qm_mcc_ceetm_lfqmt_config lfqmt_config;
-+
-+ if (cq->parent->dcp_idx == qm_dc_portal_fman0) {
-+ ret = qman_alloc_ceetm0_lfqid(&lfqid);
-+ } else if (cq->parent->dcp_idx == qm_dc_portal_fman1) {
-+ ret = qman_alloc_ceetm1_lfqid(&lfqid);
-+ } else {
-+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
-+ cq->parent->dcp_idx);
-+ return -EINVAL;
-+ }
-+
-+ if (ret) {
-+ pr_err("There is no lfqid avalaible for CQ#%d!\n", cq->idx);
-+ return -ENODEV;
-+ }
-+ p = kmalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p)
-+ return -ENOMEM;
-+ p->idx = lfqid;
-+ p->dctidx = (u16)(lfqid & CEETM_LFQMT_LFQID_LSB);
-+ p->parent = cq->parent;
-+ list_add_tail(&p->node, &cq->bound_lfqids);
-+
-+ lfqmt_config.lfqid = cpu_to_be24(CEETM_LFQMT_LFQID_MSB |
-+ (cq->parent->dcp_idx << 16) |
-+ (lfqid & CEETM_LFQMT_LFQID_LSB));
-+ lfqmt_config.cqid = cpu_to_be16((cq->parent->idx << 4) | (cq->idx));
-+ lfqmt_config.dctidx = cpu_to_be16(p->dctidx);
-+ if (qman_ceetm_configure_lfqmt(&lfqmt_config)) {
-+ pr_err("Can't configure LFQMT for LFQID#%d @ CQ#%d\n",
-+ lfqid, cq->idx);
-+ list_del(&p->node);
-+ kfree(p);
-+ return -EINVAL;
-+ }
-+ *lfq = p;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lfq_claim);
-+
-+int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq)
-+{
-+ if (lfq->parent->dcp_idx == qm_dc_portal_fman0) {
-+ qman_release_ceetm0_lfqid(lfq->idx);
-+ } else if (lfq->parent->dcp_idx == qm_dc_portal_fman1) {
-+ qman_release_ceetm1_lfqid(lfq->idx);
-+ } else {
-+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
-+ lfq->parent->dcp_idx);
-+ return -EINVAL;
-+ }
-+ list_del(&lfq->node);
-+ kfree(lfq);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lfq_release);
-+
-+int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, u64 context_a,
-+ u32 context_b)
-+{
-+ struct qm_mcc_ceetm_dct_config dct_config;
-+ lfq->context_a = context_a;
-+ lfq->context_b = context_b;
-+ dct_config.dctidx = cpu_to_be16((u16)lfq->dctidx);
-+ dct_config.dcpid = lfq->parent->dcp_idx;
-+ dct_config.context_b = cpu_to_be32(context_b);
-+ dct_config.context_a = cpu_to_be64(context_a);
-+
-+ return qman_ceetm_configure_dct(&dct_config);
-+}
-+EXPORT_SYMBOL(qman_ceetm_lfq_set_context);
-+
-+int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, u64 *context_a,
-+ u32 *context_b)
-+{
-+ struct qm_mcc_ceetm_dct_query dct_query;
-+ struct qm_mcr_ceetm_dct_query query_result;
-+
-+ dct_query.dctidx = cpu_to_be16(lfq->dctidx);
-+ dct_query.dcpid = lfq->parent->dcp_idx;
-+ if (qman_ceetm_query_dct(&dct_query, &query_result)) {
-+ pr_err("Can't query LFQID#%d's context!\n", lfq->idx);
-+ return -EINVAL;
-+ }
-+ *context_a = be64_to_cpu(query_result.context_a);
-+ *context_b = be32_to_cpu(query_result.context_b);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_lfq_get_context);
-+
-+int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq)
-+{
-+ spin_lock_init(&fq->fqlock);
-+ fq->fqid = lfq->idx;
-+ fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
-+ if (lfq->ern)
-+ fq->cb.ern = lfq->ern;
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ if (unlikely(find_empty_fq_table_entry(&fq->key, fq)))
-+ return -ENOMEM;
-+#endif
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_create_fq);
-+
-+#define MAX_CCG_IDX 0x000F
-+int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg,
-+ struct qm_ceetm_channel *channel,
-+ unsigned int idx,
-+ void (*cscn)(struct qm_ceetm_ccg *,
-+ void *cb_ctx,
-+ int congested),
-+ void *cb_ctx)
-+{
-+ struct qm_ceetm_ccg *p;
-+
-+ if (idx > MAX_CCG_IDX) {
-+ pr_err("The given ccg index is out of range\n");
-+ return -EINVAL;
-+ }
-+
-+ list_for_each_entry(p, &channel->ccgs, node) {
-+ if (p->idx == idx) {
-+ pr_err("The CCG#%d has been claimed\n", idx);
-+ return -EINVAL;
-+ }
-+ }
-+
-+ p = kmalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p) {
-+ pr_err("Can't allocate memory for CCG#%d!\n", idx);
-+ return -ENOMEM;
-+ }
-+
-+ list_add_tail(&p->node, &channel->ccgs);
-+
-+ p->idx = idx;
-+ p->parent = channel;
-+ p->cb = cscn;
-+ p->cb_ctx = cb_ctx;
-+ INIT_LIST_HEAD(&p->cb_node);
-+
-+ *ccg = p;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_ccg_claim);
-+
-+int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg)
-+{
-+ unsigned long irqflags __maybe_unused;
-+ struct qm_mcc_ceetm_ccgr_config config_opts;
-+ int ret = 0;
-+ struct qman_portal *p = get_affine_portal();
-+
-+ memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config));
-+ spin_lock_irqsave(&p->ccgr_lock, irqflags);
-+ if (!list_empty(&ccg->cb_node))
-+ list_del(&ccg->cb_node);
-+ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE |
-+ (ccg->parent->idx << 4) | ccg->idx);
-+ config_opts.dcpid = ccg->parent->dcp_idx;
-+ config_opts.we_mask = cpu_to_be16(QM_CCGR_WE_CSCN_TUPD);
-+ config_opts.cm_config.cscn_tupd = cpu_to_be16(PORTAL_IDX(p));
-+ ret = qman_ceetm_configure_ccgr(&config_opts);
-+ spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
-+ put_affine_portal();
-+
-+ list_del(&ccg->node);
-+ kfree(ccg);
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_ceetm_ccg_release);
-+
-+int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, u16 we_mask,
-+ const struct qm_ceetm_ccg_params *params)
-+{
-+ struct qm_mcc_ceetm_ccgr_config config_opts;
-+ unsigned long irqflags __maybe_unused;
-+ int ret;
-+ struct qman_portal *p;
-+
-+ if (((ccg->parent->idx << 4) | ccg->idx) >= (2 * __CGR_NUM))
-+ return -EINVAL;
-+
-+ p = get_affine_portal();
-+
-+ memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config));
-+ spin_lock_irqsave(&p->ccgr_lock, irqflags);
-+
-+ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE |
-+ (ccg->parent->idx << 4) | ccg->idx);
-+ config_opts.dcpid = ccg->parent->dcp_idx;
-+ config_opts.we_mask = we_mask;
-+ if (we_mask & QM_CCGR_WE_CSCN_EN) {
-+ config_opts.we_mask |= QM_CCGR_WE_CSCN_TUPD;
-+ config_opts.cm_config.cscn_tupd = cpu_to_be16(
-+ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p));
-+ }
-+ config_opts.we_mask = cpu_to_be16(config_opts.we_mask);
-+ config_opts.cm_config.ctl_wr_en_g = params->wr_en_g;
-+ config_opts.cm_config.ctl_wr_en_y = params->wr_en_y;
-+ config_opts.cm_config.ctl_wr_en_r = params->wr_en_r;
-+ config_opts.cm_config.ctl_td_en = params->td_en;
-+ config_opts.cm_config.ctl_td_mode = params->td_mode;
-+ config_opts.cm_config.ctl_cscn_en = params->cscn_en;
-+ config_opts.cm_config.ctl_mode = params->mode;
-+ config_opts.cm_config.oal = params->oal;
-+ config_opts.cm_config.cs_thres.hword =
-+ cpu_to_be16(params->cs_thres_in.hword);
-+ config_opts.cm_config.cs_thres_x.hword =
-+ cpu_to_be16(params->cs_thres_out.hword);
-+ config_opts.cm_config.td_thres.hword =
-+ cpu_to_be16(params->td_thres.hword);
-+ config_opts.cm_config.wr_parm_g.word =
-+ cpu_to_be32(params->wr_parm_g.word);
-+ config_opts.cm_config.wr_parm_y.word =
-+ cpu_to_be32(params->wr_parm_y.word);
-+ config_opts.cm_config.wr_parm_r.word =
-+ cpu_to_be32(params->wr_parm_r.word);
-+ ret = qman_ceetm_configure_ccgr(&config_opts);
-+ if (ret) {
-+ pr_err("Configure CCGR CM failed!\n");
-+ goto release_lock;
-+ }
-+
-+ if (we_mask & QM_CCGR_WE_CSCN_EN)
-+ if (list_empty(&ccg->cb_node))
-+ list_add(&ccg->cb_node,
-+ &p->ccgr_cbs[ccg->parent->dcp_idx]);
-+release_lock:
-+ spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_ceetm_ccg_set);
-+
-+#define CEETM_CCGR_CTL_MASK 0x01
-+int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg,
-+ struct qm_ceetm_ccg_params *params)
-+{
-+ struct qm_mcc_ceetm_ccgr_query query_opts;
-+ struct qm_mcr_ceetm_ccgr_query query_result;
-+
-+ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY |
-+ (ccg->parent->idx << 4) | ccg->idx);
-+ query_opts.dcpid = ccg->parent->dcp_idx;
-+
-+ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
-+ pr_err("Can't query CCGR#%d\n", ccg->idx);
-+ return -EINVAL;
-+ }
-+
-+ params->wr_parm_r.word = query_result.cm_query.wr_parm_r.word;
-+ params->wr_parm_y.word = query_result.cm_query.wr_parm_y.word;
-+ params->wr_parm_g.word = query_result.cm_query.wr_parm_g.word;
-+ params->td_thres.hword = query_result.cm_query.td_thres.hword;
-+ params->cs_thres_out.hword = query_result.cm_query.cs_thres_x.hword;
-+ params->cs_thres_in.hword = query_result.cm_query.cs_thres.hword;
-+ params->oal = query_result.cm_query.oal;
-+ params->wr_en_g = query_result.cm_query.ctl_wr_en_g;
-+ params->wr_en_y = query_result.cm_query.ctl_wr_en_y;
-+ params->wr_en_r = query_result.cm_query.ctl_wr_en_r;
-+ params->td_en = query_result.cm_query.ctl_td_en;
-+ params->td_mode = query_result.cm_query.ctl_td_mode;
-+ params->cscn_en = query_result.cm_query.ctl_cscn_en;
-+ params->mode = query_result.cm_query.ctl_mode;
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_ccg_get);
-+
-+int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags,
-+ u64 *frame_count, u64 *byte_count)
-+{
-+ struct qm_mcr_ceetm_statistics_query result;
-+ u16 cid, command_type;
-+ enum qm_dc_portal dcp_idx;
-+ int ret;
-+
-+ cid = cpu_to_be16((ccg->parent->idx << 4) | ccg->idx);
-+ dcp_idx = ccg->parent->dcp_idx;
-+ if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER)
-+ command_type = CEETM_QUERY_REJECT_CLEAR_STATISTICS;
-+ else
-+ command_type = CEETM_QUERY_REJECT_STATISTICS;
-+
-+ ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result);
-+ if (ret) {
-+ pr_err("Can't query the statistics of CCG#%d!\n", ccg->idx);
-+ return -EINVAL;
-+ }
-+
-+ *frame_count = be40_to_cpu(result.frm_cnt);
-+ *byte_count = be48_to_cpu(result.byte_cnt);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_ccg_get_reject_statistics);
-+
-+int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg,
-+ u16 swp_idx,
-+ unsigned int *cscn_enabled)
-+{
-+ struct qm_mcc_ceetm_ccgr_query query_opts;
-+ struct qm_mcr_ceetm_ccgr_query query_result;
-+ int i;
-+
-+ DPA_ASSERT(swp_idx < 127);
-+ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY |
-+ (ccg->parent->idx << 4) | ccg->idx);
-+ query_opts.dcpid = ccg->parent->dcp_idx;
-+
-+ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
-+ pr_err("Can't query CCGR#%d\n", ccg->idx);
-+ return -EINVAL;
-+ }
-+
-+ i = swp_idx / 32;
-+ i = 3 - i;
-+ *cscn_enabled = query_result.cm_query.cscn_targ_swp[i] >>
-+ (31 - swp_idx % 32);
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cscn_swp_get);
-+
-+int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg,
-+ u16 dcp_idx,
-+ u8 vcgid,
-+ unsigned int cscn_enabled,
-+ u16 we_mask,
-+ const struct qm_ceetm_ccg_params *params)
-+{
-+ struct qm_mcc_ceetm_ccgr_config config_opts;
-+ int ret;
-+
-+ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE |
-+ (ccg->parent->idx << 4) | ccg->idx);
-+ config_opts.dcpid = ccg->parent->dcp_idx;
-+ config_opts.we_mask = cpu_to_be16(we_mask | QM_CCGR_WE_CSCN_TUPD |
-+ QM_CCGR_WE_CDV);
-+ config_opts.cm_config.cdv = vcgid;
-+ config_opts.cm_config.cscn_tupd = cpu_to_be16((cscn_enabled << 15) |
-+ QM_CGR_TARG_UDP_CTRL_DCP | dcp_idx);
-+ config_opts.cm_config.ctl_wr_en_g = params->wr_en_g;
-+ config_opts.cm_config.ctl_wr_en_y = params->wr_en_y;
-+ config_opts.cm_config.ctl_wr_en_r = params->wr_en_r;
-+ config_opts.cm_config.ctl_td_en = params->td_en;
-+ config_opts.cm_config.ctl_td_mode = params->td_mode;
-+ config_opts.cm_config.ctl_cscn_en = params->cscn_en;
-+ config_opts.cm_config.ctl_mode = params->mode;
-+ config_opts.cm_config.cs_thres.hword =
-+ cpu_to_be16(params->cs_thres_in.hword);
-+ config_opts.cm_config.cs_thres_x.hword =
-+ cpu_to_be16(params->cs_thres_out.hword);
-+ config_opts.cm_config.td_thres.hword =
-+ cpu_to_be16(params->td_thres.hword);
-+ config_opts.cm_config.wr_parm_g.word =
-+ cpu_to_be32(params->wr_parm_g.word);
-+ config_opts.cm_config.wr_parm_y.word =
-+ cpu_to_be32(params->wr_parm_y.word);
-+ config_opts.cm_config.wr_parm_r.word =
-+ cpu_to_be32(params->wr_parm_r.word);
-+
-+ ret = qman_ceetm_configure_ccgr(&config_opts);
-+ if (ret) {
-+ pr_err("Configure CSCN_TARG_DCP failed!\n");
-+ return -EINVAL;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cscn_dcp_set);
-+
-+int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg,
-+ u16 dcp_idx,
-+ u8 *vcgid,
-+ unsigned int *cscn_enabled)
-+{
-+ struct qm_mcc_ceetm_ccgr_query query_opts;
-+ struct qm_mcr_ceetm_ccgr_query query_result;
-+
-+ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY |
-+ (ccg->parent->idx << 4) | ccg->idx);
-+ query_opts.dcpid = ccg->parent->dcp_idx;
-+
-+ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
-+ pr_err("Can't query CCGR#%d\n", ccg->idx);
-+ return -EINVAL;
-+ }
-+
-+ *vcgid = query_result.cm_query.cdv;
-+ *cscn_enabled = (query_result.cm_query.cscn_targ_dcp >> dcp_idx) & 0x1;
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_ceetm_cscn_dcp_get);
-+
-+int qman_ceetm_querycongestion(struct __qm_mcr_querycongestion *ccg_state,
-+ unsigned int dcp_idx)
-+{
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ u8 res;
-+ int i, j;
-+
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+
-+ mcc = qm_mc_start(&p->p);
-+ for (i = 0; i < 2; i++) {
-+ mcc->ccgr_query.ccgrid =
-+ cpu_to_be16(CEETM_QUERY_CONGESTION_STATE | i);
-+ mcc->ccgr_query.dcpid = dcp_idx;
-+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
-+
-+ while (!(mcr = qm_mc_result(&p->p)))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_CEETM_VERB_CCGR_QUERY);
-+ res = mcr->result;
-+ if (res == QM_MCR_RESULT_OK) {
-+ for (j = 0; j < 8; j++)
-+ mcr->ccgr_query.congestion_state.state.
-+ __state[j] = be32_to_cpu(mcr->ccgr_query.
-+ congestion_state.state.__state[j]);
-+ *(ccg_state + i) =
-+ mcr->ccgr_query.congestion_state.state;
-+ } else {
-+ pr_err("QUERY CEETM CONGESTION STATE failed\n");
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ return -EIO;
-+ }
-+ }
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return 0;
-+}
-+
-+int qman_set_wpm(int wpm_enable)
-+{
-+ return qm_set_wpm(wpm_enable);
-+}
-+EXPORT_SYMBOL(qman_set_wpm);
-+
-+int qman_get_wpm(int *wpm_enable)
-+{
-+ return qm_get_wpm(wpm_enable);
-+}
-+EXPORT_SYMBOL(qman_get_wpm);
-+
-+int qman_shutdown_fq(u32 fqid)
-+{
-+ struct qman_portal *p;
-+ unsigned long irqflags __maybe_unused;
-+ int ret;
-+ struct qm_portal *low_p;
-+ p = get_affine_portal();
-+ PORTAL_IRQ_LOCK(p, irqflags);
-+ low_p = &p->p;
-+ ret = qm_shutdown_fq(&low_p, 1, fqid);
-+ PORTAL_IRQ_UNLOCK(p, irqflags);
-+ put_affine_portal();
-+ return ret;
-+}
-+
-+const struct qm_portal_config *qman_get_qm_portal_config(
-+ struct qman_portal *portal)
-+{
-+ return portal->sharing_redirect ? NULL : portal->config;
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_low.h
-@@ -0,0 +1,1427 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "qman_private.h"
-+
-+/***************************/
-+/* Portal register assists */
-+/***************************/
-+
-+/* Cache-inhibited register offsets */
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+
-+#define QM_REG_EQCR_PI_CINH 0x0000
-+#define QM_REG_EQCR_CI_CINH 0x0004
-+#define QM_REG_EQCR_ITR 0x0008
-+#define QM_REG_DQRR_PI_CINH 0x0040
-+#define QM_REG_DQRR_CI_CINH 0x0044
-+#define QM_REG_DQRR_ITR 0x0048
-+#define QM_REG_DQRR_DCAP 0x0050
-+#define QM_REG_DQRR_SDQCR 0x0054
-+#define QM_REG_DQRR_VDQCR 0x0058
-+#define QM_REG_DQRR_PDQCR 0x005c
-+#define QM_REG_MR_PI_CINH 0x0080
-+#define QM_REG_MR_CI_CINH 0x0084
-+#define QM_REG_MR_ITR 0x0088
-+#define QM_REG_CFG 0x0100
-+#define QM_REG_ISR 0x0e00
-+#define QM_REG_IIR 0x0e0c
-+#define QM_REG_ITPR 0x0e14
-+
-+/* Cache-enabled register offsets */
-+#define QM_CL_EQCR 0x0000
-+#define QM_CL_DQRR 0x1000
-+#define QM_CL_MR 0x2000
-+#define QM_CL_EQCR_PI_CENA 0x3000
-+#define QM_CL_EQCR_CI_CENA 0x3100
-+#define QM_CL_DQRR_PI_CENA 0x3200
-+#define QM_CL_DQRR_CI_CENA 0x3300
-+#define QM_CL_MR_PI_CENA 0x3400
-+#define QM_CL_MR_CI_CENA 0x3500
-+#define QM_CL_CR 0x3800
-+#define QM_CL_RR0 0x3900
-+#define QM_CL_RR1 0x3940
-+
-+#endif
-+
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+
-+#define QM_REG_EQCR_PI_CINH 0x3000
-+#define QM_REG_EQCR_CI_CINH 0x3040
-+#define QM_REG_EQCR_ITR 0x3080
-+#define QM_REG_DQRR_PI_CINH 0x3100
-+#define QM_REG_DQRR_CI_CINH 0x3140
-+#define QM_REG_DQRR_ITR 0x3180
-+#define QM_REG_DQRR_DCAP 0x31C0
-+#define QM_REG_DQRR_SDQCR 0x3200
-+#define QM_REG_DQRR_VDQCR 0x3240
-+#define QM_REG_DQRR_PDQCR 0x3280
-+#define QM_REG_MR_PI_CINH 0x3300
-+#define QM_REG_MR_CI_CINH 0x3340
-+#define QM_REG_MR_ITR 0x3380
-+#define QM_REG_CFG 0x3500
-+#define QM_REG_ISR 0x3600
-+#define QM_REG_IIR 0x36C0
-+#define QM_REG_ITPR 0x3740
-+
-+/* Cache-enabled register offsets */
-+#define QM_CL_EQCR 0x0000
-+#define QM_CL_DQRR 0x1000
-+#define QM_CL_MR 0x2000
-+#define QM_CL_EQCR_PI_CENA 0x3000
-+#define QM_CL_EQCR_CI_CENA 0x3040
-+#define QM_CL_DQRR_PI_CENA 0x3100
-+#define QM_CL_DQRR_CI_CENA 0x3140
-+#define QM_CL_MR_PI_CENA 0x3300
-+#define QM_CL_MR_CI_CENA 0x3340
-+#define QM_CL_CR 0x3800
-+#define QM_CL_RR0 0x3900
-+#define QM_CL_RR1 0x3940
-+
-+#endif
-+
-+
-+/* BTW, the drivers (and h/w programming model) already obtain the required
-+ * synchronisation for portal accesses via lwsync(), hwsync(), and
-+ * data-dependencies. Use of barrier()s or other order-preserving primitives
-+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which
-+ * simply ensure that the compiler treats the portal registers as volatile (ie.
-+ * non-coherent). */
-+
-+/* Cache-inhibited register access. */
-+#define __qm_in(qm, o) be32_to_cpu(__raw_readl((qm)->addr_ci + (o)))
-+#define __qm_out(qm, o, val) __raw_writel((cpu_to_be32(val)), \
-+ (qm)->addr_ci + (o));
-+#define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg)
-+#define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val)
-+
-+/* Cache-enabled (index) register access */
-+#define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->addr_ce + (o))
-+#define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->addr_ce + (o))
-+#define __qm_cl_in(qm, o) be32_to_cpu(__raw_readl((qm)->addr_ce + (o)))
-+#define __qm_cl_out(qm, o, val) \
-+ do { \
-+ u32 *__tmpclout = (qm)->addr_ce + (o); \
-+ __raw_writel(cpu_to_be32(val), __tmpclout); \
-+ dcbf(__tmpclout); \
-+ } while (0)
-+#define __qm_cl_invalidate(qm, o) dcbi((qm)->addr_ce + (o))
-+#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA)
-+#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA)
-+#define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA)
-+#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val)
-+#define qm_cl_invalidate(reg)\
-+ __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA)
-+
-+/* Cache-enabled ring access */
-+#define qm_cl(base, idx) ((void *)base + ((idx) << 6))
-+
-+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
-+ * analysis, look at using the "extra" bit in the ring index registers to avoid
-+ * cyclic issues. */
-+static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
-+{
-+ /* 'first' is included, 'last' is excluded */
-+ if (first <= last)
-+ return last - first;
-+ return ringsize + last - first;
-+}
-+
-+/* Portal modes.
-+ * Enum types;
-+ * pmode == production mode
-+ * cmode == consumption mode,
-+ * dmode == h/w dequeue mode.
-+ * Enum values use 3 letter codes. First letter matches the portal mode,
-+ * remaining two letters indicate;
-+ * ci == cache-inhibited portal register
-+ * ce == cache-enabled portal register
-+ * vb == in-band valid-bit (cache-enabled)
-+ * dc == DCA (Discrete Consumption Acknowledgement), DQRR-only
-+ * As for "enum qm_dqrr_dmode", it should be self-explanatory.
-+ */
-+enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */
-+ qm_eqcr_pci = 0, /* PI index, cache-inhibited */
-+ qm_eqcr_pce = 1, /* PI index, cache-enabled */
-+ qm_eqcr_pvb = 2 /* valid-bit */
-+};
-+enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */
-+ qm_dqrr_dpush = 0, /* SDQCR + VDQCR */
-+ qm_dqrr_dpull = 1 /* PDQCR */
-+};
-+enum qm_dqrr_pmode { /* s/w-only */
-+ qm_dqrr_pci, /* reads DQRR_PI_CINH */
-+ qm_dqrr_pce, /* reads DQRR_PI_CENA */
-+ qm_dqrr_pvb /* reads valid-bit */
-+};
-+enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */
-+ qm_dqrr_cci = 0, /* CI index, cache-inhibited */
-+ qm_dqrr_cce = 1, /* CI index, cache-enabled */
-+ qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgement */
-+};
-+enum qm_mr_pmode { /* s/w-only */
-+ qm_mr_pci, /* reads MR_PI_CINH */
-+ qm_mr_pce, /* reads MR_PI_CENA */
-+ qm_mr_pvb /* reads valid-bit */
-+};
-+enum qm_mr_cmode { /* matches QCSP_CFG::MM */
-+ qm_mr_cci = 0, /* CI index, cache-inhibited */
-+ qm_mr_cce = 1 /* CI index, cache-enabled */
-+};
-+
-+
-+/* ------------------------- */
-+/* --- Portal structures --- */
-+
-+#define QM_EQCR_SIZE 8
-+#define QM_DQRR_SIZE 16
-+#define QM_MR_SIZE 8
-+
-+struct qm_eqcr {
-+ struct qm_eqcr_entry *ring, *cursor;
-+ u8 ci, available, ithresh, vbit;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ u32 busy;
-+ enum qm_eqcr_pmode pmode;
-+#endif
-+};
-+
-+struct qm_dqrr {
-+ const struct qm_dqrr_entry *ring, *cursor;
-+ u8 pi, ci, fill, ithresh, vbit;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ enum qm_dqrr_dmode dmode;
-+ enum qm_dqrr_pmode pmode;
-+ enum qm_dqrr_cmode cmode;
-+#endif
-+};
-+
-+struct qm_mr {
-+ const struct qm_mr_entry *ring, *cursor;
-+ u8 pi, ci, fill, ithresh, vbit;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ enum qm_mr_pmode pmode;
-+ enum qm_mr_cmode cmode;
-+#endif
-+};
-+
-+struct qm_mc {
-+ struct qm_mc_command *cr;
-+ struct qm_mc_result *rr;
-+ u8 rridx, vbit;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ enum {
-+ /* Can be _mc_start()ed */
-+ qman_mc_idle,
-+ /* Can be _mc_commit()ed or _mc_abort()ed */
-+ qman_mc_user,
-+ /* Can only be _mc_retry()ed */
-+ qman_mc_hw
-+ } state;
-+#endif
-+};
-+
-+#define QM_PORTAL_ALIGNMENT ____cacheline_aligned
-+
-+struct qm_addr {
-+ void __iomem *addr_ce; /* cache-enabled */
-+ void __iomem *addr_ci; /* cache-inhibited */
-+};
-+
-+struct qm_portal {
-+ /* In the non-CONFIG_FSL_DPA_CHECKING case, the following stuff up to
-+ * and including 'mc' fits within a cacheline (yay!). The 'config' part
-+ * is setup-only, so isn't a cause for a concern. In other words, don't
-+ * rearrange this structure on a whim, there be dragons ... */
-+ struct qm_addr addr;
-+ struct qm_eqcr eqcr;
-+ struct qm_dqrr dqrr;
-+ struct qm_mr mr;
-+ struct qm_mc mc;
-+} QM_PORTAL_ALIGNMENT;
-+
-+
-+/* ---------------- */
-+/* --- EQCR API --- */
-+
-+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
-+#define EQCR_CARRYCLEAR(p) \
-+ (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6)))
-+
-+/* Bit-wise logic to convert a ring pointer to a ring index */
-+static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e)
-+{
-+ return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1);
-+}
-+
-+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
-+static inline void EQCR_INC(struct qm_eqcr *eqcr)
-+{
-+ /* NB: this is odd-looking, but experiments show that it generates fast
-+ * code with essentially no branching overheads. We increment to the
-+ * next EQCR pointer and handle overflow and 'vbit'. */
-+ struct qm_eqcr_entry *partial = eqcr->cursor + 1;
-+ eqcr->cursor = EQCR_CARRYCLEAR(partial);
-+ if (partial != eqcr->cursor)
-+ eqcr->vbit ^= QM_EQCR_VERB_VBIT;
-+}
-+
-+static inline int qm_eqcr_init(struct qm_portal *portal,
-+ enum qm_eqcr_pmode pmode,
-+ unsigned int eq_stash_thresh,
-+ int eq_stash_prio)
-+{
-+ /* This use of 'register', as well as all other occurrences, is because
-+ * it has been observed to generate much faster code with gcc than is
-+ * otherwise the case. */
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ u32 cfg;
-+ u8 pi;
-+
-+ eqcr->ring = portal->addr.addr_ce + QM_CL_EQCR;
-+ eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
-+ qm_cl_invalidate(EQCR_CI);
-+ pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
-+ eqcr->cursor = eqcr->ring + pi;
-+ eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
-+ QM_EQCR_VERB_VBIT : 0;
-+ eqcr->available = QM_EQCR_SIZE - 1 -
-+ qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
-+ eqcr->ithresh = qm_in(EQCR_ITR);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 0;
-+ eqcr->pmode = pmode;
-+#endif
-+ cfg = (qm_in(CFG) & 0x00ffffff) |
-+ (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
-+ (eq_stash_prio << 26) | /* QCSP_CFG: EP */
-+ ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
-+ qm_out(CFG, cfg);
-+ return 0;
-+}
-+
-+static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal)
-+{
-+ return (qm_in(CFG) >> 28) & 0x7;
-+}
-+
-+static inline void qm_eqcr_finish(struct qm_portal *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ u8 pi, ci;
-+ u32 cfg;
-+
-+ /*
-+ * Disable EQCI stashing because the QMan only
-+ * presents the value it previously stashed to
-+ * maintain coherency. Setting the stash threshold
-+ * to 1 then 0 ensures that QMan has resyncronized
-+ * its internal copy so that the portal is clean
-+ * when it is reinitialized in the future
-+ */
-+ cfg = (qm_in(CFG) & 0x0fffffff) |
-+ (1 << 28); /* QCSP_CFG: EST */
-+ qm_out(CFG, cfg);
-+ cfg &= 0x0fffffff; /* stash threshold = 0 */
-+ qm_out(CFG, cfg);
-+
-+ pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
-+ ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
-+
-+ /* Refresh EQCR CI cache value */
-+ qm_cl_invalidate(EQCR_CI);
-+ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
-+
-+ DPA_ASSERT(!eqcr->busy);
-+ if (pi != EQCR_PTR2IDX(eqcr->cursor))
-+ pr_crit("losing uncommited EQCR entries\n");
-+ if (ci != eqcr->ci)
-+ pr_crit("missing existing EQCR completions\n");
-+ if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
-+ pr_crit("EQCR destroyed unquiesced\n");
-+}
-+
-+static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
-+ *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ DPA_ASSERT(!eqcr->busy);
-+ if (!eqcr->available)
-+ return NULL;
-+
-+
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 1;
-+#endif
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(eqcr->cursor);
-+#endif
-+ return eqcr->cursor;
-+}
-+
-+static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
-+ *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ u8 diff, old_ci;
-+
-+ DPA_ASSERT(!eqcr->busy);
-+ if (!eqcr->available) {
-+ old_ci = eqcr->ci;
-+ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
-+ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
-+ eqcr->available += diff;
-+ if (!diff)
-+ return NULL;
-+ }
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 1;
-+#endif
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(eqcr->cursor);
-+#endif
-+ return eqcr->cursor;
-+}
-+
-+static inline void qm_eqcr_abort(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
-+ DPA_ASSERT(eqcr->busy);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 0;
-+#endif
-+}
-+
-+static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next(
-+ struct qm_portal *portal, u8 myverb)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ DPA_ASSERT(eqcr->busy);
-+ DPA_ASSERT(eqcr->pmode != qm_eqcr_pvb);
-+ if (eqcr->available == 1)
-+ return NULL;
-+ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
-+ dcbf(eqcr->cursor);
-+ EQCR_INC(eqcr);
-+ eqcr->available--;
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(eqcr->cursor);
-+#endif
-+ return eqcr->cursor;
-+}
-+
-+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-+#define EQCR_COMMIT_CHECKS(eqcr) \
-+do { \
-+ DPA_ASSERT(eqcr->busy); \
-+ DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0xffffff00)); \
-+ DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0xffffff00)); \
-+} while (0)
-+#else
-+#define EQCR_COMMIT_CHECKS(eqcr) \
-+do { \
-+ DPA_ASSERT(eqcr->busy); \
-+ DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & \
-+ cpu_to_be32(0x00ffffff))); \
-+ DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & \
-+ cpu_to_be32(0x00ffffff))); \
-+} while (0)
-+#endif
-+
-+static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ EQCR_COMMIT_CHECKS(eqcr);
-+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pci);
-+ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
-+ EQCR_INC(eqcr);
-+ eqcr->available--;
-+ dcbf(eqcr->cursor);
-+ hwsync();
-+ qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor));
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 0;
-+#endif
-+}
-+
-+static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
-+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pce);
-+ qm_cl_invalidate(EQCR_PI);
-+ qm_cl_touch_rw(EQCR_PI);
-+}
-+
-+static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ EQCR_COMMIT_CHECKS(eqcr);
-+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pce);
-+ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
-+ EQCR_INC(eqcr);
-+ eqcr->available--;
-+ dcbf(eqcr->cursor);
-+ lwsync();
-+ qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor));
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 0;
-+#endif
-+}
-+
-+static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ struct qm_eqcr_entry *eqcursor;
-+ EQCR_COMMIT_CHECKS(eqcr);
-+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
-+ lwsync();
-+ eqcursor = eqcr->cursor;
-+ eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit;
-+ dcbf(eqcursor);
-+ EQCR_INC(eqcr);
-+ eqcr->available--;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ eqcr->busy = 0;
-+#endif
-+}
-+
-+static inline u8 qm_eqcr_cci_update(struct qm_portal *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ u8 diff, old_ci = eqcr->ci;
-+ eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
-+ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
-+ eqcr->available += diff;
-+ return diff;
-+}
-+
-+static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
-+ qm_cl_touch_ro(EQCR_CI);
-+}
-+
-+static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ u8 diff, old_ci = eqcr->ci;
-+ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
-+ qm_cl_invalidate(EQCR_CI);
-+ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
-+ eqcr->available += diff;
-+ return diff;
-+}
-+
-+static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ return eqcr->ithresh;
-+}
-+
-+static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ eqcr->ithresh = ithresh;
-+ qm_out(EQCR_ITR, ithresh);
-+}
-+
-+static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ return eqcr->available;
-+}
-+
-+static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
-+{
-+ register struct qm_eqcr *eqcr = &portal->eqcr;
-+ return QM_EQCR_SIZE - 1 - eqcr->available;
-+}
-+
-+
-+/* ---------------- */
-+/* --- DQRR API --- */
-+
-+/* FIXME: many possible improvements;
-+ * - look at changing the API to use pointer rather than index parameters now
-+ * that 'cursor' is a pointer,
-+ * - consider moving other parameters to pointer if it could help (ci)
-+ */
-+
-+#define DQRR_CARRYCLEAR(p) \
-+ (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6)))
-+
-+static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e)
-+{
-+ return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1);
-+}
-+
-+static inline const struct qm_dqrr_entry *DQRR_INC(
-+ const struct qm_dqrr_entry *e)
-+{
-+ return DQRR_CARRYCLEAR(e + 1);
-+}
-+
-+static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
-+{
-+ qm_out(CFG, (qm_in(CFG) & 0xff0fffff) |
-+ ((mf & (QM_DQRR_SIZE - 1)) << 20));
-+}
-+
-+static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cci);
-+ dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
-+ qm_out(DQRR_CI_CINH, dqrr->ci);
-+}
-+
-+static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
-+ dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
-+ qm_cl_out(DQRR_CI, dqrr->ci);
-+}
-+
-+static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
-+ qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
-+ ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
-+ dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
-+ dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
-+}
-+
-+static inline int qm_dqrr_init(struct qm_portal *portal,
-+ const struct qm_portal_config *config,
-+ enum qm_dqrr_dmode dmode,
-+ __maybe_unused enum qm_dqrr_pmode pmode,
-+ enum qm_dqrr_cmode cmode, u8 max_fill)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ u32 cfg;
-+
-+ /* Make sure the DQRR will be idle when we enable */
-+ qm_out(DQRR_SDQCR, 0);
-+ qm_out(DQRR_VDQCR, 0);
-+ qm_out(DQRR_PDQCR, 0);
-+ dqrr->ring = portal->addr.addr_ce + QM_CL_DQRR;
-+ dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
-+ dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
-+ dqrr->cursor = dqrr->ring + dqrr->ci;
-+ dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
-+ dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
-+ QM_DQRR_VERB_VBIT : 0;
-+ dqrr->ithresh = qm_in(DQRR_ITR);
-+
-+ /* Free up pending DQRR entries if any as per current DCM */
-+ if (dqrr->fill) {
-+ enum qm_dqrr_cmode dcm = (qm_in(CFG) >> 16) & 3;
-+
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ dqrr->cmode = dcm;
-+#endif
-+ switch (dcm) {
-+ case qm_dqrr_cci:
-+ qm_dqrr_cci_consume(portal, dqrr->fill);
-+ break;
-+ case qm_dqrr_cce:
-+ qm_dqrr_cce_consume(portal, dqrr->fill);
-+ break;
-+ case qm_dqrr_cdc:
-+ qm_dqrr_cdc_consume_n(portal, (QM_DQRR_SIZE - 1));
-+ break;
-+ default:
-+ DPA_ASSERT(0);
-+ }
-+ }
-+
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ dqrr->dmode = dmode;
-+ dqrr->pmode = pmode;
-+ dqrr->cmode = cmode;
-+#endif
-+ /* Invalidate every ring entry before beginning */
-+ for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
-+ dcbi(qm_cl(dqrr->ring, cfg));
-+ cfg = (qm_in(CFG) & 0xff000f00) |
-+ ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
-+ ((dmode & 1) << 18) | /* DP */
-+ ((cmode & 3) << 16) | /* DCM */
-+ 0xa0 | /* RE+SE */
-+ (0 ? 0x40 : 0) | /* Ignore RP */
-+ (0 ? 0x10 : 0); /* Ignore SP */
-+ qm_out(CFG, cfg);
-+ qm_dqrr_set_maxfill(portal, max_fill);
-+ return 0;
-+}
-+
-+static inline void qm_dqrr_finish(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if ((dqrr->cmode != qm_dqrr_cdc) &&
-+ (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
-+ pr_crit("Ignoring completed DQRR entries\n");
-+#endif
-+}
-+
-+static inline const struct qm_dqrr_entry *qm_dqrr_current(
-+ struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ if (!dqrr->fill)
-+ return NULL;
-+ return dqrr->cursor;
-+}
-+
-+static inline u8 qm_dqrr_cursor(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ return DQRR_PTR2IDX(dqrr->cursor);
-+}
-+
-+static inline u8 qm_dqrr_next(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->fill);
-+ dqrr->cursor = DQRR_INC(dqrr->cursor);
-+ return --dqrr->fill;
-+}
-+
-+static inline u8 qm_dqrr_pci_update(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ u8 diff, old_pi = dqrr->pi;
-+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pci);
-+ dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
-+ diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
-+ dqrr->fill += diff;
-+ return diff;
-+}
-+
-+static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pce);
-+ qm_cl_invalidate(DQRR_PI);
-+ qm_cl_touch_ro(DQRR_PI);
-+}
-+
-+static inline u8 qm_dqrr_pce_update(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ u8 diff, old_pi = dqrr->pi;
-+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pce);
-+ dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1);
-+ diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
-+ dqrr->fill += diff;
-+ return diff;
-+}
-+
-+static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
-+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
-+#if (defined CONFIG_PPC || defined CONFIG_PPC64) && !defined CONFIG_FSL_PAMU
-+ /*
-+ * On PowerPC platforms if PAMU is not available we need to
-+ * manually invalidate the cache. When PAMU is available the
-+ * cache is updated by stashing operations generated by QMan
-+ */
-+ dcbi(res);
-+ dcbt_ro(res);
-+#endif
-+
-+ /* when accessing 'verb', use __raw_readb() to ensure that compiler
-+ * inlining doesn't try to optimise out "excess reads". */
-+ if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
-+ dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
-+ if (!dqrr->pi)
-+ dqrr->vbit ^= QM_DQRR_VERB_VBIT;
-+ dqrr->fill++;
-+ }
-+}
-+
-+
-+static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cci);
-+ dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
-+ qm_out(DQRR_CI_CINH, dqrr->ci);
-+}
-+
-+static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
-+ qm_cl_invalidate(DQRR_CI);
-+ qm_cl_touch_rw(DQRR_CI);
-+}
-+
-+static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
-+ dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
-+ qm_cl_out(DQRR_CI, dqrr->ci);
-+}
-+
-+static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx,
-+ int park)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
-+ DPA_ASSERT(idx < QM_DQRR_SIZE);
-+ qm_out(DQRR_DCAP, (0 << 8) | /* S */
-+ ((park ? 1 : 0) << 6) | /* PK */
-+ idx); /* DCAP_CI */
-+}
-+
-+static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
-+ const struct qm_dqrr_entry *dq,
-+ int park)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ u8 idx = DQRR_PTR2IDX(dq);
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
-+ DPA_ASSERT((dqrr->ring + idx) == dq);
-+ DPA_ASSERT(idx < QM_DQRR_SIZE);
-+ qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
-+ ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */
-+ idx); /* DQRR_DCAP::DCAP_CI */
-+}
-+
-+static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
-+ return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
-+}
-+
-+static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
-+ qm_cl_invalidate(DQRR_CI);
-+ qm_cl_touch_ro(DQRR_CI);
-+}
-+
-+static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
-+ return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1);
-+}
-+
-+static inline u8 qm_dqrr_get_ci(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
-+ return dqrr->ci;
-+}
-+
-+static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx)
-+{
-+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
-+ qm_out(DQRR_DCAP, (0 << 8) | /* S */
-+ (1 << 6) | /* PK */
-+ (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */
-+}
-+
-+static inline void qm_dqrr_park_current(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
-+ qm_out(DQRR_DCAP, (0 << 8) | /* S */
-+ (1 << 6) | /* PK */
-+ DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */
-+}
-+
-+static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
-+{
-+ qm_out(DQRR_SDQCR, sdqcr);
-+}
-+
-+static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal)
-+{
-+ return qm_in(DQRR_SDQCR);
-+}
-+
-+static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
-+{
-+ qm_out(DQRR_VDQCR, vdqcr);
-+}
-+
-+static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal)
-+{
-+ return qm_in(DQRR_VDQCR);
-+}
-+
-+static inline void qm_dqrr_pdqcr_set(struct qm_portal *portal, u32 pdqcr)
-+{
-+ qm_out(DQRR_PDQCR, pdqcr);
-+}
-+
-+static inline u32 qm_dqrr_pdqcr_get(struct qm_portal *portal)
-+{
-+ return qm_in(DQRR_PDQCR);
-+}
-+
-+static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal)
-+{
-+ register struct qm_dqrr *dqrr = &portal->dqrr;
-+ return dqrr->ithresh;
-+}
-+
-+static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
-+{
-+ qm_out(DQRR_ITR, ithresh);
-+}
-+
-+static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal)
-+{
-+ return (qm_in(CFG) & 0x00f00000) >> 20;
-+}
-+
-+
-+/* -------------- */
-+/* --- MR API --- */
-+
-+#define MR_CARRYCLEAR(p) \
-+ (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6)))
-+
-+static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e)
-+{
-+ return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1);
-+}
-+
-+static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e)
-+{
-+ return MR_CARRYCLEAR(e + 1);
-+}
-+
-+static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
-+ enum qm_mr_cmode cmode)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ u32 cfg;
-+
-+ mr->ring = portal->addr.addr_ce + QM_CL_MR;
-+ mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
-+ mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
-+ mr->cursor = mr->ring + mr->ci;
-+ mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
-+ mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
-+ mr->ithresh = qm_in(MR_ITR);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mr->pmode = pmode;
-+ mr->cmode = cmode;
-+#endif
-+ cfg = (qm_in(CFG) & 0xfffff0ff) |
-+ ((cmode & 1) << 8); /* QCSP_CFG:MM */
-+ qm_out(CFG, cfg);
-+ return 0;
-+}
-+
-+static inline void qm_mr_finish(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ if (mr->ci != MR_PTR2IDX(mr->cursor))
-+ pr_crit("Ignoring completed MR entries\n");
-+}
-+
-+static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ if (!mr->fill)
-+ return NULL;
-+ return mr->cursor;
-+}
-+
-+static inline u8 qm_mr_cursor(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ return MR_PTR2IDX(mr->cursor);
-+}
-+
-+static inline u8 qm_mr_next(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->fill);
-+ mr->cursor = MR_INC(mr->cursor);
-+ return --mr->fill;
-+}
-+
-+static inline u8 qm_mr_pci_update(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ u8 diff, old_pi = mr->pi;
-+ DPA_ASSERT(mr->pmode == qm_mr_pci);
-+ mr->pi = qm_in(MR_PI_CINH);
-+ diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi);
-+ mr->fill += diff;
-+ return diff;
-+}
-+
-+static inline void qm_mr_pce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->pmode == qm_mr_pce);
-+ qm_cl_invalidate(MR_PI);
-+ qm_cl_touch_ro(MR_PI);
-+}
-+
-+static inline u8 qm_mr_pce_update(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ u8 diff, old_pi = mr->pi;
-+ DPA_ASSERT(mr->pmode == qm_mr_pce);
-+ mr->pi = qm_cl_in(MR_PI) & (QM_MR_SIZE - 1);
-+ diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi);
-+ mr->fill += diff;
-+ return diff;
-+}
-+
-+static inline void qm_mr_pvb_update(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
-+ DPA_ASSERT(mr->pmode == qm_mr_pvb);
-+ /* when accessing 'verb', use __raw_readb() to ensure that compiler
-+ * inlining doesn't try to optimise out "excess reads". */
-+ if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
-+ mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
-+ if (!mr->pi)
-+ mr->vbit ^= QM_MR_VERB_VBIT;
-+ mr->fill++;
-+ res = MR_INC(res);
-+ }
-+ dcbit_ro(res);
-+}
-+
-+static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->cmode == qm_mr_cci);
-+ mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
-+ qm_out(MR_CI_CINH, mr->ci);
-+}
-+
-+static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->cmode == qm_mr_cci);
-+ mr->ci = MR_PTR2IDX(mr->cursor);
-+ qm_out(MR_CI_CINH, mr->ci);
-+}
-+
-+static inline void qm_mr_cce_prefetch(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->cmode == qm_mr_cce);
-+ qm_cl_invalidate(MR_CI);
-+ qm_cl_touch_rw(MR_CI);
-+}
-+
-+static inline void qm_mr_cce_consume(struct qm_portal *portal, u8 num)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->cmode == qm_mr_cce);
-+ mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
-+ qm_cl_out(MR_CI, mr->ci);
-+}
-+
-+static inline void qm_mr_cce_consume_to_current(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ DPA_ASSERT(mr->cmode == qm_mr_cce);
-+ mr->ci = MR_PTR2IDX(mr->cursor);
-+ qm_cl_out(MR_CI, mr->ci);
-+}
-+
-+static inline u8 qm_mr_get_ci(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ return mr->ci;
-+}
-+
-+static inline u8 qm_mr_get_ithresh(struct qm_portal *portal)
-+{
-+ register struct qm_mr *mr = &portal->mr;
-+ return mr->ithresh;
-+}
-+
-+static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
-+{
-+ qm_out(MR_ITR, ithresh);
-+}
-+
-+
-+/* ------------------------------ */
-+/* --- Management command API --- */
-+
-+static inline int qm_mc_init(struct qm_portal *portal)
-+{
-+ register struct qm_mc *mc = &portal->mc;
-+ mc->cr = portal->addr.addr_ce + QM_CL_CR;
-+ mc->rr = portal->addr.addr_ce + QM_CL_RR0;
-+ mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
-+ QM_MCC_VERB_VBIT) ? 0 : 1;
-+ mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = qman_mc_idle;
-+#endif
-+ return 0;
-+}
-+
-+static inline void qm_mc_finish(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_mc *mc = &portal->mc;
-+ DPA_ASSERT(mc->state == qman_mc_idle);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ if (mc->state != qman_mc_idle)
-+ pr_crit("Losing incomplete MC command\n");
-+#endif
-+}
-+
-+static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal)
-+{
-+ register struct qm_mc *mc = &portal->mc;
-+ DPA_ASSERT(mc->state == qman_mc_idle);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = qman_mc_user;
-+#endif
-+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
-+ dcbz_64(mc->cr);
-+#endif
-+ return mc->cr;
-+}
-+
-+static inline void qm_mc_abort(struct qm_portal *portal)
-+{
-+ __maybe_unused register struct qm_mc *mc = &portal->mc;
-+ DPA_ASSERT(mc->state == qman_mc_user);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = qman_mc_idle;
-+#endif
-+}
-+
-+static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
-+{
-+ register struct qm_mc *mc = &portal->mc;
-+ struct qm_mc_result *rr = mc->rr + mc->rridx;
-+ DPA_ASSERT(mc->state == qman_mc_user);
-+ lwsync();
-+ mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
-+ dcbf(mc->cr);
-+ dcbit_ro(rr);
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = qman_mc_hw;
-+#endif
-+}
-+
-+static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal)
-+{
-+ register struct qm_mc *mc = &portal->mc;
-+ struct qm_mc_result *rr = mc->rr + mc->rridx;
-+ DPA_ASSERT(mc->state == qman_mc_hw);
-+ /* The inactive response register's verb byte always returns zero until
-+ * its command is submitted and completed. This includes the valid-bit,
-+ * in case you were wondering... */
-+ if (!__raw_readb(&rr->verb)) {
-+ dcbit_ro(rr);
-+ return NULL;
-+ }
-+ mc->rridx ^= 1;
-+ mc->vbit ^= QM_MCC_VERB_VBIT;
-+#ifdef CONFIG_FSL_DPA_CHECKING
-+ mc->state = qman_mc_idle;
-+#endif
-+ return rr;
-+}
-+
-+
-+/* ------------------------------------- */
-+/* --- Portal interrupt register API --- */
-+
-+static inline int qm_isr_init(__always_unused struct qm_portal *portal)
-+{
-+ return 0;
-+}
-+
-+static inline void qm_isr_finish(__always_unused struct qm_portal *portal)
-+{
-+}
-+
-+static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod)
-+{
-+ qm_out(ITPR, iperiod);
-+}
-+
-+static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n)
-+{
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ return __qm_in(&portal->addr, QM_REG_ISR + (n << 6));
-+#else
-+ return __qm_in(&portal->addr, QM_REG_ISR + (n << 2));
-+#endif
-+}
-+
-+static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n,
-+ u32 val)
-+{
-+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-+ __qm_out(&portal->addr, QM_REG_ISR + (n << 6), val);
-+#else
-+ __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val);
-+#endif
-+}
-+
-+/* Cleanup FQs */
-+static inline int qm_shutdown_fq(struct qm_portal **portal, int portal_count,
-+ u32 fqid)
-+{
-+
-+ struct qm_mc_command *mcc;
-+ struct qm_mc_result *mcr;
-+ u8 state;
-+ int orl_empty, fq_empty, i, drain = 0;
-+ u32 result;
-+ u32 channel, wq;
-+ u16 dest_wq;
-+
-+ /* Determine the state of the FQID */
-+ mcc = qm_mc_start(portal[0]);
-+ mcc->queryfq_np.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ_NP);
-+ while (!(mcr = qm_mc_result(portal[0])))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
-+ state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
-+ if (state == QM_MCR_NP_STATE_OOS)
-+ return 0; /* Already OOS, no need to do anymore checks */
-+
-+ /* Query which channel the FQ is using */
-+ mcc = qm_mc_start(portal[0]);
-+ mcc->queryfq.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ);
-+ while (!(mcr = qm_mc_result(portal[0])))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
-+
-+ /* Need to store these since the MCR gets reused */
-+ dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
-+ wq = dest_wq & 0x7;
-+ channel = dest_wq>>3;
-+
-+ switch (state) {
-+ case QM_MCR_NP_STATE_TEN_SCHED:
-+ case QM_MCR_NP_STATE_TRU_SCHED:
-+ case QM_MCR_NP_STATE_ACTIVE:
-+ case QM_MCR_NP_STATE_PARKED:
-+ orl_empty = 0;
-+ mcc = qm_mc_start(portal[0]);
-+ mcc->alterfq.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_RETIRE);
-+ while (!(mcr = qm_mc_result(portal[0])))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_MCR_VERB_ALTER_RETIRE);
-+ result = mcr->result; /* Make a copy as we reuse MCR below */
-+
-+ if (result == QM_MCR_RESULT_PENDING) {
-+ /* Need to wait for the FQRN in the message ring, which
-+ will only occur once the FQ has been drained. In
-+ order for the FQ to drain the portal needs to be set
-+ to dequeue from the channel the FQ is scheduled on */
-+ const struct qm_mr_entry *msg;
-+ const struct qm_dqrr_entry *dqrr = NULL;
-+ int found_fqrn = 0;
-+ u16 dequeue_wq = 0;
-+
-+ /* Flag that we need to drain FQ */
-+ drain = 1;
-+
-+ if (channel >= qm_channel_pool1 &&
-+ channel < (qm_channel_pool1 + 15)) {
-+ /* Pool channel, enable the bit in the portal */
-+ dequeue_wq = (channel -
-+ qm_channel_pool1 + 1)<<4 | wq;
-+ } else if (channel < qm_channel_pool1) {
-+ /* Dedicated channel */
-+ dequeue_wq = wq;
-+ } else {
-+ pr_info("Cannot recover FQ 0x%x, it is "
-+ "scheduled on channel 0x%x",
-+ fqid, channel);
-+ return -EBUSY;
-+ }
-+ /* Set the sdqcr to drain this channel */
-+ if (channel < qm_channel_pool1)
-+ for (i = 0; i < portal_count; i++)
-+ qm_dqrr_sdqcr_set(portal[i],
-+ QM_SDQCR_TYPE_ACTIVE |
-+ QM_SDQCR_CHANNELS_DEDICATED);
-+ else
-+ for (i = 0; i < portal_count; i++)
-+ qm_dqrr_sdqcr_set(
-+ portal[i],
-+ QM_SDQCR_TYPE_ACTIVE |
-+ QM_SDQCR_CHANNELS_POOL_CONV
-+ (channel));
-+ while (!found_fqrn) {
-+ /* Keep draining DQRR while checking the MR*/
-+ for (i = 0; i < portal_count; i++) {
-+ qm_dqrr_pvb_update(portal[i]);
-+ dqrr = qm_dqrr_current(portal[i]);
-+ while (dqrr) {
-+ qm_dqrr_cdc_consume_1ptr(
-+ portal[i], dqrr, 0);
-+ qm_dqrr_pvb_update(portal[i]);
-+ qm_dqrr_next(portal[i]);
-+ dqrr = qm_dqrr_current(
-+ portal[i]);
-+ }
-+ /* Process message ring too */
-+ qm_mr_pvb_update(portal[i]);
-+ msg = qm_mr_current(portal[i]);
-+ while (msg) {
-+ if ((msg->verb &
-+ QM_MR_VERB_TYPE_MASK)
-+ == QM_MR_VERB_FQRN)
-+ found_fqrn = 1;
-+ qm_mr_next(portal[i]);
-+ qm_mr_cci_consume_to_current(
-+ portal[i]);
-+ qm_mr_pvb_update(portal[i]);
-+ msg = qm_mr_current(portal[i]);
-+ }
-+ cpu_relax();
-+ }
-+ }
-+ }
-+ if (result != QM_MCR_RESULT_OK &&
-+ result != QM_MCR_RESULT_PENDING) {
-+ /* error */
-+ pr_err("qman_retire_fq failed on FQ 0x%x, result=0x%x\n",
-+ fqid, result);
-+ return -1;
-+ }
-+ if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
-+ /* ORL had no entries, no need to wait until the
-+ ERNs come in */
-+ orl_empty = 1;
-+ }
-+ /* Retirement succeeded, check to see if FQ needs
-+ to be drained */
-+ if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
-+ /* FQ is Not Empty, drain using volatile DQ commands */
-+ fq_empty = 0;
-+ do {
-+ const struct qm_dqrr_entry *dqrr = NULL;
-+ u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
-+ qm_dqrr_vdqcr_set(portal[0], vdqcr);
-+
-+ /* Wait for a dequeue to occur */
-+ while (dqrr == NULL) {
-+ qm_dqrr_pvb_update(portal[0]);
-+ dqrr = qm_dqrr_current(portal[0]);
-+ if (!dqrr)
-+ cpu_relax();
-+ }
-+ /* Process the dequeues, making sure to
-+ empty the ring completely */
-+ while (dqrr) {
-+ if (be32_to_cpu(dqrr->fqid) == fqid &&
-+ dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
-+ fq_empty = 1;
-+ qm_dqrr_cdc_consume_1ptr(portal[0],
-+ dqrr, 0);
-+ qm_dqrr_pvb_update(portal[0]);
-+ qm_dqrr_next(portal[0]);
-+ dqrr = qm_dqrr_current(portal[0]);
-+ }
-+ } while (fq_empty == 0);
-+ }
-+ for (i = 0; i < portal_count; i++)
-+ qm_dqrr_sdqcr_set(portal[i], 0);
-+
-+ /* Wait for the ORL to have been completely drained */
-+ while (orl_empty == 0) {
-+ const struct qm_mr_entry *msg;
-+ qm_mr_pvb_update(portal[0]);
-+ msg = qm_mr_current(portal[0]);
-+ while (msg) {
-+ if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
-+ QM_MR_VERB_FQRL)
-+ orl_empty = 1;
-+ qm_mr_next(portal[0]);
-+ qm_mr_cci_consume_to_current(portal[0]);
-+ qm_mr_pvb_update(portal[0]);
-+ msg = qm_mr_current(portal[0]);
-+ }
-+ cpu_relax();
-+ }
-+ mcc = qm_mc_start(portal[0]);
-+ mcc->alterfq.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS);
-+ while (!(mcr = qm_mc_result(portal[0])))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_MCR_VERB_ALTER_OOS);
-+ if (mcr->result != QM_MCR_RESULT_OK) {
-+ pr_err("OOS after drain Failed on FQID 0x%x, result 0x%x\n",
-+ fqid, mcr->result);
-+ return -1;
-+ }
-+ return 0;
-+ case QM_MCR_NP_STATE_RETIRED:
-+ /* Send OOS Command */
-+ mcc = qm_mc_start(portal[0]);
-+ mcc->alterfq.fqid = cpu_to_be32(fqid);
-+ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS);
-+ while (!(mcr = qm_mc_result(portal[0])))
-+ cpu_relax();
-+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
-+ QM_MCR_VERB_ALTER_OOS);
-+ if (mcr->result) {
-+ pr_err("OOS Failed on FQID 0x%x\n", fqid);
-+ return -1;
-+ }
-+ return 0;
-+ }
-+ return -1;
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_private.h
-@@ -0,0 +1,398 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "dpa_sys.h"
-+#include <linux/fsl_qman.h>
-+#include <linux/iommu.h>
-+
-+#if defined(CONFIG_FSL_PAMU)
-+#include <asm/fsl_pamu_stash.h>
-+#endif
-+
-+#if !defined(CONFIG_FSL_QMAN_FQ_LOOKUP) && defined(CONFIG_PPC64)
-+#error "_PPC64 requires _FSL_QMAN_FQ_LOOKUP"
-+#endif
-+
-+#define QBMAN_ANY_PORTAL_IDX 0xffffffff
-+ /* ----------------- */
-+ /* Congestion Groups */
-+ /* ----------------- */
-+/* This wrapper represents a bit-array for the state of the 256 Qman congestion
-+ * groups. Is also used as a *mask* for congestion groups, eg. so we ignore
-+ * those that don't concern us. We harness the structure and accessor details
-+ * already used in the management command to query congestion groups. */
-+struct qman_cgrs {
-+ struct __qm_mcr_querycongestion q;
-+};
-+static inline void qman_cgrs_init(struct qman_cgrs *c)
-+{
-+ memset(c, 0, sizeof(*c));
-+}
-+static inline void qman_cgrs_fill(struct qman_cgrs *c)
-+{
-+ memset(c, 0xff, sizeof(*c));
-+}
-+static inline int qman_cgrs_get(struct qman_cgrs *c, int num)
-+{
-+ return QM_MCR_QUERYCONGESTION(&c->q, num);
-+}
-+static inline void qman_cgrs_set(struct qman_cgrs *c, int num)
-+{
-+ c->q.__state[__CGR_WORD(num)] |= (0x80000000 >> __CGR_SHIFT(num));
-+}
-+static inline void qman_cgrs_unset(struct qman_cgrs *c, int num)
-+{
-+ c->q.__state[__CGR_WORD(num)] &= ~(0x80000000 >> __CGR_SHIFT(num));
-+}
-+static inline int qman_cgrs_next(struct qman_cgrs *c, int num)
-+{
-+ while ((++num < __CGR_NUM) && !qman_cgrs_get(c, num))
-+ ;
-+ return num;
-+}
-+static inline void qman_cgrs_cp(struct qman_cgrs *dest,
-+ const struct qman_cgrs *src)
-+{
-+ *dest = *src;
-+}
-+static inline void qman_cgrs_and(struct qman_cgrs *dest,
-+ const struct qman_cgrs *a, const struct qman_cgrs *b)
-+{
-+ int ret;
-+ u32 *_d = dest->q.__state;
-+ const u32 *_a = a->q.__state;
-+ const u32 *_b = b->q.__state;
-+ for (ret = 0; ret < 8; ret++)
-+ *(_d++) = *(_a++) & *(_b++);
-+}
-+static inline void qman_cgrs_xor(struct qman_cgrs *dest,
-+ const struct qman_cgrs *a, const struct qman_cgrs *b)
-+{
-+ int ret;
-+ u32 *_d = dest->q.__state;
-+ const u32 *_a = a->q.__state;
-+ const u32 *_b = b->q.__state;
-+ for (ret = 0; ret < 8; ret++)
-+ *(_d++) = *(_a++) ^ *(_b++);
-+}
-+
-+ /* ----------------------- */
-+ /* CEETM Congestion Groups */
-+ /* ----------------------- */
-+/* This wrapper represents a bit-array for the state of the 512 Qman CEETM
-+ * congestion groups.
-+ */
-+struct qman_ccgrs {
-+ struct __qm_mcr_querycongestion q[2];
-+};
-+static inline void qman_ccgrs_init(struct qman_ccgrs *c)
-+{
-+ memset(c, 0, sizeof(*c));
-+}
-+static inline void qman_ccgrs_fill(struct qman_ccgrs *c)
-+{
-+ memset(c, 0xff, sizeof(*c));
-+}
-+static inline int qman_ccgrs_get(struct qman_ccgrs *c, int num)
-+{
-+ if (num < __CGR_NUM)
-+ return QM_MCR_QUERYCONGESTION(&c->q[0], num);
-+ else
-+ return QM_MCR_QUERYCONGESTION(&c->q[1], (num - __CGR_NUM));
-+}
-+static inline int qman_ccgrs_next(struct qman_ccgrs *c, int num)
-+{
-+ while ((++num < __CGR_NUM) && !qman_ccgrs_get(c, num))
-+ ;
-+ return num;
-+}
-+static inline void qman_ccgrs_cp(struct qman_ccgrs *dest,
-+ const struct qman_ccgrs *src)
-+{
-+ *dest = *src;
-+}
-+static inline void qman_ccgrs_and(struct qman_ccgrs *dest,
-+ const struct qman_ccgrs *a, const struct qman_ccgrs *b)
-+{
-+ int ret, i;
-+ u32 *_d;
-+ const u32 *_a, *_b;
-+ for (i = 0; i < 2; i++) {
-+ _d = dest->q[i].__state;
-+ _a = a->q[i].__state;
-+ _b = b->q[i].__state;
-+ for (ret = 0; ret < 8; ret++)
-+ *(_d++) = *(_a++) & *(_b++);
-+ }
-+}
-+static inline void qman_ccgrs_xor(struct qman_ccgrs *dest,
-+ const struct qman_ccgrs *a, const struct qman_ccgrs *b)
-+{
-+ int ret, i;
-+ u32 *_d;
-+ const u32 *_a, *_b;
-+ for (i = 0; i < 2; i++) {
-+ _d = dest->q[i].__state;
-+ _a = a->q[i].__state;
-+ _b = b->q[i].__state;
-+ for (ret = 0; ret < 8; ret++)
-+ *(_d++) = *(_a++) ^ *(_b++);
-+ }
-+}
-+
-+/* used by CCSR and portal interrupt code */
-+enum qm_isr_reg {
-+ qm_isr_status = 0,
-+ qm_isr_enable = 1,
-+ qm_isr_disable = 2,
-+ qm_isr_inhibit = 3
-+};
-+
-+struct qm_portal_config {
-+ /* Corenet portal addresses;
-+ * [0]==cache-enabled, [1]==cache-inhibited. */
-+ __iomem void *addr_virt[2];
-+ struct resource addr_phys[2];
-+ struct device dev;
-+ struct iommu_domain *iommu_domain;
-+ /* Allow these to be joined in lists */
-+ struct list_head list;
-+ /* User-visible portal configuration settings */
-+ struct qman_portal_config public_cfg;
-+ /* power management saved data */
-+ u32 saved_isdr;
-+};
-+
-+/* Revision info (for errata and feature handling) */
-+#define QMAN_REV11 0x0101
-+#define QMAN_REV12 0x0102
-+#define QMAN_REV20 0x0200
-+#define QMAN_REV30 0x0300
-+#define QMAN_REV31 0x0301
-+#define QMAN_REV32 0x0302
-+
-+/* QMan REV_2 register contains the Cfg option */
-+#define QMAN_REV_CFG_0 0x0
-+#define QMAN_REV_CFG_1 0x1
-+#define QMAN_REV_CFG_2 0x2
-+#define QMAN_REV_CFG_3 0x3
-+
-+extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */
-+extern u8 qman_ip_cfg;
-+extern u32 qman_clk;
-+extern u16 qman_portal_max;
-+
-+#ifdef CONFIG_FSL_QMAN_CONFIG
-+/* Hooks from qman_driver.c to qman_config.c */
-+int qman_init_ccsr(struct device_node *node);
-+void qman_liodn_fixup(u16 channel);
-+int qman_set_sdest(u16 channel, unsigned int cpu_idx);
-+size_t get_qman_fqd_size(void);
-+#else
-+static inline size_t get_qman_fqd_size(void)
-+{
-+ return (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ);
-+}
-+#endif
-+
-+int qm_set_wpm(int wpm);
-+int qm_get_wpm(int *wpm);
-+
-+/* Hooks from qman_driver.c in to qman_high.c */
-+struct qman_portal *qman_create_portal(
-+ struct qman_portal *portal,
-+ const struct qm_portal_config *config,
-+ const struct qman_cgrs *cgrs);
-+
-+struct qman_portal *qman_create_affine_portal(
-+ const struct qm_portal_config *config,
-+ const struct qman_cgrs *cgrs);
-+struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect,
-+ int cpu);
-+const struct qm_portal_config *qman_destroy_affine_portal(void);
-+void qman_destroy_portal(struct qman_portal *qm);
-+
-+/* Hooks from fsl_usdpaa.c to qman_driver.c */
-+struct qm_portal_config *qm_get_unused_portal(void);
-+struct qm_portal_config *qm_get_unused_portal_idx(uint32_t idx);
-+
-+void qm_put_unused_portal(struct qm_portal_config *pcfg);
-+void qm_set_liodns(struct qm_portal_config *pcfg);
-+
-+/* This CGR feature is supported by h/w and required by unit-tests and the
-+ * debugfs hooks, so is implemented in the driver. However it allows an explicit
-+ * corruption of h/w fields by s/w that are usually incorruptible (because the
-+ * counters are usually maintained entirely within h/w). As such, we declare
-+ * this API internally. */
-+int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
-+ struct qm_mcr_cgrtestwrite *result);
-+
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+/* If the fq object pointer is greater than the size of context_b field,
-+ * than a lookup table is required. */
-+int qman_setup_fq_lookup_table(size_t num_entries);
-+#endif
-+
-+
-+/*************************************************/
-+/* QMan s/w corenet portal, low-level i/face */
-+/*************************************************/
-+
-+/* Note: most functions are only used by the high-level interface, so are
-+ * inlined from qman_low.h. The stuff below is for use by other parts of the
-+ * driver. */
-+
-+/* For qm_dqrr_sdqcr_set(); Choose one SOURCE. Choose one COUNT. Choose one
-+ * dequeue TYPE. Choose TOKEN (8-bit).
-+ * If SOURCE == CHANNELS,
-+ * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL(n).
-+ * You can choose DEDICATED_PRECEDENCE if the portal channel should have
-+ * priority.
-+ * If SOURCE == SPECIFICWQ,
-+ * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the
-+ * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the
-+ * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the
-+ * same value.
-+ */
-+#define QM_SDQCR_SOURCE_CHANNELS 0x0
-+#define QM_SDQCR_SOURCE_SPECIFICWQ 0x40000000
-+#define QM_SDQCR_COUNT_EXACT1 0x0
-+#define QM_SDQCR_COUNT_UPTO3 0x20000000
-+#define QM_SDQCR_DEDICATED_PRECEDENCE 0x10000000
-+#define QM_SDQCR_TYPE_MASK 0x03000000
-+#define QM_SDQCR_TYPE_NULL 0x0
-+#define QM_SDQCR_TYPE_PRIO_QOS 0x01000000
-+#define QM_SDQCR_TYPE_ACTIVE_QOS 0x02000000
-+#define QM_SDQCR_TYPE_ACTIVE 0x03000000
-+#define QM_SDQCR_TOKEN_MASK 0x00ff0000
-+#define QM_SDQCR_TOKEN_SET(v) (((v) & 0xff) << 16)
-+#define QM_SDQCR_TOKEN_GET(v) (((v) >> 16) & 0xff)
-+#define QM_SDQCR_CHANNELS_DEDICATED 0x00008000
-+#define QM_SDQCR_SPECIFICWQ_MASK 0x000000f7
-+#define QM_SDQCR_SPECIFICWQ_DEDICATED 0x00000000
-+#define QM_SDQCR_SPECIFICWQ_POOL(n) ((n) << 4)
-+#define QM_SDQCR_SPECIFICWQ_WQ(n) (n)
-+
-+/* For qm_dqrr_vdqcr_set(): use FQID(n) to fill in the frame queue ID */
-+#define QM_VDQCR_FQID_MASK 0x00ffffff
-+#define QM_VDQCR_FQID(n) ((n) & QM_VDQCR_FQID_MASK)
-+
-+/* For qm_dqrr_pdqcr_set(); Choose one MODE. Choose one COUNT.
-+ * If MODE==SCHEDULED
-+ * Choose SCHEDULED_CHANNELS or SCHEDULED_SPECIFICWQ. Choose one dequeue TYPE.
-+ * If CHANNELS,
-+ * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL() channels.
-+ * You can choose DEDICATED_PRECEDENCE if the portal channel should have
-+ * priority.
-+ * If SPECIFICWQ,
-+ * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the
-+ * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the
-+ * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the
-+ * same value.
-+ * If MODE==UNSCHEDULED
-+ * Choose FQID().
-+ */
-+#define QM_PDQCR_MODE_SCHEDULED 0x0
-+#define QM_PDQCR_MODE_UNSCHEDULED 0x80000000
-+#define QM_PDQCR_SCHEDULED_CHANNELS 0x0
-+#define QM_PDQCR_SCHEDULED_SPECIFICWQ 0x40000000
-+#define QM_PDQCR_COUNT_EXACT1 0x0
-+#define QM_PDQCR_COUNT_UPTO3 0x20000000
-+#define QM_PDQCR_DEDICATED_PRECEDENCE 0x10000000
-+#define QM_PDQCR_TYPE_MASK 0x03000000
-+#define QM_PDQCR_TYPE_NULL 0x0
-+#define QM_PDQCR_TYPE_PRIO_QOS 0x01000000
-+#define QM_PDQCR_TYPE_ACTIVE_QOS 0x02000000
-+#define QM_PDQCR_TYPE_ACTIVE 0x03000000
-+#define QM_PDQCR_CHANNELS_DEDICATED 0x00008000
-+#define QM_PDQCR_CHANNELS_POOL(n) (0x00008000 >> (n))
-+#define QM_PDQCR_SPECIFICWQ_MASK 0x000000f7
-+#define QM_PDQCR_SPECIFICWQ_DEDICATED 0x00000000
-+#define QM_PDQCR_SPECIFICWQ_POOL(n) ((n) << 4)
-+#define QM_PDQCR_SPECIFICWQ_WQ(n) (n)
-+#define QM_PDQCR_FQID(n) ((n) & 0xffffff)
-+
-+/* Used by all portal interrupt registers except 'inhibit'
-+ * Channels with frame availability
-+ */
-+#define QM_PIRQ_DQAVAIL 0x0000ffff
-+
-+/* The DQAVAIL interrupt fields break down into these bits; */
-+#define QM_DQAVAIL_PORTAL 0x8000 /* Portal channel */
-+#define QM_DQAVAIL_POOL(n) (0x8000 >> (n)) /* Pool channel, n==[1..15] */
-+#define QM_DQAVAIL_MASK 0xffff
-+/* This mask contains all the "irqsource" bits visible to API users */
-+#define QM_PIRQ_VISIBLE (QM_PIRQ_SLOW | QM_PIRQ_DQRI)
-+
-+/* These are qm_<reg>_<verb>(). So for example, qm_disable_write() means "write
-+ * the disable register" rather than "disable the ability to write". */
-+#define qm_isr_status_read(qm) __qm_isr_read(qm, qm_isr_status)
-+#define qm_isr_status_clear(qm, m) __qm_isr_write(qm, qm_isr_status, m)
-+#define qm_isr_enable_read(qm) __qm_isr_read(qm, qm_isr_enable)
-+#define qm_isr_enable_write(qm, v) __qm_isr_write(qm, qm_isr_enable, v)
-+#define qm_isr_disable_read(qm) __qm_isr_read(qm, qm_isr_disable)
-+#define qm_isr_disable_write(qm, v) __qm_isr_write(qm, qm_isr_disable, v)
-+/* TODO: unfortunate name-clash here, reword? */
-+#define qm_isr_inhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 1)
-+#define qm_isr_uninhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 0)
-+
-+#ifdef CONFIG_FSL_QMAN_CONFIG
-+int qman_have_ccsr(void);
-+#else
-+#define qman_have_ccsr 0
-+#endif
-+
-+__init int qman_init(void);
-+__init int qman_resource_init(void);
-+
-+/* CEETM related */
-+#define QMAN_CEETM_MAX 2
-+extern u8 num_ceetms;
-+extern struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX];
-+int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal);
-+int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal);
-+int qman_ceetm_set_prescaler(enum qm_dc_portal portal);
-+int qman_ceetm_get_prescaler(u16 *pres);
-+int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid,
-+ struct qm_mcr_ceetm_cq_query *cq_query);
-+int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query,
-+ struct qm_mcr_ceetm_ccgr_query *response);
-+int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num);
-+
-+extern void *affine_portals[NR_CPUS];
-+const struct qm_portal_config *qman_get_qm_portal_config(
-+ struct qman_portal *portal);
-+
-+/* power management */
-+#ifdef CONFIG_SUSPEND
-+void suspend_unused_qportal(void);
-+void resume_unused_qportal(void);
-+#endif
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_test.c
-@@ -0,0 +1,57 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "qman_test.h"
-+
-+MODULE_AUTHOR("Geoff Thorpe");
-+MODULE_LICENSE("Dual BSD/GPL");
-+MODULE_DESCRIPTION("Qman testing");
-+
-+static int test_init(void)
-+{
-+ int loop = 1;
-+ while (loop--) {
-+#ifdef CONFIG_FSL_QMAN_TEST_STASH_POTATO
-+ qman_test_hotpotato();
-+#endif
-+#ifdef CONFIG_FSL_QMAN_TEST_HIGH
-+ qman_test_high();
-+#endif
-+ }
-+ return 0;
-+}
-+
-+static void test_exit(void)
-+{
-+}
-+
-+module_init(test_init);
-+module_exit(test_exit);
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_test.h
-@@ -0,0 +1,45 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include <linux/kernel.h>
-+#include <linux/errno.h>
-+#include <linux/io.h>
-+#include <linux/slab.h>
-+#include <linux/module.h>
-+#include <linux/interrupt.h>
-+#include <linux/delay.h>
-+#include <linux/sched.h>
-+
-+#include <linux/fsl_qman.h>
-+
-+void qman_test_hotpotato(void);
-+void qman_test_high(void);
-+
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_test_high.c
-@@ -0,0 +1,216 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "qman_test.h"
-+
-+/*************/
-+/* constants */
-+/*************/
-+
-+#define CGR_ID 27
-+#define POOL_ID 2
-+#define FQ_FLAGS QMAN_FQ_FLAG_DYNAMIC_FQID
-+#define NUM_ENQUEUES 10
-+#define NUM_PARTIAL 4
-+#define PORTAL_SDQCR (QM_SDQCR_SOURCE_CHANNELS | \
-+ QM_SDQCR_TYPE_PRIO_QOS | \
-+ QM_SDQCR_TOKEN_SET(0x98) | \
-+ QM_SDQCR_CHANNELS_DEDICATED | \
-+ QM_SDQCR_CHANNELS_POOL(POOL_ID))
-+#define PORTAL_OPAQUE ((void *)0xf00dbeef)
-+#define VDQCR_FLAGS (QMAN_VOLATILE_FLAG_WAIT | QMAN_VOLATILE_FLAG_FINISH)
-+
-+/*************************************/
-+/* Predeclarations (eg. for fq_base) */
-+/*************************************/
-+
-+static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *,
-+ struct qman_fq *,
-+ const struct qm_dqrr_entry *);
-+static void cb_ern(struct qman_portal *, struct qman_fq *,
-+ const struct qm_mr_entry *);
-+static void cb_fqs(struct qman_portal *, struct qman_fq *,
-+ const struct qm_mr_entry *);
-+
-+/***************/
-+/* global vars */
-+/***************/
-+
-+static struct qm_fd fd, fd_dq;
-+static struct qman_fq fq_base = {
-+ .cb.dqrr = cb_dqrr,
-+ .cb.ern = cb_ern,
-+ .cb.fqs = cb_fqs
-+};
-+static DECLARE_WAIT_QUEUE_HEAD(waitqueue);
-+static int retire_complete, sdqcr_complete;
-+
-+/**********************/
-+/* internal functions */
-+/**********************/
-+
-+/* Helpers for initialising and "incrementing" a frame descriptor */
-+static void fd_init(struct qm_fd *__fd)
-+{
-+ qm_fd_addr_set64(__fd, 0xabdeadbeefLLU);
-+ __fd->format = qm_fd_contig_big;
-+ __fd->length29 = 0x0000ffff;
-+ __fd->cmd = 0xfeedf00d;
-+}
-+
-+static void fd_inc(struct qm_fd *__fd)
-+{
-+ u64 t = qm_fd_addr_get64(__fd);
-+ int z = t >> 40;
-+ t <<= 1;
-+ if (z)
-+ t |= 1;
-+ qm_fd_addr_set64(__fd, t);
-+ __fd->length29--;
-+ __fd->cmd++;
-+}
-+
-+/* The only part of the 'fd' we can't memcmp() is the ppid */
-+static int fd_cmp(const struct qm_fd *a, const struct qm_fd *b)
-+{
-+ int r = (qm_fd_addr_get64(a) == qm_fd_addr_get64(b)) ? 0 : -1;
-+ if (!r)
-+ r = a->format - b->format;
-+ if (!r)
-+ r = a->opaque - b->opaque;
-+ if (!r)
-+ r = a->cmd - b->cmd;
-+ return r;
-+}
-+
-+/********/
-+/* test */
-+/********/
-+
-+static void do_enqueues(struct qman_fq *fq)
-+{
-+ unsigned int loop;
-+ for (loop = 0; loop < NUM_ENQUEUES; loop++) {
-+ if (qman_enqueue(fq, &fd, QMAN_ENQUEUE_FLAG_WAIT |
-+ (((loop + 1) == NUM_ENQUEUES) ?
-+ QMAN_ENQUEUE_FLAG_WAIT_SYNC : 0)))
-+ panic("qman_enqueue() failed\n");
-+ fd_inc(&fd);
-+ }
-+}
-+
-+void qman_test_high(void)
-+{
-+ unsigned int flags;
-+ int res;
-+ struct qman_fq *fq = &fq_base;
-+
-+ pr_info("qman_test_high starting\n");
-+ fd_init(&fd);
-+ fd_init(&fd_dq);
-+
-+ /* Initialise (parked) FQ */
-+ if (qman_create_fq(0, FQ_FLAGS, fq))
-+ panic("qman_create_fq() failed\n");
-+ if (qman_init_fq(fq, QMAN_INITFQ_FLAG_LOCAL, NULL))
-+ panic("qman_init_fq() failed\n");
-+
-+ /* Do enqueues + VDQCR, twice. (Parked FQ) */
-+ do_enqueues(fq);
-+ pr_info("VDQCR (till-empty);\n");
-+ if (qman_volatile_dequeue(fq, VDQCR_FLAGS,
-+ QM_VDQCR_NUMFRAMES_TILLEMPTY))
-+ panic("qman_volatile_dequeue() failed\n");
-+ do_enqueues(fq);
-+ pr_info("VDQCR (%d of %d);\n", NUM_PARTIAL, NUM_ENQUEUES);
-+ if (qman_volatile_dequeue(fq, VDQCR_FLAGS,
-+ QM_VDQCR_NUMFRAMES_SET(NUM_PARTIAL)))
-+ panic("qman_volatile_dequeue() failed\n");
-+ pr_info("VDQCR (%d of %d);\n", NUM_ENQUEUES - NUM_PARTIAL,
-+ NUM_ENQUEUES);
-+ if (qman_volatile_dequeue(fq, VDQCR_FLAGS,
-+ QM_VDQCR_NUMFRAMES_SET(NUM_ENQUEUES - NUM_PARTIAL)))
-+ panic("qman_volatile_dequeue() failed\n");
-+
-+ do_enqueues(fq);
-+ pr_info("scheduled dequeue (till-empty)\n");
-+ if (qman_schedule_fq(fq))
-+ panic("qman_schedule_fq() failed\n");
-+ wait_event(waitqueue, sdqcr_complete);
-+
-+ /* Retire and OOS the FQ */
-+ res = qman_retire_fq(fq, &flags);
-+ if (res < 0)
-+ panic("qman_retire_fq() failed\n");
-+ wait_event(waitqueue, retire_complete);
-+ if (flags & QMAN_FQ_STATE_BLOCKOOS)
-+ panic("leaking frames\n");
-+ if (qman_oos_fq(fq))
-+ panic("qman_oos_fq() failed\n");
-+ qman_destroy_fq(fq, 0);
-+ pr_info("qman_test_high finished\n");
-+}
-+
-+static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *p,
-+ struct qman_fq *fq,
-+ const struct qm_dqrr_entry *dq)
-+{
-+ if (fd_cmp(&fd_dq, &dq->fd)) {
-+ pr_err("BADNESS: dequeued frame doesn't match;\n");
-+ pr_err("Expected 0x%llx, got 0x%llx\n",
-+ (unsigned long long)fd_dq.length29,
-+ (unsigned long long)dq->fd.length29);
-+ BUG();
-+ }
-+ fd_inc(&fd_dq);
-+ if (!(dq->stat & QM_DQRR_STAT_UNSCHEDULED) && !fd_cmp(&fd_dq, &fd)) {
-+ sdqcr_complete = 1;
-+ wake_up(&waitqueue);
-+ }
-+ return qman_cb_dqrr_consume;
-+}
-+
-+static void cb_ern(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_mr_entry *msg)
-+{
-+ panic("cb_ern() unimplemented");
-+}
-+
-+static void cb_fqs(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_mr_entry *msg)
-+{
-+ u8 verb = (msg->verb & QM_MR_VERB_TYPE_MASK);
-+ if ((verb != QM_MR_VERB_FQRN) && (verb != QM_MR_VERB_FQRNI))
-+ panic("unexpected FQS message");
-+ pr_info("Retirement message received\n");
-+ retire_complete = 1;
-+ wake_up(&waitqueue);
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_test_hotpotato.c
-@@ -0,0 +1,502 @@
-+/* Copyright 2009-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include <linux/kthread.h>
-+#include <linux/platform_device.h>
-+#include <linux/dma-mapping.h>
-+#include "qman_test.h"
-+
-+/* Algorithm:
-+ *
-+ * Each cpu will have HP_PER_CPU "handlers" set up, each of which incorporates
-+ * an rx/tx pair of FQ objects (both of which are stashed on dequeue). The
-+ * organisation of FQIDs is such that the HP_PER_CPU*NUM_CPUS handlers will
-+ * shuttle a "hot potato" frame around them such that every forwarding action
-+ * moves it from one cpu to another. (The use of more than one handler per cpu
-+ * is to allow enough handlers/FQs to truly test the significance of caching -
-+ * ie. when cache-expiries are occurring.)
-+ *
-+ * The "hot potato" frame content will be HP_NUM_WORDS*4 bytes in size, and the
-+ * first and last words of the frame data will undergo a transformation step on
-+ * each forwarding action. To achieve this, each handler will be assigned a
-+ * 32-bit "mixer", that is produced using a 32-bit LFSR. When a frame is
-+ * received by a handler, the mixer of the expected sender is XOR'd into all
-+ * words of the entire frame, which is then validated against the original
-+ * values. Then, before forwarding, the entire frame is XOR'd with the mixer of
-+ * the current handler. Apart from validating that the frame is taking the
-+ * expected path, this also provides some quasi-realistic overheads to each
-+ * forwarding action - dereferencing *all* the frame data, computation, and
-+ * conditional branching. There is a "special" handler designated to act as the
-+ * instigator of the test by creating an enqueuing the "hot potato" frame, and
-+ * to determine when the test has completed by counting HP_LOOPS iterations.
-+ *
-+ * Init phases:
-+ *
-+ * 1. prepare each cpu's 'hp_cpu' struct using on_each_cpu(,,1) and link them
-+ * into 'hp_cpu_list'. Specifically, set processor_id, allocate HP_PER_CPU
-+ * handlers and link-list them (but do no other handler setup).
-+ *
-+ * 2. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each
-+ * hp_cpu's 'iterator' to point to its first handler. With each loop,
-+ * allocate rx/tx FQIDs and mixer values to the hp_cpu's iterator handler
-+ * and advance the iterator for the next loop. This includes a final fixup,
-+ * which connects the last handler to the first (and which is why phase 2
-+ * and 3 are separate).
-+ *
-+ * 3. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each
-+ * hp_cpu's 'iterator' to point to its first handler. With each loop,
-+ * initialise FQ objects and advance the iterator for the next loop.
-+ * Moreover, do this initialisation on the cpu it applies to so that Rx FQ
-+ * initialisation targets the correct cpu.
-+ */
-+
-+/* helper to run something on all cpus (can't use on_each_cpu(), as that invokes
-+ * the fn from irq context, which is too restrictive). */
-+struct bstrap {
-+ void (*fn)(void);
-+ atomic_t started;
-+};
-+static int bstrap_fn(void *__bstrap)
-+{
-+ struct bstrap *bstrap = __bstrap;
-+ atomic_inc(&bstrap->started);
-+ bstrap->fn();
-+ while (!kthread_should_stop())
-+ msleep(1);
-+ return 0;
-+}
-+static int on_all_cpus(void (*fn)(void))
-+{
-+ int cpu;
-+ for_each_cpu(cpu, cpu_online_mask) {
-+ struct bstrap bstrap = {
-+ .fn = fn,
-+ .started = ATOMIC_INIT(0)
-+ };
-+ struct task_struct *k = kthread_create(bstrap_fn, &bstrap,
-+ "hotpotato%d", cpu);
-+ int ret;
-+ if (IS_ERR(k))
-+ return -ENOMEM;
-+ kthread_bind(k, cpu);
-+ wake_up_process(k);
-+ /* If we call kthread_stop() before the "wake up" has had an
-+ * effect, then the thread may exit with -EINTR without ever
-+ * running the function. So poll until it's started before
-+ * requesting it to stop. */
-+ while (!atomic_read(&bstrap.started))
-+ msleep(10);
-+ ret = kthread_stop(k);
-+ if (ret)
-+ return ret;
-+ }
-+ return 0;
-+}
-+
-+struct hp_handler {
-+
-+ /* The following data is stashed when 'rx' is dequeued; */
-+ /* -------------- */
-+ /* The Rx FQ, dequeues of which will stash the entire hp_handler */
-+ struct qman_fq rx;
-+ /* The Tx FQ we should forward to */
-+ struct qman_fq tx;
-+ /* The value we XOR post-dequeue, prior to validating */
-+ u32 rx_mixer;
-+ /* The value we XOR pre-enqueue, after validating */
-+ u32 tx_mixer;
-+ /* what the hotpotato address should be on dequeue */
-+ dma_addr_t addr;
-+ u32 *frame_ptr;
-+
-+ /* The following data isn't (necessarily) stashed on dequeue; */
-+ /* -------------- */
-+ u32 fqid_rx, fqid_tx;
-+ /* list node for linking us into 'hp_cpu' */
-+ struct list_head node;
-+ /* Just to check ... */
-+ unsigned int processor_id;
-+} ____cacheline_aligned;
-+
-+struct hp_cpu {
-+ /* identify the cpu we run on; */
-+ unsigned int processor_id;
-+ /* root node for the per-cpu list of handlers */
-+ struct list_head handlers;
-+ /* list node for linking us into 'hp_cpu_list' */
-+ struct list_head node;
-+ /* when repeatedly scanning 'hp_list', each time linking the n'th
-+ * handlers together, this is used as per-cpu iterator state */
-+ struct hp_handler *iterator;
-+};
-+
-+/* Each cpu has one of these */
-+static DEFINE_PER_CPU(struct hp_cpu, hp_cpus);
-+
-+/* links together the hp_cpu structs, in first-come first-serve order. */
-+static LIST_HEAD(hp_cpu_list);
-+static spinlock_t hp_lock = __SPIN_LOCK_UNLOCKED(hp_lock);
-+
-+static unsigned int hp_cpu_list_length;
-+
-+/* the "special" handler, that starts and terminates the test. */
-+static struct hp_handler *special_handler;
-+static int loop_counter;
-+
-+/* handlers are allocated out of this, so they're properly aligned. */
-+static struct kmem_cache *hp_handler_slab;
-+
-+/* this is the frame data */
-+static void *__frame_ptr;
-+static u32 *frame_ptr;
-+static dma_addr_t frame_dma;
-+
-+/* the main function waits on this */
-+static DECLARE_WAIT_QUEUE_HEAD(queue);
-+
-+#define HP_PER_CPU 2
-+#define HP_LOOPS 8
-+/* 80 bytes, like a small ethernet frame, and bleeds into a second cacheline */
-+#define HP_NUM_WORDS 80
-+/* First word of the LFSR-based frame data */
-+#define HP_FIRST_WORD 0xabbaf00d
-+
-+static inline u32 do_lfsr(u32 prev)
-+{
-+ return (prev >> 1) ^ (-(prev & 1u) & 0xd0000001u);
-+}
-+
-+static void allocate_frame_data(void)
-+{
-+ u32 lfsr = HP_FIRST_WORD;
-+ int loop;
-+ struct platform_device *pdev = platform_device_alloc("foobar", -1);
-+ if (!pdev)
-+ panic("platform_device_alloc() failed");
-+ if (platform_device_add(pdev))
-+ panic("platform_device_add() failed");
-+ __frame_ptr = kmalloc(4 * HP_NUM_WORDS, GFP_KERNEL);
-+ if (!__frame_ptr)
-+ panic("kmalloc() failed");
-+ frame_ptr = (void *)(((unsigned long)__frame_ptr + 63) &
-+ ~(unsigned long)63);
-+ for (loop = 0; loop < HP_NUM_WORDS; loop++) {
-+ frame_ptr[loop] = lfsr;
-+ lfsr = do_lfsr(lfsr);
-+ }
-+ frame_dma = dma_map_single(&pdev->dev, frame_ptr, 4 * HP_NUM_WORDS,
-+ DMA_BIDIRECTIONAL);
-+ platform_device_del(pdev);
-+ platform_device_put(pdev);
-+}
-+
-+static void deallocate_frame_data(void)
-+{
-+ kfree(__frame_ptr);
-+}
-+
-+static inline void process_frame_data(struct hp_handler *handler,
-+ const struct qm_fd *fd)
-+{
-+ u32 *p = handler->frame_ptr;
-+ u32 lfsr = HP_FIRST_WORD;
-+ int loop;
-+ if (qm_fd_addr_get64(fd) != (handler->addr & 0xffffffffff)) {
-+ pr_err("Got 0x%llx expected 0x%llx\n",
-+ qm_fd_addr_get64(fd), handler->addr);
-+ panic("bad frame address");
-+ }
-+ for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) {
-+ *p ^= handler->rx_mixer;
-+ if (*p != lfsr)
-+ panic("corrupt frame data");
-+ *p ^= handler->tx_mixer;
-+ lfsr = do_lfsr(lfsr);
-+ }
-+}
-+
-+static enum qman_cb_dqrr_result normal_dqrr(struct qman_portal *portal,
-+ struct qman_fq *fq,
-+ const struct qm_dqrr_entry *dqrr)
-+{
-+ struct hp_handler *handler = (struct hp_handler *)fq;
-+
-+ process_frame_data(handler, &dqrr->fd);
-+ if (qman_enqueue(&handler->tx, &dqrr->fd, 0))
-+ panic("qman_enqueue() failed");
-+ return qman_cb_dqrr_consume;
-+}
-+
-+static enum qman_cb_dqrr_result special_dqrr(struct qman_portal *portal,
-+ struct qman_fq *fq,
-+ const struct qm_dqrr_entry *dqrr)
-+{
-+ struct hp_handler *handler = (struct hp_handler *)fq;
-+
-+ process_frame_data(handler, &dqrr->fd);
-+ if (++loop_counter < HP_LOOPS) {
-+ if (qman_enqueue(&handler->tx, &dqrr->fd, 0))
-+ panic("qman_enqueue() failed");
-+ } else {
-+ pr_info("Received final (%dth) frame\n", loop_counter);
-+ wake_up(&queue);
-+ }
-+ return qman_cb_dqrr_consume;
-+}
-+
-+static void create_per_cpu_handlers(void)
-+{
-+ struct hp_handler *handler;
-+ int loop;
-+ struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus);
-+
-+ hp_cpu->processor_id = smp_processor_id();
-+ spin_lock(&hp_lock);
-+ list_add_tail(&hp_cpu->node, &hp_cpu_list);
-+ hp_cpu_list_length++;
-+ spin_unlock(&hp_lock);
-+ INIT_LIST_HEAD(&hp_cpu->handlers);
-+ for (loop = 0; loop < HP_PER_CPU; loop++) {
-+ handler = kmem_cache_alloc(hp_handler_slab, GFP_KERNEL);
-+ if (!handler)
-+ panic("kmem_cache_alloc() failed");
-+ handler->processor_id = hp_cpu->processor_id;
-+ handler->addr = frame_dma;
-+ handler->frame_ptr = frame_ptr;
-+ list_add_tail(&handler->node, &hp_cpu->handlers);
-+ }
-+ put_cpu_var(hp_cpus);
-+}
-+
-+static void destroy_per_cpu_handlers(void)
-+{
-+ struct list_head *loop, *tmp;
-+ struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus);
-+
-+ spin_lock(&hp_lock);
-+ list_del(&hp_cpu->node);
-+ spin_unlock(&hp_lock);
-+ list_for_each_safe(loop, tmp, &hp_cpu->handlers) {
-+ u32 flags;
-+ struct hp_handler *handler = list_entry(loop, struct hp_handler,
-+ node);
-+ if (qman_retire_fq(&handler->rx, &flags))
-+ panic("qman_retire_fq(rx) failed");
-+ BUG_ON(flags & QMAN_FQ_STATE_BLOCKOOS);
-+ if (qman_oos_fq(&handler->rx))
-+ panic("qman_oos_fq(rx) failed");
-+ qman_destroy_fq(&handler->rx, 0);
-+ qman_destroy_fq(&handler->tx, 0);
-+ qman_release_fqid(handler->fqid_rx);
-+ list_del(&handler->node);
-+ kmem_cache_free(hp_handler_slab, handler);
-+ }
-+ put_cpu_var(hp_cpus);
-+}
-+
-+static inline u8 num_cachelines(u32 offset)
-+{
-+ u8 res = (offset + (L1_CACHE_BYTES - 1))
-+ / (L1_CACHE_BYTES);
-+ if (res > 3)
-+ return 3;
-+ return res;
-+}
-+#define STASH_DATA_CL \
-+ num_cachelines(HP_NUM_WORDS * 4)
-+#define STASH_CTX_CL \
-+ num_cachelines(offsetof(struct hp_handler, fqid_rx))
-+
-+static void init_handler(void *__handler)
-+{
-+ struct qm_mcc_initfq opts;
-+ struct hp_handler *handler = __handler;
-+ BUG_ON(handler->processor_id != smp_processor_id());
-+ /* Set up rx */
-+ memset(&handler->rx, 0, sizeof(handler->rx));
-+ if (handler == special_handler)
-+ handler->rx.cb.dqrr = special_dqrr;
-+ else
-+ handler->rx.cb.dqrr = normal_dqrr;
-+ if (qman_create_fq(handler->fqid_rx, 0, &handler->rx))
-+ panic("qman_create_fq(rx) failed");
-+ memset(&opts, 0, sizeof(opts));
-+ opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA;
-+ opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING;
-+ opts.fqd.context_a.stashing.data_cl = STASH_DATA_CL;
-+ opts.fqd.context_a.stashing.context_cl = STASH_CTX_CL;
-+ if (qman_init_fq(&handler->rx, QMAN_INITFQ_FLAG_SCHED |
-+ QMAN_INITFQ_FLAG_LOCAL, &opts))
-+ panic("qman_init_fq(rx) failed");
-+ /* Set up tx */
-+ memset(&handler->tx, 0, sizeof(handler->tx));
-+ if (qman_create_fq(handler->fqid_tx, QMAN_FQ_FLAG_NO_MODIFY,
-+ &handler->tx))
-+ panic("qman_create_fq(tx) failed");
-+}
-+
-+static void init_phase2(void)
-+{
-+ int loop;
-+ u32 fqid = 0;
-+ u32 lfsr = 0xdeadbeef;
-+ struct hp_cpu *hp_cpu;
-+ struct hp_handler *handler;
-+
-+ for (loop = 0; loop < HP_PER_CPU; loop++) {
-+ list_for_each_entry(hp_cpu, &hp_cpu_list, node) {
-+ int ret;
-+ if (!loop)
-+ hp_cpu->iterator = list_first_entry(
-+ &hp_cpu->handlers,
-+ struct hp_handler, node);
-+ else
-+ hp_cpu->iterator = list_entry(
-+ hp_cpu->iterator->node.next,
-+ struct hp_handler, node);
-+ /* Rx FQID is the previous handler's Tx FQID */
-+ hp_cpu->iterator->fqid_rx = fqid;
-+ /* Allocate new FQID for Tx */
-+ ret = qman_alloc_fqid(&fqid);
-+ if (ret)
-+ panic("qman_alloc_fqid() failed");
-+ hp_cpu->iterator->fqid_tx = fqid;
-+ /* Rx mixer is the previous handler's Tx mixer */
-+ hp_cpu->iterator->rx_mixer = lfsr;
-+ /* Get new mixer for Tx */
-+ lfsr = do_lfsr(lfsr);
-+ hp_cpu->iterator->tx_mixer = lfsr;
-+ }
-+ }
-+ /* Fix up the first handler (fqid_rx==0, rx_mixer=0xdeadbeef) */
-+ hp_cpu = list_first_entry(&hp_cpu_list, struct hp_cpu, node);
-+ handler = list_first_entry(&hp_cpu->handlers, struct hp_handler, node);
-+ BUG_ON((handler->fqid_rx != 0) || (handler->rx_mixer != 0xdeadbeef));
-+ handler->fqid_rx = fqid;
-+ handler->rx_mixer = lfsr;
-+ /* and tag it as our "special" handler */
-+ special_handler = handler;
-+}
-+
-+static void init_phase3(void)
-+{
-+ int loop;
-+ struct hp_cpu *hp_cpu;
-+
-+ for (loop = 0; loop < HP_PER_CPU; loop++) {
-+ list_for_each_entry(hp_cpu, &hp_cpu_list, node) {
-+ if (!loop)
-+ hp_cpu->iterator = list_first_entry(
-+ &hp_cpu->handlers,
-+ struct hp_handler, node);
-+ else
-+ hp_cpu->iterator = list_entry(
-+ hp_cpu->iterator->node.next,
-+ struct hp_handler, node);
-+ preempt_disable();
-+ if (hp_cpu->processor_id == smp_processor_id())
-+ init_handler(hp_cpu->iterator);
-+ else
-+ smp_call_function_single(hp_cpu->processor_id,
-+ init_handler, hp_cpu->iterator, 1);
-+ preempt_enable();
-+ }
-+ }
-+}
-+
-+static void send_first_frame(void *ignore)
-+{
-+ u32 *p = special_handler->frame_ptr;
-+ u32 lfsr = HP_FIRST_WORD;
-+ int loop;
-+ struct qm_fd fd;
-+
-+ BUG_ON(special_handler->processor_id != smp_processor_id());
-+ memset(&fd, 0, sizeof(fd));
-+ qm_fd_addr_set64(&fd, special_handler->addr);
-+ fd.format = qm_fd_contig_big;
-+ fd.length29 = HP_NUM_WORDS * 4;
-+ for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) {
-+ if (*p != lfsr)
-+ panic("corrupt frame data");
-+ *p ^= special_handler->tx_mixer;
-+ lfsr = do_lfsr(lfsr);
-+ }
-+ pr_info("Sending first frame\n");
-+ if (qman_enqueue(&special_handler->tx, &fd, 0))
-+ panic("qman_enqueue() failed");
-+}
-+
-+void qman_test_hotpotato(void)
-+{
-+ if (cpumask_weight(cpu_online_mask) < 2) {
-+ pr_info("qman_test_hotpotato, skip - only 1 CPU\n");
-+ return;
-+ }
-+
-+ pr_info("qman_test_hotpotato starting\n");
-+
-+ hp_cpu_list_length = 0;
-+ loop_counter = 0;
-+ hp_handler_slab = kmem_cache_create("hp_handler_slab",
-+ sizeof(struct hp_handler), L1_CACHE_BYTES,
-+ SLAB_HWCACHE_ALIGN, NULL);
-+ if (!hp_handler_slab)
-+ panic("kmem_cache_create() failed");
-+
-+ allocate_frame_data();
-+
-+ /* Init phase 1 */
-+ pr_info("Creating %d handlers per cpu...\n", HP_PER_CPU);
-+ if (on_all_cpus(create_per_cpu_handlers))
-+ panic("on_each_cpu() failed");
-+ pr_info("Number of cpus: %d, total of %d handlers\n",
-+ hp_cpu_list_length, hp_cpu_list_length * HP_PER_CPU);
-+
-+ init_phase2();
-+
-+ init_phase3();
-+
-+ preempt_disable();
-+ if (special_handler->processor_id == smp_processor_id())
-+ send_first_frame(NULL);
-+ else
-+ smp_call_function_single(special_handler->processor_id,
-+ send_first_frame, NULL, 1);
-+ preempt_enable();
-+
-+ wait_event(queue, loop_counter == HP_LOOPS);
-+ deallocate_frame_data();
-+ if (on_all_cpus(destroy_per_cpu_handlers))
-+ panic("on_each_cpu() failed");
-+ kmem_cache_destroy(hp_handler_slab);
-+ pr_info("qman_test_hotpotato finished\n");
-+}
---- /dev/null
-+++ b/drivers/staging/fsl_qbman/qman_utility.c
-@@ -0,0 +1,129 @@
-+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#include "qman_private.h"
-+
-+/* ----------------- */
-+/* --- FQID Pool --- */
-+
-+struct qman_fqid_pool {
-+ /* Base and size of the FQID range */
-+ u32 fqid_base;
-+ u32 total;
-+ /* Number of FQIDs currently "allocated" */
-+ u32 used;
-+ /* Allocation optimisation. When 'used<total', it is the index of an
-+ * available FQID. Otherwise there are no available FQIDs, and this
-+ * will be set when the next deallocation occurs. */
-+ u32 next;
-+ /* A bit-field representation of the FQID range. */
-+ unsigned long *bits;
-+};
-+
-+#define QLONG_BYTES sizeof(unsigned long)
-+#define QLONG_BITS (QLONG_BYTES * 8)
-+/* Number of 'longs' required for the given number of bits */
-+#define QNUM_LONGS(b) (((b) + QLONG_BITS - 1) / QLONG_BITS)
-+/* Shorthand for the number of bytes of same (kmalloc, memset, etc) */
-+#define QNUM_BYTES(b) (QNUM_LONGS(b) * QLONG_BYTES)
-+/* And in bits */
-+#define QNUM_BITS(b) (QNUM_LONGS(b) * QLONG_BITS)
-+
-+struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num)
-+{
-+ struct qman_fqid_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
-+ unsigned int i;
-+
-+ BUG_ON(!num);
-+ if (!pool)
-+ return NULL;
-+ pool->fqid_base = fqid_start;
-+ pool->total = num;
-+ pool->used = 0;
-+ pool->next = 0;
-+ pool->bits = kzalloc(QNUM_BYTES(num), GFP_KERNEL);
-+ if (!pool->bits) {
-+ kfree(pool);
-+ return NULL;
-+ }
-+ /* If num is not an even multiple of QLONG_BITS (or even 8, for
-+ * byte-oriented searching) then we fill the trailing bits with 1, to
-+ * make them look allocated (permanently). */
-+ for (i = num + 1; i < QNUM_BITS(num); i++)
-+ set_bit(i, pool->bits);
-+ return pool;
-+}
-+EXPORT_SYMBOL(qman_fqid_pool_create);
-+
-+int qman_fqid_pool_destroy(struct qman_fqid_pool *pool)
-+{
-+ int ret = pool->used;
-+ kfree(pool->bits);
-+ kfree(pool);
-+ return ret;
-+}
-+EXPORT_SYMBOL(qman_fqid_pool_destroy);
-+
-+int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid)
-+{
-+ int ret;
-+ if (pool->used == pool->total)
-+ return -ENOMEM;
-+ *fqid = pool->fqid_base + pool->next;
-+ ret = test_and_set_bit(pool->next, pool->bits);
-+ BUG_ON(ret);
-+ if (++pool->used == pool->total)
-+ return 0;
-+ pool->next = find_next_zero_bit(pool->bits, pool->total, pool->next);
-+ if (pool->next >= pool->total)
-+ pool->next = find_first_zero_bit(pool->bits, pool->total);
-+ BUG_ON(pool->next >= pool->total);
-+ return 0;
-+}
-+EXPORT_SYMBOL(qman_fqid_pool_alloc);
-+
-+void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid)
-+{
-+ int ret;
-+
-+ fqid -= pool->fqid_base;
-+ ret = test_and_clear_bit(fqid, pool->bits);
-+ BUG_ON(!ret);
-+ if (pool->used-- == pool->total)
-+ pool->next = fqid;
-+}
-+EXPORT_SYMBOL(qman_fqid_pool_free);
-+
-+u32 qman_fqid_pool_used(struct qman_fqid_pool *pool)
-+{
-+ return pool->used;
-+}
-+EXPORT_SYMBOL(qman_fqid_pool_used);
---- /dev/null
-+++ b/include/linux/fsl_bman.h
-@@ -0,0 +1,532 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef FSL_BMAN_H
-+#define FSL_BMAN_H
-+
-+#ifdef __cplusplus
-+extern "C" {
-+#endif
-+
-+/* Last updated for v00.79 of the BG */
-+
-+/* Portal processing (interrupt) sources */
-+#define BM_PIRQ_RCRI 0x00000002 /* RCR Ring (below threshold) */
-+#define BM_PIRQ_BSCN 0x00000001 /* Buffer depletion State Change */
-+
-+/* This wrapper represents a bit-array for the depletion state of the 64 Bman
-+ * buffer pools. */
-+struct bman_depletion {
-+ u32 __state[2];
-+};
-+#define BMAN_DEPLETION_EMPTY { { 0x00000000, 0x00000000 } }
-+#define BMAN_DEPLETION_FULL { { 0xffffffff, 0xffffffff } }
-+#define __bmdep_word(x) ((x) >> 5)
-+#define __bmdep_shift(x) ((x) & 0x1f)
-+#define __bmdep_bit(x) (0x80000000 >> __bmdep_shift(x))
-+static inline void bman_depletion_init(struct bman_depletion *c)
-+{
-+ c->__state[0] = c->__state[1] = 0;
-+}
-+static inline void bman_depletion_fill(struct bman_depletion *c)
-+{
-+ c->__state[0] = c->__state[1] = ~0;
-+}
-+static inline int bman_depletion_get(const struct bman_depletion *c, u8 bpid)
-+{
-+ return c->__state[__bmdep_word(bpid)] & __bmdep_bit(bpid);
-+}
-+static inline void bman_depletion_set(struct bman_depletion *c, u8 bpid)
-+{
-+ c->__state[__bmdep_word(bpid)] |= __bmdep_bit(bpid);
-+}
-+static inline void bman_depletion_unset(struct bman_depletion *c, u8 bpid)
-+{
-+ c->__state[__bmdep_word(bpid)] &= ~__bmdep_bit(bpid);
-+}
-+
-+/* ------------------------------------------------------- */
-+/* --- Bman data structures (and associated constants) --- */
-+
-+/* Represents s/w corenet portal mapped data structures */
-+struct bm_rcr_entry; /* RCR (Release Command Ring) entries */
-+struct bm_mc_command; /* MC (Management Command) command */
-+struct bm_mc_result; /* MC result */
-+
-+/* Code-reduction, define a wrapper for 48-bit buffers. In cases where a buffer
-+ * pool id specific to this buffer is needed (BM_RCR_VERB_CMD_BPID_MULTI,
-+ * BM_MCC_VERB_ACQUIRE), the 'bpid' field is used. */
-+struct bm_buffer {
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 __reserved1;
-+ u8 bpid;
-+ u16 hi; /* High 16-bits of 48-bit address */
-+ u32 lo; /* Low 32-bits of 48-bit address */
-+#else
-+ u32 lo;
-+ u16 hi;
-+ u8 bpid;
-+ u8 __reserved;
-+#endif
-+ };
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u64 __notaddress:16;
-+ u64 addr:48;
-+#else
-+ u64 addr:48;
-+ u64 __notaddress:16;
-+#endif
-+ };
-+ u64 opaque;
-+ };
-+} __aligned(8);
-+static inline u64 bm_buffer_get64(const struct bm_buffer *buf)
-+{
-+ return buf->addr;
-+}
-+static inline dma_addr_t bm_buf_addr(const struct bm_buffer *buf)
-+{
-+ return (dma_addr_t)buf->addr;
-+}
-+/* Macro, so we compile better if 'v' isn't always 64-bit */
-+#define bm_buffer_set64(buf, v) \
-+ do { \
-+ struct bm_buffer *__buf931 = (buf); \
-+ __buf931->hi = upper_32_bits(v); \
-+ __buf931->lo = lower_32_bits(v); \
-+ } while (0)
-+
-+/* See 1.5.3.5.4: "Release Command" */
-+struct bm_rcr_entry {
-+ union {
-+ struct {
-+ u8 __dont_write_directly__verb;
-+ u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
-+ u8 __reserved1[62];
-+ };
-+ struct bm_buffer bufs[8];
-+ };
-+} __packed;
-+#define BM_RCR_VERB_VBIT 0x80
-+#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */
-+#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20
-+#define BM_RCR_VERB_CMD_BPID_MULTI 0x30
-+#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */
-+
-+/* See 1.5.3.1: "Acquire Command" */
-+/* See 1.5.3.2: "Query Command" */
-+struct bm_mcc_acquire {
-+ u8 bpid;
-+ u8 __reserved1[62];
-+} __packed;
-+struct bm_mcc_query {
-+ u8 __reserved2[63];
-+} __packed;
-+struct bm_mc_command {
-+ u8 __dont_write_directly__verb;
-+ union {
-+ struct bm_mcc_acquire acquire;
-+ struct bm_mcc_query query;
-+ };
-+} __packed;
-+#define BM_MCC_VERB_VBIT 0x80
-+#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */
-+#define BM_MCC_VERB_CMD_ACQUIRE 0x10
-+#define BM_MCC_VERB_CMD_QUERY 0x40
-+#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */
-+
-+/* See 1.5.3.3: "Acquire Response" */
-+/* See 1.5.3.4: "Query Response" */
-+struct bm_pool_state {
-+ u8 __reserved1[32];
-+ /* "availability state" and "depletion state" */
-+ struct {
-+ u8 __reserved1[8];
-+ /* Access using bman_depletion_***() */
-+ struct bman_depletion state;
-+ } as, ds;
-+};
-+struct bm_mc_result {
-+ union {
-+ struct {
-+ u8 verb;
-+ u8 __reserved1[63];
-+ };
-+ union {
-+ struct {
-+ u8 __reserved1;
-+ u8 bpid;
-+ u8 __reserved2[62];
-+ };
-+ struct bm_buffer bufs[8];
-+ } acquire;
-+ struct bm_pool_state query;
-+ };
-+} __packed;
-+#define BM_MCR_VERB_VBIT 0x80
-+#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK
-+#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE
-+#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY
-+#define BM_MCR_VERB_CMD_ERR_INVALID 0x60
-+#define BM_MCR_VERB_CMD_ERR_ECC 0x70
-+#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
-+/* Determine the "availability state" of pool 'p' from a query result 'r' */
-+#define BM_MCR_QUERY_AVAILABILITY(r, p) \
-+ bman_depletion_get(&r->query.as.state, p)
-+/* Determine the "depletion state" of pool 'p' from a query result 'r' */
-+#define BM_MCR_QUERY_DEPLETION(r, p) \
-+ bman_depletion_get(&r->query.ds.state, p)
-+
-+/*******************************************************************/
-+/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */
-+/*******************************************************************/
-+
-+ /* Portal and Buffer Pools */
-+ /* ----------------------- */
-+/* Represents a managed portal */
-+struct bman_portal;
-+
-+/* This object type represents Bman buffer pools. */
-+struct bman_pool;
-+
-+struct bman_portal_config {
-+ /* This is used for any "core-affine" portals, ie. default portals
-+ * associated to the corresponding cpu. -1 implies that there is no core
-+ * affinity configured. */
-+ int cpu;
-+ /* portal interrupt line */
-+ int irq;
-+ /* the unique index of this portal */
-+ u32 index;
-+ /* Is this portal shared? (If so, it has coarser locking and demuxes
-+ * processing on behalf of other CPUs.) */
-+ int is_shared;
-+ /* These are the buffer pool IDs that may be used via this portal. */
-+ struct bman_depletion mask;
-+};
-+
-+/* This callback type is used when handling pool depletion entry/exit. The
-+ * 'cb_ctx' value is the opaque value associated with the pool object in
-+ * bman_new_pool(). 'depleted' is non-zero on depletion-entry, and zero on
-+ * depletion-exit. */
-+typedef void (*bman_cb_depletion)(struct bman_portal *bm,
-+ struct bman_pool *pool, void *cb_ctx, int depleted);
-+
-+/* This struct specifies parameters for a bman_pool object. */
-+struct bman_pool_params {
-+ /* index of the buffer pool to encapsulate (0-63), ignored if
-+ * BMAN_POOL_FLAG_DYNAMIC_BPID is set. */
-+ u32 bpid;
-+ /* bit-mask of BMAN_POOL_FLAG_*** options */
-+ u32 flags;
-+ /* depletion-entry/exit callback, if BMAN_POOL_FLAG_DEPLETION is set */
-+ bman_cb_depletion cb;
-+ /* opaque user value passed as a parameter to 'cb' */
-+ void *cb_ctx;
-+ /* depletion-entry/exit thresholds, if BMAN_POOL_FLAG_THRESH is set. NB:
-+ * this is only allowed if BMAN_POOL_FLAG_DYNAMIC_BPID is used *and*
-+ * when run in the control plane (which controls Bman CCSR). This array
-+ * matches the definition of bm_pool_set(). */
-+ u32 thresholds[4];
-+};
-+
-+/* Flags to bman_new_pool() */
-+#define BMAN_POOL_FLAG_NO_RELEASE 0x00000001 /* can't release to pool */
-+#define BMAN_POOL_FLAG_ONLY_RELEASE 0x00000002 /* can only release to pool */
-+#define BMAN_POOL_FLAG_DEPLETION 0x00000004 /* track depletion entry/exit */
-+#define BMAN_POOL_FLAG_DYNAMIC_BPID 0x00000008 /* (de)allocate bpid */
-+#define BMAN_POOL_FLAG_THRESH 0x00000010 /* set depletion thresholds */
-+#define BMAN_POOL_FLAG_STOCKPILE 0x00000020 /* stockpile to reduce hw ops */
-+
-+/* Flags to bman_release() */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+#define BMAN_RELEASE_FLAG_WAIT 0x00000001 /* wait if RCR is full */
-+#define BMAN_RELEASE_FLAG_WAIT_INT 0x00000002 /* if we wait, interruptible? */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+#define BMAN_RELEASE_FLAG_WAIT_SYNC 0x00000004 /* if wait, until consumed? */
-+#endif
-+#endif
-+#define BMAN_RELEASE_FLAG_NOW 0x00000008 /* issue immediate release */
-+
-+/* Flags to bman_acquire() */
-+#define BMAN_ACQUIRE_FLAG_STOCKPILE 0x00000001 /* no hw op, stockpile only */
-+
-+ /* Portal Management */
-+ /* ----------------- */
-+/**
-+ * bman_get_portal_config - get portal configuration settings
-+ *
-+ * This returns a read-only view of the current cpu's affine portal settings.
-+ */
-+const struct bman_portal_config *bman_get_portal_config(void);
-+
-+/**
-+ * bman_irqsource_get - return the portal work that is interrupt-driven
-+ *
-+ * Returns a bitmask of BM_PIRQ_**I processing sources that are currently
-+ * enabled for interrupt handling on the current cpu's affine portal. These
-+ * sources will trigger the portal interrupt and the interrupt handler (or a
-+ * tasklet/bottom-half it defers to) will perform the corresponding processing
-+ * work. The bman_poll_***() functions will only process sources that are not in
-+ * this bitmask. If the current CPU is sharing a portal hosted on another CPU,
-+ * this always returns zero.
-+ */
-+u32 bman_irqsource_get(void);
-+
-+/**
-+ * bman_irqsource_add - add processing sources to be interrupt-driven
-+ * @bits: bitmask of BM_PIRQ_**I processing sources
-+ *
-+ * Adds processing sources that should be interrupt-driven (rather than
-+ * processed via bman_poll_***() functions). Returns zero for success, or
-+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU. */
-+int bman_irqsource_add(u32 bits);
-+
-+/**
-+ * bman_irqsource_remove - remove processing sources from being interrupt-driven
-+ * @bits: bitmask of BM_PIRQ_**I processing sources
-+ *
-+ * Removes processing sources from being interrupt-driven, so that they will
-+ * instead be processed via bman_poll_***() functions. Returns zero for success,
-+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. */
-+int bman_irqsource_remove(u32 bits);
-+
-+/**
-+ * bman_affine_cpus - return a mask of cpus that have affine portals
-+ */
-+const cpumask_t *bman_affine_cpus(void);
-+
-+/**
-+ * bman_poll_slow - process anything that isn't interrupt-driven.
-+ *
-+ * This function does any portal processing that isn't interrupt-driven. If the
-+ * current CPU is sharing a portal hosted on another CPU, this function will
-+ * return -EINVAL, otherwise the return value is a bitmask of BM_PIRQ_* sources
-+ * indicating what interrupt sources were actually processed by the call.
-+ *
-+ * NB, unlike the legacy wrapper bman_poll(), this function will
-+ * deterministically check for the presence of portal processing work and do it,
-+ * which implies some latency even if there's nothing to do. The bman_poll()
-+ * wrapper on the other hand (like the qman_poll() wrapper) attenuates this by
-+ * checking for (and doing) portal processing infrequently. Ie. such that
-+ * qman_poll() and bman_poll() can be called from core-processing loops. Use
-+ * bman_poll_slow() when you yourself are deciding when to incur the overhead of
-+ * processing.
-+ */
-+u32 bman_poll_slow(void);
-+
-+/**
-+ * bman_poll - process anything that isn't interrupt-driven.
-+ *
-+ * Dispatcher logic on a cpu can use this to trigger any maintenance of the
-+ * affine portal. This function does whatever processing is not triggered by
-+ * interrupts. This is a legacy wrapper that can be used in core-processing
-+ * loops but mitigates the performance overhead of portal processing by
-+ * adaptively bypassing true portal processing most of the time. (Processing is
-+ * done once every 10 calls if the previous processing revealed that work needed
-+ * to be done, or once very 1000 calls if the previous processing revealed no
-+ * work needed doing.) If you wish to control this yourself, call
-+ * bman_poll_slow() instead, which always checks for portal processing work.
-+ */
-+void bman_poll(void);
-+
-+/**
-+ * bman_rcr_is_empty - Determine if portal's RCR is empty
-+ *
-+ * For use in situations where a cpu-affine caller needs to determine when all
-+ * releases for the local portal have been processed by Bman but can't use the
-+ * BMAN_RELEASE_FLAG_WAIT_SYNC flag to do this from the final bman_release().
-+ * The function forces tracking of RCR consumption (which normally doesn't
-+ * happen until release processing needs to find space to put new release
-+ * commands), and returns zero if the ring still has unprocessed entries,
-+ * non-zero if it is empty.
-+ */
-+int bman_rcr_is_empty(void);
-+
-+/**
-+ * bman_alloc_bpid_range - Allocate a contiguous range of BPIDs
-+ * @result: is set by the API to the base BPID of the allocated range
-+ * @count: the number of BPIDs required
-+ * @align: required alignment of the allocated range
-+ * @partial: non-zero if the API can return fewer than @count BPIDs
-+ *
-+ * Returns the number of buffer pools allocated, or a negative error code. If
-+ * @partial is non zero, the allocation request may return a smaller range of
-+ * BPs than requested (though alignment will be as requested). If @partial is
-+ * zero, the return value will either be 'count' or negative.
-+ */
-+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial);
-+static inline int bman_alloc_bpid(u32 *result)
-+{
-+ int ret = bman_alloc_bpid_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+
-+/**
-+ * bman_release_bpid_range - Release the specified range of buffer pool IDs
-+ * @bpid: the base BPID of the range to deallocate
-+ * @count: the number of BPIDs in the range
-+ *
-+ * This function can also be used to seed the allocator with ranges of BPIDs
-+ * that it can subsequently allocate from.
-+ */
-+void bman_release_bpid_range(u32 bpid, unsigned int count);
-+static inline void bman_release_bpid(u32 bpid)
-+{
-+ bman_release_bpid_range(bpid, 1);
-+}
-+
-+int bman_reserve_bpid_range(u32 bpid, unsigned int count);
-+static inline int bman_reserve_bpid(u32 bpid)
-+{
-+ return bman_reserve_bpid_range(bpid, 1);
-+}
-+
-+void bman_seed_bpid_range(u32 bpid, unsigned int count);
-+
-+
-+int bman_shutdown_pool(u32 bpid);
-+
-+ /* Pool management */
-+ /* --------------- */
-+/**
-+ * bman_new_pool - Allocates a Buffer Pool object
-+ * @params: parameters specifying the buffer pool ID and behaviour
-+ *
-+ * Creates a pool object for the given @params. A portal and the depletion
-+ * callback field of @params are only used if the BMAN_POOL_FLAG_DEPLETION flag
-+ * is set. NB, the fields from @params are copied into the new pool object, so
-+ * the structure provided by the caller can be released or reused after the
-+ * function returns.
-+ */
-+struct bman_pool *bman_new_pool(const struct bman_pool_params *params);
-+
-+/**
-+ * bman_free_pool - Deallocates a Buffer Pool object
-+ * @pool: the pool object to release
-+ *
-+ */
-+void bman_free_pool(struct bman_pool *pool);
-+
-+/**
-+ * bman_get_params - Returns a pool object's parameters.
-+ * @pool: the pool object
-+ *
-+ * The returned pointer refers to state within the pool object so must not be
-+ * modified and can no longer be read once the pool object is destroyed.
-+ */
-+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool);
-+
-+/**
-+ * bman_release - Release buffer(s) to the buffer pool
-+ * @pool: the buffer pool object to release to
-+ * @bufs: an array of buffers to release
-+ * @num: the number of buffers in @bufs (1-8)
-+ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options
-+ *
-+ * Adds the given buffers to RCR entries. If the portal @p was created with the
-+ * "COMPACT" flag, then it will be using a compaction algorithm to improve
-+ * utilisation of RCR. As such, these buffers may join an existing ring entry
-+ * and/or it may not be issued right away so as to allow future releases to join
-+ * the same ring entry. Use the BMAN_RELEASE_FLAG_NOW flag to override this
-+ * behaviour by committing the RCR entry (or entries) right away. If the RCR
-+ * ring is full, the function will return -EBUSY unless BMAN_RELEASE_FLAG_WAIT
-+ * is selected, in which case it will sleep waiting for space to become
-+ * available in RCR. If the function receives a signal before such time (and
-+ * BMAN_RELEASE_FLAG_WAIT_INT is set), the function returns -EINTR. Otherwise,
-+ * it returns zero.
-+ */
-+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
-+ u32 flags);
-+
-+/**
-+ * bman_acquire - Acquire buffer(s) from a buffer pool
-+ * @pool: the buffer pool object to acquire from
-+ * @bufs: array for storing the acquired buffers
-+ * @num: the number of buffers desired (@bufs is at least this big)
-+ *
-+ * Issues an "Acquire" command via the portal's management command interface.
-+ * The return value will be the number of buffers obtained from the pool, or a
-+ * negative error code if a h/w error or pool starvation was encountered. In
-+ * the latter case, the content of @bufs is undefined.
-+ */
-+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
-+ u32 flags);
-+
-+/**
-+ * bman_flush_stockpile - Flush stockpile buffer(s) to the buffer pool
-+ * @pool: the buffer pool object the stockpile belongs
-+ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options
-+ *
-+ * Adds stockpile buffers to RCR entries until the stockpile is empty.
-+ * The return value will be a negative error code if a h/w error occurred.
-+ * If BMAN_RELEASE_FLAG_NOW flag is passed and RCR ring is full,
-+ * -EAGAIN will be returned.
-+ */
-+int bman_flush_stockpile(struct bman_pool *pool, u32 flags);
-+
-+/**
-+ * bman_query_pools - Query all buffer pool states
-+ * @state: storage for the queried availability and depletion states
-+ */
-+int bman_query_pools(struct bm_pool_state *state);
-+
-+#ifdef CONFIG_FSL_BMAN_CONFIG
-+/**
-+ * bman_query_free_buffers - Query how many free buffers are in buffer pool
-+ * @pool: the buffer pool object to query
-+ *
-+ * Return the number of the free buffers
-+ */
-+u32 bman_query_free_buffers(struct bman_pool *pool);
-+
-+/**
-+ * bman_update_pool_thresholds - Change the buffer pool's depletion thresholds
-+ * @pool: the buffer pool object to which the thresholds will be set
-+ * @thresholds: the new thresholds
-+ */
-+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds);
-+#endif
-+
-+/**
-+ * The below bman_p_***() variant might be called in a situation that the cpu
-+ * which the portal affine to is not online yet.
-+ * @bman_portal specifies which portal the API will use.
-+*/
-+int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits);
-+#ifdef __cplusplus
-+}
-+#endif
-+
-+#endif /* FSL_BMAN_H */
---- /dev/null
-+++ b/include/linux/fsl_qman.h
-@@ -0,0 +1,3888 @@
-+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are met:
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in the
-+ * documentation and/or other materials provided with the distribution.
-+ * * Neither the name of Freescale Semiconductor nor the
-+ * names of its contributors may be used to endorse or promote products
-+ * derived from this software without specific prior written permission.
-+ *
-+ *
-+ * ALTERNATIVELY, this software may be distributed under the terms of the
-+ * GNU General Public License ("GPL") as published by the Free Software
-+ * Foundation, either version 2 of that License or (at your option) any
-+ * later version.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
-+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
-+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ */
-+
-+#ifndef FSL_QMAN_H
-+#define FSL_QMAN_H
-+
-+#ifdef __cplusplus
-+extern "C" {
-+#endif
-+
-+/* Last updated for v00.800 of the BG */
-+
-+/* Hardware constants */
-+#define QM_CHANNEL_SWPORTAL0 0
-+#define QMAN_CHANNEL_POOL1 0x21
-+#define QMAN_CHANNEL_CAAM 0x80
-+#define QMAN_CHANNEL_PME 0xa0
-+#define QMAN_CHANNEL_POOL1_REV3 0x401
-+#define QMAN_CHANNEL_CAAM_REV3 0x840
-+#define QMAN_CHANNEL_PME_REV3 0x860
-+#define QMAN_CHANNEL_DCE 0x8a0
-+#define QMAN_CHANNEL_DCE_QMANREV312 0x880
-+extern u16 qm_channel_pool1;
-+extern u16 qm_channel_caam;
-+extern u16 qm_channel_pme;
-+extern u16 qm_channel_dce;
-+enum qm_dc_portal {
-+ qm_dc_portal_fman0 = 0,
-+ qm_dc_portal_fman1 = 1,
-+ qm_dc_portal_caam = 2,
-+ qm_dc_portal_pme = 3,
-+ qm_dc_portal_rman = 4,
-+ qm_dc_portal_dce = 5
-+};
-+
-+/* Portal processing (interrupt) sources */
-+#define QM_PIRQ_CCSCI 0x00200000 /* CEETM Congestion State Change */
-+#define QM_PIRQ_CSCI 0x00100000 /* Congestion State Change */
-+#define QM_PIRQ_EQCI 0x00080000 /* Enqueue Command Committed */
-+#define QM_PIRQ_EQRI 0x00040000 /* EQCR Ring (below threshold) */
-+#define QM_PIRQ_DQRI 0x00020000 /* DQRR Ring (non-empty) */
-+#define QM_PIRQ_MRI 0x00010000 /* MR Ring (non-empty) */
-+/* This mask contains all the interrupt sources that need handling except DQRI,
-+ * ie. that if present should trigger slow-path processing. */
-+#define QM_PIRQ_SLOW (QM_PIRQ_CSCI | QM_PIRQ_EQCI | QM_PIRQ_EQRI | \
-+ QM_PIRQ_MRI | QM_PIRQ_CCSCI)
-+
-+/* --- Clock speed --- */
-+/* A qman driver instance may or may not know the current qman clock speed.
-+ * However, certain CEETM calculations may not be possible if this is not known.
-+ * The 'set' function will only succeed (return zero) if the driver did not
-+ * already know the clock speed. Likewise, the 'get' function will only succeed
-+ * if the driver does know the clock speed (either because it knew when booting,
-+ * or was told via 'set'). In cases where software is running on a driver
-+ * instance that does not know the clock speed (eg. on a hypervised data-plane),
-+ * and the user can obtain the current qman clock speed by other means (eg. from
-+ * a message sent from the control-plane), then the 'set' function can be used
-+ * to enable rate-calculations in a driver where it would otherwise not be
-+ * possible. */
-+int qm_get_clock(u64 *clock_hz);
-+int qm_set_clock(u64 clock_hz);
-+
-+/* For qman_static_dequeue_*** APIs */
-+#define QM_SDQCR_CHANNELS_POOL_MASK 0x00007fff
-+/* for n in [1,15] */
-+#define QM_SDQCR_CHANNELS_POOL(n) (0x00008000 >> (n))
-+/* for conversion from n of qm_channel */
-+static inline u32 QM_SDQCR_CHANNELS_POOL_CONV(u16 channel)
-+{
-+ return QM_SDQCR_CHANNELS_POOL(channel + 1 - qm_channel_pool1);
-+}
-+
-+/* For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use
-+ * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use
-+ * FQID(n) to fill in the frame queue ID. */
-+#define QM_VDQCR_PRECEDENCE_VDQCR 0x0
-+#define QM_VDQCR_PRECEDENCE_SDQCR 0x80000000
-+#define QM_VDQCR_EXACT 0x40000000
-+#define QM_VDQCR_NUMFRAMES_MASK 0x3f000000
-+#define QM_VDQCR_NUMFRAMES_SET(n) (((n) & 0x3f) << 24)
-+#define QM_VDQCR_NUMFRAMES_GET(n) (((n) >> 24) & 0x3f)
-+#define QM_VDQCR_NUMFRAMES_TILLEMPTY QM_VDQCR_NUMFRAMES_SET(0)
-+
-+
-+/* ------------------------------------------------------- */
-+/* --- Qman data structures (and associated constants) --- */
-+
-+/* Represents s/w corenet portal mapped data structures */
-+struct qm_eqcr_entry; /* EQCR (EnQueue Command Ring) entries */
-+struct qm_dqrr_entry; /* DQRR (DeQueue Response Ring) entries */
-+struct qm_mr_entry; /* MR (Message Ring) entries */
-+struct qm_mc_command; /* MC (Management Command) command */
-+struct qm_mc_result; /* MC result */
-+
-+/* See David Lapp's "Frame formats" document, "dpateam", Jan 07, 2008 */
-+#define QM_FD_FORMAT_SG 0x4
-+#define QM_FD_FORMAT_LONG 0x2
-+#define QM_FD_FORMAT_COMPOUND 0x1
-+enum qm_fd_format {
-+ /* 'contig' implies a contiguous buffer, whereas 'sg' implies a
-+ * scatter-gather table. 'big' implies a 29-bit length with no offset
-+ * field, otherwise length is 20-bit and offset is 9-bit. 'compound'
-+ * implies a s/g-like table, where each entry itself represents a frame
-+ * (contiguous or scatter-gather) and the 29-bit "length" is
-+ * interpreted purely for congestion calculations, ie. a "congestion
-+ * weight". */
-+ qm_fd_contig = 0,
-+ qm_fd_contig_big = QM_FD_FORMAT_LONG,
-+ qm_fd_sg = QM_FD_FORMAT_SG,
-+ qm_fd_sg_big = QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG,
-+ qm_fd_compound = QM_FD_FORMAT_COMPOUND
-+};
-+
-+/* Capitalised versions are un-typed but can be used in static expressions */
-+#define QM_FD_CONTIG 0
-+#define QM_FD_CONTIG_BIG QM_FD_FORMAT_LONG
-+#define QM_FD_SG QM_FD_FORMAT_SG
-+#define QM_FD_SG_BIG (QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG)
-+#define QM_FD_COMPOUND QM_FD_FORMAT_COMPOUND
-+
-+/* See 1.5.1.1: "Frame Descriptor (FD)" */
-+struct qm_fd {
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 dd:2; /* dynamic debug */
-+ u8 liodn_offset:6;
-+ u8 bpid:8; /* Buffer Pool ID */
-+ u8 eliodn_offset:4;
-+ u8 __reserved:4;
-+ u8 addr_hi; /* high 8-bits of 40-bit address */
-+ u32 addr_lo; /* low 32-bits of 40-bit address */
-+#else
-+ u32 addr_lo; /* low 32-bits of 40-bit address */
-+ u8 addr_hi; /* high 8-bits of 40-bit address */
-+ u8 __reserved:4;
-+ u8 eliodn_offset:4;
-+ u8 bpid:8; /* Buffer Pool ID */
-+ u8 liodn_offset:6;
-+ u8 dd:2; /* dynamic debug */
-+#endif
-+ };
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u64 __notaddress:24;
-+ u64 addr:40;
-+#else
-+ u64 addr:40;
-+ u64 __notaddress:24;
-+#endif
-+ };
-+ u64 opaque_addr;
-+ };
-+ /* The 'format' field indicates the interpretation of the remaining 29
-+ * bits of the 32-bit word. For packing reasons, it is duplicated in the
-+ * other union elements. Note, union'd structs are difficult to use with
-+ * static initialisation under gcc, in which case use the "opaque" form
-+ * with one of the macros. */
-+ union {
-+ /* For easier/faster copying of this part of the fd (eg. from a
-+ * DQRR entry to an EQCR entry) copy 'opaque' */
-+ u32 opaque;
-+ /* If 'format' is _contig or _sg, 20b length and 9b offset */
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ enum qm_fd_format format:3;
-+ u16 offset:9;
-+ u32 length20:20;
-+#else
-+ u32 length20:20;
-+ u16 offset:9;
-+ enum qm_fd_format format:3;
-+#endif
-+ };
-+ /* If 'format' is _contig_big or _sg_big, 29b length */
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ enum qm_fd_format _format1:3;
-+ u32 length29:29;
-+#else
-+ u32 length29:29;
-+ enum qm_fd_format _format1:3;
-+#endif
-+ };
-+ /* If 'format' is _compound, 29b "congestion weight" */
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ enum qm_fd_format _format2:3;
-+ u32 cong_weight:29;
-+#else
-+ u32 cong_weight:29;
-+ enum qm_fd_format _format2:3;
-+#endif
-+ };
-+ };
-+ union {
-+ u32 cmd;
-+ u32 status;
-+ };
-+} __aligned(8);
-+#define QM_FD_DD_NULL 0x00
-+#define QM_FD_PID_MASK 0x3f
-+static inline u64 qm_fd_addr_get64(const struct qm_fd *fd)
-+{
-+ return fd->addr;
-+}
-+
-+static inline dma_addr_t qm_fd_addr(const struct qm_fd *fd)
-+{
-+ return (dma_addr_t)fd->addr;
-+}
-+/* Macro, so we compile better if 'v' isn't always 64-bit */
-+#define qm_fd_addr_set64(fd, v) \
-+ do { \
-+ struct qm_fd *__fd931 = (fd); \
-+ __fd931->addr = v; \
-+ } while (0)
-+
-+/* For static initialisation of FDs (which is complicated by the use of unions
-+ * in "struct qm_fd"), use the following macros. Note that;
-+ * - 'dd', 'pid' and 'bpid' are ignored because there's no static initialisation
-+ * use-case),
-+ * - use capitalised QM_FD_*** formats for static initialisation.
-+ */
-+#define QM_FD_FMT_20(cmd, addr_hi, addr_lo, fmt, off, len) \
-+ { 0, 0, 0, 0, 0, addr_hi, addr_lo, \
-+ { (((fmt)&0x7) << 29) | (((off)&0x1ff) << 20) | ((len)&0xfffff) }, \
-+ { cmd } }
-+#define QM_FD_FMT_29(cmd, addr_hi, addr_lo, fmt, len) \
-+ { 0, 0, 0, 0, 0, addr_hi, addr_lo, \
-+ { (((fmt)&0x7) << 29) | ((len)&0x1fffffff) }, \
-+ { cmd } }
-+
-+/* See 2.2.1.3 Multi-Core Datapath Acceleration Architecture */
-+#define QM_SG_OFFSET_MASK 0x1FFF
-+struct qm_sg_entry {
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 __reserved1[3];
-+ u8 addr_hi; /* high 8-bits of 40-bit address */
-+ u32 addr_lo; /* low 32-bits of 40-bit address */
-+#else
-+ u32 addr_lo; /* low 32-bits of 40-bit address */
-+ u8 addr_hi; /* high 8-bits of 40-bit address */
-+ u8 __reserved1[3];
-+#endif
-+ };
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u64 __notaddress:24;
-+ u64 addr:40;
-+#else
-+ u64 addr:40;
-+ u64 __notaddress:24;
-+#endif
-+ };
-+ u64 opaque;
-+ };
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 extension:1; /* Extension bit */
-+ u32 final:1; /* Final bit */
-+ u32 length:30;
-+#else
-+ u32 length:30;
-+ u32 final:1; /* Final bit */
-+ u32 extension:1; /* Extension bit */
-+#endif
-+ };
-+ u32 sgt_efl;
-+ };
-+ u8 __reserved2;
-+ u8 bpid;
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 __reserved3:3;
-+ u16 offset:13;
-+#else
-+ u16 offset:13;
-+ u16 __reserved3:3;
-+#endif
-+ };
-+ u16 opaque_offset;
-+ };
-+} __packed;
-+union qm_sg_efl {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 extension:1; /* Extension bit */
-+ u32 final:1; /* Final bit */
-+ u32 length:30;
-+#else
-+ u32 length:30;
-+ u32 final:1; /* Final bit */
-+ u32 extension:1; /* Extension bit */
-+#endif
-+ };
-+ u32 efl;
-+};
-+static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg)
-+{
-+ return (dma_addr_t)be64_to_cpu(sg->opaque) & 0xffffffffffULL;
-+}
-+static inline u8 qm_sg_entry_get_ext(const struct qm_sg_entry *sg)
-+{
-+ union qm_sg_efl u;
-+
-+ u.efl = be32_to_cpu(sg->sgt_efl);
-+ return u.extension;
-+}
-+static inline u8 qm_sg_entry_get_final(const struct qm_sg_entry *sg)
-+{
-+ union qm_sg_efl u;
-+
-+ u.efl = be32_to_cpu(sg->sgt_efl);
-+ return u.final;
-+}
-+static inline u32 qm_sg_entry_get_len(const struct qm_sg_entry *sg)
-+{
-+ union qm_sg_efl u;
-+
-+ u.efl = be32_to_cpu(sg->sgt_efl);
-+ return u.length;
-+}
-+static inline u8 qm_sg_entry_get_bpid(const struct qm_sg_entry *sg)
-+{
-+ return sg->bpid;
-+}
-+static inline u16 qm_sg_entry_get_offset(const struct qm_sg_entry *sg)
-+{
-+ u32 opaque_offset = be16_to_cpu(sg->opaque_offset);
-+
-+ return opaque_offset & 0x1fff;
-+}
-+
-+/* Macro, so we compile better if 'v' isn't always 64-bit */
-+#define qm_sg_entry_set64(sg, v) \
-+ do { \
-+ struct qm_sg_entry *__sg931 = (sg); \
-+ __sg931->opaque = cpu_to_be64(v); \
-+ } while (0)
-+#define qm_sg_entry_set_ext(sg, v) \
-+ do { \
-+ union qm_sg_efl __u932; \
-+ __u932.efl = be32_to_cpu((sg)->sgt_efl); \
-+ __u932.extension = v; \
-+ (sg)->sgt_efl = cpu_to_be32(__u932.efl); \
-+ } while (0)
-+#define qm_sg_entry_set_final(sg, v) \
-+ do { \
-+ union qm_sg_efl __u933; \
-+ __u933.efl = be32_to_cpu((sg)->sgt_efl); \
-+ __u933.final = v; \
-+ (sg)->sgt_efl = cpu_to_be32(__u933.efl); \
-+ } while (0)
-+#define qm_sg_entry_set_len(sg, v) \
-+ do { \
-+ union qm_sg_efl __u934; \
-+ __u934.efl = be32_to_cpu((sg)->sgt_efl); \
-+ __u934.length = v; \
-+ (sg)->sgt_efl = cpu_to_be32(__u934.efl); \
-+ } while (0)
-+#define qm_sg_entry_set_bpid(sg, v) \
-+ do { \
-+ struct qm_sg_entry *__u935 = (sg); \
-+ __u935->bpid = v; \
-+ } while (0)
-+#define qm_sg_entry_set_offset(sg, v) \
-+ do { \
-+ struct qm_sg_entry *__u936 = (sg); \
-+ __u936->opaque_offset = cpu_to_be16(v); \
-+ } while (0)
-+
-+/* See 1.5.8.1: "Enqueue Command" */
-+struct qm_eqcr_entry {
-+ u8 __dont_write_directly__verb;
-+ u8 dca;
-+ u16 seqnum;
-+ u32 orp; /* 24-bit */
-+ u32 fqid; /* 24-bit */
-+ u32 tag;
-+ struct qm_fd fd;
-+ u8 __reserved3[32];
-+} __packed;
-+#define QM_EQCR_VERB_VBIT 0x80
-+#define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */
-+#define QM_EQCR_VERB_CMD_ENQUEUE 0x01
-+#define QM_EQCR_VERB_COLOUR_MASK 0x18 /* 4 possible values; */
-+#define QM_EQCR_VERB_COLOUR_GREEN 0x00
-+#define QM_EQCR_VERB_COLOUR_YELLOW 0x08
-+#define QM_EQCR_VERB_COLOUR_RED 0x10
-+#define QM_EQCR_VERB_COLOUR_OVERRIDE 0x18
-+#define QM_EQCR_VERB_INTERRUPT 0x04 /* on command consumption */
-+#define QM_EQCR_VERB_ORP 0x02 /* enable order restoration */
-+#define QM_EQCR_DCA_ENABLE 0x80
-+#define QM_EQCR_DCA_PARK 0x40
-+#define QM_EQCR_DCA_IDXMASK 0x0f /* "DQRR::idx" goes here */
-+#define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */
-+#define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */
-+#define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */
-+#define QM_EQCR_FQID_NULL 0 /* eg. for an ORP seqnum hole */
-+
-+/* See 1.5.8.2: "Frame Dequeue Response" */
-+struct qm_dqrr_entry {
-+ u8 verb;
-+ u8 stat;
-+ u16 seqnum; /* 15-bit */
-+ u8 tok;
-+ u8 __reserved2[3];
-+ u32 fqid; /* 24-bit */
-+ u32 contextB;
-+ struct qm_fd fd;
-+ u8 __reserved4[32];
-+};
-+#define QM_DQRR_VERB_VBIT 0x80
-+#define QM_DQRR_VERB_MASK 0x7f /* where the verb contains; */
-+#define QM_DQRR_VERB_FRAME_DEQUEUE 0x60 /* "this format" */
-+#define QM_DQRR_STAT_FQ_EMPTY 0x80 /* FQ empty */
-+#define QM_DQRR_STAT_FQ_HELDACTIVE 0x40 /* FQ held active */
-+#define QM_DQRR_STAT_FQ_FORCEELIGIBLE 0x20 /* FQ was force-eligible'd */
-+#define QM_DQRR_STAT_FD_VALID 0x10 /* has a non-NULL FD */
-+#define QM_DQRR_STAT_UNSCHEDULED 0x02 /* Unscheduled dequeue */
-+#define QM_DQRR_STAT_DQCR_EXPIRED 0x01 /* VDQCR or PDQCR expired*/
-+
-+/* See 1.5.8.3: "ERN Message Response" */
-+/* See 1.5.8.4: "FQ State Change Notification" */
-+struct qm_mr_entry {
-+ u8 verb;
-+ union {
-+ struct {
-+ u8 dca;
-+ u16 seqnum;
-+ u8 rc; /* Rejection Code */
-+ u32 orp:24;
-+ u32 fqid; /* 24-bit */
-+ u32 tag;
-+ struct qm_fd fd;
-+ } __packed ern;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */
-+ u8 __reserved1:3;
-+ enum qm_dc_portal portal:3;
-+#else
-+ enum qm_dc_portal portal:3;
-+ u8 __reserved1:3;
-+ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */
-+#endif
-+ u16 __reserved2;
-+ u8 rc; /* Rejection Code */
-+ u32 __reserved3:24;
-+ u32 fqid; /* 24-bit */
-+ u32 tag;
-+ struct qm_fd fd;
-+ } __packed dcern;
-+ struct {
-+ u8 fqs; /* Frame Queue Status */
-+ u8 __reserved1[6];
-+ u32 fqid; /* 24-bit */
-+ u32 contextB;
-+ u8 __reserved2[16];
-+ } __packed fq; /* FQRN/FQRNI/FQRL/FQPN */
-+ };
-+ u8 __reserved2[32];
-+} __packed;
-+#define QM_MR_VERB_VBIT 0x80
-+/* The "ern" VERB bits match QM_EQCR_VERB_*** so aren't reproduced here. ERNs
-+ * originating from direct-connect portals ("dcern") use 0x20 as a verb which
-+ * would be invalid as a s/w enqueue verb. A s/w ERN can be distinguished from
-+ * the other MR types by noting if the 0x20 bit is unset. */
-+#define QM_MR_VERB_TYPE_MASK 0x27
-+#define QM_MR_VERB_DC_ERN 0x20
-+#define QM_MR_VERB_FQRN 0x21
-+#define QM_MR_VERB_FQRNI 0x22
-+#define QM_MR_VERB_FQRL 0x23
-+#define QM_MR_VERB_FQPN 0x24
-+#define QM_MR_RC_MASK 0xf0 /* contains one of; */
-+#define QM_MR_RC_CGR_TAILDROP 0x00
-+#define QM_MR_RC_WRED 0x10
-+#define QM_MR_RC_ERROR 0x20
-+#define QM_MR_RC_ORPWINDOW_EARLY 0x30
-+#define QM_MR_RC_ORPWINDOW_LATE 0x40
-+#define QM_MR_RC_FQ_TAILDROP 0x50
-+#define QM_MR_RC_ORPWINDOW_RETIRED 0x60
-+#define QM_MR_RC_ORP_ZERO 0x70
-+#define QM_MR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */
-+#define QM_MR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */
-+#define QM_MR_DCERN_COLOUR_GREEN 0x00
-+#define QM_MR_DCERN_COLOUR_YELLOW 0x01
-+#define QM_MR_DCERN_COLOUR_RED 0x02
-+#define QM_MR_DCERN_COLOUR_OVERRIDE 0x03
-+
-+/* An identical structure of FQD fields is present in the "Init FQ" command and
-+ * the "Query FQ" result, it's suctioned out into the "struct qm_fqd" type.
-+ * Within that, the 'stashing' and 'taildrop' pieces are also factored out, the
-+ * latter has two inlines to assist with converting to/from the mant+exp
-+ * representation. */
-+struct qm_fqd_stashing {
-+ /* See QM_STASHING_EXCL_<...> */
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 exclusive;
-+ u8 __reserved1:2;
-+ /* Numbers of cachelines */
-+ u8 annotation_cl:2;
-+ u8 data_cl:2;
-+ u8 context_cl:2;
-+#else
-+ u8 context_cl:2;
-+ u8 data_cl:2;
-+ u8 annotation_cl:2;
-+ u8 __reserved1:2;
-+ u8 exclusive;
-+#endif
-+} __packed;
-+struct qm_fqd_taildrop {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 __reserved1:3;
-+ u16 mant:8;
-+ u16 exp:5;
-+#else
-+ u16 exp:5;
-+ u16 mant:8;
-+ u16 __reserved1:3;
-+#endif
-+} __packed;
-+struct qm_fqd_oac {
-+ /* See QM_OAC_<...> */
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 oac:2; /* "Overhead Accounting Control" */
-+ u8 __reserved1:6;
-+#else
-+ u8 __reserved1:6;
-+ u8 oac:2; /* "Overhead Accounting Control" */
-+#endif
-+ /* Two's-complement value (-128 to +127) */
-+ signed char oal; /* "Overhead Accounting Length" */
-+} __packed;
-+struct qm_fqd {
-+ union {
-+ u8 orpc;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 __reserved1:2;
-+ u8 orprws:3;
-+ u8 oa:1;
-+ u8 olws:2;
-+#else
-+ u8 olws:2;
-+ u8 oa:1;
-+ u8 orprws:3;
-+ u8 __reserved1:2;
-+#endif
-+ } __packed;
-+ };
-+ u8 cgid;
-+ u16 fq_ctrl; /* See QM_FQCTRL_<...> */
-+ union {
-+ u16 dest_wq;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 channel:13; /* qm_channel */
-+ u16 wq:3;
-+#else
-+ u16 wq:3;
-+ u16 channel:13; /* qm_channel */
-+#endif
-+ } __packed dest;
-+ };
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 __reserved2:1;
-+ u16 ics_cred:15;
-+#else
-+ u16 __reserved2:1;
-+ u16 ics_cred:15;
-+#endif
-+ /* For "Initialize Frame Queue" commands, the write-enable mask
-+ * determines whether 'td' or 'oac_init' is observed. For query
-+ * commands, this field is always 'td', and 'oac_query' (below) reflects
-+ * the Overhead ACcounting values. */
-+ union {
-+ struct qm_fqd_taildrop td;
-+ struct qm_fqd_oac oac_init;
-+ };
-+ u32 context_b;
-+ union {
-+ /* Treat it as 64-bit opaque */
-+ u64 opaque;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 hi;
-+ u32 lo;
-+#else
-+ u32 lo;
-+ u32 hi;
-+#endif
-+ };
-+ /* Treat it as s/w portal stashing config */
-+ /* See 1.5.6.7.1: "FQD Context_A field used for [...] */
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ struct qm_fqd_stashing stashing;
-+ /* 48-bit address of FQ context to
-+ * stash, must be cacheline-aligned */
-+ u16 context_hi;
-+ u32 context_lo;
-+#else
-+ u32 context_lo;
-+ u16 context_hi;
-+ struct qm_fqd_stashing stashing;
-+#endif
-+ } __packed;
-+ } context_a;
-+ struct qm_fqd_oac oac_query;
-+} __packed;
-+/* 64-bit converters for context_hi/lo */
-+static inline u64 qm_fqd_stashing_get64(const struct qm_fqd *fqd)
-+{
-+ return ((u64)fqd->context_a.context_hi << 32) |
-+ (u64)fqd->context_a.context_lo;
-+}
-+static inline dma_addr_t qm_fqd_stashing_addr(const struct qm_fqd *fqd)
-+{
-+ return (dma_addr_t)qm_fqd_stashing_get64(fqd);
-+}
-+static inline u64 qm_fqd_context_a_get64(const struct qm_fqd *fqd)
-+{
-+ return ((u64)fqd->context_a.hi << 32) |
-+ (u64)fqd->context_a.lo;
-+}
-+/* Macro, so we compile better when 'v' isn't necessarily 64-bit */
-+#define qm_fqd_stashing_set64(fqd, v) \
-+ do { \
-+ struct qm_fqd *__fqd931 = (fqd); \
-+ __fqd931->context_a.context_hi = upper_32_bits(v); \
-+ __fqd931->context_a.context_lo = lower_32_bits(v); \
-+ } while (0)
-+#define qm_fqd_context_a_set64(fqd, v) \
-+ do { \
-+ struct qm_fqd *__fqd931 = (fqd); \
-+ __fqd931->context_a.hi = upper_32_bits(v); \
-+ __fqd931->context_a.lo = lower_32_bits(v); \
-+ } while (0)
-+/* convert a threshold value into mant+exp representation */
-+static inline int qm_fqd_taildrop_set(struct qm_fqd_taildrop *td, u32 val,
-+ int roundup)
-+{
-+ u32 e = 0;
-+ int oddbit = 0;
-+ if (val > 0xe0000000)
-+ return -ERANGE;
-+ while (val > 0xff) {
-+ oddbit = val & 1;
-+ val >>= 1;
-+ e++;
-+ if (roundup && oddbit)
-+ val++;
-+ }
-+ td->exp = e;
-+ td->mant = val;
-+ return 0;
-+}
-+/* and the other direction */
-+static inline u32 qm_fqd_taildrop_get(const struct qm_fqd_taildrop *td)
-+{
-+ return (u32)td->mant << td->exp;
-+}
-+
-+/* See 1.5.2.2: "Frame Queue Descriptor (FQD)" */
-+/* Frame Queue Descriptor (FQD) field 'fq_ctrl' uses these constants */
-+#define QM_FQCTRL_MASK 0x07ff /* 'fq_ctrl' flags; */
-+#define QM_FQCTRL_CGE 0x0400 /* Congestion Group Enable */
-+#define QM_FQCTRL_TDE 0x0200 /* Tail-Drop Enable */
-+#define QM_FQCTRL_ORP 0x0100 /* ORP Enable */
-+#define QM_FQCTRL_CTXASTASHING 0x0080 /* Context-A stashing */
-+#define QM_FQCTRL_CPCSTASH 0x0040 /* CPC Stash Enable */
-+#define QM_FQCTRL_FORCESFDR 0x0008 /* High-priority SFDRs */
-+#define QM_FQCTRL_AVOIDBLOCK 0x0004 /* Don't block active */
-+#define QM_FQCTRL_HOLDACTIVE 0x0002 /* Hold active in portal */
-+#define QM_FQCTRL_PREFERINCACHE 0x0001 /* Aggressively cache FQD */
-+#define QM_FQCTRL_LOCKINCACHE QM_FQCTRL_PREFERINCACHE /* older naming */
-+
-+/* See 1.5.6.7.1: "FQD Context_A field used for [...] */
-+/* Frame Queue Descriptor (FQD) field 'CONTEXT_A' uses these constants */
-+#define QM_STASHING_EXCL_ANNOTATION 0x04
-+#define QM_STASHING_EXCL_DATA 0x02
-+#define QM_STASHING_EXCL_CTX 0x01
-+
-+/* See 1.5.5.3: "Intra Class Scheduling" */
-+/* FQD field 'OAC' (Overhead ACcounting) uses these constants */
-+#define QM_OAC_ICS 0x2 /* Accounting for Intra-Class Scheduling */
-+#define QM_OAC_CG 0x1 /* Accounting for Congestion Groups */
-+
-+/* See 1.5.8.4: "FQ State Change Notification" */
-+/* This struct represents the 32-bit "WR_PARM_[GYR]" parameters in CGR fields
-+ * and associated commands/responses. The WRED parameters are calculated from
-+ * these fields as follows;
-+ * MaxTH = MA * (2 ^ Mn)
-+ * Slope = SA / (2 ^ Sn)
-+ * MaxP = 4 * (Pn + 1)
-+ */
-+struct qm_cgr_wr_parm {
-+ union {
-+ u32 word;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 MA:8;
-+ u32 Mn:5;
-+ u32 SA:7; /* must be between 64-127 */
-+ u32 Sn:6;
-+ u32 Pn:6;
-+#else
-+ u32 Pn:6;
-+ u32 Sn:6;
-+ u32 SA:7; /* must be between 64-127 */
-+ u32 Mn:5;
-+ u32 MA:8;
-+#endif
-+ } __packed;
-+ };
-+} __packed;
-+/* This struct represents the 13-bit "CS_THRES" CGR field. In the corresponding
-+ * management commands, this is padded to a 16-bit structure field, so that's
-+ * how we represent it here. The congestion state threshold is calculated from
-+ * these fields as follows;
-+ * CS threshold = TA * (2 ^ Tn)
-+ */
-+struct qm_cgr_cs_thres {
-+ union {
-+ u16 hword;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 __reserved:3;
-+ u16 TA:8;
-+ u16 Tn:5;
-+#else
-+ u16 Tn:5;
-+ u16 TA:8;
-+ u16 __reserved:3;
-+#endif
-+ } __packed;
-+ };
-+} __packed;
-+/* This identical structure of CGR fields is present in the "Init/Modify CGR"
-+ * commands and the "Query CGR" result. It's suctioned out here into its own
-+ * struct. */
-+struct __qm_mc_cgr {
-+ struct qm_cgr_wr_parm wr_parm_g;
-+ struct qm_cgr_wr_parm wr_parm_y;
-+ struct qm_cgr_wr_parm wr_parm_r;
-+ u8 wr_en_g; /* boolean, use QM_CGR_EN */
-+ u8 wr_en_y; /* boolean, use QM_CGR_EN */
-+ u8 wr_en_r; /* boolean, use QM_CGR_EN */
-+ u8 cscn_en; /* boolean, use QM_CGR_EN */
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */
-+ u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */
-+#else
-+ u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */
-+ u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */
-+#endif
-+ };
-+ u32 cscn_targ; /* use QM_CGR_TARG_* */
-+ };
-+ u8 cstd_en; /* boolean, use QM_CGR_EN */
-+ u8 cs; /* boolean, only used in query response */
-+ union {
-+ /* use qm_cgr_cs_thres_set64() */
-+ struct qm_cgr_cs_thres cs_thres;
-+ u16 __cs_thres;
-+ };
-+ u8 mode; /* QMAN_CGR_MODE_FRAME not supported in rev1.0 */
-+} __packed;
-+#define QM_CGR_EN 0x01 /* For wr_en_*, cscn_en, cstd_en */
-+#define QM_CGR_TARG_UDP_CTRL_WRITE_BIT 0x8000 /* value written to portal bit*/
-+#define QM_CGR_TARG_UDP_CTRL_DCP 0x4000 /* 0: SWP, 1: DCP */
-+#define QM_CGR_TARG_PORTAL(n) (0x80000000 >> (n)) /* s/w portal, 0-9 */
-+#define QM_CGR_TARG_FMAN0 0x00200000 /* direct-connect portal: fman0 */
-+#define QM_CGR_TARG_FMAN1 0x00100000 /* : fman1 */
-+/* Convert CGR thresholds to/from "cs_thres" format */
-+static inline u64 qm_cgr_cs_thres_get64(const struct qm_cgr_cs_thres *th)
-+{
-+ return (u64)th->TA << th->Tn;
-+}
-+static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val,
-+ int roundup)
-+{
-+ u32 e = 0;
-+ int oddbit = 0;
-+ while (val > 0xff) {
-+ oddbit = val & 1;
-+ val >>= 1;
-+ e++;
-+ if (roundup && oddbit)
-+ val++;
-+ }
-+ th->Tn = e;
-+ th->TA = val;
-+ return 0;
-+}
-+
-+/* See 1.5.8.5.1: "Initialize FQ" */
-+/* See 1.5.8.5.2: "Query FQ" */
-+/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */
-+/* See 1.5.8.5.4: "Alter FQ State Commands " */
-+/* See 1.5.8.6.1: "Initialize/Modify CGR" */
-+/* See 1.5.8.6.2: "CGR Test Write" */
-+/* See 1.5.8.6.3: "Query CGR" */
-+/* See 1.5.8.6.4: "Query Congestion Group State" */
-+struct qm_mcc_initfq {
-+ u8 __reserved1;
-+ u16 we_mask; /* Write Enable Mask */
-+ u32 fqid; /* 24-bit */
-+ u16 count; /* Initialises 'count+1' FQDs */
-+ struct qm_fqd fqd; /* the FQD fields go here */
-+ u8 __reserved3[30];
-+} __packed;
-+struct qm_mcc_queryfq {
-+ u8 __reserved1[3];
-+ u32 fqid; /* 24-bit */
-+ u8 __reserved2[56];
-+} __packed;
-+struct qm_mcc_queryfq_np {
-+ u8 __reserved1[3];
-+ u32 fqid; /* 24-bit */
-+ u8 __reserved2[56];
-+} __packed;
-+struct qm_mcc_alterfq {
-+ u8 __reserved1[3];
-+ u32 fqid; /* 24-bit */
-+ u8 __reserved2;
-+ u8 count; /* number of consecutive FQID */
-+ u8 __reserved3[10];
-+ u32 context_b; /* frame queue context b */
-+ u8 __reserved4[40];
-+} __packed;
-+struct qm_mcc_initcgr {
-+ u8 __reserved1;
-+ u16 we_mask; /* Write Enable Mask */
-+ struct __qm_mc_cgr cgr; /* CGR fields */
-+ u8 __reserved2[2];
-+ u8 cgid;
-+ u8 __reserved4[32];
-+} __packed;
-+struct qm_mcc_cgrtestwrite {
-+ u8 __reserved1[2];
-+ u8 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
-+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */
-+ u8 __reserved2[23];
-+ u8 cgid;
-+ u8 __reserved3[32];
-+} __packed;
-+struct qm_mcc_querycgr {
-+ u8 __reserved1[30];
-+ u8 cgid;
-+ u8 __reserved2[32];
-+} __packed;
-+struct qm_mcc_querycongestion {
-+ u8 __reserved[63];
-+} __packed;
-+struct qm_mcc_querywq {
-+ u8 __reserved;
-+ /* select channel if verb != QUERYWQ_DEDICATED */
-+ union {
-+ u16 channel_wq; /* ignores wq (3 lsbits) */
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 id:13; /* qm_channel */
-+ u16 __reserved1:3;
-+#else
-+ u16 __reserved1:3;
-+ u16 id:13; /* qm_channel */
-+#endif
-+ } __packed channel;
-+ };
-+ u8 __reserved2[60];
-+} __packed;
-+
-+struct qm_mcc_ceetm_lfqmt_config {
-+ u8 __reserved1[4];
-+ u32 lfqid:24;
-+ u8 __reserved2[2];
-+ u16 cqid;
-+ u8 __reserved3[2];
-+ u16 dctidx;
-+ u8 __reserved4[48];
-+} __packed;
-+
-+struct qm_mcc_ceetm_lfqmt_query {
-+ u8 __reserved1[4];
-+ u32 lfqid:24;
-+ u8 __reserved2[56];
-+} __packed;
-+
-+struct qm_mcc_ceetm_cq_config {
-+ u8 __reserved1;
-+ u16 cqid;
-+ u8 dcpid;
-+ u8 __reserved2;
-+ u16 ccgid;
-+ u8 __reserved3[56];
-+} __packed;
-+
-+struct qm_mcc_ceetm_cq_query {
-+ u8 __reserved1;
-+ u16 cqid;
-+ u8 dcpid;
-+ u8 __reserved2[59];
-+} __packed;
-+
-+struct qm_mcc_ceetm_dct_config {
-+ u8 __reserved1;
-+ u16 dctidx;
-+ u8 dcpid;
-+ u8 __reserved2[15];
-+ u32 context_b;
-+ u64 context_a;
-+ u8 __reserved3[32];
-+} __packed;
-+
-+struct qm_mcc_ceetm_dct_query {
-+ u8 __reserved1;
-+ u16 dctidx;
-+ u8 dcpid;
-+ u8 __reserved2[59];
-+} __packed;
-+
-+struct qm_mcc_ceetm_class_scheduler_config {
-+ u8 __reserved1;
-+ u16 cqcid;
-+ u8 dcpid;
-+ u8 __reserved2[6];
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 gpc_reserved:1;
-+ u8 gpc_combine_flag:1;
-+ u8 gpc_prio_b:3;
-+ u8 gpc_prio_a:3;
-+#else
-+ u8 gpc_prio_a:3;
-+ u8 gpc_prio_b:3;
-+ u8 gpc_combine_flag:1;
-+ u8 gpc_reserved:1;
-+#endif
-+ u16 crem;
-+ u16 erem;
-+ u8 w[8];
-+ u8 __reserved3[40];
-+} __packed;
-+
-+struct qm_mcc_ceetm_class_scheduler_query {
-+ u8 __reserved1;
-+ u16 cqcid;
-+ u8 dcpid;
-+ u8 __reserved2[59];
-+} __packed;
-+
-+#define CEETM_COMMAND_CHANNEL_MAPPING (0 << 12)
-+#define CEETM_COMMAND_SP_MAPPING (1 << 12)
-+#define CEETM_COMMAND_CHANNEL_SHAPER (2 << 12)
-+#define CEETM_COMMAND_LNI_SHAPER (3 << 12)
-+#define CEETM_COMMAND_TCFC (4 << 12)
-+
-+#define CEETM_CCGRID_MASK 0x01FF
-+#define CEETM_CCGR_CM_CONFIGURE (0 << 14)
-+#define CEETM_CCGR_DN_CONFIGURE (1 << 14)
-+#define CEETM_CCGR_TEST_WRITE (2 << 14)
-+#define CEETM_CCGR_CM_QUERY (0 << 14)
-+#define CEETM_CCGR_DN_QUERY (1 << 14)
-+#define CEETM_CCGR_DN_QUERY_FLUSH (2 << 14)
-+#define CEETM_QUERY_CONGESTION_STATE (3 << 14)
-+
-+struct qm_mcc_ceetm_mapping_shaper_tcfc_config {
-+ u8 __reserved1;
-+ u16 cid;
-+ u8 dcpid;
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 map_shaped:1;
-+ u8 map_reserved:4;
-+ u8 map_lni_id:3;
-+#else
-+ u8 map_lni_id:3;
-+ u8 map_reserved:4;
-+ u8 map_shaped:1;
-+#endif
-+ u8 __reserved2[58];
-+ } __packed channel_mapping;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 map_reserved:5;
-+ u8 map_lni_id:3;
-+#else
-+ u8 map_lni_id:3;
-+ u8 map_reserved:5;
-+#endif
-+ u8 __reserved2[58];
-+ } __packed sp_mapping;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 cpl:1;
-+ u8 cpl_reserved:2;
-+ u8 oal:5;
-+#else
-+ u8 oal:5;
-+ u8 cpl_reserved:2;
-+ u8 cpl:1;
-+#endif
-+ u32 crtcr:24;
-+ u32 ertcr:24;
-+ u16 crtbl;
-+ u16 ertbl;
-+ u8 mps; /* This will be hardcoded by driver with 60 */
-+ u8 __reserved2[47];
-+ } __packed shaper_config;
-+ struct {
-+ u8 __reserved2[11];
-+ u64 lnitcfcc;
-+ u8 __reserved3[40];
-+ } __packed tcfc_config;
-+ };
-+} __packed;
-+
-+struct qm_mcc_ceetm_mapping_shaper_tcfc_query {
-+ u8 __reserved1;
-+ u16 cid;
-+ u8 dcpid;
-+ u8 __reserved2[59];
-+} __packed;
-+
-+struct qm_mcc_ceetm_ccgr_config {
-+ u8 __reserved1;
-+ u16 ccgrid;
-+ u8 dcpid;
-+ u8 __reserved2;
-+ u16 we_mask;
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 ctl_reserved:1;
-+ u8 ctl_wr_en_g:1;
-+ u8 ctl_wr_en_y:1;
-+ u8 ctl_wr_en_r:1;
-+ u8 ctl_td_en:1;
-+ u8 ctl_td_mode:1;
-+ u8 ctl_cscn_en:1;
-+ u8 ctl_mode:1;
-+#else
-+ u8 ctl_mode:1;
-+ u8 ctl_cscn_en:1;
-+ u8 ctl_td_mode:1;
-+ u8 ctl_td_en:1;
-+ u8 ctl_wr_en_r:1;
-+ u8 ctl_wr_en_y:1;
-+ u8 ctl_wr_en_g:1;
-+ u8 ctl_reserved:1;
-+#endif
-+ u8 cdv;
-+ u16 cscn_tupd;
-+ u8 oal;
-+ u8 __reserved3;
-+ struct qm_cgr_cs_thres cs_thres;
-+ struct qm_cgr_cs_thres cs_thres_x;
-+ struct qm_cgr_cs_thres td_thres;
-+ struct qm_cgr_wr_parm wr_parm_g;
-+ struct qm_cgr_wr_parm wr_parm_y;
-+ struct qm_cgr_wr_parm wr_parm_r;
-+ } __packed cm_config;
-+ struct {
-+ u8 dnc;
-+ u8 dn0;
-+ u8 dn1;
-+ u64 dnba:40;
-+ u8 __reserved3[2];
-+ u16 dnth_0;
-+ u8 __reserved4[2];
-+ u16 dnth_1;
-+ u8 __reserved5[8];
-+ } __packed dn_config;
-+ struct {
-+ u8 __reserved3[3];
-+ u64 i_cnt:40;
-+ u8 __reserved4[16];
-+ } __packed test_write;
-+ };
-+ u8 __reserved5[32];
-+} __packed;
-+
-+struct qm_mcc_ceetm_ccgr_query {
-+ u8 __reserved1;
-+ u16 ccgrid;
-+ u8 dcpid;
-+ u8 __reserved2[59];
-+} __packed;
-+
-+struct qm_mcc_ceetm_cq_peek_pop_xsfdrread {
-+ u8 __reserved1;
-+ u16 cqid;
-+ u8 dcpid;
-+ u8 ct;
-+ u16 xsfdr;
-+ u8 __reserved2[56];
-+} __packed;
-+
-+#define CEETM_QUERY_DEQUEUE_STATISTICS 0x00
-+#define CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS 0x01
-+#define CEETM_WRITE_DEQUEUE_STATISTICS 0x02
-+#define CEETM_QUERY_REJECT_STATISTICS 0x03
-+#define CEETM_QUERY_REJECT_CLEAR_STATISTICS 0x04
-+#define CEETM_WRITE_REJECT_STATISTICS 0x05
-+struct qm_mcc_ceetm_statistics_query_write {
-+ u8 __reserved1;
-+ u16 cid;
-+ u8 dcpid;
-+ u8 ct;
-+ u8 __reserved2[13];
-+ u64 frm_cnt:40;
-+ u8 __reserved3[2];
-+ u64 byte_cnt:48;
-+ u8 __reserved[32];
-+} __packed;
-+
-+struct qm_mc_command {
-+ u8 __dont_write_directly__verb;
-+ union {
-+ struct qm_mcc_initfq initfq;
-+ struct qm_mcc_queryfq queryfq;
-+ struct qm_mcc_queryfq_np queryfq_np;
-+ struct qm_mcc_alterfq alterfq;
-+ struct qm_mcc_initcgr initcgr;
-+ struct qm_mcc_cgrtestwrite cgrtestwrite;
-+ struct qm_mcc_querycgr querycgr;
-+ struct qm_mcc_querycongestion querycongestion;
-+ struct qm_mcc_querywq querywq;
-+ struct qm_mcc_ceetm_lfqmt_config lfqmt_config;
-+ struct qm_mcc_ceetm_lfqmt_query lfqmt_query;
-+ struct qm_mcc_ceetm_cq_config cq_config;
-+ struct qm_mcc_ceetm_cq_query cq_query;
-+ struct qm_mcc_ceetm_dct_config dct_config;
-+ struct qm_mcc_ceetm_dct_query dct_query;
-+ struct qm_mcc_ceetm_class_scheduler_config csch_config;
-+ struct qm_mcc_ceetm_class_scheduler_query csch_query;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config mst_config;
-+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query mst_query;
-+ struct qm_mcc_ceetm_ccgr_config ccgr_config;
-+ struct qm_mcc_ceetm_ccgr_query ccgr_query;
-+ struct qm_mcc_ceetm_cq_peek_pop_xsfdrread cq_ppxr;
-+ struct qm_mcc_ceetm_statistics_query_write stats_query_write;
-+ };
-+} __packed;
-+#define QM_MCC_VERB_VBIT 0x80
-+#define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */
-+#define QM_MCC_VERB_INITFQ_PARKED 0x40
-+#define QM_MCC_VERB_INITFQ_SCHED 0x41
-+#define QM_MCC_VERB_QUERYFQ 0x44
-+#define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */
-+#define QM_MCC_VERB_QUERYWQ 0x46
-+#define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47
-+#define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */
-+#define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */
-+#define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */
-+#define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */
-+#define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */
-+#define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */
-+#define QM_MCC_VERB_INITCGR 0x50
-+#define QM_MCC_VERB_MODIFYCGR 0x51
-+#define QM_MCC_VERB_CGRTESTWRITE 0x52
-+#define QM_MCC_VERB_QUERYCGR 0x58
-+#define QM_MCC_VERB_QUERYCONGESTION 0x59
-+/* INITFQ-specific flags */
-+#define QM_INITFQ_WE_MASK 0x01ff /* 'Write Enable' flags; */
-+#define QM_INITFQ_WE_OAC 0x0100
-+#define QM_INITFQ_WE_ORPC 0x0080
-+#define QM_INITFQ_WE_CGID 0x0040
-+#define QM_INITFQ_WE_FQCTRL 0x0020
-+#define QM_INITFQ_WE_DESTWQ 0x0010
-+#define QM_INITFQ_WE_ICSCRED 0x0008
-+#define QM_INITFQ_WE_TDTHRESH 0x0004
-+#define QM_INITFQ_WE_CONTEXTB 0x0002
-+#define QM_INITFQ_WE_CONTEXTA 0x0001
-+/* INITCGR/MODIFYCGR-specific flags */
-+#define QM_CGR_WE_MASK 0x07ff /* 'Write Enable Mask'; */
-+#define QM_CGR_WE_WR_PARM_G 0x0400
-+#define QM_CGR_WE_WR_PARM_Y 0x0200
-+#define QM_CGR_WE_WR_PARM_R 0x0100
-+#define QM_CGR_WE_WR_EN_G 0x0080
-+#define QM_CGR_WE_WR_EN_Y 0x0040
-+#define QM_CGR_WE_WR_EN_R 0x0020
-+#define QM_CGR_WE_CSCN_EN 0x0010
-+#define QM_CGR_WE_CSCN_TARG 0x0008
-+#define QM_CGR_WE_CSTD_EN 0x0004
-+#define QM_CGR_WE_CS_THRES 0x0002
-+#define QM_CGR_WE_MODE 0x0001
-+
-+/* See 1.5.9.7 CEETM Management Commands */
-+#define QM_CEETM_VERB_LFQMT_CONFIG 0x70
-+#define QM_CEETM_VERB_LFQMT_QUERY 0x71
-+#define QM_CEETM_VERB_CQ_CONFIG 0x72
-+#define QM_CEETM_VERB_CQ_QUERY 0x73
-+#define QM_CEETM_VERB_DCT_CONFIG 0x74
-+#define QM_CEETM_VERB_DCT_QUERY 0x75
-+#define QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG 0x76
-+#define QM_CEETM_VERB_CLASS_SCHEDULER_QUERY 0x77
-+#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG 0x78
-+#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY 0x79
-+#define QM_CEETM_VERB_CCGR_CONFIG 0x7A
-+#define QM_CEETM_VERB_CCGR_QUERY 0x7B
-+#define QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD 0x7C
-+#define QM_CEETM_VERB_STATISTICS_QUERY_WRITE 0x7D
-+
-+/* See 1.5.8.5.1: "Initialize FQ" */
-+/* See 1.5.8.5.2: "Query FQ" */
-+/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */
-+/* See 1.5.8.5.4: "Alter FQ State Commands " */
-+/* See 1.5.8.6.1: "Initialize/Modify CGR" */
-+/* See 1.5.8.6.2: "CGR Test Write" */
-+/* See 1.5.8.6.3: "Query CGR" */
-+/* See 1.5.8.6.4: "Query Congestion Group State" */
-+struct qm_mcr_initfq {
-+ u8 __reserved1[62];
-+} __packed;
-+struct qm_mcr_queryfq {
-+ u8 __reserved1[8];
-+ struct qm_fqd fqd; /* the FQD fields are here */
-+ u8 __reserved2[30];
-+} __packed;
-+struct qm_mcr_queryfq_np {
-+ u8 __reserved1;
-+ u8 state; /* QM_MCR_NP_STATE_*** */
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 __reserved2;
-+ u32 fqd_link:24;
-+ u16 __reserved3:2;
-+ u16 odp_seq:14;
-+ u16 __reserved4:2;
-+ u16 orp_nesn:14;
-+ u16 __reserved5:1;
-+ u16 orp_ea_hseq:15;
-+ u16 __reserved6:1;
-+ u16 orp_ea_tseq:15;
-+ u8 __reserved7;
-+ u32 orp_ea_hptr:24;
-+ u8 __reserved8;
-+ u32 orp_ea_tptr:24;
-+ u8 __reserved9;
-+ u32 pfdr_hptr:24;
-+ u8 __reserved10;
-+ u32 pfdr_tptr:24;
-+ u8 __reserved11[5];
-+ u8 __reserved12:7;
-+ u8 is:1;
-+ u16 ics_surp;
-+ u32 byte_cnt;
-+ u8 __reserved13;
-+ u32 frm_cnt:24;
-+ u32 __reserved14;
-+ u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */
-+ u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */
-+ u16 __reserved15;
-+ u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */
-+ u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */
-+ u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */
-+#else
-+ u8 __reserved2;
-+ u32 fqd_link:24;
-+
-+ u16 odp_seq:14;
-+ u16 __reserved3:2;
-+
-+ u16 orp_nesn:14;
-+ u16 __reserved4:2;
-+
-+ u16 orp_ea_hseq:15;
-+ u16 __reserved5:1;
-+
-+ u16 orp_ea_tseq:15;
-+ u16 __reserved6:1;
-+
-+ u8 __reserved7;
-+ u32 orp_ea_hptr:24;
-+
-+ u8 __reserved8;
-+ u32 orp_ea_tptr:24;
-+
-+ u8 __reserved9;
-+ u32 pfdr_hptr:24;
-+
-+ u8 __reserved10;
-+ u32 pfdr_tptr:24;
-+
-+ u8 __reserved11[5];
-+ u8 is:1;
-+ u8 __reserved12:7;
-+ u16 ics_surp;
-+ u32 byte_cnt;
-+ u8 __reserved13;
-+ u32 frm_cnt:24;
-+ u32 __reserved14;
-+ u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */
-+ u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */
-+ u16 __reserved15;
-+ u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */
-+ u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */
-+ u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */
-+#endif
-+} __packed;
-+
-+
-+struct qm_mcr_alterfq {
-+ u8 fqs; /* Frame Queue Status */
-+ u8 __reserved1[61];
-+} __packed;
-+struct qm_mcr_initcgr {
-+ u8 __reserved1[62];
-+} __packed;
-+struct qm_mcr_cgrtestwrite {
-+ u16 __reserved1;
-+ struct __qm_mc_cgr cgr; /* CGR fields */
-+ u8 __reserved2[3];
-+ u32 __reserved3:24;
-+ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
-+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */
-+ u32 __reserved4:24;
-+ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */
-+ u32 a_bcnt_lo; /* low 32-bits of 40-bit */
-+ u16 lgt; /* Last Group Tick */
-+ u16 wr_prob_g;
-+ u16 wr_prob_y;
-+ u16 wr_prob_r;
-+ u8 __reserved5[8];
-+} __packed;
-+struct qm_mcr_querycgr {
-+ u16 __reserved1;
-+ struct __qm_mc_cgr cgr; /* CGR fields */
-+ u8 __reserved2[3];
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved3:24;
-+ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
-+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */
-+#else
-+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */
-+ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
-+ u32 __reserved3:24;
-+#endif
-+ };
-+ u64 i_bcnt;
-+ };
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u32 __reserved4:24;
-+ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */
-+ u32 a_bcnt_lo; /* low 32-bits of 40-bit */
-+#else
-+ u32 a_bcnt_lo; /* low 32-bits of 40-bit */
-+ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */
-+ u32 __reserved4:24;
-+#endif
-+ };
-+ u64 a_bcnt;
-+ };
-+ union {
-+ u32 cscn_targ_swp[4];
-+ u8 __reserved5[16];
-+ };
-+} __packed;
-+static inline u64 qm_mcr_querycgr_i_get64(const struct qm_mcr_querycgr *q)
-+{
-+ return be64_to_cpu(q->i_bcnt);
-+}
-+static inline u64 qm_mcr_querycgr_a_get64(const struct qm_mcr_querycgr *q)
-+{
-+ return be64_to_cpu(q->a_bcnt);
-+}
-+static inline u64 qm_mcr_cgrtestwrite_i_get64(
-+ const struct qm_mcr_cgrtestwrite *q)
-+{
-+ return be64_to_cpu(((u64)q->i_bcnt_hi << 32) | (u64)q->i_bcnt_lo);
-+}
-+static inline u64 qm_mcr_cgrtestwrite_a_get64(
-+ const struct qm_mcr_cgrtestwrite *q)
-+{
-+ return be64_to_cpu(((u64)q->a_bcnt_hi << 32) | (u64)q->a_bcnt_lo);
-+}
-+/* Macro, so we compile better if 'v' isn't always 64-bit */
-+#define qm_mcr_querycgr_i_set64(q, v) \
-+ do { \
-+ struct qm_mcr_querycgr *__q931 = (fd); \
-+ __q931->i_bcnt_hi = upper_32_bits(v); \
-+ __q931->i_bcnt_lo = lower_32_bits(v); \
-+ } while (0)
-+#define qm_mcr_querycgr_a_set64(q, v) \
-+ do { \
-+ struct qm_mcr_querycgr *__q931 = (fd); \
-+ __q931->a_bcnt_hi = upper_32_bits(v); \
-+ __q931->a_bcnt_lo = lower_32_bits(v); \
-+ } while (0)
-+struct __qm_mcr_querycongestion {
-+ u32 __state[8];
-+};
-+struct qm_mcr_querycongestion {
-+ u8 __reserved[30];
-+ /* Access this struct using QM_MCR_QUERYCONGESTION() */
-+ struct __qm_mcr_querycongestion state;
-+} __packed;
-+struct qm_mcr_querywq {
-+ union {
-+ u16 channel_wq; /* ignores wq (3 lsbits) */
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u16 id:13; /* qm_channel */
-+ u16 __reserved:3;
-+#else
-+ u16 __reserved:3;
-+ u16 id:13; /* qm_channel */
-+#endif
-+ } __packed channel;
-+ };
-+ u8 __reserved[28];
-+ u32 wq_len[8];
-+} __packed;
-+
-+/* QMAN CEETM Management Command Response */
-+struct qm_mcr_ceetm_lfqmt_config {
-+ u8 __reserved1[62];
-+} __packed;
-+struct qm_mcr_ceetm_lfqmt_query {
-+ u8 __reserved1[8];
-+ u16 cqid;
-+ u8 __reserved2[2];
-+ u16 dctidx;
-+ u8 __reserved3[2];
-+ u16 ccgid;
-+ u8 __reserved4[44];
-+} __packed;
-+
-+struct qm_mcr_ceetm_cq_config {
-+ u8 __reserved1[62];
-+} __packed;
-+
-+struct qm_mcr_ceetm_cq_query {
-+ u8 __reserved1[4];
-+ u16 ccgid;
-+ u16 state;
-+ u32 pfdr_hptr:24;
-+ u32 pfdr_tptr:24;
-+ u16 od1_xsfdr;
-+ u16 od2_xsfdr;
-+ u16 od3_xsfdr;
-+ u16 od4_xsfdr;
-+ u16 od5_xsfdr;
-+ u16 od6_xsfdr;
-+ u16 ra1_xsfdr;
-+ u16 ra2_xsfdr;
-+ u8 __reserved2;
-+ u32 frm_cnt:24;
-+ u8 __reserved333[28];
-+} __packed;
-+
-+struct qm_mcr_ceetm_dct_config {
-+ u8 __reserved1[62];
-+} __packed;
-+
-+struct qm_mcr_ceetm_dct_query {
-+ u8 __reserved1[18];
-+ u32 context_b;
-+ u64 context_a;
-+ u8 __reserved2[32];
-+} __packed;
-+
-+struct qm_mcr_ceetm_class_scheduler_config {
-+ u8 __reserved1[62];
-+} __packed;
-+
-+struct qm_mcr_ceetm_class_scheduler_query {
-+ u8 __reserved1[9];
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 gpc_reserved:1;
-+ u8 gpc_combine_flag:1;
-+ u8 gpc_prio_b:3;
-+ u8 gpc_prio_a:3;
-+#else
-+ u8 gpc_prio_a:3;
-+ u8 gpc_prio_b:3;
-+ u8 gpc_combine_flag:1;
-+ u8 gpc_reserved:1;
-+#endif
-+ u16 crem;
-+ u16 erem;
-+ u8 w[8];
-+ u8 __reserved2[5];
-+ u32 wbfslist:24;
-+ u32 d8;
-+ u32 d9;
-+ u32 d10;
-+ u32 d11;
-+ u32 d12;
-+ u32 d13;
-+ u32 d14;
-+ u32 d15;
-+} __packed;
-+
-+struct qm_mcr_ceetm_mapping_shaper_tcfc_config {
-+ u16 cid;
-+ u8 __reserved2[60];
-+} __packed;
-+
-+struct qm_mcr_ceetm_mapping_shaper_tcfc_query {
-+ u16 cid;
-+ u8 __reserved1;
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 map_shaped:1;
-+ u8 map_reserved:4;
-+ u8 map_lni_id:3;
-+#else
-+ u8 map_lni_id:3;
-+ u8 map_reserved:4;
-+ u8 map_shaped:1;
-+#endif
-+ u8 __reserved2[58];
-+ } __packed channel_mapping_query;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 map_reserved:5;
-+ u8 map_lni_id:3;
-+#else
-+ u8 map_lni_id:3;
-+ u8 map_reserved:5;
-+#endif
-+ u8 __reserved2[58];
-+ } __packed sp_mapping_query;
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 cpl:1;
-+ u8 cpl_reserved:2;
-+ u8 oal:5;
-+#else
-+ u8 oal:5;
-+ u8 cpl_reserved:2;
-+ u8 cpl:1;
-+#endif
-+ u32 crtcr:24;
-+ u32 ertcr:24;
-+ u16 crtbl;
-+ u16 ertbl;
-+ u8 mps;
-+ u8 __reserved2[15];
-+ u32 crat;
-+ u32 erat;
-+ u8 __reserved3[24];
-+ } __packed shaper_query;
-+ struct {
-+ u8 __reserved1[11];
-+ u64 lnitcfcc;
-+ u8 __reserved3[40];
-+ } __packed tcfc_query;
-+ };
-+} __packed;
-+
-+struct qm_mcr_ceetm_ccgr_config {
-+ u8 __reserved1[46];
-+ union {
-+ u8 __reserved2[8];
-+ struct {
-+ u16 timestamp;
-+ u16 wr_porb_g;
-+ u16 wr_prob_y;
-+ u16 wr_prob_r;
-+ } __packed test_write;
-+ };
-+ u8 __reserved3[8];
-+} __packed;
-+
-+struct qm_mcr_ceetm_ccgr_query {
-+ u8 __reserved1[6];
-+ union {
-+ struct {
-+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-+ u8 ctl_reserved:1;
-+ u8 ctl_wr_en_g:1;
-+ u8 ctl_wr_en_y:1;
-+ u8 ctl_wr_en_r:1;
-+ u8 ctl_td_en:1;
-+ u8 ctl_td_mode:1;
-+ u8 ctl_cscn_en:1;
-+ u8 ctl_mode:1;
-+#else
-+ u8 ctl_mode:1;
-+ u8 ctl_cscn_en:1;
-+ u8 ctl_td_mode:1;
-+ u8 ctl_td_en:1;
-+ u8 ctl_wr_en_r:1;
-+ u8 ctl_wr_en_y:1;
-+ u8 ctl_wr_en_g:1;
-+ u8 ctl_reserved:1;
-+#endif
-+ u8 cdv;
-+ u8 __reserved2[2];
-+ u8 oal;
-+ u8 __reserved3;
-+ struct qm_cgr_cs_thres cs_thres;
-+ struct qm_cgr_cs_thres cs_thres_x;
-+ struct qm_cgr_cs_thres td_thres;
-+ struct qm_cgr_wr_parm wr_parm_g;
-+ struct qm_cgr_wr_parm wr_parm_y;
-+ struct qm_cgr_wr_parm wr_parm_r;
-+ u16 cscn_targ_dcp;
-+ u8 dcp_lsn;
-+ u64 i_cnt:40;
-+ u8 __reserved4[3];
-+ u64 a_cnt:40;
-+ u32 cscn_targ_swp[4];
-+ } __packed cm_query;
-+ struct {
-+ u8 dnc;
-+ u8 dn0;
-+ u8 dn1;
-+ u64 dnba:40;
-+ u8 __reserved2[2];
-+ u16 dnth_0;
-+ u8 __reserved3[2];
-+ u16 dnth_1;
-+ u8 __reserved4[10];
-+ u16 dnacc_0;
-+ u8 __reserved5[2];
-+ u16 dnacc_1;
-+ u8 __reserved6[24];
-+ } __packed dn_query;
-+ struct {
-+ u8 __reserved2[24];
-+ struct __qm_mcr_querycongestion state;
-+ } __packed congestion_state;
-+
-+ };
-+} __packed;
-+
-+struct qm_mcr_ceetm_cq_peek_pop_xsfdrread {
-+ u8 stat;
-+ u8 __reserved1[11];
-+ u16 dctidx;
-+ struct qm_fd fd;
-+ u8 __reserved2[32];
-+} __packed;
-+
-+struct qm_mcr_ceetm_statistics_query {
-+ u8 __reserved1[17];
-+ u64 frm_cnt:40;
-+ u8 __reserved2[2];
-+ u64 byte_cnt:48;
-+ u8 __reserved3[32];
-+} __packed;
-+
-+struct qm_mc_result {
-+ u8 verb;
-+ u8 result;
-+ union {
-+ struct qm_mcr_initfq initfq;
-+ struct qm_mcr_queryfq queryfq;
-+ struct qm_mcr_queryfq_np queryfq_np;
-+ struct qm_mcr_alterfq alterfq;
-+ struct qm_mcr_initcgr initcgr;
-+ struct qm_mcr_cgrtestwrite cgrtestwrite;
-+ struct qm_mcr_querycgr querycgr;
-+ struct qm_mcr_querycongestion querycongestion;
-+ struct qm_mcr_querywq querywq;
-+ struct qm_mcr_ceetm_lfqmt_config lfqmt_config;
-+ struct qm_mcr_ceetm_lfqmt_query lfqmt_query;
-+ struct qm_mcr_ceetm_cq_config cq_config;
-+ struct qm_mcr_ceetm_cq_query cq_query;
-+ struct qm_mcr_ceetm_dct_config dct_config;
-+ struct qm_mcr_ceetm_dct_query dct_query;
-+ struct qm_mcr_ceetm_class_scheduler_config csch_config;
-+ struct qm_mcr_ceetm_class_scheduler_query csch_query;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_config mst_config;
-+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query mst_query;
-+ struct qm_mcr_ceetm_ccgr_config ccgr_config;
-+ struct qm_mcr_ceetm_ccgr_query ccgr_query;
-+ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread cq_ppxr;
-+ struct qm_mcr_ceetm_statistics_query stats_query;
-+ };
-+} __packed;
-+
-+#define QM_MCR_VERB_RRID 0x80
-+#define QM_MCR_VERB_MASK QM_MCC_VERB_MASK
-+#define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED
-+#define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED
-+#define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ
-+#define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP
-+#define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ
-+#define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED
-+#define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED
-+#define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE
-+#define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE
-+#define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS
-+#define QM_MCR_RESULT_NULL 0x00
-+#define QM_MCR_RESULT_OK 0xf0
-+#define QM_MCR_RESULT_ERR_FQID 0xf1
-+#define QM_MCR_RESULT_ERR_FQSTATE 0xf2
-+#define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */
-+#define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4
-+#define QM_MCR_RESULT_PENDING 0xf8
-+#define QM_MCR_RESULT_ERR_BADCOMMAND 0xff
-+#define QM_MCR_NP_STATE_FE 0x10
-+#define QM_MCR_NP_STATE_R 0x08
-+#define QM_MCR_NP_STATE_MASK 0x07 /* Reads FQD::STATE; */
-+#define QM_MCR_NP_STATE_OOS 0x00
-+#define QM_MCR_NP_STATE_RETIRED 0x01
-+#define QM_MCR_NP_STATE_TEN_SCHED 0x02
-+#define QM_MCR_NP_STATE_TRU_SCHED 0x03
-+#define QM_MCR_NP_STATE_PARKED 0x04
-+#define QM_MCR_NP_STATE_ACTIVE 0x05
-+#define QM_MCR_NP_PTR_MASK 0x07ff /* for RA[12] & OD[123] */
-+#define QM_MCR_NP_RA1_NRA(v) (((v) >> 14) & 0x3) /* FQD::NRA */
-+#define QM_MCR_NP_RA2_IT(v) (((v) >> 14) & 0x1) /* FQD::IT */
-+#define QM_MCR_NP_OD1_NOD(v) (((v) >> 14) & 0x3) /* FQD::NOD */
-+#define QM_MCR_NP_OD3_NPC(v) (((v) >> 14) & 0x3) /* FQD::NPC */
-+#define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */
-+#define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */
-+/* This extracts the state for congestion group 'n' from a query response.
-+ * Eg.
-+ * u8 cgr = [...];
-+ * struct qm_mc_result *res = [...];
-+ * printf("congestion group %d congestion state: %d\n", cgr,
-+ * QM_MCR_QUERYCONGESTION(&res->querycongestion.state, cgr));
-+ */
-+#define __CGR_WORD(num) (num >> 5)
-+#define __CGR_SHIFT(num) (num & 0x1f)
-+#define __CGR_NUM (sizeof(struct __qm_mcr_querycongestion) << 3)
-+static inline int QM_MCR_QUERYCONGESTION(struct __qm_mcr_querycongestion *p,
-+ u8 cgr)
-+{
-+ return p->__state[__CGR_WORD(cgr)] & (0x80000000 >> __CGR_SHIFT(cgr));
-+}
-+
-+
-+/*********************/
-+/* Utility interface */
-+/*********************/
-+
-+/* Represents an allocator over a range of FQIDs. NB, accesses are not locked,
-+ * spinlock them yourself if needed. */
-+struct qman_fqid_pool;
-+
-+/* Create/destroy a FQID pool, num must be a multiple of 32. NB, _destroy()
-+ * always succeeds, but returns non-zero if there were "leaked" FQID
-+ * allocations. */
-+struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num);
-+int qman_fqid_pool_destroy(struct qman_fqid_pool *pool);
-+/* Alloc/free a FQID from the range. _alloc() returns zero for success. */
-+int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid);
-+void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid);
-+u32 qman_fqid_pool_used(struct qman_fqid_pool *pool);
-+
-+/*******************************************************************/
-+/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */
-+/*******************************************************************/
-+
-+ /* Portal and Frame Queues */
-+ /* ----------------------- */
-+/* Represents a managed portal */
-+struct qman_portal;
-+
-+/* This object type represents Qman frame queue descriptors (FQD), it is
-+ * cacheline-aligned, and initialised by qman_create_fq(). The structure is
-+ * defined further down. */
-+struct qman_fq;
-+
-+/* This object type represents a Qman congestion group, it is defined further
-+ * down. */
-+struct qman_cgr;
-+
-+struct qman_portal_config {
-+ /* If the caller enables DQRR stashing (and thus wishes to operate the
-+ * portal from only one cpu), this is the logical CPU that the portal
-+ * will stash to. Whether stashing is enabled or not, this setting is
-+ * also used for any "core-affine" portals, ie. default portals
-+ * associated to the corresponding cpu. -1 implies that there is no core
-+ * affinity configured. */
-+ int cpu;
-+ /* portal interrupt line */
-+ int irq;
-+ /* the unique index of this portal */
-+ u32 index;
-+ /* Is this portal shared? (If so, it has coarser locking and demuxes
-+ * processing on behalf of other CPUs.) */
-+ int is_shared;
-+ /* The portal's dedicated channel id, use this value for initialising
-+ * frame queues to target this portal when scheduled. */
-+ u16 channel;
-+ /* A mask of which pool channels this portal has dequeue access to
-+ * (using QM_SDQCR_CHANNELS_POOL(n) for the bitmask) */
-+ u32 pools;
-+};
-+
-+/* This enum, and the callback type that returns it, are used when handling
-+ * dequeued frames via DQRR. Note that for "null" callbacks registered with the
-+ * portal object (for handling dequeues that do not demux because contextB is
-+ * NULL), the return value *MUST* be qman_cb_dqrr_consume. */
-+enum qman_cb_dqrr_result {
-+ /* DQRR entry can be consumed */
-+ qman_cb_dqrr_consume,
-+ /* Like _consume, but requests parking - FQ must be held-active */
-+ qman_cb_dqrr_park,
-+ /* Does not consume, for DCA mode only. This allows out-of-order
-+ * consumes by explicit calls to qman_dca() and/or the use of implicit
-+ * DCA via EQCR entries. */
-+ qman_cb_dqrr_defer,
-+ /* Stop processing without consuming this ring entry. Exits the current
-+ * qman_poll_dqrr() or interrupt-handling, as appropriate. If within an
-+ * interrupt handler, the callback would typically call
-+ * qman_irqsource_remove(QM_PIRQ_DQRI) before returning this value,
-+ * otherwise the interrupt will reassert immediately. */
-+ qman_cb_dqrr_stop,
-+ /* Like qman_cb_dqrr_stop, but consumes the current entry. */
-+ qman_cb_dqrr_consume_stop
-+};
-+typedef enum qman_cb_dqrr_result (*qman_cb_dqrr)(struct qman_portal *qm,
-+ struct qman_fq *fq,
-+ const struct qm_dqrr_entry *dqrr);
-+
-+/* This callback type is used when handling ERNs, FQRNs and FQRLs via MR. They
-+ * are always consumed after the callback returns. */
-+typedef void (*qman_cb_mr)(struct qman_portal *qm, struct qman_fq *fq,
-+ const struct qm_mr_entry *msg);
-+
-+/* This callback type is used when handling DCP ERNs */
-+typedef void (*qman_cb_dc_ern)(struct qman_portal *qm,
-+ const struct qm_mr_entry *msg);
-+
-+/* s/w-visible states. Ie. tentatively scheduled + truly scheduled + active +
-+ * held-active + held-suspended are just "sched". Things like "retired" will not
-+ * be assumed until it is complete (ie. QMAN_FQ_STATE_CHANGING is set until
-+ * then, to indicate it's completing and to gate attempts to retry the retire
-+ * command). Note, park commands do not set QMAN_FQ_STATE_CHANGING because it's
-+ * technically impossible in the case of enqueue DCAs (which refer to DQRR ring
-+ * index rather than the FQ that ring entry corresponds to), so repeated park
-+ * commands are allowed (if you're silly enough to try) but won't change FQ
-+ * state, and the resulting park notifications move FQs from "sched" to
-+ * "parked". */
-+enum qman_fq_state {
-+ qman_fq_state_oos,
-+ qman_fq_state_parked,
-+ qman_fq_state_sched,
-+ qman_fq_state_retired
-+};
-+
-+/* Frame queue objects (struct qman_fq) are stored within memory passed to
-+ * qman_create_fq(), as this allows stashing of caller-provided demux callback
-+ * pointers at no extra cost to stashing of (driver-internal) FQ state. If the
-+ * caller wishes to add per-FQ state and have it benefit from dequeue-stashing,
-+ * they should;
-+ *
-+ * (a) extend the qman_fq structure with their state; eg.
-+ *
-+ * // myfq is allocated and driver_fq callbacks filled in;
-+ * struct my_fq {
-+ * struct qman_fq base;
-+ * int an_extra_field;
-+ * [ ... add other fields to be associated with each FQ ...]
-+ * } *myfq = some_my_fq_allocator();
-+ * struct qman_fq *fq = qman_create_fq(fqid, flags, &myfq->base);
-+ *
-+ * // in a dequeue callback, access extra fields from 'fq' via a cast;
-+ * struct my_fq *myfq = (struct my_fq *)fq;
-+ * do_something_with(myfq->an_extra_field);
-+ * [...]
-+ *
-+ * (b) when and if configuring the FQ for context stashing, specify how ever
-+ * many cachelines are required to stash 'struct my_fq', to accelerate not
-+ * only the Qman driver but the callback as well.
-+ */
-+
-+struct qman_fq_cb {
-+ qman_cb_dqrr dqrr; /* for dequeued frames */
-+ qman_cb_mr ern; /* for s/w ERNs */
-+ qman_cb_mr fqs; /* frame-queue state changes*/
-+};
-+
-+struct qman_fq {
-+ /* Caller of qman_create_fq() provides these demux callbacks */
-+ struct qman_fq_cb cb;
-+ /* These are internal to the driver, don't touch. In particular, they
-+ * may change, be removed, or extended (so you shouldn't rely on
-+ * sizeof(qman_fq) being a constant). */
-+ spinlock_t fqlock;
-+ u32 fqid;
-+ volatile unsigned long flags;
-+ enum qman_fq_state state;
-+ int cgr_groupid;
-+ struct rb_node node;
-+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-+ u32 key;
-+#endif
-+};
-+
-+/* This callback type is used when handling congestion group entry/exit.
-+ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. */
-+typedef void (*qman_cb_cgr)(struct qman_portal *qm,
-+ struct qman_cgr *cgr, int congested);
-+
-+struct qman_cgr {
-+ /* Set these prior to qman_create_cgr() */
-+ u32 cgrid; /* 0..255, but u32 to allow specials like -1, 256, etc.*/
-+ qman_cb_cgr cb;
-+ /* These are private to the driver */
-+ u16 chan; /* portal channel this object is created on */
-+ struct list_head node;
-+};
-+
-+/* Flags to qman_create_fq() */
-+#define QMAN_FQ_FLAG_NO_ENQUEUE 0x00000001 /* can't enqueue */
-+#define QMAN_FQ_FLAG_NO_MODIFY 0x00000002 /* can only enqueue */
-+#define QMAN_FQ_FLAG_TO_DCPORTAL 0x00000004 /* consumed by CAAM/PME/Fman */
-+#define QMAN_FQ_FLAG_LOCKED 0x00000008 /* multi-core locking */
-+#define QMAN_FQ_FLAG_AS_IS 0x00000010 /* query h/w state */
-+#define QMAN_FQ_FLAG_DYNAMIC_FQID 0x00000020 /* (de)allocate fqid */
-+
-+/* Flags to qman_destroy_fq() */
-+#define QMAN_FQ_DESTROY_PARKED 0x00000001 /* FQ can be parked or OOS */
-+
-+/* Flags from qman_fq_state() */
-+#define QMAN_FQ_STATE_CHANGING 0x80000000 /* 'state' is changing */
-+#define QMAN_FQ_STATE_NE 0x40000000 /* retired FQ isn't empty */
-+#define QMAN_FQ_STATE_ORL 0x20000000 /* retired FQ has ORL */
-+#define QMAN_FQ_STATE_BLOCKOOS 0xe0000000 /* if any are set, no OOS */
-+#define QMAN_FQ_STATE_CGR_EN 0x10000000 /* CGR enabled */
-+#define QMAN_FQ_STATE_VDQCR 0x08000000 /* being volatile dequeued */
-+
-+/* Flags to qman_init_fq() */
-+#define QMAN_INITFQ_FLAG_SCHED 0x00000001 /* schedule rather than park */
-+#define QMAN_INITFQ_FLAG_LOCAL 0x00000004 /* set dest portal */
-+
-+/* Flags to qman_volatile_dequeue() */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+#define QMAN_VOLATILE_FLAG_WAIT 0x00000001 /* wait if VDQCR is in use */
-+#define QMAN_VOLATILE_FLAG_WAIT_INT 0x00000002 /* if wait, interruptible? */
-+#define QMAN_VOLATILE_FLAG_FINISH 0x00000004 /* wait till VDQCR completes */
-+#endif
-+
-+/* Flags to qman_enqueue(). NB, the strange numbering is to align with hardware,
-+ * bit-wise. (NB: the PME API is sensitive to these precise numberings too, so
-+ * any change here should be audited in PME.) */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT
-+#define QMAN_ENQUEUE_FLAG_WAIT 0x00010000 /* wait if EQCR is full */
-+#define QMAN_ENQUEUE_FLAG_WAIT_INT 0x00020000 /* if wait, interruptible? */
-+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
-+#define QMAN_ENQUEUE_FLAG_WAIT_SYNC 0x00000004 /* if wait, until consumed? */
-+#endif
-+#endif
-+#define QMAN_ENQUEUE_FLAG_WATCH_CGR 0x00080000 /* watch congestion state */
-+#define QMAN_ENQUEUE_FLAG_DCA 0x00008000 /* perform enqueue-DCA */
-+#define QMAN_ENQUEUE_FLAG_DCA_PARK 0x00004000 /* If DCA, requests park */
-+#define QMAN_ENQUEUE_FLAG_DCA_PTR(p) /* If DCA, p is DQRR entry */ \
-+ (((u32)(p) << 2) & 0x00000f00)
-+#define QMAN_ENQUEUE_FLAG_C_GREEN 0x00000000 /* choose one C_*** flag */
-+#define QMAN_ENQUEUE_FLAG_C_YELLOW 0x00000008
-+#define QMAN_ENQUEUE_FLAG_C_RED 0x00000010
-+#define QMAN_ENQUEUE_FLAG_C_OVERRIDE 0x00000018
-+/* For the ORP-specific qman_enqueue_orp() variant;
-+ * - this flag indicates "Not Last In Sequence", ie. all but the final fragment
-+ * of a frame. */
-+#define QMAN_ENQUEUE_FLAG_NLIS 0x01000000
-+/* - this flag performs no enqueue but fills in an ORP sequence number that
-+ * would otherwise block it (eg. if a frame has been dropped). */
-+#define QMAN_ENQUEUE_FLAG_HOLE 0x02000000
-+/* - this flag performs no enqueue but advances NESN to the given sequence
-+ * number. */
-+#define QMAN_ENQUEUE_FLAG_NESN 0x04000000
-+
-+/* Flags to qman_modify_cgr() */
-+#define QMAN_CGR_FLAG_USE_INIT 0x00000001
-+#define QMAN_CGR_MODE_FRAME 0x00000001
-+
-+ /* Portal Management */
-+ /* ----------------- */
-+/**
-+ * qman_get_portal_config - get portal configuration settings
-+ *
-+ * This returns a read-only view of the current cpu's affine portal settings.
-+ */
-+const struct qman_portal_config *qman_get_portal_config(void);
-+
-+/**
-+ * qman_irqsource_get - return the portal work that is interrupt-driven
-+ *
-+ * Returns a bitmask of QM_PIRQ_**I processing sources that are currently
-+ * enabled for interrupt handling on the current cpu's affine portal. These
-+ * sources will trigger the portal interrupt and the interrupt handler (or a
-+ * tasklet/bottom-half it defers to) will perform the corresponding processing
-+ * work. The qman_poll_***() functions will only process sources that are not in
-+ * this bitmask. If the current CPU is sharing a portal hosted on another CPU,
-+ * this always returns zero.
-+ */
-+u32 qman_irqsource_get(void);
-+
-+/**
-+ * qman_irqsource_add - add processing sources to be interrupt-driven
-+ * @bits: bitmask of QM_PIRQ_**I processing sources
-+ *
-+ * Adds processing sources that should be interrupt-driven (rather than
-+ * processed via qman_poll_***() functions). Returns zero for success, or
-+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU.
-+ */
-+int qman_irqsource_add(u32 bits);
-+
-+/**
-+ * qman_irqsource_remove - remove processing sources from being interrupt-driven
-+ * @bits: bitmask of QM_PIRQ_**I processing sources
-+ *
-+ * Removes processing sources from being interrupt-driven, so that they will
-+ * instead be processed via qman_poll_***() functions. Returns zero for success,
-+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU.
-+ */
-+int qman_irqsource_remove(u32 bits);
-+
-+/**
-+ * qman_affine_cpus - return a mask of cpus that have affine portals
-+ */
-+const cpumask_t *qman_affine_cpus(void);
-+
-+/**
-+ * qman_affine_channel - return the channel ID of an portal
-+ * @cpu: the cpu whose affine portal is the subject of the query
-+ *
-+ * If @cpu is -1, the affine portal for the current CPU will be used. It is a
-+ * bug to call this function for any value of @cpu (other than -1) that is not a
-+ * member of the mask returned from qman_affine_cpus().
-+ */
-+u16 qman_affine_channel(int cpu);
-+
-+/**
-+ * qman_get_affine_portal - return the portal pointer affine to cpu
-+ * @cpu: the cpu whose affine portal is the subject of the query
-+ *
-+ */
-+void *qman_get_affine_portal(int cpu);
-+
-+/**
-+ * qman_poll_dqrr - process DQRR (fast-path) entries
-+ * @limit: the maximum number of DQRR entries to process
-+ *
-+ * Use of this function requires that DQRR processing not be interrupt-driven.
-+ * Ie. the value returned by qman_irqsource_get() should not include
-+ * QM_PIRQ_DQRI. If the current CPU is sharing a portal hosted on another CPU,
-+ * this function will return -EINVAL, otherwise the return value is >=0 and
-+ * represents the number of DQRR entries processed.
-+ */
-+int qman_poll_dqrr(unsigned int limit);
-+
-+/**
-+ * qman_poll_slow - process anything (except DQRR) that isn't interrupt-driven.
-+ *
-+ * This function does any portal processing that isn't interrupt-driven. If the
-+ * current CPU is sharing a portal hosted on another CPU, this function will
-+ * return (u32)-1, otherwise the return value is a bitmask of QM_PIRQ_* sources
-+ * indicating what interrupt sources were actually processed by the call.
-+ */
-+u32 qman_poll_slow(void);
-+
-+/**
-+ * qman_poll - legacy wrapper for qman_poll_dqrr() and qman_poll_slow()
-+ *
-+ * Dispatcher logic on a cpu can use this to trigger any maintenance of the
-+ * affine portal. There are two classes of portal processing in question;
-+ * fast-path (which involves demuxing dequeue ring (DQRR) entries and tracking
-+ * enqueue ring (EQCR) consumption), and slow-path (which involves EQCR
-+ * thresholds, congestion state changes, etc). This function does whatever
-+ * processing is not triggered by interrupts.
-+ *
-+ * Note, if DQRR and some slow-path processing are poll-driven (rather than
-+ * interrupt-driven) then this function uses a heuristic to determine how often
-+ * to run slow-path processing - as slow-path processing introduces at least a
-+ * minimum latency each time it is run, whereas fast-path (DQRR) processing is
-+ * close to zero-cost if there is no work to be done. Applications can tune this
-+ * behaviour themselves by using qman_poll_dqrr() and qman_poll_slow() directly
-+ * rather than going via this wrapper.
-+ */
-+void qman_poll(void);
-+
-+/**
-+ * qman_stop_dequeues - Stop h/w dequeuing to the s/w portal
-+ *
-+ * Disables DQRR processing of the portal. This is reference-counted, so
-+ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to
-+ * truly re-enable dequeuing.
-+ */
-+void qman_stop_dequeues(void);
-+
-+/**
-+ * qman_start_dequeues - (Re)start h/w dequeuing to the s/w portal
-+ *
-+ * Enables DQRR processing of the portal. This is reference-counted, so
-+ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to
-+ * truly re-enable dequeuing.
-+ */
-+void qman_start_dequeues(void);
-+
-+/**
-+ * qman_static_dequeue_add - Add pool channels to the portal SDQCR
-+ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n)
-+ *
-+ * Adds a set of pool channels to the portal's static dequeue command register
-+ * (SDQCR). The requested pools are limited to those the portal has dequeue
-+ * access to.
-+ */
-+void qman_static_dequeue_add(u32 pools);
-+
-+/**
-+ * qman_static_dequeue_del - Remove pool channels from the portal SDQCR
-+ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n)
-+ *
-+ * Removes a set of pool channels from the portal's static dequeue command
-+ * register (SDQCR). The requested pools are limited to those the portal has
-+ * dequeue access to.
-+ */
-+void qman_static_dequeue_del(u32 pools);
-+
-+/**
-+ * qman_static_dequeue_get - return the portal's current SDQCR
-+ *
-+ * Returns the portal's current static dequeue command register (SDQCR). The
-+ * entire register is returned, so if only the currently-enabled pool channels
-+ * are desired, mask the return value with QM_SDQCR_CHANNELS_POOL_MASK.
-+ */
-+u32 qman_static_dequeue_get(void);
-+
-+/**
-+ * qman_dca - Perform a Discrete Consumption Acknowledgement
-+ * @dq: the DQRR entry to be consumed
-+ * @park_request: indicates whether the held-active @fq should be parked
-+ *
-+ * Only allowed in DCA-mode portals, for DQRR entries whose handler callback had
-+ * previously returned 'qman_cb_dqrr_defer'. NB, as with the other APIs, this
-+ * does not take a 'portal' argument but implies the core affine portal from the
-+ * cpu that is currently executing the function. For reasons of locking, this
-+ * function must be called from the same CPU as that which processed the DQRR
-+ * entry in the first place.
-+ */
-+void qman_dca(struct qm_dqrr_entry *dq, int park_request);
-+
-+/**
-+ * qman_eqcr_is_empty - Determine if portal's EQCR is empty
-+ *
-+ * For use in situations where a cpu-affine caller needs to determine when all
-+ * enqueues for the local portal have been processed by Qman but can't use the
-+ * QMAN_ENQUEUE_FLAG_WAIT_SYNC flag to do this from the final qman_enqueue().
-+ * The function forces tracking of EQCR consumption (which normally doesn't
-+ * happen until enqueue processing needs to find space to put new enqueue
-+ * commands), and returns zero if the ring still has unprocessed entries,
-+ * non-zero if it is empty.
-+ */
-+int qman_eqcr_is_empty(void);
-+
-+/**
-+ * qman_set_dc_ern - Set the handler for DCP enqueue rejection notifications
-+ * @handler: callback for processing DCP ERNs
-+ * @affine: whether this handler is specific to the locally affine portal
-+ *
-+ * If a hardware block's interface to Qman (ie. its direct-connect portal, or
-+ * DCP) is configured not to receive enqueue rejections, then any enqueues
-+ * through that DCP that are rejected will be sent to a given software portal.
-+ * If @affine is non-zero, then this handler will only be used for DCP ERNs
-+ * received on the portal affine to the current CPU. If multiple CPUs share a
-+ * portal and they all call this function, they will be setting the handler for
-+ * the same portal! If @affine is zero, then this handler will be global to all
-+ * portals handled by this instance of the driver. Only those portals that do
-+ * not have their own affine handler will use the global handler.
-+ */
-+void qman_set_dc_ern(qman_cb_dc_ern handler, int affine);
-+
-+ /* FQ management */
-+ /* ------------- */
-+/**
-+ * qman_create_fq - Allocates a FQ
-+ * @fqid: the index of the FQD to encapsulate, must be "Out of Service"
-+ * @flags: bit-mask of QMAN_FQ_FLAG_*** options
-+ * @fq: memory for storing the 'fq', with callbacks filled in
-+ *
-+ * Creates a frame queue object for the given @fqid, unless the
-+ * QMAN_FQ_FLAG_DYNAMIC_FQID flag is set in @flags, in which case a FQID is
-+ * dynamically allocated (or the function fails if none are available). Once
-+ * created, the caller should not touch the memory at 'fq' except as extended to
-+ * adjacent memory for user-defined fields (see the definition of "struct
-+ * qman_fq" for more info). NO_MODIFY is only intended for enqueuing to
-+ * pre-existing frame-queues that aren't to be otherwise interfered with, it
-+ * prevents all other modifications to the frame queue. The TO_DCPORTAL flag
-+ * causes the driver to honour any contextB modifications requested in the
-+ * qm_init_fq() API, as this indicates the frame queue will be consumed by a
-+ * direct-connect portal (PME, CAAM, or Fman). When frame queues are consumed by
-+ * software portals, the contextB field is controlled by the driver and can't be
-+ * modified by the caller. If the AS_IS flag is specified, management commands
-+ * will be used on portal @p to query state for frame queue @fqid and construct
-+ * a frame queue object based on that, rather than assuming/requiring that it be
-+ * Out of Service.
-+ */
-+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq);
-+
-+/**
-+ * qman_destroy_fq - Deallocates a FQ
-+ * @fq: the frame queue object to release
-+ * @flags: bit-mask of QMAN_FQ_FREE_*** options
-+ *
-+ * The memory for this frame queue object ('fq' provided in qman_create_fq()) is
-+ * not deallocated but the caller regains ownership, to do with as desired. The
-+ * FQ must be in the 'out-of-service' state unless the QMAN_FQ_FREE_PARKED flag
-+ * is specified, in which case it may also be in the 'parked' state.
-+ */
-+void qman_destroy_fq(struct qman_fq *fq, u32 flags);
-+
-+/**
-+ * qman_fq_fqid - Queries the frame queue ID of a FQ object
-+ * @fq: the frame queue object to query
-+ */
-+u32 qman_fq_fqid(struct qman_fq *fq);
-+
-+/**
-+ * qman_fq_state - Queries the state of a FQ object
-+ * @fq: the frame queue object to query
-+ * @state: pointer to state enum to return the FQ scheduling state
-+ * @flags: pointer to state flags to receive QMAN_FQ_STATE_*** bitmask
-+ *
-+ * Queries the state of the FQ object, without performing any h/w commands.
-+ * This captures the state, as seen by the driver, at the time the function
-+ * executes.
-+ */
-+void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags);
-+
-+/**
-+ * qman_init_fq - Initialises FQ fields, leaves the FQ "parked" or "scheduled"
-+ * @fq: the frame queue object to modify, must be 'parked' or new.
-+ * @flags: bit-mask of QMAN_INITFQ_FLAG_*** options
-+ * @opts: the FQ-modification settings, as defined in the low-level API
-+ *
-+ * The @opts parameter comes from the low-level portal API. Select
-+ * QMAN_INITFQ_FLAG_SCHED in @flags to cause the frame queue to be scheduled
-+ * rather than parked. NB, @opts can be NULL.
-+ *
-+ * Note that some fields and options within @opts may be ignored or overwritten
-+ * by the driver;
-+ * 1. the 'count' and 'fqid' fields are always ignored (this operation only
-+ * affects one frame queue: @fq).
-+ * 2. the QM_INITFQ_WE_CONTEXTB option of the 'we_mask' field and the associated
-+ * 'fqd' structure's 'context_b' field are sometimes overwritten;
-+ * - if @fq was not created with QMAN_FQ_FLAG_TO_DCPORTAL, then context_b is
-+ * initialised to a value used by the driver for demux.
-+ * - if context_b is initialised for demux, so is context_a in case stashing
-+ * is requested (see item 4).
-+ * (So caller control of context_b is only possible for TO_DCPORTAL frame queue
-+ * objects.)
-+ * 3. if @flags contains QMAN_INITFQ_FLAG_LOCAL, the 'fqd' structure's
-+ * 'dest::channel' field will be overwritten to match the portal used to issue
-+ * the command. If the WE_DESTWQ write-enable bit had already been set by the
-+ * caller, the channel workqueue will be left as-is, otherwise the write-enable
-+ * bit is set and the workqueue is set to a default of 4. If the "LOCAL" flag
-+ * isn't set, the destination channel/workqueue fields and the write-enable bit
-+ * are left as-is.
-+ * 4. if the driver overwrites context_a/b for demux, then if
-+ * QM_INITFQ_WE_CONTEXTA is set, the driver will only overwrite
-+ * context_a.address fields and will leave the stashing fields provided by the
-+ * user alone, otherwise it will zero out the context_a.stashing fields.
-+ */
-+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts);
-+
-+/**
-+ * qman_schedule_fq - Schedules a FQ
-+ * @fq: the frame queue object to schedule, must be 'parked'
-+ *
-+ * Schedules the frame queue, which must be Parked, which takes it to
-+ * Tentatively-Scheduled or Truly-Scheduled depending on its fill-level.
-+ */
-+int qman_schedule_fq(struct qman_fq *fq);
-+
-+/**
-+ * qman_retire_fq - Retires a FQ
-+ * @fq: the frame queue object to retire
-+ * @flags: FQ flags (as per qman_fq_state) if retirement completes immediately
-+ *
-+ * Retires the frame queue. This returns zero if it succeeds immediately, +1 if
-+ * the retirement was started asynchronously, otherwise it returns negative for
-+ * failure. When this function returns zero, @flags is set to indicate whether
-+ * the retired FQ is empty and/or whether it has any ORL fragments (to show up
-+ * as ERNs). Otherwise the corresponding flags will be known when a subsequent
-+ * FQRN message shows up on the portal's message ring.
-+ *
-+ * NB, if the retirement is asynchronous (the FQ was in the Truly Scheduled or
-+ * Active state), the completion will be via the message ring as a FQRN - but
-+ * the corresponding callback may occur before this function returns!! Ie. the
-+ * caller should be prepared to accept the callback as the function is called,
-+ * not only once it has returned.
-+ */
-+int qman_retire_fq(struct qman_fq *fq, u32 *flags);
-+
-+/**
-+ * qman_oos_fq - Puts a FQ "out of service"
-+ * @fq: the frame queue object to be put out-of-service, must be 'retired'
-+ *
-+ * The frame queue must be retired and empty, and if any order restoration list
-+ * was released as ERNs at the time of retirement, they must all be consumed.
-+ */
-+int qman_oos_fq(struct qman_fq *fq);
-+
-+/**
-+ * qman_fq_flow_control - Set the XON/XOFF state of a FQ
-+ * @fq: the frame queue object to be set to XON/XOFF state, must not be 'oos',
-+ * or 'retired' or 'parked' state
-+ * @xon: boolean to set fq in XON or XOFF state
-+ *
-+ * The frame should be in Tentatively Scheduled state or Truly Schedule sate,
-+ * otherwise the IFSI interrupt will be asserted.
-+ */
-+int qman_fq_flow_control(struct qman_fq *fq, int xon);
-+
-+/**
-+ * qman_query_fq - Queries FQD fields (via h/w query command)
-+ * @fq: the frame queue object to be queried
-+ * @fqd: storage for the queried FQD fields
-+ */
-+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd);
-+
-+/**
-+ * qman_query_fq_np - Queries non-programmable FQD fields
-+ * @fq: the frame queue object to be queried
-+ * @np: storage for the queried FQD fields
-+ */
-+int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np);
-+
-+/**
-+ * qman_query_wq - Queries work queue lengths
-+ * @query_dedicated: If non-zero, query length of WQs in the channel dedicated
-+ * to this software portal. Otherwise, query length of WQs in a
-+ * channel specified in wq.
-+ * @wq: storage for the queried WQs lengths. Also specified the channel to
-+ * to query if query_dedicated is zero.
-+ */
-+int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq);
-+
-+/**
-+ * qman_volatile_dequeue - Issue a volatile dequeue command
-+ * @fq: the frame queue object to dequeue from
-+ * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options
-+ * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set()
-+ *
-+ * Attempts to lock access to the portal's VDQCR volatile dequeue functionality.
-+ * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and
-+ * the VDQCR is already in use, otherwise returns non-zero for failure. If
-+ * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once
-+ * the VDQCR command has finished executing (ie. once the callback for the last
-+ * DQRR entry resulting from the VDQCR command has been called). If not using
-+ * the FINISH flag, completion can be determined either by detecting the
-+ * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits
-+ * in the "stat" field of the "struct qm_dqrr_entry" passed to the FQ's dequeue
-+ * callback, or by waiting for the QMAN_FQ_STATE_VDQCR bit to disappear from the
-+ * "flags" retrieved from qman_fq_state().
-+ */
-+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr);
-+
-+/**
-+ * qman_enqueue - Enqueue a frame to a frame queue
-+ * @fq: the frame queue object to enqueue to
-+ * @fd: a descriptor of the frame to be enqueued
-+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options
-+ *
-+ * Fills an entry in the EQCR of portal @qm to enqueue the frame described by
-+ * @fd. The descriptor details are copied from @fd to the EQCR entry, the 'pid'
-+ * field is ignored. The return value is non-zero on error, such as ring full
-+ * (and FLAG_WAIT not specified), congestion avoidance (FLAG_WATCH_CGR
-+ * specified), etc. If the ring is full and FLAG_WAIT is specified, this
-+ * function will block. If FLAG_INTERRUPT is set, the EQCI bit of the portal
-+ * interrupt will assert when Qman consumes the EQCR entry (subject to "status
-+ * disable", "enable", and "inhibit" registers). If FLAG_DCA is set, Qman will
-+ * perform an implied "discrete consumption acknowledgement" on the dequeue
-+ * ring's (DQRR) entry, at the ring index specified by the FLAG_DCA_IDX(x)
-+ * macro. (As an alternative to issuing explicit DCA actions on DQRR entries,
-+ * this implicit DCA can delay the release of a "held active" frame queue
-+ * corresponding to a DQRR entry until Qman consumes the EQCR entry - providing
-+ * order-preservation semantics in packet-forwarding scenarios.) If FLAG_DCA is
-+ * set, then FLAG_DCA_PARK can also be set to imply that the DQRR consumption
-+ * acknowledgement should "park request" the "held active" frame queue. Ie.
-+ * when the portal eventually releases that frame queue, it will be left in the
-+ * Parked state rather than Tentatively Scheduled or Truly Scheduled. If the
-+ * portal is watching congestion groups, the QMAN_ENQUEUE_FLAG_WATCH_CGR flag
-+ * is requested, and the FQ is a member of a congestion group, then this
-+ * function returns -EAGAIN if the congestion group is currently congested.
-+ * Note, this does not eliminate ERNs, as the async interface means we can be
-+ * sending enqueue commands to an un-congested FQ that becomes congested before
-+ * the enqueue commands are processed, but it does minimise needless thrashing
-+ * of an already busy hardware resource by throttling many of the to-be-dropped
-+ * enqueues "at the source".
-+ */
-+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags);
-+
-+typedef int (*qman_cb_precommit) (void *arg);
-+/**
-+ * qman_enqueue_precommit - Enqueue a frame to a frame queue and call cb
-+ * @fq: the frame queue object to enqueue to
-+ * @fd: a descriptor of the frame to be enqueued
-+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options
-+ * @cb: user supplied callback function to invoke before writing commit verb.
-+ * @cb_arg: callback function argument
-+ *
-+ * This is similar to qman_enqueue except that it will invoke a user supplied
-+ * callback function just before writng the commit verb. This is useful
-+ * when the user want to do something *just before* enqueuing the request and
-+ * the enqueue can't fail.
-+ */
-+int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd,
-+ u32 flags, qman_cb_precommit cb, void *cb_arg);
-+
-+/**
-+ * qman_enqueue_orp - Enqueue a frame to a frame queue using an ORP
-+ * @fq: the frame queue object to enqueue to
-+ * @fd: a descriptor of the frame to be enqueued
-+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options
-+ * @orp: the frame queue object used as an order restoration point.
-+ * @orp_seqnum: the sequence number of this frame in the order restoration path
-+ *
-+ * Similar to qman_enqueue(), but with the addition of an Order Restoration
-+ * Point (@orp) and corresponding sequence number (@orp_seqnum) for this
-+ * enqueue operation to employ order restoration. Each frame queue object acts
-+ * as an Order Definition Point (ODP) by providing each frame dequeued from it
-+ * with an incrementing sequence number, this value is generally ignored unless
-+ * that sequence of dequeued frames will need order restoration later. Each
-+ * frame queue object also encapsulates an Order Restoration Point (ORP), which
-+ * is a re-assembly context for re-ordering frames relative to their sequence
-+ * numbers as they are enqueued. The ORP does not have to be within the frame
-+ * queue that receives the enqueued frame, in fact it is usually the frame
-+ * queue from which the frames were originally dequeued. For the purposes of
-+ * order restoration, multiple frames (or "fragments") can be enqueued for a
-+ * single sequence number by setting the QMAN_ENQUEUE_FLAG_NLIS flag for all
-+ * enqueues except the final fragment of a given sequence number. Ordering
-+ * between sequence numbers is guaranteed, even if fragments of different
-+ * sequence numbers are interlaced with one another. Fragments of the same
-+ * sequence number will retain the order in which they are enqueued. If no
-+ * enqueue is to performed, QMAN_ENQUEUE_FLAG_HOLE indicates that the given
-+ * sequence number is to be "skipped" by the ORP logic (eg. if a frame has been
-+ * dropped from a sequence), or QMAN_ENQUEUE_FLAG_NESN indicates that the given
-+ * sequence number should become the ORP's "Next Expected Sequence Number".
-+ *
-+ * Side note: a frame queue object can be used purely as an ORP, without
-+ * carrying any frames at all. Care should be taken not to deallocate a frame
-+ * queue object that is being actively used as an ORP, as a future allocation
-+ * of the frame queue object may start using the internal ORP before the
-+ * previous use has finished.
-+ */
-+int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
-+ struct qman_fq *orp, u16 orp_seqnum);
-+
-+/**
-+ * qman_alloc_fqid_range - Allocate a contiguous range of FQIDs
-+ * @result: is set by the API to the base FQID of the allocated range
-+ * @count: the number of FQIDs required
-+ * @align: required alignment of the allocated range
-+ * @partial: non-zero if the API can return fewer than @count FQIDs
-+ *
-+ * Returns the number of frame queues allocated, or a negative error code. If
-+ * @partial is non zero, the allocation request may return a smaller range of
-+ * FQs than requested (though alignment will be as requested). If @partial is
-+ * zero, the return value will either be 'count' or negative.
-+ */
-+int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial);
-+static inline int qman_alloc_fqid(u32 *result)
-+{
-+ int ret = qman_alloc_fqid_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+
-+/**
-+ * qman_release_fqid_range - Release the specified range of frame queue IDs
-+ * @fqid: the base FQID of the range to deallocate
-+ * @count: the number of FQIDs in the range
-+ *
-+ * This function can also be used to seed the allocator with ranges of FQIDs
-+ * that it can subsequently allocate from.
-+ */
-+void qman_release_fqid_range(u32 fqid, unsigned int count);
-+static inline void qman_release_fqid(u32 fqid)
-+{
-+ qman_release_fqid_range(fqid, 1);
-+}
-+
-+void qman_seed_fqid_range(u32 fqid, unsigned int count);
-+
-+
-+int qman_shutdown_fq(u32 fqid);
-+
-+/**
-+ * qman_reserve_fqid_range - Reserve the specified range of frame queue IDs
-+ * @fqid: the base FQID of the range to deallocate
-+ * @count: the number of FQIDs in the range
-+ */
-+int qman_reserve_fqid_range(u32 fqid, unsigned int count);
-+static inline int qman_reserve_fqid(u32 fqid)
-+{
-+ return qman_reserve_fqid_range(fqid, 1);
-+}
-+
-+ /* Pool-channel management */
-+ /* ----------------------- */
-+/**
-+ * qman_alloc_pool_range - Allocate a contiguous range of pool-channel IDs
-+ * @result: is set by the API to the base pool-channel ID of the allocated range
-+ * @count: the number of pool-channel IDs required
-+ * @align: required alignment of the allocated range
-+ * @partial: non-zero if the API can return fewer than @count
-+ *
-+ * Returns the number of pool-channel IDs allocated, or a negative error code.
-+ * If @partial is non zero, the allocation request may return a smaller range of
-+ * than requested (though alignment will be as requested). If @partial is zero,
-+ * the return value will either be 'count' or negative.
-+ */
-+int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial);
-+static inline int qman_alloc_pool(u32 *result)
-+{
-+ int ret = qman_alloc_pool_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+
-+/**
-+ * qman_release_pool_range - Release the specified range of pool-channel IDs
-+ * @id: the base pool-channel ID of the range to deallocate
-+ * @count: the number of pool-channel IDs in the range
-+ */
-+void qman_release_pool_range(u32 id, unsigned int count);
-+static inline void qman_release_pool(u32 id)
-+{
-+ qman_release_pool_range(id, 1);
-+}
-+
-+/**
-+ * qman_reserve_pool_range - Reserve the specified range of pool-channel IDs
-+ * @id: the base pool-channel ID of the range to reserve
-+ * @count: the number of pool-channel IDs in the range
-+ */
-+int qman_reserve_pool_range(u32 id, unsigned int count);
-+static inline int qman_reserve_pool(u32 id)
-+{
-+ return qman_reserve_pool_range(id, 1);
-+}
-+
-+void qman_seed_pool_range(u32 id, unsigned int count);
-+
-+ /* CGR management */
-+ /* -------------- */
-+/**
-+ * qman_create_cgr - Register a congestion group object
-+ * @cgr: the 'cgr' object, with fields filled in
-+ * @flags: QMAN_CGR_FLAG_* values
-+ * @opts: optional state of CGR settings
-+ *
-+ * Registers this object to receiving congestion entry/exit callbacks on the
-+ * portal affine to the cpu portal on which this API is executed. If opts is
-+ * NULL then only the callback (cgr->cb) function is registered. If @flags
-+ * contains QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset
-+ * any unspecified parameters) will be used rather than a modify hw hardware
-+ * (which only modifies the specified parameters).
-+ */
-+int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
-+ struct qm_mcc_initcgr *opts);
-+
-+/**
-+ * qman_create_cgr_to_dcp - Register a congestion group object to DCP portal
-+ * @cgr: the 'cgr' object, with fields filled in
-+ * @flags: QMAN_CGR_FLAG_* values
-+ * @dcp_portal: the DCP portal to which the cgr object is registered.
-+ * @opts: optional state of CGR settings
-+ *
-+ */
-+int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
-+ struct qm_mcc_initcgr *opts);
-+
-+/**
-+ * qman_delete_cgr - Deregisters a congestion group object
-+ * @cgr: the 'cgr' object to deregister
-+ *
-+ * "Unplugs" this CGR object from the portal affine to the cpu on which this API
-+ * is executed. This must be excuted on the same affine portal on which it was
-+ * created.
-+ */
-+int qman_delete_cgr(struct qman_cgr *cgr);
-+
-+/**
-+ * qman_delete_cgr_safe - Deregisters a congestion group object from any CPU
-+ * @cgr: the 'cgr' object to deregister
-+ *
-+ * This will select the proper CPU and run there qman_delete_cgr().
-+ */
-+void qman_delete_cgr_safe(struct qman_cgr *cgr);
-+
-+/**
-+ * qman_modify_cgr - Modify CGR fields
-+ * @cgr: the 'cgr' object to modify
-+ * @flags: QMAN_CGR_FLAG_* values
-+ * @opts: the CGR-modification settings
-+ *
-+ * The @opts parameter comes from the low-level portal API, and can be NULL.
-+ * Note that some fields and options within @opts may be ignored or overwritten
-+ * by the driver, in particular the 'cgrid' field is ignored (this operation
-+ * only affects the given CGR object). If @flags contains
-+ * QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset any
-+ * unspecified parameters) will be used rather than a modify hw hardware (which
-+ * only modifies the specified parameters).
-+ */
-+int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
-+ struct qm_mcc_initcgr *opts);
-+
-+/**
-+* qman_query_cgr - Queries CGR fields
-+* @cgr: the 'cgr' object to query
-+* @result: storage for the queried congestion group record
-+*/
-+int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *result);
-+
-+/**
-+ * qman_query_congestion - Queries the state of all congestion groups
-+ * @congestion: storage for the queried state of all congestion groups
-+ */
-+int qman_query_congestion(struct qm_mcr_querycongestion *congestion);
-+
-+/**
-+ * qman_alloc_cgrid_range - Allocate a contiguous range of CGR IDs
-+ * @result: is set by the API to the base CGR ID of the allocated range
-+ * @count: the number of CGR IDs required
-+ * @align: required alignment of the allocated range
-+ * @partial: non-zero if the API can return fewer than @count
-+ *
-+ * Returns the number of CGR IDs allocated, or a negative error code.
-+ * If @partial is non zero, the allocation request may return a smaller range of
-+ * than requested (though alignment will be as requested). If @partial is zero,
-+ * the return value will either be 'count' or negative.
-+ */
-+int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial);
-+static inline int qman_alloc_cgrid(u32 *result)
-+{
-+ int ret = qman_alloc_cgrid_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+
-+/**
-+ * qman_release_cgrid_range - Release the specified range of CGR IDs
-+ * @id: the base CGR ID of the range to deallocate
-+ * @count: the number of CGR IDs in the range
-+ */
-+void qman_release_cgrid_range(u32 id, unsigned int count);
-+static inline void qman_release_cgrid(u32 id)
-+{
-+ qman_release_cgrid_range(id, 1);
-+}
-+
-+/**
-+ * qman_reserve_cgrid_range - Reserve the specified range of CGR ID
-+ * @id: the base CGR ID of the range to reserve
-+ * @count: the number of CGR IDs in the range
-+ */
-+int qman_reserve_cgrid_range(u32 id, unsigned int count);
-+static inline int qman_reserve_cgrid(u32 id)
-+{
-+ return qman_reserve_cgrid_range(id, 1);
-+}
-+
-+void qman_seed_cgrid_range(u32 id, unsigned int count);
-+
-+
-+ /* Helpers */
-+ /* ------- */
-+/**
-+ * qman_poll_fq_for_init - Check if an FQ has been initialised from OOS
-+ * @fqid: the FQID that will be initialised by other s/w
-+ *
-+ * In many situations, a FQID is provided for communication between s/w
-+ * entities, and whilst the consumer is responsible for initialising and
-+ * scheduling the FQ, the producer(s) generally create a wrapper FQ object using
-+ * and only call qman_enqueue() (no FQ initialisation, scheduling, etc). Ie;
-+ * qman_create_fq(..., QMAN_FQ_FLAG_NO_MODIFY, ...);
-+ * However, data can not be enqueued to the FQ until it is initialised out of
-+ * the OOS state - this function polls for that condition. It is particularly
-+ * useful for users of IPC functions - each endpoint's Rx FQ is the other
-+ * endpoint's Tx FQ, so each side can initialise and schedule their Rx FQ object
-+ * and then use this API on the (NO_MODIFY) Tx FQ object in order to
-+ * synchronise. The function returns zero for success, +1 if the FQ is still in
-+ * the OOS state, or negative if there was an error.
-+ */
-+static inline int qman_poll_fq_for_init(struct qman_fq *fq)
-+{
-+ struct qm_mcr_queryfq_np np;
-+ int err;
-+ err = qman_query_fq_np(fq, &np);
-+ if (err)
-+ return err;
-+ if ((np.state & QM_MCR_NP_STATE_MASK) == QM_MCR_NP_STATE_OOS)
-+ return 1;
-+ return 0;
-+}
-+
-+ /* -------------- */
-+ /* CEETM :: types */
-+ /* -------------- */
-+/**
-+ * Token Rate Structure
-+ * Shaping rates are based on a "credit" system and a pre-configured h/w
-+ * internal timer. The following type represents a shaper "rate" parameter as a
-+ * fractional number of "tokens". Here's how it works. This (fractional) number
-+ * of tokens is added to the shaper's "credit" every time the h/w timer elapses
-+ * (up to a limit which is set by another shaper parameter). Every time a frame
-+ * is enqueued through a shaper, the shaper deducts as many tokens as there are
-+ * bytes of data in the enqueued frame. A shaper will not allow itself to
-+ * enqueue any frames if its token count is negative. As such;
-+ *
-+ * The rate at which data is enqueued is limited by the
-+ * rate at which tokens are added.
-+ *
-+ * Therefore if the user knows the period between these h/w timer updates in
-+ * seconds, they can calculate the maximum traffic rate of the shaper (in
-+ * bytes-per-second) from the token rate. And vice versa, they can calculate
-+ * the token rate to use in order to achieve a given traffic rate.
-+ */
-+struct qm_ceetm_rate {
-+ /* The token rate is; whole + (fraction/8192) */
-+ u32 whole:11; /* 0..2047 */
-+ u32 fraction:13; /* 0..8191 */
-+};
-+
-+struct qm_ceetm_weight_code {
-+ /* The weight code is; 5 msbits + 3 lsbits */
-+ u8 y:5;
-+ u8 x:3;
-+};
-+
-+struct qm_ceetm {
-+ unsigned int idx;
-+ struct list_head sub_portals;
-+ struct list_head lnis;
-+ unsigned int sp_range[2];
-+ unsigned int lni_range[2];
-+};
-+
-+struct qm_ceetm_sp {
-+ struct list_head node;
-+ unsigned int idx;
-+ unsigned int dcp_idx;
-+ int is_claimed;
-+ struct qm_ceetm_lni *lni;
-+};
-+
-+/* Logical Network Interface */
-+struct qm_ceetm_lni {
-+ struct list_head node;
-+ unsigned int idx;
-+ unsigned int dcp_idx;
-+ int is_claimed;
-+ struct qm_ceetm_sp *sp;
-+ struct list_head channels;
-+ int shaper_enable;
-+ int shaper_couple;
-+ int oal;
-+ struct qm_ceetm_rate cr_token_rate;
-+ struct qm_ceetm_rate er_token_rate;
-+ u16 cr_token_bucket_limit;
-+ u16 er_token_bucket_limit;
-+};
-+
-+/* Class Queue Channel */
-+struct qm_ceetm_channel {
-+ struct list_head node;
-+ unsigned int idx;
-+ unsigned int lni_idx;
-+ unsigned int dcp_idx;
-+ struct list_head class_queues;
-+ struct list_head ccgs;
-+ u8 shaper_enable;
-+ u8 shaper_couple;
-+ struct qm_ceetm_rate cr_token_rate;
-+ struct qm_ceetm_rate er_token_rate;
-+ u16 cr_token_bucket_limit;
-+ u16 er_token_bucket_limit;
-+};
-+
-+struct qm_ceetm_ccg;
-+
-+/* This callback type is used when handling congestion entry/exit. The
-+ * 'cb_ctx' value is the opaque value associated with ccg object.
-+ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit.
-+ */
-+typedef void (*qman_cb_ccgr)(struct qm_ceetm_ccg *ccg, void *cb_ctx,
-+ int congested);
-+
-+/* Class Congestion Group */
-+struct qm_ceetm_ccg {
-+ struct qm_ceetm_channel *parent;
-+ struct list_head node;
-+ struct list_head cb_node;
-+ qman_cb_ccgr cb;
-+ void *cb_ctx;
-+ unsigned int idx;
-+};
-+
-+/* Class Queue */
-+struct qm_ceetm_cq {
-+ struct qm_ceetm_channel *parent;
-+ struct qm_ceetm_ccg *ccg;
-+ struct list_head node;
-+ unsigned int idx;
-+ int is_claimed;
-+ struct list_head bound_lfqids;
-+ struct list_head binding_node;
-+};
-+
-+/* Logical Frame Queue */
-+struct qm_ceetm_lfq {
-+ struct qm_ceetm_channel *parent;
-+ struct list_head node;
-+ unsigned int idx;
-+ unsigned int dctidx;
-+ u64 context_a;
-+ u32 context_b;
-+ qman_cb_mr ern;
-+};
-+
-+/**
-+ * qman_ceetm_bps2tokenrate - Given a desired rate 'bps' measured in bps
-+ * (ie. bits-per-second), compute the 'token_rate' fraction that best
-+ * approximates that rate.
-+ * @bps: the desired shaper rate in bps.
-+ * @token_rate: the output token rate computed with the given kbps.
-+ * @rounding: dictates how to round if an exact conversion is not possible; if
-+ * it is negative then 'token_rate' will round down to the highest value that
-+ * does not exceed the desired rate, if it is positive then 'token_rate' will
-+ * round up to the lowest value that is greater than or equal to the desired
-+ * rate, and if it is zero then it will round to the nearest approximation,
-+ * whether that be up or down.
-+ *
-+ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available.
-+ */
-+int qman_ceetm_bps2tokenrate(u64 bps,
-+ struct qm_ceetm_rate *token_rate,
-+ int rounding);
-+
-+/**
-+ * qman_ceetm_tokenrate2bps - Given a 'token_rate', compute the
-+ * corresponding number of 'bps'.
-+ * @token_rate: the input desired token_rate fraction.
-+ * @bps: the output shaper rate in bps computed with the give token rate.
-+ * @rounding: has the same semantics as the previous function.
-+ *
-+ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available.
-+ */
-+int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate,
-+ u64 *bps,
-+ int rounding);
-+
-+int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align,
-+ int partial);
-+static inline int qman_alloc_ceetm0_channel(u32 *result)
-+{
-+ int ret = qman_alloc_ceetm0_channel_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+void qman_release_ceetm0_channel_range(u32 channelid, u32 count);
-+static inline void qman_release_ceetm0_channelid(u32 channelid)
-+{
-+ qman_release_ceetm0_channel_range(channelid, 1);
-+}
-+
-+int qman_reserve_ceetm0_channel_range(u32 channelid, u32 count);
-+static inline int qman_reserve_ceetm0_channelid(u32 channelid)
-+{
-+ return qman_reserve_ceetm0_channel_range(channelid, 1);
-+}
-+
-+void qman_seed_ceetm0_channel_range(u32 channelid, u32 count);
-+
-+
-+int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align,
-+ int partial);
-+static inline int qman_alloc_ceetm1_channel(u32 *result)
-+{
-+ int ret = qman_alloc_ceetm1_channel_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+void qman_release_ceetm1_channel_range(u32 channelid, u32 count);
-+static inline void qman_release_ceetm1_channelid(u32 channelid)
-+{
-+ qman_release_ceetm1_channel_range(channelid, 1);
-+}
-+int qman_reserve_ceetm1_channel_range(u32 channelid, u32 count);
-+static inline int qman_reserve_ceetm1_channelid(u32 channelid)
-+{
-+ return qman_reserve_ceetm1_channel_range(channelid, 1);
-+}
-+
-+void qman_seed_ceetm1_channel_range(u32 channelid, u32 count);
-+
-+
-+int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align,
-+ int partial);
-+static inline int qman_alloc_ceetm0_lfqid(u32 *result)
-+{
-+ int ret = qman_alloc_ceetm0_lfqid_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count);
-+static inline void qman_release_ceetm0_lfqid(u32 lfqid)
-+{
-+ qman_release_ceetm0_lfqid_range(lfqid, 1);
-+}
-+int qman_reserve_ceetm0_lfqid_range(u32 lfqid, u32 count);
-+static inline int qman_reserve_ceetm0_lfqid(u32 lfqid)
-+{
-+ return qman_reserve_ceetm0_lfqid_range(lfqid, 1);
-+}
-+
-+void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count);
-+
-+
-+int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align,
-+ int partial);
-+static inline int qman_alloc_ceetm1_lfqid(u32 *result)
-+{
-+ int ret = qman_alloc_ceetm1_lfqid_range(result, 1, 0, 0);
-+ return (ret > 0) ? 0 : ret;
-+}
-+void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count);
-+static inline void qman_release_ceetm1_lfqid(u32 lfqid)
-+{
-+ qman_release_ceetm1_lfqid_range(lfqid, 1);
-+}
-+int qman_reserve_ceetm1_lfqid_range(u32 lfqid, u32 count);
-+static inline int qman_reserve_ceetm1_lfqid(u32 lfqid)
-+{
-+ return qman_reserve_ceetm1_lfqid_range(lfqid, 1);
-+}
-+
-+void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count);
-+
-+
-+ /* ----------------------------- */
-+ /* CEETM :: sub-portals */
-+ /* ----------------------------- */
-+
-+/**
-+ * qman_ceetm_sp_claim - Claims the given sub-portal, provided it is available
-+ * to us and configured for traffic-management.
-+ * @sp: the returned sub-portal object, if successful.
-+ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM
-+ * instance),
-+ * @sp_idx" is the desired sub-portal index from 0 to 15.
-+ *
-+ * Returns zero for success, or -ENODEV if the sub-portal is in use, or -EINVAL
-+ * if the sp_idx is out of range.
-+ *
-+ * Note that if there are multiple driver domains (eg. a linux kernel versus
-+ * user-space drivers in USDPAA, or multiple guests running under a hypervisor)
-+ * then a sub-portal may be accessible by more than one instance of a qman
-+ * driver and so it may be claimed multiple times. If this is the case, it is
-+ * up to the system architect to prevent conflicting configuration actions
-+ * coming from the different driver domains. The qman drivers do not have any
-+ * behind-the-scenes coordination to prevent this from happening.
-+ */
-+int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp,
-+ enum qm_dc_portal dcp_idx,
-+ unsigned int sp_idx);
-+
-+/**
-+ * qman_ceetm_sp_release - Releases a previously claimed sub-portal.
-+ * @sp: the sub-portal to be released.
-+ *
-+ * Returns 0 for success, or -EBUSY for failure if the dependencies are not
-+ * released.
-+ */
-+int qman_ceetm_sp_release(struct qm_ceetm_sp *sp);
-+
-+ /* ----------------------------------- */
-+ /* CEETM :: logical network interfaces */
-+ /* ----------------------------------- */
-+
-+/**
-+ * qman_ceetm_lni_claim - Claims an unclaimed LNI.
-+ * @lni: the returned LNI object, if successful.
-+ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM
-+ * instance)
-+ * @lni_idx: is the desired LNI index.
-+ *
-+ * Returns zero for success, or -EINVAL on failure, which will happen if the LNI
-+ * is not available or has already been claimed (and not yet successfully
-+ * released), or lni_dix is out of range.
-+ *
-+ * Note that there may be multiple driver domains (or instances) that need to
-+ * transmit out the same LNI, so this claim is only guaranteeing exclusivity
-+ * within the domain of the driver being called. See qman_ceetm_sp_claim() and
-+ * qman_ceetm_sp_get_lni() for more information.
-+ */
-+int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni,
-+ enum qm_dc_portal dcp_id,
-+ unsigned int lni_idx);
-+
-+/**
-+ * qman_ceetm_lni_releaes - Releases a previously claimed LNI.
-+ * @lni: the lni needs to be released.
-+ *
-+ * This will only succeed if all dependent objects have been released.
-+ * Returns zero for success, or -EBUSY if the dependencies are not released.
-+ */
-+int qman_ceetm_lni_release(struct qm_ceetm_lni *lni);
-+
-+/**
-+ * qman_ceetm_sp_set_lni
-+ * qman_ceetm_sp_get_lni - Set/get the LNI that the sub-portal is currently
-+ * mapped to.
-+ * @sp: the given sub-portal.
-+ * @lni(in "set"function): the LNI object which the sp will be mappaed to.
-+ * @lni_idx(in "get" function): the LNI index which the sp is mapped to.
-+ *
-+ * Returns zero for success, or -EINVAL for the "set" function when this sp-lni
-+ * mapping has been set, or configure mapping command returns error, and
-+ * -EINVAL for "get" function when this sp-lni mapping is not set or the query
-+ * mapping command returns error.
-+ *
-+ * This may be useful in situations where multiple driver domains have access
-+ * to the same sub-portals in order to all be able to transmit out the same
-+ * physical interface (perhaps they're on different IP addresses or VPNs, so
-+ * Fman is splitting Rx traffic and here we need to converge Tx traffic). In
-+ * that case, a control-plane is likely to use qman_ceetm_lni_claim() followed
-+ * by qman_ceetm_sp_set_lni() to configure the sub-portal, and other domains
-+ * are likely to use qman_ceetm_sp_get_lni() followed by qman_ceetm_lni_claim()
-+ * in order to determine the LNI that the control-plane had assigned. This is
-+ * why the "get" returns an index, whereas the "set" takes an (already claimed)
-+ * LNI object.
-+ */
-+int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp,
-+ struct qm_ceetm_lni *lni);
-+int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp,
-+ unsigned int *lni_idx);
-+
-+/**
-+ * qman_ceetm_lni_enable_shaper
-+ * qman_ceetm_lni_disable_shaper - Enables/disables shaping on the LNI.
-+ * @lni: the given LNI.
-+ * @coupled: indicates whether CR and ER are coupled.
-+ * @oal: the overhead accounting length which is added to the actual length of
-+ * each frame when performing shaper calculations.
-+ *
-+ * When the number of (unused) committed-rate tokens reach the committed-rate
-+ * token limit, 'coupled' indicates whether surplus tokens should be added to
-+ * the excess-rate token count (up to the excess-rate token limit).
-+ * When LNI is claimed, the shaper is disabled by default. The enable function
-+ * will turn on this shaper for this lni.
-+ * Whenever a claimed LNI is first enabled for shaping, its committed and
-+ * excess token rates and limits are zero, so will need to be changed to do
-+ * anything useful. The shaper can subsequently be enabled/disabled without
-+ * resetting the shaping parameters, but the shaping parameters will be reset
-+ * when the LNI is released.
-+ *
-+ * Returns zero for success, or errno for "enable" function in the cases as:
-+ * a) -EINVAL if the shaper is already enabled,
-+ * b) -EIO if the configure shaper command returns error.
-+ * For "disable" function, returns:
-+ * a) -EINVAL if the shaper is has already disabled.
-+ * b) -EIO if calling configure shaper command returns error.
-+ */
-+int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled,
-+ int oal);
-+int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni);
-+
-+/**
-+ * qman_ceetm_lni_is_shaper_enabled - Check LNI shaper status
-+ * @lni: the give LNI
-+ */
-+int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni);
-+
-+/**
-+ * qman_ceetm_lni_set_commit_rate
-+ * qman_ceetm_lni_get_commit_rate
-+ * qman_ceetm_lni_set_excess_rate
-+ * qman_ceetm_lni_get_excess_rate - Set/get the shaper CR/ER token rate and
-+ * token limit for the given LNI.
-+ * @lni: the given LNI.
-+ * @token_rate: the desired token rate for "set" fuction, or the token rate of
-+ * the LNI queried by "get" function.
-+ * @token_limit: the desired token bucket limit for "set" function, or the token
-+ * limit of the given LNI queried by "get" function.
-+ *
-+ * Returns zero for success. The "set" function returns -EINVAL if the given
-+ * LNI is unshapped or -EIO if the configure shaper command returns error.
-+ * The "get" function returns -EINVAL if the token rate or the token limit is
-+ * not set or the query command returns error.
-+ */
-+int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit);
-+int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit);
-+int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit);
-+int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit);
-+/**
-+ * qman_ceetm_lni_set_commit_rate_bps
-+ * qman_ceetm_lni_get_commit_rate_bps
-+ * qman_ceetm_lni_set_excess_rate_bps
-+ * qman_ceetm_lni_get_excess_rate_bps - Set/get the shaper CR/ER rate
-+ * and token limit for the given LNI.
-+ * @lni: the given LNI.
-+ * @bps: the desired shaping rate in bps for "set" fuction, or the shaping rate
-+ * of the LNI queried by "get" function.
-+ * @token_limit: the desired token bucket limit for "set" function, or the token
-+ * limit of the given LNI queried by "get" function.
-+ *
-+ * Returns zero for success. The "set" function returns -EINVAL if the given
-+ * LNI is unshapped or -EIO if the configure shaper command returns error.
-+ * The "get" function returns -EINVAL if the token rate or the token limit is
-+ * not set or the query command returns error.
-+ */
-+int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 bps,
-+ u16 token_limit);
-+int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 *bps, u16 *token_limit);
-+int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 bps,
-+ u16 token_limit);
-+int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni,
-+ u64 *bps, u16 *token_limit);
-+
-+/**
-+ * qman_ceetm_lni_set_tcfcc
-+ * qman_ceetm_lni_get_tcfcc - Configure/query "Traffic Class Flow Control".
-+ * @lni: the given LNI.
-+ * @cq_level: is between 0 and 15, representing individual class queue levels
-+ * (CQ0 to CQ7 for every channel) and grouped class queue levels (CQ8 to CQ15
-+ * for every channel).
-+ * @traffic_class: is between 0 and 7 when associating a given class queue level
-+ * to a traffic class, or -1 when disabling traffic class flow control for this
-+ * class queue level.
-+ *
-+ * Return zero for success, or -EINVAL if the cq_level or traffic_class is out
-+ * of range as indicated above, or -EIO if the configure/query tcfcc command
-+ * returns error.
-+ *
-+ * Refer to the section of QMan CEETM traffic class flow control in the
-+ * Reference Manual.
-+ */
-+int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni,
-+ unsigned int cq_level,
-+ int traffic_class);
-+int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni,
-+ unsigned int cq_level,
-+ int *traffic_class);
-+
-+ /* ----------------------------- */
-+ /* CEETM :: class queue channels */
-+ /* ----------------------------- */
-+
-+/**
-+ * qman_ceetm_channel_claim - Claims an unclaimed CQ channel that is mapped to
-+ * the given LNI.
-+ * @channel: the returned class queue channel object, if successful.
-+ * @lni: the LNI that the channel belongs to.
-+ *
-+ * Channels are always initially "unshaped".
-+ *
-+ * Return zero for success, or -ENODEV if there is no channel available(all 32
-+ * channels are claimed) or -EINVAL if the channel mapping command returns
-+ * error.
-+ */
-+int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel,
-+ struct qm_ceetm_lni *lni);
-+
-+/**
-+ * qman_ceetm_channel_release - Releases a previously claimed CQ channel.
-+ * @channel: the channel needs to be released.
-+ *
-+ * Returns zero for success, or -EBUSY if the dependencies are still in use.
-+ *
-+ * Note any shaping of the channel will be cleared to leave it in an unshaped
-+ * state.
-+ */
-+int qman_ceetm_channel_release(struct qm_ceetm_channel *channel);
-+
-+/**
-+ * qman_ceetm_channel_enable_shaper
-+ * qman_ceetm_channel_disable_shaper - Enables/disables shaping on the channel.
-+ * @channel: the given channel.
-+ * @coupled: indicates whether surplus CR tokens should be added to the
-+ * excess-rate token count (up to the excess-rate token limit) when the number
-+ * of (unused) committed-rate tokens reach the committed_rate token limit.
-+ *
-+ * Whenever a claimed channel is first enabled for shaping, its committed and
-+ * excess token rates and limits are zero, so will need to be changed to do
-+ * anything useful. The shaper can subsequently be enabled/disabled without
-+ * resetting the shaping parameters, but the shaping parameters will be reset
-+ * when the channel is released.
-+ *
-+ * Return 0 for success, or -EINVAL for failure, in the case that the channel
-+ * shaper has been enabled/disabled or the management command returns error.
-+ */
-+int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel,
-+ int coupled);
-+int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel);
-+
-+/**
-+ * qman_ceetm_channel_is_shaper_enabled - Check channel shaper status.
-+ * @channel: the give channel.
-+ */
-+int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel);
-+
-+/**
-+ * qman_ceetm_channel_set_commit_rate
-+ * qman_ceetm_channel_get_commit_rate
-+ * qman_ceetm_channel_set_excess_rate
-+ * qman_ceetm_channel_get_excess_rate - Set/get channel CR/ER shaper parameters.
-+ * @channel: the given channel.
-+ * @token_rate: the desired token rate for "set" function, or the queried token
-+ * rate for "get" function.
-+ * @token_limit: the desired token limit for "set" function, or the queried
-+ * token limit for "get" function.
-+ *
-+ * Return zero for success. The "set" function returns -EINVAL if the channel
-+ * is unshaped, or -EIO if the configure shapper command returns error. The
-+ * "get" function returns -EINVAL if token rate of token limit is not set, or
-+ * the query shaper command returns error.
-+ */
-+int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit);
-+int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit);
-+int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel,
-+ const struct qm_ceetm_rate *token_rate,
-+ u16 token_limit);
-+int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel,
-+ struct qm_ceetm_rate *token_rate,
-+ u16 *token_limit);
-+/**
-+ * qman_ceetm_channel_set_commit_rate_bps
-+ * qman_ceetm_channel_get_commit_rate_bps
-+ * qman_ceetm_channel_set_excess_rate_bps
-+ * qman_ceetm_channel_get_excess_rate_bps - Set/get channel CR/ER shaper
-+ * parameters.
-+ * @channel: the given channel.
-+ * @token_rate: the desired shaper rate in bps for "set" function, or the
-+ * shaper rate in bps for "get" function.
-+ * @token_limit: the desired token limit for "set" function, or the queried
-+ * token limit for "get" function.
-+ *
-+ * Return zero for success. The "set" function returns -EINVAL if the channel
-+ * is unshaped, or -EIO if the configure shapper command returns error. The
-+ * "get" function returns -EINVAL if token rate of token limit is not set, or
-+ * the query shaper command returns error.
-+ */
-+int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 bps, u16 token_limit);
-+int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 *bps, u16 *token_limit);
-+int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 bps, u16 token_limit);
-+int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel,
-+ u64 *bps, u16 *token_limit);
-+
-+/**
-+ * qman_ceetm_channel_set_weight
-+ * qman_ceetm_channel_get_weight - Set/get the weight for unshaped channel
-+ * @channel: the given channel.
-+ * @token_limit: the desired token limit as the weight of the unshaped channel
-+ * for "set" function, or the queried token limit for "get" function.
-+ *
-+ * The algorithm of unshaped fair queuing (uFQ) is used for unshaped channel.
-+ * It allows the unshaped channels to be included in the CR time eligible list,
-+ * and thus use the configured CR token limit value as their fair queuing
-+ * weight.
-+ *
-+ * Return zero for success, or -EINVAL if the channel is a shaped channel or
-+ * the management command returns error.
-+ */
-+int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel,
-+ u16 token_limit);
-+int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel,
-+ u16 *token_limit);
-+
-+/**
-+ * qman_ceetm_channel_set_group
-+ * qman_ceetm_channel_get_group - Set/get the grouping of the class scheduler.
-+ * @channel: the given channel.
-+ * @group_b: indicates whether there is group B in this channel.
-+ * @prio_a: the priority of group A.
-+ * @prio_b: the priority of group B.
-+ *
-+ * There are 8 individual class queues (CQ0-CQ7), and 8 grouped class queues
-+ * (CQ8-CQ15). If 'group_b' is zero, then all the grouped class queues are in
-+ * group A, otherwise they are split into group A (CQ8-11) and group B
-+ * (CQ12-C15). The individual class queues and the group(s) are in strict
-+ * priority order relative to each other. Within the group(s), the scheduling
-+ * is not strict priority order, but the result of scheduling within a group
-+ * is in strict priority order relative to the other class queues in the
-+ * channel. 'prio_a' and 'prio_b' control the priority order of the groups
-+ * relative to the individual class queues, and take values from 0-7. Eg. if
-+ * 'group_b' is non-zero, 'prio_a' is 2 and 'prio_b' is 6, then the strict
-+ * priority order would be;
-+ * CQ0, CQ1, CQ2, GROUPA, CQ3, CQ4, CQ5, CQ6, GROUPB, CQ7
-+ *
-+ * Return 0 for success. For "set" function, returns -EINVAL if prio_a or
-+ * prio_b are out of the range 0 - 7 (priority of group A or group B can not
-+ * be 0, CQ0 is always the highest class queue in this channel.), or -EIO if
-+ * the configure scheduler command returns error. For "get" function, return
-+ * -EINVAL if the query scheduler command returns error.
-+ */
-+int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel,
-+ int group_b,
-+ unsigned int prio_a,
-+ unsigned int prio_b);
-+int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel,
-+ int *group_b,
-+ unsigned int *prio_a,
-+ unsigned int *prio_b);
-+
-+/**
-+ * qman_ceetm_channel_set_group_cr_eligibility
-+ * qman_ceetm_channel_set_group_er_eligibility - Set channel group eligibility
-+ * @channel: the given channel object
-+ * @group_b: indicates whether there is group B in this channel.
-+ * @cre: the commit rate eligibility, 1 for enable, 0 for disable.
-+ *
-+ * Return zero for success, or -EINVAL if eligibility setting fails.
-+*/
-+int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel
-+ *channel, int group_b, int cre);
-+int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel
-+ *channel, int group_b, int ere);
-+
-+/**
-+ * qman_ceetm_channel_set_cq_cr_eligibility
-+ * qman_ceetm_channel_set_cq_er_eligibility - Set channel cq eligibility
-+ * @channel: the given channel object
-+ * @idx: is from 0 to 7 (representing CQ0 to CQ7).
-+ * @cre: the commit rate eligibility, 1 for enable, 0 for disable.
-+ *
-+ * Return zero for success, or -EINVAL if eligibility setting fails.
-+*/
-+int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel,
-+ unsigned int idx, int cre);
-+int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel,
-+ unsigned int idx, int ere);
-+
-+ /* --------------------- */
-+ /* CEETM :: class queues */
-+ /* --------------------- */
-+
-+/**
-+ * qman_ceetm_cq_claim - Claims an individual class queue.
-+ * @cq: the returned class queue object, if successful.
-+ * @channel: the class queue channel.
-+ * @idx: is from 0 to 7 (representing CQ0 to CQ7).
-+ * @ccg: represents the class congestion group that this class queue should be
-+ * subscribed to, or NULL if no congestion group membership is desired.
-+ *
-+ * Returns zero for success, or -EINVAL if @idx is out of range 0 - 7 or
-+ * if this class queue has been claimed, or configure class queue command
-+ * returns error, or returns -ENOMEM if allocating CQ memory fails.
-+ */
-+int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq,
-+ struct qm_ceetm_channel *channel,
-+ unsigned int idx,
-+ struct qm_ceetm_ccg *ccg);
-+
-+/**
-+ * qman_ceetm_cq_claim_A - Claims a class queue group A.
-+ * @cq: the returned class queue object, if successful.
-+ * @channel: the class queue channel.
-+ * @idx: is from 8 to 15 if only group A exits, otherwise, it is from 8 to 11.
-+ * @ccg: represents the class congestion group that this class queue should be
-+ * subscribed to, or NULL if no congestion group membership is desired.
-+ *
-+ * Return zero for success, or -EINVAL if @idx is out the range or if
-+ * this class queue has been claimed or configure class queue command returns
-+ * error, or returns -ENOMEM if allocating CQ memory fails.
-+ */
-+int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq,
-+ struct qm_ceetm_channel *channel,
-+ unsigned int idx,
-+ struct qm_ceetm_ccg *ccg);
-+
-+/**
-+ * qman_ceetm_cq_claim_B - Claims a class queue group B.
-+ * @cq: the returned class queue object, if successful.
-+ * @channel: the class queue channel.
-+ * @idx: is from 0 to 3 (CQ12 to CQ15).
-+ * @ccg: represents the class congestion group that this class queue should be
-+ * subscribed to, or NULL if no congestion group membership is desired.
-+ *
-+ * Return zero for success, or -EINVAL if @idx is out the range or if
-+ * this class queue has been claimed or configure class queue command returns
-+ * error, or returns -ENOMEM if allocating CQ memory fails.
-+ */
-+int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq,
-+ struct qm_ceetm_channel *channel,
-+ unsigned int idx,
-+ struct qm_ceetm_ccg *ccg);
-+
-+/**
-+ * qman_ceetm_cq_release - Releases a previously claimed class queue.
-+ * @cq: The class queue to be released.
-+ *
-+ * Return zero for success, or -EBUSY if the dependent objects (eg. logical
-+ * FQIDs) have not been released.
-+ */
-+int qman_ceetm_cq_release(struct qm_ceetm_cq *cq);
-+
-+/**
-+ * qman_ceetm_set_queue_weight
-+ * qman_ceetm_get_queue_weight - Configure/query the weight of a grouped class
-+ * queue.
-+ * @cq: the given class queue.
-+ * @weight_code: the desired weight code to set for the given class queue for
-+ * "set" function or the queired weight code for "get" function.
-+ *
-+ * Grouped class queues have a default weight code of zero, which corresponds to
-+ * a scheduler weighting of 1. This function can be used to modify a grouped
-+ * class queue to another weight, (Use the helpers qman_ceetm_wbfs2ratio()
-+ * and qman_ceetm_ratio2wbfs() to convert between these 'weight_code' values
-+ * and the corresponding sharing weight.)
-+ *
-+ * Returns zero for success, or -EIO if the configure weight command returns
-+ * error for "set" function, or -EINVAL if the query command returns
-+ * error for "get" function.
-+ * See section "CEETM Weighted Scheduling among Grouped Classes" in Reference
-+ * Manual for weight and weight code.
-+ */
-+int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq,
-+ struct qm_ceetm_weight_code *weight_code);
-+int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq,
-+ struct qm_ceetm_weight_code *weight_code);
-+
-+/**
-+ * qman_ceetm_set_queue_weight_in_ratio
-+ * qman_ceetm_get_queue_weight_in_ratio - Configure/query the weight of a
-+ * grouped class queue.
-+ * @cq: the given class queue.
-+ * @ratio: the weight in ratio. It should be the real ratio number multiplied
-+ * by 100 to get rid of fraction.
-+ *
-+ * Returns zero for success, or -EIO if the configure weight command returns
-+ * error for "set" function, or -EINVAL if the query command returns
-+ * error for "get" function.
-+ */
-+int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio);
-+int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio);
-+
-+/* Weights are encoded using a pseudo-exponential scheme. The weight codes 0,
-+ * 32, 64, [...] correspond to weights of 1, 2, 4, [...]. The weights
-+ * corresponding to intermediate weight codes are calculated using linear
-+ * interpolation on the inverted values. Or put another way, the inverse weights
-+ * for each 32nd weight code are 1, 1/2, 1/4, [...], and so the intervals
-+ * between these are divided linearly into 32 intermediate values, the inverses
-+ * of which form the remaining weight codes.
-+ *
-+ * The Weighted Bandwidth Fair Scheduling (WBFS) algorithm provides a form of
-+ * scheduling within a group of class queues (group A or B). Weights are used to
-+ * normalise the class queues to an underlying BFS algorithm where all class
-+ * queues are assumed to require "equal bandwidth". So the weights referred to
-+ * by the weight codes act as divisors on the size of frames being enqueued. Ie.
-+ * one class queue in a group is assigned a weight of 2 whilst the other class
-+ * queues in the group keep the default weight of 1, then the WBFS scheduler
-+ * will effectively treat all frames enqueued on the weight-2 class queue as
-+ * having half the number of bytes they really have. Ie. if all other things are
-+ * equal, that class queue would get twice as much bytes-per-second bandwidth as
-+ * the others. So weights should be chosen to provide bandwidth ratios between
-+ * members of the same class queue group. These weights have no bearing on
-+ * behaviour outside that group's WBFS mechanism though.
-+ */
-+
-+/**
-+ * qman_ceetm_wbfs2ratio - Given a weight code ('wbfs'), an accurate fractional
-+ * representation of the corresponding weight is given (in order to not lose
-+ * any precision).
-+ * @weight_code: The given weight code in WBFS.
-+ * @numerator: the numerator part of the weight computed by the weight code.
-+ * @denominator: the denominator part of the weight computed by the weight code
-+ *
-+ * Returns zero for success or -EINVAL if the given weight code is illegal.
-+ */
-+int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code,
-+ u32 *numerator,
-+ u32 *denominator);
-+/**
-+ * qman_ceetm_ratio2wbfs - Given a weight, find the nearest possible weight code
-+ * If the user needs to know how close this is, convert the resulting weight
-+ * code back to a weight and compare.
-+ * @numerator: numerator part of the given weight.
-+ * @denominator: denominator part of the given weight.
-+ * @weight_code: the weight code computed from the given weight.
-+ *
-+ * Returns zero for success, or -ERANGE if "numerator/denominator" is outside
-+ * the range of weights.
-+ */
-+int qman_ceetm_ratio2wbfs(u32 numerator,
-+ u32 denominator,
-+ struct qm_ceetm_weight_code *weight_code,
-+ int rounding);
-+
-+#define QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER 0x1
-+/**
-+ * qman_ceetm_cq_get_dequeue_statistics - Get the statistics provided by CEETM
-+ * CQ counters.
-+ * @cq: the given CQ object.
-+ * @flags: indicates whether the statistics counter will be cleared after query.
-+ * @frame_count: The number of the frames that have been counted since the
-+ * counter was cleared last time.
-+ * @byte_count: the number of bytes in all frames that have been counted.
-+ *
-+ * Return zero for success or -EINVAL if query statistics command returns error.
-+ *
-+ */
-+int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags,
-+ u64 *frame_count, u64 *byte_count);
-+
-+/**
-+ * qman_ceetm_drain_cq - drain the CQ till it is empty.
-+ * @cq: the give CQ object.
-+ * Return 0 for success or -EINVAL for unsuccessful command to empty CQ.
-+ */
-+int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq);
-+
-+ /* ---------------------- */
-+ /* CEETM :: logical FQIDs */
-+ /* ---------------------- */
-+/**
-+ * qman_ceetm_lfq_claim - Claims an unused logical FQID, associates it with
-+ * the given class queue.
-+ * @lfq: the returned lfq object, if successful.
-+ * @cq: the class queue which needs to claim a LFQID.
-+ *
-+ * Return zero for success, or -ENODEV if no LFQID is available or -ENOMEM if
-+ * allocating memory for lfq fails, or -EINVAL if configuring LFQMT fails.
-+ */
-+int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq,
-+ struct qm_ceetm_cq *cq);
-+
-+/**
-+ * qman_ceetm_lfq_release - Releases a previously claimed logical FQID.
-+ * @lfq: the lfq to be released.
-+ *
-+ * Return zero for success.
-+ */
-+int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq);
-+
-+/**
-+ * qman_ceetm_lfq_set_context
-+ * qman_ceetm_lfq_get_context - Set/get the context_a/context_b pair to the
-+ * "dequeue context table" associated with the logical FQID.
-+ * @lfq: the given logical FQ object.
-+ * @context_a: contextA of the dequeue context.
-+ * @context_b: contextB of the dequeue context.
-+ *
-+ * Returns zero for success, or -EINVAL if there is error to set/get the
-+ * context pair.
-+ */
-+int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq,
-+ u64 context_a,
-+ u32 context_b);
-+int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq,
-+ u64 *context_a,
-+ u32 *context_b);
-+
-+/**
-+ * qman_ceetm_create_fq - Initialise a FQ object for the LFQ.
-+ * @lfq: the given logic fq.
-+ * @fq: the fq object created for the given logic fq.
-+ *
-+ * The FQ object can be used in qman_enqueue() and qman_enqueue_orp() APIs to
-+ * target a logical FQID (and the class queue it is associated with).
-+ * Note that this FQ object can only be used for enqueues, and
-+ * in the case of qman_enqueue_orp() it can not be used as the 'orp' parameter,
-+ * only as 'fq'. This FQ object can not (and shouldn't) be destroyed, it is only
-+ * valid as long as the underlying 'lfq' remains claimed. It is the user's
-+ * responsibility to ensure that the underlying 'lfq' is not released until any
-+ * enqueues to this FQ object have completed. The only field the user needs to
-+ * fill in is fq->cb.ern, as that enqueue rejection handler is the callback that
-+ * could conceivably be called on this FQ object. This API can be called
-+ * multiple times to create multiple FQ objects referring to the same logical
-+ * FQID, and any enqueue rejections will respect the callback of the object that
-+ * issued the enqueue (and will identify the object via the parameter passed to
-+ * the callback too). There is no 'flags' parameter to this API as there is for
-+ * qman_create_fq() - the created FQ object behaves as though qman_create_fq()
-+ * had been called with the single flag QMAN_FQ_FLAG_NO_MODIFY.
-+ *
-+ * Returns 0 for success.
-+ */
-+int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq);
-+
-+ /* -------------------------------- */
-+ /* CEETM :: class congestion groups */
-+ /* -------------------------------- */
-+
-+/**
-+ * qman_ceetm_ccg_claim - Claims an unused CCG.
-+ * @ccg: the returned CCG object, if successful.
-+ * @channel: the given class queue channel
-+ * @cscn: the callback function of this CCG.
-+ * @cb_ctx: the corresponding context to be used used if state change
-+ * notifications are later enabled for this CCG.
-+ *
-+ * The congestion group is local to the given class queue channel, so only
-+ * class queues within the channel can be associated with that congestion group.
-+ * The association of class queues to congestion groups occurs when the class
-+ * queues are claimed, see qman_ceetm_cq_claim() and related functions.
-+ * Congestion groups are in a "zero" state when initially claimed, and they are
-+ * returned to that state when released.
-+ *
-+ * Return zero for success, or -EINVAL if no CCG in the channel is available.
-+ */
-+int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg,
-+ struct qm_ceetm_channel *channel,
-+ unsigned int idx,
-+ void (*cscn)(struct qm_ceetm_ccg *,
-+ void *cb_ctx,
-+ int congested),
-+ void *cb_ctx);
-+
-+/**
-+ * qman_ceetm_ccg_release - Releases a previously claimed CCG.
-+ * @ccg: the given ccg.
-+ *
-+ * Returns zero for success, or -EBUSY if the given ccg's dependent objects
-+ * (class queues that are associated with the CCG) have not been released.
-+ */
-+int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg);
-+
-+/* This struct is used to specify attributes for a CCG. The 'we_mask' field
-+ * controls which CCG attributes are to be updated, and the remainder specify
-+ * the values for those attributes. A CCG counts either frames or the bytes
-+ * within those frames, but not both ('mode'). A CCG can optionally cause
-+ * enqueues to be rejected, due to tail-drop or WRED, or both (they are
-+ * independent options, 'td_en' and 'wr_en_g,wr_en_y,wr_en_r'). Tail-drop can be
-+ * level-triggered due to a single threshold ('td_thres') or edge-triggered due
-+ * to a "congestion state", but not both ('td_mode'). Congestion state has
-+ * distinct entry and exit thresholds ('cs_thres_in' and 'cs_thres_out'), and
-+ * notifications can be sent to software the CCG goes in to and out of this
-+ * congested state ('cscn_en'). */
-+struct qm_ceetm_ccg_params {
-+ /* Boolean fields together in a single bitfield struct */
-+ struct {
-+ /* Whether to count bytes or frames. 1==frames */
-+ u8 mode:1;
-+ /* En/disable tail-drop. 1==enable */
-+ u8 td_en:1;
-+ /* Tail-drop on congestion-state or threshold. 1=threshold */
-+ u8 td_mode:1;
-+ /* Generate congestion state change notifications. 1==enable */
-+ u8 cscn_en:1;
-+ /* Enable WRED rejections (per colour). 1==enable */
-+ u8 wr_en_g:1;
-+ u8 wr_en_y:1;
-+ u8 wr_en_r:1;
-+ } __packed;
-+ /* Tail-drop threshold. See qm_cgr_thres_[gs]et64(). */
-+ struct qm_cgr_cs_thres td_thres;
-+ /* Congestion state thresholds, for entry and exit. */
-+ struct qm_cgr_cs_thres cs_thres_in;
-+ struct qm_cgr_cs_thres cs_thres_out;
-+ /* Overhead accounting length. Per-packet "tax", from -128 to +127 */
-+ signed char oal;
-+ /* Congestion state change notification for DCP portal, virtual CCGID*/
-+ /* WRED parameters. */
-+ struct qm_cgr_wr_parm wr_parm_g;
-+ struct qm_cgr_wr_parm wr_parm_y;
-+ struct qm_cgr_wr_parm wr_parm_r;
-+};
-+/* Bits used in 'we_mask' to qman_ceetm_ccg_set(), controls which attributes of
-+ * the CCGR are to be updated. */
-+#define QM_CCGR_WE_MODE 0x0001 /* mode (bytes/frames) */
-+#define QM_CCGR_WE_CS_THRES_IN 0x0002 /* congestion state entry threshold */
-+#define QM_CCGR_WE_TD_EN 0x0004 /* congestion state tail-drop enable */
-+#define QM_CCGR_WE_CSCN_TUPD 0x0008 /* CSCN target update */
-+#define QM_CCGR_WE_CSCN_EN 0x0010 /* congestion notification enable */
-+#define QM_CCGR_WE_WR_EN_R 0x0020 /* WRED enable - red */
-+#define QM_CCGR_WE_WR_EN_Y 0x0040 /* WRED enable - yellow */
-+#define QM_CCGR_WE_WR_EN_G 0x0080 /* WRED enable - green */
-+#define QM_CCGR_WE_WR_PARM_R 0x0100 /* WRED parameters - red */
-+#define QM_CCGR_WE_WR_PARM_Y 0x0200 /* WRED parameters - yellow */
-+#define QM_CCGR_WE_WR_PARM_G 0x0400 /* WRED parameters - green */
-+#define QM_CCGR_WE_OAL 0x0800 /* overhead accounting length */
-+#define QM_CCGR_WE_CS_THRES_OUT 0x1000 /* congestion state exit threshold */
-+#define QM_CCGR_WE_TD_THRES 0x2000 /* tail-drop threshold */
-+#define QM_CCGR_WE_TD_MODE 0x4000 /* tail-drop mode (state/threshold) */
-+#define QM_CCGR_WE_CDV 0x8000 /* cdv */
-+
-+/**
-+ * qman_ceetm_ccg_set
-+ * qman_ceetm_ccg_get - Configure/query a subset of CCG attributes.
-+ * @ccg: the given CCG object.
-+ * @we_mask: the write enable mask.
-+ * @params: the parameters setting for this ccg
-+ *
-+ * Return 0 for success, or -EIO if configure ccg command returns error for
-+ * "set" function, or -EINVAL if query ccg command returns error for "get"
-+ * function.
-+ */
-+int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg,
-+ u16 we_mask,
-+ const struct qm_ceetm_ccg_params *params);
-+int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg,
-+ struct qm_ceetm_ccg_params *params);
-+
-+/** qman_ceetm_cscn_swp_set - Add or remove a software portal from the target
-+ * mask.
-+ * qman_ceetm_cscn_swp_get - Query whether a given software portal index is
-+ * in the cscn target mask.
-+ * @ccg: the give CCG object.
-+ * @swp_idx: the index of the software portal.
-+ * @cscn_enabled: 1: Set the swp to be cscn target. 0: remove the swp from
-+ * the target mask.
-+ * @we_mask: the write enable mask.
-+ * @params: the parameters setting for this ccg
-+ *
-+ * Return 0 for success, or -EINVAL if command in set/get function fails.
-+ */
-+int qman_ceetm_cscn_swp_set(struct qm_ceetm_ccg *ccg,
-+ u16 swp_idx,
-+ unsigned int cscn_enabled,
-+ u16 we_mask,
-+ const struct qm_ceetm_ccg_params *params);
-+int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg,
-+ u16 swp_idx,
-+ unsigned int *cscn_enabled);
-+
-+/** qman_ceetm_cscn_dcp_set - Add or remove a direct connect portal from the\
-+ * target mask.
-+ * qman_ceetm_cscn_dcp_get - Query whether a given direct connect portal index
-+ * is in the cscn target mask.
-+ * @ccg: the give CCG object.
-+ * @dcp_idx: the index of the direct connect portal.
-+ * @vcgid: congestion state change notification for dcp portal, virtual CGID.
-+ * @cscn_enabled: 1: Set the dcp to be cscn target. 0: remove the dcp from
-+ * the target mask.
-+ * @we_mask: the write enable mask.
-+ * @params: the parameters setting for this ccg
-+ *
-+ * Return 0 for success, or -EINVAL if command in set/get function fails.
-+ */
-+int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg,
-+ u16 dcp_idx,
-+ u8 vcgid,
-+ unsigned int cscn_enabled,
-+ u16 we_mask,
-+ const struct qm_ceetm_ccg_params *params);
-+int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg,
-+ u16 dcp_idx,
-+ u8 *vcgid,
-+ unsigned int *cscn_enabled);
-+
-+/**
-+ * qman_ceetm_ccg_get_reject_statistics - Get the statistics provided by
-+ * CEETM CCG counters.
-+ * @ccg: the given CCG object.
-+ * @flags: indicates whether the statistics counter will be cleared after query.
-+ * @frame_count: The number of the frames that have been counted since the
-+ * counter was cleared last time.
-+ * @byte_count: the number of bytes in all frames that have been counted.
-+ *
-+ * Return zero for success or -EINVAL if query statistics command returns error.
-+ *
-+ */
-+int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags,
-+ u64 *frame_count, u64 *byte_count);
-+
-+/**
-+ * qman_ceetm_query_lfqmt - Query the logical frame queue mapping table
-+ * @lfqid: Logical Frame Queue ID
-+ * @lfqmt_query: Results of the query command
-+ *
-+ * Returns zero for success or -EIO if the query command returns error.
-+ *
-+ */
-+int qman_ceetm_query_lfqmt(int lfqid,
-+ struct qm_mcr_ceetm_lfqmt_query *lfqmt_query);
-+
-+/**
-+ * qman_ceetm_query_write_statistics - Query (and optionally write) statistics
-+ * @cid: Target ID (CQID or CCGRID)
-+ * @dcp_idx: CEETM portal ID
-+ * @command_type: One of the following:
-+ * 0 = Query dequeue statistics. CID carries the CQID to be queried.
-+ * 1 = Query and clear dequeue statistics. CID carries the CQID to be queried
-+ * 2 = Write dequeue statistics. CID carries the CQID to be written.
-+ * 3 = Query reject statistics. CID carries the CCGRID to be queried.
-+ * 4 = Query and clear reject statistics. CID carries the CCGRID to be queried
-+ * 5 = Write reject statistics. CID carries the CCGRID to be written
-+ * @frame_count: Frame count value to be written if this is a write command
-+ * @byte_count: Bytes count value to be written if this is a write command
-+ *
-+ * Returns zero for success or -EIO if the query command returns error.
-+ */
-+int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx,
-+ u16 command_type, u64 frame_count,
-+ u64 byte_count);
-+
-+/**
-+ * qman_set_wpm - Set waterfall power management
-+ *
-+ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm.
-+ *
-+ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not
-+ * accessible.
-+ */
-+int qman_set_wpm(int wpm_enable);
-+
-+/**
-+ * qman_get_wpm - Query the waterfall power management setting
-+ *
-+ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm.
-+ *
-+ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not
-+ * accessible.
-+ */
-+int qman_get_wpm(int *wpm_enable);
-+
-+/* The below qman_p_***() variants might be called in a migration situation
-+ * (e.g. cpu hotplug). They are used to continue accessing the portal that
-+ * execution was affine to prior to migration.
-+ * @qman_portal specifies which portal the APIs will use.
-+*/
-+const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal
-+ *p);
-+int qman_p_irqsource_add(struct qman_portal *p, u32 bits);
-+int qman_p_irqsource_remove(struct qman_portal *p, u32 bits);
-+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit);
-+u32 qman_p_poll_slow(struct qman_portal *p);
-+void qman_p_poll(struct qman_portal *p);
-+void qman_p_stop_dequeues(struct qman_portal *p);
-+void qman_p_start_dequeues(struct qman_portal *p);
-+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools);
-+void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools);
-+u32 qman_p_static_dequeue_get(struct qman_portal *p);
-+void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq,
-+ int park_request);
-+int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq,
-+ u32 flags __maybe_unused, u32 vdqcr);
-+int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_fd *fd, u32 flags);
-+int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_fd *fd, u32 flags,
-+ struct qman_fq *orp, u16 orp_seqnum);
-+int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq,
-+ const struct qm_fd *fd, u32 flags,
-+ qman_cb_precommit cb, void *cb_arg);
-+#ifdef __cplusplus
-+}
-+#endif
-+
-+#endif /* FSL_QMAN_H */
---- /dev/null
-+++ b/include/linux/fsl_usdpaa.h
-@@ -0,0 +1,372 @@
-+/* Copyright 2011-2012 Freescale Semiconductor, Inc.
-+ *
-+ * This file is licensed under the terms of the GNU General Public License
-+ * version 2. This program is licensed "as is" without any warranty of any
-+ * kind, whether express or implied.
-+ */
-+
-+#ifndef FSL_USDPAA_H
-+#define FSL_USDPAA_H
-+
-+#ifdef __cplusplus
-+extern "C" {
-+#endif
-+
-+#include <linux/uaccess.h>
-+#include <linux/ioctl.h>
-+#include <linux/fsl_qman.h> /* For "enum qm_channel" */
-+#include <linux/compat.h>
-+
-+#ifdef CONFIG_FSL_USDPAA
-+
-+/******************************/
-+/* Allocation of resource IDs */
-+/******************************/
-+
-+/* This enum is used to distinguish between the type of underlying object being
-+ * manipulated. */
-+enum usdpaa_id_type {
-+ usdpaa_id_fqid,
-+ usdpaa_id_bpid,
-+ usdpaa_id_qpool,
-+ usdpaa_id_cgrid,
-+ usdpaa_id_ceetm0_lfqid,
-+ usdpaa_id_ceetm0_channelid,
-+ usdpaa_id_ceetm1_lfqid,
-+ usdpaa_id_ceetm1_channelid,
-+ usdpaa_id_max /* <-- not a valid type, represents the number of types */
-+};
-+#define USDPAA_IOCTL_MAGIC 'u'
-+struct usdpaa_ioctl_id_alloc {
-+ uint32_t base; /* Return value, the start of the allocated range */
-+ enum usdpaa_id_type id_type; /* what kind of resource(s) to allocate */
-+ uint32_t num; /* how many IDs to allocate (and return value) */
-+ uint32_t align; /* must be a power of 2, 0 is treated like 1 */
-+ int partial; /* whether to allow less than 'num' */
-+};
-+struct usdpaa_ioctl_id_release {
-+ /* Input; */
-+ enum usdpaa_id_type id_type;
-+ uint32_t base;
-+ uint32_t num;
-+};
-+struct usdpaa_ioctl_id_reserve {
-+ enum usdpaa_id_type id_type;
-+ uint32_t base;
-+ uint32_t num;
-+};
-+
-+
-+/* ioctl() commands */
-+#define USDPAA_IOCTL_ID_ALLOC \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x01, struct usdpaa_ioctl_id_alloc)
-+#define USDPAA_IOCTL_ID_RELEASE \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x02, struct usdpaa_ioctl_id_release)
-+#define USDPAA_IOCTL_ID_RESERVE \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x0A, struct usdpaa_ioctl_id_reserve)
-+
-+/**********************/
-+/* Mapping DMA memory */
-+/**********************/
-+
-+/* Maximum length for a map name, including NULL-terminator */
-+#define USDPAA_DMA_NAME_MAX 16
-+/* Flags for requesting DMA maps. Maps are private+unnamed or sharable+named.
-+ * For a sharable and named map, specify _SHARED (whether creating one or
-+ * binding to an existing one). If _SHARED is specified and _CREATE is not, then
-+ * the mapping must already exist. If _SHARED and _CREATE are specified and the
-+ * mapping doesn't already exist, it will be created. If _SHARED and _CREATE are
-+ * specified and the mapping already exists, the mapping will fail unless _LAZY
-+ * is specified. When mapping to a pre-existing sharable map, the length must be
-+ * an exact match. Lengths must be a power-of-4 multiple of page size.
-+ *
-+ * Note that this does not actually map the memory to user-space, that is done
-+ * by a subsequent mmap() using the page offset returned from this ioctl(). The
-+ * ioctl() is what gives the process permission to do this, and a page-offset
-+ * with which to do so.
-+ */
-+#define USDPAA_DMA_FLAG_SHARE 0x01
-+#define USDPAA_DMA_FLAG_CREATE 0x02
-+#define USDPAA_DMA_FLAG_LAZY 0x04
-+#define USDPAA_DMA_FLAG_RDONLY 0x08
-+struct usdpaa_ioctl_dma_map {
-+ /* Output parameters - virtual and physical addresses */
-+ void *ptr;
-+ uint64_t phys_addr;
-+ /* Input parameter, the length of the region to be created (or if
-+ * mapping an existing region, this must match it). Must be a power-of-4
-+ * multiple of page size. */
-+ uint64_t len;
-+ /* Input parameter, the USDPAA_DMA_FLAG_* settings. */
-+ uint32_t flags;
-+ /* If _FLAG_SHARE is specified, the name of the region to be created (or
-+ * of the existing mapping to use). */
-+ char name[USDPAA_DMA_NAME_MAX];
-+ /* If this ioctl() creates the mapping, this is an input parameter
-+ * stating whether the region supports locking. If mapping an existing
-+ * region, this is a return value indicating the same thing. */
-+ int has_locking;
-+ /* In the case of a successful map with _CREATE and _LAZY, this return
-+ * value indicates whether we created the mapped region or whether it
-+ * already existed. */
-+ int did_create;
-+};
-+
-+#ifdef CONFIG_COMPAT
-+struct usdpaa_ioctl_dma_map_compat {
-+ /* Output parameters - virtual and physical addresses */
-+ compat_uptr_t ptr;
-+ uint64_t phys_addr;
-+ /* Input parameter, the length of the region to be created (or if
-+ * mapping an existing region, this must match it). Must be a power-of-4
-+ * multiple of page size. */
-+ uint64_t len;
-+ /* Input parameter, the USDPAA_DMA_FLAG_* settings. */
-+ uint32_t flags;
-+ /* If _FLAG_SHARE is specified, the name of the region to be created (or
-+ * of the existing mapping to use). */
-+ char name[USDPAA_DMA_NAME_MAX];
-+ /* If this ioctl() creates the mapping, this is an input parameter
-+ * stating whether the region supports locking. If mapping an existing
-+ * region, this is a return value indicating the same thing. */
-+ int has_locking;
-+ /* In the case of a successful map with _CREATE and _LAZY, this return
-+ * value indicates whether we created the mapped region or whether it
-+ * already existed. */
-+ int did_create;
-+};
-+
-+#define USDPAA_IOCTL_DMA_MAP_COMPAT \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map_compat)
-+#endif
-+
-+
-+#define USDPAA_IOCTL_DMA_MAP \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map)
-+/* munmap() does not remove the DMA map, just the user-space mapping to it.
-+ * This ioctl will do both (though you can munmap() before calling the ioctl
-+ * too). */
-+#define USDPAA_IOCTL_DMA_UNMAP \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x04, unsigned char)
-+/* We implement a cross-process locking scheme per DMA map. Call this ioctl()
-+ * with a mmap()'d address, and the process will (interruptible) sleep if the
-+ * lock is already held by another process. Process destruction will
-+ * automatically clean up any held locks. */
-+#define USDPAA_IOCTL_DMA_LOCK \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x05, unsigned char)
-+#define USDPAA_IOCTL_DMA_UNLOCK \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x06, unsigned char)
-+
-+/***************************************/
-+/* Mapping and using QMan/BMan portals */
-+/***************************************/
-+enum usdpaa_portal_type {
-+ usdpaa_portal_qman,
-+ usdpaa_portal_bman,
-+};
-+
-+#define QBMAN_ANY_PORTAL_IDX 0xffffffff
-+
-+struct usdpaa_ioctl_portal_map {
-+ /* Input parameter, is a qman or bman portal required. */
-+
-+ enum usdpaa_portal_type type;
-+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
-+ for don't care. The portal index will be populated by the
-+ driver when the ioctl() successfully completes */
-+ uint32_t index;
-+
-+ /* Return value if the map succeeds, this gives the mapped
-+ * cache-inhibited (cinh) and cache-enabled (cena) addresses. */
-+ struct usdpaa_portal_map {
-+ void *cinh;
-+ void *cena;
-+ } addr;
-+ /* Qman-specific return values */
-+ uint16_t channel;
-+ uint32_t pools;
-+};
-+
-+#ifdef CONFIG_COMPAT
-+struct compat_usdpaa_ioctl_portal_map {
-+ /* Input parameter, is a qman or bman portal required. */
-+ enum usdpaa_portal_type type;
-+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
-+ for don't care. The portal index will be populated by the
-+ driver when the ioctl() successfully completes */
-+ uint32_t index;
-+ /* Return value if the map succeeds, this gives the mapped
-+ * cache-inhibited (cinh) and cache-enabled (cena) addresses. */
-+ struct usdpaa_portal_map_compat {
-+ compat_uptr_t cinh;
-+ compat_uptr_t cena;
-+ } addr;
-+ /* Qman-specific return values */
-+ uint16_t channel;
-+ uint32_t pools;
-+};
-+#define USDPAA_IOCTL_PORTAL_MAP_COMPAT \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct compat_usdpaa_ioctl_portal_map)
-+#define USDPAA_IOCTL_PORTAL_UNMAP_COMPAT \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map_compat)
-+#endif
-+
-+#define USDPAA_IOCTL_PORTAL_MAP \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct usdpaa_ioctl_portal_map)
-+#define USDPAA_IOCTL_PORTAL_UNMAP \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map)
-+
-+struct usdpaa_ioctl_irq_map {
-+ enum usdpaa_portal_type type; /* Type of portal to map */
-+ int fd; /* File descriptor that contains the portal */
-+ void *portal_cinh; /* Cache inhibited area to identify the portal */
-+};
-+
-+#define USDPAA_IOCTL_PORTAL_IRQ_MAP \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x09, struct usdpaa_ioctl_irq_map)
-+
-+#ifdef CONFIG_COMPAT
-+
-+struct compat_ioctl_irq_map {
-+ enum usdpaa_portal_type type; /* Type of portal to map */
-+ compat_int_t fd; /* File descriptor that contains the portal */
-+ compat_uptr_t portal_cinh; /* Used identify the portal */};
-+
-+#define USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT \
-+ _IOW(USDPAA_IOCTL_MAGIC, 0x09, struct compat_ioctl_irq_map)
-+#endif
-+
-+/* ioctl to query the amount of DMA memory used in the system */
-+struct usdpaa_ioctl_dma_used {
-+ uint64_t free_bytes;
-+ uint64_t total_bytes;
-+};
-+#define USDPAA_IOCTL_DMA_USED \
-+ _IOR(USDPAA_IOCTL_MAGIC, 0x0B, struct usdpaa_ioctl_dma_used)
-+
-+/* ioctl to allocate a raw portal */
-+struct usdpaa_ioctl_raw_portal {
-+ /* inputs */
-+ enum usdpaa_portal_type type; /* Type of portal to allocate */
-+
-+ /* set to non zero to turn on stashing */
-+ uint8_t enable_stash;
-+ /* Stashing attributes for the portal */
-+ uint32_t cpu;
-+ uint32_t cache;
-+ uint32_t window;
-+
-+ /* Specifies the stash request queue this portal should use */
-+ uint8_t sdest;
-+
-+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
-+ * for don't care. The portal index will be populated by the
-+ * driver when the ioctl() successfully completes */
-+ uint32_t index;
-+
-+ /* outputs */
-+ uint64_t cinh;
-+ uint64_t cena;
-+};
-+
-+#define USDPAA_IOCTL_ALLOC_RAW_PORTAL \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct usdpaa_ioctl_raw_portal)
-+
-+#define USDPAA_IOCTL_FREE_RAW_PORTAL \
-+ _IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct usdpaa_ioctl_raw_portal)
-+
-+#ifdef CONFIG_COMPAT
-+
-+struct compat_ioctl_raw_portal {
-+ /* inputs */
-+ enum usdpaa_portal_type type; /* Type of portal to allocate */
-+
-+ /* set to non zero to turn on stashing */
-+ uint8_t enable_stash;
-+ /* Stashing attributes for the portal */
-+ uint32_t cpu;
-+ uint32_t cache;
-+ uint32_t window;
-+ /* Specifies the stash request queue this portal should use */
-+ uint8_t sdest;
-+
-+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
-+ * for don't care. The portal index will be populated by the
-+ * driver when the ioctl() successfully completes */
-+ uint32_t index;
-+
-+ /* outputs */
-+ uint64_t cinh;
-+ uint64_t cena;
-+};
-+
-+#define USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT \
-+ _IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct compat_ioctl_raw_portal)
-+
-+#define USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT \
-+ _IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct compat_ioctl_raw_portal)
-+
-+#endif
-+
-+#ifdef __KERNEL__
-+
-+/* Early-boot hook */
-+int __init fsl_usdpaa_init_early(void);
-+
-+/* Fault-handling in arch/powerpc/mm/mem.c gives USDPAA an opportunity to detect
-+ * faults within its ranges via this hook. */
-+int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size);
-+
-+#endif /* __KERNEL__ */
-+
-+#endif /* CONFIG_FSL_USDPAA */
-+
-+#ifdef __KERNEL__
-+/* This interface is needed in a few places and though it's not specific to
-+ * USDPAA as such, creating a new header for it doesn't make any sense. The
-+ * qbman kernel driver implements this interface and uses it as the backend for
-+ * both the FQID and BPID allocators. The fsl_usdpaa driver also uses this
-+ * interface for tracking per-process allocations handed out to user-space. */
-+struct dpa_alloc {
-+ struct list_head free;
-+ spinlock_t lock;
-+ struct list_head used;
-+};
-+#define DECLARE_DPA_ALLOC(name) \
-+ struct dpa_alloc name = { \
-+ .free = { \
-+ .prev = &name.free, \
-+ .next = &name.free \
-+ }, \
-+ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
-+ .used = { \
-+ .prev = &name.used, \
-+ .next = &name.used \
-+ } \
-+ }
-+static inline void dpa_alloc_init(struct dpa_alloc *alloc)
-+{
-+ INIT_LIST_HEAD(&alloc->free);
-+ INIT_LIST_HEAD(&alloc->used);
-+ spin_lock_init(&alloc->lock);
-+}
-+int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align,
-+ int partial);
-+void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count);
-+void dpa_alloc_seed(struct dpa_alloc *alloc, u32 fqid, u32 count);
-+
-+/* Like 'new' but specifies the desired range, returns -ENOMEM if the entire
-+ * desired range is not available, or 0 for success. */
-+int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base_id, u32 count);
-+/* Pops and returns contiguous ranges from the allocator. Returns -ENOMEM when
-+ * 'alloc' is empty. */
-+int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count);
-+/* Returns 1 if the specified id is alloced, 0 otherwise */
-+int dpa_alloc_check(struct dpa_alloc *list, u32 id);
-+#endif /* __KERNEL__ */
-+
-+#ifdef __cplusplus
-+}
-+#endif
-+
-+#endif /* FSL_USDPAA_H */
projects / openwrt / staging / chunkeey.git / blobdiff