[platform/kernel/linux-rpi.git] / drivers / clk / bcm / clk-bcm2835.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright (C) 2010,2015 Broadcom
 * Copyright (C) 2012 Stephen Warren
 */

/**
 * DOC: BCM2835 CPRMAN (clock manager for the "audio" domain)
 *
 * The clock tree on the 2835 has several levels.  There's a root
 * oscillator running at 19.2Mhz.  After the oscillator there are 5
 * PLLs, roughly divided as "camera", "ARM", "core", "DSI displays",
 * and "HDMI displays".  Those 5 PLLs each can divide their output to
 * produce up to 4 channels.  Finally, there is the level of clocks to
 * be consumed by other hardware components (like "H264" or "HDMI
 * state machine"), which divide off of some subset of the PLL
 * channels.
 *
 * All of the clocks in the tree are exposed in the DT, because the DT
 * may want to make assignments of the final layer of clocks to the
 * PLL channels, and some components of the hardware will actually
 * skip layers of the tree (for example, the pixel clock comes
 * directly from the PLLH PIX channel without using a CM_*CTL clock
 * generator).
 */

#include <linux/clk-provider.h>
#include <linux/clkdev.h>
#include <linux/clk.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/math.h>
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <dt-bindings/clock/bcm2835.h>
#include <soc/bcm2835/raspberrypi-firmware.h>

#define CM_PASSWORD             0x5a000000

#define CM_GNRICCTL             0x000
#define CM_GNRICDIV             0x004
# define CM_DIV_FRAC_BITS       12
# define CM_DIV_FRAC_MASK       GENMASK(CM_DIV_FRAC_BITS - 1, 0)

#define CM_VPUCTL               0x008
#define CM_VPUDIV               0x00c
#define CM_SYSCTL               0x010
#define CM_SYSDIV               0x014
#define CM_PERIACTL             0x018
#define CM_PERIADIV             0x01c
#define CM_PERIICTL             0x020
#define CM_PERIIDIV             0x024
#define CM_H264CTL              0x028
#define CM_H264DIV              0x02c
#define CM_ISPCTL               0x030
#define CM_ISPDIV               0x034
#define CM_V3DCTL               0x038
#define CM_V3DDIV               0x03c
#define CM_CAM0CTL              0x040
#define CM_CAM0DIV              0x044
#define CM_CAM1CTL              0x048
#define CM_CAM1DIV              0x04c
#define CM_CCP2CTL              0x050
#define CM_CCP2DIV              0x054
#define CM_DSI0ECTL             0x058
#define CM_DSI0EDIV             0x05c
#define CM_DSI0PCTL             0x060
#define CM_DSI0PDIV             0x064
#define CM_DPICTL               0x068
#define CM_DPIDIV               0x06c
#define CM_GP0CTL               0x070
#define CM_GP0DIV               0x074
#define CM_GP1CTL               0x078
#define CM_GP1DIV               0x07c
#define CM_GP2CTL               0x080
#define CM_GP2DIV               0x084
#define CM_HSMCTL               0x088
#define CM_HSMDIV               0x08c
#define CM_OTPCTL               0x090
#define CM_OTPDIV               0x094
#define CM_PCMCTL               0x098
#define CM_PCMDIV               0x09c
#define CM_PWMCTL               0x0a0
#define CM_PWMDIV               0x0a4
#define CM_SLIMCTL              0x0a8
#define CM_SLIMDIV              0x0ac
#define CM_SMICTL               0x0b0
#define CM_SMIDIV               0x0b4
/* no definition for 0x0b8  and 0x0bc */
#define CM_TCNTCTL              0x0c0
# define CM_TCNT_SRC1_SHIFT             12
#define CM_TCNTCNT              0x0c4
#define CM_TECCTL               0x0c8
#define CM_TECDIV               0x0cc
#define CM_TD0CTL               0x0d0
#define CM_TD0DIV               0x0d4
#define CM_TD1CTL               0x0d8
#define CM_TD1DIV               0x0dc
#define CM_TSENSCTL             0x0e0
#define CM_TSENSDIV             0x0e4
#define CM_TIMERCTL             0x0e8
#define CM_TIMERDIV             0x0ec
#define CM_UARTCTL              0x0f0
#define CM_UARTDIV              0x0f4
#define CM_VECCTL               0x0f8
#define CM_VECDIV               0x0fc
#define CM_PULSECTL             0x190
#define CM_PULSEDIV             0x194
#define CM_SDCCTL               0x1a8
#define CM_SDCDIV               0x1ac
#define CM_ARMCTL               0x1b0
#define CM_AVEOCTL              0x1b8
#define CM_AVEODIV              0x1bc
#define CM_EMMCCTL              0x1c0
#define CM_EMMCDIV              0x1c4
#define CM_EMMC2CTL             0x1d0
#define CM_EMMC2DIV             0x1d4

/* General bits for the CM_*CTL regs */
# define CM_ENABLE                      BIT(4)
# define CM_KILL                        BIT(5)
# define CM_GATE_BIT                    6
# define CM_GATE                        BIT(CM_GATE_BIT)
# define CM_BUSY                        BIT(7)
# define CM_BUSYD                       BIT(8)
# define CM_FRAC                        BIT(9)
# define CM_SRC_SHIFT                   0
# define CM_SRC_BITS                    4
# define CM_SRC_MASK                    0xf
# define CM_SRC_GND                     0
# define CM_SRC_OSC                     1
# define CM_SRC_TESTDEBUG0              2
# define CM_SRC_TESTDEBUG1              3
# define CM_SRC_PLLA_CORE               4
# define CM_SRC_PLLA_PER                4
# define CM_SRC_PLLC_CORE0              5
# define CM_SRC_PLLC_PER                5
# define CM_SRC_PLLC_CORE1              8
# define CM_SRC_PLLD_CORE               6
# define CM_SRC_PLLD_PER                6
# define CM_SRC_PLLH_AUX                7
# define CM_SRC_PLLC_CORE1              8
# define CM_SRC_PLLC_CORE2              9

#define CM_OSCCOUNT             0x100

#define CM_PLLA                 0x104
# define CM_PLL_ANARST                  BIT(8)
# define CM_PLLA_HOLDPER                BIT(7)
# define CM_PLLA_LOADPER                BIT(6)
# define CM_PLLA_HOLDCORE               BIT(5)
# define CM_PLLA_LOADCORE               BIT(4)
# define CM_PLLA_HOLDCCP2               BIT(3)
# define CM_PLLA_LOADCCP2               BIT(2)
# define CM_PLLA_HOLDDSI0               BIT(1)
# define CM_PLLA_LOADDSI0               BIT(0)

#define CM_PLLC                 0x108
# define CM_PLLC_HOLDPER                BIT(7)
# define CM_PLLC_LOADPER                BIT(6)
# define CM_PLLC_HOLDCORE2              BIT(5)
# define CM_PLLC_LOADCORE2              BIT(4)
# define CM_PLLC_HOLDCORE1              BIT(3)
# define CM_PLLC_LOADCORE1              BIT(2)
# define CM_PLLC_HOLDCORE0              BIT(1)
# define CM_PLLC_LOADCORE0              BIT(0)

#define CM_PLLD                 0x10c
# define CM_PLLD_HOLDPER                BIT(7)
# define CM_PLLD_LOADPER                BIT(6)
# define CM_PLLD_HOLDCORE               BIT(5)
# define CM_PLLD_LOADCORE               BIT(4)
# define CM_PLLD_HOLDDSI1               BIT(3)
# define CM_PLLD_LOADDSI1               BIT(2)
# define CM_PLLD_HOLDDSI0               BIT(1)
# define CM_PLLD_LOADDSI0               BIT(0)

#define CM_PLLH                 0x110
# define CM_PLLH_LOADRCAL               BIT(2)
# define CM_PLLH_LOADAUX                BIT(1)
# define CM_PLLH_LOADPIX                BIT(0)

#define CM_LOCK                 0x114
# define CM_LOCK_FLOCKH                 BIT(12)
# define CM_LOCK_FLOCKD                 BIT(11)
# define CM_LOCK_FLOCKC                 BIT(10)
# define CM_LOCK_FLOCKB                 BIT(9)
# define CM_LOCK_FLOCKA                 BIT(8)

#define CM_EVENT                0x118
#define CM_DSI1ECTL             0x158
#define CM_DSI1EDIV             0x15c
#define CM_DSI1PCTL             0x160
#define CM_DSI1PDIV             0x164
#define CM_DFTCTL               0x168
#define CM_DFTDIV               0x16c

#define CM_PLLB                 0x170
# define CM_PLLB_HOLDARM                BIT(1)
# define CM_PLLB_LOADARM                BIT(0)

#define A2W_PLLA_CTRL           0x1100
#define A2W_PLLC_CTRL           0x1120
#define A2W_PLLD_CTRL           0x1140
#define A2W_PLLH_CTRL           0x1160
#define A2W_PLLB_CTRL           0x11e0
# define A2W_PLL_CTRL_PRST_DISABLE      BIT(17)
# define A2W_PLL_CTRL_PWRDN             BIT(16)
# define A2W_PLL_CTRL_PDIV_MASK         0x000007000
# define A2W_PLL_CTRL_PDIV_SHIFT        12
# define A2W_PLL_CTRL_NDIV_MASK         0x0000003ff
# define A2W_PLL_CTRL_NDIV_SHIFT        0

#define A2W_PLLA_ANA0           0x1010
#define A2W_PLLC_ANA0           0x1030
#define A2W_PLLD_ANA0           0x1050
#define A2W_PLLH_ANA0           0x1070
#define A2W_PLLB_ANA0           0x10f0

#define A2W_PLL_KA_SHIFT        7
#define A2W_PLL_KA_MASK         GENMASK(9, 7)
#define A2W_PLL_KI_SHIFT        19
#define A2W_PLL_KI_MASK         GENMASK(21, 19)
#define A2W_PLL_KP_SHIFT        15
#define A2W_PLL_KP_MASK         GENMASK(18, 15)

#define A2W_PLLH_KA_SHIFT       19
#define A2W_PLLH_KA_MASK        GENMASK(21, 19)
#define A2W_PLLH_KI_LOW_SHIFT   22
#define A2W_PLLH_KI_LOW_MASK    GENMASK(23, 22)
#define A2W_PLLH_KI_HIGH_SHIFT  0
#define A2W_PLLH_KI_HIGH_MASK   GENMASK(0, 0)
#define A2W_PLLH_KP_SHIFT       1
#define A2W_PLLH_KP_MASK        GENMASK(4, 1)

#define A2W_XOSC_CTRL           0x1190
# define A2W_XOSC_CTRL_PLLB_ENABLE      BIT(7)
# define A2W_XOSC_CTRL_PLLA_ENABLE      BIT(6)
# define A2W_XOSC_CTRL_PLLD_ENABLE      BIT(5)
# define A2W_XOSC_CTRL_DDR_ENABLE       BIT(4)
# define A2W_XOSC_CTRL_CPR1_ENABLE      BIT(3)
# define A2W_XOSC_CTRL_USB_ENABLE       BIT(2)
# define A2W_XOSC_CTRL_HDMI_ENABLE      BIT(1)
# define A2W_XOSC_CTRL_PLLC_ENABLE      BIT(0)

#define A2W_PLLA_FRAC           0x1200
#define A2W_PLLC_FRAC           0x1220
#define A2W_PLLD_FRAC           0x1240
#define A2W_PLLH_FRAC           0x1260
#define A2W_PLLB_FRAC           0x12e0
# define A2W_PLL_FRAC_MASK              ((1 << A2W_PLL_FRAC_BITS) - 1)
# define A2W_PLL_FRAC_BITS              20

#define A2W_PLL_CHANNEL_DISABLE         BIT(8)
#define A2W_PLL_DIV_BITS                8
#define A2W_PLL_DIV_SHIFT               0

#define A2W_PLLA_DSI0           0x1300
#define A2W_PLLA_CORE           0x1400
#define A2W_PLLA_PER            0x1500
#define A2W_PLLA_CCP2           0x1600

#define A2W_PLLC_CORE2          0x1320
#define A2W_PLLC_CORE1          0x1420
#define A2W_PLLC_PER            0x1520
#define A2W_PLLC_CORE0          0x1620

#define A2W_PLLD_DSI0           0x1340
#define A2W_PLLD_CORE           0x1440
#define A2W_PLLD_PER            0x1540
#define A2W_PLLD_DSI1           0x1640

#define A2W_PLLH_AUX            0x1360
#define A2W_PLLH_RCAL           0x1460
#define A2W_PLLH_PIX            0x1560
#define A2W_PLLH_STS            0x1660

#define A2W_PLLH_CTRLR          0x1960
#define A2W_PLLH_FRACR          0x1a60
#define A2W_PLLH_AUXR           0x1b60
#define A2W_PLLH_RCALR          0x1c60
#define A2W_PLLH_PIXR           0x1d60
#define A2W_PLLH_STSR           0x1e60

#define A2W_PLLB_ARM            0x13e0
#define A2W_PLLB_SP0            0x14e0
#define A2W_PLLB_SP1            0x15e0
#define A2W_PLLB_SP2            0x16e0

#define LOCK_TIMEOUT_NS         100000000
#define BCM2835_MAX_FB_RATE     1750000000u

#define SOC_BCM2835             BIT(0)
#define SOC_BCM2711             BIT(1)
#define SOC_ALL                 (SOC_BCM2835 | SOC_BCM2711)

#define VCMSG_ID_CORE_CLOCK     4

/*
 * Names of clocks used within the driver that need to be replaced
 * with an external parent's name.  This array is in the order that
 * the clocks node in the DT references external clocks.
 */
static const char *const cprman_parent_names[] = {
        "xosc",
        "dsi0_byte",
        "dsi0_ddr2",
        "dsi0_ddr",
        "dsi1_byte",
        "dsi1_ddr2",
        "dsi1_ddr",
};

struct bcm2835_cprman {
        struct device *dev;
        void __iomem *regs;
        struct rpi_firmware *fw;
        spinlock_t regs_lock; /* spinlock for all clocks */
        unsigned int soc;

        /*
         * Real names of cprman clock parents looked up through
         * of_clk_get_parent_name(), which will be used in the
         * parent_names[] arrays for clock registration.
         */
        const char *real_parent_names[ARRAY_SIZE(cprman_parent_names)];

        /* Must be last */
        struct clk_hw_onecell_data onecell;
};

struct cprman_plat_data {
        unsigned int soc;
};

static inline void cprman_write(struct bcm2835_cprman *cprman, u32 reg, u32 val)
{
        writel(CM_PASSWORD | val, cprman->regs + reg);
}

static inline u32 cprman_read(struct bcm2835_cprman *cprman, u32 reg)
{
        return readl(cprman->regs + reg);
}

/* Does a cycle of measuring a clock through the TCNT clock, which may
 * source from many other clocks in the system.
 */
static unsigned long bcm2835_measure_tcnt_mux(struct bcm2835_cprman *cprman,
                                              u32 tcnt_mux)
{
        u32 osccount = 19200; /* 1ms */
        u32 count;
        ktime_t timeout;

        spin_lock(&cprman->regs_lock);

        cprman_write(cprman, CM_TCNTCTL, CM_KILL);

        cprman_write(cprman, CM_TCNTCTL,
                     (tcnt_mux & CM_SRC_MASK) |
                     (tcnt_mux >> CM_SRC_BITS) << CM_TCNT_SRC1_SHIFT);

        cprman_write(cprman, CM_OSCCOUNT, osccount);

        /* do a kind delay at the start */
        mdelay(1);

        /* Finish off whatever is left of OSCCOUNT */
        timeout = ktime_add_ns(ktime_get(), LOCK_TIMEOUT_NS);
        while (cprman_read(cprman, CM_OSCCOUNT)) {
                if (ktime_after(ktime_get(), timeout)) {
                        dev_err(cprman->dev, "timeout waiting for OSCCOUNT\n");
                        count = 0;
                        goto out;
                }
                cpu_relax();
        }

        /* Wait for BUSY to clear. */
        timeout = ktime_add_ns(ktime_get(), LOCK_TIMEOUT_NS);
        while (cprman_read(cprman, CM_TCNTCTL) & CM_BUSY) {
                if (ktime_after(ktime_get(), timeout)) {
                        dev_err(cprman->dev, "timeout waiting for !BUSY\n");
                        count = 0;
                        goto out;
                }
                cpu_relax();
        }

        count = cprman_read(cprman, CM_TCNTCNT);

        cprman_write(cprman, CM_TCNTCTL, 0);

out:
        spin_unlock(&cprman->regs_lock);

        return count * 1000;
}

static void bcm2835_debugfs_regset(struct bcm2835_cprman *cprman, u32 base,
                                   const struct debugfs_reg32 *regs,
                                   size_t nregs, struct dentry *dentry)
{
        struct debugfs_regset32 *regset;

        regset = devm_kzalloc(cprman->dev, sizeof(*regset), GFP_KERNEL);
        if (!regset)
                return;

        regset->regs = regs;
        regset->nregs = nregs;
        regset->base = cprman->regs + base;

        debugfs_create_regset32("regdump", S_IRUGO, dentry, regset);
}

struct bcm2835_pll_data {
        const char *name;
        u32 cm_ctrl_reg;
        u32 a2w_ctrl_reg;
        u32 frac_reg;
        u32 ana_reg_base;
        u32 reference_enable_mask;
        /* Bit in CM_LOCK to indicate when the PLL has locked. */
        u32 lock_mask;
        u32 flags;

        const struct bcm2835_pll_ana_bits *ana;

        unsigned long min_rate;
        unsigned long max_rate;
        /*
         * Highest rate for the VCO before we have to use the
         * pre-divide-by-2.
         */
        unsigned long max_fb_rate;
};

struct bcm2835_pll_ana_bits {
        u32 mask0;
        u32 set0;
        u32 mask1;
        u32 set1;
        u32 mask3;
        u32 set3;
        u32 fb_prediv_mask;
};

static const struct bcm2835_pll_ana_bits bcm2835_ana_default = {
        .mask0 = 0,
        .set0 = 0,
        .mask1 = A2W_PLL_KI_MASK | A2W_PLL_KP_MASK,
        .set1 = (2 << A2W_PLL_KI_SHIFT) | (8 << A2W_PLL_KP_SHIFT),
        .mask3 = A2W_PLL_KA_MASK,
        .set3 = (2 << A2W_PLL_KA_SHIFT),
        .fb_prediv_mask = BIT(14),
};

static const struct bcm2835_pll_ana_bits bcm2835_ana_pllh = {
        .mask0 = A2W_PLLH_KA_MASK | A2W_PLLH_KI_LOW_MASK,
        .set0 = (2 << A2W_PLLH_KA_SHIFT) | (2 << A2W_PLLH_KI_LOW_SHIFT),
        .mask1 = A2W_PLLH_KI_HIGH_MASK | A2W_PLLH_KP_MASK,
        .set1 = (6 << A2W_PLLH_KP_SHIFT),
        .mask3 = 0,
        .set3 = 0,
        .fb_prediv_mask = BIT(11),
};

struct bcm2835_pll_divider_data {
        const char *name;
        const char *source_pll;

        u32 cm_reg;
        u32 a2w_reg;

        u32 load_mask;
        u32 hold_mask;
        u32 fixed_divider;
        u32 flags;
};

struct bcm2835_clock_data {
        const char *name;

        const char *const *parents;
        int num_mux_parents;

        /* Bitmap encoding which parents accept rate change propagation. */
        unsigned int set_rate_parent;

        u32 ctl_reg;
        u32 div_reg;

        /* Number of integer bits in the divider */
        u32 int_bits;
        /* Number of fractional bits in the divider */
        u32 frac_bits;

        u32 flags;

        bool is_vpu_clock;
        bool is_mash_clock;
        bool low_jitter;

        u32 tcnt_mux;

        bool round_up;
};

struct bcm2835_gate_data {
        const char *name;
        const char *parent;

        u32 ctl_reg;
};

struct bcm2835_pll {
        struct clk_hw hw;
        struct bcm2835_cprman *cprman;
        const struct bcm2835_pll_data *data;
};

static int bcm2835_pll_is_on(struct clk_hw *hw)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        struct bcm2835_cprman *cprman = pll->cprman;
        const struct bcm2835_pll_data *data = pll->data;

        return cprman_read(cprman, data->a2w_ctrl_reg) &
                A2W_PLL_CTRL_PRST_DISABLE;
}

static u32 bcm2835_pll_get_prediv_mask(struct bcm2835_cprman *cprman,
                                       const struct bcm2835_pll_data *data)
{
        /*
         * On BCM2711 there isn't a pre-divisor available in the PLL feedback
         * loop. Bits 13:14 of ANA1 (PLLA,PLLB,PLLC,PLLD) have been re-purposed
         * for to for VCO RANGE bits.
         */
        if (cprman->soc & SOC_BCM2711)
                return 0;

        return data->ana->fb_prediv_mask;
}

static void bcm2835_pll_choose_ndiv_and_fdiv(unsigned long rate,
                                             unsigned long parent_rate,
                                             u32 *ndiv, u32 *fdiv)
{
        u64 div;

        div = (u64)rate << A2W_PLL_FRAC_BITS;
        do_div(div, parent_rate);

        *ndiv = div >> A2W_PLL_FRAC_BITS;
        *fdiv = div & ((1 << A2W_PLL_FRAC_BITS) - 1);
}

static long bcm2835_pll_rate_from_divisors(unsigned long parent_rate,
                                           u32 ndiv, u32 fdiv, u32 pdiv)
{
        u64 rate;

        if (pdiv == 0)
                return 0;

        rate = (u64)parent_rate * ((ndiv << A2W_PLL_FRAC_BITS) + fdiv);
        do_div(rate, pdiv);
        return rate >> A2W_PLL_FRAC_BITS;
}

static long bcm2835_pll_round_rate(struct clk_hw *hw, unsigned long rate,
                                   unsigned long *parent_rate)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        const struct bcm2835_pll_data *data = pll->data;
        u32 ndiv, fdiv;

        rate = clamp(rate, data->min_rate, data->max_rate);

        bcm2835_pll_choose_ndiv_and_fdiv(rate, *parent_rate, &ndiv, &fdiv);

        return bcm2835_pll_rate_from_divisors(*parent_rate, ndiv, fdiv, 1);
}

static unsigned long bcm2835_pll_get_rate(struct clk_hw *hw,
                                          unsigned long parent_rate)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        struct bcm2835_cprman *cprman = pll->cprman;
        const struct bcm2835_pll_data *data = pll->data;
        u32 a2wctrl = cprman_read(cprman, data->a2w_ctrl_reg);
        u32 ndiv, pdiv, fdiv;
        bool using_prediv;

        if (parent_rate == 0)
                return 0;

        fdiv = cprman_read(cprman, data->frac_reg) & A2W_PLL_FRAC_MASK;
        ndiv = (a2wctrl & A2W_PLL_CTRL_NDIV_MASK) >> A2W_PLL_CTRL_NDIV_SHIFT;
        pdiv = (a2wctrl & A2W_PLL_CTRL_PDIV_MASK) >> A2W_PLL_CTRL_PDIV_SHIFT;
        using_prediv = cprman_read(cprman, data->ana_reg_base + 4) &
                       bcm2835_pll_get_prediv_mask(cprman, data);

        if (using_prediv) {
                ndiv *= 2;
                fdiv *= 2;
        }

        return bcm2835_pll_rate_from_divisors(parent_rate, ndiv, fdiv, pdiv);
}

static void bcm2835_pll_off(struct clk_hw *hw)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        struct bcm2835_cprman *cprman = pll->cprman;
        const struct bcm2835_pll_data *data = pll->data;

        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, data->cm_ctrl_reg, CM_PLL_ANARST);
        cprman_write(cprman, data->a2w_ctrl_reg,
                     cprman_read(cprman, data->a2w_ctrl_reg) |
                     A2W_PLL_CTRL_PWRDN);
        spin_unlock(&cprman->regs_lock);
}

static int bcm2835_pll_on(struct clk_hw *hw)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        struct bcm2835_cprman *cprman = pll->cprman;
        const struct bcm2835_pll_data *data = pll->data;
        ktime_t timeout;

        cprman_write(cprman, data->a2w_ctrl_reg,
                     cprman_read(cprman, data->a2w_ctrl_reg) &
                     ~A2W_PLL_CTRL_PWRDN);

        /* Take the PLL out of reset. */
        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, data->cm_ctrl_reg,
                     cprman_read(cprman, data->cm_ctrl_reg) & ~CM_PLL_ANARST);
        spin_unlock(&cprman->regs_lock);

        /* Wait for the PLL to lock. */
        if (strcmp(data->name, "pllh")) {
                timeout = ktime_add_ns(ktime_get(), LOCK_TIMEOUT_NS);
                while (!(cprman_read(cprman, CM_LOCK) & data->lock_mask)) {
                        if (ktime_after(ktime_get(), timeout)) {
                                dev_err(cprman->dev, "%s: couldn't lock PLL\n",
                                        clk_hw_get_name(hw));
                                return -ETIMEDOUT;
                        }

                        cpu_relax();
                }
        }

        cprman_write(cprman, data->a2w_ctrl_reg,
                     cprman_read(cprman, data->a2w_ctrl_reg) |
                     A2W_PLL_CTRL_PRST_DISABLE);

        return 0;
}

static void
bcm2835_pll_write_ana(struct bcm2835_cprman *cprman, u32 ana_reg_base, u32 *ana)
{
        int i;

        /*
         * ANA register setup is done as a series of writes to
         * ANA3-ANA0, in that order.  This lets us write all 4
         * registers as a single cycle of the serdes interface (taking
         * 100 xosc clocks), whereas if we were to update ana0, 1, and
         * 3 individually through their partial-write registers, each
         * would be their own serdes cycle.
         */
        for (i = 3; i >= 0; i--)
                cprman_write(cprman, ana_reg_base + i * 4, ana[i]);
}

static int bcm2835_pll_set_rate(struct clk_hw *hw,
                                unsigned long rate, unsigned long parent_rate)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        struct bcm2835_cprman *cprman = pll->cprman;
        const struct bcm2835_pll_data *data = pll->data;
        u32 prediv_mask = bcm2835_pll_get_prediv_mask(cprman, data);
        bool was_using_prediv, use_fb_prediv, do_ana_setup_first;
        u32 ndiv, fdiv, a2w_ctl;
        u32 ana[4];
        int i;

        if (rate > data->max_fb_rate) {
                use_fb_prediv = true;
                rate /= 2;
        } else {
                use_fb_prediv = false;
        }

        bcm2835_pll_choose_ndiv_and_fdiv(rate, parent_rate, &ndiv, &fdiv);

        for (i = 3; i >= 0; i--)
                ana[i] = cprman_read(cprman, data->ana_reg_base + i * 4);

        was_using_prediv = ana[1] & prediv_mask;

        ana[0] &= ~data->ana->mask0;
        ana[0] |= data->ana->set0;
        ana[1] &= ~data->ana->mask1;
        ana[1] |= data->ana->set1;
        ana[3] &= ~data->ana->mask3;
        ana[3] |= data->ana->set3;

        if (was_using_prediv && !use_fb_prediv) {
                ana[1] &= ~prediv_mask;
                do_ana_setup_first = true;
        } else if (!was_using_prediv && use_fb_prediv) {
                ana[1] |= prediv_mask;
                do_ana_setup_first = false;
        } else {
                do_ana_setup_first = true;
        }

        /* Unmask the reference clock from the oscillator. */
        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, A2W_XOSC_CTRL,
                     cprman_read(cprman, A2W_XOSC_CTRL) |
                     data->reference_enable_mask);
        spin_unlock(&cprman->regs_lock);

        if (do_ana_setup_first)
                bcm2835_pll_write_ana(cprman, data->ana_reg_base, ana);

        /* Set the PLL multiplier from the oscillator. */
        cprman_write(cprman, data->frac_reg, fdiv);

        a2w_ctl = cprman_read(cprman, data->a2w_ctrl_reg);
        a2w_ctl &= ~A2W_PLL_CTRL_NDIV_MASK;
        a2w_ctl |= ndiv << A2W_PLL_CTRL_NDIV_SHIFT;
        a2w_ctl &= ~A2W_PLL_CTRL_PDIV_MASK;
        a2w_ctl |= 1 << A2W_PLL_CTRL_PDIV_SHIFT;
        cprman_write(cprman, data->a2w_ctrl_reg, a2w_ctl);

        if (!do_ana_setup_first)
                bcm2835_pll_write_ana(cprman, data->ana_reg_base, ana);

        return 0;
}

static void bcm2835_pll_debug_init(struct clk_hw *hw,
                                  struct dentry *dentry)
{
        struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw);
        struct bcm2835_cprman *cprman = pll->cprman;
        const struct bcm2835_pll_data *data = pll->data;
        struct debugfs_reg32 *regs;

        regs = devm_kcalloc(cprman->dev, 7, sizeof(*regs), GFP_KERNEL);
        if (!regs)
                return;

        regs[0].name = "cm_ctrl";
        regs[0].offset = data->cm_ctrl_reg;
        regs[1].name = "a2w_ctrl";
        regs[1].offset = data->a2w_ctrl_reg;
        regs[2].name = "frac";
        regs[2].offset = data->frac_reg;
        regs[3].name = "ana0";
        regs[3].offset = data->ana_reg_base + 0 * 4;
        regs[4].name = "ana1";
        regs[4].offset = data->ana_reg_base + 1 * 4;
        regs[5].name = "ana2";
        regs[5].offset = data->ana_reg_base + 2 * 4;
        regs[6].name = "ana3";
        regs[6].offset = data->ana_reg_base + 3 * 4;

        bcm2835_debugfs_regset(cprman, 0, regs, 7, dentry);
}

static const struct clk_ops bcm2835_pll_clk_ops = {
        .is_prepared = bcm2835_pll_is_on,
        .prepare = bcm2835_pll_on,
        .unprepare = bcm2835_pll_off,
        .recalc_rate = bcm2835_pll_get_rate,
        .set_rate = bcm2835_pll_set_rate,
        .round_rate = bcm2835_pll_round_rate,
        .debug_init = bcm2835_pll_debug_init,
};

struct bcm2835_pll_divider {
        struct clk_divider div;
        struct bcm2835_cprman *cprman;
        const struct bcm2835_pll_divider_data *data;
};

static struct bcm2835_pll_divider *
bcm2835_pll_divider_from_hw(struct clk_hw *hw)
{
        return container_of(hw, struct bcm2835_pll_divider, div.hw);
}

static int bcm2835_pll_divider_is_on(struct clk_hw *hw)
{
        struct bcm2835_pll_divider *divider = bcm2835_pll_divider_from_hw(hw);
        struct bcm2835_cprman *cprman = divider->cprman;
        const struct bcm2835_pll_divider_data *data = divider->data;

        return !(cprman_read(cprman, data->a2w_reg) & A2W_PLL_CHANNEL_DISABLE);
}

static int bcm2835_pll_divider_determine_rate(struct clk_hw *hw,
                                              struct clk_rate_request *req)
{
        return clk_divider_ops.determine_rate(hw, req);
}

static unsigned long bcm2835_pll_divider_get_rate(struct clk_hw *hw,
                                                  unsigned long parent_rate)
{
        return clk_divider_ops.recalc_rate(hw, parent_rate);
}

static void bcm2835_pll_divider_off(struct clk_hw *hw)
{
        struct bcm2835_pll_divider *divider = bcm2835_pll_divider_from_hw(hw);
        struct bcm2835_cprman *cprman = divider->cprman;
        const struct bcm2835_pll_divider_data *data = divider->data;

        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, data->cm_reg,
                     (cprman_read(cprman, data->cm_reg) &
                      ~data->load_mask) | data->hold_mask);
        cprman_write(cprman, data->a2w_reg,
                     cprman_read(cprman, data->a2w_reg) |
                     A2W_PLL_CHANNEL_DISABLE);
        spin_unlock(&cprman->regs_lock);
}

static int bcm2835_pll_divider_on(struct clk_hw *hw)
{
        struct bcm2835_pll_divider *divider = bcm2835_pll_divider_from_hw(hw);
        struct bcm2835_cprman *cprman = divider->cprman;
        const struct bcm2835_pll_divider_data *data = divider->data;

        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, data->a2w_reg,
                     cprman_read(cprman, data->a2w_reg) &
                     ~A2W_PLL_CHANNEL_DISABLE);

        cprman_write(cprman, data->cm_reg,
                     cprman_read(cprman, data->cm_reg) & ~data->hold_mask);
        spin_unlock(&cprman->regs_lock);

        return 0;
}

static int bcm2835_pll_divider_set_rate(struct clk_hw *hw,
                                        unsigned long rate,
                                        unsigned long parent_rate)
{
        struct bcm2835_pll_divider *divider = bcm2835_pll_divider_from_hw(hw);
        struct bcm2835_cprman *cprman = divider->cprman;
        const struct bcm2835_pll_divider_data *data = divider->data;
        u32 cm, div, max_div = 1 << A2W_PLL_DIV_BITS;

        div = DIV_ROUND_UP_ULL(parent_rate, rate);

        div = min(div, max_div);
        if (div == max_div)
                div = 0;

        cprman_write(cprman, data->a2w_reg, div);
        cm = cprman_read(cprman, data->cm_reg);
        cprman_write(cprman, data->cm_reg, cm | data->load_mask);
        cprman_write(cprman, data->cm_reg, cm & ~data->load_mask);

        return 0;
}

static void bcm2835_pll_divider_debug_init(struct clk_hw *hw,
                                           struct dentry *dentry)
{
        struct bcm2835_pll_divider *divider = bcm2835_pll_divider_from_hw(hw);
        struct bcm2835_cprman *cprman = divider->cprman;
        const struct bcm2835_pll_divider_data *data = divider->data;
        struct debugfs_reg32 *regs;

        regs = devm_kcalloc(cprman->dev, 7, sizeof(*regs), GFP_KERNEL);
        if (!regs)
                return;

        regs[0].name = "cm";
        regs[0].offset = data->cm_reg;
        regs[1].name = "a2w";
        regs[1].offset = data->a2w_reg;

        bcm2835_debugfs_regset(cprman, 0, regs, 2, dentry);
}

static const struct clk_ops bcm2835_pll_divider_clk_ops = {
        .is_prepared = bcm2835_pll_divider_is_on,
        .prepare = bcm2835_pll_divider_on,
        .unprepare = bcm2835_pll_divider_off,
        .recalc_rate = bcm2835_pll_divider_get_rate,
        .set_rate = bcm2835_pll_divider_set_rate,
        .determine_rate = bcm2835_pll_divider_determine_rate,
        .debug_init = bcm2835_pll_divider_debug_init,
};

/*
 * The CM dividers do fixed-point division, so we can't use the
 * generic integer divider code like the PLL dividers do (and we can't
 * fake it by having some fixed shifts preceding it in the clock tree,
 * because we'd run out of bits in a 32-bit unsigned long).
 */
struct bcm2835_clock {
        struct clk_hw hw;
        struct bcm2835_cprman *cprman;
        const struct bcm2835_clock_data *data;
};

static struct bcm2835_clock *bcm2835_clock_from_hw(struct clk_hw *hw)
{
        return container_of(hw, struct bcm2835_clock, hw);
}

static int bcm2835_clock_is_on(struct clk_hw *hw)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;

        return (cprman_read(cprman, data->ctl_reg) & CM_ENABLE) != 0;
}

static u32 bcm2835_clock_choose_div(struct clk_hw *hw,
                                    unsigned long rate,
                                    unsigned long parent_rate)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        const struct bcm2835_clock_data *data = clock->data;
        u32 unused_frac_mask =
                GENMASK(CM_DIV_FRAC_BITS - data->frac_bits, 0) >> 1;
        u64 temp = (u64)parent_rate << CM_DIV_FRAC_BITS;
        u64 rem;
        u32 div, mindiv, maxdiv;

        rem = do_div(temp, rate);
        div = temp;
        div &= ~unused_frac_mask;

        /* different clamping limits apply for a mash clock */
        if (data->is_mash_clock) {
                /* clamp to min divider of 2 */
                mindiv = 2 << CM_DIV_FRAC_BITS;
                /* clamp to the highest possible integer divider */
                maxdiv = (BIT(data->int_bits) - 1) << CM_DIV_FRAC_BITS;
        } else {
                /* clamp to min divider of 1 */
                mindiv = 1 << CM_DIV_FRAC_BITS;
                /* clamp to the highest possible fractional divider */
                maxdiv = GENMASK(data->int_bits + CM_DIV_FRAC_BITS - 1,
                                 CM_DIV_FRAC_BITS - data->frac_bits);
        }

        /* apply the clamping  limits */
        div = max_t(u32, div, mindiv);
        div = min_t(u32, div, maxdiv);

        return div;
}

static unsigned long bcm2835_clock_rate_from_divisor(struct bcm2835_clock *clock,
                                                     unsigned long parent_rate,
                                                     u32 div)
{
        const struct bcm2835_clock_data *data = clock->data;
        u64 temp;

        if (data->int_bits == 0 && data->frac_bits == 0)
                return parent_rate;

        /*
         * The divisor is a 12.12 fixed point field, but only some of
         * the bits are populated in any given clock.
         */
        div >>= CM_DIV_FRAC_BITS - data->frac_bits;
        div &= (1 << (data->int_bits + data->frac_bits)) - 1;

        if (div == 0)
                return 0;

        temp = (u64)parent_rate << data->frac_bits;

        do_div(temp, div);

        return temp;
}

static unsigned long bcm2835_round_rate(unsigned long rate)
{
        unsigned long scaler;
        unsigned long limit;

        limit = rate / 100000;

        scaler = 1;
        while (scaler < limit)
                scaler *= 10;

        /*
         * If increasing a clock by less than 0.1% changes it
         * from ..999.. to ..000.., round up.
         */
        if ((rate + scaler - 1) / scaler % 1000 == 0)
                rate = roundup(rate, scaler);

        return rate;
}

static unsigned long bcm2835_clock_get_rate(struct clk_hw *hw,
                                            unsigned long parent_rate)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;
        unsigned long rate;
        u32 div;

        if (data->int_bits == 0 && data->frac_bits == 0)
                return parent_rate;

        div = cprman_read(cprman, data->div_reg);

        rate = bcm2835_clock_rate_from_divisor(clock, parent_rate, div);

        if (data->round_up)
                rate = bcm2835_round_rate(rate);

        return rate;
}

static unsigned long bcm2835_clock_get_rate_vpu(struct clk_hw *hw,
                                                unsigned long parent_rate)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;

        if (cprman->fw) {
                struct {
                        u32 id;
                        u32 val;
                } packet;

                packet.id = VCMSG_ID_CORE_CLOCK;
                packet.val = 0;

                if (!rpi_firmware_property(cprman->fw,
                                           RPI_FIRMWARE_GET_MAX_CLOCK_RATE,
                                           &packet, sizeof(packet)))
                        return packet.val;
        }

        return bcm2835_clock_get_rate(hw, parent_rate);
}

static void bcm2835_clock_wait_busy(struct bcm2835_clock *clock)
{
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;
        ktime_t timeout = ktime_add_ns(ktime_get(), LOCK_TIMEOUT_NS);

        while (cprman_read(cprman, data->ctl_reg) & CM_BUSY) {
                if (ktime_after(ktime_get(), timeout)) {
                        dev_err(cprman->dev, "%s: couldn't lock PLL\n",
                                clk_hw_get_name(&clock->hw));
                        return;
                }
                cpu_relax();
        }
}

static void bcm2835_clock_off(struct clk_hw *hw)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;

        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, data->ctl_reg,
                     cprman_read(cprman, data->ctl_reg) & ~CM_ENABLE);
        spin_unlock(&cprman->regs_lock);

        /* BUSY will remain high until the divider completes its cycle. */
        bcm2835_clock_wait_busy(clock);
}

static int bcm2835_clock_on(struct clk_hw *hw)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;

        spin_lock(&cprman->regs_lock);
        cprman_write(cprman, data->ctl_reg,
                     cprman_read(cprman, data->ctl_reg) |
                     CM_ENABLE |
                     CM_GATE);
        spin_unlock(&cprman->regs_lock);

        /* Debug code to measure the clock once it's turned on to see
         * if it's ticking at the rate we expect.
         */
        if (data->tcnt_mux && false) {
                dev_info(cprman->dev,
                         "clk %s: rate %ld, measure %ld\n",
                         data->name,
                         clk_hw_get_rate(hw),
                         bcm2835_measure_tcnt_mux(cprman, data->tcnt_mux));
        }

        return 0;
}

static int bcm2835_clock_set_rate_and_parent(struct clk_hw *hw,
                                             unsigned long rate,
                                             unsigned long parent_rate,
                                             u8 parent)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;
        u32 div = bcm2835_clock_choose_div(hw, rate, parent_rate);
        u32 ctl;

        spin_lock(&cprman->regs_lock);

        ctl = cprman_read(cprman, data->ctl_reg);

        /* If the clock is running, we have to pause clock generation while
         * updating the control and div regs.  This is glitchless (no clock
         * signals generated faster than the rate) but each reg access is two
         * OSC cycles so the clock will slow down for a moment.
         */
        if (ctl & CM_ENABLE) {
                cprman_write(cprman, data->ctl_reg, ctl & ~CM_ENABLE);
                bcm2835_clock_wait_busy(clock);
        }

        if (parent != 0xff) {
                ctl &= ~(CM_SRC_MASK << CM_SRC_SHIFT);
                ctl |= parent << CM_SRC_SHIFT;
        }

        ctl &= ~CM_FRAC;
        ctl |= (div & CM_DIV_FRAC_MASK) ? CM_FRAC : 0;
        cprman_write(cprman, data->ctl_reg, ctl);

        cprman_write(cprman, data->div_reg, div);

        spin_unlock(&cprman->regs_lock);

        return 0;
}

static int bcm2835_clock_set_rate(struct clk_hw *hw,
                                  unsigned long rate, unsigned long parent_rate)
{
        return bcm2835_clock_set_rate_and_parent(hw, rate, parent_rate, 0xff);
}

static bool
bcm2835_clk_is_pllc(struct clk_hw *hw)
{
        if (!hw)
                return false;

        return strncmp(clk_hw_get_name(hw), "pllc", 4) == 0;
}

static unsigned long bcm2835_clock_choose_div_and_prate(struct clk_hw *hw,
                                                        int parent_idx,
                                                        unsigned long rate,
                                                        u32 *div,
                                                        unsigned long *prate,
                                                        unsigned long *avgrate)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;
        unsigned long best_rate = 0;
        u32 curdiv, mindiv, maxdiv;
        struct clk_hw *parent;

        parent = clk_hw_get_parent_by_index(hw, parent_idx);

        if (!(BIT(parent_idx) & data->set_rate_parent)) {
                *prate = clk_hw_get_rate(parent);
                *div = bcm2835_clock_choose_div(hw, rate, *prate);

                *avgrate = bcm2835_clock_rate_from_divisor(clock, *prate, *div);

                if (data->low_jitter && (*div & CM_DIV_FRAC_MASK)) {
                        unsigned long high, low;
                        u32 int_div = *div & ~CM_DIV_FRAC_MASK;

                        high = bcm2835_clock_rate_from_divisor(clock, *prate,
                                                               int_div);
                        int_div += CM_DIV_FRAC_MASK + 1;
                        low = bcm2835_clock_rate_from_divisor(clock, *prate,
                                                              int_div);

                        /*
                         * Return a value which is the maximum deviation
                         * below the ideal rate, for use as a metric.
                         */
                        return *avgrate - max(*avgrate - low, high - *avgrate);
                }
                return *avgrate;
        }

        if (data->frac_bits)
                dev_warn(cprman->dev,
                        "frac bits are not used when propagating rate change");

        /* clamp to min divider of 2 if we're dealing with a mash clock */
        mindiv = data->is_mash_clock ? 2 : 1;
        maxdiv = BIT(data->int_bits) - 1;

        /* TODO: Be smart, and only test a subset of the available divisors. */
        for (curdiv = mindiv; curdiv <= maxdiv; curdiv++) {
                unsigned long tmp_rate;

                tmp_rate = clk_hw_round_rate(parent, rate * curdiv);
                tmp_rate /= curdiv;
                if (curdiv == mindiv ||
                    (tmp_rate > best_rate && tmp_rate <= rate))
                        best_rate = tmp_rate;

                if (best_rate == rate)
                        break;
        }

        *div = curdiv << CM_DIV_FRAC_BITS;
        *prate = curdiv * best_rate;
        *avgrate = best_rate;

        return best_rate;
}

static int bcm2835_clock_determine_rate(struct clk_hw *hw,
                                        struct clk_rate_request *req)
{
        struct clk_hw *parent, *best_parent = NULL;
        bool current_parent_is_pllc;
        unsigned long rate, best_rate = 0;
        unsigned long prate, best_prate = 0;
        unsigned long avgrate, best_avgrate = 0;
        size_t i;
        u32 div;

        current_parent_is_pllc = bcm2835_clk_is_pllc(clk_hw_get_parent(hw));

        /*
         * Select parent clock that results in the closest but lower rate
         */
        for (i = 0; i < clk_hw_get_num_parents(hw); ++i) {
                parent = clk_hw_get_parent_by_index(hw, i);
                if (!parent)
                        continue;

                /*
                 * Don't choose a PLLC-derived clock as our parent
                 * unless it had been manually set that way.  PLLC's
                 * frequency gets adjusted by the firmware due to
                 * over-temp or under-voltage conditions, without
                 * prior notification to our clock consumer.
                 */
                if (bcm2835_clk_is_pllc(parent) && !current_parent_is_pllc)
                        continue;

                rate = bcm2835_clock_choose_div_and_prate(hw, i, req->rate,
                                                          &div, &prate,
                                                          &avgrate);
                if (abs(req->rate - rate) < abs(req->rate - best_rate)) {
                        best_parent = parent;
                        best_prate = prate;
                        best_rate = rate;
                        best_avgrate = avgrate;
                }
        }

        if (!best_parent)
                return -EINVAL;

        req->best_parent_hw = best_parent;
        req->best_parent_rate = best_prate;

        req->rate = best_avgrate;

        return 0;
}

static int bcm2835_clock_set_parent(struct clk_hw *hw, u8 index)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;
        u8 src = (index << CM_SRC_SHIFT) & CM_SRC_MASK;

        cprman_write(cprman, data->ctl_reg, src);
        return 0;
}

static u8 bcm2835_clock_get_parent(struct clk_hw *hw)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;
        u32 src = cprman_read(cprman, data->ctl_reg);

        return (src & CM_SRC_MASK) >> CM_SRC_SHIFT;
}

static const struct debugfs_reg32 bcm2835_debugfs_clock_reg32[] = {
        {
                .name = "ctl",
                .offset = 0,
        },
        {
                .name = "div",
                .offset = 4,
        },
};

static void bcm2835_clock_debug_init(struct clk_hw *hw,
                                    struct dentry *dentry)
{
        struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
        struct bcm2835_cprman *cprman = clock->cprman;
        const struct bcm2835_clock_data *data = clock->data;

        bcm2835_debugfs_regset(cprman, data->ctl_reg,
                bcm2835_debugfs_clock_reg32,
                ARRAY_SIZE(bcm2835_debugfs_clock_reg32),
                dentry);
}

static const struct clk_ops bcm2835_clock_clk_ops = {
        .is_prepared = bcm2835_clock_is_on,
        .prepare = bcm2835_clock_on,
        .unprepare = bcm2835_clock_off,
        .recalc_rate = bcm2835_clock_get_rate,
        .set_rate = bcm2835_clock_set_rate,
        .set_rate_and_parent = bcm2835_clock_set_rate_and_parent,
        .determine_rate = bcm2835_clock_determine_rate,
        .set_parent = bcm2835_clock_set_parent,
        .get_parent = bcm2835_clock_get_parent,
        .debug_init = bcm2835_clock_debug_init,
};

static int bcm2835_vpu_clock_is_on(struct clk_hw *hw)
{
        return true;
}

/*
 * The VPU clock can never be disabled (it doesn't have an ENABLE
 * bit), so it gets its own set of clock ops.
 */
static const struct clk_ops bcm2835_vpu_clock_clk_ops = {
        .is_prepared = bcm2835_vpu_clock_is_on,
        .recalc_rate = bcm2835_clock_get_rate_vpu,
        .set_rate = bcm2835_clock_set_rate,
        .determine_rate = bcm2835_clock_determine_rate,
        .set_parent = bcm2835_clock_set_parent,
        .get_parent = bcm2835_clock_get_parent,
        .debug_init = bcm2835_clock_debug_init,
};

static bool bcm2835_clk_is_claimed(const char *name);

static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman,
                                           const void *data)
{
        const struct bcm2835_pll_data *pll_data = data;
        struct bcm2835_pll *pll;
        struct clk_init_data init;
        int ret;

        memset(&init, 0, sizeof(init));

        /* All of the PLLs derive from the external oscillator. */
        init.parent_names = &cprman->real_parent_names[0];
        init.num_parents = 1;
        init.name = pll_data->name;
        init.ops = &bcm2835_pll_clk_ops;
        init.flags = pll_data->flags | CLK_IGNORE_UNUSED;

        if (!bcm2835_clk_is_claimed(pll_data->name))
                init.flags |= CLK_IS_CRITICAL;

        pll = kzalloc(sizeof(*pll), GFP_KERNEL);
        if (!pll)
                return NULL;

        pll->cprman = cprman;
        pll->data = pll_data;
        pll->hw.init = &init;

        ret = devm_clk_hw_register(cprman->dev, &pll->hw);
        if (ret) {
                kfree(pll);
                return NULL;
        }
        return &pll->hw;
}

static struct clk_hw *
bcm2835_register_pll_divider(struct bcm2835_cprman *cprman,
                             const void *data)
{
        const struct bcm2835_pll_divider_data *divider_data = data;
        struct bcm2835_pll_divider *divider;
        struct clk_init_data init;
        const char *divider_name;
        int ret;

        if (divider_data->fixed_divider != 1) {
                divider_name = devm_kasprintf(cprman->dev, GFP_KERNEL,
                                              "%s_prediv", divider_data->name);
                if (!divider_name)
                        return NULL;
        } else {
                divider_name = divider_data->name;
        }

        memset(&init, 0, sizeof(init));

        init.parent_names = &divider_data->source_pll;
        init.num_parents = 1;
        init.name = divider_name;
        init.ops = &bcm2835_pll_divider_clk_ops;
        init.flags = divider_data->flags | CLK_IGNORE_UNUSED;

        divider = devm_kzalloc(cprman->dev, sizeof(*divider), GFP_KERNEL);
        if (!divider)
                return NULL;

        divider->div.reg = cprman->regs + divider_data->a2w_reg;
        divider->div.shift = A2W_PLL_DIV_SHIFT;
        divider->div.width = A2W_PLL_DIV_BITS;
        divider->div.flags = CLK_DIVIDER_MAX_AT_ZERO;
        divider->div.lock = &cprman->regs_lock;
        divider->div.hw.init = &init;
        divider->div.table = NULL;

        if (!(cprman_read(cprman, divider_data->cm_reg) & divider_data->hold_mask)) {
                if (!bcm2835_clk_is_claimed(divider_data->source_pll))
                        init.flags |= CLK_IS_CRITICAL;
                if (!bcm2835_clk_is_claimed(divider_data->name))
                        divider->div.flags |= CLK_IS_CRITICAL;
        }

        divider->cprman = cprman;
        divider->data = divider_data;

        ret = devm_clk_hw_register(cprman->dev, &divider->div.hw);
        if (ret)
                return ERR_PTR(ret);

        /*
         * PLLH's channels have a fixed divide by 10 afterwards, which
         * is what our consumers are actually using.
         */
        if (divider_data->fixed_divider != 1) {
                return clk_hw_register_fixed_factor(cprman->dev,
                                                    divider_data->name,
                                                    divider_name,
                                                    CLK_SET_RATE_PARENT,
                                                    1,
                                                    divider_data->fixed_divider);
        }

        return &divider->div.hw;
}

static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
                                             const void *data)
{
        const struct bcm2835_clock_data *clock_data = data;
        struct bcm2835_clock *clock;
        struct clk_init_data init;
        const char *parents[1 << CM_SRC_BITS];
        size_t i;
        int ret;

        /*
         * Replace our strings referencing parent clocks with the
         * actual clock-output-name of the parent.
         */
        for (i = 0; i < clock_data->num_mux_parents; i++) {
                parents[i] = clock_data->parents[i];

                ret = match_string(cprman_parent_names,
                                   ARRAY_SIZE(cprman_parent_names),
                                   parents[i]);
                if (ret >= 0)
                        parents[i] = cprman->real_parent_names[ret];
        }

        memset(&init, 0, sizeof(init));
        init.parent_names = parents;
        init.num_parents = clock_data->num_mux_parents;
        init.name = clock_data->name;
        init.flags = clock_data->flags | CLK_IGNORE_UNUSED;

        /*
         * Some GPIO clocks for ethernet/wifi PLLs are marked as
         * critical (since some platforms use them), but if the
         * firmware didn't have them turned on then they clearly
         * aren't actually critical.
         */
        if ((cprman_read(cprman, clock_data->ctl_reg) & CM_ENABLE) == 0)
                init.flags &= ~CLK_IS_CRITICAL;

        /*
         * Pass the CLK_SET_RATE_PARENT flag if we are allowed to propagate
         * rate changes on at least of the parents.
         */
        if (clock_data->set_rate_parent)
                init.flags |= CLK_SET_RATE_PARENT;

        if (clock_data->is_vpu_clock) {
                init.ops = &bcm2835_vpu_clock_clk_ops;
        } else {
                init.ops = &bcm2835_clock_clk_ops;

                /* If the clock wasn't actually enabled at boot, it's not
                 * critical.
                 */
                if (!(cprman_read(cprman, clock_data->ctl_reg) & CM_ENABLE))
                        init.flags &= ~CLK_IS_CRITICAL;
        }

        clock = devm_kzalloc(cprman->dev, sizeof(*clock), GFP_KERNEL);
        if (!clock)
                return NULL;

        clock->cprman = cprman;
        clock->data = clock_data;
        clock->hw.init = &init;

        ret = devm_clk_hw_register(cprman->dev, &clock->hw);
        if (ret)
                return ERR_PTR(ret);
        return &clock->hw;
}

static struct clk_hw *bcm2835_register_gate(struct bcm2835_cprman *cprman,
                                            const void *data)
{
        const struct bcm2835_gate_data *gate_data = data;

        return clk_hw_register_gate(cprman->dev, gate_data->name,
                                    gate_data->parent,
                                    CLK_IGNORE_UNUSED | CLK_SET_RATE_GATE,
                                    cprman->regs + gate_data->ctl_reg,
                                    CM_GATE_BIT, 0, &cprman->regs_lock);
}

struct bcm2835_clk_desc {
        struct clk_hw *(*clk_register)(struct bcm2835_cprman *cprman,
                                       const void *data);
        unsigned int supported;
        const void *data;
};

/* assignment helper macros for different clock types */
#define _REGISTER(f, s, ...) { .clk_register = f, \
                               .supported = s,                          \
                               .data = __VA_ARGS__ }
#define REGISTER_PLL(s, ...)    _REGISTER(&bcm2835_register_pll,        \
                                          s,                            \
                                          &(struct bcm2835_pll_data)    \
                                          {__VA_ARGS__})
#define REGISTER_PLL_DIV(s, ...) _REGISTER(&bcm2835_register_pll_divider, \
                                           s,                             \
                                           &(struct bcm2835_pll_divider_data) \
                                           {__VA_ARGS__})
#define REGISTER_CLK(s, ...)    _REGISTER(&bcm2835_register_clock,      \
                                          s,                            \
                                          &(struct bcm2835_clock_data)  \
                                          {__VA_ARGS__})
#define REGISTER_GATE(s, ...)   _REGISTER(&bcm2835_register_gate,       \
                                          s,                            \
                                          &(struct bcm2835_gate_data)   \
                                          {__VA_ARGS__})

/* parent mux arrays plus helper macros */

/* main oscillator parent mux */
static const char *const bcm2835_clock_osc_parents[] = {
        "gnd",
        "xosc",
        "testdebug0",
        "testdebug1"
};

#define REGISTER_OSC_CLK(s, ...)        REGISTER_CLK(                   \
        s,                                                              \
        .num_mux_parents = ARRAY_SIZE(bcm2835_clock_osc_parents),       \
        .parents = bcm2835_clock_osc_parents,                           \
        __VA_ARGS__)

/* main peripherial parent mux */
static const char *const bcm2835_clock_per_parents[] = {
        "gnd",
        "xosc",
        "testdebug0",
        "testdebug1",
        "plla_per",
        "pllc_per",
        "plld_per",
        "pllh_aux",
};

#define REGISTER_PER_CLK(s, ...)        REGISTER_CLK(                   \
        s,                                                              \
        .num_mux_parents = ARRAY_SIZE(bcm2835_clock_per_parents),       \
        .parents = bcm2835_clock_per_parents,                           \
        __VA_ARGS__)

/*
 * Restrict clock sources for the PCM peripheral to the oscillator and
 * PLLD_PER because other source may have varying rates or be switched
 * off.
 *
 * Prevent other sources from being selected by replacing their names in
 * the list of potential parents with dummy entries (entry index is
 * significant).
 */
static const char *const bcm2835_pcm_per_parents[] = {
        "-",
        "xosc",
        "-",
        "-",
        "-",
        "-",
        "plld_per",
        "-",
};

#define REGISTER_PCM_CLK(s, ...)        REGISTER_CLK(                   \
        s,                                                              \
        .num_mux_parents = ARRAY_SIZE(bcm2835_pcm_per_parents),         \
        .parents = bcm2835_pcm_per_parents,                             \
        __VA_ARGS__)

/* main vpu parent mux */
static const char *const bcm2835_clock_vpu_parents[] = {
        "gnd",
        "xosc",
        "testdebug0",
        "testdebug1",
        "plla_core",
        "pllc_core0",
        "plld_core",
        "pllh_aux",
        "pllc_core1",
        "pllc_core2",
};

#define REGISTER_VPU_CLK(s, ...)        REGISTER_CLK(                   \
        s,                                                              \
        .num_mux_parents = ARRAY_SIZE(bcm2835_clock_vpu_parents),       \
        .parents = bcm2835_clock_vpu_parents,                           \
        __VA_ARGS__)

/*
 * DSI parent clocks.  The DSI byte/DDR/DDR2 clocks come from the DSI
 * analog PHY.  The _inv variants are generated internally to cprman,
 * but we don't use them so they aren't hooked up.
 */
static const char *const bcm2835_clock_dsi0_parents[] = {
        "gnd",
        "xosc",
        "testdebug0",
        "testdebug1",
        "dsi0_ddr",
        "dsi0_ddr_inv",
        "dsi0_ddr2",
        "dsi0_ddr2_inv",
        "dsi0_byte",
        "dsi0_byte_inv",
};

static const char *const bcm2835_clock_dsi1_parents[] = {
        "gnd",
        "xosc",
        "testdebug0",
        "testdebug1",
        "dsi1_ddr",
        "dsi1_ddr_inv",
        "dsi1_ddr2",
        "dsi1_ddr2_inv",
        "dsi1_byte",
        "dsi1_byte_inv",
};

#define REGISTER_DSI0_CLK(s, ...)       REGISTER_CLK(                   \
        s,                                                              \
        .num_mux_parents = ARRAY_SIZE(bcm2835_clock_dsi0_parents),      \
        .parents = bcm2835_clock_dsi0_parents,                          \
        __VA_ARGS__)

#define REGISTER_DSI1_CLK(s, ...)       REGISTER_CLK(                   \
        s,                                                              \
        .num_mux_parents = ARRAY_SIZE(bcm2835_clock_dsi1_parents),      \
        .parents = bcm2835_clock_dsi1_parents,                          \
        __VA_ARGS__)

/*
 * the real definition of all the pll, pll_dividers and clocks
 * these make use of the above REGISTER_* macros
 */
static const struct bcm2835_clk_desc clk_desc_array[] = {
        /* the PLL + PLL dividers */

        /*
         * PLLA is the auxiliary PLL, used to drive the CCP2
         * (Compact Camera Port 2) transmitter clock.
         *
         * It is in the PX LDO power domain, which is on when the
         * AUDIO domain is on.
         */
        [BCM2835_PLLA]          = REGISTER_PLL(
                SOC_ALL,
                .name = "plla",
                .cm_ctrl_reg = CM_PLLA,
                .a2w_ctrl_reg = A2W_PLLA_CTRL,
                .frac_reg = A2W_PLLA_FRAC,
                .ana_reg_base = A2W_PLLA_ANA0,
                .reference_enable_mask = A2W_XOSC_CTRL_PLLA_ENABLE,
                .lock_mask = CM_LOCK_FLOCKA,

                .ana = &bcm2835_ana_default,

                .min_rate = 600000000u,
                .max_rate = 2400000000u,
                .max_fb_rate = BCM2835_MAX_FB_RATE),
        [BCM2835_PLLA_CORE]     = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plla_core",
                .source_pll = "plla",
                .cm_reg = CM_PLLA,
                .a2w_reg = A2W_PLLA_CORE,
                .load_mask = CM_PLLA_LOADCORE,
                .hold_mask = CM_PLLA_HOLDCORE,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),

        /*
         * PLLA_PER is used for gpu clocks. Controlled by firmware, see
         * clk-raspberrypi.c.
         */

        [BCM2835_PLLA_DSI0]     = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plla_dsi0",
                .source_pll = "plla",
                .cm_reg = CM_PLLA,
                .a2w_reg = A2W_PLLA_DSI0,
                .load_mask = CM_PLLA_LOADDSI0,
                .hold_mask = CM_PLLA_HOLDDSI0,
                .fixed_divider = 1),
        [BCM2835_PLLA_CCP2]     = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plla_ccp2",
                .source_pll = "plla",
                .cm_reg = CM_PLLA,
                .a2w_reg = A2W_PLLA_CCP2,
                .load_mask = CM_PLLA_LOADCCP2,
                .hold_mask = CM_PLLA_HOLDCCP2,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),

        /* PLLB is used for the ARM's clock. */
        [BCM2835_PLLB]          = REGISTER_PLL(
                SOC_ALL,
                .name = "pllb",
                .cm_ctrl_reg = CM_PLLB,
                .a2w_ctrl_reg = A2W_PLLB_CTRL,
                .frac_reg = A2W_PLLB_FRAC,
                .ana_reg_base = A2W_PLLB_ANA0,
                .reference_enable_mask = A2W_XOSC_CTRL_PLLB_ENABLE,
                .lock_mask = CM_LOCK_FLOCKB,

                .ana = &bcm2835_ana_default,

                .min_rate = 600000000u,
                .max_rate = 3000000000u,
                .max_fb_rate = BCM2835_MAX_FB_RATE,
                .flags = CLK_GET_RATE_NOCACHE),
        [BCM2835_PLLB_ARM]      = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "pllb_arm",
                .source_pll = "pllb",
                .cm_reg = CM_PLLB,
                .a2w_reg = A2W_PLLB_ARM,
                .load_mask = CM_PLLB_LOADARM,
                .hold_mask = CM_PLLB_HOLDARM,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE),

        /*
         * PLLC is the core PLL, used to drive the core VPU clock.
         *
         * It is in the PX LDO power domain, which is on when the
         * AUDIO domain is on.
         */
        [BCM2835_PLLC]          = REGISTER_PLL(
                SOC_ALL,
                .name = "pllc",
                .cm_ctrl_reg = CM_PLLC,
                .a2w_ctrl_reg = A2W_PLLC_CTRL,
                .frac_reg = A2W_PLLC_FRAC,
                .ana_reg_base = A2W_PLLC_ANA0,
                .reference_enable_mask = A2W_XOSC_CTRL_PLLC_ENABLE,
                .lock_mask = CM_LOCK_FLOCKC,

                .ana = &bcm2835_ana_default,

                .min_rate = 600000000u,
                .max_rate = 3000000000u,
                .max_fb_rate = BCM2835_MAX_FB_RATE),
        [BCM2835_PLLC_CORE0]    = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "pllc_core0",
                .source_pll = "pllc",
                .cm_reg = CM_PLLC,
                .a2w_reg = A2W_PLLC_CORE0,
                .load_mask = CM_PLLC_LOADCORE0,
                .hold_mask = CM_PLLC_HOLDCORE0,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),
        [BCM2835_PLLC_CORE1]    = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "pllc_core1",
                .source_pll = "pllc",
                .cm_reg = CM_PLLC,
                .a2w_reg = A2W_PLLC_CORE1,
                .load_mask = CM_PLLC_LOADCORE1,
                .hold_mask = CM_PLLC_HOLDCORE1,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),
        [BCM2835_PLLC_CORE2]    = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "pllc_core2",
                .source_pll = "pllc",
                .cm_reg = CM_PLLC,
                .a2w_reg = A2W_PLLC_CORE2,
                .load_mask = CM_PLLC_LOADCORE2,
                .hold_mask = CM_PLLC_HOLDCORE2,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),
        [BCM2835_PLLC_PER]      = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "pllc_per",
                .source_pll = "pllc",
                .cm_reg = CM_PLLC,
                .a2w_reg = A2W_PLLC_PER,
                .load_mask = CM_PLLC_LOADPER,
                .hold_mask = CM_PLLC_HOLDPER,
                .fixed_divider = 1,
                .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT),

        /*
         * PLLD is the display PLL, used to drive DSI display panels.
         *
         * It is in the PX LDO power domain, which is on when the
         * AUDIO domain is on.
         */
        [BCM2835_PLLD]          = REGISTER_PLL(
                SOC_ALL,
                .name = "plld",
                .cm_ctrl_reg = CM_PLLD,
                .a2w_ctrl_reg = A2W_PLLD_CTRL,
                .frac_reg = A2W_PLLD_FRAC,
                .ana_reg_base = A2W_PLLD_ANA0,
                .reference_enable_mask = A2W_XOSC_CTRL_DDR_ENABLE,
                .lock_mask = CM_LOCK_FLOCKD,

                .ana = &bcm2835_ana_default,

                .min_rate = 600000000u,
                .max_rate = 2400000000u,
                .max_fb_rate = BCM2835_MAX_FB_RATE),
        [BCM2835_PLLD_CORE]     = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plld_core",
                .source_pll = "plld",
                .cm_reg = CM_PLLD,
                .a2w_reg = A2W_PLLD_CORE,
                .load_mask = CM_PLLD_LOADCORE,
                .hold_mask = CM_PLLD_HOLDCORE,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),
        /*
         * VPU firmware assumes that PLLD_PER isn't disabled by the ARM core.
         * Otherwise this could cause firmware lookups. That's why we mark
         * it as critical.
         */
        [BCM2835_PLLD_PER]      = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plld_per",
                .source_pll = "plld",
                .cm_reg = CM_PLLD,
                .a2w_reg = A2W_PLLD_PER,
                .load_mask = CM_PLLD_LOADPER,
                .hold_mask = CM_PLLD_HOLDPER,
                .fixed_divider = 1,
                .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT),
        [BCM2835_PLLD_DSI0]     = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plld_dsi0",
                .source_pll = "plld",
                .cm_reg = CM_PLLD,
                .a2w_reg = A2W_PLLD_DSI0,
                .load_mask = CM_PLLD_LOADDSI0,
                .hold_mask = CM_PLLD_HOLDDSI0,
                .fixed_divider = 1),
        [BCM2835_PLLD_DSI1]     = REGISTER_PLL_DIV(
                SOC_ALL,
                .name = "plld_dsi1",
                .source_pll = "plld",
                .cm_reg = CM_PLLD,
                .a2w_reg = A2W_PLLD_DSI1,
                .load_mask = CM_PLLD_LOADDSI1,
                .hold_mask = CM_PLLD_HOLDDSI1,
                .fixed_divider = 1),

        /*
         * PLLH is used to supply the pixel clock or the AUX clock for the
         * TV encoder.
         *
         * It is in the HDMI power domain.
         */
        [BCM2835_PLLH]          = REGISTER_PLL(
                SOC_BCM2835,
                "pllh",
                .cm_ctrl_reg = CM_PLLH,
                .a2w_ctrl_reg = A2W_PLLH_CTRL,
                .frac_reg = A2W_PLLH_FRAC,
                .ana_reg_base = A2W_PLLH_ANA0,
                .reference_enable_mask = A2W_XOSC_CTRL_PLLC_ENABLE,
                .lock_mask = CM_LOCK_FLOCKH,

                .ana = &bcm2835_ana_pllh,

                .min_rate = 600000000u,
                .max_rate = 3000000000u,
                .max_fb_rate = BCM2835_MAX_FB_RATE),
        [BCM2835_PLLH_RCAL]     = REGISTER_PLL_DIV(
                SOC_BCM2835,
                .name = "pllh_rcal",
                .source_pll = "pllh",
                .cm_reg = CM_PLLH,
                .a2w_reg = A2W_PLLH_RCAL,
                .load_mask = CM_PLLH_LOADRCAL,
                .hold_mask = 0,
                .fixed_divider = 10,
                .flags = CLK_SET_RATE_PARENT),
        [BCM2835_PLLH_AUX]      = REGISTER_PLL_DIV(
                SOC_BCM2835,
                .name = "pllh_aux",
                .source_pll = "pllh",
                .cm_reg = CM_PLLH,
                .a2w_reg = A2W_PLLH_AUX,
                .load_mask = CM_PLLH_LOADAUX,
                .hold_mask = 0,
                .fixed_divider = 1,
                .flags = CLK_SET_RATE_PARENT),
        [BCM2835_PLLH_PIX]      = REGISTER_PLL_DIV(
                SOC_BCM2835,
                .name = "pllh_pix",
                .source_pll = "pllh",
                .cm_reg = CM_PLLH,
                .a2w_reg = A2W_PLLH_PIX,
                .load_mask = CM_PLLH_LOADPIX,
                .hold_mask = 0,
                .fixed_divider = 10,
                .flags = CLK_SET_RATE_PARENT),

        /* the clocks */

        /* clocks with oscillator parent mux */

        /* One Time Programmable Memory clock.  Maximum 10Mhz. */
        [BCM2835_CLOCK_OTP]     = REGISTER_OSC_CLK(
                SOC_ALL,
                .name = "otp",
                .ctl_reg = CM_OTPCTL,
                .div_reg = CM_OTPDIV,
                .int_bits = 4,
                .frac_bits = 0,
                .tcnt_mux = 6),
        /*
         * Used for a 1Mhz clock for the system clocksource, and also used
         * bythe watchdog timer and the camera pulse generator.
         */
        [BCM2835_CLOCK_TIMER]   = REGISTER_OSC_CLK(
                SOC_ALL,
                .name = "timer",
                .ctl_reg = CM_TIMERCTL,
                .div_reg = CM_TIMERDIV,
                .int_bits = 6,
                .frac_bits = 12),
        /*
         * Clock for the temperature sensor.
         * Generally run at 2Mhz, max 5Mhz.
         */
        [BCM2835_CLOCK_TSENS]   = REGISTER_OSC_CLK(
                SOC_ALL,
                .name = "tsens",
                .ctl_reg = CM_TSENSCTL,
                .div_reg = CM_TSENSDIV,
                .int_bits = 5,
                .frac_bits = 0),
        [BCM2835_CLOCK_TEC]     = REGISTER_OSC_CLK(
                SOC_ALL,
                .name = "tec",
                .ctl_reg = CM_TECCTL,
                .div_reg = CM_TECDIV,
                .int_bits = 6,
                .frac_bits = 0),

        /* clocks with vpu parent mux */
        [BCM2835_CLOCK_H264]    = REGISTER_VPU_CLK(
                SOC_ALL,
                .name = "h264",
                .ctl_reg = CM_H264CTL,
                .div_reg = CM_H264DIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 1),
        [BCM2835_CLOCK_ISP]     = REGISTER_VPU_CLK(
                SOC_ALL,
                .name = "isp",
                .ctl_reg = CM_ISPCTL,
                .div_reg = CM_ISPDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 2),

        /*
         * Secondary SDRAM clock.  Used for low-voltage modes when the PLL
         * in the SDRAM controller can't be used.
         */
        [BCM2835_CLOCK_SDRAM]   = REGISTER_VPU_CLK(
                SOC_ALL,
                .name = "sdram",
                .ctl_reg = CM_SDCCTL,
                .div_reg = CM_SDCDIV,
                .int_bits = 6,
                .frac_bits = 0,
                .tcnt_mux = 3),

        /*
         * CLOCK_V3D is used for v3d clock. Controlled by firmware, see
         * clk-raspberrypi.c.
         */

        /*
         * VPU clock.  This doesn't have an enable bit, since it drives
         * the bus for everything else, and is special so it doesn't need
         * to be gated for rate changes.  It is also known as "clk_audio"
         * in various hardware documentation.
         */
        [BCM2835_CLOCK_VPU]     = REGISTER_VPU_CLK(
                SOC_ALL,
                .name = "vpu",
                .ctl_reg = CM_VPUCTL,
                .div_reg = CM_VPUDIV,
                .int_bits = 12,
                .frac_bits = 8,
                .flags = CLK_IS_CRITICAL,
                .is_vpu_clock = true,
                .tcnt_mux = 5),

        /* clocks with per parent mux */
        [BCM2835_CLOCK_AVEO]    = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "aveo",
                .ctl_reg = CM_AVEOCTL,
                .div_reg = CM_AVEODIV,
                .int_bits = 4,
                .frac_bits = 0,
                .tcnt_mux = 38),
        [BCM2835_CLOCK_CAM0]    = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "cam0",
                .ctl_reg = CM_CAM0CTL,
                .div_reg = CM_CAM0DIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 14),
        [BCM2835_CLOCK_CAM1]    = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "cam1",
                .ctl_reg = CM_CAM1CTL,
                .div_reg = CM_CAM1DIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 15),
        [BCM2835_CLOCK_DFT]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "dft",
                .ctl_reg = CM_DFTCTL,
                .div_reg = CM_DFTDIV,
                .int_bits = 5,
                .frac_bits = 0),
        [BCM2835_CLOCK_DPI]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "dpi",
                .ctl_reg = CM_DPICTL,
                .div_reg = CM_DPIDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 17),

        /* Arasan EMMC clock */
        [BCM2835_CLOCK_EMMC]    = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "emmc",
                .ctl_reg = CM_EMMCCTL,
                .div_reg = CM_EMMCDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 39),

        /* EMMC2 clock (only available for BCM2711) */
        [BCM2711_CLOCK_EMMC2]   = REGISTER_PER_CLK(
                SOC_BCM2711,
                .name = "emmc2",
                .ctl_reg = CM_EMMC2CTL,
                .div_reg = CM_EMMC2DIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 42),

        /* General purpose (GPIO) clocks */
        [BCM2835_CLOCK_GP0]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "gp0",
                .ctl_reg = CM_GP0CTL,
                .div_reg = CM_GP0DIV,
                .int_bits = 12,
                .frac_bits = 12,
                .is_mash_clock = true,
                .tcnt_mux = 20),
        [BCM2835_CLOCK_GP1]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "gp1",
                .ctl_reg = CM_GP1CTL,
                .div_reg = CM_GP1DIV,
                .int_bits = 12,
                .frac_bits = 12,
                .flags = CLK_IS_CRITICAL,
                .is_mash_clock = true,
                .tcnt_mux = 21),
        [BCM2835_CLOCK_GP2]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "gp2",
                .ctl_reg = CM_GP2CTL,
                .div_reg = CM_GP2DIV,
                .int_bits = 12,
                .frac_bits = 12,
                .flags = CLK_IS_CRITICAL),

        /* HDMI state machine */
        [BCM2835_CLOCK_HSM]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "hsm",
                .ctl_reg = CM_HSMCTL,
                .div_reg = CM_HSMDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 22),
        [BCM2835_CLOCK_PCM]     = REGISTER_PCM_CLK(
                SOC_ALL,
                .name = "pcm",
                .ctl_reg = CM_PCMCTL,
                .div_reg = CM_PCMDIV,
                .int_bits = 12,
                .frac_bits = 12,
                .is_mash_clock = true,
                .low_jitter = true,
                .tcnt_mux = 23),
        [BCM2835_CLOCK_PWM]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "pwm",
                .ctl_reg = CM_PWMCTL,
                .div_reg = CM_PWMDIV,
                .int_bits = 12,
                .frac_bits = 12,
                .is_mash_clock = true,
                .tcnt_mux = 24),
        [BCM2835_CLOCK_SLIM]    = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "slim",
                .ctl_reg = CM_SLIMCTL,
                .div_reg = CM_SLIMDIV,
                .int_bits = 12,
                .frac_bits = 12,
                .is_mash_clock = true,
                .tcnt_mux = 25),
        [BCM2835_CLOCK_SMI]     = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "smi",
                .ctl_reg = CM_SMICTL,
                .div_reg = CM_SMIDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 27),
        [BCM2835_CLOCK_UART]    = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "uart",
                .ctl_reg = CM_UARTCTL,
                .div_reg = CM_UARTDIV,
                .int_bits = 10,
                .frac_bits = 12,
                .tcnt_mux = 28,
                .round_up = true),

        /* dsi clocks */
        [BCM2835_CLOCK_DSI0E]   = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "dsi0e",
                .ctl_reg = CM_DSI0ECTL,
                .div_reg = CM_DSI0EDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 18),
        [BCM2835_CLOCK_DSI1E]   = REGISTER_PER_CLK(
                SOC_ALL,
                .name = "dsi1e",
                .ctl_reg = CM_DSI1ECTL,
                .div_reg = CM_DSI1EDIV,
                .int_bits = 4,
                .frac_bits = 8,
                .tcnt_mux = 19),
        [BCM2835_CLOCK_DSI0P]   = REGISTER_DSI0_CLK(
                SOC_ALL,
                .name = "dsi0p",
                .ctl_reg = CM_DSI0PCTL,
                .div_reg = CM_DSI0PDIV,
                .int_bits = 0,
                .frac_bits = 0,
                .tcnt_mux = 12),
        [BCM2835_CLOCK_DSI1P]   = REGISTER_DSI1_CLK(
                SOC_ALL,
                .name = "dsi1p",
                .ctl_reg = CM_DSI1PCTL,
                .div_reg = CM_DSI1PDIV,
                .int_bits = 0,
                .frac_bits = 0,
                .tcnt_mux = 13),

        /* the gates */

        /*
         * CM_PERIICTL (and CM_PERIACTL, CM_SYSCTL and CM_VPUCTL if
         * you have the debug bit set in the power manager, which we
         * don't bother exposing) are individual gates off of the
         * non-stop vpu clock.
         */
        [BCM2835_CLOCK_PERI_IMAGE] = REGISTER_GATE(
                SOC_ALL,
                .name = "peri_image",
                .parent = "vpu",
                .ctl_reg = CM_PERIICTL),
};

static bool bcm2835_clk_claimed[ARRAY_SIZE(clk_desc_array)];

/*
 * Permanently take a reference on the parent of the SDRAM clock.
 *
 * While the SDRAM is being driven by its dedicated PLL most of the
 * time, there is a little loop running in the firmware that
 * periodically switches the SDRAM to using our CM clock to do PVT
 * recalibration, with the assumption that the previously configured
 * SDRAM parent is still enabled and running.
 */
static int bcm2835_mark_sdc_parent_critical(struct clk *sdc)
{
        struct clk *parent = clk_get_parent(sdc);

        if (IS_ERR(parent))
                return PTR_ERR(parent);

        return clk_prepare_enable(parent);
}

static bool bcm2835_clk_is_claimed(const char *name)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(clk_desc_array); i++) {
                if (clk_desc_array[i].data) {
                        const char *clk_name = *(const char **)(clk_desc_array[i].data);
                        if (!strcmp(name, clk_name))
                                return bcm2835_clk_claimed[i];
                }
        }

        return false;
}

static int bcm2835_clk_probe(struct platform_device *pdev)
{
        struct device *dev = &pdev->dev;
        struct clk_hw **hws;
        struct bcm2835_cprman *cprman;
        const struct bcm2835_clk_desc *desc;
        const size_t asize = ARRAY_SIZE(clk_desc_array);
        const struct cprman_plat_data *pdata;
        struct device_node *fw_node;
        size_t i;
        u32 clk_id;
        int ret;

        pdata = of_device_get_match_data(&pdev->dev);
        if (!pdata)
                return -ENODEV;

        cprman = devm_kzalloc(dev,
                              struct_size(cprman, onecell.hws, asize),
                              GFP_KERNEL);
        if (!cprman)
                return -ENOMEM;

        spin_lock_init(&cprman->regs_lock);
        cprman->dev = dev;
        cprman->regs = devm_platform_ioremap_resource(pdev, 0);
        if (IS_ERR(cprman->regs))
                return PTR_ERR(cprman->regs);

        fw_node = of_parse_phandle(dev->of_node, "firmware", 0);
        if (fw_node) {
                struct rpi_firmware *fw = rpi_firmware_get(fw_node);
                if (!fw)
                        return -EPROBE_DEFER;
                cprman->fw = fw;
        }

        memset(bcm2835_clk_claimed, 0, sizeof(bcm2835_clk_claimed));
        for (i = 0;
             !of_property_read_u32_index(pdev->dev.of_node, "claim-clocks",
                                         i, &clk_id);
             i++)
                bcm2835_clk_claimed[clk_id]= true;

        memcpy(cprman->real_parent_names, cprman_parent_names,
               sizeof(cprman_parent_names));
        of_clk_parent_fill(dev->of_node, cprman->real_parent_names,
                           ARRAY_SIZE(cprman_parent_names));

        /*
         * Make sure the external oscillator has been registered.
         *
         * The other (DSI) clocks are not present on older device
         * trees, which we still need to support for backwards
         * compatibility.
         */
        if (!cprman->real_parent_names[0])
                return -ENODEV;

        platform_set_drvdata(pdev, cprman);

        cprman->onecell.num = asize;
        cprman->soc = pdata->soc;
        hws = cprman->onecell.hws;

        for (i = 0; i < asize; i++) {
                desc = &clk_desc_array[i];
                if (desc->clk_register && desc->data &&
                    (desc->supported & pdata->soc)) {
                        hws[i] = desc->clk_register(cprman, desc->data);
                }
        }

        ret = bcm2835_mark_sdc_parent_critical(hws[BCM2835_CLOCK_SDRAM]->clk);
        if (ret)
                return ret;

        ret = of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
                                      &cprman->onecell);
        if (ret)
                return ret;

        /* note that we have registered all the clocks */
        dev_dbg(dev, "registered %zd clocks\n", asize);

        return 0;
}

static const struct cprman_plat_data cprman_bcm2835_plat_data = {
        .soc = SOC_BCM2835,
};

static const struct cprman_plat_data cprman_bcm2711_plat_data = {
        .soc = SOC_BCM2711,
};

static const struct of_device_id bcm2835_clk_of_match[] = {
        { .compatible = "brcm,bcm2835-cprman", .data = &cprman_bcm2835_plat_data },
        { .compatible = "brcm,bcm2711-cprman", .data = &cprman_bcm2711_plat_data },
        {}
};
MODULE_DEVICE_TABLE(of, bcm2835_clk_of_match);

static struct platform_driver bcm2835_clk_driver = {
        .driver = {
                .name = "bcm2835-clk",
                .of_match_table = bcm2835_clk_of_match,
        },
        .probe          = bcm2835_clk_probe,
};

static int __init __bcm2835_clk_driver_init(void)
{
        return platform_driver_register(&bcm2835_clk_driver);
}
#ifdef CONFIG_IMA
subsys_initcall(__bcm2835_clk_driver_init);
#else
postcore_initcall(__bcm2835_clk_driver_init);
#endif

MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
MODULE_DESCRIPTION("BCM2835 clock driver");
MODULE_LICENSE("GPL");