drivers/soc/rockchip/pm_domains.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Rockchip Generic power domain support.
   4  *
   5  * Copyright (c) 2015 ROCKCHIP, Co. Ltd.
   6  */
   7
   8 #include <linux/io.h>
   9 #include <linux/iopoll.h>
  10 #include <linux/err.h>
  11 #include <linux/mutex.h>
  12 #include <linux/pm_clock.h>
  13 #include <linux/pm_domain.h>
  14 #include <linux/of_address.h>
  15 #include <linux/of_clk.h>
  16 #include <linux/of_platform.h>
  17 #include <linux/clk.h>
  18 #include <linux/regmap.h>
  19 #include <linux/mfd/syscon.h>
  20 #include <soc/rockchip/pm_domains.h>
  21 #include <dt-bindings/power/px30-power.h>
  22 #include <dt-bindings/power/rockchip,rv1126-power.h>
  23 #include <dt-bindings/power/rk3036-power.h>
  24 #include <dt-bindings/power/rk3066-power.h>
  25 #include <dt-bindings/power/rk3128-power.h>
  26 #include <dt-bindings/power/rk3188-power.h>
  27 #include <dt-bindings/power/rk3228-power.h>
  28 #include <dt-bindings/power/rk3288-power.h>
  29 #include <dt-bindings/power/rk3328-power.h>
  30 #include <dt-bindings/power/rk3366-power.h>
  31 #include <dt-bindings/power/rk3368-power.h>
  32 #include <dt-bindings/power/rk3399-power.h>
  33 #include <dt-bindings/power/rk3568-power.h>
  34 #include <dt-bindings/power/rk3588-power.h>
  35
  36 struct rockchip_domain_info {
  37         const char *name;
  38         int pwr_mask;
  39         int status_mask;
  40         int req_mask;
  41         int idle_mask;
  42         int ack_mask;
  43         bool active_wakeup;
  44         int pwr_w_mask;
  45         int req_w_mask;
  46         int mem_status_mask;
  47         int repair_status_mask;
  48         u32 pwr_offset;
  49         u32 mem_offset;
  50         u32 req_offset;
  51 };
  52
  53 struct rockchip_pmu_info {
  54         u32 pwr_offset;
  55         u32 status_offset;
  56         u32 req_offset;
  57         u32 idle_offset;
  58         u32 ack_offset;
  59         u32 mem_pwr_offset;
  60         u32 chain_status_offset;
  61         u32 mem_status_offset;
  62         u32 repair_status_offset;
  63
  64         u32 core_pwrcnt_offset;
  65         u32 gpu_pwrcnt_offset;
  66
  67         unsigned int core_power_transition_time;
  68         unsigned int gpu_power_transition_time;
  69
  70         int num_domains;
  71         const struct rockchip_domain_info *domain_info;
  72 };
  73
  74 #define MAX_QOS_REGS_NUM        5
  75 #define QOS_PRIORITY            0x08
  76 #define QOS_MODE                0x0c
  77 #define QOS_BANDWIDTH           0x10
  78 #define QOS_SATURATION          0x14
  79 #define QOS_EXTCONTROL          0x18
  80
  81 struct rockchip_pm_domain {
  82         struct generic_pm_domain genpd;
  83         const struct rockchip_domain_info *info;
  84         struct rockchip_pmu *pmu;
  85         int num_qos;
  86         struct regmap **qos_regmap;
  87         u32 *qos_save_regs[MAX_QOS_REGS_NUM];
  88         int num_clks;
  89         struct clk_bulk_data *clks;
  90 };
  91
  92 struct rockchip_pmu {
  93         struct device *dev;
  94         struct regmap *regmap;
  95         const struct rockchip_pmu_info *info;
  96         struct mutex mutex; /* mutex lock for pmu */
  97         struct genpd_onecell_data genpd_data;
  98         struct generic_pm_domain *domains[];
  99 };
 100
 101 #define to_rockchip_pd(gpd) container_of(gpd, struct rockchip_pm_domain, genpd)
 102
 103 #define DOMAIN(_name, pwr, status, req, idle, ack, wakeup)      \
 104 {                                                       \
 105         .name = _name,                          \
 106         .pwr_mask = (pwr),                              \
 107         .status_mask = (status),                        \
 108         .req_mask = (req),                              \
 109         .idle_mask = (idle),                            \
 110         .ack_mask = (ack),                              \
 111         .active_wakeup = (wakeup),                      \
 112 }
 113
 114 #define DOMAIN_M(_name, pwr, status, req, idle, ack, wakeup)    \
 115 {                                                       \
 116         .name = _name,                          \
 117         .pwr_w_mask = (pwr) << 16,                      \
 118         .pwr_mask = (pwr),                              \
 119         .status_mask = (status),                        \
 120         .req_w_mask = (req) << 16,                      \
 121         .req_mask = (req),                              \
 122         .idle_mask = (idle),                            \
 123         .ack_mask = (ack),                              \
 124         .active_wakeup = wakeup,                        \
 125 }
 126
 127 #define DOMAIN_M_O_R(_name, p_offset, pwr, status, m_offset, m_status, r_status, r_offset, req, idle, ack, wakeup)      \
 128 {                                                       \
 129         .name = _name,                                  \
 130         .pwr_offset = p_offset,                         \
 131         .pwr_w_mask = (pwr) << 16,                      \
 132         .pwr_mask = (pwr),                              \
 133         .status_mask = (status),                        \
 134         .mem_offset = m_offset,                         \
 135         .mem_status_mask = (m_status),                  \
 136         .repair_status_mask = (r_status),               \
 137         .req_offset = r_offset,                         \
 138         .req_w_mask = (req) << 16,                      \
 139         .req_mask = (req),                              \
 140         .idle_mask = (idle),                            \
 141         .ack_mask = (ack),                              \
 142         .active_wakeup = wakeup,                        \
 143 }
 144
 145 #define DOMAIN_RK3036(_name, req, ack, idle, wakeup)            \
 146 {                                                       \
 147         .name = _name,                          \
 148         .req_mask = (req),                              \
 149         .req_w_mask = (req) << 16,                      \
 150         .ack_mask = (ack),                              \
 151         .idle_mask = (idle),                            \
 152         .active_wakeup = wakeup,                        \
 153 }
 154
 155 #define DOMAIN_PX30(name, pwr, status, req, wakeup)             \
 156         DOMAIN_M(name, pwr, status, req, (req) << 16, req, wakeup)
 157
 158 #define DOMAIN_RV1126(name, pwr, req, idle, wakeup)             \
 159         DOMAIN_M(name, pwr, pwr, req, idle, idle, wakeup)
 160
 161 #define DOMAIN_RK3288(name, pwr, status, req, wakeup)           \
 162         DOMAIN(name, pwr, status, req, req, (req) << 16, wakeup)
 163
 164 #define DOMAIN_RK3328(name, pwr, status, req, wakeup)           \
 165         DOMAIN_M(name, pwr, pwr, req, (req) << 10, req, wakeup)
 166
 167 #define DOMAIN_RK3368(name, pwr, status, req, wakeup)           \
 168         DOMAIN(name, pwr, status, req, (req) << 16, req, wakeup)
 169
 170 #define DOMAIN_RK3399(name, pwr, status, req, wakeup)           \
 171         DOMAIN(name, pwr, status, req, req, req, wakeup)
 172
 173 #define DOMAIN_RK3568(name, pwr, req, wakeup)           \
 174         DOMAIN_M(name, pwr, pwr, req, req, req, wakeup)
 175
 176 /*
 177  * Dynamic Memory Controller may need to coordinate with us -- see
 178  * rockchip_pmu_block().
 179  *
 180  * dmc_pmu_mutex protects registration-time races, so DMC driver doesn't try to
 181  * block() while we're initializing the PMU.
 182  */
 183 static DEFINE_MUTEX(dmc_pmu_mutex);
 184 static struct rockchip_pmu *dmc_pmu;
 185
 186 /*
 187  * Block PMU transitions and make sure they don't interfere with ARM Trusted
 188  * Firmware operations. There are two conflicts, noted in the comments below.
 189  *
 190  * Caller must unblock PMU transitions via rockchip_pmu_unblock().
 191  */
 192 int rockchip_pmu_block(void)
 193 {
 194         struct rockchip_pmu *pmu;
 195         struct generic_pm_domain *genpd;
 196         struct rockchip_pm_domain *pd;
 197         int i, ret;
 198
 199         mutex_lock(&dmc_pmu_mutex);
 200
 201         /* No PMU (yet)? Then we just block rockchip_pmu_probe(). */
 202         if (!dmc_pmu)
 203                 return 0;
 204         pmu = dmc_pmu;
 205
 206         /*
 207          * mutex blocks all idle transitions: we can't touch the
 208          * PMU_BUS_IDLE_REQ (our ".idle_offset") register while ARM Trusted
 209          * Firmware might be using it.
 210          */
 211         mutex_lock(&pmu->mutex);
 212
 213         /*
 214          * Power domain clocks: Per Rockchip, we *must* keep certain clocks
 215          * enabled for the duration of power-domain transitions. Most
 216          * transitions are handled by this driver, but some cases (in
 217          * particular, DRAM DVFS / memory-controller idle) must be handled by
 218          * firmware. Firmware can handle most clock management via a special
 219          * "ungate" register (PMU_CRU_GATEDIS_CON0), but unfortunately, this
 220          * doesn't handle PLLs. We can assist this transition by doing the
 221          * clock management on behalf of firmware.
 222          */
 223         for (i = 0; i < pmu->genpd_data.num_domains; i++) {
 224                 genpd = pmu->genpd_data.domains[i];
 225                 if (genpd) {
 226                         pd = to_rockchip_pd(genpd);
 227                         ret = clk_bulk_enable(pd->num_clks, pd->clks);
 228                         if (ret < 0) {
 229                                 dev_err(pmu->dev,
 230                                         "failed to enable clks for domain '%s': %d\n",
 231                                         genpd->name, ret);
 232                                 goto err;
 233                         }
 234                 }
 235         }
 236
 237         return 0;
 238
 239 err:
 240         for (i = i - 1; i >= 0; i--) {
 241                 genpd = pmu->genpd_data.domains[i];
 242                 if (genpd) {
 243                         pd = to_rockchip_pd(genpd);
 244                         clk_bulk_disable(pd->num_clks, pd->clks);
 245                 }
 246         }
 247         mutex_unlock(&pmu->mutex);
 248         mutex_unlock(&dmc_pmu_mutex);
 249
 250         return ret;
 251 }
 252 EXPORT_SYMBOL_GPL(rockchip_pmu_block);
 253
 254 /* Unblock PMU transitions. */
 255 void rockchip_pmu_unblock(void)
 256 {
 257         struct rockchip_pmu *pmu;
 258         struct generic_pm_domain *genpd;
 259         struct rockchip_pm_domain *pd;
 260         int i;
 261
 262         if (dmc_pmu) {
 263                 pmu = dmc_pmu;
 264                 for (i = 0; i < pmu->genpd_data.num_domains; i++) {
 265                         genpd = pmu->genpd_data.domains[i];
 266                         if (genpd) {
 267                                 pd = to_rockchip_pd(genpd);
 268                                 clk_bulk_disable(pd->num_clks, pd->clks);
 269                         }
 270                 }
 271
 272                 mutex_unlock(&pmu->mutex);
 273         }
 274
 275         mutex_unlock(&dmc_pmu_mutex);
 276 }
 277 EXPORT_SYMBOL_GPL(rockchip_pmu_unblock);
 278
 279 #define DOMAIN_RK3588(name, p_offset, pwr, status, m_offset, m_status, r_status, r_offset, req, idle, wakeup)   \
 280         DOMAIN_M_O_R(name, p_offset, pwr, status, m_offset, m_status, r_status, r_offset, req, idle, idle, wakeup)
 281
 282 static bool rockchip_pmu_domain_is_idle(struct rockchip_pm_domain *pd)
 283 {
 284         struct rockchip_pmu *pmu = pd->pmu;
 285         const struct rockchip_domain_info *pd_info = pd->info;
 286         unsigned int val;
 287
 288         regmap_read(pmu->regmap, pmu->info->idle_offset, &val);
 289         return (val & pd_info->idle_mask) == pd_info->idle_mask;
 290 }
 291
 292 static unsigned int rockchip_pmu_read_ack(struct rockchip_pmu *pmu)
 293 {
 294         unsigned int val;
 295
 296         regmap_read(pmu->regmap, pmu->info->ack_offset, &val);
 297         return val;
 298 }
 299
 300 static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd,
 301                                          bool idle)
 302 {
 303         const struct rockchip_domain_info *pd_info = pd->info;
 304         struct generic_pm_domain *genpd = &pd->genpd;
 305         struct rockchip_pmu *pmu = pd->pmu;
 306         u32 pd_req_offset = pd_info->req_offset;
 307         unsigned int target_ack;
 308         unsigned int val;
 309         bool is_idle;
 310         int ret;
 311
 312         if (pd_info->req_mask == 0)
 313                 return 0;
 314         else if (pd_info->req_w_mask)
 315                 regmap_write(pmu->regmap, pmu->info->req_offset + pd_req_offset,
 316                              idle ? (pd_info->req_mask | pd_info->req_w_mask) :
 317                              pd_info->req_w_mask);
 318         else
 319                 regmap_update_bits(pmu->regmap, pmu->info->req_offset + pd_req_offset,
 320                                    pd_info->req_mask, idle ? -1U : 0);
 321
 322         wmb();
 323
 324         /* Wait util idle_ack = 1 */
 325         target_ack = idle ? pd_info->ack_mask : 0;
 326         ret = readx_poll_timeout_atomic(rockchip_pmu_read_ack, pmu, val,
 327                                         (val & pd_info->ack_mask) == target_ack,
 328                                         0, 10000);
 329         if (ret) {
 330                 dev_err(pmu->dev,
 331                         "failed to get ack on domain '%s', val=0x%x\n",
 332                         genpd->name, val);
 333                 return ret;
 334         }
 335
 336         ret = readx_poll_timeout_atomic(rockchip_pmu_domain_is_idle, pd,
 337                                         is_idle, is_idle == idle, 0, 10000);
 338         if (ret) {
 339                 dev_err(pmu->dev,
 340                         "failed to set idle on domain '%s', val=%d\n",
 341                         genpd->name, is_idle);
 342                 return ret;
 343         }
 344
 345         return 0;
 346 }
 347
 348 static int rockchip_pmu_save_qos(struct rockchip_pm_domain *pd)
 349 {
 350         int i;
 351
 352         for (i = 0; i < pd->num_qos; i++) {
 353                 regmap_read(pd->qos_regmap[i],
 354                             QOS_PRIORITY,
 355                             &pd->qos_save_regs[0][i]);
 356                 regmap_read(pd->qos_regmap[i],
 357                             QOS_MODE,
 358                             &pd->qos_save_regs[1][i]);
 359                 regmap_read(pd->qos_regmap[i],
 360                             QOS_BANDWIDTH,
 361                             &pd->qos_save_regs[2][i]);
 362                 regmap_read(pd->qos_regmap[i],
 363                             QOS_SATURATION,
 364                             &pd->qos_save_regs[3][i]);
 365                 regmap_read(pd->qos_regmap[i],
 366                             QOS_EXTCONTROL,
 367                             &pd->qos_save_regs[4][i]);
 368         }
 369         return 0;
 370 }
 371
 372 static int rockchip_pmu_restore_qos(struct rockchip_pm_domain *pd)
 373 {
 374         int i;
 375
 376         for (i = 0; i < pd->num_qos; i++) {
 377                 regmap_write(pd->qos_regmap[i],
 378                              QOS_PRIORITY,
 379                              pd->qos_save_regs[0][i]);
 380                 regmap_write(pd->qos_regmap[i],
 381                              QOS_MODE,
 382                              pd->qos_save_regs[1][i]);
 383                 regmap_write(pd->qos_regmap[i],
 384                              QOS_BANDWIDTH,
 385                              pd->qos_save_regs[2][i]);
 386                 regmap_write(pd->qos_regmap[i],
 387                              QOS_SATURATION,
 388                              pd->qos_save_regs[3][i]);
 389                 regmap_write(pd->qos_regmap[i],
 390                              QOS_EXTCONTROL,
 391                              pd->qos_save_regs[4][i]);
 392         }
 393
 394         return 0;
 395 }
 396
 397 static bool rockchip_pmu_domain_is_on(struct rockchip_pm_domain *pd)
 398 {
 399         struct rockchip_pmu *pmu = pd->pmu;
 400         unsigned int val;
 401
 402         if (pd->info->repair_status_mask) {
 403                 regmap_read(pmu->regmap, pmu->info->repair_status_offset, &val);
 404                 /* 1'b1: power on, 1'b0: power off */
 405                 return val & pd->info->repair_status_mask;
 406         }
 407
 408         /* check idle status for idle-only domains */
 409         if (pd->info->status_mask == 0)
 410                 return !rockchip_pmu_domain_is_idle(pd);
 411
 412         regmap_read(pmu->regmap, pmu->info->status_offset, &val);
 413
 414         /* 1'b0: power on, 1'b1: power off */
 415         return !(val & pd->info->status_mask);
 416 }
 417
 418 static bool rockchip_pmu_domain_is_mem_on(struct rockchip_pm_domain *pd)
 419 {
 420         struct rockchip_pmu *pmu = pd->pmu;
 421         unsigned int val;
 422
 423         regmap_read(pmu->regmap,
 424                     pmu->info->mem_status_offset + pd->info->mem_offset, &val);
 425
 426         /* 1'b0: power on, 1'b1: power off */
 427         return !(val & pd->info->mem_status_mask);
 428 }
 429
 430 static bool rockchip_pmu_domain_is_chain_on(struct rockchip_pm_domain *pd)
 431 {
 432         struct rockchip_pmu *pmu = pd->pmu;
 433         unsigned int val;
 434
 435         regmap_read(pmu->regmap,
 436                     pmu->info->chain_status_offset + pd->info->mem_offset, &val);
 437
 438         /* 1'b1: power on, 1'b0: power off */
 439         return val & pd->info->mem_status_mask;
 440 }
 441
 442 static int rockchip_pmu_domain_mem_reset(struct rockchip_pm_domain *pd)
 443 {
 444         struct rockchip_pmu *pmu = pd->pmu;
 445         struct generic_pm_domain *genpd = &pd->genpd;
 446         bool is_on;
 447         int ret = 0;
 448
 449         ret = readx_poll_timeout_atomic(rockchip_pmu_domain_is_chain_on, pd, is_on,
 450                                         is_on == true, 0, 10000);
 451         if (ret) {
 452                 dev_err(pmu->dev,
 453                         "failed to get chain status '%s', target_on=1, val=%d\n",
 454                         genpd->name, is_on);
 455                 goto error;
 456         }
 457
 458         udelay(20);
 459
 460         regmap_write(pmu->regmap, pmu->info->mem_pwr_offset + pd->info->pwr_offset,
 461                      (pd->info->pwr_mask | pd->info->pwr_w_mask));
 462         wmb();
 463
 464         ret = readx_poll_timeout_atomic(rockchip_pmu_domain_is_mem_on, pd, is_on,
 465                                         is_on == false, 0, 10000);
 466         if (ret) {
 467                 dev_err(pmu->dev,
 468                         "failed to get mem status '%s', target_on=0, val=%d\n",
 469                         genpd->name, is_on);
 470                 goto error;
 471         }
 472
 473         regmap_write(pmu->regmap, pmu->info->mem_pwr_offset + pd->info->pwr_offset,
 474                      pd->info->pwr_w_mask);
 475         wmb();
 476
 477         ret = readx_poll_timeout_atomic(rockchip_pmu_domain_is_mem_on, pd, is_on,
 478                                         is_on == true, 0, 10000);
 479         if (ret) {
 480                 dev_err(pmu->dev,
 481                         "failed to get mem status '%s', target_on=1, val=%d\n",
 482                         genpd->name, is_on);
 483         }
 484
 485 error:
 486         return ret;
 487 }
 488
 489 static void rockchip_do_pmu_set_power_domain(struct rockchip_pm_domain *pd,
 490                                              bool on)
 491 {
 492         struct rockchip_pmu *pmu = pd->pmu;
 493         struct generic_pm_domain *genpd = &pd->genpd;
 494         u32 pd_pwr_offset = pd->info->pwr_offset;
 495         bool is_on, is_mem_on = false;
 496
 497         if (pd->info->pwr_mask == 0)
 498                 return;
 499
 500         if (on && pd->info->mem_status_mask)
 501                 is_mem_on = rockchip_pmu_domain_is_mem_on(pd);
 502
 503         if (pd->info->pwr_w_mask)
 504                 regmap_write(pmu->regmap, pmu->info->pwr_offset + pd_pwr_offset,
 505                              on ? pd->info->pwr_w_mask :
 506                              (pd->info->pwr_mask | pd->info->pwr_w_mask));
 507         else
 508                 regmap_update_bits(pmu->regmap, pmu->info->pwr_offset + pd_pwr_offset,
 509                                    pd->info->pwr_mask, on ? 0 : -1U);
 510
 511         wmb();
 512
 513         if (is_mem_on && rockchip_pmu_domain_mem_reset(pd))
 514                 return;
 515
 516         if (readx_poll_timeout_atomic(rockchip_pmu_domain_is_on, pd, is_on,
 517                                       is_on == on, 0, 10000)) {
 518                 dev_err(pmu->dev,
 519                         "failed to set domain '%s', val=%d\n",
 520                         genpd->name, is_on);
 521                 return;
 522         }
 523 }
 524
 525 static int rockchip_pd_power(struct rockchip_pm_domain *pd, bool power_on)
 526 {
 527         struct rockchip_pmu *pmu = pd->pmu;
 528         int ret;
 529
 530         mutex_lock(&pmu->mutex);
 531
 532         if (rockchip_pmu_domain_is_on(pd) != power_on) {
 533                 ret = clk_bulk_enable(pd->num_clks, pd->clks);
 534                 if (ret < 0) {
 535                         dev_err(pmu->dev, "failed to enable clocks\n");
 536                         mutex_unlock(&pmu->mutex);
 537                         return ret;
 538                 }
 539
 540                 if (!power_on) {
 541                         rockchip_pmu_save_qos(pd);
 542
 543                         /* if powering down, idle request to NIU first */
 544                         rockchip_pmu_set_idle_request(pd, true);
 545                 }
 546
 547                 rockchip_do_pmu_set_power_domain(pd, power_on);
 548
 549                 if (power_on) {
 550                         /* if powering up, leave idle mode */
 551                         rockchip_pmu_set_idle_request(pd, false);
 552
 553                         rockchip_pmu_restore_qos(pd);
 554                 }
 555
 556                 clk_bulk_disable(pd->num_clks, pd->clks);
 557         }
 558
 559         mutex_unlock(&pmu->mutex);
 560         return 0;
 561 }
 562
 563 static int rockchip_pd_power_on(struct generic_pm_domain *domain)
 564 {
 565         struct rockchip_pm_domain *pd = to_rockchip_pd(domain);
 566
 567         return rockchip_pd_power(pd, true);
 568 }
 569
 570 static int rockchip_pd_power_off(struct generic_pm_domain *domain)
 571 {
 572         struct rockchip_pm_domain *pd = to_rockchip_pd(domain);
 573
 574         return rockchip_pd_power(pd, false);
 575 }
 576
 577 static int rockchip_pd_attach_dev(struct generic_pm_domain *genpd,
 578                                   struct device *dev)
 579 {
 580         struct clk *clk;
 581         int i;
 582         int error;
 583
 584         dev_dbg(dev, "attaching to power domain '%s'\n", genpd->name);
 585
 586         error = pm_clk_create(dev);
 587         if (error) {
 588                 dev_err(dev, "pm_clk_create failed %d\n", error);
 589                 return error;
 590         }
 591
 592         i = 0;
 593         while ((clk = of_clk_get(dev->of_node, i++)) && !IS_ERR(clk)) {
 594                 dev_dbg(dev, "adding clock '%pC' to list of PM clocks\n", clk);
 595                 error = pm_clk_add_clk(dev, clk);
 596                 if (error) {
 597                         dev_err(dev, "pm_clk_add_clk failed %d\n", error);
 598                         clk_put(clk);
 599                         pm_clk_destroy(dev);
 600                         return error;
 601                 }
 602         }
 603
 604         return 0;
 605 }
 606
 607 static void rockchip_pd_detach_dev(struct generic_pm_domain *genpd,
 608                                    struct device *dev)
 609 {
 610         dev_dbg(dev, "detaching from power domain '%s'\n", genpd->name);
 611
 612         pm_clk_destroy(dev);
 613 }
 614
 615 static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 616                                       struct device_node *node)
 617 {
 618         const struct rockchip_domain_info *pd_info;
 619         struct rockchip_pm_domain *pd;
 620         struct device_node *qos_node;
 621         int i, j;
 622         u32 id;
 623         int error;
 624
 625         error = of_property_read_u32(node, "reg", &id);
 626         if (error) {
 627                 dev_err(pmu->dev,
 628                         "%pOFn: failed to retrieve domain id (reg): %d\n",
 629                         node, error);
 630                 return -EINVAL;
 631         }
 632
 633         if (id >= pmu->info->num_domains) {
 634                 dev_err(pmu->dev, "%pOFn: invalid domain id %d\n",
 635                         node, id);
 636                 return -EINVAL;
 637         }
 638         /* RK3588 has domains with two parents (RKVDEC0/RKVDEC1) */
 639         if (pmu->genpd_data.domains[id])
 640                 return 0;
 641
 642         pd_info = &pmu->info->domain_info[id];
 643         if (!pd_info) {
 644                 dev_err(pmu->dev, "%pOFn: undefined domain id %d\n",
 645                         node, id);
 646                 return -EINVAL;
 647         }
 648
 649         pd = devm_kzalloc(pmu->dev, sizeof(*pd), GFP_KERNEL);
 650         if (!pd)
 651                 return -ENOMEM;
 652
 653         pd->info = pd_info;
 654         pd->pmu = pmu;
 655
 656         pd->num_clks = of_clk_get_parent_count(node);
 657         if (pd->num_clks > 0) {
 658                 pd->clks = devm_kcalloc(pmu->dev, pd->num_clks,
 659                                         sizeof(*pd->clks), GFP_KERNEL);
 660                 if (!pd->clks)
 661                         return -ENOMEM;
 662         } else {
 663                 dev_dbg(pmu->dev, "%pOFn: doesn't have clocks: %d\n",
 664                         node, pd->num_clks);
 665                 pd->num_clks = 0;
 666         }
 667
 668         for (i = 0; i < pd->num_clks; i++) {
 669                 pd->clks[i].clk = of_clk_get(node, i);
 670                 if (IS_ERR(pd->clks[i].clk)) {
 671                         error = PTR_ERR(pd->clks[i].clk);
 672                         dev_err(pmu->dev,
 673                                 "%pOFn: failed to get clk at index %d: %d\n",
 674                                 node, i, error);
 675                         return error;
 676                 }
 677         }
 678
 679         error = clk_bulk_prepare(pd->num_clks, pd->clks);
 680         if (error)
 681                 goto err_put_clocks;
 682
 683         pd->num_qos = of_count_phandle_with_args(node, "pm_qos",
 684                                                  NULL);
 685
 686         if (pd->num_qos > 0) {
 687                 pd->qos_regmap = devm_kcalloc(pmu->dev, pd->num_qos,
 688                                               sizeof(*pd->qos_regmap),
 689                                               GFP_KERNEL);
 690                 if (!pd->qos_regmap) {
 691                         error = -ENOMEM;
 692                         goto err_unprepare_clocks;
 693                 }
 694
 695                 for (j = 0; j < MAX_QOS_REGS_NUM; j++) {
 696                         pd->qos_save_regs[j] = devm_kcalloc(pmu->dev,
 697                                                             pd->num_qos,
 698                                                             sizeof(u32),
 699                                                             GFP_KERNEL);
 700                         if (!pd->qos_save_regs[j]) {
 701                                 error = -ENOMEM;
 702                                 goto err_unprepare_clocks;
 703                         }
 704                 }
 705
 706                 for (j = 0; j < pd->num_qos; j++) {
 707                         qos_node = of_parse_phandle(node, "pm_qos", j);
 708                         if (!qos_node) {
 709                                 error = -ENODEV;
 710                                 goto err_unprepare_clocks;
 711                         }
 712                         pd->qos_regmap[j] = syscon_node_to_regmap(qos_node);
 713                         if (IS_ERR(pd->qos_regmap[j])) {
 714                                 error = -ENODEV;
 715                                 of_node_put(qos_node);
 716                                 goto err_unprepare_clocks;
 717                         }
 718                         of_node_put(qos_node);
 719                 }
 720         }
 721
 722         if (pd->info->name)
 723                 pd->genpd.name = pd->info->name;
 724         else
 725                 pd->genpd.name = kbasename(node->full_name);
 726         pd->genpd.power_off = rockchip_pd_power_off;
 727         pd->genpd.power_on = rockchip_pd_power_on;
 728         pd->genpd.attach_dev = rockchip_pd_attach_dev;
 729         pd->genpd.detach_dev = rockchip_pd_detach_dev;
 730         pd->genpd.flags = GENPD_FLAG_PM_CLK;
 731         if (pd_info->active_wakeup)
 732                 pd->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP;
 733         pm_genpd_init(&pd->genpd, NULL,
 734                       !rockchip_pmu_domain_is_on(pd) ||
 735                       (pd->info->mem_status_mask && !rockchip_pmu_domain_is_mem_on(pd)));
 736
 737         pmu->genpd_data.domains[id] = &pd->genpd;
 738         return 0;
 739
 740 err_unprepare_clocks:
 741         clk_bulk_unprepare(pd->num_clks, pd->clks);
 742 err_put_clocks:
 743         clk_bulk_put(pd->num_clks, pd->clks);
 744         return error;
 745 }
 746
 747 static void rockchip_pm_remove_one_domain(struct rockchip_pm_domain *pd)
 748 {
 749         int ret;
 750
 751         /*
 752          * We're in the error cleanup already, so we only complain,
 753          * but won't emit another error on top of the original one.
 754          */
 755         ret = pm_genpd_remove(&pd->genpd);
 756         if (ret < 0)
 757                 dev_err(pd->pmu->dev, "failed to remove domain '%s' : %d - state may be inconsistent\n",
 758                         pd->genpd.name, ret);
 759
 760         clk_bulk_unprepare(pd->num_clks, pd->clks);
 761         clk_bulk_put(pd->num_clks, pd->clks);
 762
 763         /* protect the zeroing of pm->num_clks */
 764         mutex_lock(&pd->pmu->mutex);
 765         pd->num_clks = 0;
 766         mutex_unlock(&pd->pmu->mutex);
 767
 768         /* devm will free our memory */
 769 }
 770
 771 static void rockchip_pm_domain_cleanup(struct rockchip_pmu *pmu)
 772 {
 773         struct generic_pm_domain *genpd;
 774         struct rockchip_pm_domain *pd;
 775         int i;
 776
 777         for (i = 0; i < pmu->genpd_data.num_domains; i++) {
 778                 genpd = pmu->genpd_data.domains[i];
 779                 if (genpd) {
 780                         pd = to_rockchip_pd(genpd);
 781                         rockchip_pm_remove_one_domain(pd);
 782                 }
 783         }
 784
 785         /* devm will free our memory */
 786 }
 787
 788 static void rockchip_configure_pd_cnt(struct rockchip_pmu *pmu,
 789                                       u32 domain_reg_offset,
 790                                       unsigned int count)
 791 {
 792         /* First configure domain power down transition count ... */
 793         regmap_write(pmu->regmap, domain_reg_offset, count);
 794         /* ... and then power up count. */
 795         regmap_write(pmu->regmap, domain_reg_offset + 4, count);
 796 }
 797
 798 static int rockchip_pm_add_subdomain(struct rockchip_pmu *pmu,
 799                                      struct device_node *parent)
 800 {
 801         struct device_node *np;
 802         struct generic_pm_domain *child_domain, *parent_domain;
 803         int error;
 804
 805         for_each_child_of_node(parent, np) {
 806                 u32 idx;
 807
 808                 error = of_property_read_u32(parent, "reg", &idx);
 809                 if (error) {
 810                         dev_err(pmu->dev,
 811                                 "%pOFn: failed to retrieve domain id (reg): %d\n",
 812                                 parent, error);
 813                         goto err_out;
 814                 }
 815                 parent_domain = pmu->genpd_data.domains[idx];
 816
 817                 error = rockchip_pm_add_one_domain(pmu, np);
 818                 if (error) {
 819                         dev_err(pmu->dev, "failed to handle node %pOFn: %d\n",
 820                                 np, error);
 821                         goto err_out;
 822                 }
 823
 824                 error = of_property_read_u32(np, "reg", &idx);
 825                 if (error) {
 826                         dev_err(pmu->dev,
 827                                 "%pOFn: failed to retrieve domain id (reg): %d\n",
 828                                 np, error);
 829                         goto err_out;
 830                 }
 831                 child_domain = pmu->genpd_data.domains[idx];
 832
 833                 error = pm_genpd_add_subdomain(parent_domain, child_domain);
 834                 if (error) {
 835                         dev_err(pmu->dev, "%s failed to add subdomain %s: %d\n",
 836                                 parent_domain->name, child_domain->name, error);
 837                         goto err_out;
 838                 } else {
 839                         dev_dbg(pmu->dev, "%s add subdomain: %s\n",
 840                                 parent_domain->name, child_domain->name);
 841                 }
 842
 843                 rockchip_pm_add_subdomain(pmu, np);
 844         }
 845
 846         return 0;
 847
 848 err_out:
 849         of_node_put(np);
 850         return error;
 851 }
 852
 853 static int rockchip_pm_domain_probe(struct platform_device *pdev)
 854 {
 855         struct device *dev = &pdev->dev;
 856         struct device_node *np = dev->of_node;
 857         struct device_node *node;
 858         struct device *parent;
 859         struct rockchip_pmu *pmu;
 860         const struct of_device_id *match;
 861         const struct rockchip_pmu_info *pmu_info;
 862         int error;
 863
 864         if (!np) {
 865                 dev_err(dev, "device tree node not found\n");
 866                 return -ENODEV;
 867         }
 868
 869         match = of_match_device(dev->driver->of_match_table, dev);
 870         if (!match || !match->data) {
 871                 dev_err(dev, "missing pmu data\n");
 872                 return -EINVAL;
 873         }
 874
 875         pmu_info = match->data;
 876
 877         pmu = devm_kzalloc(dev,
 878                            struct_size(pmu, domains, pmu_info->num_domains),
 879                            GFP_KERNEL);
 880         if (!pmu)
 881                 return -ENOMEM;
 882
 883         pmu->dev = &pdev->dev;
 884         mutex_init(&pmu->mutex);
 885
 886         pmu->info = pmu_info;
 887
 888         pmu->genpd_data.domains = pmu->domains;
 889         pmu->genpd_data.num_domains = pmu_info->num_domains;
 890
 891         parent = dev->parent;
 892         if (!parent) {
 893                 dev_err(dev, "no parent for syscon devices\n");
 894                 return -ENODEV;
 895         }
 896
 897         pmu->regmap = syscon_node_to_regmap(parent->of_node);
 898         if (IS_ERR(pmu->regmap)) {
 899                 dev_err(dev, "no regmap available\n");
 900                 return PTR_ERR(pmu->regmap);
 901         }
 902
 903         /*
 904          * Configure power up and down transition delays for CORE
 905          * and GPU domains.
 906          */
 907         if (pmu_info->core_power_transition_time)
 908                 rockchip_configure_pd_cnt(pmu, pmu_info->core_pwrcnt_offset,
 909                                         pmu_info->core_power_transition_time);
 910         if (pmu_info->gpu_pwrcnt_offset)
 911                 rockchip_configure_pd_cnt(pmu, pmu_info->gpu_pwrcnt_offset,
 912                                         pmu_info->gpu_power_transition_time);
 913
 914         error = -ENODEV;
 915
 916         /*
 917          * Prevent any rockchip_pmu_block() from racing with the remainder of
 918          * setup (clocks, register initialization).
 919          */
 920         mutex_lock(&dmc_pmu_mutex);
 921
 922         for_each_available_child_of_node(np, node) {
 923                 error = rockchip_pm_add_one_domain(pmu, node);
 924                 if (error) {
 925                         dev_err(dev, "failed to handle node %pOFn: %d\n",
 926                                 node, error);
 927                         of_node_put(node);
 928                         goto err_out;
 929                 }
 930
 931                 error = rockchip_pm_add_subdomain(pmu, node);
 932                 if (error < 0) {
 933                         dev_err(dev, "failed to handle subdomain node %pOFn: %d\n",
 934                                 node, error);
 935                         of_node_put(node);
 936                         goto err_out;
 937                 }
 938         }
 939
 940         if (error) {
 941                 dev_dbg(dev, "no power domains defined\n");
 942                 goto err_out;
 943         }
 944
 945         error = of_genpd_add_provider_onecell(np, &pmu->genpd_data);
 946         if (error) {
 947                 dev_err(dev, "failed to add provider: %d\n", error);
 948                 goto err_out;
 949         }
 950
 951         /* We only expect one PMU. */
 952         if (!WARN_ON_ONCE(dmc_pmu))
 953                 dmc_pmu = pmu;
 954
 955         mutex_unlock(&dmc_pmu_mutex);
 956
 957         return 0;
 958
 959 err_out:
 960         rockchip_pm_domain_cleanup(pmu);
 961         mutex_unlock(&dmc_pmu_mutex);
 962         return error;
 963 }
 964
 965 static const struct rockchip_domain_info px30_pm_domains[] = {
 966         [PX30_PD_USB]           = DOMAIN_PX30("usb",      BIT(5),  BIT(5),  BIT(10), false),
 967         [PX30_PD_SDCARD]        = DOMAIN_PX30("sdcard",   BIT(8),  BIT(8),  BIT(9),  false),
 968         [PX30_PD_GMAC]          = DOMAIN_PX30("gmac",     BIT(10), BIT(10), BIT(6),  false),
 969         [PX30_PD_MMC_NAND]      = DOMAIN_PX30("mmc_nand", BIT(11), BIT(11), BIT(5),  false),
 970         [PX30_PD_VPU]           = DOMAIN_PX30("vpu",      BIT(12), BIT(12), BIT(14), false),
 971         [PX30_PD_VO]            = DOMAIN_PX30("vo",       BIT(13), BIT(13), BIT(7),  false),
 972         [PX30_PD_VI]            = DOMAIN_PX30("vi",       BIT(14), BIT(14), BIT(8),  false),
 973         [PX30_PD_GPU]           = DOMAIN_PX30("gpu",      BIT(15), BIT(15), BIT(2),  false),
 974 };
 975
 976 static const struct rockchip_domain_info rv1126_pm_domains[] = {
 977         [RV1126_PD_VEPU]        = DOMAIN_RV1126("vepu", BIT(2),  BIT(9),  BIT(9), false),
 978         [RV1126_PD_VI]          = DOMAIN_RV1126("vi", BIT(4),  BIT(6),  BIT(6),  false),
 979         [RV1126_PD_ISPP]        = DOMAIN_RV1126("ispp", BIT(1), BIT(8), BIT(8),  false),
 980         [RV1126_PD_VDPU]        = DOMAIN_RV1126("vdpu", BIT(3), BIT(10), BIT(10), false),
 981         [RV1126_PD_NVM]         = DOMAIN_RV1126("nvm", BIT(7), BIT(11), BIT(11),  false),
 982         [RV1126_PD_SDIO]        = DOMAIN_RV1126("sdio", BIT(8), BIT(13), BIT(13),  false),
 983         [RV1126_PD_USB]         = DOMAIN_RV1126("usb", BIT(9), BIT(15), BIT(15),  false),
 984 };
 985
 986 static const struct rockchip_domain_info rk3036_pm_domains[] = {
 987         [RK3036_PD_MSCH]        = DOMAIN_RK3036("msch", BIT(14), BIT(23), BIT(30), true),
 988         [RK3036_PD_CORE]        = DOMAIN_RK3036("core", BIT(13), BIT(17), BIT(24), false),
 989         [RK3036_PD_PERI]        = DOMAIN_RK3036("peri", BIT(12), BIT(18), BIT(25), false),
 990         [RK3036_PD_VIO]         = DOMAIN_RK3036("vio",  BIT(11), BIT(19), BIT(26), false),
 991         [RK3036_PD_VPU]         = DOMAIN_RK3036("vpu",  BIT(10), BIT(20), BIT(27), false),
 992         [RK3036_PD_GPU]         = DOMAIN_RK3036("gpu",  BIT(9),  BIT(21), BIT(28), false),
 993         [RK3036_PD_SYS]         = DOMAIN_RK3036("sys",  BIT(8),  BIT(22), BIT(29), false),
 994 };
 995
 996 static const struct rockchip_domain_info rk3066_pm_domains[] = {
 997         [RK3066_PD_GPU]         = DOMAIN("gpu",   BIT(9), BIT(9), BIT(3), BIT(24), BIT(29), false),
 998         [RK3066_PD_VIDEO]       = DOMAIN("video", BIT(8), BIT(8), BIT(4), BIT(23), BIT(28), false),
 999         [RK3066_PD_VIO]         = DOMAIN("vio",   BIT(7), BIT(7), BIT(5), BIT(22), BIT(27), false),
1000         [RK3066_PD_PERI]        = DOMAIN("peri",  BIT(6), BIT(6), BIT(2), BIT(25), BIT(30), false),
1001         [RK3066_PD_CPU]         = DOMAIN("cpu",   0,      BIT(5), BIT(1), BIT(26), BIT(31), false),
1002 };
1003
1004 static const struct rockchip_domain_info rk3128_pm_domains[] = {
1005         [RK3128_PD_CORE]        = DOMAIN_RK3288("core",  BIT(0), BIT(0), BIT(4), false),
1006         [RK3128_PD_MSCH]        = DOMAIN_RK3288("msch",  0,      0,      BIT(6), true),
1007         [RK3128_PD_VIO]         = DOMAIN_RK3288("vio",   BIT(3), BIT(3), BIT(2), false),
1008         [RK3128_PD_VIDEO]       = DOMAIN_RK3288("video", BIT(2), BIT(2), BIT(1), false),
1009         [RK3128_PD_GPU]         = DOMAIN_RK3288("gpu",   BIT(1), BIT(1), BIT(3), false),
1010 };
1011
1012 static const struct rockchip_domain_info rk3188_pm_domains[] = {
1013         [RK3188_PD_GPU]         = DOMAIN("gpu",   BIT(9), BIT(9), BIT(3), BIT(24), BIT(29), false),
1014         [RK3188_PD_VIDEO]       = DOMAIN("video", BIT(8), BIT(8), BIT(4), BIT(23), BIT(28), false),
1015         [RK3188_PD_VIO]         = DOMAIN("vio",   BIT(7), BIT(7), BIT(5), BIT(22), BIT(27), false),
1016         [RK3188_PD_PERI]        = DOMAIN("peri",  BIT(6), BIT(6), BIT(2), BIT(25), BIT(30), false),
1017         [RK3188_PD_CPU]         = DOMAIN("cpu",   BIT(5), BIT(5), BIT(1), BIT(26), BIT(31), false),
1018 };
1019
1020 static const struct rockchip_domain_info rk3228_pm_domains[] = {
1021         [RK3228_PD_CORE]        = DOMAIN_RK3036("core", BIT(0),  BIT(0),  BIT(16), true),
1022         [RK3228_PD_MSCH]        = DOMAIN_RK3036("msch", BIT(1),  BIT(1),  BIT(17), true),
1023         [RK3228_PD_BUS]         = DOMAIN_RK3036("bus",  BIT(2),  BIT(2),  BIT(18), true),
1024         [RK3228_PD_SYS]         = DOMAIN_RK3036("sys",  BIT(3),  BIT(3),  BIT(19), true),
1025         [RK3228_PD_VIO]         = DOMAIN_RK3036("vio",  BIT(4),  BIT(4),  BIT(20), false),
1026         [RK3228_PD_VOP]         = DOMAIN_RK3036("vop",  BIT(5),  BIT(5),  BIT(21), false),
1027         [RK3228_PD_VPU]         = DOMAIN_RK3036("vpu",  BIT(6),  BIT(6),  BIT(22), false),
1028         [RK3228_PD_RKVDEC]      = DOMAIN_RK3036("vdec", BIT(7),  BIT(7),  BIT(23), false),
1029         [RK3228_PD_GPU]         = DOMAIN_RK3036("gpu",  BIT(8),  BIT(8),  BIT(24), false),
1030         [RK3228_PD_PERI]        = DOMAIN_RK3036("peri", BIT(9),  BIT(9),  BIT(25), true),
1031         [RK3228_PD_GMAC]        = DOMAIN_RK3036("gmac", BIT(10), BIT(10), BIT(26), false),
1032 };
1033
1034 static const struct rockchip_domain_info rk3288_pm_domains[] = {
1035         [RK3288_PD_VIO]         = DOMAIN_RK3288("vio",   BIT(7),  BIT(7),  BIT(4), false),
1036         [RK3288_PD_HEVC]        = DOMAIN_RK3288("hevc",  BIT(14), BIT(10), BIT(9), false),
1037         [RK3288_PD_VIDEO]       = DOMAIN_RK3288("video", BIT(8),  BIT(8),  BIT(3), false),
1038         [RK3288_PD_GPU]         = DOMAIN_RK3288("gpu",   BIT(9),  BIT(9),  BIT(2), false),
1039 };
1040
1041 static const struct rockchip_domain_info rk3328_pm_domains[] = {
1042         [RK3328_PD_CORE]        = DOMAIN_RK3328("core",  0, BIT(0), BIT(0), false),
1043         [RK3328_PD_GPU]         = DOMAIN_RK3328("gpu",   0, BIT(1), BIT(1), false),
1044         [RK3328_PD_BUS]         = DOMAIN_RK3328("bus",   0, BIT(2), BIT(2), true),
1045         [RK3328_PD_MSCH]        = DOMAIN_RK3328("msch",  0, BIT(3), BIT(3), true),
1046         [RK3328_PD_PERI]        = DOMAIN_RK3328("peri",  0, BIT(4), BIT(4), true),
1047         [RK3328_PD_VIDEO]       = DOMAIN_RK3328("video", 0, BIT(5), BIT(5), false),
1048         [RK3328_PD_HEVC]        = DOMAIN_RK3328("hevc",  0, BIT(6), BIT(6), false),
1049         [RK3328_PD_VIO]         = DOMAIN_RK3328("vio",   0, BIT(8), BIT(8), false),
1050         [RK3328_PD_VPU]         = DOMAIN_RK3328("vpu",   0, BIT(9), BIT(9), false),
1051 };
1052
1053 static const struct rockchip_domain_info rk3366_pm_domains[] = {
1054         [RK3366_PD_PERI]        = DOMAIN_RK3368("peri",   BIT(10), BIT(10), BIT(6), true),
1055         [RK3366_PD_VIO]         = DOMAIN_RK3368("vio",    BIT(14), BIT(14), BIT(8), false),
1056         [RK3366_PD_VIDEO]       = DOMAIN_RK3368("video",  BIT(13), BIT(13), BIT(7), false),
1057         [RK3366_PD_RKVDEC]      = DOMAIN_RK3368("vdec",   BIT(11), BIT(11), BIT(7), false),
1058         [RK3366_PD_WIFIBT]      = DOMAIN_RK3368("wifibt", BIT(8),  BIT(8),  BIT(9), false),
1059         [RK3366_PD_VPU]         = DOMAIN_RK3368("vpu",    BIT(12), BIT(12), BIT(7), false),
1060         [RK3366_PD_GPU]         = DOMAIN_RK3368("gpu",    BIT(15), BIT(15), BIT(2), false),
1061 };
1062
1063 static const struct rockchip_domain_info rk3368_pm_domains[] = {
1064         [RK3368_PD_PERI]        = DOMAIN_RK3368("peri",  BIT(13), BIT(12), BIT(6), true),
1065         [RK3368_PD_VIO]         = DOMAIN_RK3368("vio",   BIT(15), BIT(14), BIT(8), false),
1066         [RK3368_PD_VIDEO]       = DOMAIN_RK3368("video", BIT(14), BIT(13), BIT(7), false),
1067         [RK3368_PD_GPU_0]       = DOMAIN_RK3368("gpu_0", BIT(16), BIT(15), BIT(2), false),
1068         [RK3368_PD_GPU_1]       = DOMAIN_RK3368("gpu_1", BIT(17), BIT(16), BIT(2), false),
1069 };
1070
1071 static const struct rockchip_domain_info rk3399_pm_domains[] = {
1072         [RK3399_PD_TCPD0]       = DOMAIN_RK3399("tcpd0",     BIT(8),  BIT(8),  0,       false),
1073         [RK3399_PD_TCPD1]       = DOMAIN_RK3399("tcpd1",     BIT(9),  BIT(9),  0,       false),
1074         [RK3399_PD_CCI]         = DOMAIN_RK3399("cci",       BIT(10), BIT(10), 0,       true),
1075         [RK3399_PD_CCI0]        = DOMAIN_RK3399("cci0",      0,       0,       BIT(15), true),
1076         [RK3399_PD_CCI1]        = DOMAIN_RK3399("cci1",      0,       0,       BIT(16), true),
1077         [RK3399_PD_PERILP]      = DOMAIN_RK3399("perilp",    BIT(11), BIT(11), BIT(1),  true),
1078         [RK3399_PD_PERIHP]      = DOMAIN_RK3399("perihp",    BIT(12), BIT(12), BIT(2),  true),
1079         [RK3399_PD_CENTER]      = DOMAIN_RK3399("center",    BIT(13), BIT(13), BIT(14), true),
1080         [RK3399_PD_VIO]         = DOMAIN_RK3399("vio",       BIT(14), BIT(14), BIT(17), false),
1081         [RK3399_PD_GPU]         = DOMAIN_RK3399("gpu",       BIT(15), BIT(15), BIT(0),  false),
1082         [RK3399_PD_VCODEC]      = DOMAIN_RK3399("vcodec",    BIT(16), BIT(16), BIT(3),  false),
1083         [RK3399_PD_VDU]         = DOMAIN_RK3399("vdu",       BIT(17), BIT(17), BIT(4),  false),
1084         [RK3399_PD_RGA]         = DOMAIN_RK3399("rga",       BIT(18), BIT(18), BIT(5),  false),
1085         [RK3399_PD_IEP]         = DOMAIN_RK3399("iep",       BIT(19), BIT(19), BIT(6),  false),
1086         [RK3399_PD_VO]          = DOMAIN_RK3399("vo",        BIT(20), BIT(20), 0,       false),
1087         [RK3399_PD_VOPB]        = DOMAIN_RK3399("vopb",      0,       0,       BIT(7),  false),
1088         [RK3399_PD_VOPL]        = DOMAIN_RK3399("vopl",      0,       0,       BIT(8),  false),
1089         [RK3399_PD_ISP0]        = DOMAIN_RK3399("isp0",      BIT(22), BIT(22), BIT(9),  false),
1090         [RK3399_PD_ISP1]        = DOMAIN_RK3399("isp1",      BIT(23), BIT(23), BIT(10), false),
1091         [RK3399_PD_HDCP]        = DOMAIN_RK3399("hdcp",      BIT(24), BIT(24), BIT(11), false),
1092         [RK3399_PD_GMAC]        = DOMAIN_RK3399("gmac",      BIT(25), BIT(25), BIT(23), true),
1093         [RK3399_PD_EMMC]        = DOMAIN_RK3399("emmc",      BIT(26), BIT(26), BIT(24), true),
1094         [RK3399_PD_USB3]        = DOMAIN_RK3399("usb3",      BIT(27), BIT(27), BIT(12), true),
1095         [RK3399_PD_EDP]         = DOMAIN_RK3399("edp",       BIT(28), BIT(28), BIT(22), false),
1096         [RK3399_PD_GIC]         = DOMAIN_RK3399("gic",       BIT(29), BIT(29), BIT(27), true),
1097         [RK3399_PD_SD]          = DOMAIN_RK3399("sd",        BIT(30), BIT(30), BIT(28), true),
1098         [RK3399_PD_SDIOAUDIO]   = DOMAIN_RK3399("sdioaudio", BIT(31), BIT(31), BIT(29), true),
1099 };
1100
1101 static const struct rockchip_domain_info rk3568_pm_domains[] = {
1102         [RK3568_PD_NPU]         = DOMAIN_RK3568("npu",  BIT(1), BIT(2),  false),
1103         [RK3568_PD_GPU]         = DOMAIN_RK3568("gpu",  BIT(0), BIT(1),  false),
1104         [RK3568_PD_VI]          = DOMAIN_RK3568("vi",   BIT(6), BIT(3),  false),
1105         [RK3568_PD_VO]          = DOMAIN_RK3568("vo",   BIT(7), BIT(4),  false),
1106         [RK3568_PD_RGA]         = DOMAIN_RK3568("rga",  BIT(5), BIT(5),  false),
1107         [RK3568_PD_VPU]         = DOMAIN_RK3568("vpu",  BIT(2), BIT(6),  false),
1108         [RK3568_PD_RKVDEC]      = DOMAIN_RK3568("vdec", BIT(4), BIT(8),  false),
1109         [RK3568_PD_RKVENC]      = DOMAIN_RK3568("venc", BIT(3), BIT(7),  false),
1110         [RK3568_PD_PIPE]        = DOMAIN_RK3568("pipe", BIT(8), BIT(11), false),
1111 };
1112
1113 static const struct rockchip_domain_info rk3588_pm_domains[] = {
1114         [RK3588_PD_GPU]         = DOMAIN_RK3588("gpu",     0x0, BIT(0),  0,       0x0, 0,       BIT(1),  0x0, BIT(0),  BIT(0),  false),
1115         [RK3588_PD_NPU]         = DOMAIN_RK3588("npu",     0x0, BIT(1),  BIT(1),  0x0, 0,       0,       0x0, 0,       0,       false),
1116         [RK3588_PD_VCODEC]      = DOMAIN_RK3588("vcodec",  0x0, BIT(2),  BIT(2),  0x0, 0,       0,       0x0, 0,       0,       false),
1117         [RK3588_PD_NPUTOP]      = DOMAIN_RK3588("nputop",  0x0, BIT(3),  0,       0x0, BIT(11), BIT(2),  0x0, BIT(1),  BIT(1),  false),
1118         [RK3588_PD_NPU1]        = DOMAIN_RK3588("npu1",    0x0, BIT(4),  0,       0x0, BIT(12), BIT(3),  0x0, BIT(2),  BIT(2),  false),
1119         [RK3588_PD_NPU2]        = DOMAIN_RK3588("npu2",    0x0, BIT(5),  0,       0x0, BIT(13), BIT(4),  0x0, BIT(3),  BIT(3),  false),
1120         [RK3588_PD_VENC0]       = DOMAIN_RK3588("venc0",   0x0, BIT(6),  0,       0x0, BIT(14), BIT(5),  0x0, BIT(4),  BIT(4),  false),
1121         [RK3588_PD_VENC1]       = DOMAIN_RK3588("venc1",   0x0, BIT(7),  0,       0x0, BIT(15), BIT(6),  0x0, BIT(5),  BIT(5),  false),
1122         [RK3588_PD_RKVDEC0]     = DOMAIN_RK3588("rkvdec0", 0x0, BIT(8),  0,       0x0, BIT(16), BIT(7),  0x0, BIT(6),  BIT(6),  false),
1123         [RK3588_PD_RKVDEC1]     = DOMAIN_RK3588("rkvdec1", 0x0, BIT(9),  0,       0x0, BIT(17), BIT(8),  0x0, BIT(7),  BIT(7),  false),
1124         [RK3588_PD_VDPU]        = DOMAIN_RK3588("vdpu",    0x0, BIT(10), 0,       0x0, BIT(18), BIT(9),  0x0, BIT(8),  BIT(8),  false),
1125         [RK3588_PD_RGA30]       = DOMAIN_RK3588("rga30",   0x0, BIT(11), 0,       0x0, BIT(19), BIT(10), 0x0, 0,       0,       false),
1126         [RK3588_PD_AV1]         = DOMAIN_RK3588("av1",     0x0, BIT(12), 0,       0x0, BIT(20), BIT(11), 0x0, BIT(9),  BIT(9),  false),
1127         [RK3588_PD_VI]          = DOMAIN_RK3588("vi",      0x0, BIT(13), 0,       0x0, BIT(21), BIT(12), 0x0, BIT(10), BIT(10), false),
1128         [RK3588_PD_FEC]         = DOMAIN_RK3588("fec",     0x0, BIT(14), 0,       0x0, BIT(22), BIT(13), 0x0, 0,       0,       false),
1129         [RK3588_PD_ISP1]        = DOMAIN_RK3588("isp1",    0x0, BIT(15), 0,       0x0, BIT(23), BIT(14), 0x0, BIT(11), BIT(11), false),
1130         [RK3588_PD_RGA31]       = DOMAIN_RK3588("rga31",   0x4, BIT(0),  0,       0x0, BIT(24), BIT(15), 0x0, BIT(12), BIT(12), false),
1131         [RK3588_PD_VOP]         = DOMAIN_RK3588("vop",     0x4, BIT(1),  0,       0x0, BIT(25), BIT(16), 0x0, BIT(13) | BIT(14), BIT(13) | BIT(14), false),
1132         [RK3588_PD_VO0]         = DOMAIN_RK3588("vo0",     0x4, BIT(2),  0,       0x0, BIT(26), BIT(17), 0x0, BIT(15), BIT(15), false),
1133         [RK3588_PD_VO1]         = DOMAIN_RK3588("vo1",     0x4, BIT(3),  0,       0x0, BIT(27), BIT(18), 0x4, BIT(0),  BIT(16), false),
1134         [RK3588_PD_AUDIO]       = DOMAIN_RK3588("audio",   0x4, BIT(4),  0,       0x0, BIT(28), BIT(19), 0x4, BIT(1),  BIT(17), false),
1135         [RK3588_PD_PHP]         = DOMAIN_RK3588("php",     0x4, BIT(5),  0,       0x0, BIT(29), BIT(20), 0x4, BIT(5),  BIT(21), false),
1136         [RK3588_PD_GMAC]        = DOMAIN_RK3588("gmac",    0x4, BIT(6),  0,       0x0, BIT(30), BIT(21), 0x0, 0,       0,       false),
1137         [RK3588_PD_PCIE]        = DOMAIN_RK3588("pcie",    0x4, BIT(7),  0,       0x0, BIT(31), BIT(22), 0x0, 0,       0,       true),
1138         [RK3588_PD_NVM]         = DOMAIN_RK3588("nvm",     0x4, BIT(8),  BIT(24), 0x4, 0,       0,       0x4, BIT(2),  BIT(18), false),
1139         [RK3588_PD_NVM0]        = DOMAIN_RK3588("nvm0",    0x4, BIT(9),  0,       0x4, BIT(1),  BIT(23), 0x0, 0,       0,       false),
1140         [RK3588_PD_SDIO]        = DOMAIN_RK3588("sdio",    0x4, BIT(10), 0,       0x4, BIT(2),  BIT(24), 0x4, BIT(3),  BIT(19), false),
1141         [RK3588_PD_USB]         = DOMAIN_RK3588("usb",     0x4, BIT(11), 0,       0x4, BIT(3),  BIT(25), 0x4, BIT(4),  BIT(20), true),
1142         [RK3588_PD_SDMMC]       = DOMAIN_RK3588("sdmmc",   0x4, BIT(13), 0,       0x4, BIT(5),  BIT(26), 0x0, 0,       0,       false),
1143 };
1144
1145 static const struct rockchip_pmu_info px30_pmu = {
1146         .pwr_offset = 0x18,
1147         .status_offset = 0x20,
1148         .req_offset = 0x64,
1149         .idle_offset = 0x6c,
1150         .ack_offset = 0x6c,
1151
1152         .num_domains = ARRAY_SIZE(px30_pm_domains),
1153         .domain_info = px30_pm_domains,
1154 };
1155
1156 static const struct rockchip_pmu_info rk3036_pmu = {
1157         .req_offset = 0x148,
1158         .idle_offset = 0x14c,
1159         .ack_offset = 0x14c,
1160
1161         .num_domains = ARRAY_SIZE(rk3036_pm_domains),
1162         .domain_info = rk3036_pm_domains,
1163 };
1164
1165 static const struct rockchip_pmu_info rk3066_pmu = {
1166         .pwr_offset = 0x08,
1167         .status_offset = 0x0c,
1168         .req_offset = 0x38, /* PMU_MISC_CON1 */
1169         .idle_offset = 0x0c,
1170         .ack_offset = 0x0c,
1171
1172         .num_domains = ARRAY_SIZE(rk3066_pm_domains),
1173         .domain_info = rk3066_pm_domains,
1174 };
1175
1176 static const struct rockchip_pmu_info rk3128_pmu = {
1177         .pwr_offset = 0x04,
1178         .status_offset = 0x08,
1179         .req_offset = 0x0c,
1180         .idle_offset = 0x10,
1181         .ack_offset = 0x10,
1182
1183         .num_domains = ARRAY_SIZE(rk3128_pm_domains),
1184         .domain_info = rk3128_pm_domains,
1185 };
1186
1187 static const struct rockchip_pmu_info rk3188_pmu = {
1188         .pwr_offset = 0x08,
1189         .status_offset = 0x0c,
1190         .req_offset = 0x38, /* PMU_MISC_CON1 */
1191         .idle_offset = 0x0c,
1192         .ack_offset = 0x0c,
1193
1194         .num_domains = ARRAY_SIZE(rk3188_pm_domains),
1195         .domain_info = rk3188_pm_domains,
1196 };
1197
1198 static const struct rockchip_pmu_info rk3228_pmu = {
1199         .req_offset = 0x40c,
1200         .idle_offset = 0x488,
1201         .ack_offset = 0x488,
1202
1203         .num_domains = ARRAY_SIZE(rk3228_pm_domains),
1204         .domain_info = rk3228_pm_domains,
1205 };
1206
1207 static const struct rockchip_pmu_info rk3288_pmu = {
1208         .pwr_offset = 0x08,
1209         .status_offset = 0x0c,
1210         .req_offset = 0x10,
1211         .idle_offset = 0x14,
1212         .ack_offset = 0x14,
1213
1214         .core_pwrcnt_offset = 0x34,
1215         .gpu_pwrcnt_offset = 0x3c,
1216
1217         .core_power_transition_time = 24, /* 1us */
1218         .gpu_power_transition_time = 24, /* 1us */
1219
1220         .num_domains = ARRAY_SIZE(rk3288_pm_domains),
1221         .domain_info = rk3288_pm_domains,
1222 };
1223
1224 static const struct rockchip_pmu_info rk3328_pmu = {
1225         .req_offset = 0x414,
1226         .idle_offset = 0x484,
1227         .ack_offset = 0x484,
1228
1229         .num_domains = ARRAY_SIZE(rk3328_pm_domains),
1230         .domain_info = rk3328_pm_domains,
1231 };
1232
1233 static const struct rockchip_pmu_info rk3366_pmu = {
1234         .pwr_offset = 0x0c,
1235         .status_offset = 0x10,
1236         .req_offset = 0x3c,
1237         .idle_offset = 0x40,
1238         .ack_offset = 0x40,
1239
1240         .core_pwrcnt_offset = 0x48,
1241         .gpu_pwrcnt_offset = 0x50,
1242
1243         .core_power_transition_time = 24,
1244         .gpu_power_transition_time = 24,
1245
1246         .num_domains = ARRAY_SIZE(rk3366_pm_domains),
1247         .domain_info = rk3366_pm_domains,
1248 };
1249
1250 static const struct rockchip_pmu_info rk3368_pmu = {
1251         .pwr_offset = 0x0c,
1252         .status_offset = 0x10,
1253         .req_offset = 0x3c,
1254         .idle_offset = 0x40,
1255         .ack_offset = 0x40,
1256
1257         .core_pwrcnt_offset = 0x48,
1258         .gpu_pwrcnt_offset = 0x50,
1259
1260         .core_power_transition_time = 24,
1261         .gpu_power_transition_time = 24,
1262
1263         .num_domains = ARRAY_SIZE(rk3368_pm_domains),
1264         .domain_info = rk3368_pm_domains,
1265 };
1266
1267 static const struct rockchip_pmu_info rk3399_pmu = {
1268         .pwr_offset = 0x14,
1269         .status_offset = 0x18,
1270         .req_offset = 0x60,
1271         .idle_offset = 0x64,
1272         .ack_offset = 0x68,
1273
1274         /* ARM Trusted Firmware manages power transition times */
1275
1276         .num_domains = ARRAY_SIZE(rk3399_pm_domains),
1277         .domain_info = rk3399_pm_domains,
1278 };
1279
1280 static const struct rockchip_pmu_info rk3568_pmu = {
1281         .pwr_offset = 0xa0,
1282         .status_offset = 0x98,
1283         .req_offset = 0x50,
1284         .idle_offset = 0x68,
1285         .ack_offset = 0x60,
1286
1287         .num_domains = ARRAY_SIZE(rk3568_pm_domains),
1288         .domain_info = rk3568_pm_domains,
1289 };
1290
1291 static const struct rockchip_pmu_info rk3588_pmu = {
1292         .pwr_offset = 0x14c,
1293         .status_offset = 0x180,
1294         .req_offset = 0x10c,
1295         .idle_offset = 0x120,
1296         .ack_offset = 0x118,
1297         .mem_pwr_offset = 0x1a0,
1298         .chain_status_offset = 0x1f0,
1299         .mem_status_offset = 0x1f8,
1300         .repair_status_offset = 0x290,
1301
1302         .num_domains = ARRAY_SIZE(rk3588_pm_domains),
1303         .domain_info = rk3588_pm_domains,
1304 };
1305
1306 static const struct rockchip_pmu_info rv1126_pmu = {
1307         .pwr_offset = 0x110,
1308         .status_offset = 0x108,
1309         .req_offset = 0xc0,
1310         .idle_offset = 0xd8,
1311         .ack_offset = 0xd0,
1312
1313         .num_domains = ARRAY_SIZE(rv1126_pm_domains),
1314         .domain_info = rv1126_pm_domains,
1315 };
1316
1317 static const struct of_device_id rockchip_pm_domain_dt_match[] = {
1318         {
1319                 .compatible = "rockchip,px30-power-controller",
1320                 .data = (void *)&px30_pmu,
1321         },
1322         {
1323                 .compatible = "rockchip,rk3036-power-controller",
1324                 .data = (void *)&rk3036_pmu,
1325         },
1326         {
1327                 .compatible = "rockchip,rk3066-power-controller",
1328                 .data = (void *)&rk3066_pmu,
1329         },
1330         {
1331                 .compatible = "rockchip,rk3128-power-controller",
1332                 .data = (void *)&rk3128_pmu,
1333         },
1334         {
1335                 .compatible = "rockchip,rk3188-power-controller",
1336                 .data = (void *)&rk3188_pmu,
1337         },
1338         {
1339                 .compatible = "rockchip,rk3228-power-controller",
1340                 .data = (void *)&rk3228_pmu,
1341         },
1342         {
1343                 .compatible = "rockchip,rk3288-power-controller",
1344                 .data = (void *)&rk3288_pmu,
1345         },
1346         {
1347                 .compatible = "rockchip,rk3328-power-controller",
1348                 .data = (void *)&rk3328_pmu,
1349         },
1350         {
1351                 .compatible = "rockchip,rk3366-power-controller",
1352                 .data = (void *)&rk3366_pmu,
1353         },
1354         {
1355                 .compatible = "rockchip,rk3368-power-controller",
1356                 .data = (void *)&rk3368_pmu,
1357         },
1358         {
1359                 .compatible = "rockchip,rk3399-power-controller",
1360                 .data = (void *)&rk3399_pmu,
1361         },
1362         {
1363                 .compatible = "rockchip,rk3568-power-controller",
1364                 .data = (void *)&rk3568_pmu,
1365         },
1366         {
1367                 .compatible = "rockchip,rk3588-power-controller",
1368                 .data = (void *)&rk3588_pmu,
1369         },
1370         {
1371                 .compatible = "rockchip,rv1126-power-controller",
1372                 .data = (void *)&rv1126_pmu,
1373         },
1374         { /* sentinel */ },
1375 };
1376
1377 static struct platform_driver rockchip_pm_domain_driver = {
1378         .probe = rockchip_pm_domain_probe,
1379         .driver = {
1380                 .name   = "rockchip-pm-domain",
1381                 .of_match_table = rockchip_pm_domain_dt_match,
1382                 /*
1383                  * We can't forcibly eject devices from the power
1384                  * domain, so we can't really remove power domains
1385                  * once they were added.
1386                  */
1387                 .suppress_bind_attrs = true,
1388         },
1389 };
1390
1391 static int __init rockchip_pm_domain_drv_register(void)
1392 {
1393         return platform_driver_register(&rockchip_pm_domain_driver);
1394 }
1395 postcore_initcall(rockchip_pm_domain_drv_register);