1 // SPDX-License-Identifier: GPL-2.0
3 * Interconnect framework core driver
5 * Copyright (c) 2017-2019, Linaro Ltd.
9 #include <linux/debugfs.h>
10 #include <linux/device.h>
11 #include <linux/idr.h>
12 #include <linux/init.h>
13 #include <linux/interconnect.h>
14 #include <linux/interconnect-provider.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/slab.h>
20 #include <linux/overflow.h>
22 static DEFINE_IDR(icc_idr);
23 static LIST_HEAD(icc_providers);
24 static DEFINE_MUTEX(icc_lock);
25 static struct dentry *icc_debugfs_dir;
28 * struct icc_req - constraints that are attached to each node
29 * @req_node: entry in list of requests for the particular @node
30 * @node: the interconnect node to which this constraint applies
31 * @dev: reference to the device that sets the constraints
32 * @avg_bw: an integer describing the average bandwidth in kBps
33 * @peak_bw: an integer describing the peak bandwidth in kBps
36 struct hlist_node req_node;
37 struct icc_node *node;
44 * struct icc_path - interconnect path structure
45 * @num_nodes: number of hops (nodes)
46 * @reqs: array of the requests applicable to this path of nodes
50 struct icc_req reqs[];
53 static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
58 seq_printf(s, "%-30s %12u %12u\n",
59 n->name, n->avg_bw, n->peak_bw);
62 static int icc_summary_show(struct seq_file *s, void *data)
64 struct icc_provider *provider;
66 seq_puts(s, " node avg peak\n");
67 seq_puts(s, "--------------------------------------------------------\n");
69 mutex_lock(&icc_lock);
71 list_for_each_entry(provider, &icc_providers, provider_list) {
74 list_for_each_entry(n, &provider->nodes, node_list) {
77 icc_summary_show_one(s, n);
78 hlist_for_each_entry(r, &n->req_list, req_node) {
82 seq_printf(s, " %-26s %12u %12u\n",
83 dev_name(r->dev), r->avg_bw,
89 mutex_unlock(&icc_lock);
93 DEFINE_SHOW_ATTRIBUTE(icc_summary);
95 static struct icc_node *node_find(const int id)
97 return idr_find(&icc_idr, id);
100 static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
103 struct icc_node *node = dst;
104 struct icc_path *path;
107 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL);
109 return ERR_PTR(-ENOMEM);
111 path->num_nodes = num_nodes;
113 for (i = num_nodes - 1; i >= 0; i--) {
114 node->provider->users++;
115 hlist_add_head(&path->reqs[i].req_node, &node->req_list);
116 path->reqs[i].node = node;
117 path->reqs[i].dev = dev;
118 /* reference to previous node was saved during path traversal */
119 node = node->reverse;
125 static struct icc_path *path_find(struct device *dev, struct icc_node *src,
126 struct icc_node *dst)
128 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
129 struct icc_node *n, *node = NULL;
130 struct list_head traverse_list;
131 struct list_head edge_list;
132 struct list_head visited_list;
136 INIT_LIST_HEAD(&traverse_list);
137 INIT_LIST_HEAD(&edge_list);
138 INIT_LIST_HEAD(&visited_list);
140 list_add(&src->search_list, &traverse_list);
144 list_for_each_entry_safe(node, n, &traverse_list, search_list) {
147 list_splice_init(&edge_list, &visited_list);
148 list_splice_init(&traverse_list, &visited_list);
151 for (i = 0; i < node->num_links; i++) {
152 struct icc_node *tmp = node->links[i];
155 path = ERR_PTR(-ENOENT);
159 if (tmp->is_traversed)
162 tmp->is_traversed = true;
164 list_add_tail(&tmp->search_list, &edge_list);
171 list_splice_init(&traverse_list, &visited_list);
172 list_splice_init(&edge_list, &traverse_list);
174 /* count the hops including the source */
177 } while (!list_empty(&traverse_list));
181 /* reset the traversed state */
182 list_for_each_entry_reverse(n, &visited_list, search_list)
183 n->is_traversed = false;
186 path = path_init(dev, dst, depth);
192 * We want the path to honor all bandwidth requests, so the average and peak
193 * bandwidth requirements from each consumer are aggregated at each node.
194 * The aggregation is platform specific, so each platform can customize it by
195 * implementing its own aggregate() function.
198 static int aggregate_requests(struct icc_node *node)
200 struct icc_provider *p = node->provider;
206 hlist_for_each_entry(r, &node->req_list, req_node)
207 p->aggregate(node, r->avg_bw, r->peak_bw,
208 &node->avg_bw, &node->peak_bw);
213 static int apply_constraints(struct icc_path *path)
215 struct icc_node *next, *prev = NULL;
219 for (i = 0; i < path->num_nodes; i++) {
220 next = path->reqs[i].node;
223 * Both endpoints should be valid master-slave pairs of the
224 * same interconnect provider that will be configured.
226 if (!prev || next->provider != prev->provider) {
231 /* set the constraints */
232 ret = next->provider->set(prev, next);
242 /* of_icc_xlate_onecell() - Translate function using a single index.
243 * @spec: OF phandle args to map into an interconnect node.
244 * @data: private data (pointer to struct icc_onecell_data)
246 * This is a generic translate function that can be used to model simple
247 * interconnect providers that have one device tree node and provide
248 * multiple interconnect nodes. A single cell is used as an index into
249 * an array of icc nodes specified in the icc_onecell_data struct when
250 * registering the provider.
252 struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec,
255 struct icc_onecell_data *icc_data = data;
256 unsigned int idx = spec->args[0];
258 if (idx >= icc_data->num_nodes) {
259 pr_err("%s: invalid index %u\n", __func__, idx);
260 return ERR_PTR(-EINVAL);
263 return icc_data->nodes[idx];
265 EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
268 * of_icc_get_from_provider() - Look-up interconnect node
269 * @spec: OF phandle args to use for look-up
271 * Looks for interconnect provider under the node specified by @spec and if
272 * found, uses xlate function of the provider to map phandle args to node.
274 * Returns a valid pointer to struct icc_node on success or ERR_PTR()
277 static struct icc_node *of_icc_get_from_provider(struct of_phandle_args *spec)
279 struct icc_node *node = ERR_PTR(-EPROBE_DEFER);
280 struct icc_provider *provider;
282 if (!spec || spec->args_count != 1)
283 return ERR_PTR(-EINVAL);
285 mutex_lock(&icc_lock);
286 list_for_each_entry(provider, &icc_providers, provider_list) {
287 if (provider->dev->of_node == spec->np)
288 node = provider->xlate(spec, provider->data);
292 mutex_unlock(&icc_lock);
298 * of_icc_get() - get a path handle from a DT node based on name
299 * @dev: device pointer for the consumer device
300 * @name: interconnect path name
302 * This function will search for a path between two endpoints and return an
303 * icc_path handle on success. Use icc_put() to release constraints when they
304 * are not needed anymore.
305 * If the interconnect API is disabled, NULL is returned and the consumer
306 * drivers will still build. Drivers are free to handle this specifically,
307 * but they don't have to.
309 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
310 * when the API is disabled or the "interconnects" DT property is missing.
312 struct icc_path *of_icc_get(struct device *dev, const char *name)
314 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
315 struct icc_node *src_node, *dst_node;
316 struct device_node *np = NULL;
317 struct of_phandle_args src_args, dst_args;
321 if (!dev || !dev->of_node)
322 return ERR_PTR(-ENODEV);
327 * When the consumer DT node do not have "interconnects" property
328 * return a NULL path to skip setting constraints.
330 if (!of_find_property(np, "interconnects", NULL))
334 * We use a combination of phandle and specifier for endpoint. For now
335 * lets support only global ids and extend this in the future if needed
336 * without breaking DT compatibility.
339 idx = of_property_match_string(np, "interconnect-names", name);
344 ret = of_parse_phandle_with_args(np, "interconnects",
345 "#interconnect-cells", idx * 2,
350 of_node_put(src_args.np);
352 ret = of_parse_phandle_with_args(np, "interconnects",
353 "#interconnect-cells", idx * 2 + 1,
358 of_node_put(dst_args.np);
360 src_node = of_icc_get_from_provider(&src_args);
362 if (IS_ERR(src_node)) {
363 if (PTR_ERR(src_node) != -EPROBE_DEFER)
364 dev_err(dev, "error finding src node: %ld\n",
366 return ERR_CAST(src_node);
369 dst_node = of_icc_get_from_provider(&dst_args);
371 if (IS_ERR(dst_node)) {
372 if (PTR_ERR(dst_node) != -EPROBE_DEFER)
373 dev_err(dev, "error finding dst node: %ld\n",
375 return ERR_CAST(dst_node);
378 mutex_lock(&icc_lock);
379 path = path_find(dev, src_node, dst_node);
381 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
382 mutex_unlock(&icc_lock);
386 EXPORT_SYMBOL_GPL(of_icc_get);
389 * icc_set_bw() - set bandwidth constraints on an interconnect path
390 * @path: reference to the path returned by icc_get()
391 * @avg_bw: average bandwidth in kilobytes per second
392 * @peak_bw: peak bandwidth in kilobytes per second
394 * This function is used by an interconnect consumer to express its own needs
395 * in terms of bandwidth for a previously requested path between two endpoints.
396 * The requests are aggregated and each node is updated accordingly. The entire
397 * path is locked by a mutex to ensure that the set() is completed.
398 * The @path can be NULL when the "interconnects" DT properties is missing,
399 * which will mean that no constraints will be set.
401 * Returns 0 on success, or an appropriate error code otherwise.
403 int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
405 struct icc_node *node;
406 u32 old_avg, old_peak;
410 if (!path || !path->num_nodes)
413 mutex_lock(&icc_lock);
415 old_avg = path->reqs[0].avg_bw;
416 old_peak = path->reqs[0].peak_bw;
418 for (i = 0; i < path->num_nodes; i++) {
419 node = path->reqs[i].node;
421 /* update the consumer request for this path */
422 path->reqs[i].avg_bw = avg_bw;
423 path->reqs[i].peak_bw = peak_bw;
425 /* aggregate requests for this node */
426 aggregate_requests(node);
429 ret = apply_constraints(path);
431 pr_debug("interconnect: error applying constraints (%d)\n",
434 for (i = 0; i < path->num_nodes; i++) {
435 node = path->reqs[i].node;
436 path->reqs[i].avg_bw = old_avg;
437 path->reqs[i].peak_bw = old_peak;
438 aggregate_requests(node);
440 apply_constraints(path);
443 mutex_unlock(&icc_lock);
447 EXPORT_SYMBOL_GPL(icc_set_bw);
450 * icc_get() - return a handle for path between two endpoints
451 * @dev: the device requesting the path
452 * @src_id: source device port id
453 * @dst_id: destination device port id
455 * This function will search for a path between two endpoints and return an
456 * icc_path handle on success. Use icc_put() to release
457 * constraints when they are not needed anymore.
458 * If the interconnect API is disabled, NULL is returned and the consumer
459 * drivers will still build. Drivers are free to handle this specifically,
460 * but they don't have to.
462 * Return: icc_path pointer on success, ERR_PTR() on error or NULL if the
463 * interconnect API is disabled.
465 struct icc_path *icc_get(struct device *dev, const int src_id, const int dst_id)
467 struct icc_node *src, *dst;
468 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
470 mutex_lock(&icc_lock);
472 src = node_find(src_id);
476 dst = node_find(dst_id);
480 path = path_find(dev, src, dst);
482 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
485 mutex_unlock(&icc_lock);
488 EXPORT_SYMBOL_GPL(icc_get);
491 * icc_put() - release the reference to the icc_path
492 * @path: interconnect path
494 * Use this function to release the constraints on a path when the path is
495 * no longer needed. The constraints will be re-aggregated.
497 void icc_put(struct icc_path *path)
499 struct icc_node *node;
503 if (!path || WARN_ON(IS_ERR(path)))
506 ret = icc_set_bw(path, 0, 0);
508 pr_err("%s: error (%d)\n", __func__, ret);
510 mutex_lock(&icc_lock);
511 for (i = 0; i < path->num_nodes; i++) {
512 node = path->reqs[i].node;
513 hlist_del(&path->reqs[i].req_node);
514 if (!WARN_ON(!node->provider->users))
515 node->provider->users--;
517 mutex_unlock(&icc_lock);
521 EXPORT_SYMBOL_GPL(icc_put);
523 static struct icc_node *icc_node_create_nolock(int id)
525 struct icc_node *node;
527 /* check if node already exists */
528 node = node_find(id);
532 node = kzalloc(sizeof(*node), GFP_KERNEL);
534 return ERR_PTR(-ENOMEM);
536 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL);
538 WARN(1, "%s: couldn't get idr\n", __func__);
549 * icc_node_create() - create a node
552 * Return: icc_node pointer on success, or ERR_PTR() on error
554 struct icc_node *icc_node_create(int id)
556 struct icc_node *node;
558 mutex_lock(&icc_lock);
560 node = icc_node_create_nolock(id);
562 mutex_unlock(&icc_lock);
566 EXPORT_SYMBOL_GPL(icc_node_create);
569 * icc_node_destroy() - destroy a node
572 void icc_node_destroy(int id)
574 struct icc_node *node;
576 mutex_lock(&icc_lock);
578 node = node_find(id);
580 idr_remove(&icc_idr, node->id);
581 WARN_ON(!hlist_empty(&node->req_list));
584 mutex_unlock(&icc_lock);
588 EXPORT_SYMBOL_GPL(icc_node_destroy);
591 * icc_link_create() - create a link between two nodes
592 * @node: source node id
593 * @dst_id: destination node id
595 * Create a link between two nodes. The nodes might belong to different
596 * interconnect providers and the @dst_id node might not exist (if the
597 * provider driver has not probed yet). So just create the @dst_id node
598 * and when the actual provider driver is probed, the rest of the node
601 * Return: 0 on success, or an error code otherwise
603 int icc_link_create(struct icc_node *node, const int dst_id)
605 struct icc_node *dst;
606 struct icc_node **new;
612 mutex_lock(&icc_lock);
614 dst = node_find(dst_id);
616 dst = icc_node_create_nolock(dst_id);
624 new = krealloc(node->links,
625 (node->num_links + 1) * sizeof(*node->links),
633 node->links[node->num_links++] = dst;
636 mutex_unlock(&icc_lock);
640 EXPORT_SYMBOL_GPL(icc_link_create);
643 * icc_link_destroy() - destroy a link between two nodes
644 * @src: pointer to source node
645 * @dst: pointer to destination node
647 * Return: 0 on success, or an error code otherwise
649 int icc_link_destroy(struct icc_node *src, struct icc_node *dst)
651 struct icc_node **new;
655 if (IS_ERR_OR_NULL(src))
658 if (IS_ERR_OR_NULL(dst))
661 mutex_lock(&icc_lock);
663 for (slot = 0; slot < src->num_links; slot++)
664 if (src->links[slot] == dst)
667 if (WARN_ON(slot == src->num_links)) {
672 src->links[slot] = src->links[--src->num_links];
674 new = krealloc(src->links, src->num_links * sizeof(*src->links),
680 mutex_unlock(&icc_lock);
684 EXPORT_SYMBOL_GPL(icc_link_destroy);
687 * icc_node_add() - add interconnect node to interconnect provider
688 * @node: pointer to the interconnect node
689 * @provider: pointer to the interconnect provider
691 void icc_node_add(struct icc_node *node, struct icc_provider *provider)
693 mutex_lock(&icc_lock);
695 node->provider = provider;
696 list_add_tail(&node->node_list, &provider->nodes);
698 mutex_unlock(&icc_lock);
700 EXPORT_SYMBOL_GPL(icc_node_add);
703 * icc_node_del() - delete interconnect node from interconnect provider
704 * @node: pointer to the interconnect node
706 void icc_node_del(struct icc_node *node)
708 mutex_lock(&icc_lock);
710 list_del(&node->node_list);
712 mutex_unlock(&icc_lock);
714 EXPORT_SYMBOL_GPL(icc_node_del);
717 * icc_provider_add() - add a new interconnect provider
718 * @provider: the interconnect provider that will be added into topology
720 * Return: 0 on success, or an error code otherwise
722 int icc_provider_add(struct icc_provider *provider)
724 if (WARN_ON(!provider->set))
726 if (WARN_ON(!provider->xlate))
729 mutex_lock(&icc_lock);
731 INIT_LIST_HEAD(&provider->nodes);
732 list_add_tail(&provider->provider_list, &icc_providers);
734 mutex_unlock(&icc_lock);
736 dev_dbg(provider->dev, "interconnect provider added to topology\n");
740 EXPORT_SYMBOL_GPL(icc_provider_add);
743 * icc_provider_del() - delete previously added interconnect provider
744 * @provider: the interconnect provider that will be removed from topology
746 * Return: 0 on success, or an error code otherwise
748 int icc_provider_del(struct icc_provider *provider)
750 mutex_lock(&icc_lock);
751 if (provider->users) {
752 pr_warn("interconnect provider still has %d users\n",
754 mutex_unlock(&icc_lock);
758 if (!list_empty(&provider->nodes)) {
759 pr_warn("interconnect provider still has nodes\n");
760 mutex_unlock(&icc_lock);
764 list_del(&provider->provider_list);
765 mutex_unlock(&icc_lock);
769 EXPORT_SYMBOL_GPL(icc_provider_del);
771 static int __init icc_init(void)
773 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL);
774 debugfs_create_file("interconnect_summary", 0444,
775 icc_debugfs_dir, NULL, &icc_summary_fops);
779 static void __exit icc_exit(void)
781 debugfs_remove_recursive(icc_debugfs_dir);
783 module_init(icc_init);
784 module_exit(icc_exit);
787 MODULE_DESCRIPTION("Interconnect Driver Core");
788 MODULE_LICENSE("GPL v2");