drivers/net/ethernet/intel/igc/igc_tsn.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2019 Intel Corporation */
   3
   4 #include "igc.h"
   5 #include "igc_hw.h"
   6 #include "igc_tsn.h"
   7
   8 static bool is_any_launchtime(struct igc_adapter *adapter)
   9 {
  10         int i;
  11
  12         for (i = 0; i < adapter->num_tx_queues; i++) {
  13                 struct igc_ring *ring = adapter->tx_ring[i];
  14
  15                 if (ring->launchtime_enable)
  16                         return true;
  17         }
  18
  19         return false;
  20 }
  21
  22 static bool is_cbs_enabled(struct igc_adapter *adapter)
  23 {
  24         int i;
  25
  26         for (i = 0; i < adapter->num_tx_queues; i++) {
  27                 struct igc_ring *ring = adapter->tx_ring[i];
  28
  29                 if (ring->cbs_enable)
  30                         return true;
  31         }
  32
  33         return false;
  34 }
  35
  36 static unsigned int igc_tsn_new_flags(struct igc_adapter *adapter)
  37 {
  38         unsigned int new_flags = adapter->flags & ~IGC_FLAG_TSN_ANY_ENABLED;
  39
  40         if (adapter->taprio_offload_enable)
  41                 new_flags |= IGC_FLAG_TSN_QBV_ENABLED;
  42
  43         if (is_any_launchtime(adapter))
  44                 new_flags |= IGC_FLAG_TSN_QBV_ENABLED;
  45
  46         if (is_cbs_enabled(adapter))
  47                 new_flags |= IGC_FLAG_TSN_QAV_ENABLED;
  48
  49         if (adapter->strict_priority_enable)
  50                 new_flags |= IGC_FLAG_TSN_LEGACY_ENABLED;
  51
  52         return new_flags;
  53 }
  54
  55 static bool igc_tsn_is_tx_mode_in_tsn(struct igc_adapter *adapter)
  56 {
  57         struct igc_hw *hw = &adapter->hw;
  58
  59         return !!(rd32(IGC_TQAVCTRL) & IGC_TQAVCTRL_TRANSMIT_MODE_TSN);
  60 }
  61
  62 void igc_tsn_adjust_txtime_offset(struct igc_adapter *adapter)
  63 {
  64         struct igc_hw *hw = &adapter->hw;
  65         u16 txoffset;
  66
  67         if (!igc_tsn_is_tx_mode_in_tsn(adapter))
  68                 return;
  69
  70         switch (adapter->link_speed) {
  71         case SPEED_10:
  72                 txoffset = IGC_TXOFFSET_SPEED_10;
  73                 break;
  74         case SPEED_100:
  75                 txoffset = IGC_TXOFFSET_SPEED_100;
  76                 break;
  77         case SPEED_1000:
  78                 txoffset = IGC_TXOFFSET_SPEED_1000;
  79                 break;
  80         case SPEED_2500:
  81                 txoffset = IGC_TXOFFSET_SPEED_2500;
  82                 break;
  83         default:
  84                 txoffset = 0;
  85                 break;
  86         }
  87
  88         wr32(IGC_GTXOFFSET, txoffset);
  89 }
  90
  91 static void igc_tsn_restore_retx_default(struct igc_adapter *adapter)
  92 {
  93         struct igc_hw *hw = &adapter->hw;
  94         u32 retxctl;
  95
  96         retxctl = rd32(IGC_RETX_CTL) & IGC_RETX_CTL_WATERMARK_MASK;
  97         wr32(IGC_RETX_CTL, retxctl);
  98 }
  99
 100 bool igc_tsn_is_taprio_activated_by_user(struct igc_adapter *adapter)
 101 {
 102         struct igc_hw *hw = &adapter->hw;
 103
 104         return (rd32(IGC_BASET_H) || rd32(IGC_BASET_L)) &&
 105                 adapter->taprio_offload_enable;
 106 }
 107
 108 static void igc_tsn_tx_arb(struct igc_adapter *adapter, u16 *queue_per_tc)
 109 {
 110         struct igc_hw *hw = &adapter->hw;
 111         u32 txarb;
 112
 113         txarb = rd32(IGC_TXARB);
 114
 115         txarb &= ~(IGC_TXARB_TXQ_PRIO_0_MASK |
 116                    IGC_TXARB_TXQ_PRIO_1_MASK |
 117                    IGC_TXARB_TXQ_PRIO_2_MASK |
 118                    IGC_TXARB_TXQ_PRIO_3_MASK);
 119
 120         txarb |= IGC_TXARB_TXQ_PRIO_0(queue_per_tc[3]);
 121         txarb |= IGC_TXARB_TXQ_PRIO_1(queue_per_tc[2]);
 122         txarb |= IGC_TXARB_TXQ_PRIO_2(queue_per_tc[1]);
 123         txarb |= IGC_TXARB_TXQ_PRIO_3(queue_per_tc[0]);
 124
 125         wr32(IGC_TXARB, txarb);
 126 }
 127
 128 /* Returns the TSN specific registers to their default values after
 129  * the adapter is reset.
 130  */
 131 static int igc_tsn_disable_offload(struct igc_adapter *adapter)
 132 {
 133         u16 queue_per_tc[4] = { 3, 2, 1, 0 };
 134         struct igc_hw *hw = &adapter->hw;
 135         u32 tqavctrl;
 136         int i;
 137
 138         wr32(IGC_GTXOFFSET, 0);
 139         wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
 140         wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_DEFAULT);
 141
 142         if (igc_is_device_id_i226(hw))
 143                 igc_tsn_restore_retx_default(adapter);
 144
 145         tqavctrl = rd32(IGC_TQAVCTRL);
 146         tqavctrl &= ~(IGC_TQAVCTRL_TRANSMIT_MODE_TSN |
 147                       IGC_TQAVCTRL_ENHANCED_QAV | IGC_TQAVCTRL_FUTSCDDIS);
 148
 149         wr32(IGC_TQAVCTRL, tqavctrl);
 150
 151         for (i = 0; i < adapter->num_tx_queues; i++) {
 152                 wr32(IGC_TXQCTL(i), 0);
 153                 wr32(IGC_STQT(i), 0);
 154                 wr32(IGC_ENDQT(i), NSEC_PER_SEC);
 155         }
 156
 157         wr32(IGC_QBVCYCLET_S, 0);
 158         wr32(IGC_QBVCYCLET, NSEC_PER_SEC);
 159
 160         /* Reset mqprio TC configuration. */
 161         netdev_reset_tc(adapter->netdev);
 162
 163         /* Restore the default Tx arbitration: Priority 0 has the highest
 164          * priority and is assigned to queue 0 and so on and so forth.
 165          */
 166         igc_tsn_tx_arb(adapter, queue_per_tc);
 167
 168         adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED;
 169         adapter->flags &= ~IGC_FLAG_TSN_LEGACY_ENABLED;
 170
 171         return 0;
 172 }
 173
 174 /* To partially fix i226 HW errata, reduce MAC internal buffering from 192 Bytes
 175  * to 88 Bytes by setting RETX_CTL register using the recommendation from:
 176  * a) Ethernet Controller I225/I226 Specification Update Rev 2.1
 177  *    Item 9: TSN: Packet Transmission Might Cross the Qbv Window
 178  * b) I225/6 SW User Manual Rev 1.2.4: Section 8.11.5 Retry Buffer Control
 179  */
 180 static void igc_tsn_set_retx_qbvfullthreshold(struct igc_adapter *adapter)
 181 {
 182         struct igc_hw *hw = &adapter->hw;
 183         u32 retxctl, watermark;
 184
 185         retxctl = rd32(IGC_RETX_CTL);
 186         watermark = retxctl & IGC_RETX_CTL_WATERMARK_MASK;
 187         /* Set QBVFULLTH value using watermark and set QBVFULLEN */
 188         retxctl |= (watermark << IGC_RETX_CTL_QBVFULLTH_SHIFT) |
 189                    IGC_RETX_CTL_QBVFULLEN;
 190         wr32(IGC_RETX_CTL, retxctl);
 191 }
 192
 193 static int igc_tsn_enable_offload(struct igc_adapter *adapter)
 194 {
 195         struct igc_hw *hw = &adapter->hw;
 196         u32 tqavctrl, baset_l, baset_h;
 197         u32 sec, nsec, cycle;
 198         ktime_t base_time, systim;
 199         int i;
 200
 201         wr32(IGC_TSAUXC, 0);
 202         wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_TSN);
 203         wr32(IGC_TXPBS, IGC_TXPBSIZE_TSN);
 204
 205         if (igc_is_device_id_i226(hw))
 206                 igc_tsn_set_retx_qbvfullthreshold(adapter);
 207
 208         if (adapter->strict_priority_enable) {
 209                 int err;
 210
 211                 err = netdev_set_num_tc(adapter->netdev, adapter->num_tc);
 212                 if (err)
 213                         return err;
 214
 215                 for (i = 0; i < adapter->num_tc; i++) {
 216                         err = netdev_set_tc_queue(adapter->netdev, i, 1,
 217                                                   adapter->queue_per_tc[i]);
 218                         if (err)
 219                                 return err;
 220                 }
 221
 222                 /* In case the card is configured with less than four queues. */
 223                 for (; i < IGC_MAX_TX_QUEUES; i++)
 224                         adapter->queue_per_tc[i] = i;
 225
 226                 /* Configure queue priorities according to the user provided
 227                  * mapping.
 228                  */
 229                 igc_tsn_tx_arb(adapter, adapter->queue_per_tc);
 230
 231                 /* Enable legacy TSN mode which will do strict priority without
 232                  * any other TSN features.
 233                  */
 234                 tqavctrl = rd32(IGC_TQAVCTRL);
 235                 tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN;
 236                 tqavctrl &= ~IGC_TQAVCTRL_ENHANCED_QAV;
 237                 wr32(IGC_TQAVCTRL, tqavctrl);
 238
 239                 return 0;
 240         }
 241
 242         for (i = 0; i < adapter->num_tx_queues; i++) {
 243                 struct igc_ring *ring = adapter->tx_ring[i];
 244                 u32 txqctl = 0;
 245                 u16 cbs_value;
 246                 u32 tqavcc;
 247
 248                 wr32(IGC_STQT(i), ring->start_time);
 249                 wr32(IGC_ENDQT(i), ring->end_time);
 250
 251                 if (adapter->taprio_offload_enable) {
 252                         /* If taprio_offload_enable is set we are in "taprio"
 253                          * mode and we need to be strict about the
 254                          * cycles: only transmit a packet if it can be
 255                          * completed during that cycle.
 256                          *
 257                          * If taprio_offload_enable is NOT true when
 258                          * enabling TSN offload, the cycle should have
 259                          * no external effects, but is only used internally
 260                          * to adapt the base time register after a second
 261                          * has passed.
 262                          *
 263                          * Enabling strict mode in this case would
 264                          * unnecessarily prevent the transmission of
 265                          * certain packets (i.e. at the boundary of a
 266                          * second) and thus interfere with the launchtime
 267                          * feature that promises transmission at a
 268                          * certain point in time.
 269                          */
 270                         txqctl |= IGC_TXQCTL_STRICT_CYCLE |
 271                                 IGC_TXQCTL_STRICT_END;
 272                 }
 273
 274                 if (ring->launchtime_enable)
 275                         txqctl |= IGC_TXQCTL_QUEUE_MODE_LAUNCHT;
 276
 277                 /* Skip configuring CBS for Q2 and Q3 */
 278                 if (i > 1)
 279                         goto skip_cbs;
 280
 281                 if (ring->cbs_enable) {
 282                         if (i == 0)
 283                                 txqctl |= IGC_TXQCTL_QAV_SEL_CBS0;
 284                         else
 285                                 txqctl |= IGC_TXQCTL_QAV_SEL_CBS1;
 286
 287                         /* According to i225 datasheet section 7.5.2.7, we
 288                          * should set the 'idleSlope' field from TQAVCC
 289                          * register following the equation:
 290                          *
 291                          * value = link-speed   0x7736 * BW * 0.2
 292                          *         ---------- *  -----------------         (E1)
 293                          *          100Mbps            2.5
 294                          *
 295                          * Note that 'link-speed' is in Mbps.
 296                          *
 297                          * 'BW' is the percentage bandwidth out of full
 298                          * link speed which can be found with the
 299                          * following equation. Note that idleSlope here
 300                          * is the parameter from this function
 301                          * which is in kbps.
 302                          *
 303                          *     BW =     idleSlope
 304                          *          -----------------                      (E2)
 305                          *          link-speed * 1000
 306                          *
 307                          * That said, we can come up with a generic
 308                          * equation to calculate the value we should set
 309                          * it TQAVCC register by replacing 'BW' in E1 by E2.
 310                          * The resulting equation is:
 311                          *
 312                          * value = link-speed * 0x7736 * idleSlope * 0.2
 313                          *         -------------------------------------   (E3)
 314                          *             100 * 2.5 * link-speed * 1000
 315                          *
 316                          * 'link-speed' is present in both sides of the
 317                          * fraction so it is canceled out. The final
 318                          * equation is the following:
 319                          *
 320                          *     value = idleSlope * 61036
 321                          *             -----------------                   (E4)
 322                          *                  2500000
 323                          *
 324                          * NOTE: For i225, given the above, we can see
 325                          *       that idleslope is represented in
 326                          *       40.959433 kbps units by the value at
 327                          *       the TQAVCC register (2.5Gbps / 61036),
 328                          *       which reduces the granularity for
 329                          *       idleslope increments.
 330                          *
 331                          * In i225 controller, the sendSlope and loCredit
 332                          * parameters from CBS are not configurable
 333                          * by software so we don't do any
 334                          * 'controller configuration' in respect to
 335                          * these parameters.
 336                          */
 337                         cbs_value = DIV_ROUND_UP_ULL(ring->idleslope
 338                                                      * 61036ULL, 2500000);
 339
 340                         tqavcc = rd32(IGC_TQAVCC(i));
 341                         tqavcc &= ~IGC_TQAVCC_IDLESLOPE_MASK;
 342                         tqavcc |= cbs_value | IGC_TQAVCC_KEEP_CREDITS;
 343                         wr32(IGC_TQAVCC(i), tqavcc);
 344
 345                         wr32(IGC_TQAVHC(i),
 346                              0x80000000 + ring->hicredit * 0x7736);
 347                 } else {
 348                         /* Disable any CBS for the queue */
 349                         txqctl &= ~(IGC_TXQCTL_QAV_SEL_MASK);
 350
 351                         /* Set idleSlope to zero. */
 352                         tqavcc = rd32(IGC_TQAVCC(i));
 353                         tqavcc &= ~(IGC_TQAVCC_IDLESLOPE_MASK |
 354                                     IGC_TQAVCC_KEEP_CREDITS);
 355                         wr32(IGC_TQAVCC(i), tqavcc);
 356
 357                         /* Set hiCredit to zero. */
 358                         wr32(IGC_TQAVHC(i), 0);
 359                 }
 360 skip_cbs:
 361                 wr32(IGC_TXQCTL(i), txqctl);
 362         }
 363
 364         tqavctrl = rd32(IGC_TQAVCTRL) & ~IGC_TQAVCTRL_FUTSCDDIS;
 365
 366         tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV;
 367
 368         adapter->qbv_count++;
 369
 370         cycle = adapter->cycle_time;
 371         base_time = adapter->base_time;
 372
 373         nsec = rd32(IGC_SYSTIML);
 374         sec = rd32(IGC_SYSTIMH);
 375
 376         systim = ktime_set(sec, nsec);
 377         if (ktime_compare(systim, base_time) > 0) {
 378                 s64 n = div64_s64(ktime_sub_ns(systim, base_time), cycle);
 379
 380                 base_time = ktime_add_ns(base_time, (n + 1) * cycle);
 381         } else {
 382                 if (igc_is_device_id_i226(hw)) {
 383                         ktime_t adjust_time, expires_time;
 384
 385                        /* According to datasheet section 7.5.2.9.3.3, FutScdDis bit
 386                         * has to be configured before the cycle time and base time.
 387                         * Tx won't hang if a GCL is already running,
 388                         * so in this case we don't need to set FutScdDis.
 389                         */
 390                         if (!(rd32(IGC_BASET_H) || rd32(IGC_BASET_L)))
 391                                 tqavctrl |= IGC_TQAVCTRL_FUTSCDDIS;
 392
 393                         nsec = rd32(IGC_SYSTIML);
 394                         sec = rd32(IGC_SYSTIMH);
 395                         systim = ktime_set(sec, nsec);
 396
 397                         adjust_time = adapter->base_time;
 398                         expires_time = ktime_sub_ns(adjust_time, systim);
 399                         hrtimer_start(&adapter->hrtimer, expires_time, HRTIMER_MODE_REL);
 400                 }
 401         }
 402
 403         wr32(IGC_TQAVCTRL, tqavctrl);
 404
 405         wr32(IGC_QBVCYCLET_S, cycle);
 406         wr32(IGC_QBVCYCLET, cycle);
 407
 408         baset_h = div_s64_rem(base_time, NSEC_PER_SEC, &baset_l);
 409         wr32(IGC_BASET_H, baset_h);
 410
 411         /* In i226, Future base time is only supported when FutScdDis bit
 412          * is enabled and only active for re-configuration.
 413          * In this case, initialize the base time with zero to create
 414          * "re-configuration" scenario then only set the desired base time.
 415          */
 416         if (tqavctrl & IGC_TQAVCTRL_FUTSCDDIS)
 417                 wr32(IGC_BASET_L, 0);
 418         wr32(IGC_BASET_L, baset_l);
 419
 420         return 0;
 421 }
 422
 423 int igc_tsn_reset(struct igc_adapter *adapter)
 424 {
 425         unsigned int new_flags;
 426         int err = 0;
 427
 428         new_flags = igc_tsn_new_flags(adapter);
 429
 430         if (!(new_flags & IGC_FLAG_TSN_ANY_ENABLED))
 431                 return igc_tsn_disable_offload(adapter);
 432
 433         err = igc_tsn_enable_offload(adapter);
 434         if (err < 0)
 435                 return err;
 436
 437         adapter->flags = new_flags;
 438
 439         return err;
 440 }
 441
 442 static bool igc_tsn_will_tx_mode_change(struct igc_adapter *adapter)
 443 {
 444         bool any_tsn_enabled = !!(igc_tsn_new_flags(adapter) &
 445                                   IGC_FLAG_TSN_ANY_ENABLED);
 446
 447         return (any_tsn_enabled && !igc_tsn_is_tx_mode_in_tsn(adapter)) ||
 448                (!any_tsn_enabled && igc_tsn_is_tx_mode_in_tsn(adapter));
 449 }
 450
 451 int igc_tsn_offload_apply(struct igc_adapter *adapter)
 452 {
 453         /* Per I225/6 HW Design Section 7.5.2.1 guideline, if tx mode change
 454          * from legacy->tsn or tsn->legacy, then reset adapter is needed.
 455          */
 456         if (netif_running(adapter->netdev) &&
 457             igc_tsn_will_tx_mode_change(adapter)) {
 458                 schedule_work(&adapter->reset_task);
 459                 return 0;
 460         }
 461
 462         igc_tsn_reset(adapter);
 463
 464         return 0;
 465 }