drivers/net/ipa/ipa_endpoint.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
   4  * Copyright (C) 2019-2023 Linaro Ltd.
   5  */
   6
   7 #include <linux/types.h>
   8 #include <linux/device.h>
   9 #include <linux/slab.h>
  10 #include <linux/bitfield.h>
  11 #include <linux/if_rmnet.h>
  12 #include <linux/dma-direction.h>
  13
  14 #include "gsi.h"
  15 #include "gsi_trans.h"
  16 #include "ipa.h"
  17 #include "ipa_data.h"
  18 #include "ipa_endpoint.h"
  19 #include "ipa_cmd.h"
  20 #include "ipa_mem.h"
  21 #include "ipa_modem.h"
  22 #include "ipa_table.h"
  23 #include "ipa_gsi.h"
  24 #include "ipa_power.h"
  25
  26 /* Hardware is told about receive buffers once a "batch" has been queued */
  27 #define IPA_REPLENISH_BATCH     16              /* Must be non-zero */
  28
  29 /* The amount of RX buffer space consumed by standard skb overhead */
  30 #define IPA_RX_BUFFER_OVERHEAD  (PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0))
  31
  32 /* Where to find the QMAP mux_id for a packet within modem-supplied metadata */
  33 #define IPA_ENDPOINT_QMAP_METADATA_MASK         0x000000ff /* host byte order */
  34
  35 #define IPA_ENDPOINT_RESET_AGGR_RETRY_MAX       3
  36
  37 /** enum ipa_status_opcode - IPA status opcode field hardware values */
  38 enum ipa_status_opcode {                                /* *Not* a bitmask */
  39         IPA_STATUS_OPCODE_PACKET                = 1,
  40         IPA_STATUS_OPCODE_NEW_RULE_PACKET       = 2,
  41         IPA_STATUS_OPCODE_DROPPED_PACKET        = 4,
  42         IPA_STATUS_OPCODE_SUSPENDED_PACKET      = 8,
  43         IPA_STATUS_OPCODE_LOG                   = 16,
  44         IPA_STATUS_OPCODE_DCMP                  = 32,
  45         IPA_STATUS_OPCODE_PACKET_2ND_PASS       = 64,
  46 };
  47
  48 /** enum ipa_status_exception - IPA status exception field hardware values */
  49 enum ipa_status_exception {                             /* *Not* a bitmask */
  50         /* 0 means no exception */
  51         IPA_STATUS_EXCEPTION_DEAGGR             = 1,
  52         IPA_STATUS_EXCEPTION_IPTYPE             = 4,
  53         IPA_STATUS_EXCEPTION_PACKET_LENGTH      = 8,
  54         IPA_STATUS_EXCEPTION_FRAG_RULE_MISS     = 16,
  55         IPA_STATUS_EXCEPTION_SW_FILTER          = 32,
  56         IPA_STATUS_EXCEPTION_NAT                = 64,           /* IPv4 */
  57         IPA_STATUS_EXCEPTION_IPV6_CONN_TRACK    = 64,           /* IPv6 */
  58         IPA_STATUS_EXCEPTION_UC                 = 128,
  59         IPA_STATUS_EXCEPTION_INVALID_ENDPOINT   = 129,
  60         IPA_STATUS_EXCEPTION_HEADER_INSERT      = 136,
  61         IPA_STATUS_EXCEPTION_CHEKCSUM           = 229,
  62 };
  63
  64 /** enum ipa_status_mask - IPA status mask field bitmask hardware values */
  65 enum ipa_status_mask {
  66         IPA_STATUS_MASK_FRAG_PROCESS            = BIT(0),
  67         IPA_STATUS_MASK_FILT_PROCESS            = BIT(1),
  68         IPA_STATUS_MASK_NAT_PROCESS             = BIT(2),
  69         IPA_STATUS_MASK_ROUTE_PROCESS           = BIT(3),
  70         IPA_STATUS_MASK_TAG_VALID               = BIT(4),
  71         IPA_STATUS_MASK_FRAGMENT                = BIT(5),
  72         IPA_STATUS_MASK_FIRST_FRAGMENT          = BIT(6),
  73         IPA_STATUS_MASK_V4                      = BIT(7),
  74         IPA_STATUS_MASK_CKSUM_PROCESS           = BIT(8),
  75         IPA_STATUS_MASK_AGGR_PROCESS            = BIT(9),
  76         IPA_STATUS_MASK_DEST_EOT                = BIT(10),
  77         IPA_STATUS_MASK_DEAGGR_PROCESS          = BIT(11),
  78         IPA_STATUS_MASK_DEAGG_FIRST             = BIT(12),
  79         IPA_STATUS_MASK_SRC_EOT                 = BIT(13),
  80         IPA_STATUS_MASK_PREV_EOT                = BIT(14),
  81         IPA_STATUS_MASK_BYTE_LIMIT              = BIT(15),
  82 };
  83
  84 /* Special IPA filter/router rule field value indicating "rule miss" */
  85 #define IPA_STATUS_RULE_MISS    0x3ff   /* 10-bit filter/router rule fields */
  86
  87 /** The IPA status nat_type field uses enum ipa_nat_type hardware values */
  88
  89 /* enum ipa_status_field_id - IPA packet status structure field identifiers */
  90 enum ipa_status_field_id {
  91         STATUS_OPCODE,                  /* enum ipa_status_opcode */
  92         STATUS_EXCEPTION,               /* enum ipa_status_exception */
  93         STATUS_MASK,                    /* enum ipa_status_mask (bitmask) */
  94         STATUS_LENGTH,
  95         STATUS_SRC_ENDPOINT,
  96         STATUS_DST_ENDPOINT,
  97         STATUS_METADATA,
  98         STATUS_FILTER_LOCAL,            /* Boolean */
  99         STATUS_FILTER_HASH,             /* Boolean */
 100         STATUS_FILTER_GLOBAL,           /* Boolean */
 101         STATUS_FILTER_RETAIN,           /* Boolean */
 102         STATUS_FILTER_RULE_INDEX,
 103         STATUS_ROUTER_LOCAL,            /* Boolean */
 104         STATUS_ROUTER_HASH,             /* Boolean */
 105         STATUS_UCP,                     /* Boolean */
 106         STATUS_ROUTER_TABLE,
 107         STATUS_ROUTER_RULE_INDEX,
 108         STATUS_NAT_HIT,                 /* Boolean */
 109         STATUS_NAT_INDEX,
 110         STATUS_NAT_TYPE,                /* enum ipa_nat_type */
 111         STATUS_TAG_LOW32,               /* Low-order 32 bits of 48-bit tag */
 112         STATUS_TAG_HIGH16,              /* High-order 16 bits of 48-bit tag */
 113         STATUS_SEQUENCE,
 114         STATUS_TIME_OF_DAY,
 115         STATUS_HEADER_LOCAL,            /* Boolean */
 116         STATUS_HEADER_OFFSET,
 117         STATUS_FRAG_HIT,                /* Boolean */
 118         STATUS_FRAG_RULE_INDEX,
 119 };
 120
 121 /* Size in bytes of an IPA packet status structure */
 122 #define IPA_STATUS_SIZE                 sizeof(__le32[8])
 123
 124 /* IPA status structure decoder; looks up field values for a structure */
 125 static u32 ipa_status_extract(struct ipa *ipa, const void *data,
 126                               enum ipa_status_field_id field)
 127 {
 128         enum ipa_version version = ipa->version;
 129         const __le32 *word = data;
 130
 131         switch (field) {
 132         case STATUS_OPCODE:
 133                 return le32_get_bits(word[0], GENMASK(7, 0));
 134         case STATUS_EXCEPTION:
 135                 return le32_get_bits(word[0], GENMASK(15, 8));
 136         case STATUS_MASK:
 137                 return le32_get_bits(word[0], GENMASK(31, 16));
 138         case STATUS_LENGTH:
 139                 return le32_get_bits(word[1], GENMASK(15, 0));
 140         case STATUS_SRC_ENDPOINT:
 141                 if (version < IPA_VERSION_5_0)
 142                         return le32_get_bits(word[1], GENMASK(20, 16));
 143                 return le32_get_bits(word[1], GENMASK(23, 16));
 144         /* Status word 1, bits 21-23 are reserved (not IPA v5.0+) */
 145         /* Status word 1, bits 24-26 are reserved (IPA v5.0+) */
 146         case STATUS_DST_ENDPOINT:
 147                 if (version < IPA_VERSION_5_0)
 148                         return le32_get_bits(word[1], GENMASK(28, 24));
 149                 return le32_get_bits(word[7], GENMASK(23, 16));
 150         /* Status word 1, bits 29-31 are reserved */
 151         case STATUS_METADATA:
 152                 return le32_to_cpu(word[2]);
 153         case STATUS_FILTER_LOCAL:
 154                 return le32_get_bits(word[3], GENMASK(0, 0));
 155         case STATUS_FILTER_HASH:
 156                 return le32_get_bits(word[3], GENMASK(1, 1));
 157         case STATUS_FILTER_GLOBAL:
 158                 return le32_get_bits(word[3], GENMASK(2, 2));
 159         case STATUS_FILTER_RETAIN:
 160                 return le32_get_bits(word[3], GENMASK(3, 3));
 161         case STATUS_FILTER_RULE_INDEX:
 162                 return le32_get_bits(word[3], GENMASK(13, 4));
 163         /* ROUTER_TABLE is in word 3, bits 14-21 (IPA v5.0+) */
 164         case STATUS_ROUTER_LOCAL:
 165                 if (version < IPA_VERSION_5_0)
 166                         return le32_get_bits(word[3], GENMASK(14, 14));
 167                 return le32_get_bits(word[1], GENMASK(27, 27));
 168         case STATUS_ROUTER_HASH:
 169                 if (version < IPA_VERSION_5_0)
 170                         return le32_get_bits(word[3], GENMASK(15, 15));
 171                 return le32_get_bits(word[1], GENMASK(28, 28));
 172         case STATUS_UCP:
 173                 if (version < IPA_VERSION_5_0)
 174                         return le32_get_bits(word[3], GENMASK(16, 16));
 175                 return le32_get_bits(word[7], GENMASK(31, 31));
 176         case STATUS_ROUTER_TABLE:
 177                 if (version < IPA_VERSION_5_0)
 178                         return le32_get_bits(word[3], GENMASK(21, 17));
 179                 return le32_get_bits(word[3], GENMASK(21, 14));
 180         case STATUS_ROUTER_RULE_INDEX:
 181                 return le32_get_bits(word[3], GENMASK(31, 22));
 182         case STATUS_NAT_HIT:
 183                 return le32_get_bits(word[4], GENMASK(0, 0));
 184         case STATUS_NAT_INDEX:
 185                 return le32_get_bits(word[4], GENMASK(13, 1));
 186         case STATUS_NAT_TYPE:
 187                 return le32_get_bits(word[4], GENMASK(15, 14));
 188         case STATUS_TAG_LOW32:
 189                 return le32_get_bits(word[4], GENMASK(31, 16)) |
 190                         (le32_get_bits(word[5], GENMASK(15, 0)) << 16);
 191         case STATUS_TAG_HIGH16:
 192                 return le32_get_bits(word[5], GENMASK(31, 16));
 193         case STATUS_SEQUENCE:
 194                 return le32_get_bits(word[6], GENMASK(7, 0));
 195         case STATUS_TIME_OF_DAY:
 196                 return le32_get_bits(word[6], GENMASK(31, 8));
 197         case STATUS_HEADER_LOCAL:
 198                 return le32_get_bits(word[7], GENMASK(0, 0));
 199         case STATUS_HEADER_OFFSET:
 200                 return le32_get_bits(word[7], GENMASK(10, 1));
 201         case STATUS_FRAG_HIT:
 202                 return le32_get_bits(word[7], GENMASK(11, 11));
 203         case STATUS_FRAG_RULE_INDEX:
 204                 return le32_get_bits(word[7], GENMASK(15, 12));
 205         /* Status word 7, bits 16-30 are reserved */
 206         /* Status word 7, bit 31 is reserved (not IPA v5.0+) */
 207         default:
 208                 WARN(true, "%s: bad field_id %u\n", __func__, field);
 209                 return 0;
 210         }
 211 }
 212
 213 /* Compute the aggregation size value to use for a given buffer size */
 214 static u32 ipa_aggr_size_kb(u32 rx_buffer_size, bool aggr_hard_limit)
 215 {
 216         /* A hard aggregation limit will not be crossed; aggregation closes
 217          * if saving incoming data would cross the hard byte limit boundary.
 218          *
 219          * With a soft limit, aggregation closes *after* the size boundary
 220          * has been crossed.  In that case the limit must leave enough space
 221          * after that limit to receive a full MTU of data plus overhead.
 222          */
 223         if (!aggr_hard_limit)
 224                 rx_buffer_size -= IPA_MTU + IPA_RX_BUFFER_OVERHEAD;
 225
 226         /* The byte limit is encoded as a number of kilobytes */
 227
 228         return rx_buffer_size / SZ_1K;
 229 }
 230
 231 static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,
 232                             const struct ipa_gsi_endpoint_data *all_data,
 233                             const struct ipa_gsi_endpoint_data *data)
 234 {
 235         const struct ipa_gsi_endpoint_data *other_data;
 236         struct device *dev = &ipa->pdev->dev;
 237         enum ipa_endpoint_name other_name;
 238
 239         if (ipa_gsi_endpoint_data_empty(data))
 240                 return true;
 241
 242         if (!data->toward_ipa) {
 243                 const struct ipa_endpoint_rx *rx_config;
 244                 const struct reg *reg;
 245                 u32 buffer_size;
 246                 u32 aggr_size;
 247                 u32 limit;
 248
 249                 if (data->endpoint.filter_support) {
 250                         dev_err(dev, "filtering not supported for "
 251                                         "RX endpoint %u\n",
 252                                 data->endpoint_id);
 253                         return false;
 254                 }
 255
 256                 /* Nothing more to check for non-AP RX */
 257                 if (data->ee_id != GSI_EE_AP)
 258                         return true;
 259
 260                 rx_config = &data->endpoint.config.rx;
 261
 262                 /* The buffer size must hold an MTU plus overhead */
 263                 buffer_size = rx_config->buffer_size;
 264                 limit = IPA_MTU + IPA_RX_BUFFER_OVERHEAD;
 265                 if (buffer_size < limit) {
 266                         dev_err(dev, "RX buffer size too small for RX endpoint %u (%u < %u)\n",
 267                                 data->endpoint_id, buffer_size, limit);
 268                         return false;
 269                 }
 270
 271                 if (!data->endpoint.config.aggregation) {
 272                         bool result = true;
 273
 274                         /* No aggregation; check for bogus aggregation data */
 275                         if (rx_config->aggr_time_limit) {
 276                                 dev_err(dev,
 277                                         "time limit with no aggregation for RX endpoint %u\n",
 278                                         data->endpoint_id);
 279                                 result = false;
 280                         }
 281
 282                         if (rx_config->aggr_hard_limit) {
 283                                 dev_err(dev, "hard limit with no aggregation for RX endpoint %u\n",
 284                                         data->endpoint_id);
 285                                 result = false;
 286                         }
 287
 288                         if (rx_config->aggr_close_eof) {
 289                                 dev_err(dev, "close EOF with no aggregation for RX endpoint %u\n",
 290                                         data->endpoint_id);
 291                                 result = false;
 292                         }
 293
 294                         return result;  /* Nothing more to check */
 295                 }
 296
 297                 /* For an endpoint supporting receive aggregation, the byte
 298                  * limit defines the point at which aggregation closes.  This
 299                  * check ensures the receive buffer size doesn't result in a
 300                  * limit that exceeds what's representable in the aggregation
 301                  * byte limit field.
 302                  */
 303                 aggr_size = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD,
 304                                              rx_config->aggr_hard_limit);
 305                 reg = ipa_reg(ipa, ENDP_INIT_AGGR);
 306
 307                 limit = reg_field_max(reg, BYTE_LIMIT);
 308                 if (aggr_size > limit) {
 309                         dev_err(dev, "aggregated size too large for RX endpoint %u (%u KB > %u KB)\n",
 310                                 data->endpoint_id, aggr_size, limit);
 311
 312                         return false;
 313                 }
 314
 315                 return true;    /* Nothing more to check for RX */
 316         }
 317
 318         /* Starting with IPA v4.5 sequencer replication is obsolete */
 319         if (ipa->version >= IPA_VERSION_4_5) {
 320                 if (data->endpoint.config.tx.seq_rep_type) {
 321                         dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n",
 322                                 data->endpoint_id);
 323                         return false;
 324                 }
 325         }
 326
 327         if (data->endpoint.config.status_enable) {
 328                 other_name = data->endpoint.config.tx.status_endpoint;
 329                 if (other_name >= count) {
 330                         dev_err(dev, "status endpoint name %u out of range "
 331                                         "for endpoint %u\n",
 332                                 other_name, data->endpoint_id);
 333                         return false;
 334                 }
 335
 336                 /* Status endpoint must be defined... */
 337                 other_data = &all_data[other_name];
 338                 if (ipa_gsi_endpoint_data_empty(other_data)) {
 339                         dev_err(dev, "DMA endpoint name %u undefined "
 340                                         "for endpoint %u\n",
 341                                 other_name, data->endpoint_id);
 342                         return false;
 343                 }
 344
 345                 /* ...and has to be an RX endpoint... */
 346                 if (other_data->toward_ipa) {
 347                         dev_err(dev,
 348                                 "status endpoint for endpoint %u not RX\n",
 349                                 data->endpoint_id);
 350                         return false;
 351                 }
 352
 353                 /* ...and if it's to be an AP endpoint... */
 354                 if (other_data->ee_id == GSI_EE_AP) {
 355                         /* ...make sure it has status enabled. */
 356                         if (!other_data->endpoint.config.status_enable) {
 357                                 dev_err(dev,
 358                                         "status not enabled for endpoint %u\n",
 359                                         other_data->endpoint_id);
 360                                 return false;
 361                         }
 362                 }
 363         }
 364
 365         if (data->endpoint.config.dma_mode) {
 366                 other_name = data->endpoint.config.dma_endpoint;
 367                 if (other_name >= count) {
 368                         dev_err(dev, "DMA endpoint name %u out of range "
 369                                         "for endpoint %u\n",
 370                                 other_name, data->endpoint_id);
 371                         return false;
 372                 }
 373
 374                 other_data = &all_data[other_name];
 375                 if (ipa_gsi_endpoint_data_empty(other_data)) {
 376                         dev_err(dev, "DMA endpoint name %u undefined "
 377                                         "for endpoint %u\n",
 378                                 other_name, data->endpoint_id);
 379                         return false;
 380                 }
 381         }
 382
 383         return true;
 384 }
 385
 386 /* Validate endpoint configuration data.  Return max defined endpoint ID */
 387 static u32 ipa_endpoint_max(struct ipa *ipa, u32 count,
 388                             const struct ipa_gsi_endpoint_data *data)
 389 {
 390         const struct ipa_gsi_endpoint_data *dp = data;
 391         struct device *dev = &ipa->pdev->dev;
 392         enum ipa_endpoint_name name;
 393         u32 max;
 394
 395         if (count > IPA_ENDPOINT_COUNT) {
 396                 dev_err(dev, "too many endpoints specified (%u > %u)\n",
 397                         count, IPA_ENDPOINT_COUNT);
 398                 return 0;
 399         }
 400
 401         /* Make sure needed endpoints have defined data */
 402         if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_COMMAND_TX])) {
 403                 dev_err(dev, "command TX endpoint not defined\n");
 404                 return 0;
 405         }
 406         if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_LAN_RX])) {
 407                 dev_err(dev, "LAN RX endpoint not defined\n");
 408                 return 0;
 409         }
 410         if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_MODEM_TX])) {
 411                 dev_err(dev, "AP->modem TX endpoint not defined\n");
 412                 return 0;
 413         }
 414         if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_MODEM_RX])) {
 415                 dev_err(dev, "AP<-modem RX endpoint not defined\n");
 416                 return 0;
 417         }
 418
 419         max = 0;
 420         for (name = 0; name < count; name++, dp++) {
 421                 if (!ipa_endpoint_data_valid_one(ipa, count, data, dp))
 422                         return 0;
 423                 max = max_t(u32, max, dp->endpoint_id);
 424         }
 425
 426         return max;
 427 }
 428
 429 /* Allocate a transaction to use on a non-command endpoint */
 430 static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint,
 431                                                   u32 tre_count)
 432 {
 433         struct gsi *gsi = &endpoint->ipa->gsi;
 434         u32 channel_id = endpoint->channel_id;
 435         enum dma_data_direction direction;
 436
 437         direction = endpoint->toward_ipa ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 438
 439         return gsi_channel_trans_alloc(gsi, channel_id, tre_count, direction);
 440 }
 441
 442 /* suspend_delay represents suspend for RX, delay for TX endpoints.
 443  * Note that suspend is not supported starting with IPA v4.0, and
 444  * delay mode should not be used starting with IPA v4.2.
 445  */
 446 static bool
 447 ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay)
 448 {
 449         struct ipa *ipa = endpoint->ipa;
 450         const struct reg *reg;
 451         u32 field_id;
 452         u32 offset;
 453         bool state;
 454         u32 mask;
 455         u32 val;
 456
 457         if (endpoint->toward_ipa)
 458                 WARN_ON(ipa->version >= IPA_VERSION_4_2);
 459         else
 460                 WARN_ON(ipa->version >= IPA_VERSION_4_0);
 461
 462         reg = ipa_reg(ipa, ENDP_INIT_CTRL);
 463         offset = reg_n_offset(reg, endpoint->endpoint_id);
 464         val = ioread32(ipa->reg_virt + offset);
 465
 466         field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND;
 467         mask = reg_bit(reg, field_id);
 468
 469         state = !!(val & mask);
 470
 471         /* Don't bother if it's already in the requested state */
 472         if (suspend_delay != state) {
 473                 val ^= mask;
 474                 iowrite32(val, ipa->reg_virt + offset);
 475         }
 476
 477         return state;
 478 }
 479
 480 /* We don't care what the previous state was for delay mode */
 481 static void
 482 ipa_endpoint_program_delay(struct ipa_endpoint *endpoint, bool enable)
 483 {
 484         /* Delay mode should not be used for IPA v4.2+ */
 485         WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2);
 486         WARN_ON(!endpoint->toward_ipa);
 487
 488         (void)ipa_endpoint_init_ctrl(endpoint, enable);
 489 }
 490
 491 static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint)
 492 {
 493         u32 endpoint_id = endpoint->endpoint_id;
 494         struct ipa *ipa = endpoint->ipa;
 495         u32 unit = endpoint_id / 32;
 496         const struct reg *reg;
 497         u32 val;
 498
 499         WARN_ON(!test_bit(endpoint_id, ipa->available));
 500
 501         reg = ipa_reg(ipa, STATE_AGGR_ACTIVE);
 502         val = ioread32(ipa->reg_virt + reg_n_offset(reg, unit));
 503
 504         return !!(val & BIT(endpoint_id % 32));
 505 }
 506
 507 static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint)
 508 {
 509         u32 endpoint_id = endpoint->endpoint_id;
 510         u32 mask = BIT(endpoint_id % 32);
 511         struct ipa *ipa = endpoint->ipa;
 512         u32 unit = endpoint_id / 32;
 513         const struct reg *reg;
 514
 515         WARN_ON(!test_bit(endpoint_id, ipa->available));
 516
 517         reg = ipa_reg(ipa, AGGR_FORCE_CLOSE);
 518         iowrite32(mask, ipa->reg_virt + reg_n_offset(reg, unit));
 519 }
 520
 521 /**
 522  * ipa_endpoint_suspend_aggr() - Emulate suspend interrupt
 523  * @endpoint:   Endpoint on which to emulate a suspend
 524  *
 525  *  Emulate suspend IPA interrupt to unsuspend an endpoint suspended
 526  *  with an open aggregation frame.  This is to work around a hardware
 527  *  issue in IPA version 3.5.1 where the suspend interrupt will not be
 528  *  generated when it should be.
 529  */
 530 static void ipa_endpoint_suspend_aggr(struct ipa_endpoint *endpoint)
 531 {
 532         struct ipa *ipa = endpoint->ipa;
 533
 534         if (!endpoint->config.aggregation)
 535                 return;
 536
 537         /* Nothing to do if the endpoint doesn't have aggregation open */
 538         if (!ipa_endpoint_aggr_active(endpoint))
 539                 return;
 540
 541         /* Force close aggregation */
 542         ipa_endpoint_force_close(endpoint);
 543
 544         ipa_interrupt_simulate_suspend(ipa->interrupt);
 545 }
 546
 547 /* Returns previous suspend state (true means suspend was enabled) */
 548 static bool
 549 ipa_endpoint_program_suspend(struct ipa_endpoint *endpoint, bool enable)
 550 {
 551         bool suspended;
 552
 553         if (endpoint->ipa->version >= IPA_VERSION_4_0)
 554                 return enable;  /* For IPA v4.0+, no change made */
 555
 556         WARN_ON(endpoint->toward_ipa);
 557
 558         suspended = ipa_endpoint_init_ctrl(endpoint, enable);
 559
 560         /* A client suspended with an open aggregation frame will not
 561          * generate a SUSPEND IPA interrupt.  If enabling suspend, have
 562          * ipa_endpoint_suspend_aggr() handle this.
 563          */
 564         if (enable && !suspended)
 565                 ipa_endpoint_suspend_aggr(endpoint);
 566
 567         return suspended;
 568 }
 569
 570 /* Put all modem RX endpoints into suspend mode, and stop transmission
 571  * on all modem TX endpoints.  Prior to IPA v4.2, endpoint DELAY mode is
 572  * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow
 573  * control instead.
 574  */
 575 void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable)
 576 {
 577         u32 endpoint_id = 0;
 578
 579         while (endpoint_id < ipa->endpoint_count) {
 580                 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++];
 581
 582                 if (endpoint->ee_id != GSI_EE_MODEM)
 583                         continue;
 584
 585                 if (!endpoint->toward_ipa)
 586                         (void)ipa_endpoint_program_suspend(endpoint, enable);
 587                 else if (ipa->version < IPA_VERSION_4_2)
 588                         ipa_endpoint_program_delay(endpoint, enable);
 589                 else
 590                         gsi_modem_channel_flow_control(&ipa->gsi,
 591                                                        endpoint->channel_id,
 592                                                        enable);
 593         }
 594 }
 595
 596 /* Reset all modem endpoints to use the default exception endpoint */
 597 int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)
 598 {
 599         struct gsi_trans *trans;
 600         u32 endpoint_id;
 601         u32 count;
 602
 603         /* We need one command per modem TX endpoint, plus the commands
 604          * that clear the pipeline.
 605          */
 606         count = ipa->modem_tx_count + ipa_cmd_pipeline_clear_count();
 607         trans = ipa_cmd_trans_alloc(ipa, count);
 608         if (!trans) {
 609                 dev_err(&ipa->pdev->dev,
 610                         "no transaction to reset modem exception endpoints\n");
 611                 return -EBUSY;
 612         }
 613
 614         for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) {
 615                 struct ipa_endpoint *endpoint;
 616                 const struct reg *reg;
 617                 u32 offset;
 618
 619                 /* We only reset modem TX endpoints */
 620                 endpoint = &ipa->endpoint[endpoint_id];
 621                 if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa))
 622                         continue;
 623
 624                 reg = ipa_reg(ipa, ENDP_STATUS);
 625                 offset = reg_n_offset(reg, endpoint_id);
 626
 627                 /* Value written is 0, and all bits are updated.  That
 628                  * means status is disabled on the endpoint, and as a
 629                  * result all other fields in the register are ignored.
 630                  */
 631                 ipa_cmd_register_write_add(trans, offset, 0, ~0, false);
 632         }
 633
 634         ipa_cmd_pipeline_clear_add(trans);
 635
 636         gsi_trans_commit_wait(trans);
 637
 638         ipa_cmd_pipeline_clear_wait(ipa);
 639
 640         return 0;
 641 }
 642
 643 static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint)
 644 {
 645         u32 endpoint_id = endpoint->endpoint_id;
 646         struct ipa *ipa = endpoint->ipa;
 647         enum ipa_cs_offload_en enabled;
 648         const struct reg *reg;
 649         u32 val = 0;
 650
 651         reg = ipa_reg(ipa, ENDP_INIT_CFG);
 652         /* FRAG_OFFLOAD_EN is 0 */
 653         if (endpoint->config.checksum) {
 654                 enum ipa_version version = ipa->version;
 655
 656                 if (endpoint->toward_ipa) {
 657                         u32 off;
 658
 659                         /* Checksum header offset is in 4-byte units */
 660                         off = sizeof(struct rmnet_map_header) / sizeof(u32);
 661                         val |= reg_encode(reg, CS_METADATA_HDR_OFFSET, off);
 662
 663                         enabled = version < IPA_VERSION_4_5
 664                                         ? IPA_CS_OFFLOAD_UL
 665                                         : IPA_CS_OFFLOAD_INLINE;
 666                 } else {
 667                         enabled = version < IPA_VERSION_4_5
 668                                         ? IPA_CS_OFFLOAD_DL
 669                                         : IPA_CS_OFFLOAD_INLINE;
 670                 }
 671         } else {
 672                 enabled = IPA_CS_OFFLOAD_NONE;
 673         }
 674         val |= reg_encode(reg, CS_OFFLOAD_EN, enabled);
 675         /* CS_GEN_QMB_MASTER_SEL is 0 */
 676
 677         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 678 }
 679
 680 static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint)
 681 {
 682         u32 endpoint_id = endpoint->endpoint_id;
 683         struct ipa *ipa = endpoint->ipa;
 684         const struct reg *reg;
 685         u32 val;
 686
 687         if (!endpoint->toward_ipa)
 688                 return;
 689
 690         reg = ipa_reg(ipa, ENDP_INIT_NAT);
 691         val = reg_encode(reg, NAT_EN, IPA_NAT_TYPE_BYPASS);
 692
 693         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 694 }
 695
 696 static u32
 697 ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint)
 698 {
 699         u32 header_size = sizeof(struct rmnet_map_header);
 700
 701         /* Without checksum offload, we just have the MAP header */
 702         if (!endpoint->config.checksum)
 703                 return header_size;
 704
 705         if (version < IPA_VERSION_4_5) {
 706                 /* Checksum header inserted for AP TX endpoints only */
 707                 if (endpoint->toward_ipa)
 708                         header_size += sizeof(struct rmnet_map_ul_csum_header);
 709         } else {
 710                 /* Checksum header is used in both directions */
 711                 header_size += sizeof(struct rmnet_map_v5_csum_header);
 712         }
 713
 714         return header_size;
 715 }
 716
 717 /* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */
 718 static u32 ipa_header_size_encode(enum ipa_version version,
 719                                   const struct reg *reg, u32 header_size)
 720 {
 721         u32 field_max = reg_field_max(reg, HDR_LEN);
 722         u32 val;
 723
 724         /* We know field_max can be used as a mask (2^n - 1) */
 725         val = reg_encode(reg, HDR_LEN, header_size & field_max);
 726         if (version < IPA_VERSION_4_5) {
 727                 WARN_ON(header_size > field_max);
 728                 return val;
 729         }
 730
 731         /* IPA v4.5 adds a few more most-significant bits */
 732         header_size >>= hweight32(field_max);
 733         WARN_ON(header_size > reg_field_max(reg, HDR_LEN_MSB));
 734         val |= reg_encode(reg, HDR_LEN_MSB, header_size);
 735
 736         return val;
 737 }
 738
 739 /* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */
 740 static u32 ipa_metadata_offset_encode(enum ipa_version version,
 741                                       const struct reg *reg, u32 offset)
 742 {
 743         u32 field_max = reg_field_max(reg, HDR_OFST_METADATA);
 744         u32 val;
 745
 746         /* We know field_max can be used as a mask (2^n - 1) */
 747         val = reg_encode(reg, HDR_OFST_METADATA, offset);
 748         if (version < IPA_VERSION_4_5) {
 749                 WARN_ON(offset > field_max);
 750                 return val;
 751         }
 752
 753         /* IPA v4.5 adds a few more most-significant bits */
 754         offset >>= hweight32(field_max);
 755         WARN_ON(offset > reg_field_max(reg, HDR_OFST_METADATA_MSB));
 756         val |= reg_encode(reg, HDR_OFST_METADATA_MSB, offset);
 757
 758         return val;
 759 }
 760
 761 /**
 762  * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register
 763  * @endpoint:   Endpoint pointer
 764  *
 765  * We program QMAP endpoints so each packet received is preceded by a QMAP
 766  * header structure.  The QMAP header contains a 1-byte mux_id and 2-byte
 767  * packet size field, and we have the IPA hardware populate both for each
 768  * received packet.  The header is configured (in the HDR_EXT register)
 769  * to use big endian format.
 770  *
 771  * The packet size is written into the QMAP header's pkt_len field.  That
 772  * location is defined here using the HDR_OFST_PKT_SIZE field.
 773  *
 774  * The mux_id comes from a 4-byte metadata value supplied with each packet
 775  * by the modem.  It is *not* a QMAP header, but it does contain the mux_id
 776  * value that we want, in its low-order byte.  A bitmask defined in the
 777  * endpoint's METADATA_MASK register defines which byte within the modem
 778  * metadata contains the mux_id.  And the OFST_METADATA field programmed
 779  * here indicates where the extracted byte should be placed within the QMAP
 780  * header.
 781  */
 782 static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
 783 {
 784         u32 endpoint_id = endpoint->endpoint_id;
 785         struct ipa *ipa = endpoint->ipa;
 786         const struct reg *reg;
 787         u32 val = 0;
 788
 789         reg = ipa_reg(ipa, ENDP_INIT_HDR);
 790         if (endpoint->config.qmap) {
 791                 enum ipa_version version = ipa->version;
 792                 size_t header_size;
 793
 794                 header_size = ipa_qmap_header_size(version, endpoint);
 795                 val = ipa_header_size_encode(version, reg, header_size);
 796
 797                 /* Define how to fill fields in a received QMAP header */
 798                 if (!endpoint->toward_ipa) {
 799                         u32 off;     /* Field offset within header */
 800
 801                         /* Where IPA will write the metadata value */
 802                         off = offsetof(struct rmnet_map_header, mux_id);
 803                         val |= ipa_metadata_offset_encode(version, reg, off);
 804
 805                         /* Where IPA will write the length */
 806                         off = offsetof(struct rmnet_map_header, pkt_len);
 807                         /* Upper bits are stored in HDR_EXT with IPA v4.5 */
 808                         if (version >= IPA_VERSION_4_5)
 809                                 off &= reg_field_max(reg, HDR_OFST_PKT_SIZE);
 810
 811                         val |= reg_bit(reg, HDR_OFST_PKT_SIZE_VALID);
 812                         val |= reg_encode(reg, HDR_OFST_PKT_SIZE, off);
 813                 }
 814                 /* For QMAP TX, metadata offset is 0 (modem assumes this) */
 815                 val |= reg_bit(reg, HDR_OFST_METADATA_VALID);
 816
 817                 /* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */
 818                 /* HDR_A5_MUX is 0 */
 819                 /* HDR_LEN_INC_DEAGG_HDR is 0 */
 820                 /* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */
 821         }
 822
 823         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 824 }
 825
 826 static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
 827 {
 828         u32 pad_align = endpoint->config.rx.pad_align;
 829         u32 endpoint_id = endpoint->endpoint_id;
 830         struct ipa *ipa = endpoint->ipa;
 831         const struct reg *reg;
 832         u32 val = 0;
 833
 834         reg = ipa_reg(ipa, ENDP_INIT_HDR_EXT);
 835         if (endpoint->config.qmap) {
 836                 /* We have a header, so we must specify its endianness */
 837                 val |= reg_bit(reg, HDR_ENDIANNESS);    /* big endian */
 838
 839                 /* A QMAP header contains a 6 bit pad field at offset 0.
 840                  * The RMNet driver assumes this field is meaningful in
 841                  * packets it receives, and assumes the header's payload
 842                  * length includes that padding.  The RMNet driver does
 843                  * *not* pad packets it sends, however, so the pad field
 844                  * (although 0) should be ignored.
 845                  */
 846                 if (!endpoint->toward_ipa) {
 847                         val |= reg_bit(reg, HDR_TOTAL_LEN_OR_PAD_VALID);
 848                         /* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */
 849                         val |= reg_bit(reg, HDR_PAYLOAD_LEN_INC_PADDING);
 850                         /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */
 851                 }
 852         }
 853
 854         /* HDR_PAYLOAD_LEN_INC_PADDING is 0 */
 855         if (!endpoint->toward_ipa)
 856                 val |= reg_encode(reg, HDR_PAD_TO_ALIGNMENT, pad_align);
 857
 858         /* IPA v4.5 adds some most-significant bits to a few fields,
 859          * two of which are defined in the HDR (not HDR_EXT) register.
 860          */
 861         if (ipa->version >= IPA_VERSION_4_5) {
 862                 /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */
 863                 if (endpoint->config.qmap && !endpoint->toward_ipa) {
 864                         u32 mask = reg_field_max(reg, HDR_OFST_PKT_SIZE);
 865                         u32 off;     /* Field offset within header */
 866
 867                         off = offsetof(struct rmnet_map_header, pkt_len);
 868                         /* Low bits are in the ENDP_INIT_HDR register */
 869                         off >>= hweight32(mask);
 870                         val |= reg_encode(reg, HDR_OFST_PKT_SIZE_MSB, off);
 871                         /* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */
 872                 }
 873         }
 874
 875         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 876 }
 877
 878 static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint)
 879 {
 880         u32 endpoint_id = endpoint->endpoint_id;
 881         struct ipa *ipa = endpoint->ipa;
 882         const struct reg *reg;
 883         u32 val = 0;
 884         u32 offset;
 885
 886         if (endpoint->toward_ipa)
 887                 return;         /* Register not valid for TX endpoints */
 888
 889         reg = ipa_reg(ipa,  ENDP_INIT_HDR_METADATA_MASK);
 890         offset = reg_n_offset(reg, endpoint_id);
 891
 892         /* Note that HDR_ENDIANNESS indicates big endian header fields */
 893         if (endpoint->config.qmap)
 894                 val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK);
 895
 896         iowrite32(val, ipa->reg_virt + offset);
 897 }
 898
 899 static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint)
 900 {
 901         struct ipa *ipa = endpoint->ipa;
 902         const struct reg *reg;
 903         u32 offset;
 904         u32 val;
 905
 906         if (!endpoint->toward_ipa)
 907                 return;         /* Register not valid for RX endpoints */
 908
 909         reg = ipa_reg(ipa, ENDP_INIT_MODE);
 910         if (endpoint->config.dma_mode) {
 911                 enum ipa_endpoint_name name = endpoint->config.dma_endpoint;
 912                 u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id;
 913
 914                 val = reg_encode(reg, ENDP_MODE, IPA_DMA);
 915                 val |= reg_encode(reg, DEST_PIPE_INDEX, dma_endpoint_id);
 916         } else {
 917                 val = reg_encode(reg, ENDP_MODE, IPA_BASIC);
 918         }
 919         /* All other bits unspecified (and 0) */
 920
 921         offset = reg_n_offset(reg, endpoint->endpoint_id);
 922         iowrite32(val, ipa->reg_virt + offset);
 923 }
 924
 925 /* For IPA v4.5+, times are expressed using Qtime.  A time is represented
 926  * at one of several available granularities, which are configured in
 927  * ipa_qtime_config().  Three (or, starting with IPA v5.0, four) pulse
 928  * generators are set up with different "tick" periods.  A Qtime value
 929  * encodes a tick count along with an indication of a pulse generator
 930  * (which has a fixed tick period).  Two pulse generators are always
 931  * available to the AP; a third is available starting with IPA v5.0.
 932  * This function determines which pulse generator most accurately
 933  * represents the time period provided, and returns the tick count to
 934  * use to represent that time.
 935  */
 936 static u32
 937 ipa_qtime_val(struct ipa *ipa, u32 microseconds, u32 max, u32 *select)
 938 {
 939         u32 which = 0;
 940         u32 ticks;
 941
 942         /* Pulse generator 0 has 100 microsecond granularity */
 943         ticks = DIV_ROUND_CLOSEST(microseconds, 100);
 944         if (ticks <= max)
 945                 goto out;
 946
 947         /* Pulse generator 1 has millisecond granularity */
 948         which = 1;
 949         ticks = DIV_ROUND_CLOSEST(microseconds, 1000);
 950         if (ticks <= max)
 951                 goto out;
 952
 953         if (ipa->version >= IPA_VERSION_5_0) {
 954                 /* Pulse generator 2 has 10 millisecond granularity */
 955                 which = 2;
 956                 ticks = DIV_ROUND_CLOSEST(microseconds, 100);
 957         }
 958         WARN_ON(ticks > max);
 959 out:
 960         *select = which;
 961
 962         return ticks;
 963 }
 964
 965 /* Encode the aggregation timer limit (microseconds) based on IPA version */
 966 static u32 aggr_time_limit_encode(struct ipa *ipa, const struct reg *reg,
 967                                   u32 microseconds)
 968 {
 969         u32 ticks;
 970         u32 max;
 971
 972         if (!microseconds)
 973                 return 0;       /* Nothing to compute if time limit is 0 */
 974
 975         max = reg_field_max(reg, TIME_LIMIT);
 976         if (ipa->version >= IPA_VERSION_4_5) {
 977                 u32 select;
 978
 979                 ticks = ipa_qtime_val(ipa, microseconds, max, &select);
 980
 981                 return reg_encode(reg, AGGR_GRAN_SEL, select) |
 982                        reg_encode(reg, TIME_LIMIT, ticks);
 983         }
 984
 985         /* We program aggregation granularity in ipa_hardware_config() */
 986         ticks = DIV_ROUND_CLOSEST(microseconds, IPA_AGGR_GRANULARITY);
 987         WARN(ticks > max, "aggr_time_limit too large (%u > %u usec)\n",
 988              microseconds, max * IPA_AGGR_GRANULARITY);
 989
 990         return reg_encode(reg, TIME_LIMIT, ticks);
 991 }
 992
 993 static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint)
 994 {
 995         u32 endpoint_id = endpoint->endpoint_id;
 996         struct ipa *ipa = endpoint->ipa;
 997         const struct reg *reg;
 998         u32 val = 0;
 999
1000         reg = ipa_reg(ipa, ENDP_INIT_AGGR);
1001         if (endpoint->config.aggregation) {
1002                 if (!endpoint->toward_ipa) {
1003                         const struct ipa_endpoint_rx *rx_config;
1004                         u32 buffer_size;
1005                         u32 limit;
1006
1007                         rx_config = &endpoint->config.rx;
1008                         val |= reg_encode(reg, AGGR_EN, IPA_ENABLE_AGGR);
1009                         val |= reg_encode(reg, AGGR_TYPE, IPA_GENERIC);
1010
1011                         buffer_size = rx_config->buffer_size;
1012                         limit = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD,
1013                                                  rx_config->aggr_hard_limit);
1014                         val |= reg_encode(reg, BYTE_LIMIT, limit);
1015
1016                         limit = rx_config->aggr_time_limit;
1017                         val |= aggr_time_limit_encode(ipa, reg, limit);
1018
1019                         /* AGGR_PKT_LIMIT is 0 (unlimited) */
1020
1021                         if (rx_config->aggr_close_eof)
1022                                 val |= reg_bit(reg, SW_EOF_ACTIVE);
1023                 } else {
1024                         val |= reg_encode(reg, AGGR_EN, IPA_ENABLE_DEAGGR);
1025                         val |= reg_encode(reg, AGGR_TYPE, IPA_QCMAP);
1026                         /* other fields ignored */
1027                 }
1028                 /* AGGR_FORCE_CLOSE is 0 */
1029                 /* AGGR_GRAN_SEL is 0 for IPA v4.5 */
1030         } else {
1031                 val |= reg_encode(reg, AGGR_EN, IPA_BYPASS_AGGR);
1032                 /* other fields ignored */
1033         }
1034
1035         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1036 }
1037
1038 /* The head-of-line blocking timer is defined as a tick count.  For
1039  * IPA version 4.5 the tick count is based on the Qtimer, which is
1040  * derived from the 19.2 MHz SoC XO clock.  For older IPA versions
1041  * each tick represents 128 cycles of the IPA core clock.
1042  *
1043  * Return the encoded value representing the timeout period provided
1044  * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register.
1045  */
1046 static u32 hol_block_timer_encode(struct ipa *ipa, const struct reg *reg,
1047                                   u32 microseconds)
1048 {
1049         u32 width;
1050         u32 scale;
1051         u64 ticks;
1052         u64 rate;
1053         u32 high;
1054         u32 val;
1055
1056         if (!microseconds)
1057                 return 0;       /* Nothing to compute if timer period is 0 */
1058
1059         if (ipa->version >= IPA_VERSION_4_5) {
1060                 u32 max = reg_field_max(reg, TIMER_LIMIT);
1061                 u32 select;
1062                 u32 ticks;
1063
1064                 ticks = ipa_qtime_val(ipa, microseconds, max, &select);
1065
1066                 return reg_encode(reg, TIMER_GRAN_SEL, 1) |
1067                        reg_encode(reg, TIMER_LIMIT, ticks);
1068         }
1069
1070         /* Use 64 bit arithmetic to avoid overflow */
1071         rate = ipa_core_clock_rate(ipa);
1072         ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC);
1073
1074         /* We still need the result to fit into the field */
1075         WARN_ON(ticks > reg_field_max(reg, TIMER_BASE_VALUE));
1076
1077         /* IPA v3.5.1 through v4.1 just record the tick count */
1078         if (ipa->version < IPA_VERSION_4_2)
1079                 return reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks);
1080
1081         /* For IPA v4.2, the tick count is represented by base and
1082          * scale fields within the 32-bit timer register, where:
1083          *     ticks = base << scale;
1084          * The best precision is achieved when the base value is as
1085          * large as possible.  Find the highest set bit in the tick
1086          * count, and extract the number of bits in the base field
1087          * such that high bit is included.
1088          */
1089         high = fls(ticks);              /* 1..32 (or warning above) */
1090         width = hweight32(reg_fmask(reg, TIMER_BASE_VALUE));
1091         scale = high > width ? high - width : 0;
1092         if (scale) {
1093                 /* If we're scaling, round up to get a closer result */
1094                 ticks += 1 << (scale - 1);
1095                 /* High bit was set, so rounding might have affected it */
1096                 if (fls(ticks) != high)
1097                         scale++;
1098         }
1099
1100         val = reg_encode(reg, TIMER_SCALE, scale);
1101         val |= reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks >> scale);
1102
1103         return val;
1104 }
1105
1106 /* If microseconds is 0, timeout is immediate */
1107 static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint,
1108                                               u32 microseconds)
1109 {
1110         u32 endpoint_id = endpoint->endpoint_id;
1111         struct ipa *ipa = endpoint->ipa;
1112         const struct reg *reg;
1113         u32 val;
1114
1115         /* This should only be changed when HOL_BLOCK_EN is disabled */
1116         reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_TIMER);
1117         val = hol_block_timer_encode(ipa, reg, microseconds);
1118
1119         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1120 }
1121
1122 static void
1123 ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable)
1124 {
1125         u32 endpoint_id = endpoint->endpoint_id;
1126         struct ipa *ipa = endpoint->ipa;
1127         const struct reg *reg;
1128         u32 offset;
1129         u32 val;
1130
1131         reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_EN);
1132         offset = reg_n_offset(reg, endpoint_id);
1133         val = enable ? reg_bit(reg, HOL_BLOCK_EN) : 0;
1134
1135         iowrite32(val, ipa->reg_virt + offset);
1136
1137         /* When enabling, the register must be written twice for IPA v4.5+ */
1138         if (enable && ipa->version >= IPA_VERSION_4_5)
1139                 iowrite32(val, ipa->reg_virt + offset);
1140 }
1141
1142 /* Assumes HOL_BLOCK is in disabled state */
1143 static void ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint,
1144                                                u32 microseconds)
1145 {
1146         ipa_endpoint_init_hol_block_timer(endpoint, microseconds);
1147         ipa_endpoint_init_hol_block_en(endpoint, true);
1148 }
1149
1150 static void ipa_endpoint_init_hol_block_disable(struct ipa_endpoint *endpoint)
1151 {
1152         ipa_endpoint_init_hol_block_en(endpoint, false);
1153 }
1154
1155 void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)
1156 {
1157         u32 endpoint_id = 0;
1158
1159         while (endpoint_id < ipa->endpoint_count) {
1160                 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++];
1161
1162                 if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM)
1163                         continue;
1164
1165                 ipa_endpoint_init_hol_block_disable(endpoint);
1166                 ipa_endpoint_init_hol_block_enable(endpoint, 0);
1167         }
1168 }
1169
1170 static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint)
1171 {
1172         u32 endpoint_id = endpoint->endpoint_id;
1173         struct ipa *ipa = endpoint->ipa;
1174         const struct reg *reg;
1175         u32 val = 0;
1176
1177         if (!endpoint->toward_ipa)
1178                 return;         /* Register not valid for RX endpoints */
1179
1180         reg = ipa_reg(ipa, ENDP_INIT_DEAGGR);
1181         /* DEAGGR_HDR_LEN is 0 */
1182         /* PACKET_OFFSET_VALID is 0 */
1183         /* PACKET_OFFSET_LOCATION is ignored (not valid) */
1184         /* MAX_PACKET_LEN is 0 (not enforced) */
1185
1186         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1187 }
1188
1189 static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint)
1190 {
1191         u32 resource_group = endpoint->config.resource_group;
1192         u32 endpoint_id = endpoint->endpoint_id;
1193         struct ipa *ipa = endpoint->ipa;
1194         const struct reg *reg;
1195         u32 val;
1196
1197         reg = ipa_reg(ipa, ENDP_INIT_RSRC_GRP);
1198         val = reg_encode(reg, ENDP_RSRC_GRP, resource_group);
1199
1200         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1201 }
1202
1203 static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint)
1204 {
1205         u32 endpoint_id = endpoint->endpoint_id;
1206         struct ipa *ipa = endpoint->ipa;
1207         const struct reg *reg;
1208         u32 val;
1209
1210         if (!endpoint->toward_ipa)
1211                 return;         /* Register not valid for RX endpoints */
1212
1213         reg = ipa_reg(ipa, ENDP_INIT_SEQ);
1214
1215         /* Low-order byte configures primary packet processing */
1216         val = reg_encode(reg, SEQ_TYPE, endpoint->config.tx.seq_type);
1217
1218         /* Second byte (if supported) configures replicated packet processing */
1219         if (ipa->version < IPA_VERSION_4_5)
1220                 val |= reg_encode(reg, SEQ_REP_TYPE,
1221                                   endpoint->config.tx.seq_rep_type);
1222
1223         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1224 }
1225
1226 /**
1227  * ipa_endpoint_skb_tx() - Transmit a socket buffer
1228  * @endpoint:   Endpoint pointer
1229  * @skb:        Socket buffer to send
1230  *
1231  * Returns:     0 if successful, or a negative error code
1232  */
1233 int ipa_endpoint_skb_tx(struct ipa_endpoint *endpoint, struct sk_buff *skb)
1234 {
1235         struct gsi_trans *trans;
1236         u32 nr_frags;
1237         int ret;
1238
1239         /* Make sure source endpoint's TLV FIFO has enough entries to
1240          * hold the linear portion of the skb and all its fragments.
1241          * If not, see if we can linearize it before giving up.
1242          */
1243         nr_frags = skb_shinfo(skb)->nr_frags;
1244         if (nr_frags > endpoint->skb_frag_max) {
1245                 if (skb_linearize(skb))
1246                         return -E2BIG;
1247                 nr_frags = 0;
1248         }
1249
1250         trans = ipa_endpoint_trans_alloc(endpoint, 1 + nr_frags);
1251         if (!trans)
1252                 return -EBUSY;
1253
1254         ret = gsi_trans_skb_add(trans, skb);
1255         if (ret)
1256                 goto err_trans_free;
1257         trans->data = skb;      /* transaction owns skb now */
1258
1259         gsi_trans_commit(trans, !netdev_xmit_more());
1260
1261         return 0;
1262
1263 err_trans_free:
1264         gsi_trans_free(trans);
1265
1266         return -ENOMEM;
1267 }
1268
1269 static void ipa_endpoint_status(struct ipa_endpoint *endpoint)
1270 {
1271         u32 endpoint_id = endpoint->endpoint_id;
1272         struct ipa *ipa = endpoint->ipa;
1273         const struct reg *reg;
1274         u32 val = 0;
1275
1276         reg = ipa_reg(ipa, ENDP_STATUS);
1277         if (endpoint->config.status_enable) {
1278                 val |= reg_bit(reg, STATUS_EN);
1279                 if (endpoint->toward_ipa) {
1280                         enum ipa_endpoint_name name;
1281                         u32 status_endpoint_id;
1282
1283                         name = endpoint->config.tx.status_endpoint;
1284                         status_endpoint_id = ipa->name_map[name]->endpoint_id;
1285
1286                         val |= reg_encode(reg, STATUS_ENDP, status_endpoint_id);
1287                 }
1288                 /* STATUS_LOCATION is 0, meaning IPA packet status
1289                  * precedes the packet (not present for IPA v4.5+)
1290                  */
1291                 /* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v4.0+) */
1292         }
1293
1294         iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1295 }
1296
1297 static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint,
1298                                       struct gsi_trans *trans)
1299 {
1300         struct page *page;
1301         u32 buffer_size;
1302         u32 offset;
1303         u32 len;
1304         int ret;
1305
1306         buffer_size = endpoint->config.rx.buffer_size;
1307         page = dev_alloc_pages(get_order(buffer_size));
1308         if (!page)
1309                 return -ENOMEM;
1310
1311         /* Offset the buffer to make space for skb headroom */
1312         offset = NET_SKB_PAD;
1313         len = buffer_size - offset;
1314
1315         ret = gsi_trans_page_add(trans, page, len, offset);
1316         if (ret)
1317                 put_page(page);
1318         else
1319                 trans->data = page;     /* transaction owns page now */
1320
1321         return ret;
1322 }
1323
1324 /**
1325  * ipa_endpoint_replenish() - Replenish endpoint receive buffers
1326  * @endpoint:   Endpoint to be replenished
1327  *
1328  * The IPA hardware can hold a fixed number of receive buffers for an RX
1329  * endpoint, based on the number of entries in the underlying channel ring
1330  * buffer.  If an endpoint's "backlog" is non-zero, it indicates how many
1331  * more receive buffers can be supplied to the hardware.  Replenishing for
1332  * an endpoint can be disabled, in which case buffers are not queued to
1333  * the hardware.
1334  */
1335 static void ipa_endpoint_replenish(struct ipa_endpoint *endpoint)
1336 {
1337         struct gsi_trans *trans;
1338
1339         if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags))
1340                 return;
1341
1342         /* Skip it if it's already active */
1343         if (test_and_set_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags))
1344                 return;
1345
1346         while ((trans = ipa_endpoint_trans_alloc(endpoint, 1))) {
1347                 bool doorbell;
1348
1349                 if (ipa_endpoint_replenish_one(endpoint, trans))
1350                         goto try_again_later;
1351
1352
1353                 /* Ring the doorbell if we've got a full batch */
1354                 doorbell = !(++endpoint->replenish_count % IPA_REPLENISH_BATCH);
1355                 gsi_trans_commit(trans, doorbell);
1356         }
1357
1358         clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
1359
1360         return;
1361
1362 try_again_later:
1363         gsi_trans_free(trans);
1364         clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
1365
1366         /* Whenever a receive buffer transaction completes we'll try to
1367          * replenish again.  It's unlikely, but if we fail to supply even
1368          * one buffer, nothing will trigger another replenish attempt.
1369          * If the hardware has no receive buffers queued, schedule work to
1370          * try replenishing again.
1371          */
1372         if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id))
1373                 schedule_delayed_work(&endpoint->replenish_work,
1374                                       msecs_to_jiffies(1));
1375 }
1376
1377 static void ipa_endpoint_replenish_enable(struct ipa_endpoint *endpoint)
1378 {
1379         set_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
1380
1381         /* Start replenishing if hardware currently has no buffers */
1382         if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id))
1383                 ipa_endpoint_replenish(endpoint);
1384 }
1385
1386 static void ipa_endpoint_replenish_disable(struct ipa_endpoint *endpoint)
1387 {
1388         clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
1389 }
1390
1391 static void ipa_endpoint_replenish_work(struct work_struct *work)
1392 {
1393         struct delayed_work *dwork = to_delayed_work(work);
1394         struct ipa_endpoint *endpoint;
1395
1396         endpoint = container_of(dwork, struct ipa_endpoint, replenish_work);
1397
1398         ipa_endpoint_replenish(endpoint);
1399 }
1400
1401 static void ipa_endpoint_skb_copy(struct ipa_endpoint *endpoint,
1402                                   void *data, u32 len, u32 extra)
1403 {
1404         struct sk_buff *skb;
1405
1406         if (!endpoint->netdev)
1407                 return;
1408
1409         skb = __dev_alloc_skb(len, GFP_ATOMIC);
1410         if (skb) {
1411                 /* Copy the data into the socket buffer and receive it */
1412                 skb_put(skb, len);
1413                 memcpy(skb->data, data, len);
1414                 skb->truesize += extra;
1415         }
1416
1417         ipa_modem_skb_rx(endpoint->netdev, skb);
1418 }
1419
1420 static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint,
1421                                    struct page *page, u32 len)
1422 {
1423         u32 buffer_size = endpoint->config.rx.buffer_size;
1424         struct sk_buff *skb;
1425
1426         /* Nothing to do if there's no netdev */
1427         if (!endpoint->netdev)
1428                 return false;
1429
1430         WARN_ON(len > SKB_WITH_OVERHEAD(buffer_size - NET_SKB_PAD));
1431
1432         skb = build_skb(page_address(page), buffer_size);
1433         if (skb) {
1434                 /* Reserve the headroom and account for the data */
1435                 skb_reserve(skb, NET_SKB_PAD);
1436                 skb_put(skb, len);
1437         }
1438
1439         /* Receive the buffer (or record drop if unable to build it) */
1440         ipa_modem_skb_rx(endpoint->netdev, skb);
1441
1442         return skb != NULL;
1443 }
1444
1445  /* The format of an IPA packet status structure is the same for several
1446   * status types (opcodes).  Other types aren't currently supported.
1447  */
1448 static bool ipa_status_format_packet(enum ipa_status_opcode opcode)
1449 {
1450         switch (opcode) {
1451         case IPA_STATUS_OPCODE_PACKET:
1452         case IPA_STATUS_OPCODE_DROPPED_PACKET:
1453         case IPA_STATUS_OPCODE_SUSPENDED_PACKET:
1454         case IPA_STATUS_OPCODE_PACKET_2ND_PASS:
1455                 return true;
1456         default:
1457                 return false;
1458         }
1459 }
1460
1461 static bool
1462 ipa_endpoint_status_skip(struct ipa_endpoint *endpoint, const void *data)
1463 {
1464         struct ipa *ipa = endpoint->ipa;
1465         enum ipa_status_opcode opcode;
1466         u32 endpoint_id;
1467
1468         opcode = ipa_status_extract(ipa, data, STATUS_OPCODE);
1469         if (!ipa_status_format_packet(opcode))
1470                 return true;
1471
1472         endpoint_id = ipa_status_extract(ipa, data, STATUS_DST_ENDPOINT);
1473         if (endpoint_id != endpoint->endpoint_id)
1474                 return true;
1475
1476         return false;   /* Don't skip this packet, process it */
1477 }
1478
1479 static bool
1480 ipa_endpoint_status_tag_valid(struct ipa_endpoint *endpoint, const void *data)
1481 {
1482         struct ipa_endpoint *command_endpoint;
1483         enum ipa_status_mask status_mask;
1484         struct ipa *ipa = endpoint->ipa;
1485         u32 endpoint_id;
1486
1487         status_mask = ipa_status_extract(ipa, data, STATUS_MASK);
1488         if (!status_mask)
1489                 return false;   /* No valid tag */
1490
1491         /* The status contains a valid tag.  We know the packet was sent to
1492          * this endpoint (already verified by ipa_endpoint_status_skip()).
1493          * If the packet came from the AP->command TX endpoint we know
1494          * this packet was sent as part of the pipeline clear process.
1495          */
1496         endpoint_id = ipa_status_extract(ipa, data, STATUS_SRC_ENDPOINT);
1497         command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
1498         if (endpoint_id == command_endpoint->endpoint_id) {
1499                 complete(&ipa->completion);
1500         } else {
1501                 dev_err(&ipa->pdev->dev,
1502                         "unexpected tagged packet from endpoint %u\n",
1503                         endpoint_id);
1504         }
1505
1506         return true;
1507 }
1508
1509 /* Return whether the status indicates the packet should be dropped */
1510 static bool
1511 ipa_endpoint_status_drop(struct ipa_endpoint *endpoint, const void *data)
1512 {
1513         enum ipa_status_exception exception;
1514         struct ipa *ipa = endpoint->ipa;
1515         u32 rule;
1516
1517         /* If the status indicates a tagged transfer, we'll drop the packet */
1518         if (ipa_endpoint_status_tag_valid(endpoint, data))
1519                 return true;
1520
1521         /* Deaggregation exceptions we drop; all other types we consume */
1522         exception = ipa_status_extract(ipa, data, STATUS_EXCEPTION);
1523         if (exception)
1524                 return exception == IPA_STATUS_EXCEPTION_DEAGGR;
1525
1526         /* Drop the packet if it fails to match a routing rule; otherwise no */
1527         rule = ipa_status_extract(ipa, data, STATUS_ROUTER_RULE_INDEX);
1528
1529         return rule == IPA_STATUS_RULE_MISS;
1530 }
1531
1532 static void ipa_endpoint_status_parse(struct ipa_endpoint *endpoint,
1533                                       struct page *page, u32 total_len)
1534 {
1535         u32 buffer_size = endpoint->config.rx.buffer_size;
1536         void *data = page_address(page) + NET_SKB_PAD;
1537         u32 unused = buffer_size - total_len;
1538         struct ipa *ipa = endpoint->ipa;
1539         u32 resid = total_len;
1540
1541         while (resid) {
1542                 u32 length;
1543                 u32 align;
1544                 u32 len;
1545
1546                 if (resid < IPA_STATUS_SIZE) {
1547                         dev_err(&endpoint->ipa->pdev->dev,
1548                                 "short message (%u bytes < %zu byte status)\n",
1549                                 resid, IPA_STATUS_SIZE);
1550                         break;
1551                 }
1552
1553                 /* Skip over status packets that lack packet data */
1554                 length = ipa_status_extract(ipa, data, STATUS_LENGTH);
1555                 if (!length || ipa_endpoint_status_skip(endpoint, data)) {
1556                         data += IPA_STATUS_SIZE;
1557                         resid -= IPA_STATUS_SIZE;
1558                         continue;
1559                 }
1560
1561                 /* Compute the amount of buffer space consumed by the packet,
1562                  * including the status.  If the hardware is configured to
1563                  * pad packet data to an aligned boundary, account for that.
1564                  * And if checksum offload is enabled a trailer containing
1565                  * computed checksum information will be appended.
1566                  */
1567                 align = endpoint->config.rx.pad_align ? : 1;
1568                 len = IPA_STATUS_SIZE + ALIGN(length, align);
1569                 if (endpoint->config.checksum)
1570                         len += sizeof(struct rmnet_map_dl_csum_trailer);
1571
1572                 if (!ipa_endpoint_status_drop(endpoint, data)) {
1573                         void *data2;
1574                         u32 extra;
1575
1576                         /* Client receives only packet data (no status) */
1577                         data2 = data + IPA_STATUS_SIZE;
1578
1579                         /* Have the true size reflect the extra unused space in
1580                          * the original receive buffer.  Distribute the "cost"
1581                          * proportionately across all aggregated packets in the
1582                          * buffer.
1583                          */
1584                         extra = DIV_ROUND_CLOSEST(unused * len, total_len);
1585                         ipa_endpoint_skb_copy(endpoint, data2, length, extra);
1586                 }
1587
1588                 /* Consume status and the full packet it describes */
1589                 data += len;
1590                 resid -= len;
1591         }
1592 }
1593
1594 void ipa_endpoint_trans_complete(struct ipa_endpoint *endpoint,
1595                                  struct gsi_trans *trans)
1596 {
1597         struct page *page;
1598
1599         if (endpoint->toward_ipa)
1600                 return;
1601
1602         if (trans->cancelled)
1603                 goto done;
1604
1605         /* Parse or build a socket buffer using the actual received length */
1606         page = trans->data;
1607         if (endpoint->config.status_enable)
1608                 ipa_endpoint_status_parse(endpoint, page, trans->len);
1609         else if (ipa_endpoint_skb_build(endpoint, page, trans->len))
1610                 trans->data = NULL;     /* Pages have been consumed */
1611 done:
1612         ipa_endpoint_replenish(endpoint);
1613 }
1614
1615 void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint,
1616                                 struct gsi_trans *trans)
1617 {
1618         if (endpoint->toward_ipa) {
1619                 struct ipa *ipa = endpoint->ipa;
1620
1621                 /* Nothing to do for command transactions */
1622                 if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) {
1623                         struct sk_buff *skb = trans->data;
1624
1625                         if (skb)
1626                                 dev_kfree_skb_any(skb);
1627                 }
1628         } else {
1629                 struct page *page = trans->data;
1630
1631                 if (page)
1632                         put_page(page);
1633         }
1634 }
1635
1636 void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id)
1637 {
1638         const struct reg *reg;
1639         u32 val;
1640
1641         reg = ipa_reg(ipa, ROUTE);
1642         /* ROUTE_DIS is 0 */
1643         val = reg_encode(reg, ROUTE_DEF_PIPE, endpoint_id);
1644         val |= reg_bit(reg, ROUTE_DEF_HDR_TABLE);
1645         /* ROUTE_DEF_HDR_OFST is 0 */
1646         val |= reg_encode(reg, ROUTE_FRAG_DEF_PIPE, endpoint_id);
1647         val |= reg_bit(reg, ROUTE_DEF_RETAIN_HDR);
1648
1649         iowrite32(val, ipa->reg_virt + reg_offset(reg));
1650 }
1651
1652 void ipa_endpoint_default_route_clear(struct ipa *ipa)
1653 {
1654         ipa_endpoint_default_route_set(ipa, 0);
1655 }
1656
1657 /**
1658  * ipa_endpoint_reset_rx_aggr() - Reset RX endpoint with aggregation active
1659  * @endpoint:   Endpoint to be reset
1660  *
1661  * If aggregation is active on an RX endpoint when a reset is performed
1662  * on its underlying GSI channel, a special sequence of actions must be
1663  * taken to ensure the IPA pipeline is properly cleared.
1664  *
1665  * Return:      0 if successful, or a negative error code
1666  */
1667 static int ipa_endpoint_reset_rx_aggr(struct ipa_endpoint *endpoint)
1668 {
1669         struct device *dev = &endpoint->ipa->pdev->dev;
1670         struct ipa *ipa = endpoint->ipa;
1671         struct gsi *gsi = &ipa->gsi;
1672         bool suspended = false;
1673         dma_addr_t addr;
1674         u32 retries;
1675         u32 len = 1;
1676         void *virt;
1677         int ret;
1678
1679         virt = kzalloc(len, GFP_KERNEL);
1680         if (!virt)
1681                 return -ENOMEM;
1682
1683         addr = dma_map_single(dev, virt, len, DMA_FROM_DEVICE);
1684         if (dma_mapping_error(dev, addr)) {
1685                 ret = -ENOMEM;
1686                 goto out_kfree;
1687         }
1688
1689         /* Force close aggregation before issuing the reset */
1690         ipa_endpoint_force_close(endpoint);
1691
1692         /* Reset and reconfigure the channel with the doorbell engine
1693          * disabled.  Then poll until we know aggregation is no longer
1694          * active.  We'll re-enable the doorbell (if appropriate) when
1695          * we reset again below.
1696          */
1697         gsi_channel_reset(gsi, endpoint->channel_id, false);
1698
1699         /* Make sure the channel isn't suspended */
1700         suspended = ipa_endpoint_program_suspend(endpoint, false);
1701
1702         /* Start channel and do a 1 byte read */
1703         ret = gsi_channel_start(gsi, endpoint->channel_id);
1704         if (ret)
1705                 goto out_suspend_again;
1706
1707         ret = gsi_trans_read_byte(gsi, endpoint->channel_id, addr);
1708         if (ret)
1709                 goto err_endpoint_stop;
1710
1711         /* Wait for aggregation to be closed on the channel */
1712         retries = IPA_ENDPOINT_RESET_AGGR_RETRY_MAX;
1713         do {
1714                 if (!ipa_endpoint_aggr_active(endpoint))
1715                         break;
1716                 usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
1717         } while (retries--);
1718
1719         /* Check one last time */
1720         if (ipa_endpoint_aggr_active(endpoint))
1721                 dev_err(dev, "endpoint %u still active during reset\n",
1722                         endpoint->endpoint_id);
1723
1724         gsi_trans_read_byte_done(gsi, endpoint->channel_id);
1725
1726         ret = gsi_channel_stop(gsi, endpoint->channel_id);
1727         if (ret)
1728                 goto out_suspend_again;
1729
1730         /* Finally, reset and reconfigure the channel again (re-enabling
1731          * the doorbell engine if appropriate).  Sleep for 1 millisecond to
1732          * complete the channel reset sequence.  Finish by suspending the
1733          * channel again (if necessary).
1734          */
1735         gsi_channel_reset(gsi, endpoint->channel_id, true);
1736
1737         usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
1738
1739         goto out_suspend_again;
1740
1741 err_endpoint_stop:
1742         (void)gsi_channel_stop(gsi, endpoint->channel_id);
1743 out_suspend_again:
1744         if (suspended)
1745                 (void)ipa_endpoint_program_suspend(endpoint, true);
1746         dma_unmap_single(dev, addr, len, DMA_FROM_DEVICE);
1747 out_kfree:
1748         kfree(virt);
1749
1750         return ret;
1751 }
1752
1753 static void ipa_endpoint_reset(struct ipa_endpoint *endpoint)
1754 {
1755         u32 channel_id = endpoint->channel_id;
1756         struct ipa *ipa = endpoint->ipa;
1757         bool special;
1758         int ret = 0;
1759
1760         /* On IPA v3.5.1, if an RX endpoint is reset while aggregation
1761          * is active, we need to handle things specially to recover.
1762          * All other cases just need to reset the underlying GSI channel.
1763          */
1764         special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa &&
1765                         endpoint->config.aggregation;
1766         if (special && ipa_endpoint_aggr_active(endpoint))
1767                 ret = ipa_endpoint_reset_rx_aggr(endpoint);
1768         else
1769                 gsi_channel_reset(&ipa->gsi, channel_id, true);
1770
1771         if (ret)
1772                 dev_err(&ipa->pdev->dev,
1773                         "error %d resetting channel %u for endpoint %u\n",
1774                         ret, endpoint->channel_id, endpoint->endpoint_id);
1775 }
1776
1777 static void ipa_endpoint_program(struct ipa_endpoint *endpoint)
1778 {
1779         if (endpoint->toward_ipa) {
1780                 /* Newer versions of IPA use GSI channel flow control
1781                  * instead of endpoint DELAY mode to prevent sending data.
1782                  * Flow control is disabled for newly-allocated channels,
1783                  * and we can assume flow control is not (ever) enabled
1784                  * for AP TX channels.
1785                  */
1786                 if (endpoint->ipa->version < IPA_VERSION_4_2)
1787                         ipa_endpoint_program_delay(endpoint, false);
1788         } else {
1789                 /* Ensure suspend mode is off on all AP RX endpoints */
1790                 (void)ipa_endpoint_program_suspend(endpoint, false);
1791         }
1792         ipa_endpoint_init_cfg(endpoint);
1793         ipa_endpoint_init_nat(endpoint);
1794         ipa_endpoint_init_hdr(endpoint);
1795         ipa_endpoint_init_hdr_ext(endpoint);
1796         ipa_endpoint_init_hdr_metadata_mask(endpoint);
1797         ipa_endpoint_init_mode(endpoint);
1798         ipa_endpoint_init_aggr(endpoint);
1799         if (!endpoint->toward_ipa) {
1800                 if (endpoint->config.rx.holb_drop)
1801                         ipa_endpoint_init_hol_block_enable(endpoint, 0);
1802                 else
1803                         ipa_endpoint_init_hol_block_disable(endpoint);
1804         }
1805         ipa_endpoint_init_deaggr(endpoint);
1806         ipa_endpoint_init_rsrc_grp(endpoint);
1807         ipa_endpoint_init_seq(endpoint);
1808         ipa_endpoint_status(endpoint);
1809 }
1810
1811 int ipa_endpoint_enable_one(struct ipa_endpoint *endpoint)
1812 {
1813         u32 endpoint_id = endpoint->endpoint_id;
1814         struct ipa *ipa = endpoint->ipa;
1815         struct gsi *gsi = &ipa->gsi;
1816         int ret;
1817
1818         ret = gsi_channel_start(gsi, endpoint->channel_id);
1819         if (ret) {
1820                 dev_err(&ipa->pdev->dev,
1821                         "error %d starting %cX channel %u for endpoint %u\n",
1822                         ret, endpoint->toward_ipa ? 'T' : 'R',
1823                         endpoint->channel_id, endpoint_id);
1824                 return ret;
1825         }
1826
1827         if (!endpoint->toward_ipa) {
1828                 ipa_interrupt_suspend_enable(ipa->interrupt, endpoint_id);
1829                 ipa_endpoint_replenish_enable(endpoint);
1830         }
1831
1832         __set_bit(endpoint_id, ipa->enabled);
1833
1834         return 0;
1835 }
1836
1837 void ipa_endpoint_disable_one(struct ipa_endpoint *endpoint)
1838 {
1839         u32 endpoint_id = endpoint->endpoint_id;
1840         struct ipa *ipa = endpoint->ipa;
1841         struct gsi *gsi = &ipa->gsi;
1842         int ret;
1843
1844         if (!test_bit(endpoint_id, ipa->enabled))
1845                 return;
1846
1847         __clear_bit(endpoint_id, endpoint->ipa->enabled);
1848
1849         if (!endpoint->toward_ipa) {
1850                 ipa_endpoint_replenish_disable(endpoint);
1851                 ipa_interrupt_suspend_disable(ipa->interrupt, endpoint_id);
1852         }
1853
1854         /* Note that if stop fails, the channel's state is not well-defined */
1855         ret = gsi_channel_stop(gsi, endpoint->channel_id);
1856         if (ret)
1857                 dev_err(&ipa->pdev->dev,
1858                         "error %d attempting to stop endpoint %u\n", ret,
1859                         endpoint_id);
1860 }
1861
1862 void ipa_endpoint_suspend_one(struct ipa_endpoint *endpoint)
1863 {
1864         struct device *dev = &endpoint->ipa->pdev->dev;
1865         struct gsi *gsi = &endpoint->ipa->gsi;
1866         int ret;
1867
1868         if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled))
1869                 return;
1870
1871         if (!endpoint->toward_ipa) {
1872                 ipa_endpoint_replenish_disable(endpoint);
1873                 (void)ipa_endpoint_program_suspend(endpoint, true);
1874         }
1875
1876         ret = gsi_channel_suspend(gsi, endpoint->channel_id);
1877         if (ret)
1878                 dev_err(dev, "error %d suspending channel %u\n", ret,
1879                         endpoint->channel_id);
1880 }
1881
1882 void ipa_endpoint_resume_one(struct ipa_endpoint *endpoint)
1883 {
1884         struct device *dev = &endpoint->ipa->pdev->dev;
1885         struct gsi *gsi = &endpoint->ipa->gsi;
1886         int ret;
1887
1888         if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled))
1889                 return;
1890
1891         if (!endpoint->toward_ipa)
1892                 (void)ipa_endpoint_program_suspend(endpoint, false);
1893
1894         ret = gsi_channel_resume(gsi, endpoint->channel_id);
1895         if (ret)
1896                 dev_err(dev, "error %d resuming channel %u\n", ret,
1897                         endpoint->channel_id);
1898         else if (!endpoint->toward_ipa)
1899                 ipa_endpoint_replenish_enable(endpoint);
1900 }
1901
1902 void ipa_endpoint_suspend(struct ipa *ipa)
1903 {
1904         if (!ipa->setup_complete)
1905                 return;
1906
1907         if (ipa->modem_netdev)
1908                 ipa_modem_suspend(ipa->modem_netdev);
1909
1910         ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
1911         ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
1912 }
1913
1914 void ipa_endpoint_resume(struct ipa *ipa)
1915 {
1916         if (!ipa->setup_complete)
1917                 return;
1918
1919         ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
1920         ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
1921
1922         if (ipa->modem_netdev)
1923                 ipa_modem_resume(ipa->modem_netdev);
1924 }
1925
1926 static void ipa_endpoint_setup_one(struct ipa_endpoint *endpoint)
1927 {
1928         struct gsi *gsi = &endpoint->ipa->gsi;
1929         u32 channel_id = endpoint->channel_id;
1930
1931         /* Only AP endpoints get set up */
1932         if (endpoint->ee_id != GSI_EE_AP)
1933                 return;
1934
1935         endpoint->skb_frag_max = gsi->channel[channel_id].trans_tre_max - 1;
1936         if (!endpoint->toward_ipa) {
1937                 /* RX transactions require a single TRE, so the maximum
1938                  * backlog is the same as the maximum outstanding TREs.
1939                  */
1940                 clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
1941                 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
1942                 INIT_DELAYED_WORK(&endpoint->replenish_work,
1943                                   ipa_endpoint_replenish_work);
1944         }
1945
1946         ipa_endpoint_program(endpoint);
1947
1948         __set_bit(endpoint->endpoint_id, endpoint->ipa->set_up);
1949 }
1950
1951 static void ipa_endpoint_teardown_one(struct ipa_endpoint *endpoint)
1952 {
1953         __clear_bit(endpoint->endpoint_id, endpoint->ipa->set_up);
1954
1955         if (!endpoint->toward_ipa)
1956                 cancel_delayed_work_sync(&endpoint->replenish_work);
1957
1958         ipa_endpoint_reset(endpoint);
1959 }
1960
1961 void ipa_endpoint_setup(struct ipa *ipa)
1962 {
1963         u32 endpoint_id;
1964
1965         for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count)
1966                 ipa_endpoint_setup_one(&ipa->endpoint[endpoint_id]);
1967 }
1968
1969 void ipa_endpoint_teardown(struct ipa *ipa)
1970 {
1971         u32 endpoint_id;
1972
1973         for_each_set_bit(endpoint_id, ipa->set_up, ipa->endpoint_count)
1974                 ipa_endpoint_teardown_one(&ipa->endpoint[endpoint_id]);
1975 }
1976
1977 void ipa_endpoint_deconfig(struct ipa *ipa)
1978 {
1979         ipa->available_count = 0;
1980         bitmap_free(ipa->available);
1981         ipa->available = NULL;
1982 }
1983
1984 int ipa_endpoint_config(struct ipa *ipa)
1985 {
1986         struct device *dev = &ipa->pdev->dev;
1987         const struct reg *reg;
1988         u32 endpoint_id;
1989         u32 hw_limit;
1990         u32 tx_count;
1991         u32 rx_count;
1992         u32 rx_base;
1993         u32 limit;
1994         u32 val;
1995
1996         /* Prior to IPA v3.5, the FLAVOR_0 register was not supported.
1997          * Furthermore, the endpoints were not grouped such that TX
1998          * endpoint numbers started with 0 and RX endpoints had numbers
1999          * higher than all TX endpoints, so we can't do the simple
2000          * direction check used for newer hardware below.
2001          *
2002          * For hardware that doesn't support the FLAVOR_0 register,
2003          * just set the available mask to support any endpoint, and
2004          * assume the configuration is valid.
2005          */
2006         if (ipa->version < IPA_VERSION_3_5) {
2007                 ipa->available = bitmap_zalloc(IPA_ENDPOINT_MAX, GFP_KERNEL);
2008                 if (!ipa->available)
2009                         return -ENOMEM;
2010                 ipa->available_count = IPA_ENDPOINT_MAX;
2011
2012                 bitmap_set(ipa->available, 0, IPA_ENDPOINT_MAX);
2013
2014                 return 0;
2015         }
2016
2017         /* Find out about the endpoints supplied by the hardware, and ensure
2018          * the highest one doesn't exceed the number supported by software.
2019          */
2020         reg = ipa_reg(ipa, FLAVOR_0);
2021         val = ioread32(ipa->reg_virt + reg_offset(reg));
2022
2023         /* Our RX is an IPA producer; our TX is an IPA consumer. */
2024         tx_count = reg_decode(reg, MAX_CONS_PIPES, val);
2025         rx_count = reg_decode(reg, MAX_PROD_PIPES, val);
2026         rx_base = reg_decode(reg, PROD_LOWEST, val);
2027
2028         limit = rx_base + rx_count;
2029         if (limit > IPA_ENDPOINT_MAX) {
2030                 dev_err(dev, "too many endpoints, %u > %u\n",
2031                         limit, IPA_ENDPOINT_MAX);
2032                 return -EINVAL;
2033         }
2034
2035         /* Until IPA v5.0, the max endpoint ID was 32 */
2036         hw_limit = ipa->version < IPA_VERSION_5_0 ? 32 : U8_MAX + 1;
2037         if (limit > hw_limit) {
2038                 dev_err(dev, "unexpected endpoint count, %u > %u\n",
2039                         limit, hw_limit);
2040                 return -EINVAL;
2041         }
2042
2043         /* Allocate and initialize the available endpoint bitmap */
2044         ipa->available = bitmap_zalloc(limit, GFP_KERNEL);
2045         if (!ipa->available)
2046                 return -ENOMEM;
2047         ipa->available_count = limit;
2048
2049         /* Mark all supported RX and TX endpoints as available */
2050         bitmap_set(ipa->available, 0, tx_count);
2051         bitmap_set(ipa->available, rx_base, rx_count);
2052
2053         for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) {
2054                 struct ipa_endpoint *endpoint;
2055
2056                 if (endpoint_id >= limit) {
2057                         dev_err(dev, "invalid endpoint id, %u > %u\n",
2058                                 endpoint_id, limit - 1);
2059                         goto err_free_bitmap;
2060                 }
2061
2062                 if (!test_bit(endpoint_id, ipa->available)) {
2063                         dev_err(dev, "unavailable endpoint id %u\n",
2064                                 endpoint_id);
2065                         goto err_free_bitmap;
2066                 }
2067
2068                 /* Make sure it's pointing in the right direction */
2069                 endpoint = &ipa->endpoint[endpoint_id];
2070                 if (endpoint->toward_ipa) {
2071                         if (endpoint_id < tx_count)
2072                                 continue;
2073                 } else if (endpoint_id >= rx_base) {
2074                         continue;
2075                 }
2076
2077                 dev_err(dev, "endpoint id %u wrong direction\n", endpoint_id);
2078                 goto err_free_bitmap;
2079         }
2080
2081         return 0;
2082
2083 err_free_bitmap:
2084         ipa_endpoint_deconfig(ipa);
2085
2086         return -EINVAL;
2087 }
2088
2089 static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name,
2090                                   const struct ipa_gsi_endpoint_data *data)
2091 {
2092         struct ipa_endpoint *endpoint;
2093
2094         endpoint = &ipa->endpoint[data->endpoint_id];
2095
2096         if (data->ee_id == GSI_EE_AP)
2097                 ipa->channel_map[data->channel_id] = endpoint;
2098         ipa->name_map[name] = endpoint;
2099
2100         endpoint->ipa = ipa;
2101         endpoint->ee_id = data->ee_id;
2102         endpoint->channel_id = data->channel_id;
2103         endpoint->endpoint_id = data->endpoint_id;
2104         endpoint->toward_ipa = data->toward_ipa;
2105         endpoint->config = data->endpoint.config;
2106
2107         __set_bit(endpoint->endpoint_id, ipa->defined);
2108 }
2109
2110 static void ipa_endpoint_exit_one(struct ipa_endpoint *endpoint)
2111 {
2112         __clear_bit(endpoint->endpoint_id, endpoint->ipa->defined);
2113
2114         memset(endpoint, 0, sizeof(*endpoint));
2115 }
2116
2117 void ipa_endpoint_exit(struct ipa *ipa)
2118 {
2119         u32 endpoint_id;
2120
2121         ipa->filtered = 0;
2122
2123         for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count)
2124                 ipa_endpoint_exit_one(&ipa->endpoint[endpoint_id]);
2125
2126         bitmap_free(ipa->enabled);
2127         ipa->enabled = NULL;
2128         bitmap_free(ipa->set_up);
2129         ipa->set_up = NULL;
2130         bitmap_free(ipa->defined);
2131         ipa->defined = NULL;
2132
2133         memset(ipa->name_map, 0, sizeof(ipa->name_map));
2134         memset(ipa->channel_map, 0, sizeof(ipa->channel_map));
2135 }
2136
2137 /* Returns a bitmask of endpoints that support filtering, or 0 on error */
2138 int ipa_endpoint_init(struct ipa *ipa, u32 count,
2139                       const struct ipa_gsi_endpoint_data *data)
2140 {
2141         enum ipa_endpoint_name name;
2142         u32 filtered;
2143
2144         BUILD_BUG_ON(!IPA_REPLENISH_BATCH);
2145
2146         /* Number of endpoints is one more than the maximum ID */
2147         ipa->endpoint_count = ipa_endpoint_max(ipa, count, data) + 1;
2148         if (!ipa->endpoint_count)
2149                 return -EINVAL;
2150
2151         /* Initialize endpoint state bitmaps */
2152         ipa->defined = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL);
2153         if (!ipa->defined)
2154                 return -ENOMEM;
2155
2156         ipa->set_up = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL);
2157         if (!ipa->set_up)
2158                 goto err_free_defined;
2159
2160         ipa->enabled = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL);
2161         if (!ipa->enabled)
2162                 goto err_free_set_up;
2163
2164         filtered = 0;
2165         for (name = 0; name < count; name++, data++) {
2166                 if (ipa_gsi_endpoint_data_empty(data))
2167                         continue;       /* Skip over empty slots */
2168
2169                 ipa_endpoint_init_one(ipa, name, data);
2170
2171                 if (data->endpoint.filter_support)
2172                         filtered |= BIT(data->endpoint_id);
2173                 if (data->ee_id == GSI_EE_MODEM && data->toward_ipa)
2174                         ipa->modem_tx_count++;
2175         }
2176
2177         /* Make sure the set of filtered endpoints is valid */
2178         if (!ipa_filtered_valid(ipa, filtered)) {
2179                 ipa_endpoint_exit(ipa);
2180
2181                 return -EINVAL;
2182         }
2183
2184         ipa->filtered = filtered;
2185
2186         return 0;
2187
2188 err_free_set_up:
2189         bitmap_free(ipa->set_up);
2190         ipa->set_up = NULL;
2191 err_free_defined:
2192         bitmap_free(ipa->defined);
2193         ipa->defined = NULL;
2194
2195         return -ENOMEM;
2196 }