drivers/rpmsg/qcom_glink_native.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016-2017, Linaro Ltd
   4  */
   5
   6 #include <linux/idr.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/io.h>
   9 #include <linux/list.h>
  10 #include <linux/mfd/syscon.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/of_address.h>
  14 #include <linux/platform_device.h>
  15 #include <linux/regmap.h>
  16 #include <linux/rpmsg.h>
  17 #include <linux/sizes.h>
  18 #include <linux/slab.h>
  19 #include <linux/wait.h>
  20 #include <linux/workqueue.h>
  21 #include <linux/mailbox_client.h>
  22
  23 #include "rpmsg_internal.h"
  24 #include "qcom_glink_native.h"
  25
  26 #define GLINK_NAME_SIZE         32
  27 #define GLINK_VERSION_1         1
  28
  29 #define RPM_GLINK_CID_MIN       1
  30 #define RPM_GLINK_CID_MAX       65536
  31
  32 struct glink_msg {
  33         __le16 cmd;
  34         __le16 param1;
  35         __le32 param2;
  36         u8 data[];
  37 } __packed;
  38
  39 /**
  40  * struct glink_defer_cmd - deferred incoming control message
  41  * @node:       list node
  42  * @msg:        message header
  43  * @data:       payload of the message
  44  *
  45  * Copy of a received control message, to be added to @rx_queue and processed
  46  * by @rx_work of @qcom_glink.
  47  */
  48 struct glink_defer_cmd {
  49         struct list_head node;
  50
  51         struct glink_msg msg;
  52         u8 data[];
  53 };
  54
  55 /**
  56  * struct glink_core_rx_intent - RX intent
  57  * RX intent
  58  *
  59  * @data: pointer to the data (may be NULL for zero-copy)
  60  * @id: remote or local intent ID
  61  * @size: size of the original intent (do not modify)
  62  * @reuse: To mark if the intent can be reused after first use
  63  * @in_use: To mark if intent is already in use for the channel
  64  * @offset: next write offset (initially 0)
  65  * @node:       list node
  66  */
  67 struct glink_core_rx_intent {
  68         void *data;
  69         u32 id;
  70         size_t size;
  71         bool reuse;
  72         bool in_use;
  73         u32 offset;
  74
  75         struct list_head node;
  76 };
  77
  78 /**
  79  * struct qcom_glink - driver context, relates to one remote subsystem
  80  * @dev:        reference to the associated struct device
  81  * @rx_pipe:    pipe object for receive FIFO
  82  * @tx_pipe:    pipe object for transmit FIFO
  83  * @rx_work:    worker for handling received control messages
  84  * @rx_lock:    protects the @rx_queue
  85  * @rx_queue:   queue of received control messages to be processed in @rx_work
  86  * @tx_lock:    synchronizes operations on the tx fifo
  87  * @idr_lock:   synchronizes @lcids and @rcids modifications
  88  * @lcids:      idr of all channels with a known local channel id
  89  * @rcids:      idr of all channels with a known remote channel id
  90  * @features:   remote features
  91  * @intentless: flag to indicate that there is no intent
  92  * @tx_avail_notify: Waitqueue for pending tx tasks
  93  * @sent_read_notify: flag to check cmd sent or not
  94  * @abort_tx:   flag indicating that all tx attempts should fail
  95  */
  96 struct qcom_glink {
  97         struct device *dev;
  98
  99         struct qcom_glink_pipe *rx_pipe;
 100         struct qcom_glink_pipe *tx_pipe;
 101
 102         struct work_struct rx_work;
 103         spinlock_t rx_lock;
 104         struct list_head rx_queue;
 105
 106         spinlock_t tx_lock;
 107
 108         spinlock_t idr_lock;
 109         struct idr lcids;
 110         struct idr rcids;
 111         unsigned long features;
 112
 113         bool intentless;
 114         wait_queue_head_t tx_avail_notify;
 115         bool sent_read_notify;
 116
 117         bool abort_tx;
 118 };
 119
 120 enum {
 121         GLINK_STATE_CLOSED,
 122         GLINK_STATE_OPENING,
 123         GLINK_STATE_OPEN,
 124         GLINK_STATE_CLOSING,
 125 };
 126
 127 /**
 128  * struct glink_channel - internal representation of a channel
 129  * @rpdev:      rpdev reference, only used for primary endpoints
 130  * @ept:        rpmsg endpoint this channel is associated with
 131  * @glink:      qcom_glink context handle
 132  * @refcount:   refcount for the channel object
 133  * @recv_lock:  guard for @ept.cb
 134  * @name:       unique channel name/identifier
 135  * @lcid:       channel id, in local space
 136  * @rcid:       channel id, in remote space
 137  * @intent_lock: lock for protection of @liids, @riids
 138  * @liids:      idr of all local intents
 139  * @riids:      idr of all remote intents
 140  * @intent_work: worker responsible for transmitting rx_done packets
 141  * @done_intents: list of intents that needs to be announced rx_done
 142  * @buf:        receive buffer, for gathering fragments
 143  * @buf_offset: write offset in @buf
 144  * @buf_size:   size of current @buf
 145  * @open_ack:   completed once remote has acked the open-request
 146  * @open_req:   completed once open-request has been received
 147  * @intent_req_lock: Synchronises multiple intent requests
 148  * @intent_req_result: Result of intent request
 149  * @intent_received: flag indicating that an intent has been received
 150  * @intent_req_wq: wait queue for intent_req signalling
 151  */
 152 struct glink_channel {
 153         struct rpmsg_endpoint ept;
 154
 155         struct rpmsg_device *rpdev;
 156         struct qcom_glink *glink;
 157
 158         struct kref refcount;
 159
 160         spinlock_t recv_lock;
 161
 162         char *name;
 163         unsigned int lcid;
 164         unsigned int rcid;
 165
 166         spinlock_t intent_lock;
 167         struct idr liids;
 168         struct idr riids;
 169         struct work_struct intent_work;
 170         struct list_head done_intents;
 171
 172         struct glink_core_rx_intent *buf;
 173         int buf_offset;
 174         int buf_size;
 175
 176         struct completion open_ack;
 177         struct completion open_req;
 178
 179         struct mutex intent_req_lock;
 180         int intent_req_result;
 181         bool intent_received;
 182         wait_queue_head_t intent_req_wq;
 183 };
 184
 185 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
 186
 187 static const struct rpmsg_endpoint_ops glink_endpoint_ops;
 188
 189 #define GLINK_CMD_VERSION               0
 190 #define GLINK_CMD_VERSION_ACK           1
 191 #define GLINK_CMD_OPEN                  2
 192 #define GLINK_CMD_CLOSE                 3
 193 #define GLINK_CMD_OPEN_ACK              4
 194 #define GLINK_CMD_INTENT                5
 195 #define GLINK_CMD_RX_DONE               6
 196 #define GLINK_CMD_RX_INTENT_REQ         7
 197 #define GLINK_CMD_RX_INTENT_REQ_ACK     8
 198 #define GLINK_CMD_TX_DATA               9
 199 #define GLINK_CMD_CLOSE_ACK             11
 200 #define GLINK_CMD_TX_DATA_CONT          12
 201 #define GLINK_CMD_READ_NOTIF            13
 202 #define GLINK_CMD_RX_DONE_W_REUSE       14
 203 #define GLINK_CMD_SIGNALS               15
 204
 205 #define GLINK_FEATURE_INTENTLESS        BIT(1)
 206
 207 #define NATIVE_DTR_SIG                  NATIVE_DSR_SIG
 208 #define NATIVE_DSR_SIG                  BIT(31)
 209 #define NATIVE_RTS_SIG                  NATIVE_CTS_SIG
 210 #define NATIVE_CTS_SIG                  BIT(30)
 211
 212 static void qcom_glink_rx_done_work(struct work_struct *work);
 213
 214 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
 215                                                       const char *name)
 216 {
 217         struct glink_channel *channel;
 218
 219         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 220         if (!channel)
 221                 return ERR_PTR(-ENOMEM);
 222
 223         /* Setup glink internal glink_channel data */
 224         spin_lock_init(&channel->recv_lock);
 225         spin_lock_init(&channel->intent_lock);
 226         mutex_init(&channel->intent_req_lock);
 227
 228         channel->glink = glink;
 229         channel->name = kstrdup(name, GFP_KERNEL);
 230         if (!channel->name) {
 231                 kfree(channel);
 232                 return ERR_PTR(-ENOMEM);
 233         }
 234
 235         init_completion(&channel->open_req);
 236         init_completion(&channel->open_ack);
 237         init_waitqueue_head(&channel->intent_req_wq);
 238
 239         INIT_LIST_HEAD(&channel->done_intents);
 240         INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
 241
 242         idr_init(&channel->liids);
 243         idr_init(&channel->riids);
 244         kref_init(&channel->refcount);
 245
 246         return channel;
 247 }
 248
 249 static void qcom_glink_channel_release(struct kref *ref)
 250 {
 251         struct glink_channel *channel = container_of(ref, struct glink_channel,
 252                                                      refcount);
 253         struct glink_core_rx_intent *intent;
 254         struct glink_core_rx_intent *tmp;
 255         unsigned long flags;
 256         int iid;
 257
 258         /* cancel pending rx_done work */
 259         cancel_work_sync(&channel->intent_work);
 260
 261         spin_lock_irqsave(&channel->intent_lock, flags);
 262         /* Free all non-reuse intents pending rx_done work */
 263         list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 264                 if (!intent->reuse) {
 265                         kfree(intent->data);
 266                         kfree(intent);
 267                 }
 268         }
 269
 270         idr_for_each_entry(&channel->liids, tmp, iid) {
 271                 kfree(tmp->data);
 272                 kfree(tmp);
 273         }
 274         idr_destroy(&channel->liids);
 275
 276         idr_for_each_entry(&channel->riids, tmp, iid)
 277                 kfree(tmp);
 278         idr_destroy(&channel->riids);
 279         spin_unlock_irqrestore(&channel->intent_lock, flags);
 280
 281         kfree(channel->name);
 282         kfree(channel);
 283 }
 284
 285 static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
 286 {
 287         return glink->rx_pipe->avail(glink->rx_pipe);
 288 }
 289
 290 static void qcom_glink_rx_peek(struct qcom_glink *glink,
 291                                void *data, unsigned int offset, size_t count)
 292 {
 293         glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
 294 }
 295
 296 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
 297 {
 298         glink->rx_pipe->advance(glink->rx_pipe, count);
 299 }
 300
 301 static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
 302 {
 303         return glink->tx_pipe->avail(glink->tx_pipe);
 304 }
 305
 306 static void qcom_glink_tx_write(struct qcom_glink *glink,
 307                                 const void *hdr, size_t hlen,
 308                                 const void *data, size_t dlen)
 309 {
 310         glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
 311 }
 312
 313 static void qcom_glink_tx_kick(struct qcom_glink *glink)
 314 {
 315         glink->tx_pipe->kick(glink->tx_pipe);
 316 }
 317
 318 static void qcom_glink_send_read_notify(struct qcom_glink *glink)
 319 {
 320         struct glink_msg msg;
 321
 322         msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
 323         msg.param1 = 0;
 324         msg.param2 = 0;
 325
 326         qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0);
 327
 328         qcom_glink_tx_kick(glink);
 329 }
 330
 331 static int qcom_glink_tx(struct qcom_glink *glink,
 332                          const void *hdr, size_t hlen,
 333                          const void *data, size_t dlen, bool wait)
 334 {
 335         unsigned int tlen = hlen + dlen;
 336         unsigned long flags;
 337         int ret = 0;
 338
 339         /* Reject packets that are too big */
 340         if (tlen >= glink->tx_pipe->length)
 341                 return -EINVAL;
 342
 343         spin_lock_irqsave(&glink->tx_lock, flags);
 344
 345         if (glink->abort_tx) {
 346                 ret = -EIO;
 347                 goto out;
 348         }
 349
 350         while (qcom_glink_tx_avail(glink) < tlen) {
 351                 if (!wait) {
 352                         ret = -EAGAIN;
 353                         goto out;
 354                 }
 355
 356                 if (glink->abort_tx) {
 357                         ret = -EIO;
 358                         goto out;
 359                 }
 360
 361                 if (!glink->sent_read_notify) {
 362                         glink->sent_read_notify = true;
 363                         qcom_glink_send_read_notify(glink);
 364                 }
 365
 366                 /* Wait without holding the tx_lock */
 367                 spin_unlock_irqrestore(&glink->tx_lock, flags);
 368
 369                 wait_event_timeout(glink->tx_avail_notify,
 370                                    qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
 371
 372                 spin_lock_irqsave(&glink->tx_lock, flags);
 373
 374                 if (qcom_glink_tx_avail(glink) >= tlen)
 375                         glink->sent_read_notify = false;
 376         }
 377
 378         qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
 379         qcom_glink_tx_kick(glink);
 380
 381 out:
 382         spin_unlock_irqrestore(&glink->tx_lock, flags);
 383
 384         return ret;
 385 }
 386
 387 static int qcom_glink_send_version(struct qcom_glink *glink)
 388 {
 389         struct glink_msg msg;
 390
 391         msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
 392         msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 393         msg.param2 = cpu_to_le32(glink->features);
 394
 395         return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 396 }
 397
 398 static void qcom_glink_send_version_ack(struct qcom_glink *glink)
 399 {
 400         struct glink_msg msg;
 401
 402         msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
 403         msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 404         msg.param2 = cpu_to_le32(glink->features);
 405
 406         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 407 }
 408
 409 static void qcom_glink_send_open_ack(struct qcom_glink *glink,
 410                                      struct glink_channel *channel)
 411 {
 412         struct glink_msg msg;
 413
 414         msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
 415         msg.param1 = cpu_to_le16(channel->rcid);
 416         msg.param2 = cpu_to_le32(0);
 417
 418         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 419 }
 420
 421 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
 422                                              unsigned int cid, bool granted)
 423 {
 424         struct glink_channel *channel;
 425         unsigned long flags;
 426
 427         spin_lock_irqsave(&glink->idr_lock, flags);
 428         channel = idr_find(&glink->rcids, cid);
 429         spin_unlock_irqrestore(&glink->idr_lock, flags);
 430         if (!channel) {
 431                 dev_err(glink->dev, "unable to find channel\n");
 432                 return;
 433         }
 434
 435         WRITE_ONCE(channel->intent_req_result, granted);
 436         wake_up_all(&channel->intent_req_wq);
 437 }
 438
 439 static void qcom_glink_intent_req_abort(struct glink_channel *channel)
 440 {
 441         WRITE_ONCE(channel->intent_req_result, 0);
 442         wake_up_all(&channel->intent_req_wq);
 443 }
 444
 445 /**
 446  * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
 447  * @glink: Ptr to the glink edge
 448  * @channel: Ptr to the channel that the open req is sent
 449  *
 450  * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
 451  * Will return with refcount held, regardless of outcome.
 452  *
 453  * Return: 0 on success, negative errno otherwise.
 454  */
 455 static int qcom_glink_send_open_req(struct qcom_glink *glink,
 456                                     struct glink_channel *channel)
 457 {
 458         struct {
 459                 struct glink_msg msg;
 460                 u8 name[GLINK_NAME_SIZE];
 461         } __packed req;
 462         int name_len = strlen(channel->name) + 1;
 463         int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
 464         int ret;
 465         unsigned long flags;
 466
 467         kref_get(&channel->refcount);
 468
 469         spin_lock_irqsave(&glink->idr_lock, flags);
 470         ret = idr_alloc_cyclic(&glink->lcids, channel,
 471                                RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
 472                                GFP_ATOMIC);
 473         spin_unlock_irqrestore(&glink->idr_lock, flags);
 474         if (ret < 0)
 475                 return ret;
 476
 477         channel->lcid = ret;
 478
 479         req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN);
 480         req.msg.param1 = cpu_to_le16(channel->lcid);
 481         req.msg.param2 = cpu_to_le32(name_len);
 482         strcpy(req.name, channel->name);
 483
 484         ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true);
 485         if (ret)
 486                 goto remove_idr;
 487
 488         return 0;
 489
 490 remove_idr:
 491         spin_lock_irqsave(&glink->idr_lock, flags);
 492         idr_remove(&glink->lcids, channel->lcid);
 493         channel->lcid = 0;
 494         spin_unlock_irqrestore(&glink->idr_lock, flags);
 495
 496         return ret;
 497 }
 498
 499 static void qcom_glink_send_close_req(struct qcom_glink *glink,
 500                                       struct glink_channel *channel)
 501 {
 502         struct glink_msg req;
 503
 504         req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
 505         req.param1 = cpu_to_le16(channel->lcid);
 506         req.param2 = 0;
 507
 508         qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 509 }
 510
 511 static void qcom_glink_send_close_ack(struct qcom_glink *glink,
 512                                       unsigned int rcid)
 513 {
 514         struct glink_msg req;
 515
 516         req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
 517         req.param1 = cpu_to_le16(rcid);
 518         req.param2 = 0;
 519
 520         qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 521 }
 522
 523 static void qcom_glink_rx_done_work(struct work_struct *work)
 524 {
 525         struct glink_channel *channel = container_of(work, struct glink_channel,
 526                                                      intent_work);
 527         struct qcom_glink *glink = channel->glink;
 528         struct glink_core_rx_intent *intent, *tmp;
 529         struct {
 530                 u16 id;
 531                 u16 lcid;
 532                 u32 liid;
 533         } __packed cmd;
 534
 535         unsigned int cid = channel->lcid;
 536         unsigned int iid;
 537         bool reuse;
 538         unsigned long flags;
 539
 540         spin_lock_irqsave(&channel->intent_lock, flags);
 541         list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 542                 list_del(&intent->node);
 543                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 544                 iid = intent->id;
 545                 reuse = intent->reuse;
 546
 547                 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
 548                 cmd.lcid = cid;
 549                 cmd.liid = iid;
 550
 551                 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 552                 if (!reuse) {
 553                         kfree(intent->data);
 554                         kfree(intent);
 555                 }
 556                 spin_lock_irqsave(&channel->intent_lock, flags);
 557         }
 558         spin_unlock_irqrestore(&channel->intent_lock, flags);
 559 }
 560
 561 static void qcom_glink_rx_done(struct qcom_glink *glink,
 562                                struct glink_channel *channel,
 563                                struct glink_core_rx_intent *intent)
 564 {
 565         /* We don't send RX_DONE to intentless systems */
 566         if (glink->intentless) {
 567                 kfree(intent->data);
 568                 kfree(intent);
 569                 return;
 570         }
 571
 572         /* Take it off the tree of receive intents */
 573         if (!intent->reuse) {
 574                 spin_lock(&channel->intent_lock);
 575                 idr_remove(&channel->liids, intent->id);
 576                 spin_unlock(&channel->intent_lock);
 577         }
 578
 579         /* Schedule the sending of a rx_done indication */
 580         spin_lock(&channel->intent_lock);
 581         list_add_tail(&intent->node, &channel->done_intents);
 582         spin_unlock(&channel->intent_lock);
 583
 584         schedule_work(&channel->intent_work);
 585 }
 586
 587 /**
 588  * qcom_glink_receive_version() - receive version/features from remote system
 589  *
 590  * @glink:      pointer to transport interface
 591  * @version:    remote version
 592  * @features:   remote features
 593  *
 594  * This function is called in response to a remote-initiated version/feature
 595  * negotiation sequence.
 596  */
 597 static void qcom_glink_receive_version(struct qcom_glink *glink,
 598                                        u32 version,
 599                                        u32 features)
 600 {
 601         switch (version) {
 602         case 0:
 603                 break;
 604         case GLINK_VERSION_1:
 605                 glink->features &= features;
 606                 fallthrough;
 607         default:
 608                 qcom_glink_send_version_ack(glink);
 609                 break;
 610         }
 611 }
 612
 613 /**
 614  * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
 615  *
 616  * @glink:      pointer to transport interface
 617  * @version:    remote version response
 618  * @features:   remote features response
 619  *
 620  * This function is called in response to a local-initiated version/feature
 621  * negotiation sequence and is the counter-offer from the remote side based
 622  * upon the initial version and feature set requested.
 623  */
 624 static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
 625                                            u32 version,
 626                                            u32 features)
 627 {
 628         switch (version) {
 629         case 0:
 630                 /* Version negotiation failed */
 631                 break;
 632         case GLINK_VERSION_1:
 633                 if (features == glink->features)
 634                         break;
 635
 636                 glink->features &= features;
 637                 fallthrough;
 638         default:
 639                 qcom_glink_send_version(glink);
 640                 break;
 641         }
 642 }
 643
 644 /**
 645  * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
 646  *      wire format and transmit
 647  * @glink:      The transport to transmit on.
 648  * @channel:    The glink channel
 649  * @granted:    The request response to encode.
 650  *
 651  * Return: 0 on success or standard Linux error code.
 652  */
 653 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
 654                                           struct glink_channel *channel,
 655                                           bool granted)
 656 {
 657         struct glink_msg msg;
 658
 659         msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
 660         msg.param1 = cpu_to_le16(channel->lcid);
 661         msg.param2 = cpu_to_le32(granted);
 662
 663         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 664
 665         return 0;
 666 }
 667
 668 /**
 669  * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
 670  *                         transmit
 671  * @glink:      The transport to transmit on.
 672  * @channel:    The local channel
 673  * @intent:     The intent to pass on to remote.
 674  *
 675  * Return: 0 on success or standard Linux error code.
 676  */
 677 static int qcom_glink_advertise_intent(struct qcom_glink *glink,
 678                                        struct glink_channel *channel,
 679                                        struct glink_core_rx_intent *intent)
 680 {
 681         struct command {
 682                 __le16 id;
 683                 __le16 lcid;
 684                 __le32 count;
 685                 __le32 size;
 686                 __le32 liid;
 687         } __packed;
 688         struct command cmd;
 689
 690         cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
 691         cmd.lcid = cpu_to_le16(channel->lcid);
 692         cmd.count = cpu_to_le32(1);
 693         cmd.size = cpu_to_le32(intent->size);
 694         cmd.liid = cpu_to_le32(intent->id);
 695
 696         qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 697
 698         return 0;
 699 }
 700
 701 static struct glink_core_rx_intent *
 702 qcom_glink_alloc_intent(struct qcom_glink *glink,
 703                         struct glink_channel *channel,
 704                         size_t size,
 705                         bool reuseable)
 706 {
 707         struct glink_core_rx_intent *intent;
 708         int ret;
 709         unsigned long flags;
 710
 711         intent = kzalloc(sizeof(*intent), GFP_KERNEL);
 712         if (!intent)
 713                 return NULL;
 714
 715         intent->data = kzalloc(size, GFP_KERNEL);
 716         if (!intent->data)
 717                 goto free_intent;
 718
 719         spin_lock_irqsave(&channel->intent_lock, flags);
 720         ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
 721         if (ret < 0) {
 722                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 723                 goto free_data;
 724         }
 725         spin_unlock_irqrestore(&channel->intent_lock, flags);
 726
 727         intent->id = ret;
 728         intent->size = size;
 729         intent->reuse = reuseable;
 730
 731         return intent;
 732
 733 free_data:
 734         kfree(intent->data);
 735 free_intent:
 736         kfree(intent);
 737         return NULL;
 738 }
 739
 740 static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
 741                                       u32 cid, uint32_t iid,
 742                                       bool reuse)
 743 {
 744         struct glink_core_rx_intent *intent;
 745         struct glink_channel *channel;
 746         unsigned long flags;
 747
 748         spin_lock_irqsave(&glink->idr_lock, flags);
 749         channel = idr_find(&glink->rcids, cid);
 750         spin_unlock_irqrestore(&glink->idr_lock, flags);
 751         if (!channel) {
 752                 dev_err(glink->dev, "invalid channel id received\n");
 753                 return;
 754         }
 755
 756         spin_lock_irqsave(&channel->intent_lock, flags);
 757         intent = idr_find(&channel->riids, iid);
 758
 759         if (!intent) {
 760                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 761                 dev_err(glink->dev, "invalid intent id received\n");
 762                 return;
 763         }
 764
 765         intent->in_use = false;
 766
 767         if (!reuse) {
 768                 idr_remove(&channel->riids, intent->id);
 769                 kfree(intent);
 770         }
 771         spin_unlock_irqrestore(&channel->intent_lock, flags);
 772
 773         if (reuse) {
 774                 WRITE_ONCE(channel->intent_received, true);
 775                 wake_up_all(&channel->intent_req_wq);
 776         }
 777 }
 778
 779 /**
 780  * qcom_glink_handle_intent_req() - Receive a request for rx_intent
 781  *                                          from remote side
 782  * @glink:      Pointer to the transport interface
 783  * @cid:        Remote channel ID
 784  * @size:       size of the intent
 785  *
 786  * The function searches for the local channel to which the request for
 787  * rx_intent has arrived and allocates and notifies the remote back
 788  */
 789 static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
 790                                          u32 cid, size_t size)
 791 {
 792         struct glink_core_rx_intent *intent;
 793         struct glink_channel *channel;
 794         unsigned long flags;
 795
 796         spin_lock_irqsave(&glink->idr_lock, flags);
 797         channel = idr_find(&glink->rcids, cid);
 798         spin_unlock_irqrestore(&glink->idr_lock, flags);
 799
 800         if (!channel) {
 801                 pr_err("%s channel not found for cid %d\n", __func__, cid);
 802                 return;
 803         }
 804
 805         intent = qcom_glink_alloc_intent(glink, channel, size, false);
 806         if (intent)
 807                 qcom_glink_advertise_intent(glink, channel, intent);
 808
 809         qcom_glink_send_intent_req_ack(glink, channel, !!intent);
 810 }
 811
 812 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
 813 {
 814         struct glink_defer_cmd *dcmd;
 815
 816         extra = ALIGN(extra, 8);
 817
 818         if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
 819                 dev_dbg(glink->dev, "Insufficient data in rx fifo");
 820                 return -ENXIO;
 821         }
 822
 823         dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
 824         if (!dcmd)
 825                 return -ENOMEM;
 826
 827         INIT_LIST_HEAD(&dcmd->node);
 828
 829         qcom_glink_rx_peek(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra);
 830
 831         spin_lock(&glink->rx_lock);
 832         list_add_tail(&dcmd->node, &glink->rx_queue);
 833         spin_unlock(&glink->rx_lock);
 834
 835         schedule_work(&glink->rx_work);
 836         qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
 837
 838         return 0;
 839 }
 840
 841 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
 842 {
 843         struct glink_core_rx_intent *intent;
 844         struct glink_channel *channel;
 845         struct {
 846                 struct glink_msg msg;
 847                 __le32 chunk_size;
 848                 __le32 left_size;
 849         } __packed hdr;
 850         unsigned int chunk_size;
 851         unsigned int left_size;
 852         unsigned int rcid;
 853         unsigned int liid;
 854         int ret = 0;
 855         unsigned long flags;
 856
 857         if (avail < sizeof(hdr)) {
 858                 dev_dbg(glink->dev, "Not enough data in fifo\n");
 859                 return -EAGAIN;
 860         }
 861
 862         qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr));
 863         chunk_size = le32_to_cpu(hdr.chunk_size);
 864         left_size = le32_to_cpu(hdr.left_size);
 865
 866         if (avail < sizeof(hdr) + chunk_size) {
 867                 dev_dbg(glink->dev, "Payload not yet in fifo\n");
 868                 return -EAGAIN;
 869         }
 870
 871         rcid = le16_to_cpu(hdr.msg.param1);
 872         spin_lock_irqsave(&glink->idr_lock, flags);
 873         channel = idr_find(&glink->rcids, rcid);
 874         spin_unlock_irqrestore(&glink->idr_lock, flags);
 875         if (!channel) {
 876                 dev_dbg(glink->dev, "Data on non-existing channel\n");
 877
 878                 /* Drop the message */
 879                 goto advance_rx;
 880         }
 881
 882         if (glink->intentless) {
 883                 /* Might have an ongoing, fragmented, message to append */
 884                 if (!channel->buf) {
 885                         intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 886                         if (!intent)
 887                                 return -ENOMEM;
 888
 889                         intent->data = kmalloc(chunk_size + left_size,
 890                                                GFP_ATOMIC);
 891                         if (!intent->data) {
 892                                 kfree(intent);
 893                                 return -ENOMEM;
 894                         }
 895
 896                         intent->id = 0xbabababa;
 897                         intent->size = chunk_size + left_size;
 898                         intent->offset = 0;
 899
 900                         channel->buf = intent;
 901                 } else {
 902                         intent = channel->buf;
 903                 }
 904         } else {
 905                 liid = le32_to_cpu(hdr.msg.param2);
 906
 907                 spin_lock_irqsave(&channel->intent_lock, flags);
 908                 intent = idr_find(&channel->liids, liid);
 909                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 910
 911                 if (!intent) {
 912                         dev_err(glink->dev,
 913                                 "no intent found for channel %s intent %d",
 914                                 channel->name, liid);
 915                         ret = -ENOENT;
 916                         goto advance_rx;
 917                 }
 918         }
 919
 920         if (intent->size - intent->offset < chunk_size) {
 921                 dev_err(glink->dev, "Insufficient space in intent\n");
 922
 923                 /* The packet header lied, drop payload */
 924                 goto advance_rx;
 925         }
 926
 927         qcom_glink_rx_peek(glink, intent->data + intent->offset,
 928                            sizeof(hdr), chunk_size);
 929         intent->offset += chunk_size;
 930
 931         /* Handle message when no fragments remain to be received */
 932         if (!left_size) {
 933                 spin_lock(&channel->recv_lock);
 934                 if (channel->ept.cb) {
 935                         channel->ept.cb(channel->ept.rpdev,
 936                                         intent->data,
 937                                         intent->offset,
 938                                         channel->ept.priv,
 939                                         RPMSG_ADDR_ANY);
 940                 }
 941                 spin_unlock(&channel->recv_lock);
 942
 943                 intent->offset = 0;
 944                 channel->buf = NULL;
 945
 946                 qcom_glink_rx_done(glink, channel, intent);
 947         }
 948
 949 advance_rx:
 950         qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
 951
 952         return ret;
 953 }
 954
 955 static void qcom_glink_handle_intent(struct qcom_glink *glink,
 956                                      unsigned int cid,
 957                                      unsigned int count,
 958                                      size_t avail)
 959 {
 960         struct glink_core_rx_intent *intent;
 961         struct glink_channel *channel;
 962         struct intent_pair {
 963                 __le32 size;
 964                 __le32 iid;
 965         };
 966
 967         struct {
 968                 struct glink_msg msg;
 969                 struct intent_pair intents[];
 970         } __packed * msg;
 971
 972         const size_t msglen = struct_size(msg, intents, count);
 973         int ret;
 974         int i;
 975         unsigned long flags;
 976
 977         if (avail < msglen) {
 978                 dev_dbg(glink->dev, "Not enough data in fifo\n");
 979                 return;
 980         }
 981
 982         spin_lock_irqsave(&glink->idr_lock, flags);
 983         channel = idr_find(&glink->rcids, cid);
 984         spin_unlock_irqrestore(&glink->idr_lock, flags);
 985         if (!channel) {
 986                 dev_err(glink->dev, "intents for non-existing channel\n");
 987                 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
 988                 return;
 989         }
 990
 991         msg = kmalloc(msglen, GFP_ATOMIC);
 992         if (!msg)
 993                 return;
 994
 995         qcom_glink_rx_peek(glink, msg, 0, msglen);
 996
 997         for (i = 0; i < count; ++i) {
 998                 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 999                 if (!intent)
1000                         break;
1001
1002                 intent->id = le32_to_cpu(msg->intents[i].iid);
1003                 intent->size = le32_to_cpu(msg->intents[i].size);
1004
1005                 spin_lock_irqsave(&channel->intent_lock, flags);
1006                 ret = idr_alloc(&channel->riids, intent,
1007                                 intent->id, intent->id + 1, GFP_ATOMIC);
1008                 spin_unlock_irqrestore(&channel->intent_lock, flags);
1009
1010                 if (ret < 0)
1011                         dev_err(glink->dev, "failed to store remote intent\n");
1012         }
1013
1014         WRITE_ONCE(channel->intent_received, true);
1015         wake_up_all(&channel->intent_req_wq);
1016
1017         kfree(msg);
1018         qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1019 }
1020
1021 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1022 {
1023         struct glink_channel *channel;
1024
1025         spin_lock(&glink->idr_lock);
1026         channel = idr_find(&glink->lcids, lcid);
1027         spin_unlock(&glink->idr_lock);
1028         if (!channel) {
1029                 dev_err(glink->dev, "Invalid open ack packet\n");
1030                 return -EINVAL;
1031         }
1032
1033         complete_all(&channel->open_ack);
1034
1035         return 0;
1036 }
1037
1038 /**
1039  * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1040  * @ept:        Rpmsg endpoint for channel.
1041  * @pause:      Pause transmission
1042  * @dst:        destination address of the endpoint
1043  *
1044  * Return: 0 on success or standard Linux error code.
1045  */
1046 static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1047 {
1048         struct glink_channel *channel = to_glink_channel(ept);
1049         struct qcom_glink *glink = channel->glink;
1050         struct glink_msg msg;
1051         u32 sigs = 0;
1052
1053         if (pause)
1054                 sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1055
1056         msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1057         msg.param1 = cpu_to_le16(channel->lcid);
1058         msg.param2 = cpu_to_le32(sigs);
1059
1060         return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
1061 }
1062
1063 static void qcom_glink_handle_signals(struct qcom_glink *glink,
1064                                       unsigned int rcid, unsigned int sigs)
1065 {
1066         struct glink_channel *channel;
1067         unsigned long flags;
1068         bool enable;
1069
1070         spin_lock_irqsave(&glink->idr_lock, flags);
1071         channel = idr_find(&glink->rcids, rcid);
1072         spin_unlock_irqrestore(&glink->idr_lock, flags);
1073         if (!channel)
1074                 dev_err(glink->dev, "signal for non-existing channel\n");
1075
1076         enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1077
1078         if (channel->ept.flow_cb)
1079                 channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1080 }
1081
1082 void qcom_glink_native_rx(struct qcom_glink *glink)
1083 {
1084         struct glink_msg msg;
1085         unsigned int param1;
1086         unsigned int param2;
1087         unsigned int avail;
1088         unsigned int cmd;
1089         int ret = 0;
1090
1091         /* To wakeup any blocking writers */
1092         wake_up_all(&glink->tx_avail_notify);
1093
1094         for (;;) {
1095                 avail = qcom_glink_rx_avail(glink);
1096                 if (avail < sizeof(msg))
1097                         break;
1098
1099                 qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg));
1100
1101                 cmd = le16_to_cpu(msg.cmd);
1102                 param1 = le16_to_cpu(msg.param1);
1103                 param2 = le32_to_cpu(msg.param2);
1104
1105                 switch (cmd) {
1106                 case GLINK_CMD_VERSION:
1107                 case GLINK_CMD_VERSION_ACK:
1108                 case GLINK_CMD_CLOSE:
1109                 case GLINK_CMD_CLOSE_ACK:
1110                 case GLINK_CMD_RX_INTENT_REQ:
1111                         ret = qcom_glink_rx_defer(glink, 0);
1112                         break;
1113                 case GLINK_CMD_OPEN_ACK:
1114                         ret = qcom_glink_rx_open_ack(glink, param1);
1115                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1116                         break;
1117                 case GLINK_CMD_OPEN:
1118                         ret = qcom_glink_rx_defer(glink, param2);
1119                         break;
1120                 case GLINK_CMD_TX_DATA:
1121                 case GLINK_CMD_TX_DATA_CONT:
1122                         ret = qcom_glink_rx_data(glink, avail);
1123                         break;
1124                 case GLINK_CMD_READ_NOTIF:
1125                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1126                         qcom_glink_tx_kick(glink);
1127                         break;
1128                 case GLINK_CMD_INTENT:
1129                         qcom_glink_handle_intent(glink, param1, param2, avail);
1130                         break;
1131                 case GLINK_CMD_RX_DONE:
1132                         qcom_glink_handle_rx_done(glink, param1, param2, false);
1133                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1134                         break;
1135                 case GLINK_CMD_RX_DONE_W_REUSE:
1136                         qcom_glink_handle_rx_done(glink, param1, param2, true);
1137                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1138                         break;
1139                 case GLINK_CMD_RX_INTENT_REQ_ACK:
1140                         qcom_glink_handle_intent_req_ack(glink, param1, param2);
1141                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1142                         break;
1143                 case GLINK_CMD_SIGNALS:
1144                         qcom_glink_handle_signals(glink, param1, param2);
1145                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1146                         break;
1147                 default:
1148                         dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1149                         ret = -EINVAL;
1150                         break;
1151                 }
1152
1153                 if (ret)
1154                         break;
1155         }
1156 }
1157 EXPORT_SYMBOL(qcom_glink_native_rx);
1158
1159 /* Locally initiated rpmsg_create_ept */
1160 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1161                                                      const char *name)
1162 {
1163         struct glink_channel *channel;
1164         int ret;
1165         unsigned long flags;
1166
1167         channel = qcom_glink_alloc_channel(glink, name);
1168         if (IS_ERR(channel))
1169                 return ERR_CAST(channel);
1170
1171         ret = qcom_glink_send_open_req(glink, channel);
1172         if (ret)
1173                 goto release_channel;
1174
1175         ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1176         if (!ret)
1177                 goto err_timeout;
1178
1179         ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1180         if (!ret)
1181                 goto err_timeout;
1182
1183         qcom_glink_send_open_ack(glink, channel);
1184
1185         return channel;
1186
1187 err_timeout:
1188         /* qcom_glink_send_open_req() did register the channel in lcids*/
1189         spin_lock_irqsave(&glink->idr_lock, flags);
1190         idr_remove(&glink->lcids, channel->lcid);
1191         spin_unlock_irqrestore(&glink->idr_lock, flags);
1192
1193 release_channel:
1194         /* Release qcom_glink_send_open_req() reference */
1195         kref_put(&channel->refcount, qcom_glink_channel_release);
1196         /* Release qcom_glink_alloc_channel() reference */
1197         kref_put(&channel->refcount, qcom_glink_channel_release);
1198
1199         return ERR_PTR(-ETIMEDOUT);
1200 }
1201
1202 /* Remote initiated rpmsg_create_ept */
1203 static int qcom_glink_create_remote(struct qcom_glink *glink,
1204                                     struct glink_channel *channel)
1205 {
1206         int ret;
1207
1208         qcom_glink_send_open_ack(glink, channel);
1209
1210         ret = qcom_glink_send_open_req(glink, channel);
1211         if (ret)
1212                 goto close_link;
1213
1214         ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1215         if (!ret) {
1216                 ret = -ETIMEDOUT;
1217                 goto close_link;
1218         }
1219
1220         return 0;
1221
1222 close_link:
1223         /*
1224          * Send a close request to "undo" our open-ack. The close-ack will
1225          * release qcom_glink_send_open_req() reference and the last reference
1226          * will be relesed after receiving remote_close or transport unregister
1227          * by calling qcom_glink_native_remove().
1228          */
1229         qcom_glink_send_close_req(glink, channel);
1230
1231         return ret;
1232 }
1233
1234 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1235                                                     rpmsg_rx_cb_t cb,
1236                                                     void *priv,
1237                                                     struct rpmsg_channel_info
1238                                                                         chinfo)
1239 {
1240         struct glink_channel *parent = to_glink_channel(rpdev->ept);
1241         struct glink_channel *channel;
1242         struct qcom_glink *glink = parent->glink;
1243         struct rpmsg_endpoint *ept;
1244         const char *name = chinfo.name;
1245         int cid;
1246         int ret;
1247         unsigned long flags;
1248
1249         spin_lock_irqsave(&glink->idr_lock, flags);
1250         idr_for_each_entry(&glink->rcids, channel, cid) {
1251                 if (!strcmp(channel->name, name))
1252                         break;
1253         }
1254         spin_unlock_irqrestore(&glink->idr_lock, flags);
1255
1256         if (!channel) {
1257                 channel = qcom_glink_create_local(glink, name);
1258                 if (IS_ERR(channel))
1259                         return NULL;
1260         } else {
1261                 ret = qcom_glink_create_remote(glink, channel);
1262                 if (ret)
1263                         return NULL;
1264         }
1265
1266         ept = &channel->ept;
1267         ept->rpdev = rpdev;
1268         ept->cb = cb;
1269         ept->priv = priv;
1270         ept->ops = &glink_endpoint_ops;
1271
1272         return ept;
1273 }
1274
1275 static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1276 {
1277         struct glink_channel *channel = to_glink_channel(rpdev->ept);
1278         struct device_node *np = rpdev->dev.of_node;
1279         struct qcom_glink *glink = channel->glink;
1280         struct glink_core_rx_intent *intent;
1281         const struct property *prop = NULL;
1282         __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1283         int num_intents;
1284         int num_groups = 1;
1285         __be32 *val = defaults;
1286         int size;
1287
1288         if (glink->intentless || !completion_done(&channel->open_ack))
1289                 return 0;
1290
1291         prop = of_find_property(np, "qcom,intents", NULL);
1292         if (prop) {
1293                 val = prop->value;
1294                 num_groups = prop->length / sizeof(u32) / 2;
1295         }
1296
1297         /* Channel is now open, advertise base set of intents */
1298         while (num_groups--) {
1299                 size = be32_to_cpup(val++);
1300                 num_intents = be32_to_cpup(val++);
1301                 while (num_intents--) {
1302                         intent = qcom_glink_alloc_intent(glink, channel, size,
1303                                                          true);
1304                         if (!intent)
1305                                 break;
1306
1307                         qcom_glink_advertise_intent(glink, channel, intent);
1308                 }
1309         }
1310         return 0;
1311 }
1312
1313 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1314 {
1315         struct glink_channel *channel = to_glink_channel(ept);
1316         struct qcom_glink *glink = channel->glink;
1317         unsigned long flags;
1318
1319         spin_lock_irqsave(&channel->recv_lock, flags);
1320         channel->ept.cb = NULL;
1321         spin_unlock_irqrestore(&channel->recv_lock, flags);
1322
1323         /* Decouple the potential rpdev from the channel */
1324         channel->rpdev = NULL;
1325
1326         qcom_glink_send_close_req(glink, channel);
1327 }
1328
1329 static int qcom_glink_request_intent(struct qcom_glink *glink,
1330                                      struct glink_channel *channel,
1331                                      size_t size)
1332 {
1333         struct {
1334                 u16 id;
1335                 u16 cid;
1336                 u32 size;
1337         } __packed cmd;
1338
1339         int ret;
1340
1341         mutex_lock(&channel->intent_req_lock);
1342
1343         WRITE_ONCE(channel->intent_req_result, -1);
1344         WRITE_ONCE(channel->intent_received, false);
1345
1346         cmd.id = GLINK_CMD_RX_INTENT_REQ;
1347         cmd.cid = channel->lcid;
1348         cmd.size = size;
1349
1350         ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1351         if (ret)
1352                 goto unlock;
1353
1354         ret = wait_event_timeout(channel->intent_req_wq,
1355                                  READ_ONCE(channel->intent_req_result) >= 0 &&
1356                                  READ_ONCE(channel->intent_received),
1357                                  10 * HZ);
1358         if (!ret) {
1359                 dev_err(glink->dev, "intent request timed out\n");
1360                 ret = -ETIMEDOUT;
1361         } else {
1362                 ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED;
1363         }
1364
1365 unlock:
1366         mutex_unlock(&channel->intent_req_lock);
1367         return ret;
1368 }
1369
1370 static int __qcom_glink_send(struct glink_channel *channel,
1371                              void *data, int len, bool wait)
1372 {
1373         struct qcom_glink *glink = channel->glink;
1374         struct glink_core_rx_intent *intent = NULL;
1375         struct glink_core_rx_intent *tmp;
1376         int iid = 0;
1377         struct {
1378                 struct glink_msg msg;
1379                 __le32 chunk_size;
1380                 __le32 left_size;
1381         } __packed req;
1382         int ret;
1383         unsigned long flags;
1384         int chunk_size = len;
1385         size_t offset = 0;
1386
1387         if (!glink->intentless) {
1388                 while (!intent) {
1389                         spin_lock_irqsave(&channel->intent_lock, flags);
1390                         idr_for_each_entry(&channel->riids, tmp, iid) {
1391                                 if (tmp->size >= len && !tmp->in_use) {
1392                                         if (!intent)
1393                                                 intent = tmp;
1394                                         else if (intent->size > tmp->size)
1395                                                 intent = tmp;
1396                                         if (intent->size == len)
1397                                                 break;
1398                                 }
1399                         }
1400                         if (intent)
1401                                 intent->in_use = true;
1402                         spin_unlock_irqrestore(&channel->intent_lock, flags);
1403
1404                         /* We found an available intent */
1405                         if (intent)
1406                                 break;
1407
1408                         if (!wait)
1409                                 return -EBUSY;
1410
1411                         ret = qcom_glink_request_intent(glink, channel, len);
1412                         if (ret < 0)
1413                                 return ret;
1414                 }
1415
1416                 iid = intent->id;
1417         }
1418
1419         while (offset < len) {
1420                 chunk_size = len - offset;
1421                 if (chunk_size > SZ_8K && wait)
1422                         chunk_size = SZ_8K;
1423
1424                 req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1425                 req.msg.param1 = cpu_to_le16(channel->lcid);
1426                 req.msg.param2 = cpu_to_le32(iid);
1427                 req.chunk_size = cpu_to_le32(chunk_size);
1428                 req.left_size = cpu_to_le32(len - offset - chunk_size);
1429
1430                 ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait);
1431                 if (ret) {
1432                         /* Mark intent available if we failed */
1433                         if (intent)
1434                                 intent->in_use = false;
1435                         return ret;
1436                 }
1437
1438                 offset += chunk_size;
1439         }
1440
1441         return 0;
1442 }
1443
1444 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1445 {
1446         struct glink_channel *channel = to_glink_channel(ept);
1447
1448         return __qcom_glink_send(channel, data, len, true);
1449 }
1450
1451 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1452 {
1453         struct glink_channel *channel = to_glink_channel(ept);
1454
1455         return __qcom_glink_send(channel, data, len, false);
1456 }
1457
1458 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1459 {
1460         struct glink_channel *channel = to_glink_channel(ept);
1461
1462         return __qcom_glink_send(channel, data, len, true);
1463 }
1464
1465 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1466 {
1467         struct glink_channel *channel = to_glink_channel(ept);
1468
1469         return __qcom_glink_send(channel, data, len, false);
1470 }
1471
1472 /*
1473  * Finds the device_node for the glink child interested in this channel.
1474  */
1475 static struct device_node *qcom_glink_match_channel(struct device_node *node,
1476                                                     const char *channel)
1477 {
1478         struct device_node *child;
1479         const char *name;
1480         const char *key;
1481         int ret;
1482
1483         for_each_available_child_of_node(node, child) {
1484                 key = "qcom,glink-channels";
1485                 ret = of_property_read_string(child, key, &name);
1486                 if (ret)
1487                         continue;
1488
1489                 if (strcmp(name, channel) == 0)
1490                         return child;
1491         }
1492
1493         return NULL;
1494 }
1495
1496 static const struct rpmsg_device_ops glink_device_ops = {
1497         .create_ept = qcom_glink_create_ept,
1498         .announce_create = qcom_glink_announce_create,
1499 };
1500
1501 static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1502         .destroy_ept = qcom_glink_destroy_ept,
1503         .send = qcom_glink_send,
1504         .sendto = qcom_glink_sendto,
1505         .trysend = qcom_glink_trysend,
1506         .trysendto = qcom_glink_trysendto,
1507         .set_flow_control = qcom_glink_set_flow_control,
1508 };
1509
1510 static void qcom_glink_rpdev_release(struct device *dev)
1511 {
1512         struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1513
1514         kfree(rpdev->driver_override);
1515         kfree(rpdev);
1516 }
1517
1518 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1519                               char *name)
1520 {
1521         struct glink_channel *channel;
1522         struct rpmsg_device *rpdev;
1523         bool create_device = false;
1524         struct device_node *node;
1525         int lcid;
1526         int ret;
1527         unsigned long flags;
1528
1529         spin_lock_irqsave(&glink->idr_lock, flags);
1530         idr_for_each_entry(&glink->lcids, channel, lcid) {
1531                 if (!strcmp(channel->name, name))
1532                         break;
1533         }
1534         spin_unlock_irqrestore(&glink->idr_lock, flags);
1535
1536         if (!channel) {
1537                 channel = qcom_glink_alloc_channel(glink, name);
1538                 if (IS_ERR(channel))
1539                         return PTR_ERR(channel);
1540
1541                 /* The opening dance was initiated by the remote */
1542                 create_device = true;
1543         }
1544
1545         spin_lock_irqsave(&glink->idr_lock, flags);
1546         ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1547         if (ret < 0) {
1548                 dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1549                 spin_unlock_irqrestore(&glink->idr_lock, flags);
1550                 goto free_channel;
1551         }
1552         channel->rcid = ret;
1553         spin_unlock_irqrestore(&glink->idr_lock, flags);
1554
1555         complete_all(&channel->open_req);
1556
1557         if (create_device) {
1558                 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1559                 if (!rpdev) {
1560                         ret = -ENOMEM;
1561                         goto rcid_remove;
1562                 }
1563
1564                 rpdev->ept = &channel->ept;
1565                 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE);
1566                 rpdev->src = RPMSG_ADDR_ANY;
1567                 rpdev->dst = RPMSG_ADDR_ANY;
1568                 rpdev->ops = &glink_device_ops;
1569
1570                 node = qcom_glink_match_channel(glink->dev->of_node, name);
1571                 rpdev->dev.of_node = node;
1572                 rpdev->dev.parent = glink->dev;
1573                 rpdev->dev.release = qcom_glink_rpdev_release;
1574
1575                 ret = rpmsg_register_device(rpdev);
1576                 if (ret)
1577                         goto rcid_remove;
1578
1579                 channel->rpdev = rpdev;
1580         }
1581
1582         return 0;
1583
1584 rcid_remove:
1585         spin_lock_irqsave(&glink->idr_lock, flags);
1586         idr_remove(&glink->rcids, channel->rcid);
1587         channel->rcid = 0;
1588         spin_unlock_irqrestore(&glink->idr_lock, flags);
1589 free_channel:
1590         /* Release the reference, iff we took it */
1591         if (create_device)
1592                 kref_put(&channel->refcount, qcom_glink_channel_release);
1593
1594         return ret;
1595 }
1596
1597 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1598 {
1599         struct rpmsg_channel_info chinfo;
1600         struct glink_channel *channel;
1601         unsigned long flags;
1602
1603         spin_lock_irqsave(&glink->idr_lock, flags);
1604         channel = idr_find(&glink->rcids, rcid);
1605         spin_unlock_irqrestore(&glink->idr_lock, flags);
1606         if (WARN(!channel, "close request on unknown channel\n"))
1607                 return;
1608
1609         /* cancel pending rx_done work */
1610         cancel_work_sync(&channel->intent_work);
1611
1612         if (channel->rpdev) {
1613                 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1614                 chinfo.src = RPMSG_ADDR_ANY;
1615                 chinfo.dst = RPMSG_ADDR_ANY;
1616
1617                 rpmsg_unregister_device(glink->dev, &chinfo);
1618         }
1619         channel->rpdev = NULL;
1620
1621         qcom_glink_send_close_ack(glink, channel->rcid);
1622
1623         spin_lock_irqsave(&glink->idr_lock, flags);
1624         idr_remove(&glink->rcids, channel->rcid);
1625         channel->rcid = 0;
1626         spin_unlock_irqrestore(&glink->idr_lock, flags);
1627
1628         kref_put(&channel->refcount, qcom_glink_channel_release);
1629 }
1630
1631 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1632 {
1633         struct rpmsg_channel_info chinfo;
1634         struct glink_channel *channel;
1635         unsigned long flags;
1636
1637         /* To wakeup any blocking writers */
1638         wake_up_all(&glink->tx_avail_notify);
1639
1640         spin_lock_irqsave(&glink->idr_lock, flags);
1641         channel = idr_find(&glink->lcids, lcid);
1642         if (WARN(!channel, "close ack on unknown channel\n")) {
1643                 spin_unlock_irqrestore(&glink->idr_lock, flags);
1644                 return;
1645         }
1646
1647         idr_remove(&glink->lcids, channel->lcid);
1648         channel->lcid = 0;
1649         spin_unlock_irqrestore(&glink->idr_lock, flags);
1650
1651         /* Decouple the potential rpdev from the channel */
1652         if (channel->rpdev) {
1653                 strscpy(chinfo.name, channel->name, sizeof(chinfo.name));
1654                 chinfo.src = RPMSG_ADDR_ANY;
1655                 chinfo.dst = RPMSG_ADDR_ANY;
1656
1657                 rpmsg_unregister_device(glink->dev, &chinfo);
1658         }
1659         channel->rpdev = NULL;
1660
1661         kref_put(&channel->refcount, qcom_glink_channel_release);
1662 }
1663
1664 static void qcom_glink_work(struct work_struct *work)
1665 {
1666         struct qcom_glink *glink = container_of(work, struct qcom_glink,
1667                                                 rx_work);
1668         struct glink_defer_cmd *dcmd;
1669         struct glink_msg *msg;
1670         unsigned long flags;
1671         unsigned int param1;
1672         unsigned int param2;
1673         unsigned int cmd;
1674
1675         for (;;) {
1676                 spin_lock_irqsave(&glink->rx_lock, flags);
1677                 if (list_empty(&glink->rx_queue)) {
1678                         spin_unlock_irqrestore(&glink->rx_lock, flags);
1679                         break;
1680                 }
1681                 dcmd = list_first_entry(&glink->rx_queue,
1682                                         struct glink_defer_cmd, node);
1683                 list_del(&dcmd->node);
1684                 spin_unlock_irqrestore(&glink->rx_lock, flags);
1685
1686                 msg = &dcmd->msg;
1687                 cmd = le16_to_cpu(msg->cmd);
1688                 param1 = le16_to_cpu(msg->param1);
1689                 param2 = le32_to_cpu(msg->param2);
1690
1691                 switch (cmd) {
1692                 case GLINK_CMD_VERSION:
1693                         qcom_glink_receive_version(glink, param1, param2);
1694                         break;
1695                 case GLINK_CMD_VERSION_ACK:
1696                         qcom_glink_receive_version_ack(glink, param1, param2);
1697                         break;
1698                 case GLINK_CMD_OPEN:
1699                         qcom_glink_rx_open(glink, param1, msg->data);
1700                         break;
1701                 case GLINK_CMD_CLOSE:
1702                         qcom_glink_rx_close(glink, param1);
1703                         break;
1704                 case GLINK_CMD_CLOSE_ACK:
1705                         qcom_glink_rx_close_ack(glink, param1);
1706                         break;
1707                 case GLINK_CMD_RX_INTENT_REQ:
1708                         qcom_glink_handle_intent_req(glink, param1, param2);
1709                         break;
1710                 default:
1711                         WARN(1, "Unknown defer object %d\n", cmd);
1712                         break;
1713                 }
1714
1715                 kfree(dcmd);
1716         }
1717 }
1718
1719 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1720 {
1721         struct glink_defer_cmd *dcmd;
1722         struct glink_defer_cmd *tmp;
1723
1724         /* cancel any pending deferred rx_work */
1725         cancel_work_sync(&glink->rx_work);
1726
1727         list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1728                 kfree(dcmd);
1729 }
1730
1731 static ssize_t rpmsg_name_show(struct device *dev,
1732                                struct device_attribute *attr, char *buf)
1733 {
1734         int ret = 0;
1735         const char *name;
1736
1737         ret = of_property_read_string(dev->of_node, "label", &name);
1738         if (ret < 0)
1739                 name = dev->of_node->name;
1740
1741         return sysfs_emit(buf, "%s\n", name);
1742 }
1743 static DEVICE_ATTR_RO(rpmsg_name);
1744
1745 static struct attribute *qcom_glink_attrs[] = {
1746         &dev_attr_rpmsg_name.attr,
1747         NULL
1748 };
1749 ATTRIBUTE_GROUPS(qcom_glink);
1750
1751 static void qcom_glink_device_release(struct device *dev)
1752 {
1753         struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1754         struct glink_channel *channel = to_glink_channel(rpdev->ept);
1755
1756         /* Release qcom_glink_alloc_channel() reference */
1757         kref_put(&channel->refcount, qcom_glink_channel_release);
1758         kfree(rpdev->driver_override);
1759         kfree(rpdev);
1760 }
1761
1762 static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1763 {
1764         struct rpmsg_device *rpdev;
1765         struct glink_channel *channel;
1766
1767         rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1768         if (!rpdev)
1769                 return -ENOMEM;
1770
1771         channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");
1772         if (IS_ERR(channel)) {
1773                 kfree(rpdev);
1774                 return PTR_ERR(channel);
1775         }
1776         channel->rpdev = rpdev;
1777
1778         rpdev->ept = &channel->ept;
1779         rpdev->ops = &glink_device_ops;
1780         rpdev->dev.parent = glink->dev;
1781         rpdev->dev.release = qcom_glink_device_release;
1782
1783         return rpmsg_ctrldev_register_device(rpdev);
1784 }
1785
1786 struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1787                                            unsigned long features,
1788                                            struct qcom_glink_pipe *rx,
1789                                            struct qcom_glink_pipe *tx,
1790                                            bool intentless)
1791 {
1792         int ret;
1793         struct qcom_glink *glink;
1794
1795         glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1796         if (!glink)
1797                 return ERR_PTR(-ENOMEM);
1798
1799         glink->dev = dev;
1800         glink->tx_pipe = tx;
1801         glink->rx_pipe = rx;
1802
1803         glink->features = features;
1804         glink->intentless = intentless;
1805
1806         spin_lock_init(&glink->tx_lock);
1807         spin_lock_init(&glink->rx_lock);
1808         INIT_LIST_HEAD(&glink->rx_queue);
1809         INIT_WORK(&glink->rx_work, qcom_glink_work);
1810         init_waitqueue_head(&glink->tx_avail_notify);
1811
1812         spin_lock_init(&glink->idr_lock);
1813         idr_init(&glink->lcids);
1814         idr_init(&glink->rcids);
1815
1816         glink->dev->groups = qcom_glink_groups;
1817
1818         ret = device_add_groups(dev, qcom_glink_groups);
1819         if (ret)
1820                 dev_err(dev, "failed to add groups\n");
1821
1822         ret = qcom_glink_send_version(glink);
1823         if (ret)
1824                 return ERR_PTR(ret);
1825
1826         ret = qcom_glink_create_chrdev(glink);
1827         if (ret)
1828                 dev_err(glink->dev, "failed to register chrdev\n");
1829
1830         return glink;
1831 }
1832 EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1833
1834 static int qcom_glink_remove_device(struct device *dev, void *data)
1835 {
1836         device_unregister(dev);
1837
1838         return 0;
1839 }
1840
1841 void qcom_glink_native_remove(struct qcom_glink *glink)
1842 {
1843         struct glink_channel *channel;
1844         unsigned long flags;
1845         int cid;
1846         int ret;
1847
1848         qcom_glink_cancel_rx_work(glink);
1849
1850         /* Fail all attempts at sending messages */
1851         spin_lock_irqsave(&glink->tx_lock, flags);
1852         glink->abort_tx = true;
1853         wake_up_all(&glink->tx_avail_notify);
1854         spin_unlock_irqrestore(&glink->tx_lock, flags);
1855
1856         /* Abort any senders waiting for intent requests */
1857         spin_lock_irqsave(&glink->idr_lock, flags);
1858         idr_for_each_entry(&glink->lcids, channel, cid)
1859                 qcom_glink_intent_req_abort(channel);
1860         spin_unlock_irqrestore(&glink->idr_lock, flags);
1861
1862         ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1863         if (ret)
1864                 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1865
1866         /* Release any defunct local channels, waiting for close-ack */
1867         idr_for_each_entry(&glink->lcids, channel, cid)
1868                 kref_put(&channel->refcount, qcom_glink_channel_release);
1869
1870         /* Release any defunct local channels, waiting for close-req */
1871         idr_for_each_entry(&glink->rcids, channel, cid)
1872                 kref_put(&channel->refcount, qcom_glink_channel_release);
1873
1874         idr_destroy(&glink->lcids);
1875         idr_destroy(&glink->rcids);
1876 }
1877 EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1878
1879 MODULE_DESCRIPTION("Qualcomm GLINK driver");
1880 MODULE_LICENSE("GPL v2");