From 93ceb885b2e9e86e6f6aaa5133a074f731ab1402 Mon Sep 17 00:00:00 2001 From: ahmadexp Date: Mon, 28 Sep 2026 16:58:22 -0700 Subject: [PATCH 1/4] dt-bindings: net: document RP1 GEM MACB compatible --- .../devicetree/bindings/net/cdns,macb.yaml | 92 +++++++++++++++++-- 1 file changed, 82 insertions(+), 10 deletions(-) diff --git a/Documentation/devicetree/bindings/net/cdns,macb.yaml b/Documentation/devicetree/bindings/net/cdns,macb.yaml index 62c1f8cb7300ed..dcd0c619b4cdb4 100644 --- a/Documentation/devicetree/bindings/net/cdns,macb.yaml +++ b/Documentation/devicetree/bindings/net/cdns,macb.yaml @@ -39,6 +39,10 @@ properties: - microchip,mpfs-macb # Microchip PolarFire SoC - const: cdns,macb # Generic + - items: + - const: raspberrypi,rp1-gem # Raspberry Pi RP1 gigabit ethernet interface + - const: cdns,macb # Generic + - items: - enum: - atmel,sama5d3-macb # 10/100Mbit IP on Atmel sama5d3 SoCs @@ -70,7 +74,13 @@ properties: minItems: 1 items: - description: Basic register set - - description: GEMGXL Management block registers on SiFive FU540-C000 SoC + - description: SoC-specific auxiliary register set + + reg-names: + minItems: 1 + items: + - const: macb + - enum: [gemxl_mgmt, eth_cfg] interrupts: minItems: 1 @@ -87,7 +97,7 @@ properties: - enum: [ ether_clk, hclk, pclk ] - enum: [ hclk, pclk ] - enum: [ tx_clk, tsu_clk ] - - enum: [ rx_clk, tsu_clk ] + - enum: [ rx_clk, tx_clk, tsu_clk ] - const: tsu_clk local-mac-address: true @@ -96,6 +106,26 @@ properties: phy-handle: true + raspberrypi,pio: + $ref: /schemas/types.yaml#/definitions/phandle + description: + RP1 PIO provider used for Ethernet PTP GPIO timestamping and output. + + raspberrypi,gpio-controller: + $ref: /schemas/types.yaml#/definitions/phandle + description: + RP1 GPIO provider used to reserve and route PTP-capable pins. + + phy-reset-gpios: + maxItems: 1 + deprecated: true + description: GPIO used to reset the Ethernet PHY. + + phy-reset-duration: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: PHY reset duration in milliseconds. + phys: maxItems: 1 @@ -146,12 +176,16 @@ properties: be direct children. cdns,aw2w-max-pipe: - $ref: /schemas/types.yaml#/definitions/uint32 + oneOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - $ref: /schemas/types.yaml#/definitions/uint8 description: Maximum number of outstanding AXI write requests cdns,ar2r-max-pipe: - $ref: /schemas/types.yaml#/definitions/uint32 + oneOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - $ref: /schemas/types.yaml#/definitions/uint8 description: Maximum number of outstanding AXI read requests @@ -169,6 +203,19 @@ patternProperties: properties: reset-gpios: true + brcm,powerdown-enable: + type: boolean + description: + Allow the Broadcom PHY to enter its low-power mode when the link is + down. + + led-modes: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 2 + maxItems: 2 + description: + Broadcom PHY LED mode values for the two integrated LED outputs. + magic-packet: type: boolean deprecated: true @@ -189,15 +236,40 @@ allOf: - $ref: ethernet-controller.yaml# - if: - not: - properties: - compatible: - contains: - const: sifive,fu540-c000-gem + properties: + compatible: + contains: + const: raspberrypi,rp1-gem then: properties: reg: - maxItems: 1 + minItems: 2 + maxItems: 2 + reg-names: + items: + - const: macb + - const: eth_cfg + clock-names: + items: + - const: pclk + - const: hclk + - const: tsu_clk + - const: tx_clk + required: + - reg-names + - raspberrypi,pio + - raspberrypi,gpio-controller + else: + if: + not: + properties: + compatible: + contains: + const: sifive,fu540-c000-gem + then: + properties: + reg: + maxItems: 1 unevaluatedProperties: false From 8df6082ca4b8e86bd23f60f5ef0a8528448ce692 Mon Sep 17 00:00:00 2001 From: ahmadexp Date: Mon, 28 Sep 2026 16:58:57 -0700 Subject: [PATCH 2/4] rp1: add firmware PIO consumer and GPIO reservation APIs --- drivers/firmware/rp1-fw.c | 64 ++++- drivers/misc/rp1-pio.c | 493 ++++++++++++++++++++++++++++++++-- drivers/pinctrl/pinctrl-rp1.c | 400 +++++++++++++++++++++++++-- include/linux/pinctrl/rp1.h | 17 ++ include/linux/pio_rp1.h | 18 ++ include/linux/rp1-firmware.h | 12 + 6 files changed, 952 insertions(+), 52 deletions(-) create mode 100644 include/linux/pinctrl/rp1.h diff --git a/drivers/firmware/rp1-fw.c b/drivers/firmware/rp1-fw.c index 96d80d21f9e632..52984b15489e6e 100644 --- a/drivers/firmware/rp1-fw.c +++ b/drivers/firmware/rp1-fw.c @@ -42,6 +42,43 @@ struct rp1_get_feature_resp { }; static DEFINE_MUTEX(transaction_lock); +/* The mailbox protocol has no transaction identifier or cancellation token. */ +static int transaction_error; + +int rp1_firmware_transport_error(struct rp1_firmware *fw) +{ + (void)fw; + return READ_ONCE(transaction_error); +} +EXPORT_SYMBOL_GPL(rp1_firmware_transport_error); + +/* Caller holds transaction_lock and a firmware reference. */ +static int rp1_firmware_fault(struct rp1_firmware *fw, int err) +{ + if (!transaction_error) { + /* Preserve the completion and callback context for a late reply. */ + kref_get(&fw->consumers); + __module_get(THIS_MODULE); + WRITE_ONCE(transaction_error, err); + dev_err(fw->cl.dev, + "RP1 firmware transport fault %d; power cycle required\n", err); + } + return transaction_error; +} + +int rp1_firmware_report_fault(struct rp1_firmware *fw, int err) +{ + int ret; + + if (!fw || err >= 0) + return -EINVAL; + + mutex_lock(&transaction_lock); + ret = rp1_firmware_fault(fw, err); + mutex_unlock(&transaction_lock); + return ret; +} +EXPORT_SYMBOL_GPL(rp1_firmware_report_fault); static const struct of_device_id rp1_firmware_of_match[] = { { .compatible = "raspberrypi,rp1-firmware", }, @@ -69,13 +106,19 @@ int rp1_firmware_message(struct rp1_firmware *fw, uint16_t op, int ret; u32 rc; - if (data_len + 4 > fw->buf_size) + if (!fw || fw->buf_size < 4 || data_len > U16_MAX || + data_len > fw->buf_size - 4 || (data_len && !data) || + (resp_space && !resp)) return -EINVAL; mutex_lock(&transaction_lock); + if (transaction_error) { + ret = transaction_error; + goto out; + } memcpy_toio(&fw->buf[1], data, data_len); - writel((op << 16) | data_len, fw->buf); + writel(((u32)op << 16) | data_len, fw->buf); reinit_completion(&fw->c); ret = mbox_send_message(fw->chan, NULL); @@ -90,7 +133,7 @@ int rp1_firmware_message(struct rp1_firmware *fw, uint16_t op, if (wait_for_completion_timeout(&fw->c, HZ)) ret = 0; else - ret = -ETIMEDOUT; + ret = rp1_firmware_fault(fw, -ETIMEDOUT); } else { dev_err(fw->cl.dev, "mbox_send_message returned %d\n", ret); } @@ -99,12 +142,16 @@ int rp1_firmware_message(struct rp1_firmware *fw, uint16_t op, rc = readl(fw->buf); if (rc & 0x80000000) { ret = (int32_t)rc; + } else if (rc > fw->buf_size - 4 || rc > resp_space) { + ret = rp1_firmware_fault(fw, -EPROTO); } else { - ret = min(rc, resp_space); - memcpy_fromio(resp, &fw->buf[1], ret); + ret = rc; + if (ret) + memcpy_fromio(resp, &fw->buf[1], ret); } } +out: mutex_unlock(&transaction_lock); return ret; @@ -155,6 +202,7 @@ static void devm_rp1_firmware_put(void *data) /** * rp1_firmware_get - Get pointer to rp1_firmware structure. + * @client: Device Tree node of the firmware consumer. * * The reference to rp1_firmware has to be released with rp1_firmware_put(). * @@ -202,9 +250,11 @@ EXPORT_SYMBOL_GPL(rp1_firmware_get); /** * devm_rp1_firmware_get - Get pointer to rp1_firmware structure. - * @firmware_node: Pointer to the firmware Device Tree node. + * @dev: Device associated with the managed reference. + * @client: Device Tree node of the firmware consumer. * - * Returns NULL is the firmware device is not ready. + * Returns NULL if the firmware device is not ready, or an error pointer on + * failure. The reference is released when @dev is detached. */ struct rp1_firmware *devm_rp1_firmware_get(struct device *dev, struct device_node *client) { diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c index 62f9d3fcf09fa4..2302dbc52daca5 100644 --- a/drivers/misc/rp1-pio.c +++ b/drivers/misc/rp1-pio.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -113,6 +114,7 @@ struct irq_info { struct rp1_pio_device { struct platform_device *pdev; + void __iomem *regs; struct rp1_firmware *fw; uint16_t fw_pio_base; uint16_t fw_pio_count; @@ -126,7 +128,10 @@ struct rp1_pio_device { uint32_t claimed_dmas; spinlock_t lock; struct mutex instr_mutex; + /* Covers each PIO request through response validation and fault latching. */ + struct mutex message_mutex; struct dma_info dma_configs[RP1_PIO_SMS_COUNT][RP1_PIO_DIR_COUNT]; + struct dma_chan *external_dma[RP1_PIO_SMS_COUNT][RP1_PIO_DIR_COUNT]; uint32_t used_instrs; uint8_t instr_refcounts[RP1_PIO_INSTR_COUNT]; uint16_t instrs[RP1_PIO_INSTR_COUNT]; @@ -146,6 +151,7 @@ struct rp1_pio_client { }; static struct rp1_pio_device *g_pio; +static DEFINE_MUTEX(rp1_pio_lock); static int rp1_pio_message(struct rp1_pio_device *pio, uint16_t op, const void *data, unsigned int data_len) @@ -155,11 +161,15 @@ static int rp1_pio_message(struct rp1_pio_device *pio, if (op >= pio->fw_pio_count) return -EOPNOTSUPP; + mutex_lock(&pio->message_mutex); ret = rp1_firmware_message(pio->fw, pio->fw_pio_base + op, data, data_len, &rc, sizeof(rc)); - if (ret == 4) - ret = rc; + if (ret == sizeof(rc)) + ret = (int32_t)rc; + else if (ret >= 0) + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); + mutex_unlock(&pio->message_mutex); return ret; } @@ -172,25 +182,32 @@ static int rp1_pio_message_resp(struct rp1_pio_device *pio, if (op >= pio->fw_pio_count) return -EOPNOTSUPP; - if (resp_len + 4 >= sizeof(resp_buf)) + if (resp_len > sizeof(resp_buf) - 4) return -EINVAL; if (!resp && !userbuf) return -EINVAL; + mutex_lock(&pio->message_mutex); ret = rp1_firmware_message(pio->fw, pio->fw_pio_base + op, data, data_len, resp_buf, resp_len + 4); - if (ret >= 4 && !resp_buf[0]) { + if (ret >= 0 && ret < 4) { + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); + } else if (ret >= 4 && resp_buf[0]) { + ret = (int32_t)resp_buf[0]; + if (ret == -1) + ret = -EIO; + } else if (ret >= 4 && ret != resp_len + 4) { + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); + } else if (ret >= 4) { ret -= 4; + } + mutex_unlock(&pio->message_mutex); + /* User copies must not serialize other PIO/firmware clients. */ + if (ret >= 0) { if (resp) memcpy(resp, &resp_buf[1], ret); else if (copy_to_user(userbuf, &resp_buf[1], ret)) ret = -EFAULT; - } else if (ret == 4) { - ret = (int)resp_buf[0]; - if (ret == -1) - ret = -EIO; - } else if (ret >= 0) { - ret = -EIO; } return ret; } @@ -209,14 +226,17 @@ static int rp1_pio_write_hw(struct rp1_pio_client *client, void *param) struct rp1_pio_device *pio = client->pio; struct rp1_access_hw_args *args = param; uint32_t write_buf[32 + 1]; - int len; + int len, ret; len = min(args->len, sizeof(write_buf) - 4); write_buf[0] = args->addr; if (copy_from_user(&write_buf[1], args->data, len)) return -EFAULT; - return rp1_firmware_message(pio->fw, pio->fw_pio_base + WRITE_HW, - write_buf, 4 + len, NULL, 0); + mutex_lock(&pio->message_mutex); + ret = rp1_firmware_message(pio->fw, pio->fw_pio_base + WRITE_HW, + write_buf, 4 + len, NULL, 0); + mutex_unlock(&pio->message_mutex); + return ret; } static int rp1_pio_find_program(struct rp1_pio_device *pio, @@ -251,13 +271,18 @@ int rp1_pio_can_add_program(struct rp1_pio_client *client, void *param) struct rp1_pio_device *pio = client->pio; int offset; - if (args->num_instrs > RP1_PIO_INSTR_COUNT || - ((args->origin != RP1_PIO_ORIGIN_ANY) && - (args->origin >= RP1_PIO_INSTR_COUNT || - ((args->origin + args->num_instrs) > RP1_PIO_INSTR_COUNT)))) + if (!args->num_instrs || args->num_instrs > RP1_PIO_INSTR_COUNT || + (args->origin != RP1_PIO_ORIGIN_ANY && + (args->origin >= RP1_PIO_INSTR_COUNT || + args->origin + args->num_instrs > RP1_PIO_INSTR_COUNT))) return -EINVAL; mutex_lock(&pio->instr_mutex); + offset = rp1_firmware_transport_error(pio->fw); + if (offset) { + mutex_unlock(&pio->instr_mutex); + return offset; + } offset = rp1_pio_find_program(pio, args); mutex_unlock(&pio->instr_mutex); if (offset >= 0) @@ -276,16 +301,23 @@ int rp1_pio_add_program(struct rp1_pio_client *client, void *param) int offset; uint i; - if (args->num_instrs > RP1_PIO_INSTR_COUNT || - ((args->origin != RP1_PIO_ORIGIN_ANY) && - (args->origin >= RP1_PIO_INSTR_COUNT || - ((args->origin + args->num_instrs) > RP1_PIO_INSTR_COUNT)))) + if (!args->num_instrs || args->num_instrs > RP1_PIO_INSTR_COUNT || + (args->origin != RP1_PIO_ORIGIN_ANY && + (args->origin >= RP1_PIO_INSTR_COUNT || + args->origin + args->num_instrs > RP1_PIO_INSTR_COUNT))) return -EINVAL; mutex_lock(&pio->instr_mutex); + offset = rp1_firmware_transport_error(pio->fw); + if (offset) { + mutex_unlock(&pio->instr_mutex); + return offset; + } offset = rp1_pio_find_program(pio, args); if (offset < 0) offset = rp1_pio_message(client->pio, PIO_ADD_PROGRAM, args, sizeof(*args)); + if (offset >= 0 && offset > RP1_PIO_INSTR_COUNT - args->num_instrs) + offset = rp1_firmware_report_fault(pio->fw, -EPROTO); if (offset >= 0) { uint32_t used_mask; @@ -370,8 +402,13 @@ int rp1_pio_sm_claim(struct rp1_pio_client *client, void *param) struct rp1_pio_device *pio = client->pio; int ret; + if (args->mask & ~GENMASK(RP1_PIO_SMS_COUNT - 1, 0)) + return -EINVAL; + mutex_lock(&pio->instr_mutex); ret = rp1_pio_message(client->pio, PIO_SM_CLAIM, args, sizeof(*args)); + if (!args->mask && ret >= RP1_PIO_SMS_COUNT) + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); if (ret >= 0) { if (args->mask) client->claimed_sms |= args->mask; @@ -1565,19 +1602,56 @@ struct handler_info { HANDLER(INTERRUPT_CLEAR, interrupt_clear), }; +static int rp1_pio_link_firmware(struct device *dev) +{ + struct device_node *fw_node; + struct platform_device *fw_pdev; + struct device_link *link; + enum device_link_state status; + int ret = 0; + + fw_node = of_parse_phandle(dev->of_node, "firmware", 0); + if (!fw_node) + return -ENODEV; + + fw_pdev = of_find_device_by_node(fw_node); + of_node_put(fw_node); + if (!fw_pdev) + return -EPROBE_DEFER; + + link = device_link_add(dev, &fw_pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + ret = -ENOMEM; + } else { + status = READ_ONCE(link->status); + if (status != DL_STATE_ACTIVE && status != DL_STATE_CONSUMER_PROBE) + ret = -EPROBE_DEFER; + } + + put_device(&fw_pdev->dev); + return ret; +} + struct rp1_pio_client *rp1_pio_open(void) { struct rp1_pio_client *client; int i; - if (!g_pio) - return ERR_PTR(-EPROBE_DEFER); - if (IS_ERR(g_pio)) - return ERR_CAST(g_pio); + mutex_lock(&rp1_pio_lock); + if (!g_pio) { + client = ERR_PTR(-EPROBE_DEFER); + goto out; + } + if (IS_ERR(g_pio)) { + client = ERR_CAST(g_pio); + goto out; + } client = kzalloc(sizeof(*client), GFP_KERNEL); - if (!client) - return ERR_PTR(-ENOMEM); + if (!client) { + client = ERR_PTR(-ENOMEM); + goto out; + } for (i = 0; i < RP1_PIO_IRQ_COUNT; i++) client->irqs[i] = -1; @@ -1585,11 +1659,360 @@ struct rp1_pio_client *rp1_pio_open(void) client->pio = g_pio; spin_lock_init(&client->lock); init_completion(&client->completion); - +out: + mutex_unlock(&rp1_pio_lock); return client; } EXPORT_SYMBOL_GPL(rp1_pio_open); +/** + * rp1_pio_open_for_device - open a PIO client and order its supplier + * @consumer: device which will use the returned client + * @provider_node: device-tree node for the RP1 PIO provider + * + * The device link keeps the PIO provider bound until the consumer unbinds. + * The consumer must serialize use and close of the returned client with its + * own removal path. Repeated calls for the same device pair reuse the link. + */ +struct rp1_pio_client * +rp1_pio_open_for_device(struct device *consumer, + struct device_node *provider_node) +{ + struct platform_device *pdev; + struct device_link *link; + struct rp1_pio_client *client; + enum device_link_state state; + int i; + + if (!consumer || !provider_node) + return ERR_PTR(-EINVAL); + + pdev = of_find_device_by_node(provider_node); + if (!pdev) + return ERR_PTR(-EPROBE_DEFER); + + link = device_link_add(consumer, &pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + client = ERR_PTR(-ENODEV); + goto put_device; + } + state = READ_ONCE(link->status); + if (state != DL_STATE_ACTIVE && state != DL_STATE_CONSUMER_PROBE) { + client = ERR_PTR(-EPROBE_DEFER); + goto put_device; + } + + mutex_lock(&rp1_pio_lock); + if (!g_pio || IS_ERR(g_pio) || g_pio->pdev != pdev) { + client = ERR_PTR(-EPROBE_DEFER); + goto unlock; + } + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) { + client = ERR_PTR(-ENOMEM); + goto unlock; + } + for (i = 0; i < RP1_PIO_IRQ_COUNT; i++) + client->irqs[i] = -1; + client->pio = g_pio; + spin_lock_init(&client->lock); + init_completion(&client->completion); +unlock: + mutex_unlock(&rp1_pio_lock); +put_device: + put_device(&pdev->dev); + return client; +} +EXPORT_SYMBOL_GPL(rp1_pio_open_for_device); + +/** + * rp1_pio_sm_put_mmio - append a word to an owned state-machine TX FIFO + * @client: PIO client which claimed @sm + * @sm: state-machine index + * @data: word to append + * + * This avoids a firmware round trip. The caller must ensure the FIFO has room + * and serialize writers. The MMIO write is ordered, but may be posted. + */ +int rp1_pio_sm_put_mmio(struct rp1_pio_client *client, unsigned int sm, u32 data) +{ + struct rp1_pio_device *pio; + + if (!client || sm >= RP1_PIO_SMS_COUNT || + !(client->claimed_sms & BIT(sm))) + return -EINVAL; + + pio = client->pio; + if (!pio->regs) + return -ENODEV; + + writel(data, pio->regs + RP1_PIO_FIFO_TX0 + sizeof(data) * sm); + return 0; +} +EXPORT_SYMBOL_GPL(rp1_pio_sm_put_mmio); + +/** + * rp1_pio_dma_request - request a DMA channel for an owned state machine + * @client: PIO client which owns @sm + * @sm: state-machine index + * @dir: data direction relative to the state machine + * + * The returned channel transfers one 32-bit word per FIFO request. The PIO + * client must release it after stopping its DMA activity and before closing. + */ +struct dma_chan *rp1_pio_dma_request(struct rp1_pio_client *client, + unsigned int sm, enum pio_xfer_dir dir) +{ + struct rp1_pio_device *pio; + struct dma_slave_config config = {}; + struct dma_chan *chan; + unsigned long flags; + phys_addr_t fifo_addr; + char name[5]; + u32 mask; + int ret; + + if (!client || sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT) + return ERR_PTR(-EINVAL); + pio = client->pio; + mask = BIT(sm * RP1_PIO_DIR_COUNT + dir); + + spin_lock_irqsave(&pio->lock, flags); + if (!(client->claimed_sms & BIT(sm))) { + spin_unlock_irqrestore(&pio->lock, flags); + return ERR_PTR(-EPERM); + } + if (pio->claimed_dmas & mask) { + spin_unlock_irqrestore(&pio->lock, flags); + return ERR_PTR(-EBUSY); + } + pio->claimed_dmas |= mask; + client->claimed_dmas |= mask; + spin_unlock_irqrestore(&pio->lock, flags); + + name[0] = dir == PIO_DIR_TO_SM ? 't' : 'r'; + name[1] = 'x'; + name[2] = '0' + sm; + name[3] = '\0'; + name[4] = '\0'; + chan = dma_request_chan(&pio->pdev->dev, name); + if (IS_ERR(chan)) { + ret = PTR_ERR(chan); + goto err_unclaim; + } + + fifo_addr = pio->phys_addr + sm * sizeof(u32); + if (dir == PIO_DIR_FROM_SM) + fifo_addr += RP1_PIO_FIFO_RX0; + else + fifo_addr += RP1_PIO_FIFO_TX0; + config.src_addr = fifo_addr; + config.dst_addr = fifo_addr; + config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + config.src_maxburst = 1; + config.dst_maxburst = 1; + config.direction = dir == PIO_DIR_FROM_SM ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; + ret = dmaengine_slave_config(chan, &config); + if (ret) + goto err_release; + + spin_lock_irqsave(&pio->lock, flags); + if (pio->external_dma[sm][dir]) { + spin_unlock_irqrestore(&pio->lock, flags); + ret = -EBUSY; + goto err_release; + } + pio->external_dma[sm][dir] = chan; + spin_unlock_irqrestore(&pio->lock, flags); + return chan; + +err_release: + dma_release_channel(chan); +err_unclaim: + spin_lock_irqsave(&pio->lock, flags); + pio->claimed_dmas &= ~mask; + client->claimed_dmas &= ~mask; + spin_unlock_irqrestore(&pio->lock, flags); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(rp1_pio_dma_request); + +/** + * rp1_pio_dma_release - stop and release a channel acquired by this client + * @client: PIO client which owns @sm + * @sm: state-machine index + * @dir: data direction relative to the state machine + * @chan: channel returned by rp1_pio_dma_request() + * + * If synchronous termination fails, the channel and PIO ownership are retained + * so the caller can retry cleanup. + */ +int rp1_pio_dma_release(struct rp1_pio_client *client, unsigned int sm, + enum pio_xfer_dir dir, struct dma_chan *chan) +{ + struct rp1_pio_device *pio; + unsigned long flags; + u32 mask; + int ret; + + if (!client || !chan || sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT) + return -EINVAL; + pio = client->pio; + mask = BIT(sm * RP1_PIO_DIR_COUNT + dir); + + spin_lock_irqsave(&pio->lock, flags); + if (!(client->claimed_dmas & mask) || pio->external_dma[sm][dir] != chan) { + spin_unlock_irqrestore(&pio->lock, flags); + return -EPERM; + } + spin_unlock_irqrestore(&pio->lock, flags); + + ret = dmaengine_terminate_sync(chan); + if (ret) + return ret; + dma_release_channel(chan); + + spin_lock_irqsave(&pio->lock, flags); + pio->external_dma[sm][dir] = NULL; + pio->claimed_dmas &= ~mask; + client->claimed_dmas &= ~mask; + spin_unlock_irqrestore(&pio->lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(rp1_pio_dma_release); + +/** + * rp1_pio_gpio_is_synchronized - report whether an RP1 GPIO uses input sync + * @client: PIO client + * @gpio: RP1 GPIO number + * @synchronized: set to true when the GPIO input synchronizer is enabled + */ +int rp1_pio_gpio_is_synchronized(struct rp1_pio_client *client, + unsigned int gpio, bool *synchronized) +{ + u32 request[2] = { 0xf000003c, sizeof(u32) }; + u32 mask; + int ret; + + if (!client || !synchronized || gpio >= 28) + return -EINVAL; + ret = rp1_pio_message_resp(client->pio, READ_HW, request, sizeof(request), + &mask, NULL, sizeof(mask)); + if (ret < 0) + return ret; + if (ret != sizeof(mask)) + return rp1_firmware_report_fault(client->pio->fw, -EPROTO); + /* READ_HW returns the bypass mask, so a clear bit means synchronized. */ + *synchronized = !(mask & BIT(gpio)); + return 0; +} +EXPORT_SYMBOL_GPL(rp1_pio_gpio_is_synchronized); + +/* Caller holds instr_mutex. Remove only acknowledged, exclusively-owned runs. */ +static int rp1_pio_remove_checked(struct rp1_pio_client *client) +{ + struct rp1_pio_device *pio = client->pio; + struct rp1_pio_remove_program_args args; + unsigned int i, count; + u32 mask; + int ret; + + for (i = 0; i < RP1_PIO_INSTR_COUNT; i++) { + if (!(client->claimed_instrs & BIT(i))) + continue; + if (!pio->instr_refcounts[i]) + return rp1_firmware_report_fault(pio->fw, -EUCLEAN); + if (pio->instr_refcounts[i] > 1) { + pio->instr_refcounts[i]--; + client->claimed_instrs &= ~BIT(i); + continue; + } + + for (count = 1; i + count < RP1_PIO_INSTR_COUNT; count++) { + if (!(client->claimed_instrs & BIT(i + count)) || + pio->instr_refcounts[i + count] != 1) + break; + } + args.origin = i; + args.num_instrs = count; + ret = rp1_pio_message(pio, PIO_REMOVE_PROGRAM, &args, sizeof(args)); + if (ret > 0) + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); + if (ret) + return ret; + + mask = GENMASK(i + count - 1, i); + memset(&pio->instr_refcounts[i], 0, count); + pio->used_instrs &= ~mask; + client->claimed_instrs &= ~mask; + i += count - 1; + } + + return 0; +} + +/** + * rp1_pio_close_checked - stop and release kernel-client PIO ownership + * @client: exclusively accessed client; caller has drained its own DMA + * + * Clients using the PIO bounce-buffer DMA helper are not supported here. + * A zero return consumes the client. An error retains it for retry or + * quarantine, with acknowledged releases reflected in its ownership state. + */ +int rp1_pio_close_checked(struct rp1_pio_client *client) +{ + struct rp1_pio_device *pio; + struct rp1_pio_sm_set_enabled_args stop = {}; + struct rp1_pio_sm_claim_args release = {}; + int ret; + + if (!client) + return -EINVAL; + pio = client->pio; + + mutex_lock(&pio->instr_mutex); + ret = rp1_firmware_transport_error(pio->fw); + if (ret) + goto out; + if (client->claimed_dmas) { + ret = -EOPNOTSUPP; + goto out; + } + + stop.mask = client->claimed_sms; + if (stop.mask) { + ret = rp1_pio_message(pio, PIO_SM_SET_ENABLED, &stop, sizeof(stop)); + if (ret > 0) + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); + if (ret) + goto out; + } + + ret = rp1_pio_remove_checked(client); + if (ret) + goto out; + + release.mask = client->claimed_sms; + if (release.mask) { + ret = rp1_pio_message(pio, PIO_SM_UNCLAIM, &release, sizeof(release)); + if (ret > 0) + ret = rp1_firmware_report_fault(pio->fw, -EPROTO); + if (ret) + goto out; + pio->claimed_sms &= ~release.mask; + client->claimed_sms = 0; + } + + mutex_unlock(&pio->instr_mutex); + kfree(client); + return 0; +out: + mutex_unlock(&pio->instr_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(rp1_pio_close_checked); + void rp1_pio_close(struct rp1_pio_client *client) { struct rp1_pio_device *pio = client->pio; @@ -1868,7 +2291,7 @@ static int rp1_pio_probe(struct platform_device *pdev) uint32_t op_base = 0; struct device *cdev; char dev_name[16]; - void *p; + void __iomem *p; int irq; int ret; int i; @@ -1885,6 +2308,12 @@ static int rp1_pio_probe(struct platform_device *pdev) if (pdev->id < 0) return dev_err_probe(dev, pdev->id, "alias is missing\n"); + /* PIO accesses the firmware provider's device-managed shared mapping. */ + ret = rp1_pio_link_firmware(dev); + if (ret) + return dev_err_probe(dev, ret, + "failed to link RP1 firmware supplier\n"); + fw = devm_rp1_firmware_get(dev, dev->of_node); if (!fw) return dev_err_probe(dev, -EPROBE_DEFER, "failed to find RP1 firmware driver\n"); @@ -1910,12 +2339,14 @@ static int rp1_pio_probe(struct platform_device *pdev) pio->fw = fw; spin_lock_init(&pio->lock); mutex_init(&pio->instr_mutex); + mutex_init(&pio->message_mutex); p = devm_platform_get_and_ioremap_resource(pdev, 0, &ioresource); if (IS_ERR(p)) { ret = PTR_ERR(p); goto out_err; } + pio->regs = p; if (pio->fw_pio_count > PIO_INTERRUPT_CLEAR) { for (i = 0; i < ARRAY_SIZE(pio->irqs); i++) { @@ -1966,7 +2397,9 @@ static int rp1_pio_probe(struct platform_device *pdev) goto out_cdev_del; } + mutex_lock(&rp1_pio_lock); g_pio = pio; + mutex_unlock(&rp1_pio_lock); dev_info(dev, "Created instance as %s (op count %d, %d interrupts)\n", dev_name, pio->fw_pio_count, pio->irq_count); @@ -1992,8 +2425,10 @@ static void rp1_pio_remove(struct platform_device *pdev) /* There should be no clients */ + mutex_lock(&rp1_pio_lock); if (g_pio == pio) g_pio = NULL; + mutex_unlock(&rp1_pio_lock); device_destroy(pio->dev_class, pio->dev_num); cdev_del(&pio->cdev); diff --git a/drivers/pinctrl/pinctrl-rp1.c b/drivers/pinctrl/pinctrl-rp1.c index 69eb3b0e84a26b..4d16aab9255aac 100644 --- a/drivers/pinctrl/pinctrl-rp1.c +++ b/drivers/pinctrl/pinctrl-rp1.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -22,12 +24,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -45,6 +49,7 @@ #define RP1_SET_OFFSET 0x2000 #define RP1_CLR_OFFSET 0x3000 +#define RP1_GPIO_STATUS_OETOPAD BIT(13) #define RP1_GPIO_STATUS 0x0000 #define RP1_GPIO_CTRL 0x0004 @@ -81,6 +86,7 @@ #define RP1_GPIO_CTRL_IRQEN_F_HIGH BIT(27) #define RP1_GPIO_CTRL_IRQRESET BIT(28) #define RP1_GPIO_CTRL_IRQOVER_LSB 30 +#define RP1_GPIO_CTRL_IRQEN_MASK 0x0ff00000 #define RP1_GPIO_CTRL_IRQOVER_MASK 0xc0000000 #define RP1_INT_EDGE_FALLING BIT(0) @@ -100,6 +106,7 @@ #define RP1_FSEL_ALT0 0x00 #define RP1_FSEL_GPIO 0x05 #define RP1_FSEL_NONE 0x09 +#define RP1_FSEL_PIO 0x07 #define RP1_FSEL_NONE_HW 0x1f #define RP1_DIR_OUTPUT 0 @@ -191,6 +198,11 @@ struct rp1_pin_info { u8 offset; u8 fsel; u8 irq_type; + raw_spinlock_t pio_lock; + bool pio_restore; + bool pio_changed; + u32 pio_saved_ctrl; + u32 pio_saved_pad; void __iomem *gpio; void __iomem *rio; @@ -288,6 +300,12 @@ struct rp1_pinctrl { raw_spinlock_t irq_lock[RP1_NUM_BANKS]; }; +struct rp1_pinctrl_pio_pin { + struct rp1_pin_info *pin; + struct gpio_desc *desc; + unsigned int gpio; +}; + const struct rp1_iobank_desc rp1_iobanks[RP1_NUM_BANKS] = { /* gpio inte ints rio pads */ { 0, 28, 0x0000, 0x011c, 0x0124, 0x0000, 0x0004 }, @@ -753,25 +771,60 @@ static int rp1_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) static int rp1_gpio_direction_input(struct gpio_chip *chip, unsigned offset) { struct rp1_pin_info *pin = rp1_get_pin(chip, offset); + unsigned long flags; + int ret = 0; if (!pin) return -EINVAL; + raw_spin_lock_irqsave(&pin->pio_lock, flags); + if (pin->pio_restore) { + ret = -EBUSY; + goto out; + } rp1_set_dir(pin, RP1_DIR_INPUT); rp1_set_fsel(pin, RP1_FSEL_GPIO); - return 0; +out: + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + return ret; } static int rp1_gpio_direction_output(struct gpio_chip *chip, unsigned offset, int value) { struct rp1_pin_info *pin = rp1_get_pin(chip, offset); + unsigned long flags; + int ret = 0; if (!pin) return -EINVAL; + raw_spin_lock_irqsave(&pin->pio_lock, flags); + if (pin->pio_restore) { + ret = -EBUSY; + goto out; + } rp1_set_value(pin, value); rp1_set_dir(pin, RP1_DIR_OUTPUT); rp1_set_fsel(pin, RP1_FSEL_GPIO); - return 0; +out: + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + return ret; +} + +static int rp1_gpio_request(struct gpio_chip *chip, unsigned int offset) +{ + struct rp1_pin_info *pin = rp1_get_pin(chip, offset); + unsigned long flags; + int ret; + + if (!pin) + return -EINVAL; + raw_spin_lock_irqsave(&pin->pio_lock, flags); + ret = pin->pio_restore ? -EBUSY : 0; + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + if (ret) + return ret; + + return gpiochip_generic_request(chip, offset); } static int rp1_gpio_set_config(struct gpio_chip *gc, unsigned offset, @@ -787,7 +840,7 @@ static int rp1_gpio_set_config(struct gpio_chip *gc, unsigned offset, static const struct gpio_chip rp1_gpio_chip = { .label = MODULE_NAME, .owner = THIS_MODULE, - .request = gpiochip_generic_request, + .request = rp1_gpio_request, .free = gpiochip_generic_free, .direction_input = rp1_gpio_direction_input, .direction_output = rp1_gpio_direction_output, @@ -967,21 +1020,35 @@ static int rp1_gpio_irq_set_affinity(struct irq_data *data, const struct cpumask return -EINVAL; } -static int rp1_gpio_irq_reqres(struct irq_data *d) +static int rp1_gpio_irq_request_resources(struct irq_data *data) { - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + unsigned int offset = irqd_to_hwirq(data); + struct rp1_pin_info *pin = rp1_get_pin(gc, offset); + unsigned long flags; int ret; - ret = gpiochip_irq_reqres(d); - if (!ret) - ret = rp1_gpio_direction_input(gc, d->hwirq); - + raw_spin_lock_irqsave(&pin->pio_lock, flags); + ret = pin->pio_restore ? -EBUSY : gpiochip_irq_reqres(data); + if (!ret) { + ret = rp1_gpio_direction_input(gc, offset); + if (ret) + gpiochip_irq_relres(data); + } + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); return ret; } -static void rp1_gpio_irq_relres(struct irq_data *d) +static void rp1_gpio_irq_release_resources(struct irq_data *data) { - return gpiochip_irq_relres(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + unsigned int offset = irqd_to_hwirq(data); + struct rp1_pin_info *pin = rp1_get_pin(gc, offset); + unsigned long flags; + + raw_spin_lock_irqsave(&pin->pio_lock, flags); + gpiochip_irq_relres(data); + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); } static struct irq_chip rp1_gpio_irq_chip = { @@ -993,8 +1060,8 @@ static struct irq_chip rp1_gpio_irq_chip = { .irq_mask = rp1_gpio_irq_disable, .irq_unmask = rp1_gpio_irq_enable, .irq_set_affinity = rp1_gpio_irq_set_affinity, - .irq_request_resources = rp1_gpio_irq_reqres, - .irq_release_resources = rp1_gpio_irq_relres, + .irq_request_resources = rp1_gpio_irq_request_resources, + .irq_release_resources = rp1_gpio_irq_release_resources, .flags = IRQCHIP_IMMUTABLE, }; @@ -1262,11 +1329,303 @@ static const struct pinctrl_ops rp1_pctl_ops = { .dt_free_map = rp1_pctl_dt_free_map, }; +static int rp1_pio_restore(struct rp1_pin_info *pin) +{ + unsigned long flags; + u32 ctrl; + int ret = 0; + + raw_spin_lock_irqsave(&pin->pio_lock, flags); + if (!pin->pio_restore) + goto out; + if (!pin->pio_changed) { + pin->pio_restore = false; + ret = 1; + goto out; + } + + ctrl = rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL); + FLD_SET(ctrl, RP1_GPIO_CTRL_OEOVER, RP1_OEOVER_DISABLE); + FLD_SET(ctrl, RP1_GPIO_CTRL_OUTOVER, RP1_OUTOVER_LOW); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + rp1_pin_writel(pin->pio_saved_pad, pin->dummy, pin->pad); + rp1_pin_writel(pin->pio_saved_ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + if (rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != pin->pio_saved_ctrl || + rp1_pin_readl(pin->pad) != pin->pio_saved_pad) { + ret = -EIO; + goto out; + } + pin->pio_restore = false; + pin->pio_changed = false; + ret = 1; +out: + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + return ret; +} + +/** + * rp1_pinctrl_pio_request - reserve and configure an RP1 GPIO for PIO + * @consumer: device using the pin + * @provider_node: GPIO/pinctrl provider node + * @gpio: RP1 GPIO number, in the range 0 to 27 + * + * The returned handle owns the GPIO descriptor. The driver core keeps the + * managed supplier link until the consumer device is unbound. PIO output + * remains inhibited until rp1_pinctrl_pio_set_output() enables it. + */ +struct rp1_pinctrl_pio_pin * +rp1_pinctrl_pio_request(struct device *consumer, struct device_node *provider_node, + unsigned int gpio) +{ + struct rp1_pinctrl_pio_pin *request; + struct platform_device *pdev; + struct device_link *link; + struct rp1_pinctrl *pc; + struct rp1_pin_info *pin; + enum device_link_state state; + unsigned long flags; + u32 ctrl, pad, status; + int ret; + + if (!consumer || !provider_node || gpio >= 28) + return ERR_PTR(-EINVAL); + + pdev = of_find_device_by_node(provider_node); + if (!pdev) + return ERR_PTR(-EPROBE_DEFER); + + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (!request) { + ret = -ENOMEM; + goto put_device; + } + + /* The driver core owns this managed link until the consumer is unbound. */ + link = device_link_add(consumer, &pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + ret = -ENODEV; + goto free_request; + } + state = READ_ONCE(link->status); + if (state != DL_STATE_ACTIVE && state != DL_STATE_CONSUMER_PROBE) { + ret = -EPROBE_DEFER; + goto free_request; + } + + pc = dev_get_drvdata(&pdev->dev); + if (!pc || !pc->pctl_dev) { + ret = -EPROBE_DEFER; + goto free_request; + } + + request->desc = gpiochip_request_own_desc(&pc->gpio_chip, gpio, + "rp1-macb-ptp", 0, GPIOD_ASIS); + if (IS_ERR(request->desc)) { + ret = PTR_ERR(request->desc); + request->desc = NULL; + goto free_request; + } + + pin = &pc->pins[gpio]; + request->pin = pin; + request->gpio = gpio; + raw_spin_lock_irqsave(&pin->pio_lock, flags); + if (pin->pio_restore) { + ret = -EBUSY; + goto unlock_free_desc; + } + ctrl = rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL); + pad = rp1_pin_readl(pin->pad); + status = rp1_pin_readl(pin->gpio + RP1_GPIO_STATUS); + if (((ctrl & RP1_GPIO_CTRL_FUNCSEL_MASK) != RP1_FSEL_GPIO && + (ctrl & RP1_GPIO_CTRL_FUNCSEL_MASK) != RP1_FSEL_NONE_HW) || + ((status & RP1_GPIO_STATUS_OETOPAD) && + !(pad & RP1_PAD_OUT_DISABLE_MASK)) || + gpiochip_line_is_irq(&pc->gpio_chip, gpio) || + (ctrl & (RP1_GPIO_CTRL_INOVER_MASK | RP1_GPIO_CTRL_IRQEN_MASK | + RP1_GPIO_CTRL_IRQOVER_MASK))) { + ret = -EBUSY; + goto unlock_free_desc; + } + + pin->pio_saved_ctrl = ctrl; + pin->pio_saved_pad = pad; + pin->pio_restore = true; + pin->pio_changed = true; + FLD_SET(ctrl, RP1_GPIO_CTRL_OEOVER, RP1_OEOVER_DISABLE); + FLD_SET(ctrl, RP1_GPIO_CTRL_OUTOVER, RP1_OUTOVER_LOW); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + rp1_pin_writel(pad | RP1_PAD_OUT_DISABLE_MASK | RP1_PAD_IN_ENABLE_MASK, + pin->dummy, pin->pad); + FLD_SET(ctrl, RP1_GPIO_CTRL_FUNCSEL, RP1_FSEL_PIO); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + if (rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != ctrl || + rp1_pin_readl(pin->pad) != + (pad | RP1_PAD_OUT_DISABLE_MASK | RP1_PAD_IN_ENABLE_MASK)) + ret = -EIO; + else + ret = 0; + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + if (!ret) { + put_device(&pdev->dev); + return request; + } + + /* Keep the pin inhibited while restoring it after a partial setup. */ + rp1_pio_restore(pin); + gpiochip_free_own_desc(request->desc); + request->desc = NULL; + goto free_request; + +unlock_free_desc: + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + gpiochip_free_own_desc(request->desc); + request->desc = NULL; +free_request: + kfree(request); +put_device: + put_device(&pdev->dev); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(rp1_pinctrl_pio_request); + +/** + * rp1_pinctrl_pio_set_output - safely enable or inhibit an owned PIO output + * @request: handle returned by rp1_pinctrl_pio_request() + * @enable: release the low override to PIO, or force the pad low + * + * Disabling the output first forces the pad low, then inhibits it. This path + * remains available when the PIO firmware transport has failed. + */ +int rp1_pinctrl_pio_set_output(struct rp1_pinctrl_pio_pin *request, + bool enable) +{ + struct rp1_pin_info *pin; + unsigned long flags; + u32 ctrl, pad; + int ret = 0; + + if (!request || !request->pin) + return -EINVAL; + pin = request->pin; + + raw_spin_lock_irqsave(&pin->pio_lock, flags); + if (!pin->pio_restore || !pin->pio_changed) { + ret = -EPERM; + goto out; + } + ctrl = rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL); + if (FLD_GET(ctrl, RP1_GPIO_CTRL_FUNCSEL) != RP1_FSEL_PIO) { + ret = -EPERM; + goto out; + } + + FLD_SET(ctrl, RP1_GPIO_CTRL_OUTOVER, RP1_OUTOVER_LOW); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + if (rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != ctrl) { + ret = -EIO; + goto out; + } + if (!enable) { + pad = rp1_pin_readl(pin->pad) | RP1_PAD_OUT_DISABLE_MASK; + rp1_pin_writel(pad, pin->dummy, pin->pad); + if (rp1_pin_readl(pin->pad) != pad) { + ret = -EIO; + goto out; + } + FLD_SET(ctrl, RP1_GPIO_CTRL_OEOVER, RP1_OEOVER_DISABLE); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + if (rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != ctrl) + ret = -EIO; + goto out; + } + + pad = rp1_pin_readl(pin->pad) & ~RP1_PAD_OUT_DISABLE_MASK; + rp1_pin_writel(pad, pin->dummy, pin->pad); + if (rp1_pin_readl(pin->pad) != pad) { + ret = -EIO; + goto force_low; + } + + FLD_SET(ctrl, RP1_GPIO_CTRL_OEOVER, RP1_OEOVER_PERI); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + if (rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != ctrl) { + ret = -EIO; + goto force_low; + } + + FLD_SET(ctrl, RP1_GPIO_CTRL_OUTOVER, RP1_OUTOVER_PERI); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + if (rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != ctrl) { + ret = -EIO; + goto force_low; + } + goto out; + +force_low: + ctrl = rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL); + FLD_SET(ctrl, RP1_GPIO_CTRL_OUTOVER, RP1_OUTOVER_LOW); + rp1_pin_writel(ctrl, pin->dummy, pin->gpio + RP1_GPIO_CTRL); + rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL); +out: + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(rp1_pinctrl_pio_set_output); + +/** + * rp1_pinctrl_pio_release - restore an RP1 GPIO and release its ownership + * @request: handle returned by rp1_pinctrl_pio_request() + * + * On failure the handle remains valid and the GPIO remains reserved so the + * caller can retry cleanup. + */ +int rp1_pinctrl_pio_release(struct rp1_pinctrl_pio_pin *request) +{ + struct rp1_pin_info *pin; + int ret; + + if (!request || !request->pin) + return -EINVAL; + pin = request->pin; + + ret = rp1_pinctrl_pio_set_output(request, false); + if (ret && ret != -EPERM) + return ret; + if (request->desc) { + gpiochip_free_own_desc(request->desc); + request->desc = NULL; + } + + /* gpiochip_free_own_desc() runs rp1_pmx_free(), which restores the pin. */ + ret = rp1_pio_restore(pin); + if (ret < 0 || rp1_pin_readl(pin->gpio + RP1_GPIO_CTRL) != pin->pio_saved_ctrl || + rp1_pin_readl(pin->pad) != pin->pio_saved_pad) { + unsigned long flags; + + raw_spin_lock_irqsave(&pin->pio_lock, flags); + pin->pio_restore = true; + pin->pio_changed = true; + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + return ret < 0 ? ret : -EIO; + } + + kfree(request); + return 0; +} +EXPORT_SYMBOL_GPL(rp1_pinctrl_pio_release); + static int rp1_pmx_free(struct pinctrl_dev *pctldev, unsigned offset) { struct rp1_pin_info *pin = rp1_get_pin_pctl(pctldev, offset); - u32 fsel = rp1_get_fsel(pin); + u32 fsel; + + int ret = rp1_pio_restore(pin); + if (ret) + return 0; + fsel = rp1_get_fsel(pin); /* Return all pins to GPIO_IN, unless persist_gpio_outputs is set */ if (persist_gpio_outputs && fsel == RP1_FSEL_GPIO) return 0; @@ -1341,11 +1700,19 @@ static int rp1_pmx_gpio_set_direction(struct pinctrl_dev *pctldev, bool input) { struct rp1_pin_info *pin = rp1_get_pin_pctl(pctldev, offset); + unsigned long flags; + int ret = 0; + raw_spin_lock_irqsave(&pin->pio_lock, flags); + if (pin->pio_restore) { + ret = -EBUSY; + goto out; + } rp1_set_dir(pin, input); rp1_set_fsel(pin, RP1_FSEL_GPIO); - - return 0; +out: + raw_spin_unlock_irqrestore(&pin->pio_lock, flags); + return ret; } static bool rp1_pmx_function_is_gpio(struct pinctrl_dev *pctldev, @@ -1640,6 +2007,7 @@ static int rp1_pinctrl_probe(struct platform_device *pdev) struct rp1_pin_info *pin = &pc->pins[bank->min_gpio + j]; + raw_spin_lock_init(&pin->pio_lock); pin->num = bank->min_gpio + j; pin->bank = i; pin->offset = j; diff --git a/include/linux/pinctrl/rp1.h b/include/linux/pinctrl/rp1.h new file mode 100644 index 00000000000000..9bbbe96ce6769e --- /dev/null +++ b/include/linux/pinctrl/rp1.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_PINCTRL_RP1_H +#define __LINUX_PINCTRL_RP1_H + +#include + +struct device; +struct device_node; +struct rp1_pinctrl_pio_pin; + +struct rp1_pinctrl_pio_pin * +rp1_pinctrl_pio_request(struct device *consumer, struct device_node *provider_node, + unsigned int gpio); +int rp1_pinctrl_pio_set_output(struct rp1_pinctrl_pio_pin *pin, bool enable); +int rp1_pinctrl_pio_release(struct rp1_pinctrl_pio_pin *pin); + +#endif /* __LINUX_PINCTRL_RP1_H */ diff --git a/include/linux/pio_rp1.h b/include/linux/pio_rp1.h index 54cdda69c9bad2..da95d44082a331 100644 --- a/include/linux/pio_rp1.h +++ b/include/linux/pio_rp1.h @@ -9,6 +9,11 @@ #include +struct device; +struct device_node; +struct dma_chan; +struct rp1_pio_client; + #define PARAM_WARNINGS_ENABLED 1 #ifdef DEBUG @@ -224,7 +229,20 @@ static struct rp1_pio_client *g_client; int rp1_pio_init(void); PIO rp1_pio_open(void); +struct rp1_pio_client * +rp1_pio_open_for_device(struct device *consumer, + struct device_node *provider_node); +struct dma_chan *rp1_pio_dma_request(struct rp1_pio_client *client, + unsigned int sm, enum pio_xfer_dir dir); +int rp1_pio_dma_release(struct rp1_pio_client *client, unsigned int sm, + enum pio_xfer_dir dir, struct dma_chan *chan); +int rp1_pio_gpio_is_synchronized(struct rp1_pio_client *client, + unsigned int gpio, bool *synchronized); void rp1_pio_close(struct rp1_pio_client *client); +/* On error, ownership remains with the caller. No PIO-helper DMA is released. */ +int rp1_pio_close_checked(struct rp1_pio_client *client); +int rp1_pio_sm_put_mmio(struct rp1_pio_client *client, unsigned int sm, + uint32_t data); void rp1_pio_set_error(struct rp1_pio_client *client, int err); int rp1_pio_get_error(const struct rp1_pio_client *client); void rp1_pio_clear_error(struct rp1_pio_client *client); diff --git a/include/linux/rp1-firmware.h b/include/linux/rp1-firmware.h index 19f11d6d79b167..9ea7f808747249 100644 --- a/include/linux/rp1-firmware.h +++ b/include/linux/rp1-firmware.h @@ -17,6 +17,8 @@ struct rp1_firmware; int rp1_firmware_message(struct rp1_firmware *fw, uint16_t op, const void *data, unsigned int data_len, void *resp, unsigned int resp_space); +int rp1_firmware_transport_error(struct rp1_firmware *fw); +int rp1_firmware_report_fault(struct rp1_firmware *fw, int err); void rp1_firmware_put(struct rp1_firmware *fw); struct rp1_firmware *rp1_firmware_get(struct device_node *fwnode); struct rp1_firmware *devm_rp1_firmware_get(struct device *dev, struct device_node *fwnode); @@ -30,6 +32,16 @@ static inline int rp1_firmware_message(struct rp1_firmware *fw, uint16_t op, return -EOPNOTSUPP; } +static inline int rp1_firmware_transport_error(struct rp1_firmware *fw) +{ + return -EOPNOTSUPP; +} + +static inline int rp1_firmware_report_fault(struct rp1_firmware *fw, int err) +{ + return -EOPNOTSUPP; +} + static inline void rp1_firmware_put(struct rp1_firmware *fw) { } static inline struct rp1_firmware *rp1_firmware_get(struct device_node *fwnode) From eef7b9714c5010a3a15f23c2614c2827ff292d32 Mon Sep 17 00:00:00 2001 From: ahmadexp Date: Mon, 28 Sep 2026 16:59:23 -0700 Subject: [PATCH 3/4] net: macb: add RP1 PHC GPIO timestamp and PPS support --- .../ethernet/cadence/macb-rp1-ptp.rst | 104 + .../device_drivers/ethernet/index.rst | 1 + arch/arm64/boot/dts/broadcom/rp1.dtsi | 6 +- arch/arm64/configs/bcm2712_defconfig | 7 +- drivers/net/ethernet/cadence/Kconfig | 11 + drivers/net/ethernet/cadence/Makefile | 4 + drivers/net/ethernet/cadence/macb.h | 4 + drivers/net/ethernet/cadence/macb_main.c | 15 +- drivers/net/ethernet/cadence/macb_ptp.c | 77 +- drivers/net/ethernet/cadence/macb_rp1_ptp.c | 2646 +++++++++++++++++ drivers/net/ethernet/cadence/macb_rp1_ptp.h | 69 + .../ethernet/cadence/macb_rp1_ptp_counter.h | 378 +++ .../net/ethernet/cadence/macb_rp1_ptp_math.h | 68 + 13 files changed, 3376 insertions(+), 14 deletions(-) create mode 100644 Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst create mode 100644 drivers/net/ethernet/cadence/macb_rp1_ptp.c create mode 100644 drivers/net/ethernet/cadence/macb_rp1_ptp.h create mode 100644 drivers/net/ethernet/cadence/macb_rp1_ptp_counter.h create mode 100644 drivers/net/ethernet/cadence/macb_rp1_ptp_math.h diff --git a/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst b/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst new file mode 100644 index 00000000000000..c51126067a46b9 --- /dev/null +++ b/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst @@ -0,0 +1,104 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================= +RP1 Ethernet PTP GPIO support +============================= + +The Raspberry Pi RP1 Ethernet controller exposes its Precision Time Protocol +hardware clock (PHC) through the standard Linux PTP device interface. With +``CONFIG_MACB_RP1_PPS`` enabled, the driver also exposes RP1 GPIO pins for +external timestamp input and periodic output. Applications use the standard +PTP clock ioctls; no private userspace interface is added. + +The driver advertises GPIO0 through GPIO27 as PTP pins. Select a BCM/RP1 GPIO +number, not a 40-pin header position. Pin reservation is checked when a +channel is enabled. A pin in use by another peripheral or GPIO interrupt is +rejected. Board wiring and other pin users can make a listed pin unavailable. +This PHC interface is separate from the ``pps-rp1`` overlay, which configures +the LinuxPPS ``pps-gpio`` driver. Do not configure both interfaces on the same +GPIO. + +The implementation provides two external timestamp channels and one periodic +output channel. Without PEROUT, channel 0 uses the Ethernet timestamp latch; +channel 1 requires PEROUT to supply a PHC anchor. With PEROUT active, both +EXTTS channels use separate RP1 PIO counter and DMA lanes, and each input must +use a GPIO different from the output GPIO and from the other input GPIO. The +simultaneous counter path currently supports rising edges only. Stop channel 1 +before disabling PEROUT. Channel 0 returns to direct latch capture when PEROUT +is stopped. + +Periodic output supports a 1-second period. The default output pulse is high +for 500 ms. A duty-cycle request can select a high time from 1 ms through +500 ms, in 5 ns increments. These are hardware programming increments, not a +statement of physical pin accuracy. Counter-derived input timestamps are +mapped into the PHC domain from output-pad monitor samples and PEROUT TSU +markers. The mapping has not been electrically calibrated, so it does not +establish a board-level accuracy bound. + +Discover the PHC and the pin table first. The PHC device number may change +between boots or driver registration: + +.. code-block:: sh + + sudo testptp -d /dev/ptpN -c + sudo testptp -d /dev/ptpN -l + +For rising-edge input on GPIO18 on channel 0, map the pin to the external +timestamp function, then request events. ``testptp`` disables the capture +request after reading the requested number of events: + +.. code-block:: sh + + sudo testptp -d /dev/ptpN -i 0 -L 18,1 + sudo testptp -d /dev/ptpN -i 0 -E 1 -e 8 + +Channel 0 direct capture also accepts ``-E 2`` for falling edges and ``-E 3`` +for both edges. The PTP channel index is selected with ``-i``. The mapping can +be cleared with ``-L 18,0``. + +For 1 Hz output on GPIO23 with a 10 ms high pulse: + +.. code-block:: sh + + sudo testptp -d /dev/ptpN -i 0 -L 23,2 + sudo testptp -d /dev/ptpN -i 0 -p 1000000000 -w 10000000 + +For simultaneous output and two rising-edge inputs, map three distinct GPIOs. +Enable PEROUT first, then start one capture process per EXTTS channel: + +.. code-block:: sh + + sudo testptp -d /dev/ptpN -i 0 -L 23,2 + sudo testptp -d /dev/ptpN -i 0 -L 18,1 + sudo testptp -d /dev/ptpN -i 1 -L 24,1 + sudo testptp -d /dev/ptpN -i 0 -p 1000000000 -w 10000000 + sudo testptp -d /dev/ptpN -i 0 -E 1 -e 8 + sudo testptp -d /dev/ptpN -i 1 -E 1 -e 8 + +Disable input mappings before stopping PEROUT. The channel-1 mapping and +capture depend on active output markers: + +.. code-block:: sh + + sudo testptp -d /dev/ptpN -i 1 -L 24,0 + sudo testptp -d /dev/ptpN -i 0 -L 18,0 + sudo testptp -d /dev/ptpN -i 0 -p 0 + sudo testptp -d /dev/ptpN -i 0 -L 23,0 + +The output request remains enabled after ``testptp`` exits. Disable it +explicitly before changing the pin mapping or removing the Ethernet device: + +.. code-block:: sh + + sudo testptp -d /dev/ptpN -i 0 -p 0 + sudo testptp -d /dev/ptpN -i 0 -L 23,0 + +Add ``-H `` to request a phase within the 1-second period. The +output is aligned to the PHC, but physical pin delay and jitter have not been +specified by this interface. PHC time steps temporarily inhibit output while +the driver re-establishes the requested phase grid. + +The driver creates diagnostic counters under debugfs when it is enabled. They +report capture collection age, input re-arm dead time, output latch error and +frequency-change handling. Debugfs records are diagnostic measurements, not a +calibration certificate or an accuracy guarantee. diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst index 7cfcd183054f84..09bf0be8190a52 100644 --- a/Documentation/networking/device_drivers/ethernet/index.rst +++ b/Documentation/networking/device_drivers/ethernet/index.rst @@ -18,6 +18,7 @@ Contents: amd/pds_vdpa amd/pds_vfio_pci aquantia/atlantic + cadence/macb-rp1-ptp chelsio/cxgb cirrus/cs89x0 dlink/dl2k diff --git a/arch/arm64/boot/dts/broadcom/rp1.dtsi b/arch/arm64/boot/dts/broadcom/rp1.dtsi index 7141dfcff4e708..c2a40fb25da400 100644 --- a/arch/arm64/boot/dts/broadcom/rp1.dtsi +++ b/arch/arm64/boot/dts/broadcom/rp1.dtsi @@ -997,8 +997,12 @@ }; rp1_eth: ethernet@100000 { - reg = <0xc0 0x40100000 0x0 0x4000>; + reg = <0xc0 0x40100000 0x0 0x4000>, + <0xc0 0x40104000 0x0 0x4000>; + reg-names = "macb", "eth_cfg"; compatible = "raspberrypi,rp1-gem", "cdns,macb"; + raspberrypi,pio = <&rp1_pio>; + raspberrypi,gpio-controller = <&rp1_gpio>; #address-cells = <1>; #size-cells = <0>; interrupts = ; diff --git a/arch/arm64/configs/bcm2712_defconfig b/arch/arm64/configs/bcm2712_defconfig index 80ad05b0bb2448..eb8ca7b291cecf 100644 --- a/arch/arm64/configs/bcm2712_defconfig +++ b/arch/arm64/configs/bcm2712_defconfig @@ -458,7 +458,7 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_BRCMSTB_GISB_ARB is not set CONFIG_RASPBERRYPI_FIRMWARE=y -CONFIG_FIRMWARE_RP1=m +CONFIG_FIRMWARE_RP1=y # CONFIG_EFI_VARS_PSTORE is not set CONFIG_MTD=m CONFIG_MTD_BLOCK=m @@ -482,7 +482,7 @@ CONFIG_ATA_OVER_ETH=m CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=y CONFIG_NVME_HWMON=y -CONFIG_RP1_PIO=m +CONFIG_RP1_PIO=y CONFIG_WS2812_PIO_RP1=m CONFIG_SRAM=y CONFIG_EEPROM_AT24=m @@ -535,6 +535,7 @@ CONFIG_NET_DSA_MICROCHIP_KSZ_COMMON=m CONFIG_NET_DSA_MICROCHIP_KSZ_SPI=m CONFIG_BCMGENET=y CONFIG_MACB=y +CONFIG_MACB_RP1_PPS=y CONFIG_IGB=m CONFIG_IXGBE=m CONFIG_I40E=m @@ -1562,7 +1563,7 @@ CONFIG_COMMON_CLK_SI5351=m CONFIG_CLK_RASPBERRYPI=y CONFIG_MAILBOX=y CONFIG_BCM2835_MBOX=y -CONFIG_MBOX_RP1=m +CONFIG_MBOX_RP1=y CONFIG_BCM2712_IOMMU=y CONFIG_RASPBERRYPI_POWER=y CONFIG_IIO=m diff --git a/drivers/net/ethernet/cadence/Kconfig b/drivers/net/ethernet/cadence/Kconfig index 5b2a461dfd28c3..e5ba47db1bea25 100644 --- a/drivers/net/ethernet/cadence/Kconfig +++ b/drivers/net/ethernet/cadence/Kconfig @@ -42,6 +42,17 @@ config MACB_USE_HWSTAMP help Enable IEEE 1588 Precision Time Protocol (PTP) support for MACB. +config MACB_RP1_PPS + bool "RP1 Ethernet PTP GPIO timestamp and periodic output" + depends on MACB_USE_HWSTAMP + depends on FIRMWARE_RP1=y && RP1_PIO=y && PINCTRL_RP1=y + default n + help + Enable RP1-specific external timestamp input and periodic output on + the RP1 Ethernet PHC. The firmware, PIO and pinctrl providers must be + built into the kernel so their lifetimes are ordered with the built-in + MACB driver. + config MACB_PCI tristate "Cadence PCI MACB/GEM support" depends on MACB && PCI diff --git a/drivers/net/ethernet/cadence/Makefile b/drivers/net/ethernet/cadence/Makefile index 1f33cdca9a3c09..66b6638f62bd53 100644 --- a/drivers/net/ethernet/cadence/Makefile +++ b/drivers/net/ethernet/cadence/Makefile @@ -8,5 +8,9 @@ ifeq ($(CONFIG_MACB_USE_HWSTAMP),y) macb-y += macb_ptp.o endif +ifeq ($(CONFIG_MACB_RP1_PPS),y) +macb-y += macb_rp1_ptp.o +endif + obj-$(CONFIG_MACB) += macb.o obj-$(CONFIG_MACB_PCI) += macb_pci.o diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index d046a922a1a4ef..c9f56f828918ce 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -1218,6 +1218,7 @@ static const struct gem_statistic queue_statistics[] = { struct macb; struct macb_queue; +struct rp1_extts; struct macb_or_gem_ops { int (*mog_alloc_rx_buffers)(struct macb *bp); @@ -1323,6 +1324,9 @@ struct ethtool_rx_fs_list { struct macb { void __iomem *regs; + void __iomem *rp1_eth_cfg; + resource_size_t rp1_eth_cfg_phys; + struct rp1_extts *rp1_extts; bool native_io; /* hardware IO accessors */ diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 95bc3839173c15..3f7c4471b85d5a 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -5826,9 +5826,9 @@ static int macb_probe(struct platform_device *pdev) bool native_io; phy_interface_t interface; struct net_device *dev; - struct resource *regs; + struct resource *regs, *eth_cfg_res; u32 wtrmrk_rst_val; - void __iomem *mem; + void __iomem *mem, *eth_cfg = NULL; struct macb *bp; int err, val; @@ -5836,6 +5836,15 @@ static int macb_probe(struct platform_device *pdev) if (IS_ERR(mem)) return PTR_ERR(mem); + /* Older RP1 DTBs do not describe ETH_CFG, so keep the PHC usable. */ + eth_cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "eth_cfg"); + if (eth_cfg_res) { + eth_cfg = devm_ioremap_resource(&pdev->dev, eth_cfg_res); + if (IS_ERR(eth_cfg)) + return dev_err_probe(&pdev->dev, PTR_ERR(eth_cfg), + "failed to map RP1 ETH_CFG\n"); + } + if (np) { const struct of_device_id *match; @@ -5873,6 +5882,8 @@ static int macb_probe(struct platform_device *pdev) bp->pdev = pdev; bp->dev = dev; bp->regs = mem; + bp->rp1_eth_cfg = eth_cfg; + bp->rp1_eth_cfg_phys = eth_cfg_res ? eth_cfg_res->start : 0; bp->native_io = native_io; if (native_io) { bp->macb_reg_readl = hw_readl_native; diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c index d961a5faf355df..9ca9a623478075 100644 --- a/drivers/net/ethernet/cadence/macb_ptp.c +++ b/drivers/net/ethernet/cadence/macb_ptp.c @@ -20,8 +20,10 @@ #include #include #include +#include #include "macb.h" +#include "macb_rp1_ptp.h" #define GEM_PTP_TIMER_NAME "gem-ptp-timer" @@ -74,8 +76,8 @@ static int gem_tsu_get_time(struct ptp_clock_info *ptp, struct timespec64 *ts, return 0; } -static int gem_tsu_set_time(struct ptp_clock_info *ptp, - const struct timespec64 *ts) +static int __gem_tsu_set_time(struct ptp_clock_info *ptp, + const struct timespec64 *ts) { struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); unsigned long flags; @@ -99,6 +101,20 @@ static int gem_tsu_set_time(struct ptp_clock_info *ptp, return 0; } +static int gem_tsu_set_time(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); + int ret, unlock_ret; + + ret = rp1_extts_clock_lock(bp); + if (ret) + return ret; + ret = __gem_tsu_set_time(ptp, ts); + unlock_ret = rp1_extts_clock_unlock(bp); + return ret ? ret : unlock_ret; +} + static int gem_tsu_incr_set(struct macb *bp, struct tsu_incr *incr_spec) { unsigned long flags; @@ -119,7 +135,7 @@ static int gem_tsu_incr_set(struct macb *bp, struct tsu_incr *incr_spec) return 0; } -static int gem_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +static int __gem_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) { struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); struct tsu_incr incr_spec; @@ -154,7 +170,18 @@ static int gem_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) return 0; } -static int gem_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +static int gem_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); + int ret; + + rp1_extts_rate_lock(bp); + ret = __gem_ptp_adjfine(ptp, scaled_ppm); + rp1_extts_rate_unlock(bp); + return ret; +} + +static int __gem_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) { struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); struct timespec64 now, then = ns_to_timespec64(delta); @@ -169,8 +196,8 @@ static int gem_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) gem_tsu_get_time(&bp->ptp_clock_info, &now, NULL); now = timespec64_add(now, then); - gem_tsu_set_time(&bp->ptp_clock_info, - (const struct timespec64 *)&now); + __gem_tsu_set_time(&bp->ptp_clock_info, + (const struct timespec64 *)&now); } else { adj = (sign << GEM_ADDSUB_OFFSET) | delta; @@ -180,10 +207,33 @@ static int gem_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) return 0; } +static int gem_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); + int ret, unlock_ret; + + ret = rp1_extts_clock_lock(bp); + if (ret) + return ret; + ret = __gem_ptp_adjtime(ptp, delta); + unlock_ret = rp1_extts_clock_unlock(bp); + return ret ? ret : unlock_ret; +} + static int gem_ptp_enable(struct ptp_clock_info *ptp, struct ptp_clock_request *rq, int on) { - return -EOPNOTSUPP; + struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); + + return rp1_extts_enable(bp, rq, on); +} + +static int gem_ptp_verify(struct ptp_clock_info *ptp, unsigned int pin, + enum ptp_pin_function func, unsigned int chan) +{ + struct macb *bp = container_of(ptp, struct macb, ptp_clock_info); + + return rp1_extts_verify(bp, pin, func, chan); } static const struct ptp_clock_info gem_ptp_caps_template = { @@ -200,6 +250,7 @@ static const struct ptp_clock_info gem_ptp_caps_template = { .gettimex64 = gem_tsu_get_time, .settime64 = gem_tsu_set_time, .enable = gem_ptp_enable, + .verify = gem_ptp_verify, }; static void gem_ptp_init_timer(struct macb *bp) @@ -333,14 +384,23 @@ void gem_ptp_init(struct net_device *netdev) bp->ptp_clock_info.max_adj = bp->ptp_info->get_ptp_max_adj(); gem_ptp_init_timer(bp); gem_ptp_init_tsu(bp); + if (of_device_is_compatible(bp->pdev->dev.of_node, "raspberrypi,rp1-gem")) { + int err = rp1_extts_create(bp); + + if (err) + dev_err(&bp->pdev->dev, + "failed to initialize RP1 PTP GPIO support: %d\n", err); + } bp->ptp_clock = ptp_clock_register(&bp->ptp_clock_info, &netdev->dev); if (IS_ERR(bp->ptp_clock)) { pr_err("ptp clock register failed: %ld\n", PTR_ERR(bp->ptp_clock)); bp->ptp_clock = NULL; + rp1_extts_destroy(bp); return; } else if (bp->ptp_clock == NULL) { pr_err("ptp clock register failed\n"); + rp1_extts_destroy(bp); return; } @@ -352,10 +412,12 @@ void gem_ptp_remove(struct net_device *ndev) { struct macb *bp = netdev_priv(ndev); + rp1_extts_quiesce(bp); if (bp->ptp_clock) { ptp_clock_unregister(bp->ptp_clock); bp->ptp_clock = NULL; } + rp1_extts_destroy(bp); gem_ptp_clear_timer(bp); @@ -462,4 +524,3 @@ int gem_set_hwtst(struct net_device *dev, return 0; } - diff --git a/drivers/net/ethernet/cadence/macb_rp1_ptp.c b/drivers/net/ethernet/cadence/macb_rp1_ptp.c new file mode 100644 index 00000000000000..7f6709d7d2ca68 --- /dev/null +++ b/drivers/net/ethernet/cadence/macb_rp1_ptp.c @@ -0,0 +1,2646 @@ +// SPDX-License-Identifier: GPL-2.0 +/* RP1 PIO -> local DMA -> Ethernet TSU latch, exposed on the Ethernet PHC. + * The direct path supports one EXTS channel and 28 dynamically routable GPIOs. + * While PEROUT is active, up to two independent EXTS lanes use synchronized + * PIO counters mapped to the output pad monitor's TSU markers. The PIO waits + * for an arm token before detecting each direct-path transition. Host rearm + * creates dead time; unarmed transitions cannot be counted. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "macb.h" +#include "macb_rp1_ptp.h" +#include "macb_rp1_ptp_counter.h" +#include "macb_rp1_ptp_math.h" + +#define EXT_GPIO_COUNT 28 +#define EXT_MAX_AGE_NS 100000000ULL +#define EXT_WATCHDOG_MS 100 +#define EXT_INPUT_BIAS_NS 0 +#define EXT_OUTPUT_BIAS_NS 0 + +#define OUT_RECORDS 128 +#define RP1_MAX_EXTTS 2 +#define RP1_OUTPUT_PROGRAM_WORDS 22 +#define RP1_OUTPUT_STARTUP_PROGRAM_WORDS 26 +#define RP1_COUNTER_PROGRAM_WORDS 10 +#define RP1_COUNTER_RING_WORDS 256 + +/* The compact output program generates both the pad waveform and TSU markers. + * Do not add an input-direction monitor SM on this same GPIO: its direction + * request removes the pad output enable on RP1. + */ +static const u16 rp1_compact_output_program[RP1_OUTPUT_PROGRAM_WORDS] = { + 0x80a0, 0xa027, 0x0042, 0x80a0, 0xa0c3, 0x8020, 0xa0c7, 0xa04b, + 0xa023, 0x8080, 0xa027, 0x00b4, 0x0034, 0xa046, 0xa0c3, 0x9000, + 0xb022, 0x1051, 0xa042, 0x0000, 0x0089, 0x0009, +}; + +/* Rising-edge counter shared by the output monitor and external-input lanes. + * PIO_ADD_PROGRAM relocates its JMP addresses. + */ +static const u16 rp1_counter_program[RP1_COUNTER_PROGRAM_WORDS] = { + 0x00c7, 0x0080, 0x0000, 0x00c5, 0x0080, + 0x0083, 0x0003, 0xa0c2, 0x8000, 0x0003, +}; + +struct out_record { u64 sequence, target, stamp; s64 error; }; + +struct rp1_counter_anchor { + u64 sequence; + u64 stamp; + u64 counter_sequence; + u64 wraps; + u32 y; + u32 rate; + bool marker_valid; + bool counter_valid; +}; + +struct rp1_input_sample { + u64 sequence; + u64 wraps; + u32 y; +}; + +struct rp1_duplex_input { + struct rp1_extts *parent; + struct rp1_pinctrl_pio_pin *pin; + struct dma_chan *chan; + struct device *dma_dev; + u32 *ring; + dma_addr_t ring_dma; + struct rp1_input_sample *pending; + int gpio; + int sm; + int error; + int requested_gpio; + unsigned int requested_edges; + u32 saved_dma; + bool saved_dma_valid; + bool requested; + bool prepared; + bool started; + atomic64_t produced; + u64 consumed; + u64 samples; + u64 overruns; + u64 sequence; + u64 wraps; + u32 last_y; + bool have_last_y; + u64 events; + u64 last_event_ns; + u64 pending_head; + u64 pending_tail; + u64 max_uncertainty_ns; +}; + +struct rate_record { + u64 sequence, before, after, target, pulse; + u32 old_rate, new_rate, deadline_word; + u64 sent_at, drained_at; + int action; +}; + +struct rp1_extts { + struct macb *bp; + struct mutex lock; /* Serializes PTP, PIO, DMA and GPIO state. */ + struct delayed_work work; + wait_queue_head_t monitor_wait; + struct workqueue_struct *wq; + struct dentry *debug; + struct rp1_pio_client *pio; + struct rp1_pinctrl_pio_pin *pin; + void __iomem *control; + resource_size_t control_phys; + struct dma_chan *chan; + struct dma_chan *monitor_chan; + struct device *monitor_dma_dev; + u32 *monitor_ring; + dma_addr_t monitor_ring_dma; + dma_addr_t target; + dma_cookie_t cookie; + atomic_t done; + struct ptp_pin_desc pins[EXT_GPIO_COUNT]; + int sm, offset, words, gpio, edge, last_error; + int monitor_sm, monitor_offset, monitor_error; + bool mapped, saved_dma_valid, enabled, armed, dying; + bool monitor_program_loaded, monitor_started, monitor_saved_dma_valid; + u32 saved_dma; + u32 monitor_saved_dma; + u64 armed_at, last_stamp, events, faults, timeouts, clock_steps; + atomic64_t monitor_produced; + u64 monitor_consumed, monitor_samples, monitor_overruns, monitor_last; + u64 monitor_wraps; + bool monitor_have_last_y; + u64 last_age, max_age, last_deadtime, max_deadtime; + bool output, compact, calibrating, preparing; + u32 out_rate, out_high, out_width, out_phase; + u64 map_at, map_cycles_fp, out_retargets, out_late_skips; + u64 out_max_update_ns; + u32 map_rate, out_deadline; + bool map_valid, rate_irqs_held; + unsigned long rate_irq_flags; + u64 out_first, out_target, out_count, out_anchors, rate_changes; + u64 out_prepares, out_prepare_stamp, out_prepare_age, out_max_prepare_age; + u64 out_min_lead, out_last_latch; + s64 out_error, out_min_error, out_max_error; + struct rp1_duplex_input input[RP1_MAX_EXTTS]; + struct rp1_counter_anchor anchors[OUT_RECORDS]; + u64 anchor_sequence; + u64 anchor_out_base; + u64 anchor_mismatches; + int out_startup_origin; + int out_compact_origin; + struct out_record out_records[OUT_RECORDS]; + u64 rate_before, rate_count, out_rate_epoch, rate_history_floor; + struct rate_record rate_records[OUT_RECORDS]; +}; + +static int out_start(struct rp1_extts *s); +static int out_event(struct rp1_extts *s, u64 stamp, u64 now); +static int out_enable(struct rp1_extts *s, struct ptp_perout_request *rq, int on); +static int out_monitor_prepare(struct rp1_extts *s); +static int out_monitor_start(struct rp1_extts *s); +static int out_monitor_release(struct rp1_extts *s); +static int duplex_input_enable(struct rp1_extts *s, unsigned int index, + unsigned int edges, int on); +static int duplex_inputs_resync(struct rp1_extts *s, bool *disrupted); +static int duplex_input_prepare(struct rp1_extts *s, unsigned int index, + int gpio); +static int duplex_input_release(struct rp1_extts *s, unsigned int index); +static void ext_fail(struct rp1_extts *s, int err); +static int duplex_input_submit(struct rp1_duplex_input *in); +static int duplex_monitor_submit(struct rp1_extts *s); +static void duplex_anchor_counter(struct rp1_extts *s, u64 sequence, + u32 y, u64 wraps); + +static int out_inhibit(struct rp1_extts *s) +{ + if (!s->output || !s->pin) + return 0; + /* The provider asserts the low override before disabling the pad. */ + return rp1_pinctrl_pio_set_output(s->pin, false); +} + +static void ext_config(struct rp1_extts *s, struct rp1_pio_sm_init_args *init) +{ + unsigned int wrap_bottom = s->offset + + (s->output && !s->compact ? 5 : 0); + + init->sm = s->sm; + init->initial_pc = s->offset; + init->config.clkdiv = 1 << 16; + init->config.execctrl = wrap_bottom << 7 | + (s->offset + s->words - 1) << 12; + init->config.shiftctrl = BIT(18) | BIT(19); + if (s->output) + init->config.pinctrl = BIT(29) | (s->gpio << 10); +} + +static int ext_sm_enable(struct rp1_extts *s, bool enable) +{ + struct rp1_pio_sm_set_enabled_args args = { + .mask = BIT(s->sm), .enable = enable, + }; + int ret = rp1_pio_sm_set_enabled(s->pio, &args); + + return ret < 0 ? ret : 0; +} + +static int ext_dmactrl(struct rp1_extts *s, u32 ctrl) +{ + struct rp1_pio_sm_set_dmactrl_args args = { .sm = s->sm, .ctrl = ctrl }; + int ret = rp1_pio_sm_set_dmactrl(s->pio, &args); + + return ret < 0 ? ret : 0; +} + +static int ext_dmactrl_sm(struct rp1_extts *s, int sm, u32 ctrl) +{ + struct rp1_pio_sm_set_dmactrl_args args = { .sm = sm, .ctrl = ctrl }; + int ret = rp1_pio_sm_set_dmactrl(s->pio, &args); + + return ret < 0 ? ret : 0; +} + +static int ext_sm_set_enabled_mask(struct rp1_extts *s, u16 mask, bool enable) +{ + struct rp1_pio_sm_set_enabled_args args = { + .mask = mask, .enable = enable, + }; + int ret = rp1_pio_sm_set_enabled(s->pio, &args); + + return ret < 0 ? ret : 0; +} + +static int ext_now(struct rp1_extts *s, u64 *ns) +{ + struct timespec64 ts; + u64 sec; + u32 nsec; + int ret = s->bp->ptp_clock_info.gettimex64(&s->bp->ptp_clock_info, &ts, NULL); + + if (ret) + return ret; + if (ts.tv_sec < 0 || ts.tv_nsec < 0 || ts.tv_nsec >= NSEC_PER_SEC) + return -ERANGE; + sec = ts.tv_sec; + nsec = ts.tv_nsec; + if (sec > div_u64(U64_MAX - nsec, NSEC_PER_SEC)) + return -ERANGE; + *ns = sec * NSEC_PER_SEC + nsec; + return 0; +} + +static int ext_stamp(struct rp1_extts *s, u64 *ns) +{ + u32 hi = readl(s->bp->regs + 0x1c4); + u32 lo = readl(s->bp->regs + 0x1c8); + u32 nsec = readl(s->bp->regs + 0x1cc); + u64 sec = (u64)hi << 32 | lo; + + if (hi & 0xffff0000 || nsec >= NSEC_PER_SEC || + sec > div_u64(U64_MAX - NSEC_PER_SEC, NSEC_PER_SEC)) + return -ERANGE; + *ns = sec * NSEC_PER_SEC + nsec; + return 0; +} + +/* Caller holds s->lock. The callback never takes this mutex. */ +static int ext_stop(struct rp1_extts *s) +{ + struct rp1_pio_sm_clear_fifos_args args = { .sm = s->sm }; + int first_err = 0; + unsigned int i; + int ret; + + s->map_valid = false; + ret = out_inhibit(s); + if (ret) + first_err = ret; + if (s->pio && s->sm >= 0) { + ret = ext_sm_enable(s, false); + if (ret && !first_err) + first_err = ret; + if (s->monitor_sm >= 0) { + ret = ext_sm_set_enabled_mask(s, BIT(s->monitor_sm), false); + if (ret && !first_err) + first_err = ret; + } + if (s->chan) { + ret = dmaengine_terminate_sync(s->chan); + if (ret && !first_err) + first_err = ret; + } + if (s->monitor_chan) { + ret = dmaengine_terminate_sync(s->monitor_chan); + if (ret && !first_err) + first_err = ret; + } + { + unsigned int i; + + for (i = 0; i < RP1_MAX_EXTTS; i++) { + struct rp1_duplex_input *in = &s->input[i]; + + if (in->sm >= 0) { + ret = ext_sm_set_enabled_mask(s, BIT(in->sm), false); + if (ret && !first_err) + first_err = ret; + } + if (in->chan) { + ret = dmaengine_terminate_sync(in->chan); + if (ret && !first_err) + first_err = ret; + } + } + } + if (!first_err) { + ret = rp1_pio_sm_clear_fifos(s->pio, &args); + if (ret < 0) + first_err = ret; + } + } else if (s->chan) { + ret = dmaengine_terminate_sync(s->chan); + if (ret && !first_err) + first_err = ret; + } + s->monitor_started = false; + for (i = 0; i < RP1_MAX_EXTTS; i++) + s->input[i].started = false; + if (s->armed) { + /* 6 -> 0 creates a housekeeping capture. It is never published. */ + writel(0, s->control); + if (readl(s->control)) + first_err = first_err ?: -EIO; + else + s->armed = false; + } + atomic_set(&s->done, 0); + cancel_delayed_work(&s->work); + if (!s->enabled) + wake_up_all(&s->monitor_wait); + return first_err; +} + +static int ext_release(struct rp1_extts *s) +{ + int ret; + + ret = ext_stop(s); + if (ret) + return ret; + { + unsigned int i; + + for (i = 0; i < RP1_MAX_EXTTS; i++) { + if (s->input[i].prepared || s->input[i].sm >= 0 || + s->input[i].pin || s->input[i].chan || + s->input[i].ring) { + ret = duplex_input_release(s, i); + if (ret) + return ret; + } + } + } + ret = out_monitor_release(s); + if (ret) + return ret; + if (s->chan) { + if (s->mapped) { + dma_unmap_resource(s->chan->device->dev, s->target, 4, DMA_FROM_DEVICE, 0); + s->mapped = false; + } + ret = rp1_pio_dma_release(s->pio, s->sm, PIO_DIR_FROM_SM, s->chan); + if (ret) + return ret; + s->chan = NULL; + } + if (s->pio) { + if (s->saved_dma_valid) { + ret = ext_dmactrl(s, s->saved_dma); + if (ret) + return ret; + } + ret = rp1_pio_close_checked(s->pio); + if (ret) + return ret; + s->pio = NULL; + } + if (s->pin) { + ret = rp1_pinctrl_pio_release(s->pin); + if (ret) + return ret; + s->pin = NULL; + } + s->sm = -1; + s->offset = -1; + s->compact = false; + s->saved_dma_valid = false; + return 0; +} + +static void out_monitor_dma_done(void *arg) +{ + struct rp1_extts *s = arg; + + atomic64_inc(&s->monitor_produced); + if (!READ_ONCE(s->dying)) + mod_delayed_work(s->wq, &s->work, 0); +} + +static int out_monitor_release(struct rp1_extts *s) +{ + struct rp1_pio_sm_claim_args unclaim; + struct rp1_pio_remove_program_args remove; + int ret; + + if (s->monitor_chan) { + ret = rp1_pio_dma_release(s->pio, s->monitor_sm, + PIO_DIR_FROM_SM, s->monitor_chan); + if (ret) + return ret; + s->monitor_chan = NULL; + } + if (s->monitor_ring) { + if (!s->monitor_dma_dev) + return -EINVAL; + dma_free_coherent(s->monitor_dma_dev, + RP1_COUNTER_RING_WORDS * sizeof(*s->monitor_ring), + s->monitor_ring, s->monitor_ring_dma); + s->monitor_ring = NULL; + s->monitor_ring_dma = 0; + } + if (s->monitor_dma_dev) { + put_device(s->monitor_dma_dev); + s->monitor_dma_dev = NULL; + } + if (s->monitor_program_loaded) { + remove.num_instrs = RP1_COUNTER_PROGRAM_WORDS; + remove.origin = s->monitor_offset; + ret = rp1_pio_remove_program(s->pio, &remove); + if (ret < 0) + return ret; + s->monitor_program_loaded = false; + s->monitor_offset = -1; + } + if (s->monitor_sm >= 0) { + if (s->monitor_saved_dma_valid) { + ret = ext_dmactrl_sm(s, s->monitor_sm, s->monitor_saved_dma); + if (ret) + return ret; + s->monitor_saved_dma_valid = false; + } + unclaim.mask = BIT(s->monitor_sm); + ret = rp1_pio_sm_unclaim(s->pio, &unclaim); + if (ret < 0) + return ret; + s->monitor_sm = -1; + } + s->monitor_started = false; + s->monitor_offset = -1; + s->monitor_consumed = 0; + s->monitor_samples = 0; + s->monitor_overruns = 0; + s->monitor_last = 0; + s->monitor_wraps = 0; + s->monitor_have_last_y = false; + atomic64_set(&s->monitor_produced, 0); + return 0; +} + +static int out_monitor_prepare(struct rp1_extts *s) +{ + struct rp1_pio_sm_claim_args claim = {}; + struct rp1_pio_add_program_args prog = { + .num_instrs = RP1_COUNTER_PROGRAM_WORDS, + }; + struct rp1_pio_sm_init_args init = {}; + struct rp1_pio_sm_clear_fifos_args clear = {}; + struct rp1_pio_sm_exec_args exec = {}; + struct rp1_pio_sm_set_dmactrl_args dma = {}; + struct rp1_pio_sm_get_dmactrl_args old_dma = {}; + struct device *dma_dev; + int ret; + + if (s->monitor_sm >= 0) + return 0; + if (!s->compact || s->offset < 0 || + s->offset + RP1_OUTPUT_PROGRAM_WORDS + RP1_COUNTER_PROGRAM_WORDS > + RP1_PIO_INSTRUCTION_COUNT) + return -ENOSPC; + + ret = rp1_pio_sm_claim(s->pio, &claim); + if (ret < 0) + return ret; + s->monitor_sm = ret; + s->monitor_offset = s->offset + RP1_OUTPUT_PROGRAM_WORDS; + prog.origin = s->monitor_offset; + memcpy(prog.instrs, rp1_counter_program, sizeof(rp1_counter_program)); + ret = rp1_pio_can_add_program(s->pio, &prog); + if (ret < 0) + return ret; + if (!ret) + return -ENOSPC; + ret = rp1_pio_add_program(s->pio, &prog); + if (ret < 0) + return ret; + if (ret != s->monitor_offset) { + struct rp1_pio_remove_program_args remove = { + .num_instrs = RP1_COUNTER_PROGRAM_WORDS, + .origin = ret, + }; + int remove_ret = rp1_pio_remove_program(s->pio, &remove); + + return remove_ret < 0 ? remove_ret : -EPROTO; + } + s->monitor_program_loaded = true; + + init.sm = s->monitor_sm; + init.initial_pc = s->monitor_offset; + init.config.clkdiv = 1 << 16; + init.config.execctrl = (s->monitor_offset << 7) | + ((s->monitor_offset + RP1_COUNTER_PROGRAM_WORDS - 1) << 12) | + (s->gpio << 24); + init.config.shiftctrl = BIT(18) | BIT(19); + /* Deliberately omit PINCTRL and SET_PINDIRS. This SM only samples JMP PIN. */ + ret = rp1_pio_sm_init(s->pio, &init); + if (ret < 0) + return ret; + clear.sm = s->monitor_sm; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + old_dma.sm = s->monitor_sm; + ret = rp1_pio_sm_get_dmactrl(s->pio, &old_dma); + if (ret < 0) + return ret; + s->monitor_saved_dma = old_dma.ctrl; + s->monitor_saved_dma_valid = true; + exec.sm = s->monitor_sm; + exec.instr = 0xa04b; /* MOV Y, ~NULL */ + ret = rp1_pio_sm_exec(s->pio, &exec); + if (ret < 0) + return ret; + dma.sm = s->monitor_sm; + dma.ctrl = 0x80000101; + ret = rp1_pio_sm_set_dmactrl(s->pio, &dma); + if (ret < 0) + return ret; + s->monitor_chan = rp1_pio_dma_request(s->pio, s->monitor_sm, + PIO_DIR_FROM_SM); + if (IS_ERR(s->monitor_chan)) { + ret = PTR_ERR(s->monitor_chan); + s->monitor_chan = NULL; + return ret; + } + dma_dev = s->monitor_chan->device->dev; + get_device(dma_dev); + s->monitor_dma_dev = dma_dev; + s->monitor_ring = dma_alloc_coherent(dma_dev, + RP1_COUNTER_RING_WORDS * sizeof(*s->monitor_ring), + &s->monitor_ring_dma, GFP_KERNEL); + if (!s->monitor_ring) + return -ENOMEM; + memset(s->monitor_ring, 0, + RP1_COUNTER_RING_WORDS * sizeof(*s->monitor_ring)); + atomic64_set(&s->monitor_produced, 0); + s->monitor_consumed = 0; + s->monitor_samples = 0; + s->monitor_overruns = 0; + s->monitor_last = 0; + /* Keep the channel prepared but unissued until both counter lanes can + * establish their epoch together. The descriptor is submitted at start so + * PHC-step recovery can reuse the same resources safely. + */ + s->monitor_started = false; + return 0; +} + +static int out_monitor_start(struct rp1_extts *s) +{ + struct rp1_pio_sm_enable_sync_args enable = {}; + unsigned int i; + int ret; + + if (s->monitor_sm < 0 || !s->monitor_chan || !s->monitor_ring) + return -ENODEV; + if (s->monitor_started) + return -EALREADY; + for (i = 0; i < RP1_MAX_EXTTS; i++) { + if (!s->input[i].requested || s->input[i].prepared) + continue; + ret = duplex_input_prepare(s, i, s->input[i].requested_gpio); + if (ret) + return ret; + } + ret = duplex_monitor_submit(s); + if (ret) + return ret; + enable.mask = BIT(s->sm) | BIT(s->monitor_sm); + memset(s->monitor_ring, 0, + RP1_COUNTER_RING_WORDS * sizeof(*s->monitor_ring)); + atomic64_set(&s->monitor_produced, 0); + s->monitor_consumed = 0; + s->monitor_samples = 0; + s->monitor_wraps = 0; + s->monitor_have_last_y = false; + for (i = 0; i < RP1_MAX_EXTTS; i++) { + if (!s->input[i].prepared) + continue; + memset(s->input[i].ring, 0, + RP1_COUNTER_RING_WORDS * sizeof(*s->input[i].ring)); + atomic64_set(&s->input[i].produced, 0); + s->input[i].consumed = 0; + s->input[i].samples = 0; + s->input[i].overruns = 0; + s->input[i].sequence = 0; + s->input[i].wraps = 0; + s->input[i].have_last_y = false; + s->input[i].pending_head = 0; + s->input[i].pending_tail = 0; + s->input[i].last_event_ns = 0; + ret = duplex_input_submit(&s->input[i]); + if (ret) + return ret; + dma_async_issue_pending(s->input[i].chan); + enable.mask |= BIT(s->input[i].sm); + } + dma_async_issue_pending(s->monitor_chan); + s->anchor_out_base = s->out_count; + s->anchor_sequence = 0; + memset(s->anchors, 0, sizeof(s->anchors)); + ret = rp1_pio_sm_enable_sync(s->pio, &enable); + if (ret < 0) { + dmaengine_terminate_sync(s->monitor_chan); + for (i = 0; i < RP1_MAX_EXTTS; i++) + if (s->input[i].prepared) + dmaengine_terminate_sync(s->input[i].chan); + return ret; + } + s->monitor_started = true; + s->monitor_error = 0; + for (i = 0; i < RP1_MAX_EXTTS; i++) + s->input[i].started = s->input[i].prepared; + wake_up_all(&s->monitor_wait); + return 0; +} + +static int out_monitor_drain(struct rp1_extts *s) +{ + u64 produced = atomic64_read(&s->monitor_produced), first; + u32 available; + int ret; + + if (!s->monitor_started || !s->monitor_ring) + return 0; + ret = rp1_counter_ring_window(produced, s->monitor_consumed, + RP1_COUNTER_RING_WORDS, &first, &available); + if (ret == -EOVERFLOW) { + s->monitor_overruns += first - s->monitor_consumed; + s->monitor_consumed = first; + } else if (ret) { + return ret; + } + while (available--) { + u32 value; + u32 index = s->monitor_consumed % RP1_COUNTER_RING_WORDS; + u64 sequence; + + dma_rmb(); + value = READ_ONCE(s->monitor_ring[index]); + sequence = s->monitor_consumed + 1; + if (s->monitor_have_last_y && value > s->monitor_last) + s->monitor_wraps++; + s->monitor_last = value; + s->monitor_consumed++; + s->monitor_samples = sequence; + s->monitor_have_last_y = true; + duplex_anchor_counter(s, sequence, value, s->monitor_wraps); + } + if (ret == -EOVERFLOW) + s->monitor_error = ret; + return ret == -EOVERFLOW ? 0 : ret; +} + +static void duplex_input_dma_done(void *arg) +{ + struct rp1_duplex_input *in = arg; + struct rp1_extts *s = in->parent; + + atomic64_inc(&in->produced); + if (!READ_ONCE(s->dying)) + mod_delayed_work(s->wq, &s->work, 0); +} + +static int duplex_input_submit(struct rp1_duplex_input *in) +{ + struct dma_async_tx_descriptor *desc; + dma_cookie_t cookie; + + desc = dmaengine_prep_dma_cyclic(in->chan, in->ring_dma, + RP1_COUNTER_RING_WORDS * sizeof(*in->ring), + sizeof(*in->ring), DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) + return -EIO; + desc->callback = duplex_input_dma_done; + desc->callback_param = in; + cookie = dmaengine_submit(desc); + return dma_submit_error(cookie); +} + +static int duplex_input_release(struct rp1_extts *s, unsigned int index) +{ + struct rp1_duplex_input *in = &s->input[index]; + struct rp1_pio_sm_claim_args unclaim; + int ret; + + if (in->sm >= 0 && s->pio) { + ret = ext_sm_set_enabled_mask(s, BIT(in->sm), false); + if (ret) + return ret; + } + if (in->chan) { + ret = dmaengine_terminate_sync(in->chan); + if (ret) + return ret; + ret = rp1_pio_dma_release(s->pio, in->sm, PIO_DIR_FROM_SM, in->chan); + if (ret) + return ret; + in->chan = NULL; + } + if (in->ring) { + if (!in->dma_dev) + return -EINVAL; + dma_free_coherent(in->dma_dev, + RP1_COUNTER_RING_WORDS * sizeof(*in->ring), + in->ring, in->ring_dma); + in->ring = NULL; + in->ring_dma = 0; + } + kfree(in->pending); + in->pending = NULL; + if (in->dma_dev) { + put_device(in->dma_dev); + in->dma_dev = NULL; + } + if (in->sm >= 0 && s->pio) { + if (in->saved_dma_valid) { + ret = ext_dmactrl_sm(s, in->sm, in->saved_dma); + if (ret) + return ret; + in->saved_dma_valid = false; + } + unclaim.mask = BIT(in->sm); + ret = rp1_pio_sm_unclaim(s->pio, &unclaim); + if (ret < 0) + return ret; + in->sm = -1; + } + if (in->pin) { + ret = rp1_pinctrl_pio_release(in->pin); + if (ret) + return ret; + in->pin = NULL; + } + in->prepared = false; + in->started = false; + in->gpio = -1; + in->consumed = 0; + in->samples = 0; + in->overruns = 0; + in->sequence = 0; + in->wraps = 0; + in->have_last_y = false; + in->pending_head = 0; + in->pending_tail = 0; + atomic64_set(&in->produced, 0); + return 0; +} + +static int duplex_input_prepare(struct rp1_extts *s, unsigned int index, + int gpio) +{ + struct rp1_duplex_input *in = &s->input[index]; + struct rp1_pio_sm_claim_args claim = {}; + struct rp1_pio_sm_init_args init = {}; + struct rp1_pio_sm_set_pindirs_args dirs = {}; + struct rp1_pio_sm_clear_fifos_args clear = {}; + struct rp1_pio_sm_get_dmactrl_args old_dma = {}; + struct rp1_pio_sm_exec_args exec = {}; + struct device_node *provider_node; + bool synchronized; + int ret; + + if (in->prepared || !s->monitor_program_loaded || s->monitor_offset < 0) + return -EBUSY; + { + unsigned int i; + + if (gpio == s->gpio) + return -EBUSY; + for (i = 0; i < RP1_MAX_EXTTS; i++) + if (i != index && s->input[i].prepared && + gpio == s->input[i].gpio) + return -EBUSY; + } + provider_node = of_parse_phandle(s->bp->pdev->dev.of_node, + "raspberrypi,gpio-controller", 0); + if (!provider_node) + return -EINVAL; + in->pin = rp1_pinctrl_pio_request(&s->bp->pdev->dev, provider_node, gpio); + of_node_put(provider_node); + if (IS_ERR(in->pin)) { + ret = PTR_ERR(in->pin); + in->pin = NULL; + return ret; + } + in->parent = s; + in->gpio = gpio; + ret = rp1_pio_gpio_is_synchronized(s->pio, gpio, &synchronized); + if (ret || !synchronized) + return ret ? ret : -EOPNOTSUPP; + ret = rp1_pio_sm_claim(s->pio, &claim); + if (ret < 0) + return ret; + in->sm = ret; + init.sm = in->sm; + init.initial_pc = s->monitor_offset; + init.config.clkdiv = 1 << 16; + init.config.execctrl = (s->monitor_offset << 7) | + ((s->monitor_offset + RP1_COUNTER_PROGRAM_WORDS - 1) << 12) | + (gpio << 24); + init.config.shiftctrl = BIT(18) | BIT(19); + ret = rp1_pio_sm_init(s->pio, &init); + if (ret < 0) + return ret; + dirs.sm = in->sm; + dirs.mask = BIT(gpio); + ret = rp1_pio_sm_set_pindirs(s->pio, &dirs); + if (ret < 0) + return ret; + clear.sm = in->sm; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + old_dma.sm = in->sm; + ret = rp1_pio_sm_get_dmactrl(s->pio, &old_dma); + if (ret < 0) + return ret; + in->saved_dma = old_dma.ctrl; + in->saved_dma_valid = true; + ret = ext_dmactrl_sm(s, in->sm, 0x80000101); + if (ret) + return ret; + exec.sm = in->sm; + exec.instr = 0xa04b; /* MOV Y, ~NULL */ + ret = rp1_pio_sm_exec(s->pio, &exec); + if (ret < 0) + return ret; + in->chan = rp1_pio_dma_request(s->pio, in->sm, PIO_DIR_FROM_SM); + if (IS_ERR(in->chan)) { + ret = PTR_ERR(in->chan); + in->chan = NULL; + return ret; + } + in->dma_dev = in->chan->device->dev; + get_device(in->dma_dev); + in->ring = dma_alloc_coherent(in->dma_dev, + RP1_COUNTER_RING_WORDS * sizeof(*in->ring), + &in->ring_dma, GFP_KERNEL); + if (!in->ring) + return -ENOMEM; + memset(in->ring, 0, RP1_COUNTER_RING_WORDS * sizeof(*in->ring)); + in->pending = kcalloc(RP1_COUNTER_RING_WORDS, + sizeof(*in->pending), GFP_KERNEL); + if (!in->pending) + return -ENOMEM; + atomic64_set(&in->produced, 0); + in->prepared = true; + in->error = 0; + return 0; +} + +static int duplex_monitor_submit(struct rp1_extts *s) +{ + struct dma_async_tx_descriptor *desc; + dma_cookie_t cookie; + int ret; + + desc = dmaengine_prep_dma_cyclic(s->monitor_chan, s->monitor_ring_dma, + RP1_COUNTER_RING_WORDS * sizeof(*s->monitor_ring), + sizeof(*s->monitor_ring), DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) + return -EIO; + desc->callback = out_monitor_dma_done; + desc->callback_param = s; + cookie = dmaengine_submit(desc); + ret = dma_submit_error(cookie); + return ret; +} + +static int duplex_inputs_resync(struct rp1_extts *s, bool *disrupted) +{ + struct rp1_pio_sm_enable_sync_args enable = {}; + struct rp1_pio_sm_clear_fifos_args clear = {}; + struct rp1_pio_sm_exec_args exec = {}; + u16 mask = 0; + u64 now; + unsigned int i; + int ret; + + *disrupted = false; + if (s->monitor_sm < 0 || !s->monitor_chan || !s->monitor_ring) + return -ENODEV; + ret = ext_now(s, &now); + if (ret) + return ret; + if (s->out_target <= now + 50000000ULL) + return -ETIME; + mask |= BIT(s->monitor_sm); + for (i = 0; i < RP1_MAX_EXTTS; i++) + if (s->input[i].prepared) + mask |= BIT(s->input[i].sm); + /* A failed firmware call may still have changed part of the mask. */ + *disrupted = true; + ret = ext_sm_set_enabled_mask(s, mask, false); + if (ret) + return ret; + s->monitor_started = false; + for (i = 0; i < RP1_MAX_EXTTS; i++) + s->input[i].started = false; + ret = dmaengine_terminate_sync(s->monitor_chan); + if (ret) + return ret; + for (i = 0; i < RP1_MAX_EXTTS; i++) { + struct rp1_duplex_input *in = &s->input[i]; + + if (!in->prepared) + continue; + ret = dmaengine_terminate_sync(in->chan); + if (ret) + return ret; + } + for (i = 0; i <= RP1_MAX_EXTTS; i++) { + int sm = i ? s->input[i - 1].sm : s->monitor_sm; + + if (i && !s->input[i - 1].prepared) + continue; + clear.sm = sm; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + exec.sm = sm; + exec.instr = 0xa04b; + ret = rp1_pio_sm_exec(s->pio, &exec); + if (ret < 0) + return ret; + } + memset(s->monitor_ring, 0, + RP1_COUNTER_RING_WORDS * sizeof(*s->monitor_ring)); + atomic64_set(&s->monitor_produced, 0); + s->monitor_consumed = 0; + s->monitor_samples = 0; + s->monitor_overruns = 0; + s->monitor_last = 0; + s->monitor_wraps = 0; + s->monitor_have_last_y = false; + s->anchor_sequence = 0; + s->anchor_out_base = s->out_count; + memset(s->anchors, 0, sizeof(s->anchors)); + ret = duplex_monitor_submit(s); + if (ret) + return ret; + for (i = 0; i < RP1_MAX_EXTTS; i++) { + struct rp1_duplex_input *in = &s->input[i]; + + if (!in->prepared) + continue; + memset(in->ring, 0, + RP1_COUNTER_RING_WORDS * sizeof(*in->ring)); + atomic64_set(&in->produced, 0); + in->consumed = 0; + in->samples = 0; + in->overruns = 0; + in->sequence = 0; + in->wraps = 0; + in->have_last_y = false; + in->pending_head = 0; + in->pending_tail = 0; + in->last_event_ns = 0; + ret = duplex_input_submit(in); + if (ret) + return ret; + } + dma_async_issue_pending(s->monitor_chan); + for (i = 0; i < RP1_MAX_EXTTS; i++) + if (s->input[i].prepared) + dma_async_issue_pending(s->input[i].chan); + enable.mask = mask; + ret = rp1_pio_sm_enable_sync(s->pio, &enable); + if (ret < 0) { + dmaengine_terminate_sync(s->monitor_chan); + for (i = 0; i < RP1_MAX_EXTTS; i++) + if (s->input[i].prepared) + dmaengine_terminate_sync(s->input[i].chan); + return ret; + } + s->monitor_started = true; + s->monitor_error = 0; + for (i = 0; i < RP1_MAX_EXTTS; i++) + s->input[i].started = s->input[i].prepared; + wake_up_all(&s->monitor_wait); + return 0; +} + +static struct rp1_counter_anchor *duplex_anchor_get(struct rp1_extts *s, + u64 sequence) +{ + struct rp1_counter_anchor *anchor = + &s->anchors[sequence % OUT_RECORDS]; + + if (anchor->sequence != sequence) { + memset(anchor, 0, sizeof(*anchor)); + anchor->sequence = sequence; + } + return anchor; +} + +static void duplex_anchor_counter(struct rp1_extts *s, u64 sequence, + u32 y, u64 wraps) +{ + struct rp1_counter_anchor *anchor; + + if (!sequence) + return; + anchor = duplex_anchor_get(s, sequence); + anchor->counter_sequence = sequence; + anchor->wraps = wraps; + anchor->y = y; + anchor->counter_valid = true; + s->anchor_sequence = max(s->anchor_sequence, sequence); +} + +static void duplex_anchor_marker(struct rp1_extts *s, u64 output_sequence, + u64 stamp) +{ + struct rp1_counter_anchor *anchor; + u64 sequence; + + if (!s->monitor_started || output_sequence <= s->anchor_out_base) + return; + sequence = output_sequence - s->anchor_out_base; + anchor = duplex_anchor_get(s, sequence); + anchor->stamp = stamp; + anchor->rate = s->out_rate; + anchor->marker_valid = true; +} + +static struct rp1_counter_anchor *duplex_latest_anchor(struct rp1_extts *s) +{ + struct rp1_counter_anchor *latest = NULL; + unsigned int i; + + for (i = 0; i < OUT_RECORDS; i++) { + struct rp1_counter_anchor *anchor = &s->anchors[i]; + + if (!anchor->counter_valid || !anchor->marker_valid || + anchor->sequence != anchor->counter_sequence) + continue; + if (!latest || anchor->stamp > latest->stamp) + latest = anchor; + } + return latest; +} + +static int duplex_map_input_sample(struct rp1_extts *s, + struct rp1_duplex_input *in, + const struct rp1_input_sample *sample, + u64 now, u64 *stamp, u64 *uncertainty) +{ + struct rp1_counter_rate_change *changes; + struct rp1_counter_anchor *anchor = duplex_latest_anchor(s); + u64 start, i; + unsigned int count = 0; + s64 cycle_delta; + u64 oldest; + int ret; + + if (!anchor) + return -EAGAIN; + if (anchor->stamp > now || !anchor->rate) + return -ESTALE; + if (sample->sequence > RP1_S64_MAX || + anchor->counter_sequence > RP1_S64_MAX) + return -ERANGE; + if (rp1_counter_signed_cycle_delta(anchor->y, + anchor->counter_sequence, + anchor->wraps, sample->y, + sample->sequence, sample->wraps, + &cycle_delta)) + return -ESTALE; + changes = kcalloc(OUT_RECORDS, sizeof(*changes), GFP_KERNEL); + if (!changes) + return -ENOMEM; + start = s->rate_count > OUT_RECORDS ? s->rate_count - OUT_RECORDS : 0; + start = max(start, s->out_rate_epoch); + for (i = start; i < s->rate_count; i++) { + struct rate_record *record = &s->rate_records[i % OUT_RECORDS]; + struct rp1_counter_rate_change *change; + + if (record->sequence != i + 1) { + ret = -ESTALE; + goto out_free; + } + if (cycle_delta >= 0) { + if (record->before < anchor->stamp) { + if (record->after > anchor->stamp) { + ret = -ESTALE; + goto out_free; + } + continue; + } + if (record->after > now) + continue; + } else { + if (record->after > anchor->stamp) { + if (record->before < anchor->stamp) { + ret = -ESTALE; + goto out_free; + } + continue; + } + } + if (count == OUT_RECORDS) { + ret = -E2BIG; + goto out_free; + } + change = &changes[count++]; + change->before_ns = record->before; + change->after_ns = record->after; + change->old_rate = record->old_rate; + change->new_rate = record->new_rate; + } + oldest = anchor->stamp > 5000000000ULL ? anchor->stamp - 5000000000ULL : 0; + oldest = max(oldest, s->rate_history_floor); + ret = rp1_counter_sample_to_phc(anchor->stamp, anchor->rate, + anchor->y, anchor->counter_sequence, + anchor->wraps, sample->y, + sample->sequence, sample->wraps, + oldest, now, changes, count, stamp, + uncertainty); +out_free: + kfree(changes); + return ret; +} + +static int duplex_input_publish_pending(struct rp1_extts *s, + unsigned int index) +{ + struct rp1_duplex_input *in = &s->input[index]; + u64 now; + int ret; + + if (!in->prepared || !in->started) + return 0; + while (in->pending_head < in->pending_tail) { + struct rp1_input_sample *sample = + &in->pending[in->pending_head % RP1_COUNTER_RING_WORDS]; + struct ptp_clock_event ev = { + .type = PTP_CLOCK_EXTTS, + .index = index, + }; + u64 uncertainty; + + ret = ext_now(s, &now); + if (ret) + return ret; + ret = duplex_map_input_sample(s, in, sample, now, + &ev.timestamp, &uncertainty); + if (ret == -EAGAIN) + return 0; + if (ret) + return ret; + if (ev.timestamp <= in->last_event_ns) + return -ESTALE; + ptp_clock_event(s->bp->ptp_clock, &ev); + in->last_event_ns = ev.timestamp; + in->events++; + in->max_uncertainty_ns = max(in->max_uncertainty_ns, uncertainty); + in->pending_head++; + } + return 0; +} + +static int duplex_input_drain(struct rp1_extts *s, unsigned int index) +{ + struct rp1_duplex_input *in = &s->input[index]; + u64 produced = atomic64_read(&in->produced), first; + u32 available; + int ret; + + if (!in->started || !in->ring) + return 0; + ret = rp1_counter_ring_window(produced, in->consumed, + RP1_COUNTER_RING_WORDS, &first, &available); + if (ret == -EOVERFLOW) { + in->overruns += first - in->consumed; + in->consumed = first; + return ret; + } + if (ret) + return ret; + while (available--) { + u32 y; + struct rp1_input_sample *sample; + + dma_rmb(); + y = READ_ONCE(in->ring[in->consumed % RP1_COUNTER_RING_WORDS]); + if (in->pending_tail - in->pending_head >= RP1_COUNTER_RING_WORDS) + return -EOVERFLOW; + if (in->have_last_y && y > in->last_y) + in->wraps++; + in->sequence = in->consumed + 1; + sample = &in->pending[in->pending_tail % RP1_COUNTER_RING_WORDS]; + sample->sequence = in->sequence; + sample->wraps = in->wraps; + sample->y = y; + in->pending_tail++; + in->last_y = y; + in->have_last_y = true; + in->consumed++; + in->samples++; + } + return duplex_input_publish_pending(s, index); +} + +static int duplex_input_enable(struct rp1_extts *s, unsigned int index, + unsigned int edges, int on) +{ + struct rp1_duplex_input *in; + bool disrupted; + int gpio, ret; + + if (index >= RP1_MAX_EXTTS) + return -EOPNOTSUPP; + in = &s->input[index]; + if (!on) { + ret = duplex_input_release(s, index); + if (!ret) { + in->requested = false; + in->requested_gpio = -1; + in->requested_edges = 0; + } + return ret; + } + if (edges != PTP_RISING_EDGE) + return -EOPNOTSUPP; + if (!s->output || !s->enabled || !s->monitor_started) + return -EOPNOTSUPP; + if (in->prepared) + return in->gpio == ptp_find_pin(s->bp->ptp_clock, + PTP_PF_EXTTS, index) ? 0 : -EBUSY; + gpio = ptp_find_pin(s->bp->ptp_clock, PTP_PF_EXTTS, index); + if (gpio < 0 || gpio >= EXT_GPIO_COUNT) + return -EINVAL; + ret = duplex_input_prepare(s, index, gpio); + if (ret) + goto fail; + ret = duplex_inputs_resync(s, &disrupted); + if (ret) { + if (disrupted) { + /* Resync has stopped the shared counter epoch. Releasing only the + * new lane would leave existing lanes registered but silent. + */ + ext_fail(s, ret); + return ret; + } + goto fail; + } + in->requested = true; + in->requested_gpio = gpio; + in->requested_edges = edges; + return 0; +fail: + if (duplex_input_release(s, index)) + dev_err(&s->bp->pdev->dev, + "RP1 EXTS%u cleanup retained resources after error %d\n", + index, ret); + return ret; +} + +static void ext_fail(struct rp1_extts *s, int err) +{ + s->last_error = err; + s->faults++; + s->enabled = false; + wake_up_all(&s->monitor_wait); + if (ext_release(s)) + netdev_err(s->bp->dev, "RP1 cleanup retained resources after error %d\n", + s->last_error); + netdev_warn(s->bp->dev, "RP1 %s stopped (%d), enable the channel to retry\n", + s->output ? "PEROUT" : "EXTTS", err); +} + +static void ext_done(void *arg) +{ + struct rp1_extts *s = arg; + + atomic_set(&s->done, 1); + if (!READ_ONCE(s->dying)) + mod_delayed_work(s->wq, &s->work, 0); +} + +static int ext_arm(struct rp1_extts *s) +{ + struct dma_async_tx_descriptor *desc; + struct rp1_pio_sm_put_args put = { .sm = s->sm, .data = 0 }; + struct rp1_pio_sm_fifo_state_args fifo = { .sm = s->sm }; + u64 now; + int ret; + + if (readl(s->control)) + return -EBUSY; + ret = rp1_pio_sm_fifo_state(s->pio, &fifo); + if (ret < 0 || fifo.level) + return ret < 0 ? ret : -ESTALE; + ret = ext_now(s, &now); + if (ret) + return ret; + desc = dmaengine_prep_slave_single(s->chan, s->target, 4, + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) + return -EIO; + desc->callback = ext_done; + desc->callback_param = s; + s->cookie = dmaengine_submit(desc); + ret = dma_submit_error(s->cookie); + if (ret) + return ret; + atomic_set(&s->done, 0); + s->armed_at = now; + s->armed = true; + writel(6, s->control); + if (readl(s->control) != 6) + return -EIO; + mod_delayed_work(s->wq, &s->work, msecs_to_jiffies(EXT_WATCHDOG_MS)); + dma_wmb(); + dma_async_issue_pending(s->chan); + if (s->output) + return 0; + /* Allow a new edge only after its DMA is ready. No queued old tokens. */ + ret = rp1_pio_sm_put(s->pio, &put); + if (ret < 0) + return ret; + return 0; +} + +static void ext_work(struct work_struct *work) +{ + struct rp1_extts *s = container_of(to_delayed_work(work), struct rp1_extts, work); + struct ptp_clock_event ev = { .type = PTP_CLOCK_EXTTS, .index = 0 }; + u64 stamp, now, rearmed; + int ret; + + mutex_lock(&s->lock); + if (s->monitor_started) { + ret = out_monitor_drain(s); + if (ret) + s->monitor_error = ret; + if (s->monitor_error) { + unsigned int i; + + for (i = 0; i < RP1_MAX_EXTTS; i++) + if (s->input[i].prepared) { + ext_fail(s, s->monitor_error); + goto out; + } + } + } + { + unsigned int i; + + for (i = 0; i < RP1_MAX_EXTTS; i++) { + ret = duplex_input_drain(s, i); + if (ret) { + s->input[i].error = ret; + ext_fail(s, ret); + goto out; + } + } + } + if (s->dying || !s->enabled || !s->armed) + goto out; + if (!atomic_read(&s->done)) { + if (dma_async_is_tx_complete(s->chan, s->cookie, NULL, NULL) == DMA_ERROR) { + ext_fail(s, -EIO); + goto out; + } + if (s->output) { + ret = ext_now(s, &now); + if (ret || now > s->out_target + 200000000ULL) { + s->timeouts++; + ext_fail(s, ret ? ret : -ETIMEDOUT); + goto out; + } + } + /* An absent source is idle, not a fabricated timestamp or error. */ + mod_delayed_work(s->wq, &s->work, msecs_to_jiffies(EXT_WATCHDOG_MS)); + goto out; + } + if (dma_async_is_tx_complete(s->chan, s->cookie, NULL, NULL) != DMA_COMPLETE || + readl(s->control)) { + ext_fail(s, -EIO); + goto out; + } + s->armed = false; + ret = ext_stamp(s, &stamp); + if (!ret) + ret = ext_now(s, &now); + if (ret || stamp < s->armed_at || stamp > now || + (s->last_stamp && stamp <= s->last_stamp) || + (!s->output && stamp < EXT_INPUT_BIAS_NS)) { + ext_fail(s, ret ? ret : -ESTALE); + goto out; + } + s->last_age = now - stamp; + s->max_age = max(s->max_age, s->last_age); + if (s->last_age > EXT_MAX_AGE_NS) { + s->timeouts++; + ext_fail(s, -ETIMEDOUT); + goto out; + } + s->last_stamp = stamp; + if (s->output) { + ret = out_event(s, stamp, now); + if (!ret) { + unsigned int i; + + for (i = 0; i < RP1_MAX_EXTTS; i++) { + ret = duplex_input_publish_pending(s, i); + if (ret) + break; + } + } + if (ret) + ext_fail(s, ret); + goto out; + } + s->events++; + ev.timestamp = stamp - EXT_INPUT_BIAS_NS; + ptp_clock_event(s->bp->ptp_clock, &ev); + ret = ext_arm(s); + if (!ret && !ext_now(s, &rearmed) && rearmed >= stamp) { + s->last_deadtime = rearmed - stamp; + s->max_deadtime = max(s->max_deadtime, s->last_deadtime); + } + if (ret) + ext_fail(s, ret); +out: + mutex_unlock(&s->lock); +} + +static int ext_pin_get(struct rp1_extts *s) +{ + struct device_node *provider_node; + + provider_node = of_parse_phandle(s->bp->pdev->dev.of_node, + "raspberrypi,gpio-controller", 0); + if (!provider_node) + return -EINVAL; + s->pin = rp1_pinctrl_pio_request(&s->bp->pdev->dev, provider_node, + s->gpio); + of_node_put(provider_node); + if (IS_ERR(s->pin)) { + int ret = PTR_ERR(s->pin); + + s->pin = NULL; + return ret; + } + + return 0; +} + +static int out_add_program_at_origin(struct rp1_extts *s, + const u16 *instructions, unsigned int words) +{ + struct rp1_pio_add_program_args prog = { .num_instrs = words }; + unsigned int origin; + + if (!words || words > RP1_PIO_INSTRUCTION_COUNT) + return -EINVAL; + + for (origin = 0; origin <= RP1_PIO_INSTRUCTION_COUNT - words; origin++) { + int ret; + + prog.origin = origin; + /* PIO_ADD_PROGRAM relocates JMP instructions in firmware. */ + memcpy(prog.instrs, instructions, words * sizeof(prog.instrs[0])); + + ret = rp1_pio_can_add_program(s->pio, &prog); + if (ret < 0) + return ret; + if (!ret) + continue; + ret = rp1_pio_add_program(s->pio, &prog); + if (ret < 0) + return ret; + if (ret != origin) { + struct rp1_pio_remove_program_args remove = { + .num_instrs = words, + .origin = ret, + }; + int remove_ret = rp1_pio_remove_program(s->pio, &remove); + + return remove_ret < 0 ? remove_ret : -EPROTO; + } + return ret; + } + + return -ENOSPC; +} + +static int ext_acquire(struct rp1_extts *s) +{ + struct rp1_pio_sm_claim_args claim = {}; + struct rp1_pio_add_program_args prog = { .origin = RP1_PIO_ORIGIN_ANY }; + struct rp1_pio_sm_init_args init = {}; + struct rp1_pio_sm_set_pindirs_args dirs = {}; + struct rp1_pio_sm_get_dmactrl_args dma = {}; + struct device_node *provider_node; + bool synchronized; + int ret; + + if (s->pin || s->pio || s->chan || s->mapped) + return -EBUSY; + if (!s->control || readl(s->control)) + return -EBUSY; + ret = ext_pin_get(s); + if (ret) + return ret; + provider_node = of_parse_phandle(s->bp->pdev->dev.of_node, + "raspberrypi,pio", 0); + if (!provider_node) + return -EINVAL; + s->pio = rp1_pio_open_for_device(&s->bp->pdev->dev, provider_node); + of_node_put(provider_node); + if (IS_ERR(s->pio)) { + ret = PTR_ERR(s->pio); + s->pio = NULL; + return ret; + } + ret = rp1_pio_gpio_is_synchronized(s->pio, s->gpio, &synchronized); + if (ret || (!s->output && !synchronized)) + return ret ? ret : -EOPNOTSUPP; + s->sm = rp1_pio_sm_claim(s->pio, &claim); + if (s->sm < 0) + return s->sm; + ret = ext_sm_enable(s, false); + if (ret) + return ret; + dma.sm = s->sm; + ret = rp1_pio_sm_get_dmactrl(s->pio, &dma); + if (ret < 0) + return ret; + s->saved_dma = dma.ctrl; + s->saved_dma_valid = true; + ret = ext_dmactrl(s, s->saved_dma & ~BIT(31)); + if (ret) + return ret; + /* WAIT GPIO uses the absolute GPIO, independent of IN_BASE. */ + if (s->edge == PTP_EXTTS_EDGES) { + /* Alternate WAIT LOW and WAIT HIGH to capture every transition. */ + const u16 program[] = { + 0x80a0, 0x2000 | s->gpio, 0xa0c3, 0x8020, + 0x80a0, 0x2080 | s->gpio, 0xa0c3, 0x8020, + }; + + prog.num_instrs = ARRAY_SIZE(program); + memcpy(prog.instrs, program, sizeof(program)); + } else { + /* PULL; WAIT opposite level; WAIT selected level; MOV; PUSH. */ + prog.num_instrs = 5; + prog.instrs[0] = 0x80a0; + prog.instrs[1] = 0x2000 | + (s->edge == PTP_FALLING_EDGE ? 0x80 : 0) | s->gpio; + prog.instrs[2] = 0x2000 | + (s->edge == PTP_RISING_EDGE ? 0x80 : 0) | s->gpio; + prog.instrs[3] = 0xa0c3; + prog.instrs[4] = 0x8020; + } + if (s->output) { + /* Startup -> checkpoint: G0+C+7. HIGH: H+3. + * Edge -> next checkpoint: H+C+10 (unless a LOW PULL stalls). + * Checkpoint -> edge: 9+4*(UINT_MAX-deadline) cycles. + * ISR holds HIGH; Y counts down; X is a live-replaceable deadline. + * Only C,H are queued outside the fine loop. While counting, every + * TX word is a replacement deadline; empty PULL preserves X. + */ + const u16 program[] = { + 0x80a0, 0xa027, 0xa0c3, 0x8020, 0x0044, + 0x80a0, 0xa027, 0x0047, 0x80a0, 0xa0c3, 0x8020, + 0xa0c7, 0xa04b, 0xa023, 0x8080, 0xa027, 0x00b8, + 0xa046, 0xa0c3, 0x9000, 0xb022, 0x1055, 0xa042, + 0x0005, 0x008e, 0x000e, + }; + BUILD_BUG_ON(ARRAY_SIZE(program) != RP1_OUTPUT_STARTUP_PROGRAM_WORDS); + prog.num_instrs = ARRAY_SIZE(program); + memcpy(prog.instrs, program, sizeof(program)); + } + s->words = prog.num_instrs; + s->offset = s->output ? + out_add_program_at_origin(s, prog.instrs, prog.num_instrs) : + rp1_pio_add_program(s->pio, &prog); + if (s->offset < 0) + return s->offset; + if (s->output) + s->out_startup_origin = s->offset; + ext_config(s, &init); + ret = rp1_pio_sm_init(s->pio, &init); + if (ret < 0) + return ret; + dirs.sm = s->sm; + dirs.mask = BIT(s->gpio); + ret = rp1_pio_sm_set_pindirs(s->pio, &dirs); + if (ret < 0) + return ret; + ret = ext_dmactrl(s, 0x80000101); + if (ret) + return ret; + s->chan = rp1_pio_dma_request(s->pio, s->sm, PIO_DIR_FROM_SM); + if (IS_ERR(s->chan)) { + ret = PTR_ERR(s->chan); + s->chan = NULL; + return ret; + } + s->target = dma_map_resource(s->chan->device->dev, s->control_phys, 4, + DMA_FROM_DEVICE, 0); + if (dma_mapping_error(s->chan->device->dev, s->target)) + return -EIO; + s->mapped = true; + if (s->target + 3 > dma_get_mask(s->chan->device->dev) || + (s->chan->device->dev->bus_dma_limit && + s->target + 3 > s->chan->device->dev->bus_dma_limit)) + return -ERANGE; + return 0; +} + +static int ext_start(struct rp1_extts *s) +{ + struct rp1_pio_sm_init_args init = { .sm = s->sm, .initial_pc = s->offset }; + struct rp1_pio_sm_set_pindirs_args dirs = { .sm = s->sm, .mask = BIT(s->gpio) }; + struct rp1_pio_sm_clear_fifos_args clear = { .sm = s->sm }; + int ret; + + if (s->output) + return out_start(s); + init.config.clkdiv = 1 << 16; + init.config.execctrl = s->offset << 7 | (s->offset + s->words - 1) << 12; + init.config.shiftctrl = BIT(18) | BIT(19); + ret = rp1_pio_sm_init(s->pio, &init); + if (ret < 0) + return ret; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + ret = rp1_pio_sm_set_pindirs(s->pio, &dirs); + if (ret < 0) + return ret; + s->last_stamp = 0; + ret = ext_dmactrl(s, 0x80000101); + if (!ret) + ret = ext_arm(s); + if (!ret) + ret = s->monitor_sm >= 0 ? out_monitor_start(s) : + ext_sm_enable(s, true); + return ret; +} + +static u32 out_rate(struct rp1_extts *s) +{ + unsigned long flags; + u32 ti, sub; + + spin_lock_irqsave(&s->bp->tsu_clk_lock, flags); + ti = readl(s->bp->regs + GEM_TI); + sub = readl(s->bp->regs + GEM_TISUBN); + spin_unlock_irqrestore(&s->bp->tsu_clk_lock, flags); + /* Alternate increments and another TSU source rate are not supported. */ + if ((ti & ~255U) || s->bp->tsu_rate != 50000000 || ti < 18 || ti > 22) + return 0; + return (ti << 24) | ((sub & 0xffff) << 8) | (sub >> 24); +} + +static int out_put(struct rp1_extts *s, u32 value) +{ + struct rp1_pio_sm_put_args put = { .sm = s->sm, .data = value }; + struct rp1_pio_sm_fifo_state_args fifo = { .sm = s->sm, .tx = 1 }; + int ret = rp1_pio_sm_fifo_state(s->pio, &fifo); + + if (ret < 0 || fifo.full) + return ret < 0 ? ret : -ENOSPC; + ret = rp1_pio_sm_put(s->pio, &put); + return ret < 0 ? ret : 0; +} + +static int out_load_compact_program(struct rp1_extts *s) +{ + struct rp1_pio_remove_program_args remove = { + .num_instrs = s->words, + .origin = s->offset, + }; + struct rp1_pio_sm_clear_fifos_args clear = { .sm = s->sm }; + struct rp1_pio_sm_set_pindirs_args dirs = { + .sm = s->sm, .mask = BIT(s->gpio), .dirs = BIT(s->gpio), + }; + struct rp1_pio_sm_init_args init = {}; + int ret; + + if (!s->pio || s->sm < 0 || s->offset < 0 || s->compact || + s->words != RP1_OUTPUT_STARTUP_PROGRAM_WORDS) + return -EINVAL; + + ret = ext_sm_enable(s, false); + if (ret) + return ret; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + ret = rp1_pio_remove_program(s->pio, &remove); + if (ret < 0) + return ret; + s->offset = -1; + s->words = 0; + + ret = out_add_program_at_origin(s, rp1_compact_output_program, + RP1_OUTPUT_PROGRAM_WORDS); + if (ret < 0) + return ret; + s->offset = ret; + s->out_compact_origin = ret; + s->words = RP1_OUTPUT_PROGRAM_WORDS; + s->compact = true; + ext_config(s, &init); + ret = rp1_pio_sm_init(s->pio, &init); + if (ret < 0) + return ret; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + ret = rp1_pio_sm_set_pindirs(s->pio, &dirs); + if (ret < 0) + return ret; + ret = ext_dmactrl(s, 0x80000101); + if (ret) + return ret; + /* The output SM itself emits the TSU marker on its pad transition. */ + ret = out_monitor_prepare(s); + if (ret) { + int cleanup_ret = out_monitor_release(s); + + if (cleanup_ret) + return cleanup_ret; + s->monitor_error = ret; + wake_up_all(&s->monitor_wait); + dev_dbg(&s->bp->pdev->dev, + "RP1 output pad monitor unavailable (%d); output remains active\n", + ret); + } else { + s->monitor_error = 0; + } + return 0; +} + +static int out_start(struct rp1_extts *s) +{ + struct rp1_pio_sm_init_args init = {}; + struct rp1_pio_sm_clear_fifos_args clear = { .sm = s->sm }; + struct rp1_pio_sm_set_pindirs_args dirs = { + .sm = s->sm, .mask = BIT(s->gpio), .dirs = BIT(s->gpio), + }; + unsigned int i; + bool input_requested = false; + u64 now; + int ret; + + s->out_rate = out_rate(s); + if (!s->out_rate) + return -EOPNOTSUPP; + s->out_high = out_ns_cycles(s->out_width, s->out_rate) - 3; + ret = out_inhibit(s); + if (ret) + return ret; + ext_config(s, &init); + ret = rp1_pio_sm_init(s->pio, &init); + if (ret < 0) + return ret; + ret = rp1_pio_sm_clear_fifos(s->pio, &clear); + if (ret < 0) + return ret; + ret = rp1_pio_sm_set_pindirs(s->pio, &dirs); + if (ret < 0) + return ret; + ret = ext_now(s, &now); + if (ret || s->out_first < now + OUT_MIN_LEAD_NS) + return ret ? ret : -ETIME; + s->calibrating = true; + s->last_stamp = 0; + s->map_valid = false; + s->out_rate_epoch = s->rate_count; + s->rate_history_floor = 0; + s->out_target = now; /* Watchdog for the immediate, invisible anchor. */ + ret = ext_dmactrl(s, 0x80000101); + if (!ret) + ret = out_put(s, OUT_GUARD_CYCLES); + if (!ret) + ret = ext_arm(s); + if (!ret && s->monitor_sm >= 0) { + ret = out_monitor_start(s); + } else if (!ret && s->compact) { + for (i = 0; i < RP1_MAX_EXTTS; i++) + input_requested |= s->input[i].requested; + if (input_requested) + ret = s->monitor_error ?: -ENODEV; + else + ret = ext_sm_enable(s, true); + } else if (!ret) { + ret = ext_sm_enable(s, true); + } + return ret; +} + +/* These helpers only perform arithmetic or ordered MMIO. No firmware RPC. + * s->lock serializes updates against timestamp collection and time steps. + */ +static int out_map_advance(struct rp1_extts *s, u64 at, u32 rate) +{ + if (at < s->map_at || at - s->map_at > 100000000ULL || !s->map_rate) + return -ERANGE; + s->map_cycles_fp += out_ns_cycles_fp(at - s->map_at, s->map_rate); + s->map_at = at; + s->map_rate = rate; + return 0; +} + +static int out_map_seed(struct rp1_extts *s, u64 stamp) +{ + u64 i, start = s->rate_count > OUT_RECORDS ? s->rate_count - OUT_RECORDS : 0; + bool found = false; + int ret; + + start = max(start, s->out_rate_epoch); + if (stamp < s->rate_history_floor) + return -ESTALE; + s->map_at = stamp; + s->map_cycles_fp = 0; + s->map_rate = s->out_rate; + /* Include updates between this frozen checkpoint and its collection. */ + for (i = start; i < s->rate_count; i++) { + struct rate_record *r = &s->rate_records[i % OUT_RECORDS]; + u64 mid = r->before + (r->after - r->before) / 2; + + if (mid < stamp) + continue; + if (!found) { + s->map_rate = r->old_rate; + found = true; + } + if (r->old_rate != s->map_rate) + return -ESTALE; + ret = out_map_advance(s, mid, r->new_rate); + if (ret) + return ret; + } + return s->map_rate == s->out_rate ? 0 : -ESTALE; +} + +static int out_send_deadline(struct rp1_extts *s, u64 *sent, u64 *drained) +{ + u64 cycles_fp, deadline = s->out_target + EXT_OUTPUT_BIAS_NS; + int ret = ext_now(s, sent); + + if (ret) + return ret; + /* No word may escape the fine loop and be mistaken for a coarse count. + * Include the previously programmed match when a new rate delays it. + */ + if (deadline < *sent + OUT_UPDATE_GUARD_NS) + return -ETIME; + if (deadline <= s->map_at || deadline - s->map_at > 100000000ULL) + return -ERANGE; + cycles_fp = s->map_cycles_fp + out_ns_cycles_fp(deadline - s->map_at, s->map_rate); + if (cycles_fp < ((u64)OUT_FINE_OFFSET_CYCLES << 16) || + cycles_fp > ((u64)10000000 << 16)) + return -ERANGE; + if (s->map_valid) { + u64 old_fp = ((u64)OUT_FINE_OFFSET_CYCLES + + (u64)(OUT_COUNTER_SEED - s->out_deadline) * + OUT_FINE_TICK_CYCLES) << 16; + u64 current_fp = s->map_cycles_fp + + out_ns_cycles_fp(*sent - s->map_at, s->map_rate); + + if (old_fp < current_fp + out_ns_cycles_fp(OUT_UPDATE_GUARD_NS, s->map_rate)) + return -ETIME; + } + s->out_deadline = out_deadline_word(cycles_fp); + ret = rp1_pio_sm_put_mmio(s->pio, s->sm, s->out_deadline); + if (ret) + return ret; + /* A following ordered PHC read drains the posted write to this endpoint. */ + ret = ext_now(s, drained); + if (ret || *drained < *sent || *drained >= deadline - OUT_UPDATE_GUARD_NS / 2) + return ret ? ret : -ETIME; + s->map_valid = true; + return 0; +} + +static int out_event(struct rp1_extts *s, u64 stamp, u64 now) +{ + u64 cycles, measured_now, sent, drained; + unsigned int i; + bool transitioned = false; + bool input_requested = false; + int ret; + + if (out_rate(s) != s->out_rate) + return -ESTALE; + if (s->calibrating) { + if (s->out_first + EXT_OUTPUT_BIAS_NS < stamp + OUT_MIN_LEAD_NS) + return -ETIME; + cycles = out_ns_cycles(s->out_first + EXT_OUTPUT_BIAS_NS - + OUT_PREP_LEAD_NS - stamp, s->out_rate); + if (cycles <= OUT_COMPACT_CHECKPOINT_CYCLES || cycles > U32_MAX) + return -ERANGE; + if (!s->compact) { + ret = out_load_compact_program(s); + if (ret) + return ret; + transitioned = true; + } + s->out_high = out_ns_cycles(s->out_width, s->out_rate) - 3; + /* The compact program starts from word zero after the startup program + * is removed. Its first checkpoint is five cycles after its count. + * OUT_GUARD_CYCLES has already elapsed before this PHC anchor. + */ + ret = out_put(s, cycles - OUT_COMPACT_CHECKPOINT_CYCLES); + if (!ret) + ret = out_put(s, s->out_high); + s->out_target = s->out_first; + s->preparing = true; + if (!ret) + ret = ext_arm(s); + if (!ret && transitioned) { + if (s->monitor_sm >= 0) { + ret = out_monitor_start(s); + } else { + for (i = 0; i < RP1_MAX_EXTTS; i++) + input_requested |= s->input[i].requested; + ret = input_requested ? + (s->monitor_error ?: -ENODEV) : + ext_sm_enable(s, true); + } + } + if (!ret) + ret = ext_now(s, &measured_now); + if (ret || measured_now - stamp + 50000000ULL >= + out_cycles_ns(OUT_GUARD_CYCLES, s->out_rate)) + return ret ? ret : -ETIME; + ret = rp1_pinctrl_pio_set_output(s->pin, true); + if (ret) + return ret; + s->calibrating = false; + s->out_anchors++; + return 0; + } + if (s->preparing) { + unsigned long flags; + + if (s->out_target + EXT_OUTPUT_BIAS_NS <= now + OUT_UPDATE_GUARD_NS || + s->out_target + EXT_OUTPUT_BIAS_NS > stamp + 30000000ULL) + return -ETIME; + s->map_valid = false; + ret = out_map_seed(s, stamp); + if (!ret) + ret = ext_arm(s); + if (ret) + return ret; + local_irq_save(flags); + ret = out_send_deadline(s, &sent, &drained); + local_irq_restore(flags); + if (ret) + return ret; + s->out_prepares++; + s->out_prepare_stamp = stamp; + s->out_prepare_age = drained - stamp; + s->out_max_prepare_age = max(s->out_max_prepare_age, s->out_prepare_age); + s->out_min_lead = min(s->out_min_lead, + s->out_target + EXT_OUTPUT_BIAS_NS - drained); + s->preparing = false; + return 0; + } + + s->map_valid = false; + s->out_last_latch = stamp; + s->out_error = (s64)stamp - (s64)(s->out_target + EXT_OUTPUT_BIAS_NS); + s->out_count++; + if (s->out_count == 1) { + s->out_min_error = s->out_error; + s->out_max_error = s->out_error; + } + s->out_min_error = min(s->out_min_error, s->out_error); + s->out_max_error = max(s->out_max_error, s->out_error); + s->out_records[(s->out_count - 1) % OUT_RECORDS] = (struct out_record) { + .sequence = s->out_count, .target = s->out_target, + .stamp = stamp, .error = s->out_error, + }; + duplex_anchor_marker(s, s->out_count, stamp); + if (abs(s->out_error) > 100000) + return -ERANGE; + cycles = out_ns_cycles(s->out_target + NSEC_PER_SEC + + EXT_OUTPUT_BIAS_NS - OUT_PREP_LEAD_NS - stamp, + s->out_rate); + if (cycles <= s->out_high + 10 || cycles > U32_MAX) + return -ERANGE; + cycles -= s->out_high + 10; + s->out_high = out_ns_cycles(s->out_width, s->out_rate) - 3; + s->out_target += NSEC_PER_SEC; + s->preparing = true; + ret = ext_arm(s); + if (!ret) + ret = out_put(s, cycles); + if (!ret) + ret = out_put(s, s->out_high); + return ret; +} + +static int out_enable(struct rp1_extts *s, struct ptp_perout_request *rq, int on) +{ + u64 now, start; + u32 width = NSEC_PER_SEC / 2; + int pin, ret = 0; + + if (rq->index) + return -EOPNOTSUPP; + if (on && (rq->flags & ~(PTP_PEROUT_PHASE | PTP_PEROUT_DUTY_CYCLE))) + return -EOPNOTSUPP; + if (on && (rq->period.sec != 1 || rq->period.nsec)) + return -EOPNOTSUPP; + if (on && (rq->flags & PTP_PEROUT_DUTY_CYCLE)) { + if (rq->on.sec || rq->on.nsec < 1000000 || + rq->on.nsec > 500000000 || rq->on.nsec % 5) + return -ERANGE; + width = rq->on.nsec; + } + mutex_lock(&s->lock); + if (s->dying) { + ret = -ENODEV; + goto out; + } + if (!on) { + if (s->output) { + if (s->input[1].requested || s->input[1].prepared) { + ret = -EBUSY; + goto out; + } + s->enabled = false; + wake_up_all(&s->monitor_wait); + ret = ext_release(s); + if (ret) + goto out; + s->output = false; + if (s->input[0].requested) { + s->gpio = s->input[0].requested_gpio; + s->edge = s->input[0].requested_edges; + s->last_stamp = 0; + ret = ext_acquire(s); + if (!ret) { + s->enabled = true; + ret = ext_start(s); + } + if (ret) { + s->enabled = false; + if (ext_release(s)) + dev_err(&s->bp->pdev->dev, + "RP1 EXTS0 restart cleanup failed\n"); + } + } + } + goto out; + } + pin = ptp_find_pin(s->bp->ptp_clock, PTP_PF_PEROUT, 0); + if (pin < 0 || pin >= EXT_GPIO_COUNT) { + ret = -EINVAL; + goto out; + } + if (s->enabled) { + if (s->output || !s->input[0].requested || + s->input[1].requested || s->input[0].requested_gpio == pin) { + ret = -EBUSY; + goto out; + } + /* Promote the direct one-channel input path into the duplex counter + * path. The requested GPIO and edge mode survive resource teardown. + */ + s->enabled = false; + ret = ext_release(s); + if (ret) + goto out; + } + ret = ext_now(s, &now); + if (ret) + goto out; + if (rq->flags & PTP_PEROUT_PHASE) { + if (rq->phase.sec || rq->phase.nsec >= NSEC_PER_SEC || rq->phase.nsec % 5) { + ret = -EINVAL; + goto out; + } + start = out_next_grid(now + NSEC_PER_SEC, rq->phase.nsec); + } else { + if (rq->start.sec < 0 || rq->start.nsec >= NSEC_PER_SEC || rq->start.nsec % 5 || + rq->start.sec > div_u64(U64_MAX - NSEC_PER_SEC, NSEC_PER_SEC)) { + ret = -EINVAL; + goto out; + } + start = (u64)rq->start.sec * NSEC_PER_SEC + rq->start.nsec; + if (start < now + OUT_MIN_LEAD_NS || start > now + 10 * NSEC_PER_SEC) { + ret = -ETIME; + goto out; + } + } + s->gpio = pin; + s->output = true; + s->out_first = start; + s->out_phase = do_div(start, NSEC_PER_SEC); + s->out_width = width; + s->out_count = 0; + s->out_anchors = 0; + s->out_prepares = 0; + s->out_prepare_stamp = 0; + s->out_prepare_age = 0; + s->out_max_prepare_age = 0; + s->out_retargets = 0; + s->out_late_skips = 0; + s->out_max_update_ns = 0; + s->out_error = 0; + s->out_min_error = 0; + s->out_max_error = 0; + s->out_min_lead = U64_MAX; + s->out_startup_origin = -1; + s->out_compact_origin = -1; + s->monitor_sm = -1; + s->monitor_offset = -1; + atomic64_set(&s->monitor_produced, 0); + ret = ext_acquire(s); + if (!ret) { + s->enabled = true; + ret = out_start(s); + } + if (ret) { + s->enabled = false; + if (ext_release(s)) + dev_err(&s->bp->pdev->dev, + "RP1 PEROUT cleanup retained resources after error %d\n", ret); + } + s->last_error = ret; +out: + mutex_unlock(&s->lock); + return ret; +} + +int rp1_extts_enable(struct macb *bp, struct ptp_clock_request *rq, int on) +{ + struct rp1_extts *s = bp->rp1_extts; + unsigned int edges; + int pin, ret = 0; + + if (s && rq->type == PTP_CLK_REQ_PEROUT) + return out_enable(s, &rq->perout, on); + if (!s || rq->type != PTP_CLK_REQ_EXTTS || + rq->extts.index >= RP1_MAX_EXTTS) + return -EOPNOTSUPP; + edges = rq->extts.flags & PTP_EXTTS_EDGES; + if (on && !edges && (rq->extts.flags & PTP_STRICT_FLAGS)) + return -EINVAL; + if (!edges) + edges = PTP_RISING_EDGE; + mutex_lock(&s->lock); + if (s->dying) { + ret = -ENODEV; + goto out; + } + if (on && s->output && s->enabled && !s->monitor_started) { + long waited; + + mutex_unlock(&s->lock); + waited = wait_event_interruptible_timeout(s->monitor_wait, + READ_ONCE(s->monitor_started) || + (READ_ONCE(s->monitor_sm) < 0 && + READ_ONCE(s->monitor_error)) || + !READ_ONCE(s->enabled) || READ_ONCE(s->dying), + msecs_to_jiffies(2000)); + if (waited < 0) + return waited; + if (!waited) + return -ETIMEDOUT; + mutex_lock(&s->lock); + if (s->dying) { + ret = -ENODEV; + goto out; + } + if (!s->monitor_started) { + ret = s->monitor_error ?: s->last_error ?: -EIO; + goto out; + } + } + if (s->output && s->enabled) { + ret = duplex_input_enable(s, rq->extts.index, edges, on); + if (ret) + s->last_error = ret; + goto out; + } + if (!on && s->input[rq->extts.index].requested) { + if (!s->output && s->enabled && !rq->extts.index) { + s->enabled = false; + ret = ext_release(s); + } else { + ret = duplex_input_release(s, rq->extts.index); + } + if (!ret) { + s->input[rq->extts.index].requested = false; + s->input[rq->extts.index].requested_gpio = -1; + s->input[rq->extts.index].requested_edges = 0; + } + goto out; + } + if (rq->extts.index) { + ret = -EOPNOTSUPP; + goto out; + } + if (!on) { + if (!s->output) { + s->enabled = false; + ret = ext_release(s); + } + goto out; + } + pin = ptp_find_pin(bp->ptp_clock, PTP_PF_EXTTS, 0); + if (pin < 0 || pin >= EXT_GPIO_COUNT) { + ret = -EINVAL; + goto out; + } + if (s->enabled) { + ret = (!s->output && s->gpio == pin && s->edge == edges) ? 0 : -EBUSY; + goto out; + } + s->gpio = pin; + s->output = false; + s->edge = edges; + s->input[0].requested = true; + s->input[0].requested_gpio = pin; + s->input[0].requested_edges = edges; + ret = ext_acquire(s); + if (!ret) { + s->enabled = true; + ret = ext_start(s); + } + if (ret) { + s->enabled = false; + s->last_error = ret; + if (ext_release(s)) + dev_err(&bp->pdev->dev, + "RP1 EXTS cleanup retained resources after error %d\n", ret); + s->input[0].requested = false; + s->input[0].requested_gpio = -1; + s->input[0].requested_edges = 0; + } else { + s->last_error = 0; + } +out: + mutex_unlock(&s->lock); + return ret; +} + +int rp1_extts_verify(struct macb *bp, unsigned int pin, + enum ptp_pin_function func, unsigned int chan) +{ + if (!bp->rp1_extts || pin >= EXT_GPIO_COUNT) + return -EINVAL; + if (func == PTP_PF_NONE) + return 0; + if (func == PTP_PF_EXTTS) + return chan < RP1_MAX_EXTTS ? 0 : -EINVAL; + if (func == PTP_PF_PEROUT) + return chan ? -EINVAL : 0; + return -EOPNOTSUPP; +} + +int rp1_extts_clock_lock(struct macb *bp) +{ + struct rp1_extts *s = bp->rp1_extts; + int ret; + + if (!s) + return 0; + mutex_lock(&s->lock); + if (!s->enabled) + return 0; + ret = ext_stop(s); + if (ret) + mutex_unlock(&s->lock); + return ret; +} + +static int ext_resume(struct rp1_extts *s) +{ + u64 now; + int ret; + + if (s->output) { + ret = ext_now(s, &now); + if (ret) + return ret; + /* Preserve the requested grid across a time step. Output + * stays LOW during reacquisition; continuity is not promised. + */ + s->out_first = out_next_grid(now + NSEC_PER_SEC, s->out_phase); + } + return ext_start(s); +} + +int rp1_extts_clock_unlock(struct macb *bp) +{ + struct rp1_extts *s = bp->rp1_extts; + int ret = 0; + + if (!s) + return 0; + s->clock_steps++; + if (s->enabled) { + ret = ext_resume(s); + if (ret) + ext_fail(s, ret); + } + mutex_unlock(&s->lock); + return ret; +} + +void rp1_extts_rate_lock(struct macb *bp) +{ + struct rp1_extts *s = bp->rp1_extts; + + if (!s) + return; + mutex_lock(&s->lock); + s->rate_before = 0; + s->rate_irqs_held = s->enabled && s->output; + if (s->rate_irqs_held) { + /* The wrapped MACB adjfine and our completion only use arithmetic + * and MMIO. Keep host preemption out of the rate-to-FIFO interval. + */ + local_irq_save(s->rate_irq_flags); + ext_now(s, &s->rate_before); + } +} + +void rp1_extts_rate_unlock(struct macb *bp) +{ + struct rp1_extts *s = bp->rp1_extts; + u32 rate; + u64 after = 0; + int ret = 0; + + if (!s) + return; + s->rate_changes++; + if (s->enabled && s->output) { + struct rate_record *r; + + rate = out_rate(s); + ext_now(s, &after); + s->rate_count++; + r = &s->rate_records[(s->rate_count - 1) % OUT_RECORDS]; + if (r->sequence > s->out_rate_epoch) + s->rate_history_floor = r->after; + *r = (struct rate_record) { + .sequence = s->rate_count, .before = s->rate_before, .after = after, + .target = s->out_target, .pulse = s->out_count, + .old_rate = s->out_rate, .new_rate = rate, + }; + if (!rate || !s->rate_before || after < s->rate_before) { + ret = -EOPNOTSUPP; + } else if (rate != s->out_rate) { + if (s->map_valid) { + ret = out_map_advance(s, s->rate_before + + (after - s->rate_before) / 2, rate); + if (!ret) { + ret = out_send_deadline(s, &r->sent_at, &r->drained_at); + if (ret == -ETIME && !r->drained_at) { + s->out_late_skips++; + r->action = 2; + /* Keep the current edge inside the final guard. */ + ret = 0; + } else if (!ret) { + s->out_retargets++; + r->action = 1; + r->deadline_word = s->out_deadline; + } + } + } + s->out_rate = rate; + } + if (r->drained_at) + s->out_max_update_ns = max(s->out_max_update_ns, r->drained_at - r->before); + } + if (s->rate_irqs_held) { + local_irq_restore(s->rate_irq_flags); + s->rate_irqs_held = false; + } + /* Resource release and firmware calls are always outside IRQ masking. */ + if (ret) + ext_fail(s, ret); + mutex_unlock(&s->lock); +} + +static int ext_status_show(struct seq_file *m, void *unused) +{ + struct rp1_extts *s = m->private; + + mutex_lock(&s->lock); + seq_printf(m, "backend=pio-dma-tsu source=%s phc=%d pins=28 extts=%d perout=1\n", + s->output ? "perout" : "physical", + s->bp->ptp_clock ? ptp_clock_index(s->bp->ptp_clock) : -1, + RP1_MAX_EXTTS); + seq_printf(m, "enabled=%d armed=%d gpio=%d edge=%s sm=%d error=%d\n", + s->enabled, s->armed, s->gpio, + s->edge == PTP_EXTTS_EDGES ? "both" : + s->edge == PTP_FALLING_EDGE ? "falling" : "rising", + s->sm, s->last_error); + seq_printf(m, "events=%llu faults=%llu timeouts=%llu clock_steps=%llu bias_ns=%d\n", + s->events, s->faults, s->timeouts, s->clock_steps, EXT_INPUT_BIAS_NS); + seq_printf(m, "last_latch_ns=%llu collection_age_ns=%llu max_collection_age_ns=%llu\n", + s->last_stamp, s->last_age, s->max_age); + seq_printf(m, "rearm_deadtime_ns=%llu max_rearm_deadtime_ns=%llu\n", + s->last_deadtime, s->max_deadtime); + seq_puts(m, "unarmed_edges=unobservable calibration=uncalibrated nominal_pio_cycle_ns=5\n"); + seq_printf(m, "out_active=%d out_calibrating=%d out_gpio=%d out_count=%llu out_anchors=%llu rate_changes=%llu startup_origin=%d compact_origin=%d\n", + s->output && s->enabled, s->output && s->enabled && s->calibrating, + s->output ? s->gpio : -1, s->out_count, s->out_anchors, + s->rate_changes, s->out_startup_origin, s->out_compact_origin); + seq_printf(m, "out_first_ns=%llu out_next_target_ns=%llu out_width_ns=%u out_phase_ns=%u out_rate_q24=%u\n", + s->out_first, s->out_target, s->out_width, s->out_phase, s->out_rate); + seq_printf(m, "out_preparing=%d out_prepares=%llu out_prepare_stamp_ns=%llu out_prepare_age_ns=%llu out_max_prepare_age_ns=%llu\n", + s->preparing, s->out_prepares, s->out_prepare_stamp, + s->out_prepare_age, s->out_max_prepare_age); + seq_printf(m, "retarget_enabled=%d out_retargets=%llu out_late_skips=%llu out_max_update_ns=%llu\n", + true, s->out_retargets, s->out_late_skips, s->out_max_update_ns); + seq_printf(m, "out_last_latch_ns=%llu out_error_ns=%lld out_min_error_ns=%lld out_max_error_ns=%lld out_min_lead_ns=%llu EXT_OUTPUT_BIAS_NS=%d\n", + s->out_last_latch, s->out_error, s->out_min_error, s->out_max_error, + s->out_min_lead == U64_MAX ? 0 : s->out_min_lead, EXT_OUTPUT_BIAS_NS); + seq_printf(m, "out_pad_monitor=%d monitor_sm=%d monitor_origin=%d monitor_samples=%llu monitor_overruns=%llu monitor_last_counter=%llu monitor_error=%d marker_source=output-pio-transition\n", + s->monitor_started, s->monitor_sm, s->monitor_offset, + s->monitor_samples, s->monitor_overruns, s->monitor_last, + s->monitor_error); + { + unsigned int i; + + for (i = 0; i < RP1_MAX_EXTTS; i++) { + struct rp1_duplex_input *in = &s->input[i]; + + seq_printf(m, "extts%u requested=%d requested_gpio=%d requested_edges=%u prepared=%d started=%d gpio=%d sm=%d samples=%llu events=%llu overruns=%llu pending=%llu last_event_ns=%llu max_uncertainty_ns=%llu error=%d\n", + i, in->requested, in->requested_gpio, + in->requested_edges, in->prepared, in->started, + in->gpio, in->sm, + in->samples, in->events, in->overruns, + in->pending_tail - in->pending_head, + in->last_event_ns, in->max_uncertainty_ns, in->error); + } + } + seq_puts(m, "perout_period_ns=1000000000 input_gpio_must_differ_from_output=1 output_accuracy=uncalibrated-marker-latch\n"); + mutex_unlock(&s->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ext_status); + +static int out_records_show(struct seq_file *m, void *unused) +{ + struct rp1_extts *s = m->private; + u64 i, start; + + mutex_lock(&s->lock); + seq_puts(m, "sequence target_ns latch_ns marker_error_ns\n"); + start = s->out_count > OUT_RECORDS ? s->out_count - OUT_RECORDS : 0; + for (i = start; i < s->out_count; i++) { + struct out_record *r = &s->out_records[i % OUT_RECORDS]; + + seq_printf(m, "%llu %llu %llu %lld\n", r->sequence, r->target, r->stamp, r->error); + } + mutex_unlock(&s->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(out_records); + +static int rate_records_show(struct seq_file *m, void *unused) +{ + struct rp1_extts *s = m->private; + u64 i, start; + + mutex_lock(&s->lock); + seq_puts(m, "sequence before_ns after_ns target_ns pulse old_rate_q24 new_rate_q24 action sent_ns drained_ns deadline_word\n"); + start = s->rate_count > OUT_RECORDS ? s->rate_count - OUT_RECORDS : 0; + for (i = start; i < s->rate_count; i++) { + struct rate_record *r = &s->rate_records[i % OUT_RECORDS]; + + seq_printf(m, "%llu %llu %llu %llu %llu %u %u %d %llu %llu %u\n", r->sequence, + r->before, r->after, r->target, r->pulse, r->old_rate, r->new_rate, + r->action, r->sent_at, r->drained_at, r->deadline_word); + } + mutex_unlock(&s->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(rate_records); + +int rp1_extts_quiesce(struct macb *bp) +{ + struct rp1_extts *s = bp->rp1_extts; + int ret; + + if (!s) + return 0; + debugfs_remove_recursive(s->debug); + s->debug = NULL; + mutex_lock(&s->lock); + WRITE_ONCE(s->dying, true); + s->enabled = false; + ret = ext_release(s); + mutex_unlock(&s->lock); + cancel_delayed_work_sync(&s->work); + return ret; +} + +int rp1_extts_destroy(struct macb *bp) +{ + struct rp1_extts *s = bp->rp1_extts; + int ret; + + if (!s) + return 0; + ret = rp1_extts_quiesce(bp); + if (ret) { + dev_err(&bp->pdev->dev, + "RP1 PTP cleanup failed (%d); retaining active resources\n", ret); + bp->rp1_extts = NULL; + s->bp = NULL; + return ret; + } + destroy_workqueue(s->wq); + bp->rp1_extts = NULL; + kfree(s); + return 0; +} + +int rp1_extts_create(struct macb *bp) +{ + struct rp1_extts *s; + struct device_node *np; + int i; + + if (!of_device_is_compatible(bp->pdev->dev.of_node, "raspberrypi,rp1-gem") || + !bp->rp1_eth_cfg || !bp->rp1_eth_cfg_phys) + return -ENODEV; + np = of_parse_phandle(bp->pdev->dev.of_node, "raspberrypi,pio", 0); + if (!np) + return -EINVAL; + of_node_put(np); + np = of_parse_phandle(bp->pdev->dev.of_node, + "raspberrypi,gpio-controller", 0); + if (!np) + return -EINVAL; + of_node_put(np); + + s = kzalloc(sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->bp = bp; + s->sm = -1; + s->offset = -1; + s->monitor_sm = -1; + s->monitor_offset = -1; + s->out_startup_origin = -1; + s->out_compact_origin = -1; + s->control = bp->rp1_eth_cfg; + s->control_phys = bp->rp1_eth_cfg_phys; + mutex_init(&s->lock); + init_waitqueue_head(&s->monitor_wait); + INIT_DELAYED_WORK(&s->work, ext_work); + s->wq = alloc_ordered_workqueue("rp1-extts", WQ_MEM_RECLAIM); + if (!s->wq) { + kfree(s); + return -ENOMEM; + } + bp->rp1_extts = s; + for (i = 0; i < RP1_MAX_EXTTS; i++) { + s->input[i].sm = -1; + s->input[i].gpio = -1; + s->input[i].requested_gpio = -1; + atomic64_set(&s->input[i].produced, 0); + } + for (i = 0; i < EXT_GPIO_COUNT; i++) { + snprintf(s->pins[i].name, sizeof(s->pins[i].name), "GPIO%d", i); + s->pins[i].index = i; + s->pins[i].func = PTP_PF_NONE; + s->pins[i].chan = 0; + } + bp->ptp_clock_info.n_pins = EXT_GPIO_COUNT; + bp->ptp_clock_info.n_ext_ts = RP1_MAX_EXTTS; + bp->ptp_clock_info.n_per_out = 1; + bp->ptp_clock_info.supported_extts_flags = + PTP_RISING_EDGE | PTP_FALLING_EDGE | PTP_STRICT_FLAGS; + bp->ptp_clock_info.supported_perout_flags = + PTP_PEROUT_PHASE | PTP_PEROUT_DUTY_CYCLE; + bp->ptp_clock_info.pin_config = s->pins; + s->debug = debugfs_create_dir("rp1-extts", NULL); + debugfs_create_file("status", 0444, s->debug, s, &ext_status_fops); + debugfs_create_file("output_records", 0444, s->debug, s, &out_records_fops); + debugfs_create_file("rate_records", 0444, s->debug, s, &rate_records_fops); + return 0; +} diff --git a/drivers/net/ethernet/cadence/macb_rp1_ptp.h b/drivers/net/ethernet/cadence/macb_rp1_ptp.h new file mode 100644 index 00000000000000..138644068f354c --- /dev/null +++ b/drivers/net/ethernet/cadence/macb_rp1_ptp.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef MACB_RP1_PTP_H +#define MACB_RP1_PTP_H + +#include +#include + +struct macb; +struct ptp_clock_request; + +#if IS_ENABLED(CONFIG_MACB_RP1_PPS) +int rp1_extts_create(struct macb *bp); +int rp1_extts_quiesce(struct macb *bp); +int rp1_extts_destroy(struct macb *bp); +int rp1_extts_enable(struct macb *bp, struct ptp_clock_request *rq, int on); +int rp1_extts_verify(struct macb *bp, unsigned int pin, + enum ptp_pin_function func, unsigned int chan); +int rp1_extts_clock_lock(struct macb *bp); +int rp1_extts_clock_unlock(struct macb *bp); +void rp1_extts_rate_lock(struct macb *bp); +void rp1_extts_rate_unlock(struct macb *bp); +#else +static inline int rp1_extts_create(struct macb *bp) +{ + return 0; +} + +static inline int rp1_extts_quiesce(struct macb *bp) +{ + return 0; +} + +static inline int rp1_extts_destroy(struct macb *bp) +{ + return 0; +} + +static inline int rp1_extts_enable(struct macb *bp, + struct ptp_clock_request *rq, int on) +{ + return -EOPNOTSUPP; +} + +static inline int rp1_extts_verify(struct macb *bp, unsigned int pin, + enum ptp_pin_function func, unsigned int chan) +{ + return -EOPNOTSUPP; +} + +static inline int rp1_extts_clock_lock(struct macb *bp) +{ + return 0; +} + +static inline int rp1_extts_clock_unlock(struct macb *bp) +{ + return 0; +} + +static inline void rp1_extts_rate_lock(struct macb *bp) +{ +} + +static inline void rp1_extts_rate_unlock(struct macb *bp) +{ +} +#endif + +#endif diff --git a/drivers/net/ethernet/cadence/macb_rp1_ptp_counter.h b/drivers/net/ethernet/cadence/macb_rp1_ptp_counter.h new file mode 100644 index 00000000000000..a5bc8448351699 --- /dev/null +++ b/drivers/net/ethernet/cadence/macb_rp1_ptp_counter.h @@ -0,0 +1,378 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef MACB_RP1_PTP_COUNTER_H +#define MACB_RP1_PTP_COUNTER_H + +#ifdef __KERNEL__ +#include +#include +#else +#include +#include +typedef long long s64; +#endif +#include "macb_rp1_ptp_math.h" + +#define RP1_S64_MAX ((s64)(~0ULL >> 1)) +#define RP1_S64_MIN (-RP1_S64_MAX - 1) + +#define RP1_COUNTER_LOOP_CYCLES 2 +#define RP1_COUNTER_EDGE_CYCLES 4 +#define RP1_COUNTER_MAX_Y_DELTA 0x7fffffffU +#define RP1_COUNTER_MAX_RATE_CHANGES 128 +#define RP1_COUNTER_MIN_TSU_RATE (18U << 24) +#define RP1_COUNTER_MAX_TSU_RATE (23U << 24) + +/* Return the oldest valid cyclic-DMA sequence and number of unread words. */ +static inline int rp1_counter_ring_window(u64 produced, u64 consumed, + u32 capacity, u64 *first, + u32 *available) +{ + if (!capacity || !first || !available || produced < consumed) + return -EINVAL; + if (produced - consumed > capacity) { + *first = produced - capacity; + *available = capacity; + return -EOVERFLOW; + } + *first = consumed; + *available = produced - consumed; + return 0; +} + +struct rp1_counter_rate_change { + u64 before_ns; + u64 after_ns; + u32 old_rate; + u32 new_rate; +}; + +static inline int rp1_counter_signed_delta(u64 sample, u64 anchor, s64 *delta) +{ + u64 magnitude; + + if (sample >= anchor) { + magnitude = sample - anchor; + if (magnitude > (u64)RP1_S64_MAX) + return -ERANGE; + *delta = magnitude; + } else { + magnitude = anchor - sample; + if (magnitude > (u64)RP1_S64_MAX) + return -ERANGE; + *delta = -(s64)magnitude; + } + + return 0; +} + +/* + * Convert a bounded interval between two synchronously started counter SMs + * into PIO cycles. Callers must select an anchor no more than half a 32-bit + * Y-counter span before the sample. The sequence correction accounts for + * four PIO instructions spent capturing each edge; the wrap correction + * accounts for the extra instruction on each Y wrap. This is only a PIO-cycle + * delta. Mapping cycles to PHC nanoseconds and electrical bias are separate. + */ +static inline int rp1_counter_cycle_delta(u32 anchor_y, u64 anchor_sequence, + u64 anchor_wraps, u32 sample_y, + u64 sample_sequence, u64 sample_wraps, + u64 *cycles) +{ + u32 y_delta = anchor_y - sample_y; + s64 sequence_delta, wrap_delta, edge_cycles, total; + int ret; + + if (!cycles || y_delta > RP1_COUNTER_MAX_Y_DELTA) + return -ERANGE; + ret = rp1_counter_signed_delta(sample_sequence, anchor_sequence, + &sequence_delta); + if (ret) + return ret; + ret = rp1_counter_signed_delta(sample_wraps, anchor_wraps, &wrap_delta); + if (ret) + return ret; + if (sequence_delta > RP1_S64_MAX / RP1_COUNTER_EDGE_CYCLES || + sequence_delta < RP1_S64_MIN / RP1_COUNTER_EDGE_CYCLES) + return -ERANGE; + edge_cycles = sequence_delta * RP1_COUNTER_EDGE_CYCLES; + total = (s64)y_delta * RP1_COUNTER_LOOP_CYCLES; + if ((edge_cycles > 0 && total > RP1_S64_MAX - edge_cycles) || + (edge_cycles < 0 && total < RP1_S64_MIN - edge_cycles)) + return -ERANGE; + total += edge_cycles; + if ((wrap_delta > 0 && total > RP1_S64_MAX - wrap_delta) || + (wrap_delta < 0 && total < RP1_S64_MIN - wrap_delta)) + return -ERANGE; + total += wrap_delta; + if (total < 0) + return -ERANGE; + *cycles = total; + return 0; +} + +/* + * Return signed cycles from an anchor sample to another lane's sample. + * Positive means the sample followed the anchor; negative means it preceded + * it. Trying both orders handles an event that lands within the four-cycle + * capture path of an anchor edge while retaining the half-range stale check. + */ +static inline int rp1_counter_signed_cycle_delta(u32 anchor_y, + u64 anchor_sequence, + u64 anchor_wraps, + u32 sample_y, + u64 sample_sequence, + u64 sample_wraps, + s64 *cycles) +{ + u64 magnitude; + int first, second; + + if (!cycles) + return -EINVAL; + first = rp1_counter_cycle_delta(anchor_y, anchor_sequence, anchor_wraps, + sample_y, sample_sequence, sample_wraps, + &magnitude); + if (!first) { + if (magnitude > (u64)RP1_S64_MAX) + return -ERANGE; + *cycles = magnitude; + return 0; + } + second = rp1_counter_cycle_delta(sample_y, sample_sequence, sample_wraps, + anchor_y, anchor_sequence, anchor_wraps, + &magnitude); + if (!second) { + if (magnitude > (u64)RP1_S64_MAX) + return -ERANGE; + *cycles = -(s64)magnitude; + return 0; + } + return first == -ERANGE ? second : first; +} + +/* + * Map a cycle delta ending at anchor_ns back into the PHC. Rate-change times + * are represented by the midpoint of the driver's before/after PHC reads; + * uncertainty accumulates the full bracket width for every crossed change. + * oldest_ns is the PHC history floor. Events older than it are rejected. + */ +static inline int rp1_counter_cycles_to_phc(u64 anchor_ns, u32 anchor_rate, + u64 cycles, u64 oldest_ns, + const struct rp1_counter_rate_change *changes, + unsigned int change_count, u64 *event_ns, + u64 *uncertainty_ns) +{ + u64 current_ns = anchor_ns; + u64 remaining_fp, uncertainty = 0; + u32 rate = anchor_rate; + unsigned int i; + + if (!event_ns || !uncertainty_ns || + anchor_rate < RP1_COUNTER_MIN_TSU_RATE || + anchor_rate >= RP1_COUNTER_MAX_TSU_RATE || + anchor_ns < oldest_ns || cycles > (~0ULL >> 16) || + (change_count && !changes)) + return -ERANGE; + if (change_count > RP1_COUNTER_MAX_RATE_CHANGES) + return -E2BIG; + + for (i = 0; i < change_count; i++) { + const struct rp1_counter_rate_change *change = &changes[i]; + + if (change->old_rate < RP1_COUNTER_MIN_TSU_RATE || + change->old_rate >= RP1_COUNTER_MAX_TSU_RATE || + change->new_rate < RP1_COUNTER_MIN_TSU_RATE || + change->new_rate >= RP1_COUNTER_MAX_TSU_RATE || + change->before_ns > change->after_ns || + change->after_ns > anchor_ns || + (i && change->before_ns < changes[i - 1].after_ns)) + return -EINVAL; + if (i && change->old_rate != changes[i - 1].new_rate) + return -ESTALE; + } + if (change_count && changes[change_count - 1].new_rate != anchor_rate) + return -ESTALE; + + remaining_fp = cycles << 16; + for (i = change_count; i > 0; i--) { + const struct rp1_counter_rate_change *change = &changes[i - 1]; + u64 midpoint = change->before_ns + + (change->after_ns - change->before_ns) / 2; + u64 segment_fp, delta_ns; + + if (midpoint > current_ns) + return -ESTALE; + if (midpoint == current_ns) { + rate = change->old_rate; + current_ns = midpoint; + continue; + } + segment_fp = out_ns_cycles_fp(current_ns - midpoint, rate); + if (remaining_fp <= segment_fp) { + delta_ns = out_cycles_fp_ns(remaining_fp, rate); + if (delta_ns > current_ns - oldest_ns) + return -ESTALE; + *event_ns = current_ns - delta_ns; + if (change->old_rate != change->new_rate && + *event_ns >= change->before_ns && + *event_ns <= change->after_ns) { + u64 width = change->after_ns - change->before_ns; + + if (uncertainty > ~0ULL - width) + return -ERANGE; + uncertainty += width; + } + *uncertainty_ns = uncertainty; + return 0; + } + + remaining_fp -= segment_fp; + if (change->old_rate != change->new_rate) { + u64 width = change->after_ns - change->before_ns; + + if (uncertainty > ~0ULL - width) + return -ERANGE; + uncertainty += width; + } + current_ns = midpoint; + rate = change->old_rate; + } + + { + u64 delta_ns = out_cycles_fp_ns(remaining_fp, rate); + + if (delta_ns > current_ns - oldest_ns) + return -ESTALE; + *event_ns = current_ns - delta_ns; + } + *uncertainty_ns = uncertainty; + return 0; +} + +/* + * Map a cycle delta forward from a PHC anchor. This is used for an input edge + * newer than the latest output-marker anchor, so it can be reported without + * waiting for the following PPS marker. Rate-change records must begin at or + * after anchor_ns and be ordered. The caller supplies the retained PHC window + * so a delayed DMA sample cannot silently extrapolate beyond it. + */ +static inline int rp1_counter_cycles_to_phc_forward(u64 anchor_ns, + u32 anchor_rate, + u64 cycles, + u64 oldest_ns, + u64 latest_ns, + const struct rp1_counter_rate_change *changes, + unsigned int change_count, + u64 *event_ns, + u64 *uncertainty_ns) +{ + u64 current_ns = anchor_ns; + u64 remaining_fp, uncertainty = 0; + u32 rate = anchor_rate; + unsigned int i; + + if (!event_ns || !uncertainty_ns || + anchor_rate < RP1_COUNTER_MIN_TSU_RATE || + anchor_rate >= RP1_COUNTER_MAX_TSU_RATE || + anchor_ns < oldest_ns || anchor_ns > latest_ns || + cycles > (~0ULL >> 16) || (change_count && !changes)) + return -ERANGE; + if (change_count > RP1_COUNTER_MAX_RATE_CHANGES) + return -E2BIG; + remaining_fp = cycles << 16; + for (i = 0; i < change_count; i++) { + const struct rp1_counter_rate_change *change = &changes[i]; + u64 midpoint, segment_fp, delta_ns; + + if (change->old_rate < RP1_COUNTER_MIN_TSU_RATE || + change->old_rate >= RP1_COUNTER_MAX_TSU_RATE || + change->new_rate < RP1_COUNTER_MIN_TSU_RATE || + change->new_rate >= RP1_COUNTER_MAX_TSU_RATE || + change->before_ns < anchor_ns || + change->before_ns > change->after_ns || + change->after_ns > latest_ns || + (i && change->before_ns < changes[i - 1].after_ns)) + return -EINVAL; + if (change->old_rate != rate) + return -ESTALE; + midpoint = change->before_ns + + (change->after_ns - change->before_ns) / 2; + segment_fp = out_ns_cycles_fp(midpoint - current_ns, rate); + if (remaining_fp <= segment_fp) { + delta_ns = out_cycles_fp_ns(remaining_fp, rate); + if (delta_ns > latest_ns - current_ns) + return -ESTALE; + *event_ns = current_ns + delta_ns; + if (change->old_rate != change->new_rate && + *event_ns >= change->before_ns && + *event_ns <= change->after_ns) { + u64 width = change->after_ns - change->before_ns; + + if (uncertainty > ~0ULL - width) + return -ERANGE; + uncertainty += width; + } + *uncertainty_ns = uncertainty; + return 0; + } + remaining_fp -= segment_fp; + if (change->old_rate != change->new_rate) { + u64 width = change->after_ns - change->before_ns; + + if (uncertainty > ~0ULL - width) + return -ERANGE; + uncertainty += width; + } + current_ns = midpoint; + rate = change->new_rate; + } + { + u64 delta_ns = out_cycles_fp_ns(remaining_fp, rate); + + if (delta_ns > latest_ns - current_ns) + return -ESTALE; + *event_ns = current_ns + delta_ns; + } + *uncertainty_ns = uncertainty; + return 0; +} + +/* Map one counter-lane sample relative to a PHC-stamped counter anchor. */ +static inline int rp1_counter_sample_to_phc(u64 anchor_ns, + u32 anchor_rate, u32 anchor_y, + u64 anchor_sequence, + u64 anchor_wraps, + u32 sample_y, + u64 sample_sequence, + u64 sample_wraps, + u64 oldest_ns, + u64 latest_ns, + const struct rp1_counter_rate_change *changes, + unsigned int change_count, + u64 *event_ns, + u64 *uncertainty_ns) +{ + s64 cycle_delta; + int ret; + + if (anchor_ns > latest_ns) + return -ERANGE; + ret = rp1_counter_signed_cycle_delta(anchor_y, anchor_sequence, + anchor_wraps, sample_y, + sample_sequence, sample_wraps, + &cycle_delta); + if (ret) + return ret; + if (cycle_delta >= 0) + return rp1_counter_cycles_to_phc_forward(anchor_ns, anchor_rate, + cycle_delta, oldest_ns, + latest_ns, changes, + change_count, event_ns, + uncertainty_ns); + return rp1_counter_cycles_to_phc(anchor_ns, anchor_rate, + (u64)-cycle_delta, oldest_ns, + changes, change_count, event_ns, + uncertainty_ns); +} + +#endif diff --git a/drivers/net/ethernet/cadence/macb_rp1_ptp_math.h b/drivers/net/ethernet/cadence/macb_rp1_ptp_math.h new file mode 100644 index 00000000000000..c2c1d7d47a121f --- /dev/null +++ b/drivers/net/ethernet/cadence/macb_rp1_ptp_math.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef RP1_PEROUT_MATH_H +#define RP1_PEROUT_MATH_H +#ifdef __KERNEL__ +#include +#define OUT_RATE_DEN BIT_ULL(26) +#else +typedef unsigned long long u64; +typedef unsigned int u32; +#define div64_u64(n, d) ((u64)(n) / (u64)(d)) +#define OUT_RATE_DEN 67108864ULL +#endif + +/* 50 MHz TSU increment in Q24 divided by four 200 MHz PIO cycles. */ +#define OUT_SECOND 1000000000ULL +#define OUT_GUARD_CYCLES 60000000U +#define OUT_COMPACT_CHECKPOINT_CYCLES 5U +#define OUT_PREP_LEAD_NS 20000000ULL +#define OUT_UPDATE_GUARD_NS 100000ULL +#define OUT_COUNTER_SEED 0xffffffffU +#define OUT_FINE_OFFSET_CYCLES 10U +#define OUT_FINE_TICK_CYCLES 4U +#define OUT_MIN_LEAD_NS 800000000ULL +static inline u64 out_cycles_ns(u64 cycles, u32 rate) +{ + return div64_u64(cycles * rate + OUT_RATE_DEN / 2, OUT_RATE_DEN); +} + +static inline u64 out_ns_cycles(u64 ns, u32 rate) +{ + return div64_u64(ns * OUT_RATE_DEN + rate / 2, rate); +} + +/* Q16 cycle accumulation preserves sub-cycle phase across rate changes. */ +static inline u64 out_ns_cycles_fp(u64 ns, u32 rate) +{ +#ifdef __KERNEL__ + return mul_u64_u64_div_u64(ns, OUT_RATE_DEN << 16, rate); +#else + return (u64)(((__uint128_t)ns * (OUT_RATE_DEN << 16)) / rate); +#endif +} + +/* Inverse mapping for a Q16 PIO-cycle delta, rounded down to PHC ns. */ +static inline u64 out_cycles_fp_ns(u64 cycles_fp, u32 rate) +{ +#ifdef __KERNEL__ + return mul_u64_u64_div_u64(cycles_fp, rate, OUT_RATE_DEN << 16); +#else + return (u64)(((__uint128_t)cycles_fp * rate) / (OUT_RATE_DEN << 16)); +#endif +} + +static inline u32 out_deadline_word(u64 cycles_fp) +{ + u64 fine_fp = cycles_fp - ((u64)OUT_FINE_OFFSET_CYCLES << 16); + u64 ticks = (fine_fp + ((u64)OUT_FINE_TICK_CYCLES << 15)) / + ((u64)OUT_FINE_TICK_CYCLES << 16); + return OUT_COUNTER_SEED - ticks; +} + +static inline u64 out_next_grid(u64 earliest, u32 phase) +{ + u64 next = div64_u64(earliest, OUT_SECOND) * OUT_SECOND + phase; + + return next < earliest ? next + OUT_SECOND : next; +} +#endif From ba244d3d9efb1f4b398245ee9659023e3542c517 Mon Sep 17 00:00:00 2001 From: ahmadexp Date: Wed, 30 Sep 2026 23:16:05 -0700 Subject: [PATCH 4/4] net: macb: document RP1 PIO programs --- .../ethernet/cadence/macb-rp1-ptp.rst | 4 +- drivers/net/ethernet/cadence/macb_rp1_ptp.c | 98 +++++++++++++++++-- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst b/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst index c51126067a46b9..3dad957b113950 100644 --- a/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst +++ b/Documentation/networking/device_drivers/ethernet/cadence/macb-rp1-ptp.rst @@ -10,8 +10,8 @@ hardware clock (PHC) through the standard Linux PTP device interface. With external timestamp input and periodic output. Applications use the standard PTP clock ioctls; no private userspace interface is added. -The driver advertises GPIO0 through GPIO27 as PTP pins. Select a BCM/RP1 GPIO -number, not a 40-pin header position. Pin reservation is checked when a +The driver advertises GPIO0 through GPIO27 as PTP pins. Select an RP1 GPIO +number, not a 40-pin header pin number. Pin reservation is checked when a channel is enabled. A pin in use by another peripheral or GPIO interrupt is rejected. Board wiring and other pin users can make a listed pin unavailable. This PHC interface is separate from the ``pps-rp1`` overlay, which configures diff --git a/drivers/net/ethernet/cadence/macb_rp1_ptp.c b/drivers/net/ethernet/cadence/macb_rp1_ptp.c index 7f6709d7d2ca68..314a11f9d86ff9 100644 --- a/drivers/net/ethernet/cadence/macb_rp1_ptp.c +++ b/drivers/net/ethernet/cadence/macb_rp1_ptp.c @@ -42,22 +42,100 @@ #define RP1_COUNTER_PROGRAM_WORDS 10 #define RP1_COUNTER_RING_WORDS 256 -/* The compact output program generates both the pad waveform and TSU markers. - * Do not add an input-direction monitor SM on this same GPIO: its direction - * request removes the pad output enable on RP1. +/* pioasm source: + * + * .program rp1_output + * .side_set 1 + * .wrap_target + * pull block side 0 + * mov x, osr side 0 + * jmp x--, 2 side 0 + * pull block side 0 + * mov isr, null side 0 + * push block side 0 + * mov isr, osr side 0 + * mov y, ~null side 0 + * mov x, null side 0 + * pull noblock side 0 + * mov x, osr side 0 + * jmp x != y, 20 side 0 + * jmp !x, 20 side 0 + * mov y, isr side 0 + * mov isr, null side 0 + * push noblock side 1 + * mov x, y side 1 + * jmp x--, 17 side 1 + * nop side 0 + * jmp 0 side 0 + * jmp y--, 9 side 0 + * jmp 9 side 0 + * .wrap + * + * The words and instruction comments below are pioasm output. The program + * generates the pad waveform and TSU markers. Do not add an input-direction + * monitor SM on this same GPIO: its direction request removes the pad output + * enable on RP1. */ static const u16 rp1_compact_output_program[RP1_OUTPUT_PROGRAM_WORDS] = { - 0x80a0, 0xa027, 0x0042, 0x80a0, 0xa0c3, 0x8020, 0xa0c7, 0xa04b, - 0xa023, 0x8080, 0xa027, 0x00b4, 0x0034, 0xa046, 0xa0c3, 0x9000, - 0xb022, 0x1051, 0xa042, 0x0000, 0x0089, 0x0009, + 0x80a0, /* 0: pull block side 0 */ + 0xa027, /* 1: mov x, osr side 0 */ + 0x0042, /* 2: jmp x--, 2 side 0 */ + 0x80a0, /* 3: pull block side 0 */ + 0xa0c3, /* 4: mov isr, null side 0 */ + 0x8020, /* 5: push block side 0 */ + 0xa0c7, /* 6: mov isr, osr side 0 */ + 0xa04b, /* 7: mov y, ~null side 0 */ + 0xa023, /* 8: mov x, null side 0 */ + 0x8080, /* 9: pull noblock side 0 */ + 0xa027, /* 10: mov x, osr side 0 */ + 0x00b4, /* 11: jmp x != y, 20 side 0 */ + 0x0034, /* 12: jmp !x, 20 side 0 */ + 0xa046, /* 13: mov y, isr side 0 */ + 0xa0c3, /* 14: mov isr, null side 0 */ + 0x9000, /* 15: push noblock side 1 */ + 0xb022, /* 16: mov x, y side 1 */ + 0x1051, /* 17: jmp x--, 17 side 1 */ + 0xa042, /* 18: nop side 0 */ + 0x0000, /* 19: jmp 0 side 0 */ + 0x0089, /* 20: jmp y--, 9 side 0 */ + 0x0009, /* 21: jmp 9 side 0 */ }; -/* Rising-edge counter shared by the output monitor and external-input lanes. - * PIO_ADD_PROGRAM relocates its JMP addresses. +/* pioasm source for the rising-edge counter shared by the output monitor and + * external-input lanes: + * + * .program rp1_counter + * .wrap_target + * start: + * jmp pin, capture + * jmp y--, start + * jmp start + * high_test: + * jmp pin, high_loop + * jmp y--, start + * high_loop: + * jmp y--, high_loop + * jmp high_loop + * capture: + * mov isr, y + * push noblock + * jmp high_test + * .wrap + * + * The words and instruction comments below are pioasm output. PIO_ADD_PROGRAM + * relocates the absolute JMP addresses when it loads this program. */ static const u16 rp1_counter_program[RP1_COUNTER_PROGRAM_WORDS] = { - 0x00c7, 0x0080, 0x0000, 0x00c5, 0x0080, - 0x0083, 0x0003, 0xa0c2, 0x8000, 0x0003, + 0x00c7, /* 0: jmp pin, 7 */ + 0x0080, /* 1: jmp y--, 0 */ + 0x0000, /* 2: jmp 0 */ + 0x00c5, /* 3: jmp pin, 5 */ + 0x0080, /* 4: jmp y--, 0 */ + 0x0085, /* 5: jmp y--, 5 */ + 0x0005, /* 6: jmp 5 */ + 0xa0c2, /* 7: mov isr, y */ + 0x8000, /* 8: push noblock */ + 0x0003, /* 9: jmp 3 */ }; struct out_record { u64 sequence, target, stamp; s64 error; };