Add Allocate Endpoint ID support for MCTP Bridge

* New mctpd method to send Allocate Endpoint ID mctp control message
* Timer for MCTP Bridge to settle ( observed it takes sometime for
Bridges like FPGA to allocate downstream eids to endpoints during
which any request on the downstream endpoint fails)
* update AssignEndpoint and AssignEndpointStatic method to consider
Endpoint nature (MCTP bridge based on pool size received)
* Check for presence of contiguous available EIDs of pool size.
* Pytest framework change

> Introduced new Endpoint of MCTP Bridge type (pool size =2)
> New test method to validate endpoint_allocate_eid()
> update existing pytest method to accomodate for new endpoint
  in present network

'''
System contains MCTP Bridge (FPGA) over USB which has
downstream endpoints such as EROTS, IROTS.

MCTP Bridge is one bus usb1-1 populating as mctpusb0 interface.

USBMON :
==================

cat /sys/kernel/debug/usb/usbmon/1u
866e6900 886711482 S Bo:1:002:1 -115 13 = 1ab4000d 010008c8 00800100 0c  -> SET EID 0xc
866e6900 886711719 C Bo:1:002:1 0 13 >
82a05f00 886711737 C Bi:1:002:2 0 15 = 1ab4000f 01080cc0 00000100 120c0f  -> Bridge size 0xf
82a05f00 886711753 S Bi:1:002:2 -115 512 <
866e6200 886712339 S Bo:1:002:1 -115 11 = 1ab4000b 010c08c8 008105  -> GET MSSG TYPE (eid : 0xc)
866e6200 886712449 C Bo:1:002:1 0 11 >
82a05f00 886712461 C Bi:1:002:2 0 14 = 1ab4000e 01080cc0 00010500 017e
82a05f00 886712474 S Bi:1:002:2 -115 512 <
866e6900 886712862 S Bo:1:002:1 -115 11 = 1ab4000b 010c08c8 008203  -> GET UUID (eid : 0xc)
866e6900 886712961 C Bo:1:002:1 0 11 >
82a05f00 886712975 C Bi:1:002:2 0 28 = 1ab4001c 01080cc0 00020300 xxxx xxxx xxxx xxxx
82a05f00 886712993 S Bi:1:002:2 -115 512 <
866e6200 886714375 S Bo:1:002:1 -115 14 = 1ab4000e 010c08c8 00830800 0f0d  -> ALLOCATE EID 0xc, bridge size 0xf, start eid 0xd
866e6200 886714455 C Bo:1:002:1 0 14 >
82a05f00 886714475 C Bi:1:002:2 0 15 = 1ab4000f 01080cc0 00030800 010f0d
82a05f00 886714490 S Bi:1:002:2 -115 512 <
85e6ab80 890712851 S Bo:1:002:1 -115 11 = 1ab4000b 010d08c8 008405 -> GET MSSG TYPE (eid : 0xd)
85e6ab80 890713119 C Bo:1:002:1 0 11 >
82a05f00 890713452 C Bi:1:002:2 0 17 = 1ab40011 01080dc0 00040500 0401057e 7f
82a05f00 890713487 S Bi:1:002:2 -115 512 <
85e6a780 890713984 S Bo:1:002:1 -115 11 = 1ab4000b 010d08c8 008503 -> GET UUID (eid : 0xd)
85e6a780 890714073 C Bo:1:002:1 0 11 >
82a05f00 890714451 C Bi:1:002:2 0 28 = 1ab4001c 01080dc0 00050300 xxxx xxxx xxxx xxxx
82a05f00 890714496 S Bi:1:002:2 -115 512 <
85e6a180 890718152 S Bo:1:002:1 -115 11 = 1ab4000b 010e08c8 008605 -> GET MSSG TYPE (eid : 0xe) (Timedout)
85e6a180 890718331 C Bo:1:002:1 0 11 >
86654980 891215635 S Bo:1:002:1 -115 11 = 1ab4000b 010e08c8 008703 -> GET UUID (eid : 0xe)  (Timedout)
86654980 891215839 C Bo:1:002:1 0 11 >
86654e00 891712957 S Bo:1:002:1 -115 11 = 1ab4000b 010f08c8 008805 -> GET MSSG TYPE (eid : 0xf)
86654e00 891713082 C Bo:1:002:1 0 11 >
82a05f00 891713576 C Bi:1:002:2 0 17 = 1ab40011 01080fc0 00080500 0401057e 7f
82a05f00 891713615 S Bi:1:002:2 -115 512 <
85e6a400 891715076 S Bo:1:002:1 -115 11 = 1ab4000b 010f08c8 008903
85e6a400 891715199 C Bo:1:002:1 0 11 >

MCTPD Traces
=======================
busctl call au.com.codeconstruct.MCTP1 /au/com/codeconstruct/mctp1/interfaces/mctpusb0 au.com.codeconstruct.MCTP.BusOwner1 AssignEndpoint ay 0
yisb 12 1 "/au/com/codeconstruct/mctp1/networks/1/endpoints/12" true

mctpd: read_message got from sockaddr_mctp_ext eid 12 net 1 type 0x00 if 3 hw len 0 0x len 6
mctpd: physaddr if 3 hw len 0 0x requested allocation of pool size = 15
mctpd: emit_endpoint_added: /au/com/codeconstruct/mctp1/networks/1/endpoints/12
endpoint_allocate_eid Asking for contiguous EIDs for pool with start eid : 13
mctpd: endpoint_send_allocate_endpoint_id Allocation was rejected as it was Allocated by other bus
endpoint_send_allocate_endpoint_id Allocated size of 15, starting from EID 13

Bridge Time expired, set pool eids
mctpd: emit_endpoint_added: /au/com/codeconstruct/mctp1/networks/1/endpoints/13
mctpd: Error getting endpoint types for peer eid 14 net 1 phys physaddr if 3 hw len 0 0x state 1. Ignoring error -110 Connection timed out
mctpd: Error getting UUID for peer eid 14 net 1 phys physaddr if 3 hw len 0 0x state 1. Ignoring error -110 Connection timed out

mctpd: emit_endpoint_added: /au/com/codeconstruct/mctp1/networks/1/endpoints/14
mctpd: emit_endpoint_added: /au/com/codeconstruct/mctp1/networks/1/endpoints/15
===============================================================================

mctp addr show
eid 8 net 1 dev mctpusb0

busctl tree au.com.codeconstruct.MCTP1
`- /au
  `- /au/com
    `- /au/com/codeconstruct
      `- /au/com/codeconstruct/mctp1
        |- /au/com/codeconstruct/mctp1/interfaces
        | |- /au/com/codeconstruct/mctp1/interfaces/lo
\        | `- /au/com/codeconstruct/mctp1/interfaces/mctpusb0
        `- /au/com/codeconstruct/mctp1/networks
          `- /au/com/codeconstruct/mctp1/networks/1
            `- /au/com/codeconstruct/mctp1/networks/1/endpoints
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/12
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/13
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/14
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/15
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/16
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/17
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/18
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/19
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/20
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/21
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/22
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/23
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/24
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/25
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/26
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/27
              |- /au/com/codeconstruct/mctp1/networks/1/endpoints/8

==========================================================================
PYTEST

ninja -C obj test
ninja: Entering directory `obj'
[0/1] Running all tests.
1/1 test-mctpd        OK             21.17s   20 subtests passed

Ok:                 1
Expected Fail:      0
Fail:               0
Unexpected Pass:    0
Skipped:            0
Timeout:            0

'''

Signed-off-by: Faizan Ali <[email protected]>

Author: faizana-nvidia

src/mctpd.c

@@ -53,6 +53,7 @@
 #define CC_MCTP_DBUS_IFACE_INTERFACE "au.com.codeconstruct.MCTP.Interface1"
 #define CC_MCTP_DBUS_NETWORK_INTERFACE "au.com.codeconstruct.MCTP.Network1"
 
+#define BRIDGE_SETTLE_DELAY_SEC 4
 // an arbitrary constant for use with sd_id128_get_machine_app_specific()
 static const char* mctpd_appid = "67369c05-4b97-4b7e-be72-65cfd8639f10";
 
@@ -243,6 +244,7 @@ static int add_local_eid(ctx *ctx, int net, int eid);
 static int del_local_eid(ctx *ctx, int net, int eid);
 static int add_net(ctx *ctx, int net);
 static int add_interface(ctx *ctx, int ifindex);
+static int endpoint_allocate_eid(peer *peer);
 
 mctp_eid_t local_addr(const ctx *ctx, int ifindex) {
 	mctp_eid_t *eids, ret = 0;
@@ -2107,6 +2109,16 @@ static int method_assign_endpoint(sd_bus_message *call, void *data, sd_bus_error
 		goto err;
 	dfree(peer_path);
 
+	/* MCTP Bridge Support */
+	/* Allocate Endopint EID while looking for pool start eid
+	 * Update pool endpoints to dbus */
+
+	 if(peer->pool_size > 0) {
+		/* new start eid will be assigned before MCTP Allocate eid control command */
+		peer->pool_start = peer->eid + 1;
+		rc = endpoint_allocate_eid(peer);
+	}
+
 	return sd_bus_reply_method_return(call, "yisb",
 		peer->eid, peer->net, peer_path, 1);
 err:
@@ -2185,8 +2197,11 @@ static int method_assign_endpoint_static(sd_bus_message *call, void *data,
 	dfree(peer_path);
 
 	/* MCTP Bridge Support */
+	/* Allocate Endopint EID
+	 * Update pool endpoints to dbus */
 	if(peer->pool_size > 0) {
 		peer->pool_start = start_eid;
+		rc = endpoint_allocate_eid(peer);
 	}
 
 	return sd_bus_reply_method_return(call, "yisb",
@@ -4494,6 +4509,160 @@ static void setup_config_defaults(ctx *ctx)
 	ctx->default_role = ENDPOINT_ROLE_BUS_OWNER;
 }
 
+static int endpoint_send_allocate_endpoint_id(peer *peer,
+	mctp_eid_t eid_start, uint8_t eid_pool_size, mctp_ctrl_cmd_alloc_eid_op oper,
+	uint8_t *allocated_pool_size, mctp_eid_t *allocated_pool_start)
+{
+	struct sockaddr_mctp_ext addr;
+	struct mctp_ctrl_cmd_alloc_eid req = {0};
+	struct mctp_ctrl_resp_alloc_eid *resp = NULL;
+	uint8_t *buf = NULL;
+	size_t buf_size;
+	uint8_t iid, stat;
+	int rc;
+
+	iid = mctp_next_iid(peer->ctx);
+	req.ctrl_hdr.rq_dgram_inst = RQDI_REQ | iid;
+	req.ctrl_hdr.command_code = MCTP_CTRL_CMD_ALLOCATE_ENDPOINT_IDS;
+	req.alloc_eid_op = alloc_eid;
+	req.pool_size = eid_pool_size;
+	req.start_eid = eid_start;
+	rc = endpoint_query_peer(peer, MCTP_CTRL_HDR_MSG_TYPE, &req, sizeof(req),
+							&buf, &buf_size, &addr);
+	if (rc < 0)
+		goto out;
+
+	rc = mctp_ctrl_validate_response(buf, buf_size, sizeof(*resp),
+									peer_tostr_short(peer), iid,
+									MCTP_CTRL_CMD_ALLOCATE_ENDPOINT_IDS);
+
+	if (rc)
+		goto out;
+
+	resp = (void *)buf;
+	if (!resp) {
+		warnx("%s Invalid response Buffer\n", __func__);
+		return -ENOMEM;
+	}
+
+	stat = resp->status & 0x03;
+	if (stat == 0x00) {
+		if(peer->ctx->verbose) {
+			fprintf(stderr, "%s Allocation Accepted \n", __func__);
+		}
+	}
+	else if (stat == 0x1) {
+		warnx("%s Allocation was rejected as it was Allocated by other bus \n", __func__);
+	}
+
+	*allocated_pool_size = resp->eid_pool_size;
+	*allocated_pool_start = resp->eid_set;
+	if(peer->ctx->verbose) {
+		fprintf(stderr, "%s Allocated size of %d, starting from EID %d\n", __func__,
+				resp->eid_pool_size, resp->eid_set);
+	}
+
+	return 0;
+out:
+	free(buf);
+	return rc;
+}
+static int cb_populate_pool_eids(sd_event_source *s, uint64_t t, void* data)
+{
+	peer *peer = data;
+	int net = peer->net;
+	struct peer *allocated_peer = NULL;
+	dest_phys dest = peer->phys;
+	int rc;
+
+	fprintf (stderr, "Bridge Time expired, set pool eids\n");
+	for (mctp_eid_t eid = peer->pool_start;
+		eid < peer->pool_start + peer->pool_size; eid++) {
+		rc = add_peer(peer->ctx, &dest, eid, net, &allocated_peer);
+		if (rc < 0) {
+			warnx("%s failed to find peer for allocated eid %d\n",
+					__func__, eid);
+			continue;
+		}
+
+		rc = setup_added_peer(allocated_peer);
+		if (rc < 0) {
+			warnx("%s failed to setup peer for allocated eid %d\n",
+					__func__, eid);
+			continue;
+		}
+	}
+
+	return 0;
+}
+
+static mctp_eid_t get_pool_start (peer *peer, mctp_eid_t eid_start, uint8_t pool_size)
+{
+	uint8_t count = 0;
+	mctp_eid_t pool_start = eid_alloc_max;
+	net_det *n = lookup_net(peer->ctx, peer->net);
+
+	if (!n) {
+		warnx("BUG: Unknown net %d : failed to get pool start\n", peer->net);
+		return eid_alloc_max;
+	}
+
+	for (mctp_eid_t e = eid_start; e <= eid_alloc_max; e++) {
+		if (n->peeridx[e] == -1) {
+			if(pool_start == eid_alloc_max) {
+				pool_start = e;
+			}
+			count++;
+			if (count == pool_size) return pool_start;
+		} else {
+			pool_start = eid_alloc_max;
+			count = 0;
+		}
+	}
+
+	return eid_alloc_max;
+}
+
+static int endpoint_allocate_eid(peer* peer)
+{
+	uint8_t allocated_pool_size = 0;
+	mctp_eid_t allocated_pool_start = 0;
+
+	/* Find pool sized contiguous unused eids to allocate on the bridge. */
+	peer->pool_start = get_pool_start(peer, peer->pool_start, peer->pool_size);
+	if(peer->pool_start == eid_alloc_max) {
+		warnx("%s failed to find contiguous EIDs of required size", __func__);
+		return 0;
+	} else {
+		if(peer->ctx->verbose)
+			fprintf(stderr, "%s Asking for contiguous EIDs for pool with start eid : %d\n", __func__, peer->pool_start);
+	}
+
+	int rc = endpoint_send_allocate_endpoint_id(peer, peer->pool_start, peer->pool_size,
+				alloc_eid, &allocated_pool_size, &allocated_pool_start);
+	if (rc) {
+		warnx("%s failed to allocate endpoints, returned %s %d\n",
+				__func__, strerror(-rc), rc);
+	} else {
+		peer->pool_size = allocated_pool_size;
+		peer->pool_start = allocated_pool_start;
+
+		//delay for settling MCTP Bridge after allocation of downstream eid
+		uint64_t now_usec, timer_usec;
+		// Get the current time in microseconds
+		if (sd_event_now(peer->ctx->event, CLOCK_MONOTONIC, &now_usec) < 0) {
+			warnx("Failed to get current time");
+			return -1;
+		}
+		timer_usec = now_usec + BRIDGE_SETTLE_DELAY_SEC * 1000000ULL;
+		rc = sd_event_add_time(peer->ctx->event, NULL,
+				CLOCK_MONOTONIC, timer_usec, 0,
+				cb_populate_pool_eids, peer);
+	}
+
+	return 0;
+}
+
 int main(int argc, char **argv)
 {
 	int rc;

tests/conftest.py

@@ -314,6 +314,12 @@ async def handle_mctp_control(self, sock, addr, data):
                 data = bytes(hdr + [0x00, len(types)] + types)
                 await sock.send(raddr, data)
 
+            elif opcode == 8:
+                # Allocate Enpoint IDs
+                (_, pool_st) = data[3:]
+                data = bytes(hdr + [0x00, 0x00, self.pool_size, pool_st])
+                await sock.send(raddr, data)
+
             else:
                 await sock.send(raddr, bytes(hdr + [0x05])) # unsupported command
 
@@ -348,9 +354,9 @@ def __init__(self):
     def add_endpoint(self, endpoint):
         self.endpoints.append(endpoint)
 
-    def lookup_endpoint(self, iface, lladdr):
+    def lookup_endpoint(self, iface, lladdr, eid):
         for ep in self.endpoints:
-            if ep.iface == iface and ep.lladdr == lladdr:
+            if ep.iface == iface and ep.lladdr == lladdr and (eid == ep.eid or not eid):
                 return ep
         return None
 
@@ -570,7 +576,7 @@ async def handle_send(self, addr, data):
         if phys is None:
             return
 
-        ep = self.network.lookup_endpoint(*phys)
+        ep = self.network.lookup_endpoint(*phys, a.eid)
         if ep is None:
             return
 
@@ -1031,8 +1037,8 @@ def name_owner_changed(name, new_owner, old_owner):
 
 """Simple system & network.
 
-Contains one interface (lladdr 0x10, local EID 8), and one endpoint (lladdr
-0x1d), that reports support for MCTP control and PLDM.
+Contains two interface (lladdr 0x10, local EID 8), (lladdr 0x11, local EID 10) with one endpoint (lladdr
+0x1d), and MCTP bridge (lladdr 0x1e, pool size 2), that reports support for MCTP control and PLDM.
 """
 @pytest.fixture
 async def sysnet():
@@ -1044,6 +1050,12 @@ async def sysnet():
     network = Network()
     network.add_endpoint(Endpoint(iface, bytes([0x1d]), types = [0, 1]))
 
+    # new interface for MCTP Bridge device
+    bridge_iface = System.Interface('mctp1', 2, 2, bytes([0x11]), 68, 254, True)
+    await system.add_interface(bridge_iface)
+    await system.add_address(System.Address(bridge_iface, 10))
+    network.add_endpoint(Endpoint(bridge_iface, bytes([0x1e]), types=[0, 1], pool_size= 0x2))
+
     return Sysnet(system, network)
 
 @pytest.fixture

tests/test_mctpd.py

@@ -531,7 +531,7 @@ def ep_removed(ep_path, interfaces):
     # Delete the endpoint from the network so its recovery will fail after
     # timeout. Note we delete at the split index as the network was already
     # populated with the default endpoint
-    del mctpd.network.endpoints[split]
+    del mctpd.network.endpoints[split + 1]
 
     # Begin recovery for the endpoint ...
     ep_ep = await ep.get_interface(MCTPD_ENDPOINT_I)
@@ -555,3 +555,50 @@ def ep_removed(ep_path, interfaces):
     with trio.move_on_after(2 * MCTPD_TRECLAIM) as expected:
         await removed.acquire()
     assert not expected.cancelled_caught
+
+""" Test that we allocate Eids to MCTP Bridge Endpoints"""
+async def test_endpoint_allocate_eid(dbus, mctpd):
+    bridge = mctpd.network.endpoints[1]
+    ep_types = [0, 1, 5]
+    bridge.types = ep_types
+    static_eid = 35
+    start_eid = 36
+
+    # mimicing MCTP Bridge by adding Bridg's Endpoints to same network and physcial address
+    for eid in range(start_eid, start_eid + bridge.pool_size + 1):
+        mctpd.network.add_endpoint(Endpoint(bridge.iface, bridge.lladdr, types = [0,1,2,3], eid= eid))
+
+    # first assign enpoint to MCTP Bridge and later involk allocate endpoint ids
+    # this will add Brige's enpoints routes and neigh and update dbus
+    mctp = await mctpd_mctp_iface_obj(dbus, bridge.iface)
+    (eid, _, path, _) = await mctp.call_assign_endpoint_static(
+        bridge.lladdr,
+        static_eid,
+        start_eid
+    )
+
+    # non blocking sleep for Allocate Eid timer expiry
+    await trio.sleep(5)
+    assert eid == static_eid
+    #  check if networks neighbours are reflecting the mctpd added bridge's neighbours
+    assert len(mctpd.system.neighbours) == (1 + bridge.pool_size)
+
+    neigh = mctpd.system.neighbours[0]
+    assert neigh.lladdr == bridge.lladdr
+    assert neigh.eid == static_eid
+
+    neigh = mctpd.system.neighbours[1]
+    assert neigh.lladdr == bridge.lladdr
+    assert neigh.eid == start_eid
+
+    neigh = mctpd.system.neighbours[2]
+    assert neigh.lladdr == bridge.lladdr
+    assert neigh.eid == start_eid + 1
+
+    # check one of Bridge's endpoint types per dbus property
+    path = path.rsplit('/endpoints', 1)[0]
+    path = path + f"/endpoints/{mctpd.network.endpoints[2].eid}"
+    ep = await mctpd_mctp_endpoint_common_obj(dbus, path)
+    query_types = list(await ep.get_supported_message_types())
+    print(mctpd.network.endpoints[2].types)
+    assert (query_types == mctpd.network.endpoints[2].types)