in my program mach_msg() hangs after it is called many times

[Da Zheng]

Hello,

I wrote a program called eth-multiplexer. It creates several virtual 
network interfaces. When it gets the packet from one interface, it will 
broadcast the packet to all interfaces, to all clients that connect to 
the interfaces. There is a BPF filter for each client, so the packet 
should go through the filter before it is delivered to the client.

The problem is that after eth-multiplexer calls mach_msg() to deliver 
packets to the clients many times, mach_msg() gets blocked.
In my test, the client is pfinet, and I use TCP to send data from one 
pfinet to another.

Here is the code of eth-multiplexer to receive the packet from one 
client and dispatch it to others.
It implements all server-side functions of device.defs, and gets packets 
from ds_device_write().

kern_return_t
ds_device_write (device_t device, mach_port_t reply_port,
                mach_msg_type_name_t reply_type, dev_mode_t mode,
                recnum_t recnum, io_buf_ptr_t data, size_t datalen,
                int *bytes_written)
{
 kern_return_t ret;
 struct vether_device *vdev = ports_lookup_port (port_bucket, device,
                                                 vdev_portclass);
 if (vdev == NULL)
   return D_NO_SUCH_DEVICE;
 ports_port_deref (vdev);
 /* The packet is forwarded to all virtual interfaces and
  * the interface which the multiplexer connects to. */
 ret = device_write (ether_port, mode , recnum ,
                      data, datalen, bytes_written);
 broadcast_pack (data, datalen);
 return ret;
}

int
broadcast_pack (char *data, int datalen)
{
 struct vether_device *vdev;
 int rval;

 for (vdev = dev_head; vdev; vdev = vdev->next)
   {
     rval=deliver_pack (data, datalen, vdev);
     if(rval)
       return rval;
   }
 return 0;
}

/* Create a message, and deliver it. */
int
deliver_pack (char *data, int datalen, struct vether_device *vdev)
{
 struct net_rcv_msg msg;
 int pack_size;
 struct ethhdr *header;
 struct packet_header *packet;

 pack_size = datalen - sizeof (struct ethhdr);
 /* remember message sizes must be rounded up */
 msg.msg_hdr.msgh_size = (((mach_msg_size_t) (sizeof(struct net_rcv_msg)
                                              - NET_RCV_MAX + 
pack_size)) + 3) & ~3;

 header = (struct ethhdr *) msg.header;
 packet = (struct packet_header *) msg.packet;
 msg.header_type = header_type;
 memcpy (header, data, sizeof (struct ethhdr));
 msg.packet_type = packet_type;
 memcpy (packet + 1, data + sizeof (struct ethhdr), pack_size);
 packet->type = header->h_proto;
 packet->length = pack_size + sizeof (struct packet_header);
 msg.packet_type.msgt_number = packet->length;

 return deliver_msg (&msg, vdev);
}

/*
* Deliver the message to all right pfinet servers that
* connects to the virtual network interface.
*/
int
deliver_msg(struct net_rcv_msg *msg, struct vether_device *vdev)
{
 mach_msg_return_t err;
 queue_head_t *if_port_list;
 net_rcv_port_t infp, nextfp;

 msg->msg_hdr.msgh_bits = MACH_MSGH_BITS (MACH_MSG_TYPE_COPY_SEND, 0);
 /* remember message sizes must be rounded up */
 msg->msg_hdr.msgh_local_port = MACH_PORT_NULL;
 msg->msg_hdr.msgh_kind = MACH_MSGH_KIND_NORMAL;
 msg->msg_hdr.msgh_id = NET_RCV_MSG_ID;

 if_port_list = &vdev->if_rcv_port_list;
 FILTER_ITERATE (if_port_list, infp, nextfp, &infp->input)
   {
     mach_port_t dest;
     net_hash_entry_t entp, *hash_headp;
     int ret_count;

     entp = (net_hash_entry_t) 0;
     ret_count = bpf_do_filter (infp,
                                msg->packet + sizeof (struct 
packet_header),
                                msg->net_rcv_msg_packet_count, msg->header,
                                sizeof (struct ethhdr), &hash_headp, 
&entp);
     if (entp == (net_hash_entry_t) 0)
       dest = infp->rcv_port;
     else
       dest = entp->rcv_port;

     if (ret_count)
       {
         debug ("before delivering the packet\n");
         msg->msg_hdr.msgh_remote_port = dest;
         err = mach_msg ((mach_msg_header_t *)msg, MACH_SEND_MSG,
                         msg->msg_hdr.msgh_size, 0, MACH_PORT_NULL,
                         MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
         if (err != MACH_MSG_SUCCESS)
           {
             mach_port_deallocate(mach_task_self (),
                                  ((mach_msg_header_t 
*)msg)->msgh_remote_port);
             error (0, err, "mach_msg");
             return -1;
           }
         debug ("after delivering the packet\n");
       }
   }
 FILTER_ITERATE_END

   return 0;
}

I wonder if the code above could cause the problem.
There is only one thread running in my program, so eth-multiplexer 
receives the next packet from a client only when it finish dispatching 
the previous one.
I wonder if it can make the queue of a port full so that mach_msg() is 
blocked.

Zheng Da