win-pvdrivers

view xennet/xennet_rx.c @ 1103:632727e57ffb

Fix bug in rx_coalesce implementation where reference counting goes bad. Add ASSERT to catch this too.
author James Harper <james.harper@bendigoit.com.au>
date Thu Sep 25 20:36:51 2014 +1000 (2014-09-25)
parents 86a97ef8cdf9
children ef445b2286f7
line source
1 /*
2 PV Drivers for Windows Xen HVM Domains
4 Copyright (c) 2014, James Harper
5 All rights reserved.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11 * Redistributions in binary form must reproduce the above copyright
12 notice, this list of conditions and the following disclaimer in the
13 documentation and/or other materials provided with the distribution.
14 * Neither the name of James Harper nor the
15 names of its contributors may be used to endorse or promote products
16 derived from this software without specific prior written permission.
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 DISCLAIMED. IN NO EVENT SHALL JAMES HARPER BE LIABLE FOR ANY
22 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
30 #include "xennet.h"
32 static __inline shared_buffer_t *
33 get_pb_from_freelist(struct xennet_info *xi)
34 {
35 shared_buffer_t *pb;
36 PVOID ptr_ref;
38 if (stack_pop(xi->rx_pb_stack, &ptr_ref))
39 {
40 pb = ptr_ref;
41 pb->ref_count = 1;
42 InterlockedDecrement(&xi->rx_pb_free);
43 return pb;
44 }
46 /* don't allocate a new one if we are shutting down */
47 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
48 return NULL;
50 pb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t), XENNET_POOL_TAG, LowPoolPriority);
51 if (!pb)
52 return NULL;
53 pb->virtual = ExAllocatePoolWithTagPriority(NonPagedPool, PAGE_SIZE, XENNET_POOL_TAG, LowPoolPriority);
54 if (!pb->virtual)
55 {
56 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
57 return NULL;
58 }
59 pb->mdl = IoAllocateMdl(pb->virtual, PAGE_SIZE, FALSE, FALSE, NULL);
60 if (!pb->mdl)
61 {
62 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
63 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
64 return NULL;
65 }
66 pb->gref = (grant_ref_t)XnGrantAccess(xi->handle,
67 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
68 if (pb->gref == INVALID_GRANT_REF)
69 {
70 IoFreeMdl(pb->mdl);
71 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
72 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
73 return NULL;
74 }
75 MmBuildMdlForNonPagedPool(pb->mdl);
76 pb->ref_count = 1;
77 return pb;
78 }
80 static __inline VOID
81 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
82 {
83 UNREFERENCED_PARAMETER(xi);
84 InterlockedIncrement(&pb->ref_count);
85 }
87 static __inline VOID
88 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
89 {
90 int ref = InterlockedDecrement(&pb->ref_count);
91 XN_ASSERT(ref >= 0);
92 if (ref == 0)
93 {
94 //NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
95 //NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
96 if (xi->rx_pb_free > RX_MAX_PB_FREELIST)
97 {
98 XnEndAccess(xi->handle, pb->gref, FALSE, (ULONG)'XNRX');
99 IoFreeMdl(pb->mdl);
100 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
101 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
102 return;
103 }
104 pb->mdl->ByteCount = PAGE_SIZE;
105 pb->mdl->Next = NULL;
106 pb->next = NULL;
107 stack_push(xi->rx_pb_stack, pb);
108 InterlockedIncrement(&xi->rx_pb_free);
109 }
110 }
112 static __inline shared_buffer_t *
113 get_hb_from_freelist(struct xennet_info *xi)
114 {
115 shared_buffer_t *hb;
116 PVOID ptr_ref;
118 if (stack_pop(xi->rx_hb_stack, &ptr_ref))
119 {
120 hb = ptr_ref;
121 InterlockedDecrement(&xi->rx_hb_free);
122 return hb;
123 }
125 /* don't allocate a new one if we are shutting down */
126 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
127 return NULL;
129 hb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, XENNET_POOL_TAG, LowPoolPriority);
130 if (!hb)
131 return NULL;
132 NdisZeroMemory(hb, sizeof(shared_buffer_t));
133 hb->mdl = IoAllocateMdl(hb + 1, MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, FALSE, FALSE, NULL);
134 if (!hb->mdl) {
135 ExFreePoolWithTag(hb, XENNET_POOL_TAG);
136 return NULL;
137 }
138 MmBuildMdlForNonPagedPool(hb->mdl);
139 return hb;
140 }
142 static __inline VOID
143 put_hb_on_freelist(struct xennet_info *xi, shared_buffer_t *hb)
144 {
145 XN_ASSERT(xi);
146 hb->mdl->ByteCount = sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH;
147 hb->mdl->Next = NULL;
148 hb->next = NULL;
149 stack_push(xi->rx_hb_stack, hb);
150 InterlockedIncrement(&xi->rx_hb_free);
151 }
153 // Called at DISPATCH_LEVEL with rx lock held
154 static VOID
155 XenNet_FillRing(struct xennet_info *xi)
156 {
157 unsigned short id;
158 shared_buffer_t *page_buf;
159 ULONG i, notify;
160 ULONG batch_target;
161 RING_IDX req_prod = xi->rx_ring.req_prod_pvt;
162 netif_rx_request_t *req;
164 //FUNCTION_ENTER();
166 if (xi->device_state != DEVICE_STATE_ACTIVE)
167 return;
169 batch_target = xi->rx_target - (req_prod - xi->rx_ring.rsp_cons);
171 if (batch_target < (xi->rx_target >> 2)) {
172 //FUNCTION_EXIT();
173 return; /* only refill if we are less than 3/4 full already */
174 }
176 for (i = 0; i < batch_target; i++) {
177 page_buf = get_pb_from_freelist(xi);
178 if (!page_buf) {
179 FUNCTION_MSG("Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target);
180 break;
181 }
182 xi->rx_id_free--;
184 /* Give to netback */
185 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
186 XN_ASSERT(xi->rx_ring_pbs[id] == NULL);
187 xi->rx_ring_pbs[id] = page_buf;
188 req = RING_GET_REQUEST(&xi->rx_ring, req_prod + i);
189 req->id = id;
190 req->gref = page_buf->gref;
191 XN_ASSERT(req->gref != INVALID_GRANT_REF);
192 }
193 KeMemoryBarrier();
194 xi->rx_ring.req_prod_pvt = req_prod + i;
195 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx_ring, notify);
196 if (notify) {
197 XnNotify(xi->handle, xi->event_channel);
198 }
200 //FUNCTION_EXIT();
202 return;
203 }
205 #if NTDDI_VERSION < NTDDI_VISTA
206 typedef struct {
207 PNDIS_PACKET first_packet;
208 PNDIS_PACKET last_packet;
209 ULONG packet_count;
210 } rx_context_t;
211 #else
212 typedef struct {
213 PNET_BUFFER_LIST first_nbl;
214 PNET_BUFFER_LIST last_nbl;
215 ULONG packet_count;
216 ULONG nbl_count;
217 } rx_context_t;
218 #endif
220 #if NTDDI_VERSION < NTDDI_VISTA
221 /*
222 NDIS5 appears to insist that the checksum on received packets is correct, and won't
223 believe us when we lie about it, which happens when the packet is generated on the
224 same bridge in Dom0. Doh!
225 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
226 */
228 static BOOLEAN
229 XenNet_SumPacketData(
230 packet_info_t *pi,
231 PNDIS_PACKET packet,
232 BOOLEAN set_csum) {
233 USHORT i;
234 PUCHAR buffer;
235 PMDL mdl;
236 UINT total_length;
237 UINT data_length;
238 UINT buffer_length;
239 USHORT buffer_offset;
240 ULONG csum;
241 PUSHORT csum_ptr;
242 USHORT remaining;
243 USHORT ip4_length;
244 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
246 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
247 if (!buffer) {
248 FUNCTION_MSG("NdisGetFirstBufferFromPacketSafe failed, buffer == NULL\n");
249 return FALSE;
250 }
251 XN_ASSERT(mdl);
253 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
254 data_length = ip4_length + XN_HDR_SIZE;
256 if ((USHORT)data_length > total_length) {
257 FUNCTION_MSG("Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length);
258 return FALSE;
259 }
261 switch (pi->ip_proto) {
262 case 6:
263 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
264 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
265 break;
266 case 17:
267 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
268 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
269 break;
270 default:
271 FUNCTION_MSG("Don't know how to calc sum for IP Proto %d\n", pi->ip_proto);
272 //FUNCTION_EXIT();
273 return FALSE; // should never happen
274 }
276 if (set_csum)
277 *csum_ptr = 0;
279 csum = 0;
280 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
281 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
282 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
284 remaining = ip4_length - pi->ip4_header_length;
286 csum += remaining;
288 csum_span = FALSE;
289 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
290 while (i < data_length) {
291 /* don't include the checksum field itself in the calculation */
292 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6)) {
293 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
294 i += 2;
295 buffer_offset += 2;
296 continue;
297 }
298 if (csum_span) {
299 /* the other half of the next bit */
300 XN_ASSERT(buffer_offset == 0);
301 csum += (USHORT)buffer[buffer_offset];
302 csum_span = FALSE;
303 i += 1;
304 buffer_offset += 1;
305 } else if (buffer_offset == buffer_length - 1) {
306 /* deal with a buffer ending on an odd byte boundary */
307 csum += (USHORT)buffer[buffer_offset] << 8;
308 csum_span = TRUE;
309 i += 1;
310 buffer_offset += 1;
311 } else {
312 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
313 i += 2;
314 buffer_offset += 2;
315 }
316 if (buffer_offset == buffer_length && i < total_length) {
317 NdisGetNextBuffer(mdl, &mdl);
318 if (mdl == NULL) {
319 FUNCTION_MSG(__DRIVER_NAME " Ran out of buffers\n");
320 return FALSE; // should never happen
321 }
322 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
323 XN_ASSERT(buffer_length);
324 buffer_offset = 0;
325 }
326 }
328 while (csum & 0xFFFF0000)
329 csum = (csum & 0xFFFF) + (csum >> 16);
331 if (set_csum) {
332 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
333 } else {
334 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
335 }
336 return TRUE;
337 }
338 #endif
340 static BOOLEAN
341 XenNet_MakePacket(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi) {
342 #if NTDDI_VERSION < NTDDI_VISTA
343 NDIS_STATUS status;
344 PNDIS_PACKET packet;
345 #else
346 PNET_BUFFER_LIST nbl;
347 PNET_BUFFER packet;
348 #endif
349 PMDL mdl_head, mdl_tail, curr_mdl;
350 PUCHAR header_va;
351 ULONG out_remaining;
352 ULONG header_extra;
353 shared_buffer_t *header_buf;
354 ULONG outstanding;
355 #if NTDDI_VERSION < NTDDI_VISTA
356 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
357 //UINT packet_length;
358 #else
359 NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csum_info;
360 #endif
361 //FUNCTION_ENTER();
363 #if NTDDI_VERSION < NTDDI_VISTA
364 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
365 if (status != NDIS_STATUS_SUCCESS) {
366 FUNCTION_MSG("No free packets\n");
367 return FALSE;
368 }
370 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
371 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
372 #else
373 nbl = NdisAllocateNetBufferList(xi->rx_nbl_pool, 0, 0);
374 if (!nbl) {
375 /* buffers will be freed in MakePackets */
376 FUNCTION_MSG("No free nbls\n");
377 //FUNCTION_EXIT();
378 return FALSE;
379 }
381 packet = NdisAllocateNetBuffer(xi->rx_packet_pool, NULL, 0, 0);
382 if (!packet) {
383 FUNCTION_MSG("No free packets\n");
384 NdisFreeNetBufferList(nbl);
385 //FUNCTION_EXIT();
386 return FALSE;
387 }
388 #endif
390 if ((!pi->first_mdl->Next || (xi->config_rx_coalesce && pi->total_length <= PAGE_SIZE)) && !pi->split_required) {
391 /* a single buffer <= MTU */
392 header_buf = NULL;
393 /* get all the packet into the header */
394 XenNet_BuildHeader(pi, pi->first_mdl_virtual, PAGE_SIZE);
396 /* have to create a partial mdl over the pb MDL as the pb mdl has a Next which breaks things */
397 curr_mdl = IoAllocateMdl(pi->first_mdl_virtual, pi->total_length, FALSE, FALSE, NULL);
398 XN_ASSERT(curr_mdl);
399 IoBuildPartialMdl(pi->first_mdl, curr_mdl, pi->first_mdl_virtual, pi->total_length);
400 #if NTDDI_VERSION < NTDDI_VISTA
401 NdisChainBufferAtBack(packet, curr_mdl);
402 PACKET_FIRST_PB(packet) = pi->first_pb;
403 #else
404 NET_BUFFER_FIRST_MDL(packet) = curr_mdl;
405 NET_BUFFER_CURRENT_MDL(packet) = curr_mdl;
406 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
407 NET_BUFFER_DATA_OFFSET(packet) = 0;
408 NET_BUFFER_DATA_LENGTH(packet) = pi->total_length;
409 NB_FIRST_PB(packet) = pi->first_pb;
410 #endif
411 ref_pb(xi, pi->first_pb);
412 } else {
413 XN_ASSERT(ndis_os_minor_version >= 1);
414 header_buf = get_hb_from_freelist(xi);
415 if (!header_buf) {
416 FUNCTION_MSG("No free header buffers\n");
417 #if NTDDI_VERSION < NTDDI_VISTA
418 NdisUnchainBufferAtFront(packet, &curr_mdl);
419 NdisFreePacket(packet);
420 #else
421 NdisFreeNetBufferList(nbl);
422 NdisFreeNetBuffer(packet);
423 #endif
424 return FALSE;
425 }
426 header_va = (PUCHAR)(header_buf + 1);
427 NdisMoveMemory(header_va, pi->header, pi->header_length);
428 //if (pi->ip_proto == 50) {
429 // FUNCTION_MSG("header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead);
430 // FUNCTION_MSG("ip4_header_length = %d\n", pi->ip4_header_length);
431 // FUNCTION_MSG("tcp_header_length = %d\n", pi->tcp_header_length);
432 //}
433 /* make sure only the header is in the first buffer (or the entire packet, but that is done in the above case) */
434 XenNet_BuildHeader(pi, header_va, MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
435 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
436 XN_ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
437 header_buf->mdl->ByteCount = pi->header_length;
438 mdl_head = mdl_tail = curr_mdl = header_buf->mdl;
439 XN_ASSERT(!header_buf->mdl->Next);
440 #if NTDDI_VERSION < NTDDI_VISTA
441 PACKET_FIRST_PB(packet) = header_buf;
442 header_buf->next = pi->curr_pb;
443 NdisChainBufferAtBack(packet, mdl_head);
444 #else
445 NB_FIRST_PB(packet) = header_buf;
446 header_buf->next = pi->curr_pb;
447 NET_BUFFER_FIRST_MDL(packet) = mdl_head;
448 NET_BUFFER_CURRENT_MDL(packet) = mdl_head;
449 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
450 NET_BUFFER_DATA_OFFSET(packet) = 0;
451 NET_BUFFER_DATA_LENGTH(packet) = pi->header_length;
452 #endif
454 if (pi->split_required) {
455 /* must be ip4 */
456 ULONG tcp_length;
457 USHORT new_ip4_length;
458 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
459 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
460 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
461 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
462 pi->tcp_seq += tcp_length;
463 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
464 /* part of the packet is already present in the header buffer for lookahead */
465 out_remaining = tcp_length - header_extra;
466 XN_ASSERT((LONG)out_remaining >= 0);
467 } else {
468 out_remaining = pi->total_length - pi->header_length;
469 XN_ASSERT((LONG)out_remaining >= 0);
470 }
472 while (out_remaining != 0) {
473 //ULONG in_buffer_offset;
474 ULONG in_buffer_length;
475 ULONG out_length;
477 //if (pi->ip_proto == 50) {
478 // FUNCTION_MSG("in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb);
479 //}
480 if (!pi->curr_mdl || !pi->curr_pb) {
481 FUNCTION_MSG("out of buffers for packet\n");
482 //KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
483 // TODO: free some stuff or we'll leak
484 /* unchain buffers then free packet */
485 //FUNCTION_EXIT();
486 return FALSE;
487 }
489 in_buffer_length = MmGetMdlByteCount(pi->curr_mdl);
490 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
491 curr_mdl = IoAllocateMdl((PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length, FALSE, FALSE, NULL);
492 XN_ASSERT(curr_mdl);
493 IoBuildPartialMdl(pi->curr_mdl, curr_mdl, (PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length);
494 mdl_tail->Next = curr_mdl;
495 mdl_tail = curr_mdl;
496 curr_mdl->Next = NULL; /* I think this might be redundant */
497 #if NTDDI_VERSION < NTDDI_VISTA
498 #else
499 NET_BUFFER_DATA_LENGTH(packet) += out_length;
500 #endif
501 ref_pb(xi, pi->curr_pb);
502 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
503 if (pi->curr_mdl_offset == in_buffer_length) {
504 pi->curr_mdl = pi->curr_mdl->Next;
505 pi->curr_pb = pi->curr_pb->next;
506 pi->curr_mdl_offset = 0;
507 }
508 out_remaining -= out_length;
509 }
510 #if NTDDI_VERSION < NTDDI_VISTA
511 if (pi->split_required) {
512 // TODO: only if Ip checksum is disabled...
513 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
514 }
515 #endif
516 if (header_extra > 0)
517 pi->header_length -= header_extra;
518 }
520 rc->packet_count++;
521 #if NTDDI_VERSION < NTDDI_VISTA
522 #else
523 NET_BUFFER_LIST_FIRST_NB(nbl) = packet;
524 #endif
526 if (pi->parse_result == PARSE_OK) {
527 #if NTDDI_VERSION < NTDDI_VISTA
528 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
529 packet, TcpIpChecksumPacketInfo);
530 csum_info->Value = 0;
531 if (pi->csum_blank || pi->data_validated || pi->split_required) {
532 BOOLEAN checksum_offload = FALSE;
533 /* we know this is IPv4, and we know Linux always validates the IPv4 checksum for us */
534 if (xi->setting_csum.V4Receive.IpChecksum) {
535 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
536 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
537 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
538 else
539 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
540 }
541 }
542 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
543 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
544 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
545 checksum_offload = TRUE;
546 }
547 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
548 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
549 checksum_offload = TRUE;
550 }
551 if (pi->csum_blank && (!xi->config_csum_rx_dont_fix || !checksum_offload)) {
552 XenNet_SumPacketData(pi, packet, TRUE);
553 }
554 } else if (xi->config_csum_rx_check && pi->ip_version == 4) {
555 if (xi->setting_csum.V4Receive.IpChecksum) {
556 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
557 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
558 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
559 else
560 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
561 }
562 }
563 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
564 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
565 if (XenNet_SumPacketData(pi, packet, FALSE)) {
566 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
567 } else {
568 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
569 }
570 }
571 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
572 if (XenNet_SumPacketData(pi, packet, FALSE)) {
573 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
574 } else {
575 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
576 }
577 }
578 }
579 #else
580 csum_info.Value = 0;
581 if (pi->csum_blank || pi->data_validated || pi->mss) {
582 if (pi->ip_proto == 6) {
583 csum_info.Receive.IpChecksumSucceeded = TRUE;
584 csum_info.Receive.TcpChecksumSucceeded = TRUE;
585 } else if (pi->ip_proto == 17) {
586 csum_info.Receive.IpChecksumSucceeded = TRUE;
587 csum_info.Receive.UdpChecksumSucceeded = TRUE;
588 }
589 }
590 NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo) = csum_info.Value;
591 #endif
592 }
594 #if NTDDI_VERSION < NTDDI_VISTA
595 if (!rc->first_packet) {
596 rc->first_packet = packet;
597 } else {
598 PACKET_NEXT_PACKET(rc->last_packet) = packet;
599 }
600 rc->last_packet = packet;
601 rc->packet_count++;
602 #else
603 if (!rc->first_nbl) {
604 rc->first_nbl = nbl;
605 } else {
606 NET_BUFFER_LIST_NEXT_NBL(rc->last_nbl) = nbl;
607 }
608 rc->last_nbl = nbl;
609 NET_BUFFER_LIST_NEXT_NBL(nbl) = NULL;
610 rc->nbl_count++;
611 if (pi->is_multicast) {
612 /* multicast */
613 xi->stats.ifHCInMulticastPkts++;
614 xi->stats.ifHCInMulticastOctets += NET_BUFFER_DATA_LENGTH(packet);
615 } else if (pi->is_broadcast) {
616 /* broadcast */
617 xi->stats.ifHCInBroadcastPkts++;
618 xi->stats.ifHCInBroadcastOctets += NET_BUFFER_DATA_LENGTH(packet);
619 } else {
620 /* unicast */
621 xi->stats.ifHCInUcastPkts++;
622 xi->stats.ifHCInUcastOctets += NET_BUFFER_DATA_LENGTH(packet);
623 }
624 #endif
626 outstanding = InterlockedIncrement(&xi->rx_outstanding);
627 #if NTDDI_VERSION < NTDDI_VISTA
628 if (outstanding > RX_PACKET_HIGH_WATER_MARK || !xi->rx_pb_free) {
629 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
630 } else {
631 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
632 }
633 #if 0
634 /* windows gets lazy about ack packets and holds on to them forever under high load situations. we don't like this */
635 NdisQueryPacketLength(packet, &packet_length);
636 if (pi->parse_result != PARSE_OK || (pi->ip_proto == 6 && packet_length <= NDIS_STATUS_RESOURCES_MAX_LENGTH))
637 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
638 else
639 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
640 #endif
641 #endif
643 //FUNCTION_EXIT();
644 return TRUE;
645 }
647 static VOID
648 XenNet_MakePackets(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi)
649 {
650 UCHAR tcp_flags;
651 shared_buffer_t *page_buf;
653 XenNet_ParsePacketHeader(pi, NULL, XN_HDR_SIZE + xi->current_lookahead);
655 if (!XenNet_FilterAcceptPacket(xi, pi)) {
656 goto done;
657 }
659 if (pi->split_required) {
660 #if NTDDI_VERSION < NTDDI_VISTA
661 /* need to split to mss for NDIS5 */
662 #else
663 switch (xi->current_gso_rx_split_type) {
664 case RX_LSO_SPLIT_HALF:
665 pi->mss = max((pi->tcp_length + 1) / 2, pi->mss);
666 break;
667 case RX_LSO_SPLIT_NONE:
668 pi->mss = 65535;
669 break;
670 }
671 #endif
672 }
674 switch (pi->ip_proto) {
675 case 6: // TCP
676 if (pi->split_required)
677 break;
678 /* fall through */
679 case 17: // UDP
680 if (!XenNet_MakePacket(xi, rc, pi)) {
681 FUNCTION_MSG("Failed to make packet\n");
682 #if NTDDI_VERSION < NTDDI_VISTA
683 xi->stat_rx_no_buffer++;
684 #else
685 xi->stats.ifInDiscards++;
686 #endif
687 goto done;
688 }
689 goto done;
690 default:
691 if (!XenNet_MakePacket(xi, rc, pi)) {
692 FUNCTION_MSG("Failed to make packet\n");
693 #if NTDDI_VERSION < NTDDI_VISTA
694 xi->stat_rx_no_buffer++;
695 #else
696 xi->stats.ifInDiscards++;
697 #endif
698 goto done;
699 }
700 goto done;
701 }
703 /* this is the split_required code */
704 pi->tcp_remaining = pi->tcp_length;
706 /* we can make certain assumptions here as the following code is only for tcp4 */
707 tcp_flags = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13];
708 /* clear all tcp flags except ack except for the last packet */
709 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] &= 0x10;
710 while (pi->tcp_remaining) {
711 if (pi->tcp_remaining <= pi->mss) {
712 /* restore tcp flags for the last packet */
713 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] = tcp_flags;
714 }
715 if (!XenNet_MakePacket(xi, rc, pi)) {
716 FUNCTION_MSG("Failed to make packet\n");
717 #if NTDDI_VERSION < NTDDI_VISTA
718 xi->stat_rx_no_buffer++;
719 #else
720 xi->stats.ifInDiscards++;
721 #endif
722 break; /* we are out of memory - just drop the packets */
723 }
724 }
725 done:
726 page_buf = pi->first_pb;
727 while (page_buf) {
728 shared_buffer_t *next_pb = page_buf->next;
729 put_pb_on_freelist(xi, page_buf); /* this doesn't actually free the page_puf if there are outstanding references */
730 page_buf = next_pb;
731 }
732 XenNet_ClearPacketInfo(pi);
733 //FUNCTION_EXIT();
734 return;
735 }
737 #if NTDDI_VERSION < NTDDI_VISTA
738 /* called at DISPATCH_LEVEL */
739 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
740 VOID
741 XenNet_ReturnPacket(NDIS_HANDLE adapter_context, PNDIS_PACKET packet) {
742 struct xennet_info *xi = adapter_context;
743 PNDIS_BUFFER buffer;
744 shared_buffer_t *page_buf = PACKET_FIRST_PB(packet);
746 //FUNCTION_ENTER();
747 NdisUnchainBufferAtFront(packet, &buffer);
749 while (buffer) {
750 shared_buffer_t *next_buf;
751 XN_ASSERT(page_buf);
752 next_buf = page_buf->next;
753 if (!page_buf->virtual) {
754 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
755 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(buffer) - 1);
756 } else {
757 if (buffer != page_buf->mdl)
758 NdisFreeBuffer(buffer);
759 put_pb_on_freelist(xi, page_buf);
760 }
761 NdisUnchainBufferAtFront(packet, &buffer);
762 page_buf = next_buf;
763 }
765 NdisFreePacket(packet);
766 InterlockedDecrement(&xi->rx_outstanding);
767 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
768 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
769 //FUNCTION_EXIT();
770 }
771 #else
772 /* called at <= DISPATCH_LEVEL */
773 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
774 VOID
775 XenNet_ReturnNetBufferLists(NDIS_HANDLE adapter_context, PNET_BUFFER_LIST curr_nbl, ULONG return_flags)
776 {
777 struct xennet_info *xi = adapter_context;
778 UNREFERENCED_PARAMETER(return_flags);
780 //FUNCTION_ENTER();
782 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
784 XN_ASSERT(xi);
785 while (curr_nbl)
786 {
787 PNET_BUFFER_LIST next_nbl;
788 PNET_BUFFER curr_nb;
790 next_nbl = NET_BUFFER_LIST_NEXT_NBL(curr_nbl);
791 curr_nb = NET_BUFFER_LIST_FIRST_NB(curr_nbl);
792 while (curr_nb)
793 {
794 PNET_BUFFER next_nb;
795 PMDL curr_mdl;
796 shared_buffer_t *page_buf;
798 next_nb = NET_BUFFER_NEXT_NB(curr_nb);
799 curr_mdl = NET_BUFFER_FIRST_MDL(curr_nb);
800 page_buf = NB_FIRST_PB(curr_nb);
801 while (curr_mdl)
802 {
803 shared_buffer_t *next_buf;
804 PMDL next_mdl;
806 XN_ASSERT(page_buf); /* make sure that there is a pb to match this mdl */
807 next_mdl = curr_mdl->Next;
808 next_buf = page_buf->next;
809 if (!page_buf->virtual)
810 {
811 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
812 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(curr_mdl) - 1);
813 }
814 else
815 {
816 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
817 if (curr_mdl != page_buf->mdl)
818 {
819 //KdPrint((__DRIVER_NAME " curr_mdl = %p, page_buf->mdl = %p\n", curr_mdl, page_buf->mdl));
820 IoFreeMdl(curr_mdl);
821 }
822 put_pb_on_freelist(xi, page_buf);
823 }
824 curr_mdl = next_mdl;
825 page_buf = next_buf;
826 }
828 NdisFreeNetBuffer(curr_nb);
829 InterlockedDecrement(&xi->rx_outstanding);
831 curr_nb = next_nb;
832 }
833 NdisFreeNetBufferList(curr_nbl);
834 curr_nbl = next_nbl;
835 }
837 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
838 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
840 //FUNCTION_EXIT();
841 }
842 #endif
844 /* We limit the number of packets per interrupt so that acks get a chance
845 under high rx load. The DPC is immediately re-scheduled */
847 #define MAXIMUM_PACKETS_PER_INDICATE 32
849 #define MAXIMUM_PACKETS_PER_INTERRUPT 2560 /* this is calculated before large packet split */
850 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
852 // Called at DISPATCH_LEVEL
853 BOOLEAN
854 XenNet_RxBufferCheck(struct xennet_info *xi)
855 {
856 RING_IDX cons, prod;
857 ULONG packet_count = 0;
858 ULONG packet_data = 0;
859 ULONG buffer_count = 0;
860 USHORT id;
861 int more_to_do = FALSE;
862 shared_buffer_t *page_buf;
863 #if NTDDI_VERSION < NTDDI_VISTA
864 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
865 PNDIS_PACKET first_header_only_packet;
866 PNDIS_PACKET last_header_only_packet;
867 #else
868 #endif
869 //ULONG nbl_count = 0;
870 ULONG interim_packet_data = 0;
871 struct netif_extra_info *ei;
872 rx_context_t rc;
873 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
874 shared_buffer_t *head_buf = NULL;
875 shared_buffer_t *tail_buf = NULL;
876 shared_buffer_t *last_buf = NULL;
877 BOOLEAN extra_info_flag = FALSE;
878 BOOLEAN more_data_flag = FALSE;
879 BOOLEAN dont_set_event;
880 //FUNCTION_ENTER();
882 #if NTDDI_VERSION < NTDDI_VISTA
883 rc.first_packet = NULL;
884 rc.last_packet = NULL;
885 rc.packet_count = 0;
886 #else
887 rc.first_nbl = NULL;
888 rc.last_nbl = NULL;
889 rc.packet_count = 0;
890 rc.nbl_count = 0;
891 #endif
893 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
894 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
896 if (xi->device_state != DEVICE_STATE_ACTIVE) {
897 /* there is a chance that our Dpc had been queued just before the shutdown... */
898 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
899 return FALSE;
900 }
902 if (xi->rx_partial_buf) {
903 head_buf = xi->rx_partial_buf;
904 tail_buf = xi->rx_partial_buf;
905 while (tail_buf->next)
906 tail_buf = tail_buf->next;
907 more_data_flag = xi->rx_partial_more_data_flag;
908 extra_info_flag = xi->rx_partial_extra_info_flag;
909 xi->rx_partial_buf = NULL;
910 }
912 do {
913 prod = xi->rx_ring.sring->rsp_prod;
914 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
916 for (cons = xi->rx_ring.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++) {
917 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
918 page_buf = xi->rx_ring_pbs[id];
919 XN_ASSERT(page_buf);
920 xi->rx_ring_pbs[id] = NULL;
921 xi->rx_id_free++;
922 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx_ring, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
923 if (!extra_info_flag) {
924 if (page_buf->rsp.status <= 0 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE) {
925 FUNCTION_MSG("Error: rsp offset %d, size %d\n",
926 page_buf->rsp.offset, page_buf->rsp.status);
927 XN_ASSERT(!extra_info_flag);
928 put_pb_on_freelist(xi, page_buf);
929 continue;
930 }
931 }
933 if (!head_buf) {
934 head_buf = page_buf;
935 tail_buf = page_buf;
936 } else {
937 tail_buf->next = page_buf;
938 tail_buf = page_buf;
939 }
940 page_buf->next = NULL;
942 if (extra_info_flag) {
943 ei = (struct netif_extra_info *)&page_buf->rsp;
944 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
945 } else {
946 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
947 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
948 interim_packet_data += page_buf->rsp.status;
949 }
951 if (!extra_info_flag && !more_data_flag) {
952 last_buf = page_buf;
953 packet_count++;
954 packet_data += interim_packet_data;
955 interim_packet_data = 0;
956 }
957 buffer_count++;
958 }
959 xi->rx_ring.rsp_cons = cons;
961 /* Give netback more buffers */
962 XenNet_FillRing(xi);
964 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
965 break;
967 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
968 if (!more_to_do) {
969 xi->rx_ring.sring->rsp_event = xi->rx_ring.rsp_cons + 1;
970 KeMemoryBarrier();
971 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
972 }
973 } while (more_to_do);
975 /* anything past last_buf belongs to an incomplete packet... */
976 if (last_buf && last_buf->next)
977 {
978 FUNCTION_MSG("Partial receive\n");
979 xi->rx_partial_buf = last_buf->next;
980 xi->rx_partial_more_data_flag = more_data_flag;
981 xi->rx_partial_extra_info_flag = extra_info_flag;
982 last_buf->next = NULL;
983 }
985 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
987 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
988 {
989 /* fire again immediately */
990 FUNCTION_MSG("Dpc Duration Exceeded\n");
991 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
992 // KeSetImportanceDpc(&xi->rxtx_dpc, MediumImportance);
993 KeInsertQueueDpc(&xi->rxtx_dpc, NULL, NULL);
994 /* dont set an event in TX path */
995 dont_set_event = TRUE;
996 }
997 else
998 {
999 /* make sure the Dpc queue is run immediately next interrupt */
1000 // KeSetImportanceDpc(&xi->rxtx_dpc, HighImportance);
1001 /* set an event in TX path */
1002 dont_set_event = FALSE;
1005 /* make packets out of the buffers */
1006 page_buf = head_buf;
1007 extra_info_flag = FALSE;
1008 more_data_flag = FALSE;
1010 while (page_buf) {
1011 shared_buffer_t *next_buf = page_buf->next;
1012 PMDL mdl;
1014 page_buf->next = NULL;
1015 if (extra_info_flag) {
1016 //KdPrint((__DRIVER_NAME " processing extra info\n"));
1017 ei = (struct netif_extra_info *)&page_buf->rsp;
1018 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
1019 switch (ei->type)
1021 case XEN_NETIF_EXTRA_TYPE_GSO:
1022 switch (ei->u.gso.type) {
1023 case XEN_NETIF_GSO_TYPE_TCPV4:
1024 pi->mss = ei->u.gso.size;
1025 // TODO - put this assertion somewhere XN_ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
1026 break;
1027 default:
1028 FUNCTION_MSG("Unknown GSO type (%d) detected\n", ei->u.gso.type);
1029 break;
1031 break;
1032 default:
1033 FUNCTION_MSG("Unknown extra info type (%d) detected\n", ei->type);
1034 break;
1036 put_pb_on_freelist(xi, page_buf);
1037 } else {
1038 XN_ASSERT(!page_buf->rsp.offset);
1039 if (!more_data_flag) { // handling the packet's 1st buffer
1040 if (page_buf->rsp.flags & NETRXF_csum_blank)
1041 pi->csum_blank = TRUE;
1042 if (page_buf->rsp.flags & NETRXF_data_validated)
1043 pi->data_validated = TRUE;
1045 mdl = page_buf->mdl;
1046 mdl->ByteCount = page_buf->rsp.status; //NdisAdjustBufferLength(mdl, page_buf->rsp.status);
1047 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
1048 if (pi->first_pb) {
1049 XN_ASSERT(pi->curr_pb);
1050 //KdPrint((__DRIVER_NAME " additional buffer\n"));
1051 pi->curr_pb->next = page_buf;
1052 pi->curr_pb = page_buf;
1053 XN_ASSERT(pi->curr_mdl);
1054 pi->curr_mdl->Next = mdl;
1055 pi->curr_mdl = mdl;
1056 } else {
1057 pi->first_pb = page_buf;
1058 pi->curr_pb = page_buf;
1059 pi->first_mdl = mdl;
1060 pi->curr_mdl = mdl;
1062 //pi->mdl_count++;
1063 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
1064 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
1065 pi->total_length = pi->total_length + page_buf->rsp.status;
1068 /* Packet done, add it to the list */
1069 if (!more_data_flag && !extra_info_flag) {
1070 pi->curr_pb = pi->first_pb;
1071 pi->curr_mdl = pi->first_mdl;
1072 XenNet_MakePackets(xi, &rc, pi);
1075 page_buf = next_buf;
1077 XN_ASSERT(!more_data_flag && !extra_info_flag);
1079 #if NTDDI_VERSION < NTDDI_VISTA
1080 packet_count = 0;
1081 first_header_only_packet = NULL;
1082 last_header_only_packet = NULL;
1084 while (rc.first_packet) {
1085 PNDIS_PACKET packet;
1086 NDIS_STATUS status;
1088 packet = rc.first_packet;
1089 XN_ASSERT(PACKET_FIRST_PB(packet));
1090 rc.first_packet = PACKET_NEXT_PACKET(packet);
1091 status = NDIS_GET_PACKET_STATUS(packet);
1092 if (status == NDIS_STATUS_RESOURCES) {
1093 if (!first_header_only_packet) {
1094 first_header_only_packet = packet;
1095 } else {
1096 PACKET_NEXT_PACKET(last_header_only_packet) = packet;
1098 last_header_only_packet = packet;
1099 PACKET_NEXT_PACKET(packet) = NULL;
1101 packets[packet_count++] = packet;
1102 /* if we indicate a packet with NDIS_STATUS_RESOURCES then any following packet can't be NDIS_STATUS_SUCCESS */
1103 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || !rc.first_packet
1104 || (NDIS_GET_PACKET_STATUS(rc.first_packet) == NDIS_STATUS_SUCCESS
1105 && status == NDIS_STATUS_RESOURCES)) {
1106 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
1107 packet_count = 0;
1110 /* now return the packets for which we indicated NDIS_STATUS_RESOURCES */
1111 while (first_header_only_packet) {
1112 PNDIS_PACKET packet = first_header_only_packet;
1113 first_header_only_packet = PACKET_NEXT_PACKET(packet);
1114 XenNet_ReturnPacket(xi, packet);
1116 #else
1117 if (rc.first_nbl) {
1118 NdisMIndicateReceiveNetBufferLists(xi->adapter_handle, rc.first_nbl,
1119 NDIS_DEFAULT_PORT_NUMBER, rc.nbl_count,
1120 NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL
1121 //| NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE
1122 | NDIS_RECEIVE_FLAGS_PERFECT_FILTERED);
1124 #endif
1125 //FUNCTION_EXIT();
1126 return dont_set_event;
1129 static VOID
1130 XenNet_BufferFree(xennet_info_t *xi)
1132 shared_buffer_t *sb;
1133 int i;
1135 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1136 if (xi->rx_ring_pbs[i] != NULL) {
1137 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
1138 xi->rx_ring_pbs[i] = NULL;
1142 /* because we are shutting down this won't allocate new ones */
1143 while ((sb = get_pb_from_freelist(xi)) != NULL) {
1144 XnEndAccess(xi->handle,
1145 sb->gref, FALSE, (ULONG)'XNRX');
1146 IoFreeMdl(sb->mdl);
1147 ExFreePoolWithTag(sb->virtual, XENNET_POOL_TAG);
1148 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
1150 while ((sb = get_hb_from_freelist(xi)) != NULL) {
1151 IoFreeMdl(sb->mdl);
1152 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
1156 BOOLEAN
1157 XenNet_RxInit(xennet_info_t *xi) {
1158 #if NTDDI_VERSION < NTDDI_VISTA
1159 NDIS_STATUS status;
1160 #else
1161 NET_BUFFER_LIST_POOL_PARAMETERS nbl_pool_parameters;
1162 NET_BUFFER_POOL_PARAMETERS nb_pool_parameters;
1163 #endif
1164 int ret;
1165 int i;
1167 FUNCTION_ENTER();
1169 // this stuff needs to be done once only...
1170 KeInitializeSpinLock(&xi->rx_lock);
1171 KeInitializeEvent(&xi->rx_idle_event, SynchronizationEvent, FALSE);
1172 #if NTDDI_VERSION < NTDDI_VISTA
1173 FUNCTION_MSG("NdisSystemProcessorCount = %d\n", NdisSystemProcessorCount());
1174 xi->rxpi = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG, NormalPoolPriority);
1175 #else
1176 FUNCTION_MSG("KeQueryActiveProcessorCount = %d\n", KeQueryActiveProcessorCount(NULL));
1177 xi->rxpi = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(packet_info_t) * KeQueryActiveProcessorCount(NULL), XENNET_POOL_TAG, NormalPoolPriority);
1178 #endif
1179 if (!xi->rxpi) {
1180 FUNCTION_MSG("ExAllocatePoolWithTagPriority failed\n");
1181 return FALSE;
1183 #if NTDDI_VERSION < NTDDI_VISTA
1184 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
1185 #else
1186 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * KeQueryActiveProcessorCount(NULL));
1187 #endif
1188 ret = stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
1189 if (!ret) {
1190 FUNCTION_MSG("Failed to allocate rx_pb_stack\n");
1191 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1192 return FALSE;
1194 ret = stack_new(&xi->rx_hb_stack, NET_RX_RING_SIZE * 4);
1195 if (!ret) {
1196 FUNCTION_MSG("Failed to allocate rx_hb_stack\n");
1197 stack_delete(xi->rx_pb_stack, NULL, NULL);
1198 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1199 return FALSE;
1202 xi->rx_id_free = NET_RX_RING_SIZE;
1203 xi->rx_outstanding = 0;
1205 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1206 xi->rx_ring_pbs[i] = NULL;
1209 #if NTDDI_VERSION < NTDDI_VISTA
1210 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4, PROTOCOL_RESERVED_SIZE_IN_PACKET);
1211 if (status != NDIS_STATUS_SUCCESS) {
1212 FUNCTION_MSG("NdisAllocatePacketPool failed with 0x%x\n", status);
1213 return FALSE;
1215 #else
1216 nbl_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1217 nbl_pool_parameters.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1218 nbl_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1219 nbl_pool_parameters.ProtocolId = NDIS_PROTOCOL_ID_DEFAULT;
1220 nbl_pool_parameters.fAllocateNetBuffer = FALSE;
1221 nbl_pool_parameters.ContextSize = 0;
1222 nbl_pool_parameters.PoolTag = XENNET_POOL_TAG;
1223 nbl_pool_parameters.DataSize = 0; /* NET_BUFFERS are always allocated separately */
1225 xi->rx_nbl_pool = NdisAllocateNetBufferListPool(xi->adapter_handle, &nbl_pool_parameters);
1226 if (!xi->rx_nbl_pool) {
1227 FUNCTION_MSG("NdisAllocateNetBufferListPool failed\n");
1228 return FALSE;
1231 nb_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1232 nb_pool_parameters.Header.Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1233 nb_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1234 nb_pool_parameters.PoolTag = XENNET_POOL_TAG;
1235 nb_pool_parameters.DataSize = 0; /* the buffers come from the ring */
1236 xi->rx_packet_pool = NdisAllocateNetBufferPool(xi->adapter_handle, &nb_pool_parameters);
1237 if (!xi->rx_packet_pool) {
1238 FUNCTION_MSG("NdisAllocateNetBufferPool (rx_packet_pool) failed\n");
1239 return FALSE;
1241 #endif
1242 XenNet_FillRing(xi);
1244 FUNCTION_EXIT();
1246 return TRUE;
1249 VOID
1250 XenNet_RxShutdown(xennet_info_t *xi) {
1251 KIRQL old_irql;
1252 UNREFERENCED_PARAMETER(xi);
1254 FUNCTION_ENTER();
1256 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1257 while (xi->rx_outstanding) {
1258 FUNCTION_MSG("Waiting for %d packets to be returned\n", xi->rx_outstanding);
1259 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1260 KeWaitForSingleObject(&xi->rx_idle_event, Executive, KernelMode, FALSE, NULL);
1261 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1263 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1265 XenNet_BufferFree(xi);
1267 stack_delete(xi->rx_pb_stack, NULL, NULL);
1268 stack_delete(xi->rx_hb_stack, NULL, NULL);
1271 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1273 #if NTDDI_VERSION < NTDDI_VISTA
1274 NdisFreePacketPool(xi->rx_packet_pool);
1275 #else
1276 NdisFreeNetBufferPool(xi->rx_packet_pool);
1277 NdisFreeNetBufferListPool(xi->rx_nbl_pool);
1278 #endif
1280 FUNCTION_EXIT();
1281 return;