--------------------- PatchSet 10353 Date: 2008/01/02 17:29:31 Author: adri Branch: s27_adri Tag: (none) Log: The first, very naive, implementation of the bufregion stuff in stmem. * Toss the appending logic in storeAppend() for now and just allocate a new buffer each call. Worry about growing buffer regions once this and other stuff has had the kinks worked out of it. * Convert the mem_node_ref stuff over to use the bufregion stuff. Eventually mem_node_ref's may become bufregion_t's; I'm not sure yet. Its entirely possible they won't become bufregion_t's.. * Remove some of the old assert()s in client_side.c which don't matter anymore. Rework one other one to make sure ther'es at least the data required. * Enforce a returned 'size' limit on the store copy callback - I was returning the size of the available data in that buffer rather than capping at SM_PAGE_SIZE. Some code may -require- data chunks of that length (eg I think netdb? or asn?) and I dont' want to break them. I'll modify client_side.c later on to issue copies for up to $LARGE size (say 64k). Ideally the storeAppend() interface will grow back the "append data to existing buffer if possible" logic one other bugs have been identified and fixed. But I'm concentrating on getting reference-counted data into the store right now and the next big move will be modifying String's to take buffer regions rather than straight buf_t's (and figure out how to allow buffer regions to be read-only or read-write.) Thats a prerequisite for reference-based parsing. Members: src/client_side.c:1.202.2.9.4.23->1.202.2.9.4.24 src/defines.h:1.43.8.4->1.43.8.5 src/squid.h:1.36.24.9->1.36.24.10 src/stmem.c:1.10.2.4.4.6->1.10.2.4.4.7 src/store_client.c:1.25.10.5.4.3->1.25.10.5.4.4 src/store_swapout.c:1.22.24.3->1.22.24.4 src/structs.h:1.158.2.5.4.11->1.158.2.5.4.12 Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.202.2.9.4.23 retrieving revision 1.202.2.9.4.24 diff -u -r1.202.2.9.4.23 -r1.202.2.9.4.24 --- squid/src/client_side.c 28 Dec 2007 04:05:37 -0000 1.202.2.9.4.23 +++ squid/src/client_side.c 2 Jan 2008 17:29:31 -0000 1.202.2.9.4.24 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.202.2.9.4.23 2007/12/28 04:05:37 adri Exp $ + * $Id: client_side.c,v 1.202.2.9.4.24 2008/01/02 17:29:31 adri Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -3031,7 +3031,6 @@ int fd = conn->fd; MemBuf mb; debug(33, 5) ("clientSendMoreData: %s, %d bytes\n", http->uri, (int) size); - assert(size + ref.offset <= SM_PAGE_SIZE); assert(size <= SM_PAGE_SIZE); assert(http->request != NULL); dlinkDelete(&http->active, &ClientActiveRequests); @@ -3061,7 +3060,7 @@ return; } assert(! MEMREF_ISNULL(ref)); - assert(MEMREF_LEN(ref) <= size); + assert(MEMREF_LEN(ref) >= size); buf = MEMREF_BUF(ref); if (!http->request->range && !http->request->flags.chunked_response) { /* Avoid copying to MemBuf for non-range requests */ Index: squid/src/defines.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/defines.h,v retrieving revision 1.43.8.4 retrieving revision 1.43.8.5 diff -u -r1.43.8.4 -r1.43.8.5 --- squid/src/defines.h 28 Dec 2007 08:03:34 -0000 1.43.8.4 +++ squid/src/defines.h 2 Jan 2008 17:29:33 -0000 1.43.8.5 @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.43.8.4 2007/12/28 08:03:34 adri Exp $ + * $Id: defines.h,v 1.43.8.5 2008/01/02 17:29:33 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -356,8 +356,8 @@ #define LOGFILE_SEQNO(n) ( (n)->sequence_number ) -#define MEMREF_BUF(r) ( buf_buf((r).node->buf) + (r).offset ) -#define MEMREF_LEN(r) ( buf_len((r).node->buf) - (r).offset ) -#define MEMREF_ISNULL(r) ( (r).node == NULL && (r).node->buf == NULL ) +#define MEMREF_BUF(r) ( bufRegionBuf( (r).node->br ) + (r).offset ) +#define MEMREF_LEN(r) ( bufRegionLen( (r).node->br ) - (r).offset ) +#define MEMREF_ISNULL(r) ( (r).node == NULL && bufRegionIsNull( (r).node->br ) ) #endif /* SQUID_DEFINES_H */ Index: squid/src/squid.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/squid.h,v retrieving revision 1.36.24.9 retrieving revision 1.36.24.10 diff -u -r1.36.24.9 -r1.36.24.10 --- squid/src/squid.h 17 Dec 2007 08:17:51 -0000 1.36.24.9 +++ squid/src/squid.h 2 Jan 2008 17:29:33 -0000 1.36.24.10 @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.36.24.9 2007/12/17 08:17:51 adri Exp $ + * $Id: squid.h,v 1.36.24.10 2008/01/02 17:29:33 adri Exp $ * * AUTHOR: Duane Wessels * @@ -380,6 +380,7 @@ /* libbuf */ #include "../libbuf/buf.h" +#include "../libbuf/bufregion.h" #include "../libbuf/StringMap.h" #include "../libbuf/String.h" Index: squid/src/stmem.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/stmem.c,v retrieving revision 1.10.2.4.4.6 retrieving revision 1.10.2.4.4.7 diff -u -r1.10.2.4.4.6 -r1.10.2.4.4.7 --- squid/src/stmem.c 29 Dec 2007 07:21:43 -0000 1.10.2.4.4.6 +++ squid/src/stmem.c 2 Jan 2008 17:29:33 -0000 1.10.2.4.4.7 @@ -1,6 +1,6 @@ /* - * $Id: stmem.c,v 1.10.2.4.4.6 2007/12/29 07:21:43 adri Exp $ + * $Id: stmem.c,v 1.10.2.4.4.7 2008/01/02 17:29:33 adri Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Harvest Derived @@ -39,7 +39,7 @@ stmemNodeFree(mem_node *p) { if (!p->uses) { - p->buf = buf_deref(p->buf); + bufregion_free(&p->br); memFree(p, MEM_MEM_NODE); } else @@ -52,7 +52,7 @@ mem_node *p; while ((p = mem->head)) { mem->head = p->next; - store_mem_size -= buf_capacity(p->buf); + store_mem_size -= buf_capacity(p->br.b); stmemNodeFree(p); } mem->head = mem->tail = NULL; @@ -64,7 +64,7 @@ { squid_off_t current_offset = mem->origin_offset; mem_node *p = mem->head; - while (p && ((current_offset + buf_len(p->buf)) <= target_offset)) { + while (p && ((current_offset + bufRegionLen(p->br)) <= target_offset)) { if (p == mem->tail) { /* keep the last one to avoid change to other part of code */ mem->head = mem->tail; @@ -73,8 +73,8 @@ } else { mem_node *lastp = p; p = p->next; - current_offset += buf_len(lastp->buf); - store_mem_size -= buf_capacity(lastp->buf); + current_offset += bufRegionLen(lastp->br); + store_mem_size -= buf_capacity(lastp->br.b); stmemNodeFree(lastp); } } @@ -108,31 +108,28 @@ stmemAppend(mem_hdr * mem, const char *data, int len) { mem_node *p; - int avail_len; - int len_to_copy; + buf_t *b; debug(19, 6) ("memAppend: len %d\n", len); - /* Does the last block still contain empty space? - * If so, fill out the block before dropping into the - * allocation loop */ - if (mem->head && mem->tail && buf_len(mem->tail->buf) < SM_PAGE_SIZE) { - avail_len = SM_PAGE_SIZE - buf_len(mem->tail->buf); - len_to_copy = XMIN(avail_len, len); - /* Just in case the buffer suddenly "becomes bigger" for some reason! */ - store_mem_size -= buf_capacity(mem->tail->buf); - buf_append(mem->tail->buf, data, len_to_copy, BF_NONE); - store_mem_size += buf_capacity(mem->tail->buf); - /* Adjust the ptr and len according to what was deposited in the page */ - data += len_to_copy; - len -= len_to_copy; - } - while (len > 0) { - len_to_copy = XMIN(len, SM_PAGE_SIZE); + + /* + * For now? Just create a new buffer of exactly the same size, copy into it, and then + * create a region for it. + */ + + /* Once this works? Create a buffer of ST_MEM_SIZE, copy into it, region it. + * If more Append() copy data comes along and the tail buffer is ours AND the buffer region + * matches the end of the used buffer, copy into the buffer and grow the reference. + * Else create a new buffer. + */ + p = memAllocate(MEM_MEM_NODE); /* This is a non-zero'ed buffer; make sure you fully initialise it */ - p->buf = buf_create_size(SM_PAGE_SIZE); p->next = NULL; p->uses = 0; - buf_append(p->buf, data, len_to_copy, BF_NONE); - store_mem_size += buf_capacity(p->buf); + b = buf_create_size(len); + buf_append(b, data, len, BF_NONE); + store_mem_size += buf_capacity(b); + bufregion_init_buf(&p->br, b, 0, len); + b = buf_deref(b); if (!mem->head) { /* The chain is empty */ mem->head = mem->tail = p; @@ -141,9 +138,6 @@ mem->tail->next = p; mem->tail = p; } - len -= len_to_copy; - data += len_to_copy; - } } /* @@ -172,8 +166,8 @@ if (p == NULL) return 0; /* Seek our way into store */ - while ((t_off + buf_len(p->buf)) <= offset) { - t_off += buf_len(p->buf); + while ((t_off + bufRegionLen(p->br)) <= offset) { + t_off += bufRegionLen(p->br); if (!p->next) { debug(19, 1) ("memRef: p->next == NULL\n"); return 0; @@ -187,19 +181,23 @@ r->offset = offset - t_off; assert(r->offset >= 0); assert(r->offset >= 0); - assert(buf_len(p->buf) + t_off - offset > 0); - debug(19, 3) ("memRef: returning node %p, offset %d, %d bytes\n", p, (int) r->offset, (int) (buf_len(p->buf) + t_off - offset)); - return buf_len(p->buf) + t_off - offset; + assert(bufRegionLen(p->br) + t_off - offset > 0); + debug(19, 3) ("memRef: returning node %p, offset %d, %d bytes\n", p, (int) r->offset, (int) (bufRegionLen(p->br) + t_off - offset)); + return bufRegionLen(p->br) + t_off - offset; } void stmemNodeRefCreate(mem_node_ref * r) { + buf_t *b; + assert(1==0); /* XXX this code shouldn't be called! */ assert(r->node == NULL); r->node = memAllocate(MEM_MEM_NODE); r->node->uses = 0; r->node->next = NULL; - r->node->buf = buf_create_size(SM_PAGE_SIZE); + b = buf_create_size(SM_PAGE_SIZE); + bufregion_init_buf(&r->node->br, b, 0, 0); + b = buf_deref(b); r->offset = 0; } Index: squid/src/store_client.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_client.c,v retrieving revision 1.25.10.5.4.3 retrieving revision 1.25.10.5.4.4 diff -u -r1.25.10.5.4.3 -r1.25.10.5.4.4 --- squid/src/store_client.c 28 Dec 2007 07:52:30 -0000 1.25.10.5.4.3 +++ squid/src/store_client.c 2 Jan 2008 17:29:33 -0000 1.25.10.5.4.4 @@ -1,6 +1,6 @@ /* - * $Id: store_client.c,v 1.25.10.5.4.3 2007/12/28 07:52:30 adri Exp $ + * $Id: store_client.c,v 1.25.10.5.4.4 2008/01/02 17:29:33 adri Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -386,7 +386,7 @@ } if (EBIT_TEST(e->flags, RELEASE_REQUEST)) storeSwapOutMaintainMemObject(e); - storeClientCallback(sc, sz); + storeClientCallback(sc, XMIN(sz, sc->copy_size)); return; } /* What the client wants is not in memory. Schedule a disk read */ @@ -403,10 +403,18 @@ if ((!sc->callback) && (!sc->new_callback)) { fatal("storeClientFileRead: missing a callback!\n"); } + assert(1==0); + + /* + * XXX We need to read into a buffer and then hand that buffer to storeClientReadHeader. + * XXX We shouldn't be reading into a reference like this. Thats such a hacky way of doing it. + * XXX This will have to be rethought! */ + + assert(!sc->flags.disk_io_pending); sc->flags.disk_io_pending = 1; assert(sc->node_ref.node == NULL); /* We should never, ever have a node here; or we'd leak! */ - stmemNodeRefCreate(&sc->node_ref); /* Creates an entry with reference count == 0 (which is "a reference" in that code) */ + //stmemNodeRefCreate(&sc->node_ref); /* Creates an entry with reference count == 0 (which is "a reference" in that code) */ /* XXX these macros should be const char! [ahc] */ if (mem->swap_hdr_sz == 0) { storeRead(sc->swapin_sio, Index: squid/src/store_swapout.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/store_swapout.c,v retrieving revision 1.22.24.3 retrieving revision 1.22.24.4 diff -u -r1.22.24.3 -r1.22.24.4 --- squid/src/store_swapout.c 28 Dec 2007 08:03:34 -0000 1.22.24.3 +++ squid/src/store_swapout.c 2 Jan 2008 17:29:34 -0000 1.22.24.4 @@ -1,6 +1,6 @@ /* - * $Id: store_swapout.c,v 1.22.24.3 2007/12/28 08:03:34 adri Exp $ + * $Id: store_swapout.c,v 1.22.24.4 2008/01/02 17:29:34 adri Exp $ * * DEBUG: section 20 Storage Manager Swapout Functions * AUTHOR: Duane Wessels @@ -277,7 +277,8 @@ * but we can look at this at a later date or whenever the code results * in bad swapouts, whichever happens first. :-) */ - swap_buf_len = buf_len(mem->swapout.memnode->buf); + fatal("swapout path isn't working yet; it needs to be converted!\n"); + //swap_buf_len = buf_len(mem->swapout.memnode->buf); debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); assert(swap_buf_len > 0); @@ -286,7 +287,6 @@ mem->swapout.queue_offset += swap_buf_len; /* XXX for now, this is disabled. the whole FREE * overloading to unref the stmem node is just plain * XXX stupid and must be fleshed out to be a proper completion callback. [ahc] */ - assert(1==0); #if 0 storeWrite(mem->swapout.sio, stmemNodeGet(mem->swapout.memnode), swap_buf_len, stmemNodeFree); #endif Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/structs.h,v retrieving revision 1.158.2.5.4.11 retrieving revision 1.158.2.5.4.12 diff -u -r1.158.2.5.4.11 -r1.158.2.5.4.12 --- squid/src/structs.h 29 Dec 2007 10:57:49 -0000 1.158.2.5.4.11 +++ squid/src/structs.h 2 Jan 2008 17:29:34 -0000 1.158.2.5.4.12 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.158.2.5.4.11 2007/12/29 10:57:49 adri Exp $ + * $Id: structs.h,v 1.158.2.5.4.12 2008/01/02 17:29:34 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1627,9 +1627,7 @@ struct _mem_node { /* This is a bit pointless considering that buffers already have their own reference counting; worry about that later */ - buf_t *buf; - int offset; - int length; + bufregion_t br; int uses; mem_node *next; };