This patch is generated from the modio branch of HEAD in squid Sun Jan 25 14:36:32 2004 GMT See http://devel.squid-cache.org/ Index: squid/include/version.h diff -u squid/include/version.h:1.4 squid/include/version.h:1.2.2.2 --- squid/include/version.h:1.4 Tue Nov 14 05:03:47 2000 +++ squid/include/version.h Wed Dec 20 07:27:39 2000 @@ -4,7 +4,7 @@ * SQUID_VERSION - String for version id of this distribution */ #ifndef SQUID_VERSION -#define SQUID_VERSION "2.5.DEVEL" +#define SQUID_VERSION "2.5.DEVEL-modio" #endif #ifndef SQUID_RELEASE_TIME Index: squid/src/HttpHdrRange.c diff -u squid/src/HttpHdrRange.c:1.6 squid/src/HttpHdrRange.c:1.2.2.4 --- squid/src/HttpHdrRange.c:1.6 Wed Feb 7 11:11:47 2001 +++ squid/src/HttpHdrRange.c Tue Feb 13 12:54:28 2001 @@ -467,7 +467,7 @@ assert(http); stringAppend(&b, full_appname_string, strlen(full_appname_string)); stringAppend(&b, ":", 1); - key = storeKeyText(http->entry->hash.key); + key = storeKeyPublic(storeKeyUrl(http->entry), METHOD_GET); stringAppend(&b, key, strlen(key)); return b; } Index: squid/src/HttpRequest.c diff -u squid/src/HttpRequest.c:1.6 squid/src/HttpRequest.c:1.2.2.5 --- squid/src/HttpRequest.c:1.6 Fri Jan 12 00:20:32 2001 +++ squid/src/HttpRequest.c Sun Mar 18 14:51:21 2001 @@ -35,10 +35,24 @@ #include "squid.h" +CBDATA_TYPE(request_t); + +static int request_initialised = 0; + +void +requestInit(void) +{ + if (request_initialised) + return; + + request_initialised = 1; + CBDATA_INIT_TYPE(request_t); +} + request_t * requestCreate(method_t method, protocol_t protocol, const char *urlpath) { - request_t *req = memAllocate(MEM_REQUEST_T); + request_t *req = cbdataAlloc(request_t); req->method = method; req->protocol = protocol; if (urlpath) @@ -51,6 +65,7 @@ return req; } + void requestDestroy(request_t * req) { @@ -64,9 +79,7 @@ httpHeaderClean(&req->header); if (req->cache_control) httpHdrCcDestroy(req->cache_control); - if (req->range) - httpHdrRangeDestroy(req->range); - memFree(req, MEM_REQUEST_T); + cbdataFree(req); } request_t * Index: squid/src/Makefile.in diff -u squid/src/Makefile.in:1.7 squid/src/Makefile.in:1.2.2.6 --- squid/src/Makefile.in:1.7 Sat Feb 10 08:49:03 2001 +++ squid/src/Makefile.in Tue Feb 13 12:54:29 2001 @@ -183,6 +183,8 @@ wais.o \ wccp.o \ whois.o \ + reply_internal.o \ + reply_network.o \ $(XTRA_OBJS) SNMP_OBJS = \ Index: squid/src/asn.c diff -u squid/src/asn.c:1.10 squid/src/asn.c:1.2.2.12 --- squid/src/asn.c:1.10 Sat Mar 3 02:44:31 2001 +++ squid/src/asn.c Sun Mar 18 14:33:31 2001 @@ -195,18 +195,19 @@ assert(NULL != req); asState->request = requestLink(req); if ((e = storeGetPublic(asres, METHOD_GET)) == NULL) { - e = storeCreateEntry(asres, asres, null_request_flags, METHOD_GET); - asState->sc = storeClientListAdd(e, asState); + /* I'm guessing that this should be private .. */ + e = storeCreateEntry(asres, asres, null_request_flags, METHOD_GET, + REPLY_OBJ_INTERNAL); + storeClientRegister(e, asState); fwdStart(-1, e, asState->request); } else { storeLockObject(e); - asState->sc = storeClientListAdd(e, asState); + storeClientRegister(e, asState); } asState->entry = e; asState->seen = 0; asState->offset = 0; - storeClientCopy(asState->sc, - e, + storeClientCopy(e, asState->seen, asState->offset, 4096, @@ -228,7 +229,7 @@ asStateFree(asState); return; } - if (size == 0 && e->mem_obj->inmem_hi > 0) { + if (size == 0 && storeMemHiOffset(e) > 0) { memFree(buf, MEM_4K_BUF); asStateFree(asState); return; @@ -267,18 +268,16 @@ asState->seen, asState->offset); if (e->store_status == STORE_PENDING) { debug(53, 3) ("asHandleReply: store_status == STORE_PENDING: %s\n", storeUrl(e)); - storeClientCopy(asState->sc, - e, + storeClientCopy(e, asState->seen, asState->offset, 4096, buf, asHandleReply, asState); - } else if (asState->seen < e->mem_obj->inmem_hi) { - debug(53, 3) ("asHandleReply: asState->seen < e->mem_obj->inmem_hi %s\n", storeUrl(e)); - storeClientCopy(asState->sc, - e, + } else if (asState->seen < storeMemHiOffset(e)) { + debug(53, 3) ("asHandleReply: asState->seen < storeMemHiOffset(e) %s\n", storeUrl(e)); + storeClientCopy(e, asState->seen, asState->offset, 4096, @@ -297,7 +296,7 @@ { ASState *asState = data; debug(53, 3) ("asnStateFree: %s\n", storeUrl(asState->entry)); - storeUnregister(asState->sc, asState->entry, asState); + storeClientUnregister(asState->entry, asState); storeUnlockObject(asState->entry); requestUnlink(asState->request); cbdataFree(asState); Index: squid/src/cbdata.c diff -u squid/src/cbdata.c:1.10 squid/src/cbdata.c:1.2.2.6 --- squid/src/cbdata.c:1.10 Sat Mar 3 02:44:31 2001 +++ squid/src/cbdata.c Sun Mar 18 14:51:21 2001 @@ -134,7 +134,6 @@ #define CREATE_CBDATA_FREE(type, free_func) cbdataInitType(CBDATA_##type, #type, sizeof(type), free_func) CREATE_CBDATA(acl_access); CREATE_CBDATA(aclCheck_t); - CREATE_CBDATA(clientHttpRequest); CREATE_CBDATA(ConnStateData); CREATE_CBDATA(ErrorState); CREATE_CBDATA(FwdState); Index: squid/src/client_side.c diff -u squid/src/client_side.c:1.24 squid/src/client_side.c:1.2.2.25 --- squid/src/client_side.c:1.24 Sun Mar 4 05:45:04 2001 +++ squid/src/client_side.c Sun Mar 18 14:51:21 2001 @@ -78,7 +78,6 @@ /* Local functions */ static CWCB clientWriteComplete; -static CWCB clientWriteBodyComplete; static PF clientReadRequest; static PF connStateFree; static PF requestTimeout; @@ -103,8 +102,6 @@ static STCB clientSendMoreData; static STCB clientCacheHit; static void clientSetKeepaliveFlag(clientHttpRequest *); -static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); -static void clientPackTermBound(String boundary, MemBuf * mb); static void clientInterpretRequestHeaders(clientHttpRequest *); static void clientProcessRequest(clientHttpRequest *); static void clientProcessExpired(void *data); @@ -113,11 +110,27 @@ static int clientHierarchical(clientHttpRequest * http); static int clientCheckContentLength(request_t * r); static DEFER httpAcceptDefer; -static log_type clientProcessRequest2(clientHttpRequest * http); +static log_type clientProcessRequest2(clientHttpRequest * http, StoreEntry *e); static int clientReplyBodyTooLarge(int clen); static int clientRequestBodyTooLarge(int clen); +static void httpMemFree(void *); +static STGETDONE clientPurgeRequest; +static STGETDONE clientLookupDone; +static STGETDONE clientLookupHeadDone; +static void clientProcessRequestDone(clientHttpRequest *http); static void clientProcessBody(ConnStateData * conn); + +CBDATA_TYPE(clientHttpRequest); + +static void httpMemFree(void *data) +{ + clientHttpRequest *h = data; + + memFree(h->buf, MEM_CLIENT_SOCK_BUF); +} + + static int checkAccelOnly(clientHttpRequest * http) { @@ -192,7 +205,8 @@ } StoreEntry * -clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags) +clientCreateStoreEntry(clientHttpRequest * h, method_t m, request_flags flags, + reply_obj_t reply_obj) { StoreEntry *e; /* @@ -201,13 +215,13 @@ */ if (h->request == NULL) h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); - e = storeCreateEntry(h->uri, h->log_uri, flags, m); - h->sc = storeClientListAdd(e, h); + e = storeCreateEntry(h->uri, h->log_uri, flags, m, reply_obj); + storeClientRegister(e, h); #if DELAY_POOLS delaySetStoreClient(h->sc, delayClient(h->request)); #endif - storeClientCopy(h->sc, e, 0, 0, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), clientSendMoreData, h); + h->bufofs = 0; + storeClientCopy(e, 0, 0, CLIENT_SOCK_SZ, h->buf, clientSendMoreData, h); return e; } @@ -246,7 +260,7 @@ page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName); http->log_type = LOG_TCP_DENIED; http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); + null_request_flags, REPLY_OBJ_INTERNAL); if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) { if (!http->flags.accel) { /* Proxy authorisation needed */ @@ -367,19 +381,19 @@ } http->request->flags.refresh = 1; http->old_entry = http->entry; - http->old_sc = http->sc; /* * Assert that 'http' is already a client of old_entry. If * it is not, then the beginning of the object data might get * freed from memory before we need to access it. */ - assert(http->sc->callback_data == http); + assert(http->entry->sc.callback_data == http); entry = storeCreateEntry(url, http->log_uri, http->request->flags, - http->request->method); + http->request->method, + REPLY_OBJ_NETWORK); /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ - http->sc = storeClientListAdd(entry, http); + storeClientRegister(entry, http); #if DELAY_POOLS /* delay_id is already set on original store client */ delaySetStoreClient(http->sc, delayClient(http->request)); @@ -392,11 +406,12 @@ /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientHandleIMSReply, http); } @@ -452,11 +467,9 @@ http_status status; debug(33, 3) ("clientHandleIMSReply: %s, %d bytes\n", url, (int) size); if (entry == NULL) { - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } mem = entry->mem_obj; @@ -466,10 +479,9 @@ /* We have an existing entry, but failed to validate it */ /* Its okay to send the old one anyway */ http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeUnregister(http->sc, entry, http); + storeClientUnregister(entry, http); storeUnlockObject(entry); entry = http->entry = http->old_entry; - http->sc = http->old_sc; } else if (STORE_PENDING == entry->store_status && 0 == status) { debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); if (size >= CLIENT_SOCK_SZ) { @@ -477,17 +489,18 @@ debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); /* use old entry, this repeats the code abovez */ http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeUnregister(http->sc, entry, http); + storeClientUnregister(entry, http); storeUnlockObject(entry); entry = http->entry = http->old_entry; - http->sc = http->old_sc; /* continue */ } else { - storeClientCopy(http->sc, entry, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->bufofs += size; + assert(http->bufofs <= CLIENT_SOCK_SZ); + storeClientCopy(entry, + http->out.offset + http->bufofs, + http->out.offset + http->bufofs, + CLIENT_SOCK_SZ - http->bufofs, + buf + http->bufofs, clientHandleIMSReply, http); return; @@ -508,8 +521,7 @@ * not the body they refer to. */ httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); storeTimestampsSet(oldentry); - storeUnregister(http->sc, entry, http); - http->sc = http->old_sc; + storeClientUnregister(entry, http); storeUnlockObject(entry); entry = http->entry = oldentry; entry->timestamp = squid_curtime; @@ -526,15 +538,15 @@ storeTimestampsSet(http->old_entry); http->log_type = LOG_TCP_REFRESH_HIT; } - storeUnregister(http->old_sc, http->old_entry, http); + storeClientUnregister(http->old_entry, http); storeUnlockObject(http->old_entry); recopy = 0; } http->old_entry = NULL; /* done with old_entry */ - http->old_sc = NULL; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); if (recopy) { - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, @@ -581,27 +593,38 @@ } void -clientPurgeRequest(clientHttpRequest * http) +clientStartPurgeRequest(clientHttpRequest * http) { - StoreEntry *entry; ErrorState *err = NULL; - HttpReply *r; - http_status status; - http_version_t version; + debug(33, 3) ("Config2.onoff.enable_purge = %d\n", Config2.onoff.enable_purge); if (!Config2.onoff.enable_purge) { http->log_type = LOG_TCP_DENIED; err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); err->request = requestLink(http->request); err->src_addr = http->conn->peer.sin_addr; - http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); return; } http->log_type = LOG_TCP_MISS; /* Release both IP and object cache entries */ ipcacheInvalidate(http->request->host); - if ((entry = storeGetPublic(http->uri, METHOD_GET)) == NULL) { + /* Initiate a lookup to try and get said object */ + storeDoubleCheckRequest(http->request, http->uri, METHOD_GET); + storeLookup(http->request, clientPurgeRequest, http); +} + +static void +clientPurgeRequest(void *data, StoreEntry *entry) +{ + clientHttpRequest *http = data; + HttpReply *r; + http_status status; + http_version_t version; + + if (entry == NULL) { status = HTTP_NOT_FOUND; } else { storeRelease(entry); @@ -613,7 +636,8 @@ * Make a new entry to hold the reply to be written * to the client. */ - http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); httpReplyReset(r = http->entry->mem_obj->reply); httpBuildVersion(&version, 1, 0); httpReplySetHeaders(r, version, status, NULL, NULL, 0, 0, -1); @@ -718,7 +742,7 @@ */ if ((e = http->entry)) { http->entry = NULL; - storeUnregister(http->sc, e, http); + storeClientUnregister(e, http); storeUnlockObject(e); } #endif @@ -773,19 +797,16 @@ safe_free(http->al.headers.request); safe_free(http->al.headers.reply); safe_free(http->redirect.location); - stringClean(&http->range_iter.boundary); if ((e = http->entry)) { http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); } /* old_entry might still be set if we didn't yet get the reply * code in clientHandleIMSReply() */ if ((e = http->old_entry)) { http->old_entry = NULL; - storeUnregister(http->old_sc, e, http); - http->old_sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); } requestUnlink(http->request); @@ -888,12 +909,15 @@ #endif request->flags.nocache = 1; } + /* ignore range header in non-GETs */ if (request->method == METHOD_GET) { - request->range = httpHeaderGetRange(req_hdr); - if (request->range) - request->flags.range = 1; + if (httpHeaderHas(req_hdr, HDR_RANGE)) + request->flags.range = 1; + if (httpHeaderHas(req_hdr, HDR_REQUEST_RANGE)) + request->flags.range = 1; } + if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) request->flags.auth = 1; if (request->login[0] != '\0') @@ -1069,149 +1093,6 @@ return 0; } -/* - * returns true if If-Range specs match reply, false otherwise - */ -static int -clientIfRangeMatch(clientHttpRequest * http, HttpReply * rep) -{ - const TimeOrTag spec = httpHeaderGetTimeOrTag(&http->request->header, HDR_IF_RANGE); - /* check for parsing falure */ - if (!spec.valid) - return 0; - /* got an ETag? */ - if (spec.tag.str) { - ETag rep_tag = httpHeaderGetETag(&rep->header, HDR_ETAG); - debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n", - spec.tag.str, rep_tag.str ? rep_tag.str : ""); - if (!rep_tag.str) - return 0; /* entity has no etag to compare with! */ - if (spec.tag.weak || rep_tag.weak) { - debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n", - spec.tag.str, rep_tag.str); - return 0; /* must use strong validator for sub-range requests */ - } - return etagIsEqual(&rep_tag, &spec.tag); - } - /* got modification time? */ - if (spec.time >= 0) { - return http->entry->lastmod <= spec.time; - } - assert(0); /* should not happen */ - return 0; -} - -/* returns expected content length for multi-range replies - * note: assumes that httpHdrRangeCanonize has already been called - * warning: assumes that HTTP headers for individual ranges at the - * time of the actuall assembly will be exactly the same as - * the headers when clientMRangeCLen() is called */ -static int -clientMRangeCLen(clientHttpRequest * http) -{ - int clen = 0; - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec; - MemBuf mb; - - assert(http->entry->mem_obj); - - memBufDefInit(&mb); - while ((spec = httpHdrRangeGetSpec(http->request->range, &pos))) { - - /* account for headers for this range */ - memBufReset(&mb); - clientPackRangeHdr(http->entry->mem_obj->reply, - spec, http->range_iter.boundary, &mb); - clen += mb.size; - - /* account for range content */ - clen += spec->length; - - debug(33, 6) ("clientMRangeCLen: (clen += %d + %d) == %d\n", - mb.size, spec->length, clen); - } - /* account for the terminating boundary */ - memBufReset(&mb); - clientPackTermBound(http->range_iter.boundary, &mb); - clen += mb.size; - - memBufClean(&mb); - return clen; -} - -/* adds appropriate Range headers if needed */ -static void -clientBuildRangeHeader(clientHttpRequest * http, HttpReply * rep) -{ - HttpHeader *hdr = rep ? &rep->header : 0; - const char *range_err = NULL; - request_t *request = http->request; - assert(request->range); - /* check if we still want to do ranges */ - if (!rep) - range_err = "no [parse-able] reply"; - else if (rep->sline.status != HTTP_OK) - range_err = "wrong status code"; - else if (httpHeaderHas(hdr, HDR_CONTENT_RANGE)) - range_err = "origin server does ranges"; - else if (rep->content_length < 0) - range_err = "unknown length"; - else if (rep->content_length != http->entry->mem_obj->reply->content_length) - range_err = "INCONSISTENT length"; /* a bug? */ - else if (httpHeaderHas(&http->request->header, HDR_IF_RANGE) && !clientIfRangeMatch(http, rep)) - range_err = "If-Range match failed"; - else if (!httpHdrRangeCanonize(http->request->range, rep->content_length)) - range_err = "canonization failed"; - else if (httpHdrRangeIsComplex(http->request->range)) - range_err = "too complex range header"; - else if (!request->flags.cachable) /* from we_do_ranges in http.c */ - range_err = "non-cachable request"; - /* get rid of our range specs on error */ - if (range_err) { - debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err); - httpHdrRangeDestroy(http->request->range); - http->request->range = NULL; - } else { - const int spec_count = http->request->range->specs.count; - int actual_clen = -1; - - debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n", - spec_count, rep->content_length); - assert(spec_count > 0); - /* ETags should not be returned with Partial Content replies? */ - httpHeaderDelById(hdr, HDR_ETAG); - /* append appropriate header(s) */ - if (spec_count == 1) { - HttpHdrRangePos pos = HttpHdrRangeInitPos; - const HttpHdrRangeSpec *spec = httpHdrRangeGetSpec(http->request->range, &pos); - assert(spec); - /* append Content-Range */ - httpHeaderAddContRange(hdr, *spec, rep->content_length); - /* set new Content-Length to the actual number of bytes - * transmitted in the message-body */ - actual_clen = spec->length; - } else { - /* multipart! */ - /* generate boundary string */ - http->range_iter.boundary = httpHdrRangeBoundaryStr(http); - /* delete old Content-Type, add ours */ - httpHeaderDelById(hdr, HDR_CONTENT_TYPE); - httpHeaderPutStrf(hdr, HDR_CONTENT_TYPE, - "multipart/byteranges; boundary=\"%s\"", - strBuf(http->range_iter.boundary)); - /* Content-Length is not required in multipart responses - * but it is always nice to have one */ - actual_clen = clientMRangeCLen(http); - } - - /* replace Content-Length header */ - assert(actual_clen >= 0); - httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); - httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, actual_clen); - debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen); - } -} /* * filters out unwanted entries from original reply header @@ -1256,9 +1137,6 @@ httpHeaderDelById(hdr, HDR_CONNECTION); stringClean(&strConnection); } - /* Handle Ranges */ - if (request->range) - clientBuildRangeHeader(http, rep); /* * Add a estimated Age header on cache hits. */ @@ -1331,19 +1209,15 @@ httpBuildVersion(&rep->sline.version, 1, 0); /* do header conversions */ clientBuildReplyHeader(http, rep); + /* if we do ranges, change status to "Partial Content" */ - if (http->request->range) + if (http->request->flags.range) httpStatusLineSet(&rep->sline, rep->sline.version, HTTP_PARTIAL_CONTENT, NULL); } else { /* parsing failure, get rid of the invalid reply */ httpReplyDestroy(rep); rep = NULL; - /* if we were going to do ranges, backoff */ - if (http->request->range) { - /* this will fail and destroy request->range */ - clientBuildRangeHeader(http, rep); - } } return rep; } @@ -1364,18 +1238,15 @@ request_t *r = http->request; debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); if (http->entry == NULL) { - memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: request aborted\n"); return; } else if (size < 0) { /* swap in failure */ - memFree(buf, MEM_CLIENT_SOCK_BUF); debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); http->log_type = LOG_TCP_SWAPFAIL_MISS; if ((e = http->entry)) { http->entry = NULL; - storeUnregister(http->sc, e, http); - http->sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); } clientProcessMiss(http); @@ -1390,18 +1261,18 @@ * punt to clientProcessMiss. */ if (e->mem_status == IN_MEMORY || e->store_status == STORE_OK) { - memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); } else if (size == CLIENT_SOCK_SZ && http->out.offset == 0) { - memFree(buf, MEM_CLIENT_SOCK_BUF); clientProcessMiss(http); } else { debug(33, 3) ("clientCacheHit: waiting for HTTP reply headers\n"); - storeClientCopy(http->sc, e, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->bufofs += size; + assert(http->bufofs <= CLIENT_SOCK_SZ); + storeClientCopy(e, + http->out.offset + http->bufofs, + http->out.offset + http->bufofs, + CLIENT_SOCK_SZ - http->bufofs, + buf + http->bufofs, clientCacheHit, http); } @@ -1465,7 +1336,6 @@ http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } - memFree(buf, MEM_CLIENT_SOCK_BUF); } else if (r->flags.ims) { /* * Handle If-Modified-Since requests from the client @@ -1473,7 +1343,6 @@ if (mem->reply->sline.status != HTTP_OK) { debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", mem->reply->sline.status); - memFree(buf, MEM_CLIENT_SOCK_BUF); http->log_type = LOG_TCP_MISS; clientProcessMiss(http); } else if (modifiedSince(e, http->request)) { @@ -1483,11 +1352,10 @@ time_t timestamp = e->timestamp; MemBuf mb = httpPacked304Reply(e->mem_obj->reply); http->log_type = LOG_TCP_IMS_HIT; - memFree(buf, MEM_CLIENT_SOCK_BUF); - storeUnregister(http->sc, e, http); - http->sc = NULL; + storeClientUnregister(e, http); storeUnlockObject(e); - e = clientCreateStoreEntry(http, http->request->method, null_request_flags); + e = clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); /* * Copy timestamp from the original entry so the 304 * reply has a meaningful Age: header. @@ -1511,168 +1379,6 @@ } } -/* put terminating boundary for multiparts */ -static void -clientPackTermBound(String boundary, MemBuf * mb) -{ - memBufPrintf(mb, "\r\n--%s--\r\n", strBuf(boundary)); - debug(33, 6) ("clientPackTermBound: buf offset: %d\n", mb->size); -} - -/* appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */ -static void -clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb) -{ - HttpHeader hdr; - Packer p; - assert(rep); - assert(spec); - - /* put boundary */ - debug(33, 5) ("clientPackRangeHdr: appending boundary: %s\n", strBuf(boundary)); - /* rfc2046 requires to _prepend_ boundary with ! */ - memBufPrintf(mb, "\r\n--%s\r\n", strBuf(boundary)); - - /* stuff the header with required entries and pack it */ - httpHeaderInit(&hdr, hoReply); - if (httpHeaderHas(&rep->header, HDR_CONTENT_TYPE)) - httpHeaderPutStr(&hdr, HDR_CONTENT_TYPE, httpHeaderGetStr(&rep->header, HDR_CONTENT_TYPE)); - httpHeaderAddContRange(&hdr, *spec, rep->content_length); - packerToMemInit(&p, mb); - httpHeaderPackInto(&hdr, &p); - packerClean(&p); - httpHeaderClean(&hdr); - - /* append (we packed a header, not a reply) */ - memBufPrintf(mb, crlf); -} - -/* - * extracts a "range" from *buf and appends them to mb, updating - * all offsets and such. - */ -static void -clientPackRange(clientHttpRequest * http, - HttpHdrRangeIter * i, - const char **buf, - ssize_t * size, - MemBuf * mb) -{ - const ssize_t copy_sz = i->debt_size <= *size ? i->debt_size : *size; - off_t body_off = http->out.offset - i->prefix_size; - assert(*size > 0); - assert(i->spec); - /* - * intersection of "have" and "need" ranges must not be empty - */ - assert(body_off < i->spec->offset + i->spec->length); - assert(body_off + *size > i->spec->offset); - /* - * put boundary and headers at the beginning of a range in a - * multi-range - */ - if (http->request->range->specs.count > 1 && i->debt_size == i->spec->length) { - assert(http->entry->mem_obj); - clientPackRangeHdr( - http->entry->mem_obj->reply, /* original reply */ - i->spec, /* current range */ - i->boundary, /* boundary, the same for all */ - mb - ); - } - /* - * append content - */ - debug(33, 3) ("clientPackRange: appending %d bytes\n", copy_sz); - memBufAppend(mb, *buf, copy_sz); - /* - * update offsets - */ - *size -= copy_sz; - i->debt_size -= copy_sz; - body_off += copy_sz; - *buf += copy_sz; - http->out.offset = body_off + i->prefix_size; /* sync */ - /* - * paranoid check - */ - assert(*size >= 0 && i->debt_size >= 0); -} - -/* returns true if there is still data available to pack more ranges - * increments iterator "i" - * used by clientPackMoreRanges */ -static int -clientCanPackMoreRanges(const clientHttpRequest * http, HttpHdrRangeIter * i, ssize_t size) -{ - /* first update "i" if needed */ - if (!i->debt_size) { - if ((i->spec = httpHdrRangeGetSpec(http->request->range, &i->pos))) - i->debt_size = i->spec->length; - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - /* continue condition: need_more_data && have_more_data */ - return i->spec && size > 0; -} - -/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */ -/* returns true if we need more data */ -static int -clientPackMoreRanges(clientHttpRequest * http, const char *buf, ssize_t size, MemBuf * mb) -{ - HttpHdrRangeIter *i = &http->range_iter; - /* offset in range specs does not count the prefix of an http msg */ - off_t body_off = http->out.offset - i->prefix_size; - assert(size >= 0); - /* check: reply was parsed and range iterator was initialized */ - assert(i->prefix_size > 0); - /* filter out data according to range specs */ - while (clientCanPackMoreRanges(http, i, size)) { - off_t start; /* offset of still missing data */ - assert(i->spec); - start = i->spec->offset + i->spec->length - i->debt_size; - debug(33, 3) ("clientPackMoreRanges: in: offset: %d size: %d\n", - (int) body_off, size); - debug(33, 3) ("clientPackMoreRanges: out: start: %d spec[%d]: [%d, %d), len: %d debt: %d\n", - (int) start, (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length, i->debt_size); - assert(body_off <= start); /* we did not miss it */ - /* skip up to start */ - if (body_off + size > start) { - const size_t skip_size = start - body_off; - body_off = start; - size -= skip_size; - buf += skip_size; - } else { - /* has not reached start yet */ - body_off += size; - size = 0; - buf = NULL; - } - /* put next chunk if any */ - if (size) { - http->out.offset = body_off + i->prefix_size; /* sync */ - clientPackRange(http, i, &buf, &size, mb); - body_off = http->out.offset - i->prefix_size; /* sync */ - } - } - assert(!i->debt_size == !i->spec); /* paranoid sync condition */ - debug(33, 3) ("clientPackMoreRanges: buf exhausted: in: offset: %d size: %d need_more: %d\n", - (int) body_off, size, i->debt_size); - if (i->debt_size) { - debug(33, 3) ("clientPackMoreRanges: need more: spec[%d]: [%d, %d), len: %d\n", - (int) i->pos, i->spec->offset, (int) (i->spec->offset + i->spec->length), i->spec->length); - /* skip the data we do not need if possible */ - if (i->debt_size == i->spec->length) /* at the start of the cur. spec */ - body_off = i->spec->offset; - else - assert(body_off == i->spec->offset + i->spec->length - i->debt_size); - } else if (http->request->range->specs.count > 1) { - /* put terminating boundary for multiparts */ - clientPackTermBound(i->boundary, mb); - } - http->out.offset = body_off + i->prefix_size; /* sync */ - return i->debt_size > 0; -} static int clientReplyBodyTooLarge(int clen) @@ -1752,22 +1458,18 @@ if (conn->chr != http) { /* there is another object in progress, defer this one */ debug(33, 1) ("clientSendMoreData: Deferring %s\n", storeUrl(entry)); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size < 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } else if (size == 0) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - memFree(buf, MEM_CLIENT_SOCK_BUF); return; } if (http->out.offset == 0) { @@ -1783,11 +1485,10 @@ if (rep && clientReplyBodyTooLarge(rep->content_length)) { ErrorState *err = errorCon(ERR_TOO_BIG, HTTP_FORBIDDEN); err->request = requestLink(http->request); - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); httpReplyDestroy(rep); return; @@ -1815,11 +1516,11 @@ ErrorState *err; err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); err->request = requestLink(http->request); - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); - http->entry = clientCreateStoreEntry(http, http->request->method, - null_request_flags); + http->entry = + clientCreateStoreEntry(http, http->request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); httpReplyDestroy(rep); return; @@ -1827,24 +1528,17 @@ aclChecklistFree(ch); } else if (size < CLIENT_SOCK_SZ && entry->store_status == STORE_PENDING) { /* wait for more to arrive */ - storeClientCopy(http->sc, entry, - http->out.offset + size, - http->out.offset, - CLIENT_SOCK_SZ, - buf, + http->bufofs += size; + assert(http->bufofs <= CLIENT_SOCK_SZ); + storeClientCopy(entry, + http->out.offset + http->bufofs, + http->out.offset + http->bufofs, + CLIENT_SOCK_SZ - http->bufofs, + buf + http->bufofs, clientSendMoreData, http); return; } - /* reset range iterator */ - http->range_iter.pos = HttpHdrRangeInitPos; - } else if (!http->request->range) { - /* Avoid copying to MemBuf for non-range requests */ - /* Note, if we're here, then 'rep' is known to be NULL */ - http->out.offset += body_size; - comm_write(fd, buf, size, clientWriteBodyComplete, http, NULL); - /* NULL because clientWriteBodyComplete frees it */ - return; } if (http->request->method == METHOD_HEAD) { if (rep) { @@ -1879,42 +1573,18 @@ memBufDefInit(&mb); } /* append body if any */ - if (http->request->range) { - /* Only GET requests should have ranges */ - assert(http->request->method == METHOD_GET); - /* clientPackMoreRanges() updates http->out.offset */ - /* force the end of the transfer if we are done */ - if (!clientPackMoreRanges(http, body_buf, body_size, &mb)) - http->flags.done_copying = 1; - } else if (body_buf && body_size) { + if (body_buf && body_size) { http->out.offset += body_size; check_size += body_size; memBufAppend(&mb, body_buf, body_size); } - if (!http->request->range && http->request->method == METHOD_GET) + if (http->request->method == METHOD_GET) assert(check_size == size); /* write */ comm_write_mbuf(fd, mb, clientWriteComplete, http); /* if we don't do it, who will? */ - memFree(buf, MEM_CLIENT_SOCK_BUF); } -/* - * clientWriteBodyComplete is called for MEM_CLIENT_SOCK_BUF's - * written directly to the client socket, versus copying to a MemBuf - * and going through comm_write_mbuf. Most non-range responses after - * the headers probably go through here. - */ -static void -clientWriteBodyComplete(int fd, char *buf, size_t size, int errflag, void *data) -{ - /* - * NOTE: clientWriteComplete doesn't currently use its "buf" - * (second) argument, so we pass in NULL. - */ - clientWriteComplete(fd, NULL, size, errflag, data); - memFree(buf, MEM_CLIENT_SOCK_BUF); -} static void clientKeepaliveNextRequest(clientHttpRequest * http) @@ -1955,14 +1625,15 @@ debug(33, 1) ("clientKeepaliveNextRequest: FD %d Sending next\n", conn->fd); assert(entry); - if (0 == storeClientCopyPending(http->sc, entry, http)) { + if (0 == storeClientCopyPending(entry, http)) { if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientSendMoreData, http); } @@ -2017,11 +1688,12 @@ * storage manager. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientWriteComplete 2: ENTRY_ABORTED\n"); - storeClientCopy(http->sc, entry, + http->bufofs = 0; + storeClientCopy(entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientSendMoreData, http); } @@ -2046,60 +1718,68 @@ err->request = requestLink(r); err->src_addr = http->conn->peer.sin_addr; if (http->entry) { - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); } - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, r->method, null_request_flags, + REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); } -/* - * Return true if we should force a cache miss on this range request. - * entry must be non-NULL. + +/* + * clientLookupDone - handle the result of the first lookup + * + * If the result of the first lookup is a MISS and the method is METHOD_HEAD, + * we can generate a HEAD reply from a cached GET object, so initiate a + * second lookup. Otherwise, call clientProcessRequest2() to get the + * cache hit/miss information, and punt to clientProcessRequestDone(). */ -static int -clientCheckRangeForceMiss(StoreEntry * entry, HttpHdrRange * range) +static void +clientLookupDone(void *data, StoreEntry *e) { - /* - * If the range_offset_limit is NOT in effect, there - * is no reason to force a miss. - */ - if (0 == httpHdrRangeOffsetLimit(range)) - return 0; - /* - * Here, we know it's possibly a hit. If we already have the - * whole object cached, we won't force a miss. - */ - if (STORE_OK == entry->store_status) - return 0; /* we have the whole object */ - /* - * Now we have a hit on a PENDING object. We need to see - * if the part we want is already cached. If so, we don't - * force a miss. - */ - assert(NULL != entry->mem_obj); - if (httpHdrRangeFirstOffset(range) <= entry->mem_obj->inmem_hi) - return 0; - /* - * Even though we have a PENDING copy of the object, we - * don't want to wait to reach the first range offset, - * so we force a miss for a new range request to the - * origin. - */ - return 1; + clientHttpRequest *http = data; + + if (http->request->method == METHOD_HEAD && e == NULL) { + /* + * Start off the GET lookup, and change the request to be a + * HEAD afterwards + */ + http->request->method = METHOD_GET; + storeDoubleCheckRequest(http->request, http->uri, METHOD_GET); + storeLookup(http->request, clientLookupHeadDone, http); + http->request->method = METHOD_HEAD; + } else { + /* We have to continue with our result, NULL or not */ + http->log_type = clientProcessRequest2(http, e); + clientProcessRequestDone(http); + } + /* NOTREACHED */ } +/* + * clientLookupHeadDone - handle the HEAD -> GET request + * + * Similar to clientLookupDone(), we simply call clientProcessRequest2 + */ +static void +clientLookupHeadDone(void *data, StoreEntry *e) +{ + clientHttpRequest *http = data; + + http->log_type = clientProcessRequest2(http, e); + clientProcessRequestDone(http); +} + +/* + * decide whether we've got a cache HIT or MISS, and then punt across + * to clientProcessRequestDone() + */ static log_type -clientProcessRequest2(clientHttpRequest * http) +clientProcessRequest2(clientHttpRequest * http, StoreEntry *e) { request_t *r = http->request; - StoreEntry *e; - e = http->entry = storeGetPublic(http->uri, r->method); - if (r->method == METHOD_HEAD && e == NULL) { - /* We can generate a HEAD reply from a cached GET object */ - e = http->entry = storeGetPublic(http->uri, METHOD_GET); - } + /* Release negatively cached IP-cache entries on reload */ if (r->flags.nocache) ipcacheInvalidate(r->host); @@ -2112,7 +1792,7 @@ #endif if (NULL == e) { /* this object isn't in the cache */ - debug(33, 3) ("clientProcessRequest2: storeGet() MISS\n"); + debug(33, 3) ("clientProcessRequest2: storeLookup() MISS\n"); return LOG_TCP_MISS; } if (Config.onoff.offline) { @@ -2151,21 +1831,14 @@ http->entry = NULL; ipcacheInvalidate(r->host); return LOG_TCP_CLIENT_REFRESH_MISS; - } - if (NULL == r->range) { - (void) 0; - } else if (httpHdrRangeWillBeComplex(r->range)) { + } else if (r->flags.range) { /* * Some clients break if we return "200 OK" for a Range * request. We would have to return "200 OK" for a _complex_ * Range request that is also a HIT. Thus, let's prevent HITs * on complex Range requests */ - debug(33, 3) ("clientProcessRequest2: complex range MISS\n"); - http->entry = NULL; - return LOG_TCP_MISS; - } else if (clientCheckRangeForceMiss(e, r->range)) { - debug(33, 3) ("clientProcessRequest2: forcing miss due to range_offset_limit\n"); + debug(33, 3) ("clientProcessRequest2: range MISS\n"); http->entry = NULL; return LOG_TCP_MISS; } @@ -2190,11 +1863,12 @@ sslStart(fd, url, r, &http->out.size, &http->al.http.code); return; } else if (r->method == METHOD_PURGE) { - clientPurgeRequest(http); + clientStartPurgeRequest(http); return; } else if (r->method == METHOD_TRACE) { if (r->max_forwards == 0) { - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, r->method, + null_request_flags, REPLY_OBJ_INTERNAL); storeReleaseRequest(http->entry); storeBuffer(http->entry); rep = httpReplyCreate(); @@ -2210,25 +1884,44 @@ /* yes, continue */ http->log_type = LOG_TCP_MISS; } else { - http->log_type = clientProcessRequest2(http); + /* We need to initiate a lookup, so defer completing the request */ + debug(33, 3) ("clientProcessRequest: looking up %s\n", http->uri); + storeDoubleCheckRequest(r, http->uri, r->method); + storeLookup(r, clientLookupDone, http); + return; } - debug(33, 4) ("clientProcessRequest: %s for '%s'\n", + /* Continue processing */ + clientProcessRequestDone(http); +} + + +/* + * clientProcessRequestDone - the completion of the request chain + * + * Either start getting data from a cached object, or punt across + * to clientProcessMiss(). + */ +static void +clientProcessRequestDone(clientHttpRequest *http) +{ + debug(33, 4) ("clientProcessRequestDone: %s for '%s'\n", log_tags[http->log_type], http->uri); http->out.offset = 0; if (NULL != http->entry) { storeLockObject(http->entry); storeCreateMemObject(http->entry, http->uri, http->log_uri); - http->entry->mem_obj->method = r->method; - http->sc = storeClientListAdd(http->entry, http); + http->entry->mem_obj->method = http->request->method; + storeClientRegister(http->entry, http); #if DELAY_POOLS - delaySetStoreClient(http->sc, delayClient(r)); + delaySetStoreClient(http->sc, delayClient(http->request)); #endif - storeClientCopy(http->sc, http->entry, + http->bufofs = 0; + storeClientCopy(http->entry, http->out.offset, http->out.offset, CLIENT_SOCK_SZ, - memAllocate(MEM_CLIENT_SOCK_BUF), + http->buf, clientCacheHit, http); } else { @@ -2258,8 +1951,7 @@ debug(33, 0) ("\tlog_type = %s\n", log_tags[http->log_type]); storeEntryDump(http->entry, 1); } - storeUnregister(http->sc, http->entry, http); - http->sc = NULL; + storeClientUnregister(http->entry, http); storeUnlockObject(http->entry); http->entry = NULL; } @@ -2275,12 +1967,14 @@ err = errorCon(ERR_ACCESS_DENIED, HTTP_FORBIDDEN); err->request = requestLink(r); err->src_addr = http->conn->peer.sin_addr; - http->entry = clientCreateStoreEntry(http, r->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, r->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); return; } assert(http->out.offset == 0); - http->entry = clientCreateStoreEntry(http, r->method, r->flags); + http->entry = clientCreateStoreEntry(http, r->method, r->flags, + REPLY_OBJ_NETWORK); if (http->redirect.status) { HttpReply *rep = httpReplyCreate(); #if LOG_TCP_REDIRECTS @@ -2302,6 +1996,7 @@ parseHttpRequestAbort(ConnStateData * conn, const char *uri) { clientHttpRequest *http; + CBDATA_INIT_TYPE_FREECB(clientHttpRequest, httpMemFree); http = cbdataAlloc(clientHttpRequest); http->conn = conn; http->start = current_time; @@ -2309,6 +2004,7 @@ http->uri = xstrdup(uri); http->log_uri = xstrndup(uri, MAX_URL); http->range_iter.boundary = StringNull; + http->buf = memAllocate(MEM_CLIENT_SOCK_BUF); dlinkAdd(http, &http->active, &ClientActiveRequests); return http; } @@ -2441,12 +2137,14 @@ assert(prefix_sz <= conn->in.offset); /* Ok, all headers are received */ + CBDATA_INIT_TYPE_FREECB(clientHttpRequest, httpMemFree); http = cbdataAlloc(clientHttpRequest); http->http_ver = http_ver; http->conn = conn; http->start = current_time; http->req_sz = prefix_sz; http->range_iter.boundary = StringNull; + http->buf = memAllocate(MEM_CLIENT_SOCK_BUF); *prefix_p = xmalloc(prefix_sz + 1); xmemcpy(*prefix_p, conn->in.buf, prefix_sz); *(*prefix_p + prefix_sz) = '\0'; @@ -2719,7 +2417,8 @@ debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd); err = errorCon(ERR_INVALID_REQ, HTTP_BAD_REQUEST); err->request_hdrs = xstrdup(conn->in.buf); - http->entry = clientCreateStoreEntry(http, method, null_request_flags); + http->entry = clientCreateStoreEntry(http, method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); safe_free(prefix); break; @@ -2730,7 +2429,8 @@ err->src_addr = conn->peer.sin_addr; err->url = xstrdup(http->uri); http->al.http.code = err->http_status; - http->entry = clientCreateStoreEntry(http, method, null_request_flags); + http->entry = clientCreateStoreEntry(http, method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); safe_free(prefix); break; @@ -2774,7 +2474,8 @@ err->request = requestLink(request); request->flags.proxy_keepalive = 0; http->al.http.code = err->http_status; - http->entry = clientCreateStoreEntry(http, request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); break; } @@ -2783,7 +2484,8 @@ err->src_addr = conn->peer.sin_addr; err->request = requestLink(request); http->al.http.code = err->http_status; - http->entry = clientCreateStoreEntry(http, request->method, null_request_flags); + http->entry = clientCreateStoreEntry(http, request->method, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); break; } @@ -2798,7 +2500,7 @@ err = errorCon(ERR_TOO_BIG, HTTP_REQUEST_ENTITY_TOO_LARGE); err->request = requestLink(request); http->entry = clientCreateStoreEntry(http, - METHOD_NONE, null_request_flags); + METHOD_NONE, null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); break; } @@ -2823,7 +2525,8 @@ /* add to the client request queue */ for (H = &conn->chr; *H; H = &(*H)->next); *H = http; - http->entry = clientCreateStoreEntry(http, METHOD_NONE, null_request_flags); + http->entry = clientCreateStoreEntry(http, METHOD_NONE, + null_request_flags, REPLY_OBJ_INTERNAL); errorAppendEntry(http->entry, err); return; } Index: squid/src/enums.h diff -u squid/src/enums.h:1.17 squid/src/enums.h:1.2.2.13 --- squid/src/enums.h:1.17 Sat Mar 3 02:44:32 2001 +++ squid/src/enums.h Sun Mar 18 14:51:21 2001 @@ -487,7 +487,6 @@ KEY_PRIVATE, ENTRY_FWD_HDR_WAIT, ENTRY_NEGCACHED, - ENTRY_VALIDATED, ENTRY_BAD_LENGTH, ENTRY_ABORTED #if UNUSED_CODE @@ -634,7 +633,6 @@ MEM_PS_STATE, MEM_REFRESH_T, MEM_RELIST, - MEM_REQUEST_T, MEM_SQUIDCONFIG, MEM_SQUIDCONFIG2, MEM_STATCOUNTERS, @@ -726,6 +724,15 @@ }; /* + * reply object types + */ +typedef enum { + REPLY_OBJ_NONE, + REPLY_OBJ_INTERNAL, + REPLY_OBJ_NETWORK +} reply_obj_t; + +/* * cbdata types. similar to the MEM_* types above, but managed * in cbdata.c. A big difference is that these types are dynamically * allocated. This list is only a list of predefined types. Other types @@ -736,7 +743,6 @@ CBDATA_UNDEF = 0, CBDATA_acl_access, CBDATA_aclCheck_t, - CBDATA_clientHttpRequest, CBDATA_ConnStateData, CBDATA_DigestFetchState, CBDATA_ErrorState, Index: squid/src/errorpage.c diff -u squid/src/errorpage.c:1.11 squid/src/errorpage.c:1.2.2.9 --- squid/src/errorpage.c:1.11 Sat Mar 3 02:44:32 2001 +++ squid/src/errorpage.c Sun Mar 18 14:33:31 2001 @@ -262,7 +262,7 @@ HttpReply *rep; MemObject *mem = entry->mem_obj; assert(mem != NULL); - assert(mem->inmem_hi == 0); + assert(storeMemHiOffset(entry) == 0); if (entry->store_status != STORE_PENDING) { /* * If the entry is not STORE_PENDING, then no clients @@ -270,7 +270,7 @@ * error message */ assert(EBIT_TEST(entry->flags, ENTRY_ABORTED)); - assert(mem->nclients == 0); + assert(storePendingNClients(entry) == 0); errorStateFree(err); return; } Index: squid/src/forward.c diff -u squid/src/forward.c:1.10 squid/src/forward.c:1.2.2.7 --- squid/src/forward.c:1.10 Sat Mar 3 02:44:32 2001 +++ squid/src/forward.c Sun Mar 18 14:33:31 2001 @@ -92,7 +92,7 @@ fwdLog(fwdState); #endif if (e->store_status == STORE_PENDING) { - if (e->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(e) == 0) { assert(fwdState->err); errorAppendEntry(e, fwdState->err); fwdState->err = NULL; @@ -130,7 +130,7 @@ { if (fwdState->entry->store_status != STORE_PENDING) return 0; - if (fwdState->entry->mem_obj->inmem_hi > 0) + if (storeMemHiOffset(fwdState->entry) > 0) return 0; if (fwdState->n_tries > 10) return 0; @@ -249,7 +249,7 @@ peer *p = fwdStateServerPeer(fwdState); debug(17, 2) ("fwdConnectTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); assert(fd == fwdState->server_fd); - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { err = errorCon(ERR_CONNECT_FAIL, HTTP_GATEWAY_TIMEOUT); err->request = requestLink(fwdState->request); err->xerrno = ETIMEDOUT; @@ -585,7 +585,7 @@ #endif if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) return rc; - if (mem->inmem_hi - storeLowestMemReaderOffset(e) < READ_AHEAD_GAP) + if (storeMemHiOffset(e) - storeLowestMemReaderOffset(e) < READ_AHEAD_GAP) return rc; return 1; } Index: squid/src/ftp.c diff -u squid/src/ftp.c:1.12 squid/src/ftp.c:1.2.2.8 --- squid/src/ftp.c:1.12 Sat Mar 3 02:44:32 2001 +++ squid/src/ftp.c Sun Mar 18 14:33:31 2001 @@ -925,11 +925,11 @@ storeAppend(entry, ftpState->data.buf, len); ftpState->data.offset = 0; } - commSetSelect(fd, - COMM_SELECT_READ, - ftpDataRead, - data, - Config.Timeout.read); + commSetSelect(fd, + COMM_SELECT_READ, + ftpDataRead, + data, + Config.Timeout.read); } } @@ -1045,7 +1045,6 @@ const char *url = storeUrl(entry); FtpStateData *ftpState; HttpReply *reply; - CBDATA_INIT_TYPE(FtpStateData); ftpState = cbdataAlloc(FtpStateData); debug(9, 3) ("ftpStart: '%s'\n", url); @@ -2002,6 +2001,11 @@ static int ftpRestartable(FtpStateData * ftpState) { + + /* With the advent of the range header in the request_range going away.. */ + return 0; + +#if 0 if (ftpState->restart_offset > 0) return 1; if (!ftpState->request->range) @@ -2015,6 +2019,7 @@ if (ftpState->restart_offset <= 0) return 0; return 1; +#endif } static void @@ -2364,7 +2369,7 @@ ftpFailed(FtpStateData * ftpState, err_type error) { StoreEntry *entry = ftpState->entry; - if (entry->mem_obj->inmem_hi == 0) + if (storeMemHiOffset(entry) == 0) ftpFailedErrorMessage(ftpState, error); if (ftpState->data.fd > -1) { comm_close(ftpState->data.fd); @@ -2480,7 +2485,7 @@ if (ftpState->flags.http_header_sent) return; ftpState->flags.http_header_sent = 1; - assert(e->mem_obj->inmem_hi == 0); + assert(storeMemHiOffset(e) == 0); EBIT_CLR(e->flags, ENTRY_FWD_HDR_WAIT); filename = (t = strRChr(urlpath, '/')) ? t + 1 : strBuf(urlpath); if (ftpState->flags.isdir) { @@ -2523,7 +2528,7 @@ httpHeaderPutStr(&reply->header, HDR_CONTENT_ENCODING, mime_enc); httpReplySwapOut(reply, e); storeBufferFlush(e); - reply->hdr_sz = e->mem_obj->inmem_hi; + reply->hdr_sz = storeMemHiOffset(e); storeTimestampsSet(e); if (ftpState->flags.authenticated) { /* Index: squid/src/globals.h diff -u squid/src/globals.h:1.7 squid/src/globals.h:1.2.2.5 --- squid/src/globals.h:1.7 Fri Jan 12 00:20:33 2001 +++ squid/src/globals.h Fri Jan 12 00:46:07 2001 @@ -125,6 +125,7 @@ extern double current_dtime; extern int store_hash_buckets; /* 0 */ extern hash_table *store_table; /* NULL */ +extern dlink_list store_list; /* { NULL, NULL } */ extern dlink_list ClientActiveRequests; extern const String StringNull; /* { 0, 0, NULL } */ extern const MemBuf MemBufNull; /* MemBufNULL */ Index: squid/src/gopher.c diff -u squid/src/gopher.c:1.8 squid/src/gopher.c:1.2.2.6 --- squid/src/gopher.c:1.8 Sat Mar 3 02:44:32 2001 +++ squid/src/gopher.c Sun Mar 18 14:33:31 2001 @@ -577,7 +577,7 @@ StoreEntry *entry = gopherState->entry; debug(10, 4) ("gopherTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); if (entry->store_status == STORE_PENDING) { - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { fwdFail(gopherState->fwdState, errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); } @@ -633,7 +633,7 @@ debug(50, 1) ("gopherReadReply: error reading: %s\n", xstrerror()); if (ignoreErrno(errno)) { commSetSelect(fd, COMM_SELECT_READ, gopherReadReply, data, 0); - } else if (entry->mem_obj->inmem_hi == 0) { + } else if (storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); err->xerrno = errno; @@ -643,7 +643,7 @@ } else { comm_close(fd); } - } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { + } else if (len == 0 && storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); err->xerrno = errno; Index: squid/src/http.c diff -u squid/src/http.c:1.12 squid/src/http.c:1.2.2.8 --- squid/src/http.c:1.12 Sat Mar 3 02:44:32 2001 +++ squid/src/http.c Sun Mar 18 14:33:31 2001 @@ -93,7 +93,7 @@ StoreEntry *entry = httpState->entry; debug(11, 4) ("httpTimeout: FD %d: '%s'\n", fd, storeUrl(entry)); if (entry->store_status == STORE_PENDING) { - if (entry->mem_obj->inmem_hi == 0) { + if (storeMemHiOffset(entry) == 0) { fwdFail(httpState->fwd, errorCon(ERR_READ_TIMEOUT, HTTP_GATEWAY_TIMEOUT)); } @@ -229,6 +229,13 @@ */ if (httpHeaderHas(hdr, HDR_VARY)) return 0; + + /* We also don't deal with range requests */ + if (httpHeaderHas(hdr, HDR_RANGE)) + return 0; + if (httpHeaderHas(hdr, HDR_REQUEST_RANGE)) + return 0; + /* Pragma: no-cache in _replies_ is not documented in HTTP, * but servers like "Active Imaging Webcast/2.0" sure do use it */ if (httpHeaderHas(hdr, HDR_PRAGMA)) { @@ -324,7 +331,7 @@ HttpReply *reply = entry->mem_obj->reply; Ctx ctx; debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", - storeKeyText(entry->hash.key)); + storeKeyUrl(entry)); if (httpState->reply_hdr == NULL) httpState->reply_hdr = memAllocate(MEM_8K_BUF); assert(httpState->reply_hdr_state == 0); @@ -405,8 +412,8 @@ httpPconnTransferDone(HttpStateData * httpState) { /* return 1 if we got the last of the data on a persistent connection */ - MemObject *mem = httpState->entry->mem_obj; - HttpReply *reply = mem->reply; + StoreEntry *entry = httpState->entry; + HttpReply *reply = entry->mem_obj->reply; int clen; debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); /* @@ -442,7 +449,7 @@ if (clen < 0) return 0; /* If the body size is known, we must wait until we've gotten all of it. */ - if (mem->inmem_hi < reply->content_length + reply->hdr_sz) + if (storeMemHiOffset(entry) < reply->content_length + reply->hdr_sz) return 0; /* We got it all */ return 1; @@ -512,7 +519,7 @@ fd, xstrerror()); if (ignoreErrno(errno)) { commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - } else if (entry->mem_obj->inmem_hi == 0) { + } else if (storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_READ_ERROR, HTTP_INTERNAL_SERVER_ERROR); err->xerrno = errno; @@ -521,7 +528,7 @@ } else { comm_close(fd); } - } else if (len == 0 && entry->mem_obj->inmem_hi == 0) { + } else if (len == 0 && storeMemHiOffset(entry) == 0) { ErrorState *err; err = errorCon(ERR_ZERO_SIZE_OBJECT, HTTP_SERVICE_UNAVAILABLE); err->xerrno = errno; @@ -647,7 +654,6 @@ LOCAL_ARRAY(char, bbuf, BBUF_SZ); String strConnection = StringNull; const HttpHeader *hdr_in = &orig_request->header; - int we_do_ranges; const HttpHeaderEntry *e; String strVia; String strFwd; @@ -657,26 +663,8 @@ if (request->lastmod > -1 && request->method == METHOD_GET) httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, request->lastmod); - /* decide if we want to do Ranges ourselves - * (and fetch the whole object now) - * We want to handle Ranges ourselves iff - * - we can actually parse client Range specs - * - the specs are expected to be simple enough (e.g. no out-of-order ranges) - * - reply will be cachable - * (If the reply will be uncachable we have to throw it away after - * serving this request, so it is better to forward ranges to - * the server and fetch only the requested content) - */ - if (NULL == orig_request->range) - we_do_ranges = 0; - else if (!orig_request->flags.cachable) - we_do_ranges = 0; - else if (httpHdrRangeOffsetLimit(orig_request->range)) - we_do_ranges = 0; - else - we_do_ranges = 1; - debug(11, 8) ("httpBuildRequestHeader: range specs: %p, cachable: %d; we_do_ranges: %d\n", - orig_request->range, orig_request->flags.cachable, we_do_ranges); + /* We are not doing ranges at all now - we're just passing them through */ + debug(11, 8) ("httpBuildRequestHeader: cachable: %d\n", orig_request->flags.cachable); strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); while ((e = httpHeaderGetEntry(hdr_in, &pos))) { @@ -737,12 +725,6 @@ httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); } break; - case HDR_RANGE: - case HDR_IF_RANGE: - case HDR_REQUEST_RANGE: - if (!we_do_ranges) - httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); - break; case HDR_PROXY_CONNECTION: case HDR_CONNECTION: case HDR_VIA: Index: squid/src/main.c diff -u squid/src/main.c:1.19 squid/src/main.c:1.2.2.10 --- squid/src/main.c:1.19 Tue Jan 30 16:27:46 2001 +++ squid/src/main.c Tue Feb 6 02:08:04 2001 @@ -479,6 +479,7 @@ #else idnsInit(); #endif + requestInit(); redirectInit(); authenticateInit(&Config.authConfig); useragentOpenLog(); @@ -501,6 +502,9 @@ #if USE_UNLINKD unlinkdInit(); #endif + reply_internal_init(); + reply_network_init(); + urlInitialize(); cachemgrInit(); statInit(); Index: squid/src/mem.c diff -u squid/src/mem.c:1.9 squid/src/mem.c:1.2.2.4 --- squid/src/mem.c:1.9 Sat Feb 17 12:56:22 2001 +++ squid/src/mem.c Tue Feb 20 07:39:42 2001 @@ -275,8 +275,6 @@ memDataInit(MEM_PS_STATE, "ps_state", sizeof(ps_state), 0); memDataInit(MEM_REFRESH_T, "refresh_t", sizeof(refresh_t), 0); memDataInit(MEM_RELIST, "relist", sizeof(relist), 0); - memDataInit(MEM_REQUEST_T, "request_t", sizeof(request_t), - Squid_MaxFD >> 3); memDataInit(MEM_SQUIDCONFIG, "SquidConfig", sizeof(SquidConfig), 0); memDataInit(MEM_SQUIDCONFIG2, "SquidConfig2", sizeof(SquidConfig2), 0); memDataInit(MEM_STATCOUNTERS, "StatCounters", sizeof(StatCounters), 0); Index: squid/src/mime.c diff -u squid/src/mime.c:1.8 squid/src/mime.c:1.2.2.9 --- squid/src/mime.c:1.8 Fri Jan 12 00:20:33 2001 +++ squid/src/mime.c Wed Feb 7 05:54:38 2001 @@ -364,12 +364,18 @@ debug(25, 5) ("mimeInit: added '%s'\n", buf); } fclose(fp); + /* * Create Icon StoreEntry's */ + + /* Remember, we're not doing icons in this fashion.. -- adrian */ +#if 0 for (m = MimeTable; m != NULL; m = m->next) mimeLoadIconFile(m->icon); debug(25, 1) ("Loaded Icons.\n"); +#endif + debug(25, 1) ("Not loading icons.\n"); } void @@ -391,62 +397,5 @@ static void mimeLoadIconFile(const char *icon) { - int fd; - int n; - request_flags flags; - struct stat sb; - StoreEntry *e; - LOCAL_ARRAY(char, path, MAXPATHLEN); - LOCAL_ARRAY(char, url, MAX_URL); - char *buf; - const char *type = mimeGetContentType(icon); - HttpReply *reply; - http_version_t version; - if (type == NULL) - fatal("Unknown icon format while reading mime.conf\n"); - buf = internalLocalUri("/squid-internal-static/icons/", icon); - xstrncpy(url, buf, MAX_URL); - if (storeGetPublic(url, METHOD_GET)) - return; - snprintf(path, MAXPATHLEN, "%s/%s", Config.icons.directory, icon); - fd = file_open(path, O_RDONLY | O_BINARY); - if (fd < 0) { - debug(25, 0) ("mimeLoadIconFile: %s: %s\n", path, xstrerror()); - return; - } - if (fstat(fd, &sb) < 0) { - debug(50, 0) ("mimeLoadIconFile: FD %d: fstat: %s\n", fd, xstrerror()); - return; - } - flags = null_request_flags; - flags.cachable = 1; - e = storeCreateEntry(url, - url, - flags, - METHOD_GET); - assert(e != NULL); - storeSetPublicKey(e); - storeBuffer(e); - e->mem_obj->request = requestLink(urlParse(METHOD_GET, url)); - httpReplyReset(reply = e->mem_obj->reply); - httpBuildVersion(&version, 1, 0); - httpReplySetHeaders(reply, version, HTTP_OK, NULL, - type, (int) sb.st_size, sb.st_mtime, -1); - reply->cache_control = httpHdrCcCreate(); - httpHdrCcSetMaxAge(reply->cache_control, 86400); - httpHeaderPutCc(&reply->header, reply->cache_control); - httpReplySwapOut(reply, e); - reply->hdr_sz = e->mem_obj->inmem_hi; /* yuk */ - /* read the file into the buffer and append it to store */ - buf = memAllocate(MEM_4K_BUF); - while ((n = read(fd, buf, 4096)) > 0) - storeAppend(e, buf, n); - file_close(fd); - EBIT_SET(e->flags, ENTRY_SPECIAL); - storeBufferFlush(e); - storeComplete(e); - storeTimestampsSet(e); - debug(25, 3) ("Loaded icon %s\n", url); - storeUnlockObject(e); - memFree(buf, MEM_4K_BUF); + fatal("I shouldn't get here yet!\n"); } Index: squid/src/neighbors.c diff -u squid/src/neighbors.c:1.9 squid/src/neighbors.c:1.2.2.7 --- squid/src/neighbors.c:1.9 Sat Mar 3 02:44:32 2001 +++ squid/src/neighbors.c Sun Mar 18 14:51:21 2001 @@ -1122,7 +1122,8 @@ assert(p->type == PEER_MULTICAST); p->mcast.flags.count_event_pending = 0; snprintf(url, MAX_URL, "http://%s/", inet_ntoa(p->in_addr.sin_addr)); - fake = storeCreateEntry(url, url, null_request_flags, METHOD_GET); + fake = storeCreateEntry(url, url, null_request_flags, METHOD_GET, + REPLY_OBJ_INTERNAL); psstate = cbdataAlloc(ps_state); psstate->request = requestLink(urlParse(METHOD_GET, url)); psstate->entry = fake; Index: squid/src/net_db.c diff -u squid/src/net_db.c:1.10 squid/src/net_db.c:1.2.2.8 --- squid/src/net_db.c:1.10 Sat Mar 3 02:44:32 2001 +++ squid/src/net_db.c Sun Mar 18 14:33:32 2001 @@ -627,8 +627,8 @@ debug(38, 3) ("netdbExchangeHandleReply: STORE_PENDING\n"); storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, ex->buf, netdbExchangeHandleReply, ex); - } else if (ex->seen < ex->e->mem_obj->inmem_hi) { - debug(38, 3) ("netdbExchangeHandleReply: ex->e->mem_obj->inmem_hi\n"); + } else if (ex->seen < storeMemHiOffset(ex->e)) { + debug(38, 3) ("netdbExchangeHandleReply: storeMemHiOffset(ex->e)\n"); storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, ex->buf, netdbExchangeHandleReply, ex); } else { @@ -644,7 +644,7 @@ debug(38, 3) ("netdbExchangeDone: %s\n", storeUrl(ex->e)); memFree(ex->buf, MEM_4K_BUF); requestUnlink(ex->r); - storeUnregister(ex->sc, ex->e, ex); + storeClientUnregister(ex->sc, ex->e, ex); storeUnlockObject(ex->e); cbdataUnlock(ex->p); cbdataFree(ex); @@ -1005,11 +1005,12 @@ requestLink(ex->r); assert(NULL != ex->r); httpBuildVersion(&ex->r->http_ver, 1, 0); - ex->e = storeCreateEntry(uri, uri, null_request_flags, METHOD_GET); + ex->e = storeCreateEntry(uri, uri, null_request_flags, METHOD_GET, + REPLY_OBJ_INTERNAL); ex->buf_sz = 4096; ex->buf = memAllocate(MEM_4K_BUF); assert(NULL != ex->e); - ex->sc = storeClientListAdd(ex->e, ex); + ex->sc = storeClientRegister(ex->e, ex); storeClientCopy(ex->sc, ex->e, ex->seen, ex->used, ex->buf_sz, ex->buf, netdbExchangeHandleReply, ex); ex->r->flags.loopdetect = 1; /* cheat! -- force direct */ Index: squid/src/peer_digest.c diff -u squid/src/peer_digest.c:1.7 squid/src/peer_digest.c:1.2.2.8 --- squid/src/peer_digest.c:1.7 Sat Mar 3 02:44:32 2001 +++ squid/src/peer_digest.c Sun Mar 18 14:33:32 2001 @@ -269,7 +269,6 @@ peer *p = pd->peer; StoreEntry *e, *old_e; char *url; - const cache_key *key; request_t *req; DigestFetchState *fetch = NULL; @@ -283,8 +282,7 @@ url = internalRemoteUri(p->host, p->http_port, "/squid-internal-periodic/", StoreDigestFileName); - key = storeKeyPublic(url, METHOD_GET); - debug(72, 2) ("peerDigestRequest: %s key: %s\n", url, storeKeyText(key)); + debug(72, 2) ("peerDigestRequest: %s\n", url); req = urlParse(METHOD_GET, url); assert(req); @@ -308,16 +306,17 @@ req->flags.cachable = 1; /* the rest is based on clientProcessExpired() */ req->flags.refresh = 1; - old_e = fetch->old_entry = storeGet(key); + old_e = fetch->old_entry = storeGetPublic(url, METHOD_GET); if (old_e) { debug(72, 5) ("peerDigestRequest: found old entry\n"); storeLockObject(old_e); storeCreateMemObject(old_e, url, url); - fetch->old_sc = storeClientListAdd(old_e, fetch); + fetch->old_sc = storeClientRegister(old_e, fetch); } - e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method); + e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method, + REPLY_OBJ_NETWORK); assert(EBIT_TEST(e->flags, KEY_PRIVATE)); - fetch->sc = storeClientListAdd(e, fetch); + fetch->sc = storeClientRegister(e, fetch); /* set lastmod to trigger IMS request if possible */ if (old_e) e->lastmod = old_e->lastmod; @@ -366,7 +365,7 @@ httpReplyUpdateOnNotModified(fetch->old_entry->mem_obj->reply, reply); storeTimestampsSet(fetch->old_entry); /* get rid of 304 reply */ - storeUnregister(fetch->sc, fetch->entry, fetch); + storeClientUnregister(fetch->sc, fetch->entry, fetch); storeUnlockObject(fetch->entry); fetch->entry = fetch->old_entry; fetch->old_entry = NULL; @@ -377,7 +376,7 @@ /* get rid of old entry if any */ if (fetch->old_entry) { debug(72, 3) ("peerDigestFetchReply: got new digest, releasing old one\n"); - storeUnregister(fetch->old_sc, fetch->old_entry, fetch); + storeClientUnregister(fetch->old_sc, fetch->old_entry, fetch); storeReleaseRequest(fetch->old_entry); storeUnlockObject(fetch->old_entry); fetch->old_entry = NULL; @@ -685,7 +684,7 @@ if (fetch->old_entry) { debug(72, 2) ("peerDigestFetchFinish: deleting old entry\n"); - storeUnregister(fetch->sc, fetch->old_entry, fetch); + storeClientUnregister(fetch->sc, fetch->old_entry, fetch); storeReleaseRequest(fetch->old_entry); storeUnlockObject(fetch->old_entry); fetch->old_entry = NULL; @@ -697,7 +696,7 @@ statCounter.cd.msgs_recv += fetch->recv.msg; /* unlock everything */ - storeUnregister(fetch->sc, fetch->entry, fetch); + storeClientUnregister(fetch->sc, fetch->entry, fetch); storeUnlockObject(fetch->entry); requestUnlink(fetch->request); fetch->entry = NULL; @@ -721,7 +720,7 @@ /* XXX: we must distinguish between 304 hits and misses here */ fetch->sent.bytes = httpRequestPrefixLen(fetch->request); fetch->recv.bytes = fetch->entry->store_status == STORE_PENDING ? - mem->inmem_hi : mem->object_sz; + storeMemHiOffset(fetch->entry) : mem->object_sz; fetch->sent.msg = fetch->recv.msg = 1; fetch->expires = fetch->entry->expires; fetch->resp_time = squid_curtime - fetch->start_time; Index: squid/src/protos.h diff -u squid/src/protos.h:1.20 squid/src/protos.h:1.2.2.19 --- squid/src/protos.h:1.20 Sat Mar 3 02:44:32 2001 +++ squid/src/protos.h Sun Mar 18 14:33:32 2001 @@ -127,11 +127,12 @@ extern void clientAccessCheckDone(int, void *); extern int modifiedSince(StoreEntry *, request_t *); extern char *clientConstructTraceEcho(clientHttpRequest *); -extern void clientPurgeRequest(clientHttpRequest *); +extern void clientStartPurgeRequest(clientHttpRequest *); extern int checkNegativeHit(StoreEntry *); extern void clientHttpConnectionsOpen(void); extern void clientHttpConnectionsClose(void); -extern StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, request_flags); +extern StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, + request_flags, reply_obj_t); extern int isTcpHit(log_type); extern void clientReadBody(request_t * req, char *buf, size_t size, CBCB * callback, void *data); extern int clientAbortBody(request_t * req); @@ -493,6 +494,7 @@ extern int httpReplyBodySize(method_t, HttpReply *); /* Http Request */ +extern void requestInit(void); extern request_t *requestCreate(method_t, protocol_t, const char *urlpath); extern void requestDestroy(request_t *); extern request_t *requestLink(request_t *); @@ -866,7 +868,8 @@ extern StoreEntry *new_StoreEntry(int, const char *, const char *); extern StoreEntry *storeGet(const cache_key *); extern StoreEntry *storeGetPublic(const char *uri, const method_t method); -extern StoreEntry *storeCreateEntry(const char *, const char *, request_flags, method_t); +extern StoreEntry *storeCreateEntry(const char *, const char *, request_flags, + method_t, reply_obj_t); extern void storeSetPublicKey(StoreEntry *); extern void storeComplete(StoreEntry *); extern void storeInit(void); @@ -893,6 +896,7 @@ extern void storeMemObjectDump(MemObject * mem); extern void storeEntryDump(const StoreEntry * e, int debug_lvl); extern const char *storeUrl(const StoreEntry *); +extern const char *storeKeyUrl(const StoreEntry *); extern void storeCreateMemObject(StoreEntry *, const char *, const char *); extern void storeCopyNotModifiedReplyHeaders(MemObject * O, MemObject * N); extern void storeBuffer(StoreEntry *); @@ -918,6 +922,10 @@ extern void storeFsDone(void); extern void storeFsAdd(char *, STSETUP *); extern void storeReplAdd(char *, REMOVALPOLICYCREATE *); +extern off_t storeMemHiOffset(const StoreEntry *); +extern off_t storeMemLoOffset(const StoreEntry *); +extern void storeDetachReply(StoreEntry *); +extern void storeDoubleCheckRequest(request_t *, char *, method_t); /* store_modules.c */ extern void storeFsSetup(void); @@ -926,6 +934,9 @@ extern void storeReplSetup(void); /* store_io.c */ +extern void storeLookup(request_t * , STGETDONE *, void *); +extern void storeLookupComplete(request_t *, STGETDONE *, void *, + StoreEntry *); extern storeIOState *storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *); extern storeIOState *storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *); extern void storeClose(storeIOState *); @@ -1029,10 +1040,10 @@ #if STORE_CLIENT_LIST_DEBUG extern store_client *storeClientListSearch(const MemObject * mem, void *data); #endif -extern store_client *storeClientListAdd(StoreEntry * e, void *data); -extern void storeClientCopy(store_client *, StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); -extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data); -extern int storeUnregister(store_client * sc, StoreEntry * e, void *data); +extern void storeClientRegister(StoreEntry * e, void *data); +extern void storeClientCopy(StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); +extern int storeClientCopyPending(StoreEntry * e, void *data); +extern int storeClientUnregister(StoreEntry * e, void *data); extern off_t storeLowestMemReaderOffset(const StoreEntry * entry); extern void InvokeHandlers(StoreEntry * e); extern int storePendingNClients(const StoreEntry * e); @@ -1289,6 +1300,16 @@ extern unsigned int url_checksum(const char *url); #endif + +/* + * reply objects + */ +extern void reply_internal_init(void); +extern void reply_internal_create(StoreEntry *); + +extern void reply_network_init(void); +extern void reply_network_create(StoreEntry *); + /* * hack to allow snmp access to the statistics counters */ Index: squid/src/reply_internal.c diff -u /dev/null squid/src/reply_internal.c:1.1.2.6 --- /dev/null Sun Jan 25 06:36:42 2004 +++ squid/src/reply_internal.c Wed Jan 3 04:11:33 2001 @@ -0,0 +1,290 @@ +/* + * reply_internal.c + * + * Handles "internal" replies, such as error pages, internal gifs, etc.. + * + * Adrian Chadd + */ + + +#include "squid.h" + +struct _reply_internal_state { + mem_hdr data_hdr; + off_t inmem_hi; + off_t inmem_lo; +}; + + + +typedef struct _reply_internal_state reply_internal_state_t; + +MemPool *reply_internal_state_pool = NULL; + +/* Prototypes */ +static void reply_internal_clientcopy(StoreEntry * e); +static void reply_internal_clientcopy3(StoreEntry * e); + + + +void +reply_internal_init(void) +{ + reply_internal_state_pool = memPoolCreate("Internal reply pool", + sizeof(reply_internal_state_t)); +} + + +/* + * reply_internal_done + */ +static void +reply_internal_done(StoreEntry *e) +{ + reply_internal_state_t *state = e->stdata; + + /* Make sure noone is left ! */ + + /* Kill the state struct */ + stmemFree(&state->data_hdr); + memPoolFree(reply_internal_state_pool, state); + + /* Done! */ +} + + +/* + * reply_internal_memhi + */ +static off_t +reply_internal_memhi(const StoreEntry *e) +{ + reply_internal_state_t *state = e->stdata; + + return state->inmem_hi; +} + + +/* + * reply_internal_memlo + */ +static off_t +reply_internal_memlo(const StoreEntry *e) +{ + reply_internal_state_t *state = e->stdata; + + return state->inmem_lo; +} + + +/* + * reply_internal_append + */ +static void +reply_internal_append(StoreEntry *e, const char *buf, int len) +{ + MemObject *mem = e->mem_obj; + reply_internal_state_t *state = e->stdata; + assert(mem != NULL); + assert(len >= 0); + assert(e->store_status == STORE_PENDING); + if (len) { + debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", + len, + storeKeyUrl(e)); + /* Used to call storeGetMemSpace here .. */ + /* storeGetMemSpace(len); */ + stmemAppend(&state->data_hdr, buf, len); + state->inmem_hi += len; + } + if (EBIT_TEST(e->flags, DELAY_SENDING)) + return; + e->stflush(e); +} + + +/* + * reply_internal_flush + */ +static void +reply_internal_flush(StoreEntry *e) +{ + debug(20, 3) ("reply_internal_flush: %s\n", storeKeyUrl(e)); + /* walk the entire list looking for valid callbacks */ + if (e->sc.callback_data == NULL) + return; + if (e->sc.callback == NULL) + return; + if (e->sc.flags.disk_io_pending) + return; + reply_internal_clientcopy3(e); +} + + +/* store client copy evilness */ + +static void +reply_internal_clientcallback(store_client * sc, ssize_t sz) +{ + STCB *callback = sc->callback; + char *buf = sc->copy_buf; + assert(sc->callback); + sc->callback = NULL; + sc->copy_buf = NULL; + if (cbdataValid(sc->callback_data)) + callback(sc->callback_data, buf, sz); +} + +static void +reply_internal_clientcopyevent(void *data) +{ + StoreEntry *e = data; + debug(20, 3) ("reply_internal_clientcopyevent: Running\n"); + e->sc.flags.copy_event_pending = 0; + if (!e->sc.valid) + return; + reply_internal_clientcopy3(e); +} + + +/* + * This function is used below to decide if we have any more data to + * send to the client. If the store_status is STORE_PENDING, then we + * do have more data to send. If its STORE_OK, then + * we continue checking. + * If the length is >= 0, then we compare it to the requested copy + * offset. + */ +static int +reply_internal_clientnomoretosend(StoreEntry * e, store_client * sc) +{ + if (e->store_status == STORE_PENDING) + return 0; + assert(objectLen(e) > -1); /* Internal objects will never be swap backed */ + if (sc->copy_offset < objectLen(e)) + return 0; + return 1; +} + +/* copy bytes requested by the client */ +static void +reply_internal_clientcopy(StoreEntry * e) +{ + store_client *sc = &e->sc; + if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) { + debug(20, 5) ("reply_internal_clientcopy: returning because ENTRY_FWD_HDR_WAIT set\n"); + return; + } + debug(20, 3) ("reply_internal_clientcopy: %s\n", storeKeyUrl(e)); + assert(sc->callback != NULL); + /* + * We used to check for ENTRY_ABORTED here. But there were some + * problems. For example, we might have a slow client (or two) and + * the server-side is reading far ahead and swapping to disk. Even + * if the server-side aborts, we want to give the client(s) + * everything we got before the abort condition occurred. + */ + reply_internal_clientcopy3(e); +} + +static void +reply_internal_clientcopy3(StoreEntry * e) +{ + store_client *sc = &e->sc; + reply_internal_state_t *state = e->stdata; + size_t sz; + + sc->flags.store_copying = 1; + if (reply_internal_clientnomoretosend(e, sc)) { + /* There is no more to send! */ + reply_internal_clientcallback(sc, 0); + goto finish; + } + if ((e->store_status == STORE_PENDING) && + (sc->copy_offset >= state->inmem_hi)) { + /* client has already seen this, wait for more */ + debug(20, 3) ("reply_internal_clientcopy3: Waiting for more\n"); + goto finish; + } + + if ((sc->copy_offset >= state->inmem_lo) && + (sc->copy_offset < state->inmem_hi)) { + /* What the client wants is in memory */ + debug(20, 3) ("reply_internal_clientcopy3: Copying from memory\n"); + sz = stmemCopy(&state->data_hdr, + sc->copy_offset, sc->copy_buf, sc->copy_size); + reply_internal_clientcallback(sc, sz); + goto finish; + } + /* We shouldn't ever get here! */ + assert(1 == 0); + +finish: + sc->flags.store_copying = 0; +} + + +/* + * reply_makepublic - make this object public + */ +static void +reply_internal_makepublic(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_makeprivate - make this object private + */ +static void +reply_internal_makeprivate(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_internal_release - object is being deleted + * + * Yes, its a no-op for us because all objects are private + */ +static void +reply_internal_release(StoreEntry *e) +{ +} + + +/* + * reply_internal_create - create an internal reply .. + */ +void +reply_internal_create(StoreEntry *e) +{ + reply_internal_state_t *state; + MemObject *mem; + + assert(e != NULL); + assert(e->mem_obj != NULL); + + mem = e->mem_obj; + + /* Create a new state struct */ + state = memPoolAlloc(reply_internal_state_pool); + + /* Initialise it */ + e->stdata = state; + + /* Assign our functions .. */ + e->stappend = reply_internal_append; + e->stflush = reply_internal_flush; + e->stmemhi = reply_internal_memhi; + e->stmemlo = reply_internal_memlo; + e->stclientcopy = reply_internal_clientcopy; + e->stmakeprivate = reply_internal_makeprivate; + e->stmakepublic = reply_internal_makepublic; + e->strelease = reply_internal_release; + e->stdone = reply_internal_done; + + /* Done! */ +} Index: squid/src/reply_network.c diff -u /dev/null squid/src/reply_network.c:1.1.2.2 --- /dev/null Sun Jan 25 06:36:42 2004 +++ squid/src/reply_network.c Sun Jan 7 15:22:47 2001 @@ -0,0 +1,296 @@ +/* + * reply_network.c + * + * Handles replies which have a network as a data server. + * This module handles limiting the amount of data which is requested + * from a server so as to not flood the client, and also writing incoming + * data to a backing object if needed. + * + * Adrian Chadd + */ + + +#include "squid.h" + +struct _reply_network_state { + mem_hdr data_hdr; + off_t inmem_hi; + off_t inmem_lo; +}; + + + +typedef struct _reply_network_state reply_network_state_t; + +MemPool *reply_network_state_pool = NULL; + +/* Prototypes */ +static void reply_network_clientcopy(StoreEntry * e); +static void reply_network_clientcopy3(StoreEntry * e); + + + +void +reply_network_init(void) +{ + reply_network_state_pool = memPoolCreate("Internal reply pool", + sizeof(reply_network_state_t)); +} + + +/* + * reply_network_done + */ +static void +reply_network_done(StoreEntry *e) +{ + reply_network_state_t *state = e->stdata; + + /* Make sure noone is left ! */ + + /* Kill the state struct */ + stmemFree(&state->data_hdr); + memPoolFree(reply_network_state_pool, state); + + /* Done! */ +} + + +/* + * reply_network_memhi + */ +static off_t +reply_network_memhi(const StoreEntry *e) +{ + reply_network_state_t *state = e->stdata; + + return state->inmem_hi; +} + + +/* + * reply_network_memlo + */ +static off_t +reply_network_memlo(const StoreEntry *e) +{ + reply_network_state_t *state = e->stdata; + + return state->inmem_lo; +} + + +/* + * reply_network_append + */ +static void +reply_network_append(StoreEntry *e, const char *buf, int len) +{ + MemObject *mem = e->mem_obj; + reply_network_state_t *state = e->stdata; + assert(mem != NULL); + assert(len >= 0); + assert(e->store_status == STORE_PENDING); + if (len) { + debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", + len, + storeKeyUrl(e)); + /* Used to call storeGetMemSpace here .. */ + /* storeGetMemSpace(len); */ + stmemAppend(&state->data_hdr, buf, len); + state->inmem_hi += len; + } + if (EBIT_TEST(e->flags, DELAY_SENDING)) + return; + e->stflush(e); +} + + +/* + * reply_network_flush + */ +static void +reply_network_flush(StoreEntry *e) +{ + debug(20, 3) ("reply_network_flush: %s\n", storeKeyUrl(e)); + /* walk the entire list looking for valid callbacks */ + if (e->sc.callback_data == NULL) + return; + if (e->sc.callback == NULL) + return; + if (e->sc.flags.disk_io_pending) + return; + reply_network_clientcopy3(e); +} + + +/* store client copy evilness */ + +static void +reply_network_clientcallback(store_client * sc, ssize_t sz) +{ + STCB *callback = sc->callback; + char *buf = sc->copy_buf; + assert(sc->callback); + sc->callback = NULL; + sc->copy_buf = NULL; + if (cbdataValid(sc->callback_data)) + callback(sc->callback_data, buf, sz); +} + +static void +reply_network_clientcopyevent(void *data) +{ + StoreEntry *e = data; + debug(20, 3) ("reply_network_clientcopyevent: Running\n"); + e->sc.flags.copy_event_pending = 0; + if (!e->sc.valid) + return; + reply_network_clientcopy3(e); +} + + +/* + * This function is used below to decide if we have any more data to + * send to the client. If the store_status is STORE_PENDING, then we + * do have more data to send. If its STORE_OK, then + * we continue checking. + * If the length is >= 0, then we compare it to the requested copy + * offset. + */ +static int +reply_network_clientnomoretosend(StoreEntry * e, store_client * sc) +{ + if (e->store_status == STORE_PENDING) + return 0; + assert(objectLen(e) > -1); /* Internal objects will never be swap backed */ + if (sc->copy_offset < objectLen(e)) + return 0; + return 1; +} + +/* copy bytes requested by the client */ +static void +reply_network_clientcopy(StoreEntry * e) +{ + store_client *sc = &e->sc; + if (EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) { + debug(20, 5) ("reply_network_clientcopy: returning because ENTRY_FWD_HDR_WAIT set\n"); + return; + } + debug(20, 3) ("reply_network_clientcopy: %s\n", storeKeyUrl(e)); + assert(sc->callback != NULL); + /* + * We used to check for ENTRY_ABORTED here. But there were some + * problems. For example, we might have a slow client (or two) and + * the server-side is reading far ahead and swapping to disk. Even + * if the server-side aborts, we want to give the client(s) + * everything we got before the abort condition occurred. + */ + reply_network_clientcopy3(e); +} + +static void +reply_network_clientcopy3(StoreEntry * e) +{ + store_client *sc = &e->sc; + reply_network_state_t *state = e->stdata; + size_t sz; + + sc->flags.store_copying = 1; + if (reply_network_clientnomoretosend(e, sc)) { + /* There is no more to send! */ + reply_network_clientcallback(sc, 0); + goto finish; + } + if ((e->store_status == STORE_PENDING) && + (sc->copy_offset >= state->inmem_hi)) { + /* client has already seen this, wait for more */ + debug(20, 3) ("reply_network_clientcopy3: Waiting for more\n"); + goto finish; + } + + if ((sc->copy_offset >= state->inmem_lo) && + (sc->copy_offset < state->inmem_hi)) { + /* What the client wants is in memory */ + debug(20, 3) ("reply_network_clientcopy3: Copying from memory\n"); + sz = stmemCopy(&state->data_hdr, + sc->copy_offset, sc->copy_buf, sc->copy_size); + state->inmem_lo = sc->copy_offset + sz; + assert(state->inmem_lo <= state->inmem_hi); + stmemFreeDataUpto(&state->data_hdr, state->inmem_lo - 1); + reply_network_clientcallback(sc, sz); + goto finish; + } + /* We shouldn't ever get here! */ + assert(1 == 0); + +finish: + sc->flags.store_copying = 0; +} + + +/* + * reply_makepublic - make this object public + */ +static void +reply_network_makepublic(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_makeprivate - make this object private + */ +static void +reply_network_makeprivate(StoreEntry *e) +{ + /* Do nothing */ +} + + +/* + * reply_network_release - object is being deleted + * + * Yes, its a no-op for us because all objects are private + */ +static void +reply_network_release(StoreEntry *e) +{ +} + + +/* + * reply_network_create - create an internal reply .. + */ +void +reply_network_create(StoreEntry *e) +{ + reply_network_state_t *state; + MemObject *mem; + + assert(e != NULL); + assert(e->mem_obj != NULL); + + mem = e->mem_obj; + + /* Create a new state struct */ + state = memPoolAlloc(reply_network_state_pool); + + /* Initialise it */ + e->stdata = state; + + /* Assign our functions .. */ + e->stappend = reply_network_append; + e->stflush = reply_network_flush; + e->stmemhi = reply_network_memhi; + e->stmemlo = reply_network_memlo; + e->stclientcopy = reply_network_clientcopy; + e->stmakeprivate = reply_network_makeprivate; + e->stmakepublic = reply_network_makepublic; + e->strelease = reply_network_release; + e->stdone = reply_network_done; + + /* Done! */ +} Index: squid/src/stat.c diff -u squid/src/stat.c:1.9 squid/src/stat.c:1.2.2.10 --- squid/src/stat.c:1.9 Sat Mar 3 02:44:32 2001 +++ squid/src/stat.c Sun Mar 18 14:33:32 2001 @@ -40,9 +40,9 @@ typedef int STOBJFLT(const StoreEntry *); typedef struct { - StoreEntry *sentry; - int bucket; - STOBJFLT *filter; + StoreEntry *sentry; /* Where we're outputting to */ + STOBJFLT *filter; /* Filter function for output */ + StoreEntry *cur_entry; /* Current StoreEntry being checked .. */ } StatObjectsState; /* LOCALS */ @@ -230,8 +230,6 @@ strcat(buf, "FWD_HDR_WAIT,"); if (EBIT_TEST(flags, ENTRY_NEGCACHED)) strcat(buf, "NEGCACHED,"); - if (EBIT_TEST(flags, ENTRY_VALIDATED)) - strcat(buf, "VALIDATED,"); if (EBIT_TEST(flags, ENTRY_BAD_LENGTH)) strcat(buf, "BAD_LENGTH,"); if (EBIT_TEST(flags, ENTRY_ABORTED)) @@ -257,10 +255,8 @@ statStoreEntry(StoreEntry * s, StoreEntry * e) { MemObject *mem = e->mem_obj; - int i; struct _store_client *sc; - dlink_node *node; - storeAppendPrintf(s, "KEY %s\n", storeKeyText(e->hash.key)); + storeAppendPrintf(s, "KEY %s\n", storeKeyUrl(e)); if (mem) storeAppendPrintf(s, "\t%s %s\n", RequestMethodStr[mem->method], mem->log_url); @@ -274,18 +270,12 @@ storeAppendPrintf(s, "\tSwap Dir %d, File %#08X\n", e->swap_dirn, e->swap_filen); if (mem != NULL) { - storeAppendPrintf(s, "\tinmem_lo: %d\n", (int) mem->inmem_lo); - storeAppendPrintf(s, "\tinmem_hi: %d\n", (int) mem->inmem_hi); - storeAppendPrintf(s, "\tswapout: %d bytes queued\n", - (int) mem->swapout.queue_offset); - if (mem->swapout.sio) - storeAppendPrintf(s, "\tswapout: %d bytes written\n", - (int) storeOffset(mem->swapout.sio)); - for (i = 0, node = mem->clients.head; node; node = node->next, i++) { - sc = (store_client *) node->data; - if (sc->callback_data == NULL) - continue; - storeAppendPrintf(s, "\tClient #%d, %p\n", i, sc->callback_data); + storeAppendPrintf(s, "\tInMem_LoOffset: %d\n", (int) storeMemLoOffset(e)); + storeAppendPrintf(s, "\tInMem_HiOffset: %d\n", (int) storeMemHiOffset(e)); + + sc = &e->sc; + if (sc->valid) { + storeAppendPrintf(s, "\tClient #%d, %p\n", 1, sc->callback_data); storeAppendPrintf(s, "\t\tcopy_offset: %d\n", (int) sc->copy_offset); storeAppendPrintf(s, "\t\tseen_offset: %d\n", @@ -305,20 +295,18 @@ storeAppendPrintf(s, "\n"); } +#define MAX_NUMOBJECTS_TO_SCAN 1000 + /* process objects list */ static void statObjects(void *data) { StatObjectsState *state = data; StoreEntry *e; - hash_link *link_ptr = NULL; - hash_link *link_next = NULL; - if (state->bucket >= store_hash_buckets) { - storeComplete(state->sentry); - storeUnlockObject(state->sentry); - cbdataFree(state); - return; - } else if (EBIT_TEST(state->sentry->flags, ENTRY_ABORTED)) { + int i; + + + if (EBIT_TEST(state->sentry->flags, ENTRY_ABORTED)) { storeUnlockObject(state->sentry); cbdataFree(state); return; @@ -326,17 +314,33 @@ eventAdd("statObjects", statObjects, state, 0.1, 1); return; } + + /* Don't return the data to the client immediately */ storeBuffer(state->sentry); - debug(49, 3) ("statObjects: Bucket #%d\n", state->bucket); - link_next = hash_get_bucket(store_table, state->bucket); - while (NULL != (link_ptr = link_next)) { - link_next = link_ptr->next; - e = (StoreEntry *) link_ptr; - if (state->filter && 0 == state->filter(e)) - continue; - statStoreEntry(state->sentry, e); + + for (i = 0; i < MAX_NUMOBJECTS_TO_SCAN; i++) { + /* Assume we were given a non-NULL object .. */ + e = state->cur_entry; + assert(e != NULL); + if (state->filter) { + if (state->filter(e)) + statStoreEntry(state->sentry, e); + } else + statStoreEntry(state->sentry, e); + storeUnlockObject(e); + + /* Make sure we have a next object .. */ + if (e->node.next == NULL) { + storeComplete(state->sentry); + storeUnlockObject(state->sentry); + cbdataFree(state); + return; + } + /* e becomes this if we get to the top of the loop again! */ + state->cur_entry = (StoreEntry *) e->node.next->data; + storeLockObject(state->cur_entry); } - state->bucket++; + eventAdd("statObjects", statObjects, state, 0.0, 1); storeBufferFlush(state->sentry); } @@ -348,6 +352,9 @@ state = cbdataAlloc(StatObjectsState); state->sentry = sentry; state->filter = filter; + state->cur_entry = (StoreEntry *) store_list.head->data; + /* This is so it doesn't get riped out from beneath us! */ + storeLockObject(state->cur_entry); storeLockObject(sentry); eventAdd("statObjects", statObjects, state, 0.0, 1); } @@ -376,8 +383,10 @@ { if (e->mem_obj == NULL) return 0; +#if 0 if (e->mem_obj->swapout.sio == NULL) return 0; +#endif return 1; } Index: squid/src/store.c diff -u squid/src/store.c:1.10 squid/src/store.c:1.2.2.21 --- squid/src/store.c:1.10 Thu Feb 22 21:51:08 2001 +++ squid/src/store.c Sun Mar 18 14:33:32 2001 @@ -77,23 +77,20 @@ * local function prototypes */ static int storeEntryValidLength(const StoreEntry *); +#if 0 static void storeGetMemSpace(int); -static void storeHashDelete(StoreEntry *); +#endif static MemObject *new_MemObject(const char *, const char *); static void destroy_MemObject(StoreEntry *); static FREE destroy_StoreEntry; +#if 0 static void storePurgeMem(StoreEntry *); -static void storeEntryReferenced(StoreEntry *); -static void storeEntryDereferenced(StoreEntry *); -static int getKeyCounter(void); -static int storeKeepInMemory(const StoreEntry *); +#endif static OBJH storeCheckCachableStats; -static EVH storeLateRelease; /* * local variables */ -static Stack LateReleaseStack; #if URL_CHECKSUM_DEBUG unsigned int @@ -132,8 +129,8 @@ { StoreEntry *e = NULL; e = memAllocate(MEM_STOREENTRY); - if (mem_obj_flag) - e->mem_obj = new_MemObject(url, log_url); + /* We're always creating MemObjects now! */ + e->mem_obj = new_MemObject(url, log_url); debug(20, 3) ("new_StoreEntry: returning %p\n", e); e->expires = e->lastmod = e->lastref = e->timestamp = -1; e->swap_filen = -1; @@ -150,16 +147,15 @@ #if URL_CHECKSUM_DEBUG assert(mem->chksum == url_checksum(mem->url)); #endif + + e->mem_obj = NULL; - if (!shutting_down) - assert(mem->swapout.sio == NULL); - stmemFree(&mem->data_hdr); - mem->inmem_hi = 0; + /* * There is no way to abort FD-less clients, so they might * still have mem->clients set if mem->fd == -1 */ - assert(mem->fd == -1 || mem->clients.head == NULL); + assert(mem->fd == -1 || e->sc.valid == 0); httpReplyDestroy(mem->reply); requestUnlink(mem->request); mem->request = NULL; @@ -175,102 +171,55 @@ StoreEntry *e = data; debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); assert(e != NULL); - if (e->mem_obj) - destroy_MemObject(e); - storeHashDelete(e); - assert(e->hash.key == NULL); - memFree(e, MEM_STOREENTRY); -} + assert(e->mem_obj != NULL); -/* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ + dlinkDelete(&e->node, &store_list); -void -storeHashInsert(StoreEntry * e, const cache_key * key) -{ - debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", - e, storeKeyText(key)); - e->hash.key = storeKeyDup(key); - hash_join(store_table, &e->hash); -} + /* Free the reply .. */ + storeDetachReply(e); -static void -storeHashDelete(StoreEntry * e) -{ - hash_remove_link(store_table, &e->hash); - storeKeyFree(e->hash.key); - e->hash.key = NULL; + /* Scrape it clean .. */ + destroy_MemObject(e); + + memFree(e, MEM_STOREENTRY); } /* -------------------------------------------------------------------------- */ +#if 0 /* get rid of memory copy of the object */ /* Only call this if storeCheckPurgeMem(e) returns 1 */ static void storePurgeMem(StoreEntry * e) { - if (e->mem_obj == NULL) - return; + assert(e->mem_obj != NULL); debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", - storeKeyText(e->hash.key)); - storeSetMemStatus(e, NOT_IN_MEMORY); + storeKeyUrl(e)); destroy_MemObject(e); if (e->swap_status != SWAPOUT_DONE) storeRelease(e); } +#endif -static void -storeEntryReferenced(StoreEntry * e) -{ - SwapDir *SD; - - /* Notify the fs that we're referencing this object again */ - if (e->swap_dirn > -1) { - SD = INDEXSD(e->swap_dirn); - if (SD->refobj) - SD->refobj(SD, e); - } - /* Notify the memory cache that we're referencing this object again */ - if (e->mem_obj) { - if (mem_policy->Referenced) - mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl); - } -} - -static void -storeEntryDereferenced(StoreEntry * e) -{ - SwapDir *SD; - - /* Notify the fs that we're not referencing this object any more */ - if (e->swap_filen > -1) { - SD = INDEXSD(e->swap_dirn); - if (SD->unrefobj != NULL) - SD->unrefobj(SD, e); - } - /* Notify the memory cache that we're not referencing this object any more */ - if (e->mem_obj) { - if (mem_policy->Dereferenced) - mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl); - } -} void storeLockObject(StoreEntry * e) { + assert(e->mem_obj != NULL); e->lock_count++; debug(20, 3) ("storeLockObject: key '%s' count=%d\n", - storeKeyText(e->hash.key), (int) e->lock_count); + storeKeyUrl(e), (int) e->lock_count); e->lastref = squid_curtime; - storeEntryReferenced(e); } void storeReleaseRequest(StoreEntry * e) { + assert(e->mem_obj != NULL); if (EBIT_TEST(e->flags, RELEASE_REQUEST)) return; - debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyUrl(e)); EBIT_SET(e->flags, RELEASE_REQUEST); /* * Clear cachable flag here because we might get called before @@ -278,6 +227,15 @@ * prevents httpMakePublic from really setting a public key. */ EBIT_CLR(e->flags, ENTRY_CACHABLE); + /* + * Notify the reply layer that we want this object released.. + * XXX an evil hack - this can be called before the reply object + * is attached. If this happens, don't try to notify the reply + * object. When the reply object is created by createNewStoreEntry + * we'll be ok. -- adrian + */ + if (e->strelease != NULL) + e->strelease(e); storeSetPrivateKey(e); } @@ -286,28 +244,30 @@ int storeUnlockObject(StoreEntry * e) { + assert(e->mem_obj != NULL); e->lock_count--; debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", - storeKeyText(e->hash.key), e->lock_count); + storeKeyUrl(e), e->lock_count); if (e->lock_count) return (int) e->lock_count; - if (e->store_status == STORE_PENDING) + + /* If the object is still pending data, we set RELEASE_REQUEST and wait */ + if (e->store_status == STORE_PENDING) { EBIT_SET(e->flags, RELEASE_REQUEST); + return 0; + } + + /* The object is not pending data, so kill it .. */ assert(storePendingNClients(e) == 0); + assert(e->store_status == STORE_OK); + + /* Only release an object if we've been requested to.. */ if (EBIT_TEST(e->flags, RELEASE_REQUEST)) - storeRelease(e); - else if (storeKeepInMemory(e)) { - storeEntryDereferenced(e); - storeSetMemStatus(e, IN_MEMORY); - requestUnlink(e->mem_obj->request); - e->mem_obj->request = NULL; - } else { - storePurgeMem(e); - storeEntryDereferenced(e); - if (EBIT_TEST(e->flags, KEY_PRIVATE)) - debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__); - } - return 0; + storeRelease(e); + else + destroy_StoreEntry(e); + + return -1; } /* Lookup an object in the cache. @@ -316,7 +276,12 @@ storeGet(const cache_key * key) { debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); + return NULL; + + /* XXX we will perform a FS lookup here .. */ +#if 0 return (StoreEntry *) hash_lookup(store_table, key); +#endif } StoreEntry * @@ -325,43 +290,23 @@ return storeGet(storeKeyPublic(uri, method)); } -static int -getKeyCounter(void) -{ - static int key_counter = 0; - if (++key_counter < 0) - key_counter = 1; - return key_counter; -} - void storeSetPrivateKey(StoreEntry * e) { - const cache_key *newkey; - MemObject *mem = e->mem_obj; - if (e->hash.key && EBIT_TEST(e->flags, KEY_PRIVATE)) + if (EBIT_TEST(e->flags, KEY_PRIVATE)) return; /* is already private */ - if (e->hash.key) { - if (e->swap_filen > -1) - storeDirSwapLog(e, SWAP_LOG_DEL); - storeHashDelete(e); - } - if (mem != NULL) { - mem->id = getKeyCounter(); - newkey = storeKeyPrivate(mem->url, mem->method, mem->id); - } else { - newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter()); - } - assert(hash_lookup(store_table, newkey) == NULL); EBIT_SET(e->flags, KEY_PRIVATE); - storeHashInsert(e, newkey); + /* XXX this lame check can go !!! -- adrian */ + if (e->stmakeprivate) + e->stmakeprivate(e); } void storeSetPublicKey(StoreEntry * e) { +#if 0 StoreEntry *e2 = NULL; - const cache_key *newkey; +#endif MemObject *mem = e->mem_obj; if (e->hash.key && !EBIT_TEST(e->flags, KEY_PRIVATE)) return; /* is already public */ @@ -382,7 +327,12 @@ e->hash.key, mem->url); #endif assert(!EBIT_TEST(e->flags, RELEASE_REQUEST)); - newkey = storeKeyPublic(mem->url, mem->method); +#if 0 + /* + * We don't need this anymore, key collisions should be handled in + * the object store(s) + * -- adrian + */ if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); storeSetPrivateKey(e2); @@ -391,14 +341,18 @@ } if (e->hash.key) storeHashDelete(e); +#endif EBIT_CLR(e->flags, KEY_PRIVATE); - storeHashInsert(e, newkey); + /* XXX this lame check can go !!! -- adrian */ + if (e->stmakepublic) + e->stmakepublic(e); if (e->swap_filen > -1) storeDirSwapLog(e, SWAP_LOG_ADD); } StoreEntry * -storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method) +storeCreateEntry(const char *url, const char *log_url, request_flags flags, + method_t method, reply_obj_t reply_obj) { StoreEntry *e = NULL; MemObject *mem = NULL; @@ -408,6 +362,20 @@ e->lock_count = 1; /* Note lock here w/o calling storeLock() */ mem = e->mem_obj; mem->method = method; + dlinkAdd(e, &e->node, &store_list); + + switch (reply_obj) { + case REPLY_OBJ_INTERNAL: + reply_internal_create(e); + break; + case REPLY_OBJ_NETWORK: + reply_network_create(e); + break; + default: + fatal("No valid reply_obj given to storeCreateEntry!\n"); + /* NOTREACHED */ + } + if (neighbors_do_private_keys || !flags.hierarchical) storeSetPrivateKey(e); else @@ -419,8 +387,8 @@ EBIT_CLR(e->flags, ENTRY_CACHABLE); storeReleaseRequest(e); } + e->store_status = STORE_PENDING; - storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAPOUT_NONE; e->swap_filen = -1; e->swap_dirn = -1; @@ -428,7 +396,6 @@ e->lastref = squid_curtime; e->timestamp = -1; /* set in storeTimestampsSet() */ e->ping_status = PING_NONE; - EBIT_SET(e->flags, ENTRY_VALIDATED); return e; } @@ -436,7 +403,7 @@ void storeExpireNow(StoreEntry * e) { - debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyUrl(e)); e->expires = squid_curtime; } @@ -444,22 +411,7 @@ void storeAppend(StoreEntry * e, const char *buf, int len) { - MemObject *mem = e->mem_obj; - assert(mem != NULL); - assert(len >= 0); - assert(e->store_status == STORE_PENDING); - if (len) { - debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", - len, - storeKeyText(e->hash.key)); - storeGetMemSpace(len); - stmemAppend(&mem->data_hdr, buf, len); - mem->inmem_hi += len; - } - if (EBIT_TEST(e->flags, DELAY_SENDING)) - return; - InvokeHandlers(e); - storeSwapOut(e); + e->stappend(e, buf, len); } void @@ -562,7 +514,7 @@ return 0; /* avoid release call below */ } else if ((e->mem_obj->reply->content_length > 0 && e->mem_obj->reply->content_length > Config.Store.maxObjectSize) || - e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { + storeMemHiOffset(e) > Config.Store.maxObjectSize) { debug(20, 2) ("storeCheckCachable: NO: too big\n"); store_check_cachable_hist.no.too_big++; } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) { @@ -629,17 +581,17 @@ void storeComplete(StoreEntry * e) { - debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeComplete: '%s'\n", storeKeyUrl(e)); if (e->store_status != STORE_PENDING) { /* * if we're not STORE_PENDING, then probably we got aborted * and there should be NO clients on this entry */ assert(EBIT_TEST(e->flags, ENTRY_ABORTED)); - assert(e->mem_obj->nclients == 0); + assert(e->sc.valid == 0); return; } - e->mem_obj->object_sz = e->mem_obj->inmem_hi; + e->mem_obj->object_sz = storeMemHiOffset(e); e->store_status = STORE_OK; assert(e->mem_status == NOT_IN_MEMORY); if (!storeEntryValidLength(e)) { @@ -656,7 +608,6 @@ * responses without content length would sometimes get released * in client_side, thinking that the response is incomplete. */ - storeSwapOut(e); InvokeHandlers(e); } @@ -671,18 +622,17 @@ MemObject *mem = e->mem_obj; assert(e->store_status == STORE_PENDING); assert(mem != NULL); - debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->hash.key)); + debug(20, 6) ("storeAbort: %s\n", storeKeyUrl(e)); storeLockObject(e); /* lock while aborting */ storeNegativeCache(e); storeReleaseRequest(e); EBIT_SET(e->flags, ENTRY_ABORTED); - storeSetMemStatus(e, NOT_IN_MEMORY); e->store_status = STORE_OK; /* * We assign an object length here. The only other place we assign * the object length is in storeComplete() */ - mem->object_sz = mem->inmem_hi; + mem->object_sz = storeMemHiOffset(e); /* Notify the server side */ if (mem->abort.callback) { eventAdd("mem->abort.callback", @@ -696,10 +646,11 @@ /* Notify the client side */ InvokeHandlers(e); /* Close any swapout file */ - storeSwapOutFileClose(e); storeUnlockObject(e); /* unlock */ } +/* This function isn't used right now .. -- adrian */ +#if 0 /* Clear Memory storage to accommodate the given object len */ static void storeGetMemSpace(int size) @@ -729,10 +680,8 @@ debug(20, 3) (" %6d HOT objects\n", hot_obj_count); debug(20, 3) (" %6d were released\n", released); } +#endif -/* The maximum objects to scan for maintain storage space */ -#define MAINTAIN_MAX_SCAN 1024 -#define MAINTAIN_MAX_REMOVE 64 /* * This routine is to be called by main loop in main.c. @@ -774,7 +723,8 @@ void storeRelease(StoreEntry * e) { - debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyUrl(e)); + /* If, for any reason we can't discard this object because of an * outstanding request, mark it for pending release */ if (storeEntryLocked(e)) { @@ -783,81 +733,24 @@ storeReleaseRequest(e); return; } - if (store_dirs_rebuilding && e->swap_filen > -1) { - storeSetPrivateKey(e); - if (e->mem_obj) { - storeSetMemStatus(e, NOT_IN_MEMORY); - destroy_MemObject(e); - } - if (e->swap_filen > -1) { - /* - * Fake a call to storeLockObject(). When rebuilding is done, - * we'll just call storeUnlockObject() on these. - */ - e->lock_count++; - EBIT_SET(e->flags, RELEASE_REQUEST); - stackPush(&LateReleaseStack, e); - return; - } else { - destroy_StoreEntry(e); - } - } + + /* Notify the reply layer to release this object */ + if (!EBIT_TEST(e->flags, RELEASE_REQUEST)) + e->strelease(e); + + /* Time to deallocate the storeentry .. */ storeLog(STORE_LOG_RELEASE, e); - if (e->swap_filen > -1) { - storeUnlink(e); - if (e->swap_status == SWAPOUT_DONE) - if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) - storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, -1); - if (!EBIT_TEST(e->flags, KEY_PRIVATE)) - storeDirSwapLog(e, SWAP_LOG_DEL); -#if 0 - /* From 2.4. I think we do this in storeUnlink? */ - storeSwapFileNumberSet(e, -1); -#endif - } - storeSetMemStatus(e, NOT_IN_MEMORY); + + /* Here we'd log a swap log delete, but thats done at the FS layer .. */ destroy_StoreEntry(e); } -static void -storeLateRelease(void *unused) -{ - StoreEntry *e; - int i; - static int n = 0; - if (store_dirs_rebuilding) { - eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); - return; - } - for (i = 0; i < 10; i++) { - e = stackPop(&LateReleaseStack); - if (e == NULL) { - /* done! */ - debug(20, 1) ("storeLateRelease: released %d objects\n", n); - return; - } - storeUnlockObject(e); - n++; - } - eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1); -} - /* return 1 if a store entry is locked */ int storeEntryLocked(const StoreEntry * e) { if (e->lock_count) return 1; - if (e->swap_status == SWAPOUT_WRITING) - return 1; - if (e->store_status == STORE_PENDING) - return 1; - /* - * SPECIAL, PUBLIC entries should be "locked" - */ - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) - if (!EBIT_TEST(e->flags, KEY_PRIVATE)) - return 1; return 0; } @@ -868,7 +761,7 @@ const HttpReply *reply; assert(e->mem_obj != NULL); reply = e->mem_obj->reply; - debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->hash.key)); + debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyUrl(e)); debug(20, 5) ("storeEntryValidLength: object_len = %d\n", objectLen(e)); debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", @@ -877,17 +770,17 @@ reply->content_length); if (reply->content_length < 0) { debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 1; } if (reply->hdr_sz == 0) { debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 1; } if (e->mem_obj->method == METHOD_HEAD) { debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 1; } if (reply->sline.status == HTTP_NOT_MODIFIED) @@ -900,7 +793,7 @@ debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", diff < 0 ? -diff : diff, diff < 0 ? "big" : "small", - storeKeyText(e->hash.key)); + storeKeyUrl(e)); return 0; } @@ -925,15 +818,13 @@ void storeInit(void) { + store_list.head = NULL; + store_list.tail = NULL; storeKeyInit(); storeInitHashValues(); - store_table = hash_create(storeKeyHashCmp, - store_hash_buckets, storeKeyHashHash); mem_policy = createRemovalPolicy(Config.memPolicy); storeDigestInit(); storeLogOpen(); - stackInit(&LateReleaseStack); - eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); storeDirInit(); storeRebuildStart(); cachemgrRegister("storedir", @@ -957,17 +848,6 @@ store_pages_max = Config.memMaxSize / SM_PAGE_SIZE; } -static int -storeKeepInMemory(const StoreEntry * e) -{ - MemObject *mem = e->mem_obj; - if (mem == NULL) - return 0; - if (mem->data_hdr.head == NULL) - return 0; - return mem->inmem_lo == 0; -} - void storeNegativeCache(StoreEntry * e) { @@ -978,8 +858,6 @@ void storeFreeMemory(void) { - hashFreeItems(store_table, destroy_StoreEntry); - hashFreeMemory(store_table); store_table = NULL; #if USE_CACHE_DIGESTS if (store_digest) @@ -1061,23 +939,9 @@ void storeMemObjectDump(MemObject * mem) { - debug(20, 1) ("MemObject->data.head: %p\n", - mem->data_hdr.head); - debug(20, 1) ("MemObject->data.tail: %p\n", - mem->data_hdr.tail); - debug(20, 1) ("MemObject->data.origin_offset: %d\n", - mem->data_hdr.origin_offset); debug(20, 1) ("MemObject->start_ping: %d.%06d\n", (int) mem->start_ping.tv_sec, (int) mem->start_ping.tv_usec); - debug(20, 1) ("MemObject->inmem_hi: %d\n", - (int) mem->inmem_hi); - debug(20, 1) ("MemObject->inmem_lo: %d\n", - (int) mem->inmem_lo); - debug(20, 1) ("MemObject->clients: %p\n", - mem->clients); - debug(20, 1) ("MemObject->nclients: %d\n", - mem->nclients); debug(20, 1) ("MemObject->reply: %p\n", mem->reply); debug(20, 1) ("MemObject->request: %p\n", @@ -1090,8 +954,7 @@ void storeEntryDump(const StoreEntry * e, int l) { - debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->hash.key)); - debug(20, l) ("StoreEntry->next: %p\n", e->hash.next); + debug(20, l) ("StoreEntry->key: %s\n", storeKeyUrl(e)); debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj); debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp); debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref); @@ -1109,40 +972,6 @@ debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status); } -/* - * NOTE, this function assumes only two mem states - */ -void -storeSetMemStatus(StoreEntry * e, int new_status) -{ - MemObject *mem = e->mem_obj; - if (new_status == e->mem_status) - return; - assert(mem != NULL); - if (new_status == IN_MEMORY) { - assert(mem->inmem_lo == 0); - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n", - mem->url); - } else { - mem_policy->Add(mem_policy, e, &mem->repl); - debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n", - mem->url); - } - hot_obj_count++; - } else { - if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { - debug(20, 4) ("storeSetMemStatus: special entry %s\n", - mem->url); - } else { - mem