--------------------- PatchSet 10170 Date: 2007/11/27 08:12:17 Author: adri Branch: store_copy Tag: (none) Log: Merge changes from Squid-2.HEAD. Members: ChangeLog:1.93->1.93.2.1 configure.in:1.184->1.184.2.1 doc/release-notes/release-2.6.html:1.47->1.47.2.1 doc/release-notes/release-2.6.sgml:1.59->1.59.2.1 helpers/basic_auth/DB/.cvsignore:1.2->1.2.2.1 helpers/basic_auth/DB/squid_db_auth.in:1.2->1.2.4.1 helpers/basic_auth/squid_radius_auth/ChangeLog:1.1->1.1.4.1(DEAD) helpers/basic_auth/squid_radius_auth/Makefile.am:1.1->1.1.4.1 helpers/basic_auth/squid_radius_auth/md5.c:1.1->1.1.4.1(DEAD) helpers/basic_auth/squid_radius_auth/md5.h:1.1->1.1.4.1(DEAD) helpers/basic_auth/squid_radius_auth/squid_rad_auth.c:1.1->1.1.4.1 helpers/basic_auth/squid_radius_auth/util.c:1.1->1.1.4.1 helpers/basic_auth/squid_radius_auth/util.h:1.1->1.1.4.1 include/Array.h:1.7->1.7.52.1 include/md5.h:1.12->1.12.34.1 lib/Array.c:1.6->1.6.52.1 lib/Makefile.am:1.10->1.10.18.1 lib/md5.c:1.12->1.12.30.1 src/HttpHeader.c:1.28->1.28.6.1 src/HttpReply.c:1.23->1.23.6.1 src/HttpRequest.c:1.17->1.17.10.1 src/Makefile.am:1.54->1.54.2.1 src/access_log.c:1.43->1.43.2.1 src/cache_cf.c:1.102->1.102.2.1 src/cf.data.depend:1.1->1.1.4.1 src/cf.data.pre:1.209->1.209.2.1 src/client_side.c:1.202.2.6->1.202.2.7 src/client_side_rewrite.c:1.2.4.1->1.2.4.2 src/client_side_storeurl_rewrite.c:1.3.2.1->1.3.2.2 src/enums.h:1.62->1.62.2.1 src/errorpage.c:1.37->1.37.6.1 src/fqdncache.c:1.22->1.22.2.1 src/helper.c:1.43->1.43.6.1 src/http.c:1.63->1.63.2.1 src/ipcache.c:1.25->1.25.2.1 src/main.c:1.80->1.80.2.1 src/mib.txt:1.9->1.9.4.1 src/protos.h:1.146.2.3->1.146.2.4 src/redirect.c:1.16->1.16.10.1 src/refresh.c:1.15->1.15.6.1 src/refresh_check.c:1.2.4.1->1.2.4.2 src/stat.c:1.42.2.2->1.42.2.3 src/store.c:1.49->1.49.4.1 src/store_client.c:1.25.10.4->1.25.10.5 src/store_key_md5.c:1.10->1.10.10.1 src/store_rewrite.c:1.1.4.1->1.1.4.2 src/store_swapmeta.c:1.12->1.12.22.1 src/store_update.c:1.2.4.1->1.2.4.2 src/structs.h:1.158.2.4->1.158.2.5 src/typedefs.h:1.43.2.2->1.43.2.3 src/fs/aufs/aiops.c:1.28->1.28.18.1 src/fs/aufs/aiops_win32.c:1.4->1.4.28.1 src/fs/aufs/store_dir_aufs.c:1.50->1.50.6.1 Index: squid/ChangeLog =================================================================== RCS file: /cvsroot/squid-sf//squid/ChangeLog,v retrieving revision 1.93 retrieving revision 1.93.2.1 diff -u -r1.93 -r1.93.2.1 --- squid/ChangeLog 23 Sep 2007 16:52:56 -0000 1.93 +++ squid/ChangeLog 27 Nov 2007 08:12:17 -0000 1.93.2.1 @@ -39,6 +39,45 @@ - Implementation of an external logfile daemon (ie, logging to a pipe); UDP logging (based in part on work done by Tim Starling) +Changes to squid-2.6.STABLE17 (26 Nov 2007) + + - Fix compile error with old GCC 2.x or other ANSI-C compilers before + C99 + - Mention the login= cache_peer option in release notes + - Fix bad cache_peer example in squid.conf + - Bug #2086: Fix a compile-time memory corruption error causing cf_gen + to fail + - Bug #2048: Clarify high_memory_warning usage + - Reject DNS responses which result in no data + - Fix version number in configuration manual + - Move cache and request/reply_header_max_size to their proper + sections + - Bug #2088: sbrk statistics broken when process size >2GB + - Move logopen() much earlier to have fatal startup errors sent to the + proper syslog facility + - Fix HTTP/0.9 responses + - Correct bad example config for tos_outgoing_tos + - Fix grammar in description of mail_program squid.conf option + - Ignore Content-Length in chunked responses instead of rejecting the + response as invalid + - Documented that http_port no longer have a default + - Cleanup of cache digest documentation + - Make aufs store rebuilding back off a little if I/O load too high + - Bug #2100: Respect DNS ttl=0 + - Update udp_(incoming|outgoing)_address documentation to reflect + current bahaviour. + - Update HTCP documentation + - Document the overlapping helper request format + - Change priority of proxy auth and extacl provided username in + login=*:pass + - pack header entries on cache updates + - Make squid_db_auth reopen the database connection on each query by + default + - Improve helper debug ouput, including the channel number + - Update cachePeerEntry MIB description to mention what is used as + index key + - Import squid_radius_auth for authenticating to RADIUS + Changes to squid-2.6.STABLE16 (5 Sep 2007) - Test for sys/capability.h linux include file to avoid failing on Index: squid/configure.in =================================================================== RCS file: /cvsroot/squid-sf//squid/configure.in,v retrieving revision 1.184 retrieving revision 1.184.2.1 diff -u -r1.184 -r1.184.2.1 --- squid/configure.in 5 Sep 2007 22:52:14 -0000 1.184 +++ squid/configure.in 27 Nov 2007 08:12:17 -0000 1.184.2.1 @@ -1,7 +1,7 @@ dnl dnl Configuration input file for Squid dnl -dnl $Id: configure.in,v 1.184 2007/09/05 22:52:14 squidadm Exp $ +dnl $Id: configure.in,v 1.184.2.1 2007/11/27 08:12:17 adri Exp $ dnl dnl dnl @@ -10,7 +10,7 @@ AM_CONFIG_HEADER(include/autoconf.h) AC_CONFIG_AUX_DIR(cfgaux) AM_INIT_AUTOMAKE -AC_REVISION($Revision: 1.184 $)dnl +AC_REVISION($Revision: 1.184.2.1 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AM_MAINTAINER_MODE @@ -686,11 +686,13 @@ fi ]) +dnl SSL is not enabled by default. AM_CONDITIONAL(ENABLE_SSL, false) SSLLIB='' SSLFLAGS='' +dnl Default is to use OpenSSL when available AC_ARG_ENABLE(ssl, [ --enable-ssl Enable ssl gatewaying support using OpenSSL], [ if test "$enableval" != "no"; then @@ -709,8 +711,7 @@ fi ]) -AM_CONDITIONAL(NEED_OWN_MD5, true) - +dnl User may specify OpenSSL is needed from a non-standard location AC_ARG_WITH(openssl, [ --with-openssl[=prefix] Compile with the OpenSSL libraries. The path to @@ -739,7 +740,6 @@ AC_DEFINE(USE_OPENSSL, 1, [Define this to make use of the OpenSSL libraries for MD5 calculation rather than Squid's own MD5 implementation or if building with SSL encryption (USE_SSL)]) - AM_CONDITIONAL(NEED_OWN_MD5, false) if test -z "$SSLLIB"; then SSLLIB="-lcrypto" # for MD5 routines fi @@ -751,6 +751,7 @@ fi AC_SUBST(SSLLIB) + AC_ARG_ENABLE(forw-via-db, [ --enable-forw-via-db Enable Forw/Via database], [ if test "$enableval" = "yes" ; then @@ -1635,6 +1636,7 @@ sys/file.h \ sys/ioctl.h \ sys/mount.h \ + sys/md5.h \ sys/msg.h \ sys/prctl.h \ sys/resource.h \ @@ -2103,7 +2105,11 @@ dnl Check for libcrypt dnl Some of our helpers use crypt(3) which may be in libc, or in dnl libcrypt (eg FreeBSD) -AC_CHECK_LIB(crypt, crypt, [CRYPTLIB="-lcrypt"]) +AC_CHECK_LIB(crypt, crypt, [CRYPTLIB=" -lcrypt "]) + +dnl Solaris10 provides MD5 natively through libmd5 +AC_CHECK_LIB(md5, MD5Init, [CRYPTLIB+=" -lmd5 "]) + AC_SUBST(CRYPTLIB) dnl Check for libdl, used by auth_modules/PAM Index: squid/doc/release-notes/release-2.6.html =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/release-notes/release-2.6.html,v retrieving revision 1.47 retrieving revision 1.47.2.1 diff -u -r1.47 -r1.47.2.1 --- squid/doc/release-notes/release-2.6.html 5 Sep 2007 22:52:15 -0000 1.47 +++ squid/doc/release-notes/release-2.6.html 27 Nov 2007 08:12:19 -0000 1.47.2.1 @@ -2,12 +2,12 @@ - Squid 2.6.STABLE16 release notes + Squid 2.6.STABLE17 release notes -

Squid 2.6.STABLE16 release notes

+

Squid 2.6.STABLE17 release notes

-

Squid Developers

$Id: release-2.6.html,v 1.47 2007/09/05 22:52:15 squidadm Exp $ +

Squid Developers

$Id: release-2.6.html,v 1.47.2.1 2007/11/27 08:12:19 adri Exp $
This document contains the release notes for version 2.6 of Squid. Squid is a WWW Cache application developed by the Web Caching community. @@ -75,6 +75,9 @@

21. Key changes squid-2.6.STABLE15 to 2.6.STABLE16

+

+

22. Key changes squid-2.6.STABLE16 to 2.6.STABLE17

+

1. Key changes from squid 2.5

@@ -153,7 +156,7 @@

-
http_port

Now takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual.

+
http_port

Now takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual. Also there is no longer a default port and you must include a http_port directive in your squid.conf if you want Squid to listen for http requests.

httpd_accel_* for transparent proxy

Now implemented by the "transparent" http_port option

httpd_accel_* for accelerator mode

Now implemented by other options and the http_port accel option. See individual directives below.

httpd_accel_host

Replaced by defaultsite http_port option and cache_peer originserver option.

@@ -207,6 +210,7 @@
check_hostnames

Control if Squid should check the sanity of host names before trying to look them up in DNS

allow_underscores

Control if _ is to be considered a valid character in hostnames or not

cache_peer carp-load-factor

Option removed. CARP now uses the weight parameter instead.

+
cache_peer login=

Controls how Squid forwards login information to peer caches and servers.

@@ -743,5 +747,20 @@

+

22. Key changes squid-2.6.STABLE16 to 2.6.STABLE17

+ +

+

+

+ Index: squid/doc/release-notes/release-2.6.sgml =================================================================== RCS file: /cvsroot/squid-sf//squid/doc/release-notes/release-2.6.sgml,v retrieving revision 1.59 retrieving revision 1.59.2.1 diff -u -r1.59 -r1.59.2.1 --- squid/doc/release-notes/release-2.6.sgml 9 Sep 2007 09:53:06 -0000 1.59 +++ squid/doc/release-notes/release-2.6.sgml 27 Nov 2007 08:12:19 -0000 1.59.2.1 @@ -1,8 +1,8 @@
-Squid 2.6.STABLE16 release notes +Squid 2.6.STABLE17 release notes Squid Developers -$Id: release-2.6.sgml,v 1.59 2007/09/09 09:53:06 squidadm Exp $ +$Id: release-2.6.sgml,v 1.59.2.1 2007/11/27 08:12:19 adri Exp $ This document contains the release notes for version 2.6 of Squid. @@ -82,7 +82,7 @@ Changes to squid.conf

-http_portNow takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual. +http_portNow takes a list of options in addition to the port address, specifying the purpose of this http_port. Default is plain Internet proxy as usual. Also there is no longer a default port and you must include a http_port directive in your squid.conf if you want Squid to listen for http requests. httpd_accel_* for transparent proxyNow implemented by the "transparent" http_port option httpd_accel_* for accelerator modeNow implemented by other options and the http_port accel option. See individual directives below. httpd_accel_hostReplaced by defaultsite http_port option and cache_peer originserver option. @@ -638,5 +638,20 @@ name="ChangeLog"> file for details. +Key changes squid-2.6.STABLE16 to 2.6.STABLE17 + +

+ + A temporary memory leak fixed on cache updates + New squid_radius_auth RADIUS authentication helper + Documentation cleanups + Code cleanups and portability fixes + Several minor bugfixes + See also the list of and the file for details. + +

Index: squid/helpers/basic_auth/DB/.cvsignore =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/DB/.cvsignore,v retrieving revision 1.2 retrieving revision 1.2.2.1 diff -u -r1.2 -r1.2.2.1 --- squid/helpers/basic_auth/DB/.cvsignore 3 Sep 2007 04:51:58 -0000 1.2 +++ squid/helpers/basic_auth/DB/.cvsignore 27 Nov 2007 08:12:20 -0000 1.2.2.1 @@ -2,4 +2,4 @@ Makefile Makefile.in squid_db_auth -squid_db_auth.8Makefile.in +squid_db_auth.8 Index: squid/helpers/basic_auth/DB/squid_db_auth.in =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/DB/squid_db_auth.in,v retrieving revision 1.2 retrieving revision 1.2.4.1 diff -u -r1.2 -r1.2.4.1 --- squid/helpers/basic_auth/DB/squid_db_auth.in 22 Aug 2007 06:52:10 -0000 1.2 +++ squid/helpers/basic_auth/DB/squid_db_auth.in 27 Nov 2007 08:12:20 -0000 1.2.4.1 @@ -21,6 +21,7 @@ my $db_passwdcol = "password"; my $db_cond = "enabled = 1"; my $plaintext = 0; +my $persist = 0; =pod @@ -66,6 +67,10 @@ Database contains plain-text passwords +=item B<--persist> + +Keep a persistent database connection open between queries. + =back =cut @@ -79,13 +84,30 @@ 'passwdcol=s' => \$db_passwdcol, 'cond=s' => \$db_cond, 'plaintext' => \$plaintext, + 'persist' => \$persist, ); -my $dbh = DBI->connect($dsn, $db_user, $db_passwd) || die ("Could not connect to $dsn\n"); +my ($_dbh, $_sth); -my ($sth) = $dbh->prepare("SELECT $db_passwdcol FROM $db_table WHERE $db_usercol = ?" . ($db_cond ne "" ? " AND $db_cond" : "")) || die; +sub close_db() +{ + return if !defined($_dbh); + $_dbh->disconnect(); + undef $_dbh; + undef $_sth; +} -my $status; +sub open_db() +{ + return $_sth if defined $_sth; + $_dbh = DBI->connect($dsn, $db_user, $db_passwd); + if (!defined $_dbh) { + warn ("Could not connect to $dsn\n"); + return undef; + } + $_sth = $_dbh->prepare("SELECT $db_passwdcol FROM $db_table WHERE $db_usercol = ?" . ($db_cond ne "" ? " AND $db_cond" : "")) || die; + return $_sth; +} sub check_password($$) { @@ -97,20 +119,34 @@ return 0; } + +sub query_db($) { + my ($user) = @_; + my ($sth) = open_db() || return undef; + if (!$sth->execute($user)) { + close_db(); + open_db() || return undef; + $sth->execute($user) || return undef;; + } + return $sth; +} +my $status; + while (<>) { my ($user, $password) = split; $status = "ERR"; $user =~ s/%(..)/pack("H*", $1)/ge; $password =~ s/%(..)/pack("H*", $1)/ge; - $status = "ERR internal error"; - $sth->execute($user) || next; + $status = "ERR database error"; + my $sth = query_db($user) || next; $status = "ERR unknown login"; - my ($row) = $sth->fetchrow_arrayref() || next; + my $row = $sth->fetchrow_arrayref() || next; $status = "ERR login failure"; next if (!check_password($password, @$row[0])); $status = "OK"; } continue { + close_db() if (!$persist); print $status . "\n"; } --- squid/helpers/basic_auth/squid_radius_auth/ChangeLog Wed Nov 28 01:20:24 2007 +++ /dev/null Wed Nov 28 01:20:24 2007 @@ -1,47 +0,0 @@ -2007-08-27 02:10 hno - - * Makefile.in: Bootstrapped - -2007-08-24 14:31 hno - - * Makefile.in: Bootstrapped - -2007-08-24 04:36 amosjeffries - - * Makefile.in: Bootstrapped. - -2007-08-18 02:11 hno - - * Makefile.in: Bootstrapped - -2007-08-17 20:58 serassio - - * Makefile.in: Bootstrapped - -2007-08-15 02:11 hno - - * Makefile.in: Bootstrapped - -2007-08-14 19:07 serassio - - * Makefile.in: Bootstrapped - -2007-08-13 10:28 serassio - - * .cvsignore: Added/updated .cvsignore files - -2007-06-25 15:57 hno - - * Makefile.in: Bootstrapped - -2007-06-25 13:02 hno - - * COPYRIGHT, Changelog, Make.inc, Makefile.am, Makefile.default, - Makefile.in, Makefile.solaris, README, md5.c, md5.h, radius.h, - squid_rad_auth.c, squid_radius_auth.8, util.c, util.h: Import - squid_radius_auth helper for basic authentication to RADIUS - -2004-08-07 20:09 hno - - * etc/squid_radius_auth.conf: Import of version 1.07 - Index: squid/helpers/basic_auth/squid_radius_auth/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/Makefile.am,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/Makefile.am 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/Makefile.am 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -1,7 +1,7 @@ # # Makefile for the Squid LDAP authentication helper # -# $Id: Makefile.am,v 1.1 2007/09/05 22:52:16 squidadm Exp $ +# $Id: Makefile.am,v 1.1.4.1 2007/11/27 08:12:20 adri Exp $ # # Uncomment and customize the following to suit your needs: # @@ -11,8 +11,8 @@ EXTRA_DIST = squid_radius_auth.8 squid_radius_auth_SOURCES = \ squid_rad_auth.c \ - md5.c md5.h \ radius.h \ util.c util.h -LDADD = $(XTRA_LIBS) +LDADD = -L$(top_builddir)/lib -lmiscutil $(SSLLIB) $(XTRA_LIBS) +INCLUDES = -I$(top_srcdir)/include --- squid/helpers/basic_auth/squid_radius_auth/md5.c Wed Nov 28 01:20:24 2007 +++ /dev/null Wed Nov 28 01:20:24 2007 @@ -1,333 +0,0 @@ -/* MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm - */ - -/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All -rights reserved. - -License to copy and use this software is granted provided that it -is identified as the "RSA Data Security, Inc. MD5 Message-Digest -Algorithm" in all material mentioning or referencing this software -or this function. - -License is also granted to make and use derivative works provided -that such works are identified as "derived from the RSA Data -Security, Inc. MD5 Message-Digest Algorithm" in all material -mentioning or referencing the derived work. - -RSA Data Security, Inc. makes no representations concerning either -the merchantability of this software or the suitability of this -software for any particular purpose. It is provided "as is" -without express or implied warranty of any kind. - -These notices must be retained in any copies of any part of this -documentation and/or software. - */ - -#include "md5.h" - -/* Constants for MD5Transform routine. - */ -#define S11 7 -#define S12 12 -#define S13 17 -#define S14 22 -#define S21 5 -#define S22 9 -#define S23 14 -#define S24 20 -#define S31 4 -#define S32 11 -#define S33 16 -#define S34 23 -#define S41 6 -#define S42 10 -#define S43 15 -#define S44 21 - -static void MD5Transform PROTO_LIST ((UINT4 [4], unsigned char [64])); -static void Encode PROTO_LIST - ((unsigned char *, UINT4 *, unsigned int)); -static void Decode PROTO_LIST - ((UINT4 *, unsigned char *, unsigned int)); -static void MD5_memcpy PROTO_LIST ((POINTER, POINTER, unsigned int)); -static void MD5_memset PROTO_LIST ((POINTER, int, unsigned int)); - -static unsigned char PADDING[64] = { - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; - -/* F, G, H and I are basic MD5 functions. - */ -#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) -#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) -#define H(x, y, z) ((x) ^ (y) ^ (z)) -#define I(x, y, z) ((y) ^ ((x) | (~z))) - -/* ROTATE_LEFT rotates x left n bits. - */ -#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n)))) - -/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. -Rotation is separate from addition to prevent recomputation. - */ -#define FF(a, b, c, d, x, s, ac) { \ - (a) += F ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } -#define GG(a, b, c, d, x, s, ac) { \ - (a) += G ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } -#define HH(a, b, c, d, x, s, ac) { \ - (a) += H ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } -#define II(a, b, c, d, x, s, ac) { \ - (a) += I ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - } - -/* MD5 initialization. Begins an MD5 operation, writing a new context. - */ -void MD5Init (context) -MD5_CTX *context; /* context */ -{ - context->count[0] = context->count[1] = 0; - /* Load magic initialization constants. -*/ - context->state[0] = 0x67452301; - context->state[1] = 0xefcdab89; - context->state[2] = 0x98badcfe; - context->state[3] = 0x10325476; -} - -/* MD5 block update operation. Continues an MD5 message-digest - operation, processing another message block, and updating the - context. - */ -void MD5Update (context, input, inputLen) -MD5_CTX *context; /* context */ -unsigned char *input; /* input block */ -unsigned int inputLen; /* length of input block */ -{ - unsigned int i, index, partLen; - - /* Compute number of bytes mod 64 */ - index = (unsigned int)((context->count[0] >> 3) & 0x3F); - - /* Update number of bits */ - if ((context->count[0] += ((UINT4)inputLen << 3)) - < ((UINT4)inputLen << 3)) - context->count[1]++; - context->count[1] += ((UINT4)inputLen >> 29); - - partLen = 64 - index; - - /* Transform as many times as possible. -*/ - if (inputLen >= partLen) { - MD5_memcpy - ((POINTER)&context->buffer[index], (POINTER)input, partLen); - MD5Transform (context->state, context->buffer); - - for (i = partLen; i + 63 < inputLen; i += 64) - MD5Transform (context->state, &input[i]); - - index = 0; - } - else - i = 0; - - /* Buffer remaining input */ - MD5_memcpy - ((POINTER)&context->buffer[index], (POINTER)&input[i], - inputLen-i); -} - -/* MD5 finalization. Ends an MD5 message-digest operation, writing the - the message digest and zeroizing the context. - */ -void MD5Final (digest, context) -unsigned char digest[16]; /* message digest */ -MD5_CTX *context; /* context */ -{ - unsigned char bits[8]; - unsigned int index, padLen; - - /* Save number of bits */ - Encode (bits, context->count, 8); - - /* Pad out to 56 mod 64. -*/ - index = (unsigned int)((context->count[0] >> 3) & 0x3f); - padLen = (index < 56) ? (56 - index) : (120 - index); - MD5Update (context, PADDING, padLen); - - /* Append length (before padding) */ - MD5Update (context, bits, 8); - - /* Store state in digest */ - Encode (digest, context->state, 16); - - /* Zeroize sensitive information. -*/ - MD5_memset ((POINTER)context, 0, sizeof (*context)); -} - -/* MD5 basic transformation. Transforms state based on block. - */ -static void MD5Transform (state, block) -UINT4 state[4]; -unsigned char block[64]; -{ - UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; - - Decode (x, block, 64); - - /* Round 1 */ - FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */ - FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */ - FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */ - FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */ - FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */ - FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */ - FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */ - FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */ - FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */ - FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */ - FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ - FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ - FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ - FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ - FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ - FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ - - /* Round 2 */ - GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */ - GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */ - GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ - GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */ - GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */ - GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */ - GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ - GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */ - GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */ - GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ - GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */ - GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */ - GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ - GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */ - GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */ - GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ - - /* Round 3 */ - HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */ - HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */ - HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ - HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ - HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */ - HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */ - HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */ - HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ - HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ - HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */ - HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */ - HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */ - HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */ - HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ - HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ - HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */ - - /* Round 4 */ - II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */ - II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */ - II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ - II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */ - II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ - II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */ - II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ - II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */ - II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */ - II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ - II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */ - II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ - II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */ - II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ - II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */ - II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */ - - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - - /* Zeroize sensitive information. -*/ - MD5_memset ((POINTER)x, 0, sizeof (x)); -} - -/* Encodes input (UINT4) into output (unsigned char). Assumes len is - a multiple of 4. - */ -static void Encode (output, input, len) -unsigned char *output; -UINT4 *input; -unsigned int len; -{ - unsigned int i, j; - - for (i = 0, j = 0; j < len; i++, j += 4) { - output[j] = (unsigned char)(input[i] & 0xff); - output[j+1] = (unsigned char)((input[i] >> 8) & 0xff); - output[j+2] = (unsigned char)((input[i] >> 16) & 0xff); - output[j+3] = (unsigned char)((input[i] >> 24) & 0xff); - } -} - -/* Decodes input (unsigned char) into output (UINT4). Assumes len is - a multiple of 4. - */ -static void Decode (output, input, len) -UINT4 *output; -unsigned char *input; -unsigned int len; -{ - unsigned int i, j; - - for (i = 0, j = 0; j < len; i++, j += 4) - output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) | - (((UINT4)input[j+2]) << 16) | (((UINT4)input[j+3]) << 24); -} - -/* Note: Replace "for loop" with standard memcpy if possible. - */ - -static void MD5_memcpy (output, input, len) -POINTER output; -POINTER input; -unsigned int len; -{ - unsigned int i; - - for (i = 0; i < len; i++) - output[i] = input[i]; -} - -/* Note: Replace "for loop" with standard memset if possible. - */ -static void MD5_memset (output, value, len) -POINTER output; -int value; -unsigned int len; -{ - unsigned int i; - - for (i = 0; i < len; i++) - ((char *)output)[i] = (char)value; -} --- squid/helpers/basic_auth/squid_radius_auth/md5.h Wed Nov 28 01:20:24 2007 +++ /dev/null Wed Nov 28 01:20:24 2007 @@ -1,77 +0,0 @@ -/* GLOBAL.H - RSAREF types and constants - */ - -/* PROTOTYPES should be set to one if and only if the compiler supports - function argument prototyping. - The following makes PROTOTYPES default to 0 if it has not already - been defined with C compiler flags. - */ -#ifndef PROTOTYPES -# if __STDC__ -# define PROTOTYPES 1 -# else -# define PROTOTYPES 0 -# endif -#endif - -/* POINTER defines a generic pointer type */ -typedef unsigned char *POINTER; - -/* UINT2 defines a two byte word */ -typedef unsigned short int UINT2; - -/* UINT4 defines a four byte word */ -#if defined(__alpha) && (defined(__osf__) || defined(__linux__)) -typedef unsigned int UINT4; -#else -typedef unsigned long int UINT4; -#endif - -/* PROTO_LIST is defined depending on how PROTOTYPES is defined above. - If using PROTOTYPES, then PROTO_LIST returns the list, otherwise it - returns an empty list. - */ -#if PROTOTYPES -#define PROTO_LIST(list) list -#else -#define PROTO_LIST(list) () -#endif - -/* MD5.H - header file for MD5C.C - */ - -/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All -rights reserved. - -License to copy and use this software is granted provided that it -is identified as the "RSA Data Security, Inc. MD5 Message-Digest -Algorithm" in all material mentioning or referencing this software -or this function. - -License is also granted to make and use derivative works provided -that such works are identified as "derived from the RSA Data -Security, Inc. MD5 Message-Digest Algorithm" in all material -mentioning or referencing the derived work. - -RSA Data Security, Inc. makes no representations concerning either -the merchantability of this software or the suitability of this -software for any particular purpose. It is provided "as is" -without express or implied warranty of any kind. - -These notices must be retained in any copies of any part of this -documentation and/or software. - */ - -/* MD5 context. */ -typedef struct { - UINT4 state[4]; /* state (ABCD) */ - UINT4 count[2]; /* number of bits, modulo 2^64 (lsb first) */ - unsigned char buffer[64]; /* input buffer */ -} MD5_CTX; - -void MD5Init PROTO_LIST ((MD5_CTX *)); -void MD5Update PROTO_LIST - ((MD5_CTX *, unsigned char *, unsigned int)); -void MD5Final PROTO_LIST ((unsigned char [16], MD5_CTX *)); -void md5_calc PROTO_LIST ((unsigned char *output, unsigned char *input, unsigned int inlen)); - Index: squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/squid_rad_auth.c 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -45,21 +45,51 @@ * and many others */ +#include "config.h" + +#if HAVE_SYS_TYPES_H #include +#endif +#if HAVE_SYS_SOCKET_H #include +#endif +#if HAVE_NETINET_IN_H #include +#endif +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_UNISTD_H #include +#endif +#if HAVE_FCNTL_H #include +#endif +#if HAVE_CTYPE_H #include +#endif +#if HAVE_STDIO_H #include +#endif +#if HAVE_UNISTD_H #include +#endif +#if HAVE_NETDB_H #include +#endif +#if HAVE_PWD_H #include +#endif +#if HAVE_STDLIB_H #include +#endif +#if HAVE_TIME_H #include +#endif +#if HAVE_STRING_H #include +#endif #include "md5.h" #include "radius.h" @@ -83,8 +113,8 @@ static char svc_name[MAXLINE] = "radius"; static int nasport = 111; static int nasporttype = 0; -static UINT4 nas_ipaddr; -static UINT4 auth_ipaddr; +static u_int32_t nas_ipaddr; +static u_int32_t auth_ipaddr; static int retries = 30; char *progname = "squid_rad_auth"; @@ -111,10 +141,22 @@ } /* + * MD5 digest + */ +static void +md5_calc(uint8_t out[16], void *in, size_t len) +{ + MD5_CTX ctx; + MD5Init(&ctx); + MD5Update(&ctx, in, len); + MD5Final(out, &ctx); +} + +/* * Receive and verify the result. */ static int -result_recv(UINT4 host, u_short udp_port, char *buffer, int length) +result_recv(u_int32_t host, u_short udp_port, char *buffer, int length) { AUTH_HDR *auth; int totallen; @@ -241,7 +283,7 @@ int secretlen; u_char cbc[AUTH_VECTOR_LEN]; int i, j; - UINT4 ui; + u_int32_t ui; struct sockaddr_in saremote; fd_set readfds; socklen_t salen; Index: squid/helpers/basic_auth/squid_radius_auth/util.c =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/util.c,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/util.c 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/util.c 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -35,18 +35,42 @@ "@(#)util.c 1.5 Copyright 1992 Livingston Enterprises Inc\n" " 2.1 Copyright 1997 Cistron Internet Services B.V."; +#include "config.h" + +#if HAVE_SYS_TYES_H #include +#endif +#if HAVE_SYS_SOCKET_H #include +#endif +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_NETINET_IN_H #include +#endif +#if HAVE_STDIO_H #include +#endif +#if HAVE_STDLIB_H #include +#endif +#if HAVE_NETDB_H #include +#endif +#if HAVE_PWD_H #include +#endif +#if HAVE_TIME_H #include +#endif +#if HAVE_CTYPE_H #include +#endif +#if HAVE_SIGNAL_H #include +#endif #include "md5.h" #include "util.h" @@ -89,37 +113,37 @@ * Return an IP address in host long notation from * one supplied in standard dot notation. */ -static UINT4 ipstr2long(char *ip_str) +static u_int32_t ipstr2long(char *ip_str) { char buf[6]; char *ptr; int i; int count; - UINT4 ipaddr; + u_int32_t ipaddr; int cur_byte; - ipaddr = (UINT4)0; + ipaddr = (u_int32_t)0; for(i = 0;i < 4;i++) { ptr = buf; count = 0; *ptr = '\0'; while(*ip_str != '.' && *ip_str != '\0' && count < 4) { if(!isdigit(*ip_str)) { - return((UINT4)0); + return((u_int32_t)0); } *ptr++ = *ip_str++; count++; } if(count >= 4 || count == 0) { - return((UINT4)0); + return((u_int32_t)0); } *ptr = '\0'; cur_byte = atoi(buf); if(cur_byte < 0 || cur_byte > 255) { - return((UINT4)0); + return((u_int32_t)0); } ip_str++; - ipaddr = ipaddr << 8 | (UINT4)cur_byte; + ipaddr = ipaddr << 8 | (u_int32_t)cur_byte; } return(ipaddr); } @@ -128,7 +152,7 @@ * Return an IP address in host long notation from a host * name or address in dot notation. */ -UINT4 get_ipaddr(char *host) +u_int32_t get_ipaddr(char *host) { struct hostent *hp; @@ -136,9 +160,9 @@ return(ipstr2long(host)); } else if((hp = gethostbyname(host)) == (struct hostent *)NULL) { - return((UINT4)0); + return((u_int32_t)0); } - return(ntohl(*(UINT4 *)hp->h_addr)); + return(ntohl(*(u_int32_t *)hp->h_addr)); } Index: squid/helpers/basic_auth/squid_radius_auth/util.h =================================================================== RCS file: /cvsroot/squid-sf//squid/helpers/basic_auth/squid_radius_auth/util.h,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/helpers/basic_auth/squid_radius_auth/util.h 5 Sep 2007 22:52:16 -0000 1.1 +++ squid/helpers/basic_auth/squid_radius_auth/util.h 27 Nov 2007 08:12:20 -0000 1.1.4.1 @@ -1,2 +1,2 @@ /* util.c */ -UINT4 get_ipaddr (char *); +u_int32_t get_ipaddr (char *); Index: squid/include/Array.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/Array.h,v retrieving revision 1.7 retrieving revision 1.7.52.1 diff -u -r1.7 -r1.7.52.1 --- squid/include/Array.h 28 Apr 2006 11:10:49 -0000 1.7 +++ squid/include/Array.h 27 Nov 2007 08:12:20 -0000 1.7.52.1 @@ -1,5 +1,5 @@ /* - * $Id: Array.h,v 1.7 2006/04/28 11:10:49 squidadm Exp $ + * $Id: Array.h,v 1.7.52.1 2007/11/27 08:12:20 adri Exp $ * * AUTHOR: Alex Rousskov * @@ -50,6 +50,8 @@ extern void arrayAppend(Array * s, void *obj); extern void arrayInsert(Array * s, void *obj, int position); extern void arrayPreAppend(Array * s, int app_count); +extern void arrayShrink(Array *a, int new_count); + #endif /* SQUID_ARRAY_H */ Index: squid/include/md5.h =================================================================== RCS file: /cvsroot/squid-sf//squid/include/md5.h,v retrieving revision 1.12 retrieving revision 1.12.34.1 diff -u -r1.12 -r1.12.34.1 --- squid/include/md5.h 29 May 2006 00:14:40 -0000 1.12 +++ squid/include/md5.h 27 Nov 2007 08:12:20 -0000 1.12.34.1 @@ -1,18 +1,18 @@ #ifndef SQUID_MD5_H #define SQUID_MD5_H -#if USE_OPENSSL +#if HAVE_SYS_TYPES_H +#include +#endif /* * If Squid is compiled with OpenSSL then we use the MD5 routines * from there via some wrapper macros, and the rest of this file is ignored.. */ +#define USE_SQUID_MD5 0 -#if HAVE_OPENSSL_MD5_H +#if USE_OPENSSL && HAVE_OPENSSL_MD5_H #include -#else -#error Cannot find OpenSSL headers -#endif /* Hack to adopt Squid to the OpenSSL syntax */ #define MD5_DIGEST_CHARS MD5_DIGEST_LENGTH @@ -21,7 +21,28 @@ #define MD5Update MD5_Update #define MD5Final MD5_Final -#else /* USE_OPENSSL */ +#elif USE_OPENSSL && !HAVE_OPENSSL_MD5_H +#error Cannot find OpenSSL MD5 headers + +#elif HAVE_SYS_MD5_H +/* + * Solaris 10 provides MD5 as part of the system. + */ +#include + +/* + * They also define MD5_CTX with different field names + * fortunately we do not access it directly in the squid code. + */ + +/* Hack to adopt Squid to the OpenSSL syntax */ +#define MD5_DIGEST_CHARS MD5_DIGEST_LENGTH + +#else /* NEED_OWN_MD5 */ + + /* Turn on internal MD5 code */ +#undef USE_SQUID_MD5 +#define USE_SQUID_MD5 1 /* * This is the header file for the MD5 message-digest algorithm. Index: squid/lib/Array.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/Array.c,v retrieving revision 1.6 retrieving revision 1.6.52.1 diff -u -r1.6 -r1.6.52.1 --- squid/lib/Array.c 28 Apr 2006 11:10:49 -0000 1.6 +++ squid/lib/Array.c 27 Nov 2007 08:12:20 -0000 1.6.52.1 @@ -1,5 +1,5 @@ /* - * $Id: Array.c,v 1.6 2006/04/28 11:10:49 squidadm Exp $ + * $Id: Array.c,v 1.6.52.1 2007/11/27 08:12:20 adri Exp $ * * AUTHOR: Alex Rousskov * @@ -138,3 +138,11 @@ /* reset, just in case */ memset(a->items + a->count, 0, (a->capacity - a->count) * sizeof(void *)); } + +void +arrayShrink(Array *a, int new_count) +{ + assert(new_count < a->capacity); + assert(new_count >= 0); + a->count = new_count; +} Index: squid/lib/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/Makefile.am,v retrieving revision 1.10 retrieving revision 1.10.18.1 diff -u -r1.10 -r1.10.18.1 --- squid/lib/Makefile.am 8 Sep 2006 19:50:55 -0000 1.10 +++ squid/lib/Makefile.am 27 Nov 2007 08:12:20 -0000 1.10.18.1 @@ -13,11 +13,6 @@ else STRSEPSOURCE= endif -if NEED_OWN_MD5 -MD5SOURCE=md5.c -else -MD5SOURCE= -endif if ENABLE_WIN32SPECIFIC LIBSSPWIN32=libsspwin32.a else @@ -41,7 +36,6 @@ @LIBREGEX@ \ $(LIBSSPWIN32) EXTRA_libmiscutil_a_SOURCES = \ - md5.c \ snprintf.c \ strsep.c \ win32lib.c @@ -53,7 +47,7 @@ heap.c \ html_quote.c \ iso3307.c \ - $(MD5SOURCE) \ + md5.c \ radix.c \ rfc1035.c \ rfc1123.c \ Index: squid/lib/md5.c =================================================================== RCS file: /cvsroot/squid-sf//squid/lib/md5.c,v retrieving revision 1.12 retrieving revision 1.12.30.1 diff -u -r1.12 -r1.12.30.1 --- squid/lib/md5.c 31 May 2006 11:51:11 -0000 1.12 +++ squid/lib/md5.c 27 Nov 2007 08:12:21 -0000 1.12.30.1 @@ -26,11 +26,16 @@ */ #include "config.h" -/* MS VisualStudio Projects are monolithic, so we need the following - * #if to exclude the MD5 code from compile process when we are - * building the SSL support. +#include "md5.h" + +/* + * Now that we have several alternatives the MD5 files are + * passed in by default. But a header-selection decides whether + * this provided version is to be built. + * TODO: may obsolete the MSV #if below. */ -#if !USE_SSL +#if USE_SQUID_MD5 + #if HAVE_STRING_H #include /* for memcpy() */ #endif @@ -41,8 +46,6 @@ #include /* for ntohl() */ #endif -#include "md5.h" - #ifdef WORDS_BIGENDIAN void byteSwap(uint32_t * buf, unsigned words) @@ -255,5 +258,5 @@ buf[3] += d; } -#endif -#endif +#endif /* !ASM_MD5 */ +#endif /* !USE_SQUID_MD5 */ Index: squid/src/HttpHeader.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpHeader.c,v retrieving revision 1.28 retrieving revision 1.28.6.1 diff -u -r1.28 -r1.28.6.1 --- squid/src/HttpHeader.c 11 Apr 2007 22:52:54 -0000 1.28 +++ squid/src/HttpHeader.c 27 Nov 2007 08:12:23 -0000 1.28.6.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.c,v 1.28 2007/04/11 22:52:54 squidadm Exp $ + * $Id: HttpHeader.c,v 1.28.6.1 2007/11/27 08:12:23 adri Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -381,12 +381,34 @@ } } +static void +httpHeaderRepack(HttpHeader * hdr) +{ + HttpHeaderPos dp = HttpHeaderInitPos; + HttpHeaderPos pos = HttpHeaderInitPos; + + /* XXX breaks layering for now! ie, getting grubby fingers in without httpHeaderEntryGet() */ + dp = 0; + pos = 0; + while (dp < hdr->entries.count) { + for (; dp < hdr->entries.count && hdr->entries.items[dp] == NULL; dp++); + assert(dp < hdr->entries.count); + hdr->entries.items[pos] = hdr->entries.items[dp]; + if (dp != pos) + hdr->entries.items[dp] = NULL; + pos++; + dp++; + } + arrayShrink(&hdr->entries, pos); +} + /* use fresh entries to replace old ones */ void httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask) { const HttpHeaderEntry *e; HttpHeaderPos pos = HttpHeaderInitPos; + assert(old && fresh); assert(old != fresh); debug(55, 7) ("updating hdr: %p <- %p\n", old, fresh); @@ -399,8 +421,17 @@ httpHeaderDelById(old, e->id); else httpHeaderDelByName(old, strBuf(e->name)); + } + pos = HttpHeaderInitPos; + while ((e = httpHeaderGetEntry(fresh, &pos))) { + /* deny bad guys (ok to check for HDR_OTHER) here */ + if (denied_mask && CBIT_TEST(*denied_mask, e->id)) + continue; httpHeaderAddClone(old, e); } + + /* And now, repack the array to "fill in the holes" */ + httpHeaderRepack(old); } /* just handy in parsing: resets and returns false */ Index: squid/src/HttpReply.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpReply.c,v retrieving revision 1.23 retrieving revision 1.23.6.1 diff -u -r1.23 -r1.23.6.1 --- squid/src/HttpReply.c 2 Jun 2007 19:52:18 -0000 1.23 +++ squid/src/HttpReply.c 27 Nov 2007 08:12:23 -0000 1.23.6.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.c,v 1.23 2007/06/02 19:52:18 squidadm Exp $ + * $Id: HttpReply.c,v 1.23.6.1 2007/11/27 08:12:23 adri Exp $ * * DEBUG: section 58 HTTP Reply (Response) * AUTHOR: Alex Rousskov @@ -42,9 +42,8 @@ static HttpHeaderMask Denied304HeadersMask; static http_hdr_type Denied304HeadersArr[] = { - HDR_ALLOW, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE, HDR_CONTENT_LENGTH, - HDR_CONTENT_LOCATION, HDR_CONTENT_RANGE, HDR_LAST_MODIFIED, HDR_LINK, - HDR_OTHER + HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE, HDR_CONTENT_LENGTH, + HDR_CONTENT_LOCATION, HDR_CONTENT_RANGE }; /* local routines */ @@ -274,6 +273,7 @@ /* clean cache */ httpReplyHdrCacheClean(rep); /* update raw headers */ + httpHeaderDelById(&rep->header, HDR_AGE); httpHeaderUpdate(&rep->header, &freshRep->header, (const HttpHeaderMask *) &Denied304HeadersMask); /* init cache */ Index: squid/src/HttpRequest.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/HttpRequest.c,v retrieving revision 1.17 retrieving revision 1.17.10.1 diff -u -r1.17 -r1.17.10.1 --- squid/src/HttpRequest.c 21 Jan 2007 14:03:34 -0000 1.17 +++ squid/src/HttpRequest.c 27 Nov 2007 08:12:24 -0000 1.17.10.1 @@ -1,6 +1,6 @@ /* - * $Id: HttpRequest.c,v 1.17 2007/01/21 14:03:34 squidadm Exp $ + * $Id: HttpRequest.c,v 1.17.10.1 2007/11/27 08:12:24 adri Exp $ * * DEBUG: section 73 HTTP Request * AUTHOR: Duane Wessels @@ -59,6 +59,7 @@ requestAbortBody(req); if (req->auth_user_request) authenticateAuthUserRequestUnlock(req->auth_user_request); + safe_free(req->store_url); safe_free(req->canonical); safe_free(req->vary_hdr); safe_free(req->vary_headers); Index: squid/src/Makefile.am =================================================================== RCS file: /cvsroot/squid-sf//squid/src/Makefile.am,v retrieving revision 1.54 retrieving revision 1.54.2.1 diff -u -r1.54 -r1.54.2.1 --- squid/src/Makefile.am 23 Sep 2007 14:51:42 -0000 1.54 +++ squid/src/Makefile.am 27 Nov 2007 08:12:24 -0000 1.54.2.1 @@ -156,6 +156,8 @@ cbdata.c \ client_db.c \ client_side.c \ + client_side_rewrite.c \ + client_side_storeurl_rewrite.c \ comm.c \ $(COMMLOOP_SOURCE) \ debug.c \ @@ -223,8 +225,10 @@ peer_userhash.c \ protos.h \ redirect.c \ + store_rewrite.c \ referer.c \ refresh.c \ + refresh_check.c \ send-announce.c \ $(SNMPSOURCE) \ squid.h \ @@ -245,6 +249,7 @@ store_swapin.c \ store_swapmeta.c \ store_swapout.c \ + store_update.c \ structs.h \ tools.c \ typedefs.h \ Index: squid/src/access_log.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/access_log.c,v retrieving revision 1.43 retrieving revision 1.43.2.1 diff -u -r1.43 -r1.43.2.1 --- squid/src/access_log.c 23 Sep 2007 14:51:42 -0000 1.43 +++ squid/src/access_log.c 27 Nov 2007 08:12:24 -0000 1.43.2.1 @@ -1,6 +1,6 @@ /* - * $Id: access_log.c,v 1.43 2007/09/23 14:51:42 squidadm Exp $ + * $Id: access_log.c,v 1.43.2.1 2007/11/27 08:12:24 adri Exp $ * * DEBUG: section 46 Access Log * AUTHOR: Duane Wessels @@ -320,6 +320,8 @@ LFT_SEQUENCE_NUMBER, + LFT_EXT_FRESHNESS, + LFT_PERCENT /* special string cases for escaped chars */ } logformat_bcode_t; @@ -428,6 +430,8 @@ {"ea", LFT_EXT_LOG}, + {"ef", LFT_EXT_FRESHNESS}, + {"%", LFT_PERCENT}, {NULL, LFT_NONE} /* this must be last */ @@ -691,6 +695,12 @@ doint = 1; break; + case LFT_EXT_FRESHNESS: + out = al->ext_refresh; + + quote = 1; + break; + case LFT_PERCENT: out = "%"; break; Index: squid/src/cache_cf.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cache_cf.c,v retrieving revision 1.102 retrieving revision 1.102.2.1 diff -u -r1.102 -r1.102.2.1 --- squid/src/cache_cf.c 23 Sep 2007 14:51:42 -0000 1.102 +++ squid/src/cache_cf.c 27 Nov 2007 08:12:24 -0000 1.102.2.1 @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.c,v 1.102 2007/09/23 14:51:42 squidadm Exp $ + * $Id: cache_cf.c,v 1.102.2.1 2007/11/27 08:12:24 adri Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -479,6 +479,7 @@ requirePathnameExists("Error Directory", Config.errorDirectory); authenticateConfigure(&Config.authConfig); externalAclConfigure(); + refreshCheckConfigure(); #if HTTP_VIOLATIONS { const refresh_t *R; @@ -494,6 +495,12 @@ debug(22, 1) ("WARNING: use of 'override-lastmod' in 'refresh_pattern' violates HTTP\n"); break; } + for (R = Config.Refresh; R; R = R->next) { + if (R->stale_while_revalidate <= 0) + continue; + debug(22, 1) ("WARNING: use of 'stale-while-revalidate' in 'refresh_pattern' violates HTTP\n"); + break; + } } #endif #if !HTTP_VIOLATIONS @@ -1610,6 +1617,34 @@ } } +static u_short +GetService(const char *proto) +{ + struct servent *port = NULL; + char *token = strtok(NULL, w_space); + if (token == NULL) { + self_destruct(); + return -1; /* NEVER REACHED */ + } + port = getservbyname(token, proto); + if (port != NULL) { + return ntohs((u_short) port->s_port); + } + return xatos(token); +} + +static u_short +GetTcpService(void) +{ + return GetService("tcp"); +} + +static u_short +GetUdpService(void) +{ + return GetService("udp"); +} + static void parse_peer(peer ** head) { @@ -1634,10 +1669,10 @@ p->options.no_digest = 1; p->options.no_netdb_exchange = 1; } - p->http_port = GetShort(); + p->http_port = GetTcpService(); if (!p->http_port) self_destruct(); - p->icp.port = GetShort(); + p->icp.port = GetUdpService(); p->connection_auth = -1; /* auto */ while ((token = strtok(NULL, w_space))) { if (!strcasecmp(token, "proxy-only")) { @@ -2146,7 +2181,13 @@ storeAppendPrintf(entry, " ignore-private"); if (head->flags.ignore_auth) storeAppendPrintf(entry, " ignore-auth"); + if (head->stale_while_revalidate > 0) + storeAppendPrintf(entry, " stale-while-revalidate=%d", head->stale_while_revalidate); #endif + if (head->max_stale >= 0) + storeAppendPrintf(entry, " max-stale=%d", head->max_stale); + if (head->negative_ttl >= 0) + storeAppendPrintf(entry, " negative-ttl=%d", head->negative_ttl); storeAppendPrintf(entry, "\n"); head = head->next; } @@ -2169,6 +2210,9 @@ int ignore_private = 0; int ignore_auth = 0; #endif + int stale_while_revalidate = -1; + int max_stale = -1; + int negative_ttl = -1; int i; refresh_t *t; regex_t comp; @@ -2213,10 +2257,18 @@ ignore_reload = 1; refresh_nocache_hack = 1; /* tell client_side.c that this is used */ + } else if (!strncmp(token, "stale-while-revalidate=", 23)) { + stale_while_revalidate = atoi(token + 23); } else #endif + if (!strncmp(token, "max-stale=", 10)) { + max_stale = atoi(token + 10); + } else if (!strncmp(token, "negative-ttl=", 13)) { + negative_ttl = atoi(token + 13); + } else { debug(22, 0) ("redreshAddToList: Unknown option '%s': %s\n", pattern, token); + } } if ((errcode = regcomp(&comp, pattern, flags)) != 0) { char errbuf[256]; @@ -2253,6 +2305,9 @@ if (ignore_auth) t->flags.ignore_auth = 1; #endif + t->stale_while_revalidate = stale_while_revalidate; + t->max_stale = max_stale; + t->negative_ttl = negative_ttl; t->next = NULL; while (*head) head = &(*head)->next; Index: squid/src/cf.data.depend =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.depend,v retrieving revision 1.1 retrieving revision 1.1.4.1 diff -u -r1.1 -r1.1.4.1 --- squid/src/cf.data.depend 5 Sep 2007 21:51:54 -0000 1.1 +++ squid/src/cf.data.depend 27 Nov 2007 08:12:24 -0000 1.1.4.1 @@ -52,3 +52,4 @@ programline extension_method errormap +refreshCheckHelper Index: squid/src/cf.data.pre =================================================================== RCS file: /cvsroot/squid-sf//squid/src/cf.data.pre,v retrieving revision 1.209 retrieving revision 1.209.2.1 diff -u -r1.209 -r1.209.2.1 --- squid/src/cf.data.pre 23 Sep 2007 14:51:43 -0000 1.209 +++ squid/src/cf.data.pre 27 Nov 2007 08:12:24 -0000 1.209.2.1 @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.209 2007/09/23 14:51:43 squidadm Exp $ +# $Id: cf.data.pre,v 1.209.2.1 2007/11/27 08:12:24 adri Exp $ # # SQUID Web Proxy Cache http://www.squid-cache.org/ # ---------------------------------------------------------- @@ -626,10 +626,18 @@ NOCOMMENT_START #Recommended minimum configuration: -acl all src 0.0.0.0/0.0.0.0 +acl all src all acl manager proto cache_object -acl localhost src 127.0.0.1/255.255.255.255 +acl localhost src 127.0.0.1/32 acl to_localhost dst 127.0.0.0/8 +# +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 10.0.0.0/8 # RFC1918 possible internal network +acl localnet src 172.16.0.0/12 # RFC1918 possible internal network +acl localnet src 192.168.0.0/16 # RFC1918 possible internal network +# acl SSL_ports port 443 acl Safe_ports port 80 # http acl Safe_ports port 21 # ftp @@ -686,11 +694,10 @@ # # INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS -# Example rule allowing access from your local networks. Adapt -# to list your (internal) IP networks from where browsing should -# be allowed -#acl our_networks src 192.168.1.0/24 192.168.2.0/24 -#http_access allow our_networks +# Example rule allowing access from your local networks. +# Adapt localnet in the ACL section to list your (internal) IP networks +# from where browsing should be allowed +http_access allow localnet # And finally deny all other access to this proxy http_access deny all @@ -740,8 +747,9 @@ See http_access for details NOCOMMENT_START -#Allow ICP queries from everyone -icp_access allow all +#Allow ICP queries from local networks only +icp_access allow localnet +icp_access deny all NOCOMMENT_END DOC_END @@ -759,8 +767,15 @@ See http_access for details -#Allow HTCP queries from everyone -htcp_access allow all + NOTE: The default if no htcp_access lines are present is to + deny all traffic. This default may cause problems with peers + using the htcp or htcp-oldsquid options. + +NOCOMMENT_START +#Allow HTCP queries from local networks only +htcp_access allow localnet +htcp_access deny all +NOCOMMENT_END DOC_END NAME: htcp_clr_access @@ -1161,7 +1176,7 @@ acl normal_service_net src 10.0.0.0/255.255.255.0 acl good_service_net src 10.0.1.0/255.255.255.0 - tcp_outgoing_tos 0x00 normal_service_net 0x00 + tcp_outgoing_tos 0x00 normal_service_net tcp_outgoing_tos 0x20 good_service_net TOS/DSCP values really only have local significance - so you should @@ -1515,8 +1530,12 @@ use 'htcp' to send HTCP, instead of ICP, queries to the neighbor. You probably also want to set the "icp port" to 4827 instead of 3130. + You must also allow this Squid htcp_access and + http_access in the peer Squid configuration. use 'htcp-oldsquid' to send HTCP to old Squid versions + You must also allow this Squid htcp_access and + http_access in the peer Squid configuration. 'originserver' causes this parent peer to be contacted as a origin server. Meant to be used in accelerator setups. @@ -2064,6 +2083,20 @@ numbers closer together. DOC_END +NAME: update_headers +IFDEF: HTTP_VIOLATIONS +COMMENT: on|off +TYPE: onoff +DEFAULT: on +LOC: Config.onoff.update_headers +DOC_START + By default Squid updates stored HTTP headers when receiving + a 304 response. Set this to off if you want to disable this + for disk I/O performance reasons. Disabling this VIOLATES the + HTTP standard, and could make you liable for problems which it + causes. +DOC_END + COMMENT_START LOGFILE OPTIONS ----------------------------------------------------------------------------- @@ -2529,6 +2562,30 @@ ----------------------------------------------------------------------------- COMMENT_END +NAME: storeurl_rewrite_program +TYPE: programline +LOC: Config.Program.store_rewrite.command +DEFAULT: none +DOC_START + +DOC_END + +NAME: storeurl_rewrite_children +TYPE: int +DEFAULT: 5 +LOC: Config.Program.store_rewrite.children +DOC_START + +DOC_END + +NAME: storeurl_rewrite_concurrency +TYPE: int +DEFAULT: 0 +LOC: Config.Program.store_rewrite.concurrency +DOC_START + +DOC_END + NAME: url_rewrite_program redirect_program TYPE: programline LOC: Config.Program.url_rewrite.command @@ -2539,7 +2596,13 @@ For each requested URL rewriter will receive on line with the format - URL client_ip "/" fqdn user method urlgroup + URL client_ip "/" fqdn user method urlgroup + [ kvpairs] + + In the future, the rewriter interface will be extended with + key=value pairs ("kvpairs" shown above). Rewriter programs + should be prepared to receive and possibly ignore additional + whitespace-separated tokens on each input line. And the rewriter may return a rewritten URL. The other components of the request line does not need to be returned (ignored if they are). @@ -2550,7 +2613,7 @@ It can also return a "urlgroup" that can subsequently be matched in cache_peer_access and similar ACL driven rules. An urlgroup is - returned by prefixing the returned url with "!urlgroup!" + returned by prefixing the returned URL with "!urlgroup!". By default, a URL rewriter is not used. DOC_END @@ -2574,6 +2637,12 @@ The number of requests each redirector helper can handle in parallel. Defaults to 0 which indicates the redirector is a old-style single threaded redirector. + + When this directive is set to a value >= 1 then the protocol + used to communicate with the helper is modified to include + a request ID in front of the request/response. The request + ID from the request must be echoed back with the response + to that request. DOC_END NAME: url_rewrite_host_header redirect_rewrites_host_header @@ -2599,6 +2668,14 @@ are sent. DOC_END +NAME: storeurl_access +TYPE: acl_access +DEFAULT: none +LOC: Config.accessList.storeurl_rewrite +DOC_START + +DOC_END + NAME: redirector_bypass TYPE: onoff LOC: Config.onoff.redirector_bypass @@ -2692,6 +2769,16 @@ NOCOMMENT_END DOC_END +NAME: max_stale +COMMENT: time-units +TYPE: time_t +LOC: Config.maxStale +DEFAULT: 1 week +DOC_START + This option puts an upper limit on how stale content Squid + will serve from the cache if cache validation fails. +DOC_END + NAME: refresh_pattern TYPE: refreshpattern LOC: Config.Refresh @@ -2722,6 +2809,9 @@ ignore-no-cache ignore-private ignore-auth + stale-while-revalidate=NN + max-stale=NN + negative-ttl=NN override-expire enforces min age even if the server sent a Expires: header. Doing this VIOLATES the HTTP @@ -2758,6 +2848,19 @@ Enabling this feature could make you liable for problems which it causes. + stale-while-revalidate=NN makes Squid perform an asyncronous + cache validation if the object isn't more stale than NN. + Doing this VIOLATES the HTTP standard. Enabling this + feature could make you liable for problems which it + causes. + + max-stale=NN provided a maximum staleness factor. Squid won't + serve objects more stale than this even if it failed to + validate the object. + + negative-ttl=NN overrides the global negative_ttl parameter + selectively for URLs matching this pattern (in seconds). + Basically a cached object is: FRESH if expires < now, else STALE @@ -3232,6 +3335,69 @@ or response to be rejected. DOC_END +NAME: external_refresh_check +TYPE: refreshCheckHelper +DEFAULT: none +LOC: Config.Program.refresh_check +DOC_START + This option defines an external helper for determining whether to + refresh a stale response. It will be called when Squid receives a + request for a cached response that is stale; the helper can either + confirm that the response is stale with a STALE response, or + extend the freshness of the response (thereby avoiding a refresh + check) with a FRESH response, along with a freshness=nnn keyword. + + external_refresh_check [options] FORMAT.. /path/to/helper [helper_args] + + If present, helper_args will be passed to the helper on the command + line verbatim. + + Options: + + children=n Number of processes to spawn to service external + refresh checks (default 5). + concurrency=n Concurrency level per process. Only used with + helpers capable of processing more than one query + at a time. + + When using the concurrency option, the protocol is changed by introducing + a query channel tag infront of the request/response. The query channel + tag is a number between 0 and concurrency−1. + + FORMAT specifications: + + %CACHE_URI The URI of the cached response + %RES{Header} HTTP response header value + %AGE The age of the cached response + + The request sent to the helper consists of the data in the format + specification in the order specified. + + The helper receives lines per the above format specification, and + returns lines starting with OK or ERR indicating the validity of + the request and optionally followed by additional keywords with + more details. URL escaping is used to protect each value in both + requests and responses. + + General result syntax: + + FRESH / STALE keyword=value ... + + Defined keywords: + + freshness=nnn The number of seconds to extend the freshness of + the response by. + log=string String to be logged in access.log. Available as + %ef in logformat specifications. + res{Header}=value + Value to update response headers with. If already + present, the supplied value completely replaces + the cached value. + + In the event of a helper−related error (e.g., overload), Squid + will always default to STALE. +DOC_END + COMMENT_START TIMEOUTS ----------------------------------------------------------------------------- @@ -3404,9 +3570,10 @@ LOC: Config.EmailProgram DOC_START Email program used to send mail if the cache dies. - The default is "mail". The specified program must complain + The default is "mail". The specified program must comply with the standard Unix mail syntax: - mail_program recipient < mailfile + mail-program recipient < mailfile + Optional command line options can be specified. DOC_END @@ -3989,8 +4156,7 @@ DEFAULT: on DOC_START This controls whether the server will generate a Cache Digest - of its contents. By default, Cache Digest generation is - enabled if Squid is compiled with USE_CACHE_DIGESTS defined. + of its contents. DOC_END NAME: digest_bits_per_entry @@ -4011,7 +4177,7 @@ LOC: Config.digest.rebuild_period DEFAULT: 1 hour DOC_START - This is the number of seconds between Cache Digest rebuilds. + This is the wait time between Cache Digest rebuilds. DOC_END NAME: digest_rewrite_period @@ -4021,8 +4187,7 @@ LOC: Config.digest.rewrite_period DEFAULT: 1 hour DOC_START - This is the number of seconds between Cache Digest writes to - disk. + This is the wait time between Cache Digest writes to disk. DOC_END NAME: digest_swapout_chunk_size @@ -4155,31 +4320,46 @@ TYPE: address LOC:Config.Addrs.udp_incoming DEFAULT: 0.0.0.0 -DOC_NONE +DOC_START + udp_incoming_address is used for UDP packets received from other + caches. + + The default behavior is to not bind to any specific address. + + Only change this if you want to have all UDP queries received on + a specific interface/address. + + NOTE: udp_incoming_address is used by the ICP, HTCP, and DNS + modules. Altering it will affect all of them in the same manner. + + see also; udp_outgoing_address + + NOTE, udp_incoming_address and udp_outgoing_address can not + have the same value since they both use the same port. +DOC_END NAME: udp_outgoing_address TYPE: address LOC: Config.Addrs.udp_outgoing DEFAULT: 255.255.255.255 DOC_START - udp_incoming_address is used for the ICP socket receiving packets - from other caches. - udp_outgoing_address is used for ICP packets sent out to other + udp_outgoing_address is used for UDP packets sent out to other caches. The default behavior is to not bind to any specific address. - A udp_incoming_address value of 0.0.0.0 indicates Squid - should listen for UDP messages on all available interfaces. - - If udp_outgoing_address is set to 255.255.255.255 (the default) - it will use the same socket as udp_incoming_address. Only - change this if you want to have ICP queries sent using another - address than where this Squid listens for ICP queries from other + Instead it will use the same socket as udp_incoming_address. + Only change this if you want to have UDP queries sent using another + address than where this Squid listens for UDP queries from other caches. + NOTE: udp_outgoing_address is used by the ICP, HTCP, and DNS + modules. Altering it will affect all of them in the same manner. + + see also; udp_incoming_address + NOTE, udp_incoming_address and udp_outgoing_address can not - have the same value since they both use port 3130. + have the same value since they both use the same port. DOC_END NAME: icp_hit_stale @@ -4580,6 +4760,18 @@ acts on cacheable requests. DOC_END +NAME: ignore_ims_on_miss +IFDEF: HTTP_VIOLATIONS +COMMENT: on|off +TYPE: onoff +DEFAULT: off +LOC: Config.onoff.ignore_ims_on_miss +DOC_START + This options makes Squid ignore If-Modified-Since on + cache misses. This is useful while the cache is + mostly empty to more quickly have the cache populated. +DOC_END + NAME: always_direct TYPE: acl_access LOC: Config.accessList.AlwaysDirect Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/client_side.c,v retrieving revision 1.202.2.6 retrieving revision 1.202.2.7 diff -u -r1.202.2.6 -r1.202.2.7 --- squid/src/client_side.c 29 Sep 2007 10:38:27 -0000 1.202.2.6 +++ squid/src/client_side.c 27 Nov 2007 08:12:25 -0000 1.202.2.7 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.202.2.6 2007/09/29 10:38:27 adri Exp $ + * $Id: client_side.c,v 1.202.2.7 2007/11/27 08:12:25 adri Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -116,14 +116,14 @@ static int clientGotNotEnough(clientHttpRequest *); static void checkFailureRatio(err_type, hier_code); static void clientProcessMiss(clientHttpRequest *); +static void clientProcessHit(clientHttpRequest * http); static void clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep); static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri); static clientHttpRequest *parseHttpRequest(ConnStateData *, HttpMsgBuf *, method_t *, int *); -static void clientRedirectStart(clientHttpRequest * http); -static RH clientRedirectDone; + static void clientCheckNoCache(clientHttpRequest *); static void clientCheckNoCacheDone(int answer, void *data); -static STNCB clientHandleIMSReply; +static STHCB clientHandleIMSReply; static int clientGetsOldEntry(StoreEntry * new, StoreEntry * old, request_t * request); #if USE_IDENT static IDCB clientIdentDone; @@ -135,14 +135,15 @@ #endif /* FOLLOW_X_FORWARDED_FOR */ static int clientOnlyIfCached(clientHttpRequest * http); static STNCB clientSendMoreData; -static STNCB clientSendMoreHeaderData; -static STNCB clientCacheHit; +static STHCB clientSendHeaders; +static STHCB clientCacheHit; static void clientSetKeepaliveFlag(clientHttpRequest *); static void clientPackRangeHdr(const HttpReply * rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb); static void clientPackTermBound(String boundary, MemBuf * mb); -static void clientInterpretRequestHeaders(clientHttpRequest *); static void clientProcessRequest(clientHttpRequest *); -static void clientProcessExpired(void *data); +static void clientProcessExpired(clientHttpRequest *); +static void clientRefreshCheck(clientHttpRequest *); +static REFRESHCHECK clientRefreshCheckDone; static void clientProcessOnlyIfCachedMiss(clientHttpRequest * http); static int clientCachable(clientHttpRequest * http); static int clientHierarchical(clientHttpRequest * http); @@ -155,7 +156,6 @@ static void clientEatRequestBody(clientHttpRequest *); static void clientAccessCheck(void *data); static void clientAccessCheckDone(int answer, void *data); -static void clientAccessCheck2(void *data); static void clientAccessCheckDone2(int answer, void *data); static BODY_HANDLER clientReadBody; static void clientAbortBody(request_t * req); @@ -167,6 +167,31 @@ static StoreEntry *clientCreateStoreEntry(clientHttpRequest *, method_t, request_flags); static inline int clientNatLookup(ConnStateData * conn); +/* Temporary here while restructuring stuff */ +static void +storeClientCopyHeadersCB(void *data, char *buf, ssize_t size) +{ + clientHttpRequest *http = data; + assert(http->header_callback); + assert(http->header_entry); + memFree(buf, MEM_STORE_CLIENT_BUF); + if (!http->header_entry) + return; + if (size < 0 || !memHaveHeaders(http->header_entry->mem_obj)) { + http->header_callback(data, NULL); + return; + } + http->header_callback(data, http->header_entry->mem_obj->reply); +} +void +storeClientCopyHeaders(store_client * sc, StoreEntry * e, STHCB * callback, void *callback_data) +{ + clientHttpRequest *http = callback_data; + http->header_callback = callback; + http->header_entry = e; + storeClientCopy(http->sc, e, 0, 0, STORE_CLIENT_BUF_SZ, memAllocate(MEM_STORE_CLIENT_BUF), storeClientCopyHeadersCB, http); +} + #if USE_IDENT static void clientIdentDone(const char *ident, void *data) @@ -177,7 +202,7 @@ #endif -static aclCheck_t * +aclCheck_t * clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http) { aclCheck_t *ch; @@ -358,7 +383,7 @@ aclNBCheck(http->acl_checklist, clientAccessCheckDone, http); } -static void +void clientAccessCheck2(void *data) { clientHttpRequest *http = data; @@ -394,12 +419,14 @@ if (h->request == NULL) h->request = requestLink(requestCreate(m, PROTO_NONE, null_string)); e = storeCreateEntry(h->uri, flags, m); + if (h->request->store_url) + storeEntrySetStoreUrl(e, h->request->store_url); h->sc = storeClientRegister(e, h); #if DELAY_POOLS if (h->log_type != LOG_TCP_DENIED) delaySetStoreClient(h->sc, delayClient(h)); #endif - storeClientRef(h->sc, e, 0, 0, SM_PAGE_SIZE, clientSendMoreHeaderData, h); + storeClientCopyHeaders(h->sc, e, clientSendHeaders, h); return e; } @@ -531,123 +558,6 @@ } static void -clientRedirectAccessCheckDone(int answer, void *data) -{ - clientHttpRequest *http = data; - http->acl_checklist = NULL; - if (answer == ACCESS_ALLOWED) - redirectStart(http, clientRedirectDone, http); - else - clientRedirectDone(http, NULL); -} - -static void -clientRedirectStart(clientHttpRequest * http) -{ - debug(33, 5) ("clientRedirectStart: '%s'\n", http->uri); - if (Config.Program.url_rewrite.command == NULL) { - clientRedirectDone(http, NULL); - return; - } - if (Config.accessList.url_rewrite) { - http->acl_checklist = clientAclChecklistCreate(Config.accessList.url_rewrite, http); - aclNBCheck(http->acl_checklist, clientRedirectAccessCheckDone, http); - } else { - redirectStart(http, clientRedirectDone, http); - } -} - -static void -clientRedirectDone(void *data, char *result) -{ - clientHttpRequest *http = data; - request_t *new_request = NULL; - request_t *old_request = http->request; - const char *urlgroup = http->conn->port->urlgroup; - debug(33, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri, - result ? result : "NULL"); - assert(http->redirect_state == REDIRECT_PENDING); - http->redirect_state = REDIRECT_DONE; - if (result) { - http_status status; - if (*result == '!') { - char *t; - if ((t = strchr(result + 1, '!')) != NULL) { - urlgroup = result + 1; - *t++ = '\0'; - result = t; - } else { - debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); - } - } - status = (http_status) atoi(result); - if (status == HTTP_MOVED_PERMANENTLY - || status == HTTP_MOVED_TEMPORARILY - || status == HTTP_SEE_OTHER - || status == HTTP_TEMPORARY_REDIRECT) { - char *t = result; - if ((t = strchr(result, ':')) != NULL) { - http->redirect.status = status; - http->redirect.location = xstrdup(t + 1); - goto redirect_parsed; - } else { - debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); - } - } else if (strcmp(result, http->uri)) - new_request = urlParse(old_request->method, result); - } - redirect_parsed: - if (new_request) { - safe_free(http->uri); - http->uri = xstrdup(urlCanonical(new_request)); - new_request->http_ver = old_request->http_ver; - httpHeaderAppend(&new_request->header, &old_request->header); - new_request->client_addr = old_request->client_addr; - new_request->client_port = old_request->client_port; -#if FOLLOW_X_FORWARDED_FOR - new_request->indirect_client_addr = old_request->indirect_client_addr; -#endif /* FOLLOW_X_FORWARDED_FOR */ - new_request->my_addr = old_request->my_addr; - new_request->my_port = old_request->my_port; - new_request->flags = old_request->flags; - new_request->flags.redirected = 1; - if (old_request->auth_user_request) { - new_request->auth_user_request = old_request->auth_user_request; - authenticateAuthUserRequestLock(new_request->auth_user_request); - } - if (old_request->body_reader) { - new_request->body_reader = old_request->body_reader; - new_request->body_reader_data = old_request->body_reader_data; - old_request->body_reader = NULL; - old_request->body_reader_data = NULL; - } - new_request->content_length = old_request->content_length; - if (strBuf(old_request->extacl_log)) - new_request->extacl_log = stringDup(&old_request->extacl_log); - if (old_request->extacl_user) - new_request->extacl_user = xstrdup(old_request->extacl_user); - if (old_request->extacl_passwd) - new_request->extacl_passwd = xstrdup(old_request->extacl_passwd); - requestUnlink(old_request); - http->request = requestLink(new_request); - } else { - /* Don't mess with urlgroup on internal request */ - if (old_request->flags.internal) - urlgroup = NULL; - } - safe_free(http->request->urlgroup); /* only paranoia. should not happen */ - if (urlgroup && *urlgroup) - http->request->urlgroup = xstrdup(urlgroup); - clientInterpretRequestHeaders(http); -#if HEADERS_LOG - headersLog(0, 1, request->method, request); -#endif - /* XXX This really should become a ref-counted string type pointer, not a copy! */ - fd_note(http->conn->fd, http->uri); - clientAccessCheck2(http); -} - -static void clientCheckNoCache(clientHttpRequest * http) { if (Config.accessList.noCache && http->request->flags.cachable) { @@ -689,74 +599,49 @@ } static void -clientHandleETagReply(void *data, mem_node_ref ref, ssize_t size) +clientHandleETagReply(void *data, HttpReply * rep) { //const char *buf = ref.node->data + ref.offset; clientHttpRequest *http = data; StoreEntry *entry = http->entry; - MemObject *mem; - const char *url = storeUrl(entry); - http_status status; - debug(33, 3) ("clientHandleETagReply: %s, %d bytes\n", url, (int) size); + const char *url = storeLookupUrl(entry); if (entry == NULL) { /* client aborted */ - stmemNodeUnref(&ref); return; } - if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - stmemNodeUnref(&ref); + if (!rep) { + debug(33, 3) ("clientHandleETagReply: FAILED '%s'\n", url); clientHandleETagMiss(http); return; } - mem = entry->mem_obj; - status = mem->reply->sline.status; if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { debug(33, 3) ("clientHandleETagReply: ABORTED '%s'\n", url); - stmemNodeUnref(&ref); clientHandleETagMiss(http); return; } - if (STORE_PENDING == entry->store_status && 0 == status) { - debug(33, 3) ("clientHandleETagReply: Incomplete headers for '%s'\n", url); - if (size >= SM_PAGE_SIZE) { - /* will not get any bigger than that */ - debug(33, 3) ("clientHandleETagReply: Reply is too large '%s'\n", url); - stmemNodeUnref(&ref); - clientHandleETagMiss(http); - } else { - stmemNodeUnref(&ref); - storeClientRef(http->sc, entry, - http->out.offset + size, - http->out.offset, - SM_PAGE_SIZE, - clientHandleETagReply, - http); - } - return; - } - if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { + debug(33, 3) ("clientHandleETagReply: %s = %d\n", url, (int) rep->sline.status); + if (HTTP_NOT_MODIFIED == rep->sline.status) { /* Remember the ETag and restart */ - stmemNodeUnref(&ref); - if (mem->reply) { + if (rep) { request_t *request = http->request; - const char *etag = httpHeaderGetStr(&mem->reply->header, HDR_ETAG); + const char *etag = httpHeaderGetStr(&rep->header, HDR_ETAG); const char *vary = request->vary_headers; - int has_vary = httpHeaderHas(&entry->mem_obj->reply->header, HDR_VARY); + int has_vary = httpHeaderHas(&rep->header, HDR_VARY); #if X_ACCELERATOR_VARY - has_vary |= httpHeaderHas(&entry->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY); + has_vary |= httpHeaderHas(&rep->header, HDR_X_ACCELERATOR_VARY); #endif if (has_vary) - vary = httpMakeVaryMark(request, mem->reply); + vary = httpMakeVaryMark(request, rep); if (etag && vary) { - storeAddVary(mem->url, mem->method, NULL, httpHeaderGetStr(&mem->reply->header, HDR_ETAG), request->vary_hdr, request->vary_headers, strBuf(request->vary_encoding)); + storeAddVary(url, entry->mem_obj->method, NULL, httpHeaderGetStr(&rep->header, HDR_ETAG), request->vary_hdr, request->vary_headers, strBuf(request->vary_encoding)); } } clientHandleETagMiss(http); return; } /* Send the new object to the client */ - clientSendMoreHeaderData(data, ref, size); + clientSendHeaders(data, rep); return; } @@ -767,6 +652,8 @@ StoreEntry *entry = NULL; debug(33, 3) ("clientProcessETag: '%s'\n", http->uri); entry = storeCreateEntry(url, http->request->flags, http->request->method); + if (http->request->store_url) + storeEntrySetStoreUrl(entry, http->request->store_url); http->sc = storeClientRegister(entry, http); #if DELAY_POOLS /* delay_id is already set on original store client */ @@ -778,18 +665,34 @@ /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessETag: found ENTRY_ABORTED object\n"); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, entry, clientHandleETagReply, http); } static void -clientProcessExpired(void *data) +clientRefreshCheck(clientHttpRequest * http) +{ + refreshCheckSubmit(http->entry, clientRefreshCheckDone, http); +} + +static void +clientRefreshCheckDone(void *data, int fresh, const char *log) { clientHttpRequest *http = data; + if (log) { + safe_free(http->al.ext_refresh); + http->al.ext_refresh = xstrdup(log); + } + if (fresh) + clientProcessHit(http); + else + clientProcessExpired(http); +} + +static void +clientProcessExpired(clientHttpRequest * http) +{ char *url = http->uri; StoreEntry *entry = NULL; int hit = 0; @@ -828,6 +731,8 @@ } if (!entry) { entry = storeCreateEntry(url, http->request->flags, http->request->method); + if (http->request->store_url) + storeEntrySetStoreUrl(entry, http->request->store_url); if (http->entry->mem_obj) { http->entry->mem_obj->refresh_timestamp = squid_curtime; if (Config.onoff.collapsed_forwarding) { @@ -836,7 +741,12 @@ } } } - /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ + if (entry->mem_obj->old_entry) { + storeUnlockObject(entry->mem_obj->old_entry); + entry->mem_obj->old_entry = NULL; + } + entry->mem_obj->old_entry = http->old_entry; + storeLockObject(entry->mem_obj->old_entry); http->sc = storeClientRegister(entry, http); #if DELAY_POOLS /* delay_id is already set on original store client */ @@ -848,6 +758,7 @@ } else http->request->lastmod = -1; debug(33, 5) ("clientProcessExpired: lastmod %ld\n", (long int) entry->lastmod); + /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */ http->entry = entry; http->out.offset = 0; if (can_revalidate) { @@ -862,10 +773,7 @@ /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n"); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, entry, clientHandleIMSReply, http); } @@ -881,8 +789,16 @@ /* If the reply is a failure then send the old object as a last * resort */ if (status >= 500 && status < 600) { - debug(33, 3) ("clientGetsOldEntry: YES, failure reply=%d\n", status); - return 1; + if (EBIT_TEST(new_entry->flags, ENTRY_NEGCACHED)) { + debug(33, 3) ("clientGetsOldEntry: NO, negatively cached failure reply=%d\n", status); + return 0; + } + if (refreshCheckStaleOK(old_entry, request)) { + debug(33, 3) ("clientGetsOldEntry: YES, failure reply=%d and old acceptable to send\n", status); + return 1; + } + debug(33, 3) ("clientGetsOldEntry: NO, failure reply=%d and old NOT acceptable to send\n", status); + return 0; } /* If the reply is not to a cache validation conditional then * we should forward it to the client */ @@ -929,7 +845,7 @@ static void -clientHandleIMSReply(void *data, mem_node_ref ref, ssize_t size) +clientHandleIMSReply(void *data, HttpReply * rep) { clientHttpRequest *http = data; StoreEntry *entry = http->entry; @@ -938,23 +854,20 @@ int unlink_request = 0; StoreEntry *oldentry; int recopy = 1; - http_status status; - debug(33, 3) ("clientHandleIMSReply: %s, %ld bytes\n", url, (long int) size); + debug(33, 3) ("clientHandleIMSReply: %s\n", url); if (http->old_entry && http->old_entry->mem_obj && http->old_entry->mem_obj->ims_entry) { storeUnlockObject(http->old_entry->mem_obj->ims_entry); http->old_entry->mem_obj->ims_entry = NULL; } if (entry == NULL) { - stmemNodeUnref(&ref); return; } - if (size < 0 && !EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - stmemNodeUnref(&ref); - return; + if (entry->mem_obj->old_entry) { + storeUnlockObject(entry->mem_obj->old_entry); + entry->mem_obj->old_entry = NULL; } mem = entry->mem_obj; - status = mem->reply->sline.status; - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + if (!rep) { debug(33, 3) ("clientHandleIMSReply: ABORTED '%s'\n", url); /* We have an existing entry, but failed to validate it */ /* Its okay to send the old one anyway */ @@ -963,45 +876,26 @@ storeUnlockObject(entry); entry = http->entry = http->old_entry; http->sc = http->old_sc; - } else if (STORE_PENDING == entry->store_status && 0 == status) { - debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url); - if (size >= SM_PAGE_SIZE) { - /* will not get any bigger than that */ - debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url); - /* use old entry, this repeats the code abovez */ - http->log_type = LOG_TCP_REFRESH_FAIL_HIT; - storeClientUnregister(http->sc, entry, http); - storeUnlockObject(entry); - entry = http->entry = http->old_entry; - http->sc = http->old_sc; - /* continue */ - } else { - stmemNodeUnref(&ref); - storeClientRef(http->sc, entry, - http->out.offset + size, - http->out.offset, - SM_PAGE_SIZE, - clientHandleIMSReply, - http); - return; - } } else if (clientGetsOldEntry(entry, http->old_entry, http->request)) { /* We initiated the IMS request, the client is not expecting * 304, so put the good one back. First, make sure the old entry * headers have been loaded from disk. */ oldentry = http->old_entry; - http->log_type = LOG_TCP_REFRESH_HIT; if (oldentry->mem_obj->request == NULL) { oldentry->mem_obj->request = requestLink(mem->request); unlink_request = 1; } - if (mem->reply->sline.status == HTTP_NOT_MODIFIED) { + if (rep->sline.status == HTTP_NOT_MODIFIED) { /* Don't memcpy() the whole reply structure here. For example, * www.thegist.com (Netscape/1.13) returns a content-length for * 304's which seems to be the length of the 304 HEADERS!!! and * not the body they refer to. */ - httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, mem->reply); + httpReplyUpdateOnNotModified(oldentry->mem_obj->reply, rep); storeTimestampsSet(oldentry); + storeUpdate(oldentry, http->request); + http->log_type = LOG_TCP_REFRESH_HIT; + } else { + http->log_type = LOG_TCP_REFRESH_FAIL_HIT; } storeClientUnregister(http->sc, entry, http); http->sc = http->old_sc; @@ -1015,11 +909,14 @@ /* the client can handle this reply, whatever it is */ http->flags.hit = 0; http->log_type = LOG_TCP_REFRESH_MISS; - if (HTTP_NOT_MODIFIED == mem->reply->sline.status) { + if (HTTP_NOT_MODIFIED == rep->sline.status) { httpReplyUpdateOnNotModified(http->old_entry->mem_obj->reply, - mem->reply); + rep); storeTimestampsSet(http->old_entry); - http->log_type = LOG_TCP_REFRESH_HIT; + if (!EBIT_TEST(http->old_entry->flags, REFRESH_FAILURE)) + http->log_type = LOG_TCP_REFRESH_HIT; + else + http->log_type = LOG_TCP_REFRESH_FAIL_HIT; } /* Get rid of the old entry if not a cache validation */ if (!http->request->flags.cache_validation) @@ -1032,16 +929,11 @@ http->old_sc = NULL; assert(!EBIT_TEST(entry->flags, ENTRY_ABORTED)); if (recopy) { - stmemNodeUnref(&ref); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, - clientSendMoreHeaderData, + storeClientCopyHeaders(http->sc, entry, + clientSendHeaders, http); } else { - clientSendMoreHeaderData(data, stmemNodeRef(&ref), size); - stmemNodeUnref(&ref); + clientSendHeaders(data, rep); } } @@ -1051,7 +943,7 @@ squid_off_t object_length; MemObject *mem = entry->mem_obj; time_t mod_time = entry->lastmod; - debug(33, 3) ("modifiedSince: '%s'\n", storeUrl(entry)); + debug(33, 3) ("modifiedSince: '%s'\n", storeLookupUrl(entry)); debug(33, 3) ("modifiedSince: mod_time = %ld\n", (long int) mod_time); if (mod_time < 0) return 1; @@ -1116,10 +1008,7 @@ http->entry->mem_obj->method = http->request->method; http->sc = storeClientRegister(http->entry, http); http->log_type = LOG_TCP_HIT; - storeClientRef(http->sc, http->entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, http->entry, clientCacheHit, http); return; @@ -1407,7 +1296,7 @@ #endif } -static void +void clientInterpretRequestHeaders(clientHttpRequest * http) { request_t *request = http->request; @@ -1680,6 +1569,8 @@ return 1; if (code == LOG_TCP_STALE_HIT) return 1; + if (code == LOG_TCP_ASYNC_HIT) + return 1; if (code == LOG_TCP_IMS_HIT) return 1; if (code == LOG_TCP_REFRESH_FAIL_HIT) @@ -2032,7 +1923,7 @@ httpHdrMangleList(hdr, request); } -/* Used exclusively by clientBuildReply() during failure cases only */ +/* Used exclusively by clientCloneReply() during failure cases only */ static void clientUnwindReply(clientHttpRequest * http, HttpReply * rep) { @@ -2054,7 +1945,7 @@ * be. */ static HttpReply * -clientBuildReply(clientHttpRequest * http) +clientCloneReply(clientHttpRequest * http, HttpReply * orig_rep) { HttpReply *rep = NULL; /* If we don't have a memobj / reply by now then we're stuffed */ @@ -2063,7 +1954,7 @@ return NULL; } /* try to grab the already-parsed header */ - rep = httpReplyClone(http->sc->entry->mem_obj->reply); + rep = httpReplyClone(orig_rep); if (rep->pstate == psParsed) { /* enforce 1.0 reply version */ httpBuildVersion(&rep->sline.version, 1, 0); @@ -2108,6 +1999,163 @@ } /* + * Perform an async refresh of an object + */ +typedef struct _clientAsyncRefreshRequest { + request_t *request; + StoreEntry *entry; + StoreEntry *old_entry; + store_client *sc; + squid_off_t offset; + size_t buf_in_use; + char readbuf[STORE_CLIENT_BUF_SZ]; + struct timeval start; +} clientAsyncRefreshRequest; + +CBDATA_TYPE(clientAsyncRefreshRequest); + +static void +clientAsyncDone(clientAsyncRefreshRequest * async) +{ + AccessLogEntry al; + static aclCheck_t *ch; + MemObject *mem = async->entry->mem_obj; + request_t *request = async->request; + memset(&al, 0, sizeof(al)); + al.icp.opcode = ICP_INVALID; + al.url = mem->url; + debug(33, 9) ("clientAsyncDone: url='%s'\n", al.url); + al.http.code = mem->reply->sline.status; + al.http.content_type = strBuf(mem->reply->content_type); + al.cache.size = async->offset; + if (async->old_entry->mem_obj) + async->old_entry->mem_obj->refresh_timestamp = 0; + if (mem->reply->sline.status == 304) { + /* Don't memcpy() the whole reply structure here. For example, + * www.thegist.com (Netscape/1.13) returns a content-length for + * 304's which seems to be the length of the 304 HEADERS!!! and + * not the body they refer to. */ + httpReplyUpdateOnNotModified(async->old_entry->mem_obj->reply, async->entry->mem_obj->reply); + storeTimestampsSet(async->old_entry); + storeUpdate(async->old_entry, async->request); + al.cache.code = LOG_TCP_ASYNC_HIT; + } else + al.cache.code = LOG_TCP_ASYNC_MISS; + al.cache.msec = tvSubMsec(async->start, current_time); + if (Config.onoff.log_mime_hdrs) { + Packer p; + MemBuf mb; + memBufDefInit(&mb); + packerToMemInit(&p, &mb); + httpHeaderPackInto(&request->header, &p); + al.headers.request = xstrdup(mb.buf); + packerClean(&p); + memBufClean(&mb); + } + al.http.method = request->method; + al.http.version = request->http_ver; + al.hier = request->hier; + if (request->auth_user_request) { + if (authenticateUserRequestUsername(request->auth_user_request)) + al.cache.authuser = xstrdup(authenticateUserRequestUsername(request->auth_user_request)); + authenticateAuthUserRequestUnlock(request->auth_user_request); + request->auth_user_request = NULL; + } else if (request->extacl_user) { + al.cache.authuser = xstrdup(request->extacl_user); + } + al.request = request; + al.reply = mem->reply; + ch = aclChecklistCreate(Config.accessList.http, request, NULL); + ch->reply = mem->reply; + if (!Config.accessList.log || aclCheckFast(Config.accessList.log, ch)) + accessLogLog(&al, ch); + aclChecklistFree(ch); + storeClientUnregister(async->sc, async->entry, async); + storeUnlockObject(async->entry); + storeUnlockObject(async->old_entry); + requestUnlink(async->request); + safe_free(al.headers.request); + safe_free(al.headers.reply); + safe_free(al.cache.authuser); + cbdataFree(async); +} + +static void +clientHandleAsyncReply(void *data, char *buf, ssize_t size) +{ + clientAsyncRefreshRequest *async = data; + StoreEntry *e = async->entry; + if (EBIT_TEST(e->flags, ENTRY_ABORTED)) { + clientAsyncDone(async); + return; + } + if (size <= 0) { + clientAsyncDone(async); + return; + } + async->offset += size; + if (e->mem_obj->reply->sline.status == 304) { + clientAsyncDone(async); + return; + } + storeClientCopy(async->sc, async->entry, + async->offset, + async->offset, + STORE_CLIENT_BUF_SZ, async->readbuf, + clientHandleAsyncReply, + async); +} + +static void +clientAsyncRefresh(clientHttpRequest * http) +{ + char *url = http->uri; + clientAsyncRefreshRequest *async; + request_t *request = http->request; + debug(33, 3) ("clientAsyncRefresh: '%s'\n", http->uri); + CBDATA_INIT_TYPE(clientAsyncRefreshRequest); + http->entry->mem_obj->refresh_timestamp = squid_curtime; + async = cbdataAlloc(clientAsyncRefreshRequest); + async->start = current_time; + async->request = requestLink(request); + async->old_entry = http->entry; + storeLockObject(async->old_entry); + async->entry = storeCreateEntry(url, + request->flags, + request->method); + async->entry->mem_obj->old_entry = async->old_entry; + storeLockObject(async->entry->mem_obj->old_entry); + async->sc = storeClientRegister(async->entry, async); + request->etags = NULL; /* Should always be null as this was a cache hit, but just in case.. */ + httpHeaderDelById(&request->header, HDR_RANGE); + httpHeaderDelById(&request->header, HDR_IF_RANGE); + httpHeaderDelById(&request->header, HDR_IF_NONE_MATCH); + httpHeaderDelById(&request->header, HDR_IF_MATCH); + if (async->old_entry->lastmod > 0) + request->lastmod = async->old_entry->lastmod; + else if (async->old_entry->mem_obj && async->old_entry->mem_obj->reply) + request->lastmod = async->old_entry->mem_obj->reply->date; + else + request->lastmod = -1; + if (!request->etag) { + const char *etag = httpHeaderGetStr(&async->old_entry->mem_obj->reply->header, HDR_ETAG); + if (etag) + async->request->etag = xstrdup(etag); + } +#if DELAY_POOLS + /* delay_id is already set on original store client */ + delaySetStoreClient(async->sc, delayClient(http)); +#endif + fwdStart(-1, async->entry, async->request); + storeClientCopy(async->sc, async->entry, + async->offset, + async->offset, + STORE_CLIENT_BUF_SZ, async->readbuf, + clientHandleAsyncReply, + async); +} + +/* * clientCacheHit should only be called until the HTTP reply headers * have been parsed. Normally this should be a single call, but * it might take more than one. As soon as we have the headers, @@ -2115,7 +2163,7 @@ * clientProcessMiss. */ static void -clientCacheHit(void *data, mem_node_ref ref, ssize_t size) +clientCacheHit(void *data, HttpReply * rep) { clientHttpRequest *http = data; StoreEntry *e = http->entry; @@ -2123,32 +2171,31 @@ request_t *r = http->request; int is_modified = -1; int stale; - debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http->uri, (int) size); http->flags.hit = 0; if (http->entry == NULL) { - stmemNodeUnref(&ref); debug(33, 3) ("clientCacheHit: request aborted\n"); return; - } else if (size <= 0) { + } else if (!rep) { /* swap in failure */ - stmemNodeUnref(&ref); debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http->uri); http->log_type = LOG_TCP_SWAPFAIL_MISS; clientProcessMiss(http); return; } - assert(size > 0); mem = e->mem_obj; + debug(33, 3) ("clientCacheHit: %s = %d\n", http->uri, rep->sline.status); assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); - if (!memHaveHeaders(mem)) { - debug(33, 1) ("clientCacheHit: No reply headers in '%s'?\n", e->mem_obj->url); - stmemNodeUnref(&ref); + + if (r->store_url && mem->store_url == NULL && strcmp(r->store_url, mem->url) != 0) { + debug(33, 1) ("clientCacheHit: request has store_url '%s'; mem object in hit doesn't and doesn't match the url '%s'!\n", r->store_url, mem->url); clientProcessMiss(http); return; - } - if (strcmp(mem->url, urlCanonical(r)) != 0) { - debug(33, 1) ("clientCacheHit: URL mismatch '%s' != '%s'?\n", e->mem_obj->url, urlCanonical(r)); - stmemNodeUnref(&ref); + } else if (r->store_url && strcmp(mem->store_url, r->store_url) != 0) { + debug(33, 1) ("clientCacheHit: store URL mismatch '%s' != '%s'?\n", mem->store_url, r->store_url); + clientProcessMiss(http); + return; + } else if ((!r->store_url) && (!mem->store_url) && strcmp(mem->url, urlCanonical(r)) != 0) { + debug(33, 1) ("clientCacheHit: (store url '%s'); URL mismatch '%s' != '%s'?\n", r->store_url, e->mem_obj->url, urlCanonical(r)); clientProcessMiss(http); return; } @@ -2184,7 +2231,6 @@ * so we only get here once. (it also takes care of cancelling loops) */ debug(33, 2) ("clientProcessHit: Vary detected!\n"); - stmemNodeUnref(&ref); return; } case VARY_RESTART: @@ -2194,13 +2240,11 @@ safe_free(r->vary_hdr); safe_free(r->vary_headers); clientProcessRequest(http); - stmemNodeUnref(&ref); return; case VARY_CANCEL: /* varyEvaluateMatch found a object loop. Process as miss */ debug(33, 1) ("clientProcessHit: Vary object loop!\n"); clientProcessMiss(http); - stmemNodeUnref(&ref); return; } if (r->method == METHOD_PURGE) { @@ -2209,7 +2253,6 @@ http->sc = NULL; storeUnlockObject(e); clientPurgeRequest(http); - stmemNodeUnref(&ref); return; } http->flags.hit = 1; @@ -2220,11 +2263,9 @@ #endif ) { http->log_type = LOG_TCP_NEGATIVE_HIT; - clientSendMoreHeaderData(data, stmemNodeRef(&ref), size); - stmemNodeUnref(&ref); + clientSendHeaders(data, rep); } else { http->log_type = LOG_TCP_MISS; - stmemNodeUnref(&ref); clientProcessMiss(http); } return; @@ -2242,7 +2283,6 @@ * Query the origin to see what should be done. */ http->log_type = LOG_TCP_MISS; - stmemNodeUnref(&ref); clientProcessMiss(http); return; } @@ -2255,7 +2295,6 @@ debug(33, 4) ("clientCacheHit: Reply code %d != 200\n", mem->reply->sline.status); http->log_type = LOG_TCP_MISS; - stmemNodeUnref(&ref); clientProcessMiss(http); return; } @@ -2290,6 +2329,10 @@ debug(33, 2) ("clientProcessHit: refresh_stale HIT\n"); http->log_type = LOG_TCP_STALE_HIT; stale = 0; + } else if (stale == -2 && e->mem_obj->refresh_timestamp + e->mem_obj->stale_while_revalidate >= squid_curtime) { + debug(33, 2) ("clientProcessHit: stale-while-revalidate HIT\n"); + http->log_type = LOG_TCP_STALE_HIT; + stale = 0; } else if (stale && http->flags.internal) { debug(33, 2) ("clientProcessHit: internal HIT\n"); stale = 0; @@ -2297,7 +2340,13 @@ debug(33, 2) ("clientProcessHit: offline HIT\n"); http->log_type = LOG_TCP_OFFLINE_HIT; stale = 0; + } else if (stale == -2) { + debug(33, 2) ("clientProcessHit: stale-while-revalidate needs revalidation\n"); + clientAsyncRefresh(http); + http->log_type = LOG_TCP_STALE_HIT; + stale = 0; } + http->is_modified = is_modified; if (stale) { debug(33, 5) ("clientCacheHit: in refreshCheck() block\n"); /* @@ -2318,12 +2367,20 @@ */ http->log_type = LOG_TCP_CLIENT_REFRESH_MISS; clientProcessMiss(http); - } else { - clientProcessExpired(http); + return; } - stmemNodeUnref(&ref); + clientRefreshCheck(http); return; } + clientProcessHit(http); +} + +static void +clientProcessHit(clientHttpRequest * http) +{ + int is_modified = http->is_modified; + StoreEntry *e = http->entry; + if (is_modified == 0) { time_t timestamp = e->timestamp; MemBuf mb = httpPacked304Reply(e->mem_obj->reply); @@ -2343,18 +2400,18 @@ storeAppend(e, mb.buf, mb.size); memBufClean(&mb); storeComplete(e); - stmemNodeUnref(&ref); return; } /* * plain ol' cache hit */ + if (EBIT_TEST(e->flags, REFRESH_FAILURE)) + http->log_type = LOG_TCP_NEGATIVE_HIT; if (e->store_status != STORE_OK) http->log_type = LOG_TCP_MISS; else if (http->log_type == LOG_TCP_HIT && e->mem_status == IN_MEMORY) http->log_type = LOG_TCP_MEM_HIT; - clientSendMoreHeaderData(data, stmemNodeRef(&ref), size); - stmemNodeUnref(&ref); + clientSendHeaders(http, e->mem_obj->reply); } /* put terminating boundary for multiparts */ @@ -2612,49 +2669,32 @@ * such, writes processed message to the client's socket */ static void -clientSendMoreHeaderData(void *data, mem_node_ref ref, ssize_t size) +clientSendHeaders(void *data, HttpReply * rep) { //const char *buf = ref.node->data + ref.offset; clientHttpRequest *http = data; StoreEntry *entry = http->entry; ConnStateData *conn = http->conn; int fd = conn->fd; - HttpReply *rep = NULL; - debug(33, 5) ("clientSendMoreHeaderData: %s, %d bytes\n", http->uri, (int) size); - assert(size <= SM_PAGE_SIZE); assert(http->request != NULL); dlinkDelete(&http->active, &ClientActiveRequests); dlinkAdd(http, &http->active, &ClientActiveRequests); - debug(33, 5) ("clientSendMoreHeaderData: FD %d '%s', out.offset=%ld \n", - fd, storeUrl(entry), (long int) http->out.offset); + debug(33, 5) ("clientSendMoreHeaderData: FD %d '%s'\n", fd, storeUrl(entry)); assert(conn->reqs.head != NULL); if (DLINK_HEAD(conn->reqs) != http) { /* there is another object in progress, defer this one */ debug(33, 2) ("clientSendMoreHeaderData: Deferring %s\n", storeUrl(entry)); - stmemNodeUnref(&ref); return; } else if (http->request->flags.reset_tcp) { comm_reset_close(fd); - stmemNodeUnref(&ref); - return; - } else if (entry && EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* call clientWriteComplete so the client socket gets closed */ - stmemNodeUnref(&ref); - clientWriteComplete(fd, NULL, 0, COMM_OK, http); - return; - } else if (size < 0) { - /* call clientWriteComplete so the client socket gets closed */ - clientWriteComplete(fd, NULL, 0, COMM_OK, http); - stmemNodeUnref(&ref); return; - } else if (size == 0) { + } else if (!rep) { /* call clientWriteComplete so the client socket gets closed */ clientWriteComplete(fd, NULL, 0, COMM_OK, http); - stmemNodeUnref(&ref); return; } assert(http->out.offset == 0); - rep = http->reply = clientBuildReply(http); + rep = http->reply = clientCloneReply(http, rep); if (!rep) { ErrorState *err = errorCon(ERR_INVALID_RESP, HTTP_BAD_GATEWAY, http->orig_request); storeClientUnregister(http->sc, http->entry, http); @@ -2663,7 +2703,6 @@ http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); errorAppendEntry(http->entry, err); - stmemNodeUnref(&ref); return; } clientMaxBodySize(http->request, http, rep); @@ -2676,7 +2715,6 @@ http->entry = clientCreateStoreEntry(http, http->request->method, null_request_flags); errorAppendEntry(http->entry, err); - stmemNodeUnref(&ref); return; } /* @@ -2691,7 +2729,6 @@ */ http->range_iter.prefix_size = rep->hdr_sz; debug(33, 3) ("clientSendMoreHeaderData: %d bytes of headers\n", rep->hdr_sz); - stmemNodeUnref(&ref); clientHttpLocationRewriteCheck(http); } @@ -2956,7 +2993,7 @@ * If we are here, then store_status == STORE_OK and it * seems we have a HEAD repsponse which is missing the * empty end-of-headers line (home.mira.net, phttpd/0.99.72 - * does this). Because clientBuildReply() fails we just + * does this). Because clientCloneReply() fails we just * call this reply a body, set the done_copying flag and * continue... */ @@ -3047,11 +3084,8 @@ if (0 == storeClientCopyPending(http->sc, entry, http)) { if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n"); - storeClientRef(http->sc, entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, - clientSendMoreHeaderData, + storeClientCopyHeaders(http->sc, entry, + clientSendHeaders, http); } } @@ -3357,10 +3391,7 @@ #if DELAY_POOLS delaySetStoreClient(http->sc, delayClient(http)); #endif - storeClientRef(http->sc, http->entry, - http->out.offset, - http->out.offset, - SM_PAGE_SIZE, + storeClientCopyHeaders(http->sc, http->entry, clientCacheHit, http); } else { --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/client_side_rewrite.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,154 @@ + +/* + * $Id: client_side_rewrite.c,v 1.2.4.2 2007/11/27 08:12:27 adri Exp $ + * + * DEBUG: section 33 Client-side Routines - URL Rewriter + * AUTHOR: Duane Wessels; Adrian Chadd + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + + +/* Local functions */ + +void +clientRedirectAccessCheckDone(int answer, void *data) +{ + clientHttpRequest *http = data; + http->acl_checklist = NULL; + if (answer == ACCESS_ALLOWED) + redirectStart(http, clientRedirectDone, http); + else + clientRedirectDone(http, NULL); +} + +void +clientRedirectStart(clientHttpRequest * http) +{ + debug(33, 5) ("clientRedirectStart: '%s'\n", http->uri); + if (Config.Program.url_rewrite.command == NULL) { + clientRedirectDone(http, NULL); + return; + } + if (Config.accessList.url_rewrite) { + http->acl_checklist = clientAclChecklistCreate(Config.accessList.url_rewrite, http); + aclNBCheck(http->acl_checklist, clientRedirectAccessCheckDone, http); + } else { + redirectStart(http, clientRedirectDone, http); + } +} + +void +clientRedirectDone(void *data, char *result) +{ + clientHttpRequest *http = data; + request_t *new_request = NULL; + request_t *old_request = http->request; + const char *urlgroup = http->conn->port->urlgroup; + debug(33, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri, + result ? result : "NULL"); + assert(http->redirect_state == REDIRECT_PENDING); + http->redirect_state = REDIRECT_DONE; + if (result) { + http_status status; + if (*result == '!') { + char *t; + if ((t = strchr(result + 1, '!')) != NULL) { + urlgroup = result + 1; + *t++ = '\0'; + result = t; + } else { + debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); + } + } + status = (http_status) atoi(result); + if (status == HTTP_MOVED_PERMANENTLY + || status == HTTP_MOVED_TEMPORARILY + || status == HTTP_SEE_OTHER + || status == HTTP_TEMPORARY_REDIRECT) { + char *t = result; + if ((t = strchr(result, ':')) != NULL) { + http->redirect.status = status; + http->redirect.location = xstrdup(t + 1); + goto redirect_parsed; + } else { + debug(33, 1) ("clientRedirectDone: bad input: %s\n", result); + } + } else if (strcmp(result, http->uri)) + new_request = urlParse(old_request->method, result); + } + redirect_parsed: + if (new_request) { + safe_free(http->uri); + http->uri = xstrdup(urlCanonical(new_request)); + new_request->http_ver = old_request->http_ver; + httpHeaderAppend(&new_request->header, &old_request->header); + new_request->client_addr = old_request->client_addr; + new_request->client_port = old_request->client_port; +#if FOLLOW_X_FORWARDED_FOR + new_request->indirect_client_addr = old_request->indirect_client_addr; +#endif /* FOLLOW_X_FORWARDED_FOR */ + new_request->my_addr = old_request->my_addr; + new_request->my_port = old_request->my_port; + new_request->flags = old_request->flags; + new_request->flags.redirected = 1; + if (old_request->auth_user_request) { + new_request->auth_user_request = old_request->auth_user_request; + authenticateAuthUserRequestLock(new_request->auth_user_request); + } + if (old_request->body_reader) { + new_request->body_reader = old_request->body_reader; + new_request->body_reader_data = old_request->body_reader_data; + old_request->body_reader = NULL; + old_request->body_reader_data = NULL; + } + new_request->content_length = old_request->content_length; + if (strBuf(old_request->extacl_log)) + new_request->extacl_log = stringDup(&old_request->extacl_log); + if (old_request->extacl_user) + new_request->extacl_user = xstrdup(old_request->extacl_user); + if (old_request->extacl_passwd) + new_request->extacl_passwd = xstrdup(old_request->extacl_passwd); + requestUnlink(old_request); + http->request = requestLink(new_request); + } else { + /* Don't mess with urlgroup on internal request */ + if (old_request->flags.internal) + urlgroup = NULL; + } + safe_free(http->request->urlgroup); /* only paranoia. should not happen */ + if (urlgroup && *urlgroup) + http->request->urlgroup = xstrdup(urlgroup); + clientInterpretRequestHeaders(http); + /* XXX This really should become a ref-counted string type pointer, not a copy! */ + fd_note(http->conn->fd, http->uri); + + clientStoreURLRewriteStart(http); +} --- /dev/null Wed Nov 28 01:20:24 2007 +++ squid/src/client_side_storeurl_rewrite.c Wed Nov 28 01:20:25 2007 @@ -0,0 +1,92 @@ + +/* + * $Id: client_side_storeurl_rewrite.c,v 1.3.2.2 2007/11/27 08:12:27 adri Exp $ + * + * DEBUG: section 85 Client-side Routines - Store URL Rewriter + * AUTHOR: Duane Wessels; Adrian Chadd + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + + +/* Local functions */ + +void +clientStoreURLRewriteAccessCheckDone(int answer, void *data) +{ + clientHttpRequest *http = data; + http->acl_checklist = NULL; + if (answer == ACCESS_ALLOWED) + storeurlStart(http, clientStoreURLRewriteDone, http); + else + clientStoreURLRewriteDone(http, NULL); +} + +void +clientStoreURLRewriteStart(clientHttpRequest * http) +{ + debug(85, 5) ("clientStoreURLRewriteStart: '%s'\n", http->uri); + if (Config.Program.store_rewrite.command == NULL) { + clientStoreURLRewriteDone(http, NULL); + return; + } + if (Config.accessList.storeurl_rewrite) { + http->acl_checklist = clientAclChecklistCreate(Config.accessList.storeurl_rewrite, http); + aclNBCheck(http->acl_checklist, clientStoreURLRewriteAccessCheckDone, http); + } else { + storeurlStart(http, clientStoreURLRewriteDone, http); + } +} + +void +clientStoreURLRewriteDone(void *data, char *result) +{ + clientHttpRequest *http = data; + + debug(85, 3) ("clientStoreURLRewriteDone: '%s' result=%s\n", http->uri, + result ? result : "NULL"); +#if 0 + assert(http->redirect_state == REDIRECT_PENDING); + http->redirect_state = REDIRECT_DONE; +#endif + + if (result) { + http->request->store_url = xstrdup(result); + debug(85, 3) ("Rewrote to %s\n", http->request->store_url); + /* XXX is this actually the right spot to do this? How about revalidation? */ + //storeEntrySetStoreUrl(http->entry, result); + } + /* This is the final part of the rewrite chain - this should be broken out! */ + clientInterpretRequestHeaders(http); +#if HEADERS_LOG + headersLog(0, 1, request->method, request); +#endif + clientAccessCheck2(http); +} Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/enums.h,v retrieving revision 1.62 retrieving revision 1.62.2.1 diff -u -r1.62 -r1.62.2.1 --- squid/src/enums.h 24 Sep 2007 13:52:08 -0000 1.62 +++ squid/src/enums.h 27 Nov 2007 08:12:27 -0000 1.62.2.1 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.62 2007/09/24 13:52:08 squidadm Exp $ + * $Id: enums.h,v 1.62.2.1 2007/11/27 08:12:27 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -52,6 +52,8 @@ LOG_TCP_REDIRECT, #endif LOG_TCP_STALE_HIT, + LOG_TCP_ASYNC_HIT, + LOG_TCP_ASYNC_MISS, LOG_UDP_HIT, LOG_UDP_MISS, LOG_UDP_DENIED, @@ -525,7 +527,7 @@ ENTRY_REVALIDATE, DELAY_SENDING, RELEASE_REQUEST, - REFRESH_REQUEST, + REFRESH_FAILURE, ENTRY_CACHABLE, ENTRY_DISPATCHED, KEY_PRIVATE, @@ -654,6 +656,7 @@ STORE_META_VARY_HEADERS, /* Stores Vary request headers */ STORE_META_STD_LFS, /* standard metadata in lfs format */ STORE_META_OBJSIZE, /* object size, if its known */ + STORE_META_STOREURL, /* the store url, if different to the normal URL */ STORE_META_END }; Index: squid/src/errorpage.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/errorpage.c,v retrieving revision 1.37 retrieving revision 1.37.6.1 diff -u -r1.37 -r1.37.6.1 --- squid/src/errorpage.c 26 Feb 2007 09:51:31 -0000 1.37 +++ squid/src/errorpage.c 27 Nov 2007 08:12:27 -0000 1.37.6.1 @@ -1,6 +1,6 @@ /* - * $Id: errorpage.c,v 1.37 2007/02/26 09:51:31 squidadm Exp $ + * $Id: errorpage.c,v 1.37.6.1 2007/11/27 08:12:27 adri Exp $ * * DEBUG: section 4 Error Generation * AUTHOR: Duane Wessels @@ -324,7 +324,7 @@ authenticateFixHeader(rep, err->auth_user_request, err->request, 0, 1); httpReplySwapOut(rep, entry); EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); - storeNegativeCache(entry); + storeExpireNow(entry); storeReleaseRequest(entry); storeBufferFlush(entry); storeComplete(entry); Index: squid/src/fqdncache.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/fqdncache.c,v retrieving revision 1.22 retrieving revision 1.22.2.1 diff -u -r1.22 -r1.22.2.1 --- squid/src/fqdncache.c 17 Sep 2007 09:51:44 -0000 1.22 +++ squid/src/fqdncache.c 27 Nov 2007 08:12:27 -0000 1.22.2.1 @@ -1,6 +1,6 @@ /* - * $Id: fqdncache.c,v 1.22 2007/09/17 09:51:44 squidadm Exp $ + * $Id: fqdncache.c,v 1.22.2.1 2007/11/27 08:12:27 adri Exp $ * * DEBUG: section 35 FQDN Cache * AUTHOR: Harvest Derived @@ -316,7 +316,7 @@ f->error_message = xstrdup("No PTR record"); return f; } - if (ttl == 0 || ttl > Config.positiveDnsTtl) + if (ttl > Config.positiveDnsTtl) ttl = Config.positiveDnsTtl; if (ttl < Config.negativeDnsTtl) ttl = Config.negativeDnsTtl; Index: squid/src/helper.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/helper.c,v retrieving revision 1.43 retrieving revision 1.43.6.1 diff -u -r1.43 -r1.43.6.1 --- squid/src/helper.c 1 Apr 2007 15:08:17 -0000 1.43 +++ squid/src/helper.c 27 Nov 2007 08:12:27 -0000 1.43.6.1 @@ -1,6 +1,6 @@ /* - * $Id: helper.c,v 1.43 2007/04/01 15:08:17 squidadm Exp $ + * $Id: helper.c,v 1.43.6.1 2007/11/27 08:12:27 adri Exp $ * * DEBUG: section 84 Helper process maintenance * AUTHOR: Harvest Derived? @@ -1026,7 +1026,7 @@ { helper *hlp = srv->parent; helper_request **ptr = NULL; - int slot; + int slot = -1; if (!cbdataValid(r->data)) { debug(84, 1) ("helperDispatch: invalid callback data\n"); helperRequestFree(r); @@ -1057,8 +1057,8 @@ helperDispatch_done, /* Handler */ srv); } - debug(84, 5) ("helperDispatch: Request sent to %s #%d, %d bytes\n", - hlp->id_name, srv->index + 1, (int) strlen(r->buf)); + debug(84, 5) ("helperDispatch: Request sent to %s #%d[%d], %d bytes\n", + hlp->id_name, srv->index + 1, slot, (int) strlen(r->buf)); srv->stats.uses++; hlp->stats.requests++; } Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/http.c,v retrieving revision 1.63 retrieving revision 1.63.2.1 diff -u -r1.63 -r1.63.2.1 --- squid/src/http.c 3 Sep 2007 04:49:47 -0000 1.63 +++ squid/src/http.c 27 Nov 2007 08:12:28 -0000 1.63.2.1 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.63 2007/09/03 04:49:47 squidadm Exp $ + * $Id: http.c,v 1.63.2.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -131,6 +131,8 @@ storeNegativeCache(entry); if (EBIT_TEST(entry->flags, ENTRY_CACHABLE)) storeSetPublicKey(entry); + if (entry->expires <= squid_curtime) + storeRelease(entry); } static void @@ -432,7 +434,7 @@ debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr.buf); httpState->reply_hdr_state += 2; httpState->chunk_size = -1; /* Terminated by EOF */ - memBufClean(&httpState->reply_hdr); + httpState->reply_hdr.size = old_size; httpBuildVersion(&reply->sline.version, 0, 9); reply->sline.status = HTTP_INVALID_HEADER; ctx_exit(ctx); @@ -506,8 +508,8 @@ stringClean(&tr); if (httpState->flags.chunked && reply->content_length >= 0) { /* Can't have a content-length in chunked encoding */ - reply->sline.status = HTTP_INVALID_HEADER; - return done; + reply->content_length = -1; + httpHeaderDelById(&reply->header, HDR_CONTENT_LENGTH); } } if (!httpState->flags.chunked) { @@ -530,6 +532,8 @@ if (strBuf(httpState->orig_request->vary_encoding)) entry->mem_obj->vary_encoding = xstrdup(strBuf(httpState->orig_request->vary_encoding)); } + if (entry->mem_obj->old_entry) + EBIT_CLR(entry->mem_obj->old_entry->flags, REFRESH_FAILURE); switch (httpCachableReply(httpState)) { case 1: httpMakePublic(entry); @@ -538,10 +542,7 @@ httpMakePrivate(entry); break; case -1: - if (Config.negativeTtl > 0) - httpCacheNegatively(entry); - else - httpMakePrivate(entry); + httpCacheNegatively(entry); break; default: assert(0); @@ -955,6 +956,8 @@ httpHeaderPutTime(&reply->header, HDR_DATE, squid_curtime); mb = httpReplyPack(reply); storeAppend(entry, mb.buf, mb.size); + storeAppend(entry, httpState->reply_hdr.buf, httpState->reply_hdr.size); + memBufClean(&httpState->reply_hdr); httpReplyReset(reply); httpReplyParse(reply, mb.buf, mb.size); memBufClean(&mb); @@ -1145,13 +1148,14 @@ /* append unless we added our own; * note: at most one client's ims header can pass through */ if (!httpHeaderHas(hdr_out, HDR_IF_MODIFIED_SINCE)) - httpHeaderAddClone(hdr_out, e); + if (!Config.onoff.ignore_ims_on_miss || !orig_request->flags.cachable || orig_request->flags.auth) + httpHeaderAddClone(hdr_out, e); break; case HDR_IF_NONE_MATCH: - /* append unless we added our own; - * note: at most one client's ims header can pass through */ + /* append unless ignore_ims_on_miss is in effect */ if (!httpHeaderHas(hdr_out, HDR_IF_NONE_MATCH)) - httpHeaderAddClone(hdr_out, e); + if (!Config.onoff.ignore_ims_on_miss || !orig_request->flags.cachable || orig_request->flags.auth) + httpHeaderAddClone(hdr_out, e); break; case HDR_MAX_FORWARDS: if (orig_request->method == METHOD_TRACE) { @@ -1253,10 +1257,10 @@ /* Special mode, to pass the username to the upstream cache */ char loginbuf[256]; const char *username = "-"; - if (orig_request->auth_user_request) - username = authenticateUserRequestUsername(orig_request->auth_user_request); - else if (orig_request->extacl_user) + if (orig_request->extacl_user) username = orig_request->extacl_user; + else if (orig_request->auth_user_request) + username = authenticateUserRequestUsername(orig_request->auth_user_request); snprintf(loginbuf, sizeof(loginbuf), "%s%s", username, orig_request->peer_login + 1); httpHeaderPutStrf(hdr_out, HDR_PROXY_AUTHORIZATION, "Basic %s", base64_encode(loginbuf)); Index: squid/src/ipcache.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/ipcache.c,v retrieving revision 1.25 retrieving revision 1.25.2.1 diff -u -r1.25 -r1.25.2.1 --- squid/src/ipcache.c 17 Sep 2007 09:51:44 -0000 1.25 +++ squid/src/ipcache.c 27 Nov 2007 08:12:28 -0000 1.25.2.1 @@ -1,6 +1,6 @@ /* - * $Id: ipcache.c,v 1.25 2007/09/17 09:51:44 squidadm Exp $ + * $Id: ipcache.c,v 1.25.2.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 14 IP Cache * AUTHOR: Harvest Derived @@ -369,7 +369,7 @@ i->addrs.count = (unsigned char) na; else i->addrs.count = 255; - if (ttl == 0 || ttl > Config.positiveDnsTtl) + if (ttl > Config.positiveDnsTtl) ttl = Config.positiveDnsTtl; if (ttl < Config.negativeDnsTtl) ttl = Config.negativeDnsTtl; Index: squid/src/main.c =================================================================== RCS file: /cvsroot/squid-sf//squid/src/main.c,v retrieving revision 1.80 retrieving revision 1.80.2.1 diff -u -r1.80 -r1.80.2.1 --- squid/src/main.c 24 Sep 2007 13:52:09 -0000 1.80 +++ squid/src/main.c 27 Nov 2007 08:12:28 -0000 1.80.2.1 @@ -1,6 +1,6 @@ /* - * $Id: main.c,v 1.80 2007/09/24 13:52:09 squidadm Exp $ + * $Id: main.c,v 1.80.2.1 2007/11/27 08:12:28 adri Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -396,9 +396,11 @@ idnsShutdown(); #endif redirectShutdown(); + storeurlShutdown(); locationRewriteShutdown(); authenticateShutdown(); externalAclShutdown(); + refreshCheckShutdown(); storeDirCloseSwapLogs(); storeLogClose(); accessLogClose(); @@ -425,9 +427,11 @@ idnsInit(); #endif redirectInit(); + storeurlInit(); locationRewriteInit(); authenticateInit(&Config.authConfig); externalAclInit(); + refreshCheckInit(); #if USE_WCCP wccpInit(); #endif @@ -463,6 +467,7 @@ locationRewriteShutdown(); authenticateShutdown(); externalAclShutdown(); + refreshCheckShutdown(); _db_rotate_log(); /* cache.log */ storeDirWriteCleanLogs(1); storeDirSync(); /* Flush pending I/O ops */ @@ -478,9 +483,11 @@ dnsInit(); #endif redirectInit(); + storeurlInit(); locationRewriteInit(); authenticateInit(&Config.authConfig); externalAclInit(); + refreshCheckInit(); } static void @@ -576,10 +583,12 @@ idnsInit(); #endif redirectInit(); + storeurlInit(); locationRewriteInit(); errorMapInit(); authenticateInit(&Config.authConfig); externalAclInit(); + refreshCheckInit(); useragentOpenLog(); refererOpenLog(); httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ @@ -715,7 +724,9 @@ #endif mainParseOptions(argc, argv); +#if HAVE_SYSLOG && defined(LOG_LOCAL4) openlog(appname, LOG_PID | LOG_NDELAY | LOG_CONS, syslog_facility); +#endif #if defined(USE_WIN32_SERVICE) && defined(_SQUID_WIN32_) if (opt_install_service) { @@ -1087,6 +1098,7 @@ #endif redirectShutdown(); externalAclShutdown(); + refreshCheckShutdown(); locationRewriteShutdown(); icpConnectionClose(); #if USE_HTCP Index: squid/src/mib.txt =================================================================== RCS file: /cvsroot/squid-sf//squid/src/mib.txt,v retrieving revision 1.9 retrieving revision 1.9.4.1 diff -u -r1.9 -r1.9.4.1 --- squid/src/mib.txt 27 Aug 2007 13:53:41 -0000 1.9 +++ squid/src/mib.txt 27 Nov 2007 08:12:28 -0000 1.9.4.1 @@ -2,7 +2,7 @@ SQUID-MIB DEFINITIONS ::= BEGIN -- --- $Id: mib.txt,v 1.9 2007/08/27 13:53:41 squidadm Exp $ +-- $Id: mib.txt,v 1.9.4.1 2007/11/27 08:12:28 adri Exp $ -- IMPORTS @@ -715,7 +715,7 @@ MAX-ACCESS not-accessible STATUS current DESCRIPTION - " An entry in cachePeerTable " + " cache_peer indexed by IP address " INDEX { cachePeerAddr } ::= { cachePeerTable 1 } @@ -724,7 +724,7 @@ MAX-ACCESS not-accessible STATUS current DESCRIPTION - " An entry in cachePeerTable " + " cache_peer indexed by position in squid.conf " INDEX { cachePeerIndex } ::= { cachePeerTable 2 } Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid-sf//squid/src/protos.h,v retrieving revision 1.146.2.3 retrieving revision 1.146.2.4 diff -u -r1.146.2.3 -r1.146.2.4 --- squid/src/protos.h 27 Sep 2007 15:09:17 -0000 1.146.2.3 +++ squid/src/protos.h 27 Nov 2007 08:12:28 -0000 1.146.2.4 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.146.2.3 2007/09/27 15:09:17 adri Exp $ + * $Id: protos.h,v 1.146.2.4 2007/11/27 08:12:28 adri Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -747,6 +747,10 @@ extern void redirectInit(void); extern void redirectShutdown(void); +extern void storeurlStart(clientHttpRequest *, RH *, void *); +extern void storeurlInit(void); +extern void storeurlShutdown(void); + extern void locationRewriteStart(HttpReply *, clientHttpRequest *, RH *, void *); extern void locationRewriteInit(void); extern void locationRewriteShutdown(void); @@ -796,9 +800,11 @@ extern int refreshIsCachable(const StoreEntry *); extern int refreshCheckHTTP(const StoreEntry *, request_t *); extern int refreshCheckHTTPStale(const StoreEntry *, request_t *); +extern int refreshCheckStaleOK(const StoreEntry *, request_t *); extern int refreshCheckICP(const StoreEntry *, request_t *); extern int refreshCheckHTCP(const StoreEntry *, request_t *); extern int refreshCheckDigest(const StoreEntry *, time_t delta); +extern refresh_cc refreshCC(const StoreEntry *, request_t *); extern time_t getMaxAge(const char *url); extern void refreshInit(void); extern const refresh_t *refreshLimits(const char *url); @@ -911,6 +917,7 @@ * store.c */ extern StoreEntry *new_StoreEntry(int, const char *); +extern void storeEntrySetStoreUrl(StoreEntry * e, const char *store_url); extern StoreEntry *storeGet(const cache_key *); extern StoreEntry *storeGetPublic(const char *uri, const method_t method); extern StoreEntry *storeGetPublicByRequest(request_t * request); @@ -924,6 +931,7 @@ extern void storeLockObjectDebug(StoreEntry *, const char *file, const int line); extern void storeRelease(StoreEntry *); extern int storeUnlockObjectDebug(StoreEntry *, const char *file, const int line); +extern const char *storeLookupUrl(const StoreEntry * e); #define storeLockObject(a) storeLockObjectDebug(a, __FILE__, __LINE__); #define storeUnlockObject(a) storeUnlockObjectDebug(a, __FILE__, __LINE__); extern EVH storeMaintainSwapSpace; @@ -970,6 +978,7 @@ void storeResumeRead(StoreEntry *); void storeResetDefer(StoreEntry *); extern int memHaveHeaders(const MemObject * mem); +extern void storeUpdate(StoreEntry * e, request_t *); /* store_modules.c */ @@ -1091,6 +1100,7 @@ extern store_client *storeClientRegister(StoreEntry * e, void *data); extern void storeClientCopy(store_client *, StoreEntry *, squid_off_t, squid_off_t, size_t, char *, STCB *, void *); extern void storeClientRef(store_client *, StoreEntry *, squid_off_t, squid_off_t, size_t, STNCB *, void *); +extern void storeClientCopyHeaders(store_client *, StoreEntry *, STHCB *, void *); extern int storeClientCopyPending(store_client *, StoreEntry * e, void *data); extern int storeClientUnregister(store_client * sc, StoreEntry * e, void *data); extern squid_off_t storeLowestMemReaderOffset(const StoreEntry * entry); @@ -1415,6 +1425,15 @@ const char *externalAclMessage(external_acl_entry * entry); +/* refresh_check.c */ +extern void parse_refreshCheckHelper(refresh_check_helper **); +extern void dump_refreshCheckHelper(StoreEntry * sentry, const char *name, const refresh_check_helper *); +extern void free_refreshCheckHelper(refresh_check_helper **); +extern void refreshCheckSubmit(StoreEntry * entry, REFRESHCHECK * callback, void *data); +extern void refreshCheckInit(void); +extern void refreshCheckConfigure(void); +extern void refreshCheckShutdown(void); + #if USE_WCCPv2 extern void parse_wccp2_service(void *v); extern void free_wccp2_service(void *v); @@ -1448,5 +1467,20 @@ extern const char *xinet_ntoa(const struct in_addr addr); +/* client_side.c */ +extern aclCheck_t *clientAclChecklistCreate(const acl_access * acl, const clientHttpRequest * http); +extern void clientInterpretRequestHeaders(clientHttpRequest * http); +extern void clientAccessCheck2(void *data); + +/* client_side_redirect.c */ +extern void clientRedirectAccessCheckDone(int answer, void *data); +extern void clientRedirectStart(clientHttpRequest * http); +extern void clientRedirectDone(void *data, char *result); + +/* client_side_storeurl_rewrite.c */ +extern void cl