Distribution EOLE » Modules » Amon

//Please refer to http://dansguardian.org/?page=copyright2

//for the license for this code.

//Written by Daniel Barron (daniel@//jadeb.com).

//For support go to http://groups.yahoo.com/group/dansguardian

//  This program is free software; you can redistribute it and/or modify

//  it under the terms of the GNU General Public License as published by

//  the Free Software Foundation; either version 2 of the License, or

//  (at your option) any later version.

//

//  This program is distributed in the hope that it will be useful,

//  but WITHOUT ANY WARRANTY; without even the implied warranty of

//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

//  GNU General Public License for more details.

//

//  You should have received a copy of the GNU General Public License

//  along with this program; if not, write to the Free Software

//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

//This file contains modifications suggested and mostly provided by

//Daniel Robbins 13/4/01 drobbins@gento.org

//Modifications include, but not limited to, getcontenttype(), << , >>

// INCLUDES

#ifdef HAVE_CONFIG_H

        #include "dgconfig.h"

#endif

#include "HTTPHeader.hpp"

#include "Socket.hpp"

#include "OptionContainer.hpp"

#include "FDTunnel.hpp"

#include <unistd.h>

#include <sys/socket.h>

#include <exception>

#include <time.h>

#include <syslog.h>

#include <cerrno>

#include <zlib.h>

// GLOBALS

extern OptionContainer o;

// regexp for decoding %xx in URLs

extern RegExp urldecode_re;

// IMPLEMENTATION

// set timeout for socket operations

void HTTPHeader::setTimeout(int t)

{

        timeout = t;

}

// reset header object for future use

void HTTPHeader::reset()

{

        if (dirty) {

                header.clear();

                //postdata.reset();

                postdata[0] = '\0';

                postdatalen = 0;

                postdatachopped = false;

                ispostupload = false;

                waspersistent = false;

                ispersistent = false;

                cachedurl = "";

                phost = NULL;

                pport = NULL;

                pcontentlength = NULL;

                pcontenttype = NULL;

                pproxyauthorization = NULL;

                pcontentdisposition = NULL;

                puseragent = NULL;

                pxforwardedfor = NULL;

                pcontentencoding = NULL;

                pproxyconnection = NULL;

                dirty = false;

        }

}

// *

// *

// * header value and type checks

// *

// *

// grab request type (GET, HEAD etc.)

String HTTPHeader::requestType()

{

        return header.front().before(" ");

}

// grab return code

int HTTPHeader::returnCode()

{

        return header.front().after(" ").before(" ").toInteger();

}

// grab content length

off_t HTTPHeader::contentLength()

{

        // code 304 - not modified - no content

        String temp(header.front().after(" "));

        if (temp.startsWith("304"))

                return 0;

        if (pcontentlength != NULL) {

                temp = pcontentlength->after(" ");

                return temp.toOffset();

        }

        // no content-length header - we don't know

        return -1;

}

// grab the auth type

String HTTPHeader::getAuthType()

{

        if (pproxyauthorization != NULL) {

                return pproxyauthorization->after(" ").before(" ");

        }

        return "";

}

// check the request's return code to see if it's an auth required message

bool HTTPHeader::authRequired()

{

        String temp(header.front().after(" "));

        if (temp.startsWith("407")) {

                return true;

        }

        return false;

}

// grab content disposition

String HTTPHeader::disposition()

{

        if (pcontentdisposition != NULL) {

                String filename(pcontentdisposition->after("filename").after("="));

                if (filename.contains(";"))

                        filename = filename.before(";");

                filename.removeWhiteSpace();  // incase of trailing space

                if (filename.contains("\"")) {

                        return filename.after("\"").before("\"");

                }

                return filename;

                // example format:

                // Content-Disposition: attachment; filename="filename.ext"

                // Content-Disposition: attachment; filename=filename.ext

                // Content-Disposition: filename="filename.ext"

                // 3rd format encountered from download script on realVNC's

                // website. notice it does not contain any semicolons! PRA 4-11-2005

        }

        return "";  // it finds the header proposed filename

}

// grab the user agent

String HTTPHeader::userAgent()

{

        if (puseragent != NULL) {

                // chop off '/r'

                String result(puseragent->after(" "));

                result.resize(result.length() - 1);

                return result;

        }

        return "";

}

// grab the content type header

String HTTPHeader::getContentType()

{

        if (pcontenttype != NULL) {

                String mimetype(pcontenttype->after(" "));

                if (mimetype.length() < 1)

                        return "-";

                unsigned char c;

                size_t j = 0;

                while (j < mimetype.length()) {

                        c = mimetype[j];

                        if (c == ' ' || c == ';' || c < 32) {        // remove the

                                mimetype = mimetype.subString(0, j);

                                // extra info not needed

                                j = 0;

                        }

                        ++j;

                }

                mimetype.toLower();

                return mimetype;

        }

        return "-";

}

// does the given content type string match our headers?

bool HTTPHeader::isContentType(const String& t)

{

        return getContentType().startsWith(t);

}

// grab contents of X-Forwarded-For header

// Modification based on a submitted patch by

// Jimmy Myrick (jmyrick@tiger1.tiger.org)

std::string HTTPHeader::getXForwardedForIP()

{

        if (pxforwardedfor != NULL) {

                String line(pxforwardedfor->after(": "));

                line.chop();

                return std::string(line.toCharArray());

        }

        return "";

}

// check the return code to see if it's a redirection request

bool HTTPHeader::isRedirection()

{

        // The 1st line of the header for a redirection is thus:

        // HTTP/1.(0|1) 3xx

        if (header.size() < 1) {

                return false;

        }                        // sometimes get called b 4 read

        String answer(header.front().after(" ").before(" "));

        if (answer[0] == '3' && answer.length() == 3) {

                return true;

        }

        return false;

}

// grab the contents of Proxy-Authorization header

// returns base64-decoding of the chunk of data after the auth type string

std::string HTTPHeader::getAuthData()

{

        if (pproxyauthorization != NULL) {

                String line(pproxyauthorization->after(" ").after(" "));

                return decodeb64(line);  // it's base64 MIME encoded

        }

        return "";

}

// grab raw contents of Proxy-Authorization header without decoding

std::string HTTPHeader::getRawAuthData()

{

        if (pproxyauthorization != NULL) {

                return pproxyauthorization->after(" ").after(" ");

        }

        return "";

}

// do we have a non-identity content encoding? this means body is compressed

bool HTTPHeader::isCompressed()

{

        if (pcontentencoding != NULL) {

                if (pcontentencoding->indexOf("identity") != -1) {

                        // http1.1 says this

                        // should not be here, but not must not

                        return false;

                }

#ifdef DGDEBUG

                std::cout << "is compressed" << std::endl;

#endif

                return true;  // i.e. encoded with something other than clear

        }

        return false;

}

// grab content encoding header

String HTTPHeader::contentEncoding()

{

        if (pcontentencoding != NULL) {

                String ce(pcontentencoding->after(": "));

                ce.toLower();

                return ce;

        }

        return "";  // we need a default don't we?

}

// *

// *

// * header modifications

// *

// *

// squid adds this so if more support it it may be useful one day

void HTTPHeader::addXForwardedFor(const std::string &clientip)

{

        std::string line("X-Forwarded-For: " + clientip + "\r");

        header.push_back(String(line.c_str()));

}

// set content length header to report given lenth

void HTTPHeader::setContentLength(int newlen)

{

        if (pcontentlength != NULL) {

                (*pcontentlength) = "Content-Length: " + String(newlen) + "\r";

        }

}

// set the proxy-connection header to allow persistence (or not)

void HTTPHeader::makePersistent(bool persist)

{

        if (persist) {

                // Only make persistent if it originally was, but now isn't.

                // The intention isn't to change browser behaviour, just to

                // un-do any connection downgrading which DG may have performed

                // earlier.

                if (waspersistent && !ispersistent) {

                        if (pproxyconnection != NULL) {

                                (*pproxyconnection) = pproxyconnection->before(":") + ": Keep-Alive\r";

                        } else {

                                header.push_back(String("Proxy-Connection: Keep-Alive\r"));

                                pproxyconnection = &(header.back());

                        }

                        ispersistent = true;

                }

        } else {

                // Only downgrade to non-persistent if it isn't currently persistent.

                if (ispersistent) {

                        if (pproxyconnection != NULL) {

                                (*pproxyconnection) = pproxyconnection->before(":") + ": Close\r";

                        } else {

                                header.push_back(String("Proxy-Connection: Close\r"));

                                pproxyconnection = &(header.back());

                        }

                        ispersistent = false;

                }

        }

}

// return a modified accept-encoding header, based on the one supplied,

// but with "identity" added and only supported encodings allowed.

String HTTPHeader::modifyEncodings(String e)

{

        // There are 4 types of encoding: gzip, deflate, compress and identity

        // deflate is in zlib format

        // compress is in unix compress format

        // identity is uncompressed and supported by all browsers (obviously)

        // we do not support compress

        e.toLower();

        String o("Accept-Encoding: identity");

#if ZLIB_VERNUM < 0x1210

#warning 'Accept-Encoding: gzip' is disabled

#else

        if (e.contains("gzip")) {

                o += ",gzip";

        }

#endif

        if (e.contains("deflate")) {

                o += ",deflate";

        }

        return o;

}

// set content length to report the given length, and strip content encoding

void HTTPHeader::removeEncoding(int newlen)

{

        if (pcontentlength != NULL) {

                (*pcontentlength) = "Content-Length: " + String(newlen) + "\r";

        }

        // this may all be overkill. since we strip everything out of the outgoing

        // accept-encoding header that we don't support, we won't be getting anything

        // back again that we don't support, in theory. leave new code commented

        // unless it proves to be necessary further down the line. PRA 20-10-2005

        if (pcontentencoding != NULL) {

/*#ifdef DGDEBUG

                std::cout << std::endl << "Stripping Content-Encoding header" <<std::endl;

                std::cout << "Old: " << header[i] <<std::endl;

#endif

                // only strip supported compression types

                String temp(header[i].after(":"));

                temp.removeWhiteSpace();

                String newheader;

                // iterate over comma-separated list of encodings

                while (temp.length() != 0) {

                        if (!(temp.startsWith("gzip") || temp.startsWith("deflate"))) {

                                // add other, unstripped encoding types back into the header

                                if (newheader.length() != 0)

                                        newheader += ", ";

                                newheader += (temp.before(",").length() != 0 ? temp.before(",") : temp);

                        }

                        temp = temp.after(",");

                        temp.removeWhiteSpace();

                }

                if (newheader.length() == 0)*/

                        (*pcontentencoding) = "X-DansGuardian-Removed: Content-Encoding\r";

/*                        else

                        header[i] = "Content-Encoding: "+newheader;

#ifdef DGDEBUG

                std::cout << "New: " << header[i] << std::endl << std::endl;

#endif*/

        }

}

// modifies the URL in all relevant header lines after a regexp search and replace

// setURL Code originally from from Ton Gorter 2004

void HTTPHeader::setURL(String &url) {

        //Modif FTRIF 

        //String hostname;

        String hostname,credentials;

        //Fin Modif FTRIF

        bool https = (url.before("://") == "https");

        int port = (https ? 443 : 80);

        if (!url.after("://").contains("/")) {

                url += "/";

        }

        hostname = url.after("://").before("/");

        if (hostname.contains("@")) { // Contains a username:password combo

                hostname = hostname.after("@");

        }

        if (hostname.contains(":")) {

                port = hostname.after(":").toInteger();

                if (port == 0 || port > 65535) {

                        port = (https ? 443 : 80);

                }

                hostname = hostname.before(":");  // chop off the port bit

        }

        //Ajout FTRIF

        //Restore stripped credentials

        credentials="";

        if (header.front().after("://").before(hostname.toCharArray()).contains("@"))

        { // Contains a username:password combo

          credentials = header.front().after("://").before(hostname.toCharArray());

        }

        //Fin ajout FTRIF

#ifdef DGDEBUG

        std::cout << "setURL: header.front() changed from: " << header.front() << std::endl;

#endif

        if (!https)

                //Modif FTRIF

                //header.front() = header.front().before(" ") + " " + url + " " + header.front().after(" ").after(" ");

                header.front() = header.front().before(" ") + " " + url.before("://") + "://" + credentials + url.after("://") + " " + header.front().after(" ").after(" ");

                //Fin Modif FTRIF

        else

                // Should take form of "CONNECT example.com:443 HTTP/1.0" for SSL

                header.front() = header.front().before(" ") + " " + hostname + ":" + String(port) + " " + header.front().after(" ").after(" ");

#ifdef DGDEBUG

        std::cout << " to: " << header.front() << std::endl;

#endif

        if (phost != NULL) {

#ifdef DGDEBUG

                std::cout << "setURL: header[] line changed from: " << (*phost) << std::endl;

#endif

                (*phost) = String("Host: ") + hostname;

                if (port != (https ? 443 : 80))

                {

                        (*phost) += ":";

                        (*phost) += String(port);

                }

                (*phost) += "\r";

#ifdef DGDEBUG

                std::cout << " to " << (*phost) << std::endl;

#endif

        }

        if (pport != NULL) {

#ifdef DGDEBUG

                std::cout << "setURL: header[] line changed from: " << (*pport) << std::endl;

#endif

                (*pport) = String("Port: ") + String(port) + "\r";

#ifdef DGDEBUG

                std::cout << " to " << (*pport) << std::endl;

#endif

        }

        // Don't just cache the URL we're sent - url() performs some other

        // processing, notably stripping the port part. Caching here will

        // bypass all that.

        //cachedurl = url.toCharArray();

}

// Does a regexp search and replace.

// urlRegExp Code originally from from Ton Gorter 2004

bool HTTPHeader::regExp(String& line, std::deque<RegExp>& regexp_list, std::deque<String>& replacement_list) {

        RegExp *re;

        String replacement;

        String repstr;

        String newLine;

        bool linemodified = false;

        unsigned int i;

        unsigned int j, k;

        unsigned int s = regexp_list.size();

        unsigned int matches, submatches;

        unsigned int match;

        unsigned int srcoff;

        unsigned int nextoffset;

        unsigned int matchlen;

        unsigned int oldlinelen;

        // iterate over our list of precompiled regexes

        for (i = 0; i < s; i++) {

                newLine = "";

                re = &(regexp_list[i]);

                if (re->match(line.toCharArray())) {

                        repstr = replacement_list[i];

                        matches = re->numberOfMatches();

                        srcoff = 0;

                        for (j = 0; j < matches; j++) {

                                nextoffset = re->offset(j);

                                matchlen = re->length(j);

                                // copy next chunk of unmodified data

                                if (nextoffset > srcoff) {

                                        newLine += line.subString(srcoff, nextoffset - srcoff);

                                        srcoff = nextoffset;

                                }

                                // Count number of submatches (brackets) in replacement string

                                for (submatches = 0; j+submatches+1 < matches; submatches++)

                                        if (re->offset(j+submatches+1) + re->length(j+submatches+1) > srcoff + matchlen)

                                                break;

                                // \1 and $1 replacement

                                replacement = "";

                                for (k = 0; k < repstr.length(); k++) {

                                        // find \1..\9 and $1..$9 and fill them in with submatched strings

                                        if ((repstr[k] == '\\' || repstr[k] == '$') && repstr[k+1] >= '1' && repstr[k+1] <= '9') {

                                                match = repstr[++k] - '0';

                                                if (match <= submatches) {

                                                        replacement += re->result(j + match).c_str();

                                                }

                                        } else {

                                                // unescape \\ and \$, and add non-backreference characters to string

                                                if (repstr[k] == '\\' && (repstr[k+1] == '\\' || repstr[k+1] == '$'))

                                                        k++;

                                                replacement += repstr.subString(k, 1);

                                        }

                                }

                                // copy filled in replacement string

                                newLine += replacement;

                                srcoff += matchlen;

                                j += submatches;

                        }

                        oldlinelen = line.length();

                        if (srcoff < oldlinelen) {

                                newLine += line.subString(srcoff, oldlinelen - srcoff);

                        }

#ifdef DGDEBUG

                        std::cout << "Line modified! (" << line << " -> " << newLine << ")" << std::endl;

#endif

                        // copy newLine into line and continue with other regexes

                        line = newLine;

                        linemodified = true;

                }

        }

        return linemodified;

}

// Perform searches and replacements on URL

bool HTTPHeader::urlRegExp(int filtergroup) {

        // exit immediately if list is empty

        if (not o.fg[filtergroup]->url_regexp_list_comp.size())

                return false;

#ifdef DGDEBUG

        std::cout << "Starting URL reg exp replace" << std::endl;

#endif

        String newUrl(url());

        if (regExp(newUrl, o.fg[filtergroup]->url_regexp_list_comp, o.fg[filtergroup]->url_regexp_list_rep)) {

                setURL(newUrl);

                return true;

        }

        return false;

}

// Perform searches and replacements on header lines

bool HTTPHeader::headerRegExp(int filtergroup) {

        // exit immediately if list is empty

        if (not o.fg[filtergroup]->header_regexp_list_comp.size())

                return false;

        bool result = false;

        for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {

#ifdef DGDEBUG

                std::cout << "Starting header reg exp replace: " << *i << std::endl;

#endif

                bool chop = false;

                if (i->endsWith("\r"))

                {

                        i->chop();

                        chop = true;

                }

                result |= regExp(*i, o.fg[filtergroup]->header_regexp_list_comp, o.fg[filtergroup]->header_regexp_list_rep);

                if (chop)

                        i->append("\r");

        }

        return result;

}

// *

// *

// * detailed header checks & fixes

// *

// *

// is a URL malformed?

bool HTTPHeader::malformedURL(const String& url)

{

        String host(url.after("://"));

        if (host.contains("/"))

                host = host.before("/");

        if (host.length() < 2) {

#ifdef DGDEBUG

                std::cout << "host len too small" << std::endl;

#endif

                return true;

        }

        if (host.contains(":"))

                host = host.before(":");

        if (host.contains("..") || host.endsWith(".")) {

#ifdef DGDEBUG

                std::cout << "double dots in domain name" << std::endl;

#endif

                return true;

        }

        int i, len;

        unsigned char c;

        len = host.length();

        bool containsletter = false;

        for (i = 0; i < len; i++) {

                c = (unsigned char) host[i];

                // If it contains something other than numbers, dots, or [a-fx] (hex encoded IPs),

                // IP obfuscation can be ruled out.

                if (!containsletter &&

                                (((c < '0') || (c > '9'))

                                 && (c != '.') && (c != 'x') && (c != 'X')

                                 && ((c < 'a') || (c > 'f'))

                                 && ((c < 'A') || (c > 'F'))))

                        containsletter = true;

                if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z')

                        && !(c >= '0' && c <= '9') && c != '.' && c != '-' && c != '_') {

#ifdef DGDEBUG

                        std::cout << "bad char in hostname" << std::endl;

#endif

                        return true;

                        // only allowed letters, digits, hiphen, dots

                }

        }

        // no IP obfuscation going on

        if (containsletter)

                return false;

#ifdef DGDEBUG

        else

                std::cout << "Checking for IP obfuscation in " << host << std::endl;

#endif

        // Check no IP obfuscation is going on

        // This includes IPs encoded as a single decimal number,

        // fully or partly hex encoded, and octal encoded

        bool first = true;

        bool obfuscation = false;

        if (host.endsWith("."))

                host.chop();

        do {

                if (!first)

                        host = host.after(".");

                first = false;

                String hostpart(host);

                if (host.contains("."))

                        hostpart = hostpart.before(".");

                // If any part of the host starts with a letter, any letter,

                // then we must have a hostname rather than an IP (obscured

                // or otherwise).  TLDs never start with a number.

                if ((hostpart[0] >= 'a' && hostpart[0] <= 'z') || (hostpart[0] >= 'A' && hostpart[0] <= 'Z'))

                        return false;

                // If any part of the host begins with 0, it may be hex or octal

                if ((hostpart[0] == '0') && (hostpart.length() > 1))

                {

                        obfuscation = true;

                        continue;

                }

                // Also check range, for decimal obfuscation.

                int part = hostpart.toInteger();

                if ((part < 0) || (part > 255))

                        obfuscation = true;

        } while (host.contains("."));

        // If we have any obfuscated parts, and haven't proven it's a hostname, it's invalid.

        return obfuscation;

}

// is this a POST request encapsulating a file upload?

bool HTTPHeader::isPostUpload(Socket &peersock)

{

        if (header.front().toCharArray()[0] != 'P') {

                return false;

        }

        /*bool answer = false;

        int postlen = postdata.buffer_length;

        int i;

        if (postlen < 14) {        // min length for there to be a match

                return false;

        }

        char *postdatablock = new char[postlen + 64];  // extra 64 for search

        try {

                postdata.copyToMemory(postdatablock);

                for (i = 0; i < postlen; i++) {        // make lowercase char by char

                        if (isupper(postdatablock[i])) {

                                postdatablock[i] = tolower(postdatablock[i]);

                        }

                }

                RegExp mysearch;

                std::string dis("content-type: ");  // signifies file upload

                char *p = new char[32];

                try {

                        for (i = 0; i < (signed) dis.length(); i++) {

                                p[i] = dis[i];  // copy it to the block of memory

                        }

                        char *pend = p + dis.length();  // pointer for search

                        char *postdatablockend = postdatablock + postlen;

                        // search the post data for the content type header

                        char *res = mysearch.search(postdatablock, postdatablockend, p, pend);

                        // if we searched all the way to the end without finding it,

                        // there is no post upload going on; otherwise, there is

                        if (res != postdatablockend) {

                                answer = true;

                        }

                }

                catch(exception & e) {

                };

                delete[]p;

        }

        catch(exception & e) {

        };

        delete[]postdatablock;

        return answer;*/

        off_t cl = contentLength();

        if (((cl > 0) && (cl < 14)) || (getContentType() == "application/x-www-form-urlencoded")) {

#ifdef DGDEBUG

                std::cout << "Based on content length/type, is not POST upload!" << std::endl;

#endif

                ispostupload = false;

                return false;

        }

        if (getContentType().length() > 0) {

#ifdef DGDEBUG

                std::cout << "Based on content length/type, is POST upload!" << std::endl;

#endif

                ispostupload = true;

                return true;

        }

#ifdef DGDEBUG

        std::cout << "Reading a line of POST data to determine content type: ";

#endif

        postdatalen = peersock.getLine(postdata, 14, 60, &postdatachopped);

#ifdef DGDEBUG

        std::cout << postdata << std::endl;

#endif

        if (postdatalen != 14) {

#ifdef DGDEBUG

                std::cout << "Is not POST upload!" << std::endl;

#endif

                ispostupload = false;

                return false;

        }

        String conttype(postdata);

        if (conttype.startsWithLower("content-type: ")) {

#ifdef DGDEBUG

                std::cout << "Is POST upload!" << std::endl;

#endif

                ispostupload = true;

                return true;

        } else {

#ifdef DGDEBUG

                std::cout << "Is not POST upload!" << std::endl;

#endif

                ispostupload = false;

                return false;

        }

}

// fix bugs in certain web servers that don't obey standards.

// actually, it's us that don't obey standards - HTTP RFC says header names

// are case-insensitive. - Anonymous SF Poster, 2006-02-23

void HTTPHeader::checkheader(bool allowpersistent)

{

        // are these headers outgoing, or incoming?

        bool outgoing = true;

        if (header.front().startsWith("HT"))

                outgoing = false;

        bool first = true;

        for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {        // check each line in the headers

                // HTTP 1.1 is persistent by default

                if (first) {

                        if (i->after("HTTP/").startsWith("1.1")) {

#ifdef DGDEBUG

                                std::cout << "CheckHeader: HTTP/1.1, so assuming persistency" << std::endl;

#endif

                                waspersistent = true;

                                ispersistent = true;

                        }

                        // Do not allow persistent connections on CONNECT requests - the browser thinks it has a tunnel

                        // directly to the external server, not a connection to the proxy, so it won't be re-used in the

                        // manner expected by DG and will result in waiting for time-outs.  Bug identified by Jason Deasi.

                        if ((*i)[0] == 'C') {

#ifdef DGDEBUG

                                std::cout << "CheckHeader: CONNECT request; disallowing persistency" << std::endl;

#endif

                                allowpersistent = false;

                        }

                        first = false;

                        // force HTTP/1.0 - we don't support chunked transfer encoding, possibly amongst other things

                        if (outgoing)

                                (*i) = i->before(" HTTP/") + " HTTP/1.0\r";

                }

                // index headers - try to perform the checks in the order the average browser sends the headers.

                // also only do the necessary checks for the header type (sent/received).

                else if (outgoing && (phost == NULL) && i->startsWithLower("host:")) {

                        phost = &(*i);

                }

                // don't allow through multiple host headers

                else if (outgoing && (phost != NULL) && i->startsWithLower("host:")) {

                        i->assign("X-DG-IgnoreMe: removed multiple host headers\r");

                }

                else if (outgoing && (puseragent == NULL) && i->startsWithLower("user-agent:")) {

                        puseragent = &(*i);

                }

                else if (outgoing && i->startsWithLower("accept-encoding:")) {

                        (*i) = "Accept-Encoding:" + i->after(":");

                        (*i) = modifyEncodings(*i) + "\r";

                }

                else if ((pcontenttype == NULL) && i->startsWithLower("content-type:")) {

                        pcontenttype = &(*i);

                }

                else if ((pcontentlength == NULL) && i->startsWithLower("content-length:")) {

                        pcontentlength = &(*i);

                }

                // is this ever sent outgoing?

                else if ((pcontentdisposition == NULL) && i->startsWithLower("content-disposition:")) {

                        pcontentdisposition = &(*i);

                }

                else if ((!outgoing) && (pcontentencoding == NULL) && i->startsWithLower("content-encoding:")) {

                        pcontentencoding = &(*i);

                }

                else if ((pproxyauthorization == NULL) && i->startsWithLower("proxy-authorization:")) {

                        pproxyauthorization = &(*i);

                }

                else if ((pproxyconnection == NULL) && (i->startsWithLower("proxy-connection:") || i->startsWithLower("connection:"))) {

#ifdef DGDEBUG

                        std::cout << "CheckHeader: Found Proxy-Connection" << std::endl;

#endif

                        if (i->contains("live")) {

#ifdef DGDEBUG

                                std::cout << "CheckHeader: P-C says keep-alive" << std::endl;

#endif

                                waspersistent = true;

                                if (!allowpersistent) {

#ifdef DGDEBUG

                                        std::cout << "CheckHeader: ... but we aren't allowed to" << std::endl;

#endif

                                        ispersistent = false;

                                        (*i) = i->before(":") + ": Close\r";

                                } else {

                                        ispersistent = true;

                                }

                        } else {

#ifdef DGDEBUG

                                std::cout << "CheckHeader: P-C says close" << std::endl;

#endif

                                ispersistent = false;

                                waspersistent = false;

                        }

                        pproxyconnection = &(*i);

                }

                else if (outgoing && (pxforwardedfor == NULL) && i->startsWithLower("x-forwarded-for:")) {

                        pxforwardedfor = &(*i);

                }

                // this one's non-standard, so check for it last

                else if (outgoing && (pport = NULL) && i->startsWithLower("port:")) {

                        pport = &(*i);

                }

#ifdef DGDEBUG

                std::cout << (*i) << std::endl;

#endif

        }

#ifdef DGDEBUG

        std::cout << "CheckHeader flags: AP=" << allowpersistent << " IP=" << ispersistent << " PPC=" << !(pproxyconnection == NULL) << std::endl;

#endif

        // if a request was HTTP 1.1 and there was no proxy-connection header, we may need to add one

        if ((!allowpersistent) && ispersistent) {

                // we should only be in this state if pproxyconnection == NULL (otherwise ispersistent will have been falsified earlier)

#ifdef DGDEBUG

                std::cout << "CheckHeader: Adding our own Proxy-Connection: Close" << std::endl;

#endif

                header.push_back("Proxy-Connection: Close\r");

                pproxyconnection = &(header.back());

                ispersistent = false;

        } else if (allowpersistent && ispersistent && (pproxyconnection == NULL)) {

#ifdef DGDEBUG

                std::cout << "CheckHeader: Adding our own Proxy-Connection: Keep-Alive" << std::endl;

#endif

                // we should only be in this state if HTTP 1.1, persistency allowed, but persistency not explicitly asked for

                header.push_back("Proxy-Connection: Keep-Alive\r");

                pproxyconnection = &(header.back());

        }

        // Normalise request headers (fix host, port, first line of header, etc. to all be consistent)

        if (outgoing)

        {

                String newurl(url(true));

                setURL(newurl);

        }

}

// A request may be in the form:

//  GET http://foo.bar:80/ HTML/1.0 (if :80 is omitted 80 is assumed)

// or:

//  GET / HTML/1.0

//  Host: foo.bar (optional header in HTTP/1.0, but like HTTP/1.1, we require it!)

//  Port: 80 (not a standard header; do any clients send it?)

// or:

//  CONNECT foo.bar:443  HTTP/1.1

// So we need to handle all 3

String HTTPHeader::url(bool withport)

{

        // Version of URL *with* port is not cached,

        // as vast majority of our code doesn't like

        // port numbers in URLs.

        if (cachedurl.length() > 0 && !withport)

                return cachedurl;

        port = 80;

        bool https = false;

        String hostname;

        String answer(header.front().after(" "));

        answer.removeMultiChar(' ');

        if (answer.after(" ").startsWith("HTTP/")) {

                answer = answer.before(" HTTP/");

        } else {

                answer = answer.before(" http/");  // just in case!

        }

        if (requestType() == "CONNECT") {

                https = true;

                port = 443;

                if (!answer.startsWith("https://")) {

                        answer = "https://" + answer;

                }

        }

        if (pport != NULL) {

                port = pport->after(" ").toInteger();

                if (port == 0 || port > 65535)

                        port = (https ? 443 : 80);

        }

        if (answer.length()) {

                if (answer[0] == '/') {        // must be the latter above

                        if (phost != NULL) {

                                hostname = phost->after(" ");

                                hostname.removeWhiteSpace();

                                if (hostname.contains(":"))

                                {

                                        port = hostname.after(":").toInteger();

                                        if (port == 0 || port > 65535) {

                                                port = (https ? 443 : 80);

                                        }

                                        hostname = hostname.before(":");

                                }

                                while (hostname.endsWith("."))

                                        hostname.chop();

                                if (withport && (port != (https ? 443 : 80)))

                                        hostname += ":" + String(port);

                                hostname = "http://" + hostname;

                                answer = hostname + answer;

                        }

                        // Squid doesn't like requests in this format. Work around the fact.

                        header.front() = requestType() + " " + answer + " HTTP/" + header.front().after(" HTTP/");

                } else {        // must be in the form GET http://foo.bar:80/ HTML/1.0

                        if (!answer.after("://").contains("/")) {

                                answer += "/";  // needed later on so correct host is extracted

                        }

                        String protocol(answer.before("://"));

                        hostname = answer.after("://");

                        String url(hostname.after("/"));

                        url.removeWhiteSpace();  // remove rubbish like ^M and blanks

                        if (url.length() > 0) {

                                url = "/" + url;

                        }

                        hostname = hostname.before("/");  // extra / was added 4 here

                        if (hostname.contains("@")) {        // Contains a username:password combo

                                hostname = hostname.after("@");

                        }

                        if (hostname.contains(":")) {

                                port = hostname.after(":").toInteger();

                                if (port == 0 || port > 65535) {

                                        port = (https ? 443 : 80);

                                }

                                hostname = hostname.before(":");  // chop off the port bit

                        }

                        while (hostname.endsWith("."))

                                hostname.chop();

                        if (withport && (port != (https ? 443 : 80)))

                                hostname += ":" + String(port);

                        answer = protocol + "://" + hostname + url;

                }

        }

        if (answer.endsWith("//")) {

                answer.chop();

        }

#ifdef DGDEBUG

        std::cout << "from header url:" << answer << std::endl;

#endif

        // Don't include port numbers in the URL in the cached version.

        // Most of the code only copes with URLs *without* port specifiers.

        if (!withport)

                cachedurl = answer.toCharArray();

        return answer;

}

// *

// *

// * Bypass URL/Cookie funcs

// *

// *

// chop the GBYPASS or GIBYPASS variable out of a bypass URL

// This function ASSUMES that you really know what you are doing

// Do NOT run this function unless you know that the URL contains a valid bypass code

// Ernest W Lessenger

void HTTPHeader::chopBypass(String url, bool infectionbypass)

{

        if (url.contains(infectionbypass ? "GIBYPASS=" : "GBYPASS=")) {

                if (url.contains(infectionbypass ? "?GIBYPASS=" : "?GBYPASS=")) {

                        String bypass(url.after(infectionbypass ? "?GIBYPASS=" : "?GBYPASS="));

                        header.front() = header.front().before(infectionbypass ? "?GIBYPASS=" : "?GBYPASS=") + header.front().after(bypass.toCharArray());

                } else {

                        String bypass(url.after(infectionbypass ? "&GIBYPASS=" : "&GBYPASS="));

                        header.front() = header.front().before(infectionbypass ? "&GIBYPASS=" : "&GBYPASS=") + header.front().after(bypass.toCharArray());

                }

        }

        cachedurl = "";

}

// same for scan bypass

void HTTPHeader::chopScanBypass(String url)

{

        if (url.contains("GSBYPASS=")) {

                if (url.contains("?GSBYPASS=")) {

                        String bypass(url.after("?GSBYPASS="));

                        header.front() = header.front().before("?GSBYPASS=") + header.front().after(bypass.toCharArray());

                } else {

                        String bypass(url.after("&GSBYPASS="));

                        header.front() = header.front().before("&GSBYPASS=") + header.front().after(bypass.toCharArray());

                }

        }

        cachedurl = "";

}

// I'm not proud of this... --Ernest

String HTTPHeader::getCookie(const char *cookie)

{

        String line;

        // TODO - do away with loop here somehow, or otherwise speed it up?

        for (std::deque<String>::iterator i = header.begin(); i != header.end(); i++) {

                if (i->startsWithLower("cookie:")) {

                        line = i->after(": ");

                        if (line.contains(cookie)) {        // We know we have the cookie

                                line = line.after(cookie);

                                line.lop();  // Get rid of the '='

                                if (line.contains(";")) {

                                        line = line.before(";");

                                }

                        }

                        // break;  // Technically there should be only one Cookie: header, but...

                }

        }

        line.removeWhiteSpace();

#ifdef DGDEBUG

        std::cout << "Found GBYPASS cookie:" << line << std::endl;

#endif

        return line;

}

// add cookie with given name & value to outgoing headers

void HTTPHeader::setCookie(const char *cookie, const char *domain, const char *value)

{

        String line("Set-Cookie: ");

        line += cookie;

        line += "=";

        line += value;

        line += "; path=/; domain=.";

        line += domain;

        line += "\r";

        header.push_back(line);

#ifdef DGDEBUG

        std::cout << "Setting cookie:" << line << std::endl;

#endif

        // no expiry specified so ends with the browser session

}

// is this a temporary filter bypass cookie?

bool HTTPHeader::isBypassCookie(String url, const char *magic, const char *clientip)

{

        String cookie(getCookie("GBYPASS"));

        if (!cookie.length()) {

#ifdef DGDEBUG

                std::cout << "No bypass cookie" << std::endl;

#endif

                return false;

        }

        String cookiehash(cookie.subString(0, 32));

        String cookietime(cookie.after(cookiehash.toCharArray()));

        String mymagic(magic);

        mymagic += clientip;

        mymagic += cookietime;

        bool matched = false;

        while(url.contains(".")) {

                String hashed(url.md5(mymagic.toCharArray()));

                if (hashed == cookiehash) {

                        matched = true;

                        break;

                }

                url = url.after(".");

        }

        if (not matched) {

#ifdef DGDEBUG

                std::cout << "Cookie GBYPASS not match" << std::endl;

#endif

                return false;

        }

        time_t timen = time(NULL);

        time_t timeu = cookietime.toLong();

        if (timeu < timen) {

#ifdef DGDEBUG

                std::cout << "Cookie GBYPASS expired: " << timeu << " " << timen << std::endl;

#endif

                return false;

        }

        return true;

}

// is this a temporary filter bypass URL?

int HTTPHeader::isBypassURL(String * url, const char *magic, const char *clientip, bool *isvirusbypass)

{

        if ((*url).length() <= 45)

                return false;  // Too short, can't be a bypass

        // check to see if this is a bypass URL, and which type it is

        bool filterbypass = false;

        bool virusbypass = false;

        if ((isvirusbypass == NULL) && ((*url).contains("GBYPASS="))) {

                filterbypass = true;

        } else if ((isvirusbypass != NULL) && (*url).contains("GIBYPASS=")) {

                virusbypass = true;

        }

        if (!(filterbypass || virusbypass))

                return 0;

#ifdef DGDEBUG

        std::cout << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " found checking..." << std::endl;

#endif

        String url_left((*url).before(filterbypass ? "GBYPASS=" : "GIBYPASS="));

        url_left.chop();  // remove the ? or &

        String url_right((*url).after(filterbypass ? "GBYPASS=" : "GIBYPASS="));

        String url_hash(url_right.subString(0, 32));

        String url_time(url_right.after(url_hash.toCharArray()));

#ifdef DGDEBUG

        std::cout << "URL: " << url_left << ", HASH: " << url_hash << ", TIME: " << url_time << std::endl;

#endif

        String mymagic(magic);

        mymagic += clientip;

        mymagic += url_time;

        String hashed(url_left.md5(mymagic.toCharArray()));

        if (hashed != url_hash) {

#ifdef DGDEBUG

                std::cout << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " hash mismatch" << std::endl;