/* http_fetcher.c - HTTP handling functions

	HTTP Fetcher 
	Copyright (C) 2001, 2003, 2004 Lyle Hanson (lhanson@users.sourceforge.net)

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Library General Public
	License as published by the Free Software Foundation; either
	version 2 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
	Library General Public License for more details.

	See LICENSE file for details

 */

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <ctype.h>
#include <errno.h>
#include <netdb.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include "http_fetcher.h"

/* Globals */
int timeout = DEFAULT_READ_TIMEOUT;
char *userAgent = NULL;
char *referer = NULL;
int hideUserAgent = 0;
int hideReferer = 1;
static int followRedirects = DEFAULT_REDIRECTS;	/* # of redirects to follow */
extern const char *http_errlist[];	/* Array of HTTP Fetcher error messages */
extern char convertedError[128];	/* Buffer to used when errors contain %d */
static int errorSource = 0;
static int http_errno = 0;
static int errorInt = 0;			/* When the error message has a %d in it,
									 *	this variable is inserted */


	/* 
	 * Actually downloads the page, registering a hit (donation)
	 *	If the fileBuf passed in is NULL, the url is downloaded and then
	 *	freed; otherwise the necessary space is allocated for fileBuf.
	 *	Returns size of download on success, -1 on error is set, 
	 */
int http_fetch(const char *url_tmp, char **fileBuf)
	{
	//fd_set rfds;
	//struct timeval tv;
	char headerBuf[HEADER_BUF_SIZE] = {0};
	char *tmp, *url, *requestBuf = NULL, *host, *charIndex; /* , *pageBuf */
	int sock, bytesRead = 0, bufsize = REQUEST_BUF_SIZE; /*, contentLength = -1 */
	int ret = -1, tempSize;
	/* int i,       For redirects
		selectRet,
		found = 0,
		redirectsFollowed = 0;*/


	if(url_tmp == NULL)
		{
		errorSource = FETCHER_ERROR;
		http_errno = HF_NULLURL;
		return -1;
		}

	/* Copy the url passed in into a buffer we can work with, change, etc. */
	url = malloc(strlen(url_tmp)+1);
	if(url == NULL)
		{
		errorSource = ERRNO;
		return -1;
		}
	strncpy(url, url_tmp, strlen(url_tmp) + 1);
	
	/* This loop allows us to follow redirects if need be.  An afterthought,
	 * added to provide this basic functionality.  Will hopefully be designed
	 * better in 2.x.x ;) */
/*	while(!found &&
		  (followRedirects < 0 || redirectsFollowed < followRedirects) )
  */  if ( 1 ) 
		{
		/* Seek to the file path portion of the url */
		charIndex = strstr(url, "://");
		if(charIndex != NULL)
			{
			/* url contains a protocol field */
			charIndex += strlen("://");
			host = charIndex;
			charIndex = strchr(charIndex, '/');
			}
		else
			{
			host = (char *)url;
			charIndex = strchr(url, '/');
			}

		/* Compose a request string */
		requestBuf = malloc(bufsize);
		if(requestBuf == NULL)
			{
			free(url);
			errorSource = ERRNO;
			return -1;
			}
		requestBuf[0] = 0;

		if(charIndex == NULL)
			{
			/* The url has no '/' in it, assume the user is making a root-level
			 *	request */ 
			tempSize = strlen("GET /") + strlen(HTTP_VERSION) + 2;
			if(_checkBufSize(&requestBuf, &bufsize, tempSize) ||
				snprintf(requestBuf, bufsize, "GET / %s\r\n", HTTP_VERSION) < 0)
				{
				free(url);
				free(requestBuf);
				errorSource = ERRNO;
				return -1;
				}
			}
		else
			{
			tempSize = strlen("GET ") + strlen(charIndex) +
  	          strlen(HTTP_VERSION) + 4;
		 	/* + 4 is for ' ', '\r', '\n', and NULL */
                                    
			if(_checkBufSize(&requestBuf, &bufsize, tempSize) ||
					snprintf(requestBuf, bufsize, "GET %s %s\r\n",
					charIndex, HTTP_VERSION) < 0)
				{
				free(url);
				free(requestBuf);
				errorSource = ERRNO;
				return -1;
				}
			}

		/* Null out the end of the hostname if need be */
		if(charIndex != NULL)
			*charIndex = 0;

		/* Use Host: even though 1.0 doesn't specify it.  Some servers
		 *	won't play nice if we don't send Host, and it shouldn't
		 *	hurt anything */
		ret = bufsize - strlen(requestBuf); /* Space left in buffer */
		tempSize = (int)strlen("Host: ") + (int)strlen(host) + 3;
        /* +3 for "\r\n\0" */
		if(_checkBufSize(&requestBuf, &bufsize, tempSize + 128))
			{
			free(url);
			free(requestBuf);
			errorSource = ERRNO;
			return -1;
			}
		strcat(requestBuf, "Host: ");
		strcat(requestBuf, host);
		strcat(requestBuf, "\r\n");

		if(!hideReferer && referer != NULL)	/* NO default referer */
			{
			tempSize = (int)strlen("Referer: ") + (int)strlen(referer) + 3;
   	        /* + 3 is for '\r', '\n', and NULL */
			if(_checkBufSize(&requestBuf, &bufsize, tempSize))
				{
				free(url);
				free(requestBuf);
				errorSource = ERRNO;
				return -1;
				}
			strcat(requestBuf, "Referer: ");
			strcat(requestBuf, referer);
			strcat(requestBuf, "\r\n");
			}

		if(!hideUserAgent && userAgent == NULL)
			{
			tempSize = (int)strlen("User-Agent: ") +
				(int)strlen(DEFAULT_USER_AGENT) + (int)strlen(VERSION) + 4;
   	        /* + 4 is for '\', '\r', '\n', and NULL */
			if(_checkBufSize(&requestBuf, &bufsize, tempSize))
				{
				free(url);
				free(requestBuf);
				errorSource = ERRNO;
				return -1;
				}
			strcat(requestBuf, "User-Agent: ");
			strcat(requestBuf, DEFAULT_USER_AGENT);
			strcat(requestBuf, "/");
			strcat(requestBuf, VERSION);
			strcat(requestBuf, "\r\n");
			}
		else if(!hideUserAgent)
			{
			tempSize = (int)strlen("User-Agent: ") + (int)strlen(userAgent) + 3;
   	        /* + 3 is for '\r', '\n', and NULL */
			if(_checkBufSize(&requestBuf, &bufsize, tempSize))
				{
				free(url);
				free(requestBuf);
				errorSource = ERRNO;
				return -1;
				}
			strcat(requestBuf, "User-Agent: ");
			strcat(requestBuf, userAgent);
			strcat(requestBuf, "\r\n");
			}

		tempSize = (int)strlen("Connection: Close\r\n\r\n");
		if(_checkBufSize(&requestBuf, &bufsize, tempSize))
			{
			free(url);
			free(requestBuf);
			errorSource = ERRNO;
			return -1;
			}
		strcat(requestBuf, "Connection: Close\r\n\r\n");

		/* Now free any excess memory allocated to the buffer */
		tmp = realloc(requestBuf, strlen(requestBuf) + 1);
		if(tmp == NULL)
			{
			free(url);
			free(requestBuf);
			errorSource = ERRNO;
			return -1;
			}
		requestBuf = tmp;

		sock = makeSocket(host);		/* errorSource set within makeSocket */
		if(sock == -1) { free(url); free(requestBuf); return -1;}

		free(url);
        url = NULL;

		if(write(sock, requestBuf, strlen(requestBuf)) == -1)
			{
			close(sock);
			free(requestBuf);
			errorSource = ERRNO;
			return -1;
			}

		free(requestBuf);
        requestBuf = NULL;

		/* Grab enough of the response to get the metadata */
		ret = _http_read_header(sock, headerBuf);	/* errorSource set within */
		if(ret < 0) { close(sock); return -1; }

charIndex = strstr(headerBuf, "</body>");
bytesRead = charIndex - headerBuf + 7;
headerBuf[bytesRead] = 0;
strcpy( *fileBuf, headerBuf );
free(url);
free(requestBuf);
close(sock);
return bytesRead;
	    } 


	}



	/*
	 * Changes the User Agent.  Returns 0 on success, -1 on error. 
	 */
int http_setUserAgent(const char *newAgent)
	{
	static int freeOldAgent = 0; /* Indicates previous malloc's */
	char *tmp;

	if(newAgent == NULL)
		{
		if(freeOldAgent) free(userAgent);
		userAgent = NULL;
		hideUserAgent = 1;
		}
	else
		{
		tmp = (char *)malloc(strlen(newAgent));
		if(tmp == NULL) { errorSource = ERRNO; return -1; }
		if(freeOldAgent) free(userAgent);
		userAgent = tmp;
		strcpy(userAgent, newAgent);
		freeOldAgent = 1;
		hideUserAgent = 0;
		}

	return 0;
	}



	/*
	 * Changes the Referer.  Returns 0 on success, -1 on error
	 */
int http_setReferer(const char *newReferer)
	{
	static int freeOldReferer = 0; /* Indicated previous malloc's */
	char *tmp;

	if(newReferer == NULL)
		{
		if(freeOldReferer) free(referer);
		referer = NULL;
		hideReferer = 1;
		}
	else
		{
		tmp = (char *)malloc(strlen(newReferer));
		if(tmp == NULL) { errorSource = ERRNO; return -1; }
		if(freeOldReferer) free(referer);
		referer = tmp;
		strcpy(referer, newReferer);
		freeOldReferer = 1;
		hideReferer = 0;
		}
	
	return 0;
	}



	/*
	 * Changes the amount of time that HTTP Fetcher will wait for data
	 *	before timing out on reads
	 */
void http_setTimeout(int seconds) { timeout = seconds; }



	/*
	 * Changes the number of HTTP redirects HTTP Fetcher will automatically
	 *	follow.  If a request returns a status code of 3XX and contains
	 *	a "Location:" field, the library will transparently follow up to
	 *	the specified number of redirects.  With this implementation
	 *	(which is just a stopgap, really) the caller won't be aware of any
	 *	redirection and will assume the returned document came from the original
	 *	URL.
	 * To disable redirects, pass a 0.  To follow unlimited redirects (probably
	 *  unwise), pass a negative value.  The default is to follow 3 redirects.
	 */
void http_setRedirects(int redirects) { followRedirects = redirects; }



	/*
	 * Puts the filename portion of the url into 'filename'.
	 * Returns:
	 *	0 on success
	 *	1 when url contains no end filename (i.e., 'www.foo.com/'),
	 *		and **filename should not be assumed to be valid
	 *	-1 on error
	 */
int http_parseFilename(const char *url, char **filename)
	{
	char *ptr;

	if(url == NULL)
		{
		errorSource = FETCHER_ERROR;
		http_errno = HF_NULLURL;
		return -1;
		}

	ptr = (char *)rindex(url, '/');
	if(ptr == NULL)
		/* Root level request, apparently */
		return 1;

	ptr++;
	if(*ptr == '\0') return 1;

	*filename = (char *)malloc(strlen(ptr));
	if(*filename == NULL) { errorSource = ERRNO; return -1; }
	strcpy(*filename, ptr);

	return 0;
	}

	
	
	/* Depending on the source of error, calls either perror() or prints
	 *	an HTTP Fetcher error message to stdout */
void http_perror(const char *string)
	{
	if(errorSource == ERRNO)
		perror(string);
	else if(errorSource == H_ERRNO)
		herror(string);
	else if(errorSource == FETCHER_ERROR)
		{
		const char *stringIndex;

		if(strstr(http_errlist[http_errno], "%d") == NULL)
			{
			fputs(string, stderr);
			fputs(": ", stderr);
			fputs(http_errlist[http_errno], stderr);
			fputs("\n", stderr);
			}
		else
			{
			/* The error string has a %d in it, we need to insert errorInt */
			stringIndex = http_errlist[http_errno];
			while(*stringIndex != '%')			/* Print up to the %d */
				{
				fputc(*stringIndex, stderr);
				stringIndex++;
				}
			fprintf(stderr, "%d", errorInt);	/* Print the number */
			stringIndex += 2;					/* Skip past the %d */
			while(*stringIndex != 0)			/* Print up to the end NULL */
				{
				fputc(*stringIndex, stderr);
				stringIndex++;
				}
			fputs("\n", stderr);
			}
		}
	}



	/* 
	 * Returns a pointer to the current error description message. The
	 *	message pointed to is only good until the next call to http_strerror(),
	 *	so if you need to hold on to the message for a while you should make
	 *	a copy of it
	 */
const char *http_strerror() {

	if(errorSource == ERRNO)
		return strerror(errno);
	else if(errorSource == H_ERRNO)
#ifdef HAVE_HSTRERROR
		return hstrerror(h_errno);
#else
		return http_errlist[HF_HERROR];
#endif
	else if(errorSource == FETCHER_ERROR)
		{
		if(strstr(http_errlist[http_errno], "%d") == NULL)
			return http_errlist[http_errno];
		else
			{
			/* The error string has a %d in it, we need to insert errorInt.
			 *	convertedError[128] has been declared for that purpose */
			char *stringIndex, *originalError;
		
			originalError = (char *)http_errlist[http_errno];
			convertedError[0] = 0;		/* Start off with NULL */
			stringIndex = strstr(originalError, "%d");
			strncat(convertedError, originalError,		/* Copy up to %d */
				abs(stringIndex - originalError));
			sprintf(&convertedError[strlen(convertedError)],"%d",errorInt);
			stringIndex += 2;		/* Skip past the %d */
			strcat(convertedError, stringIndex);

			return convertedError;
			}
		}
		
	return http_errlist[HF_METAERROR];	/* Should NEVER happen */
	}

	
	/*
	 * Reads the metadata of an HTTP response.
	 * Perhaps a little inefficient, as it reads 1 byte at a time, but
	 *	I don't think it's that much of a loss (most headers aren't HUGE).
	 * Returns:
	 *	# of bytes read on success, or
	 *	-1 on error
	 */
int _http_read_header(int sock, char *headerPtr)
	{
	fd_set rfds;
	struct timeval tv;
	int bytesRead = 0, newlines = 0, ret, selectRet;

	while(newlines != 2 && bytesRead != HEADER_BUF_SIZE)
		{
		FD_ZERO(&rfds);
		FD_SET(sock, &rfds);
		tv.tv_sec = timeout; 
		tv.tv_usec = 0;

		if(timeout >= 0)
			selectRet = select(sock+1, &rfds, NULL, NULL, &tv);
		else		/* No timeout, can block indefinately */
			selectRet = select(sock+1, &rfds, NULL, NULL, NULL);
		
		if(selectRet == 0)
			{
			errorSource = FETCHER_ERROR;
			http_errno = HF_HEADTIMEOUT;
			errorInt = timeout;
			return -1;
			}
		else if(selectRet == -1) { errorSource = ERRNO; return -1; }

		ret = read(sock, headerPtr, 1);
		if(ret == -1) { errorSource = ERRNO; return -1; }
		bytesRead++;

		if(*headerPtr == '\r')			/* Ignore CR */
			{
			/* Basically do nothing special, just don't set newlines
			 *	to 0 */
			headerPtr++;
			continue;
			}
		else if(*headerPtr == '\n')		/* LF is the separator */
			newlines++;
		else
			newlines = 0;

		headerPtr++;
		}

	headerPtr -= 3;		/* Snip the trailing LF's */
	*headerPtr = '\0';
	return bytesRead;
	}



	/*
	 * Opens a TCP socket and returns the descriptor
	 * Returns:
	 *	socket descriptor, or
	 *	-1 on error
	 */
int makeSocket(const char *host)
	{
	int sock;										/* Socket descriptor */
	struct sockaddr_in sa;							/* Socket address */
	struct hostent *hp;								/* Host entity */
	int ret;
    int port;
    char *p;
	
    /* Check for port number specified in URL */
    p = strchr(host, ':');
    if(p)
        {
        port = atoi(p + 1);
        *p = '\0';
        }
    else
        port = PORT_NUMBER;

	hp = gethostbyname(host);
	if(hp == NULL) { errorSource = H_ERRNO; return -1; }
		
	/* Copy host address from hostent to (server) socket address */
	memcpy((char *)&sa.sin_addr, (char *)hp->h_addr, hp->h_length);
	sa.sin_family = hp->h_addrtype;		/* Set service sin_family to PF_INET */
	sa.sin_port = htons(port);      	/* Put portnum into sockaddr */

	sock = socket(hp->h_addrtype, SOCK_STREAM, 0);
	if(sock == -1) { errorSource = ERRNO; return -1; }

	ret = connect(sock, (struct sockaddr *)&sa, sizeof(sa));
	if(ret == -1) { close(sock); errorSource = ERRNO; return -1; }

	return sock;
	}



	/*
	 * Determines if the given NULL-terminated buffer is large enough to
	 * 	concatenate the given number of characters.  If not, it attempts to
	 * 	grow the buffer to fit.
	 * Returns:
	 *	0 on success, or
	 *	-1 on error (original buffer is unchanged).
	 */
int _checkBufSize(char **buf, int *bufsize, int more)
	{
	char *tmp;
	int roomLeft = *bufsize - (strlen(*buf) + 1);
	if(roomLeft > more)
		return 0;
	tmp = realloc(*buf, *bufsize + more + 1);
	if(tmp == NULL)
		return -1;
	*buf = tmp;
	*bufsize += more + 1;
	return 0;
	}
