/*
*  http_get.c for ed2k_gui
*
*  GET HTML PAGE (SERVERLIST) VIA HTTP PROTOCOL
*
* (c) 2001-2002 TP Muller <t.i.m@orange.net>
*/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include <gnet.h>

#include "global.h"
#include "core-conn.h"
#include "downloads-list.h"
#include "http_get.h"

#include "search.h"
#include "servers.h"
#include "status_page.h"
#include "options.h"
#include "misc.h"
#include "misc_strings.h"
#include "global.h"
#include "tag.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>

#include <gtk/gtkmain.h>

// global variables

// local variables and defines

#define HTTP_GET_TIMEOUT	45*1000


#ifdef G_OS_UNIX
# ifdef FREEBSD
#  define OP_SYSTEM_STRING "FreeBSD"
# else
#  ifdef LINUX
#   define OP_SYSTEM_STRING "Linux"
#  else
#   ifdef NETBSD
#    define OP_SYSTEM_STRING "NetBSD"
#   else
#    ifdef OPENBSD
#     define OP_SYSTEM_STRING "OpenBSD"
#    else
#     define OP_SYSTEM_STRING "unix"				/* some other unix/platform */
#    endif /* ifdef OPENBSD */
#   endif /* ifdef NETBSD */
#  endif /* ifdef LINUX */
# endif /* ifdef FREEBSD */
#else
# define OP_SYSTEM_STRING "MSWin32"
#endif


// local functions

static void          http_get_connection_callback ( GConn *conn,
                                                    GConnEvent *event,
                                                    gpointer datastruct );

static gboolean      http_get_check_for_redirection (gchar *page, guint redirections);

static void          http_get_process_serverlist_page ( gchar *page,
                                                        guint bufsize,
                                                        gboolean clearlist );

static void          http_get_process_contactlist_page ( gchar *page,
                                                         guint bufsize,
                                                         gboolean clearlist );

static void          http_get_onContactIPResolved (GInetAddr *ia, gpointer data);


static void			 http_get_connection_cleanup (GConn *conn, struct http_get_callback_data *data);
static gchar		*http_get_beginning_of_data (const gchar *page);
static gint			 extract_server_links_from_metfile (const gchar *data, guint datasize, gboolean clearlist);
static gboolean		 http_get_ip_is_evil (guint8 n1, guint8 n2, guint8 n3, guint8 n4, const gchar **offender);


// extract_server_links
//
// Takes a page with ed2k://|server|-links and extracts them one by one and
//	adds them to the server list. It also clears the list before the first
//	link is added on request.
//
// returns number of added servers

gint
extract_server_links (const gchar *pagebuf, gboolean clear_serverlist)
{
	guint32	n1, n2, n3, n4, port, num=0, invalid=0, new=0;
	gchar	**linkarray, **link, extrapriority;
	g_return_val_if_fail (pagebuf!=NULL, 0);

	linkarray = g_strsplit (pagebuf, "|server|", 750);

	if (!linkarray) return 0;
	link = linkarray;
	while (*link)
	{
		gboolean	 foundlink;
		const gchar	*offender;

		foundlink = TRUE;

		/* We accept two formats - the standard format (5 parameter) and EXP's format, with 
		 * an additional figure '0'-'2' at the end specifying the priority of the server
		 */

		if (sscanf (*link, "%u.%u.%u.%u|%u|%c", &n1, &n2, &n3, &n4, &port, &extrapriority) != 6)
		{
			extrapriority = '0';	/* normal priority if no priority is specified */
			if (sscanf (*link, "%u.%u.%u.%u|%u|", &n1, &n2, &n3, &n4, &port) != 5)
			{
				foundlink = FALSE;
			}
		}

		if (foundlink)
		{
			if (n1==10 || (n1==172 && n2>=16 && n2<=31) || (n1==192 && n2==168) || (n1==127 && n2==0 && n3==0))
			{
				status_message_blue (_("GUI: server list: skipped useless server with private IP %u.%u.%u.%u.\n"), n1,n2,n3,n4);
				foundlink = FALSE;
			}
		}

		if (foundlink)
		{
			if (http_get_ip_is_evil (n4, n3, n2, n1, &offender))
			{
				status_message_blue (_("GUI: server list: skipped suspicious IP from '%s'.\n"), offender);
				foundlink = FALSE;
			}
		}

		if (foundlink)
		{
			guint32	ip, priority=0;
			ip = GUINT32_TO_LE (n1 | (n2 << 8) | (n3 << 16) | (n4 << 24));

			if (extrapriority == '1') priority = 1;
			if (extrapriority == '2') priority = 2;

//			g_print ("====> server %u.%u.%u.%u : %u\n", n1,n2,n3,n4,port);
			if (!servers_lookup (ip,port))
			{
				// get rid of all servers if wanted
				if (clear_serverlist==1 && num==0)
				{
					servers_cleanup ( );
				}

				gui_core_conn_send_add_server(core, ip, port);
				if (priority>0)
					gui_core_conn_send_set_server_pref(core, ip, port, (guint8) priority);
				new++;
			}
			num++;
		} else if (*(link+1)!=NULL && link!=linkarray) invalid++;	// last+first line can be invalid
		link++;
	}

	status_msg (_("Fetched %u servers. %u were new. %u links were invalid. Total now: %u servers\n"),
				 num, new, invalid, servers_get_count () + new);

	g_strfreev(linkarray);
	return num;
}



/******************************************************************************
 *
 *  http_get_onContactIPResolved
 *
 ***/

static void
http_get_onContactIPResolved (GInetAddr *ia, gpointer data)
{
	if ( ia != NULL )
	{
		gchar *command = g_strdup_printf("boot %s %u", gnet_inetaddr_get_canonical_name(ia), gnet_inetaddr_get_port(ia));

//		g_print ("*** send: %s\n", command);
		misc_boot_overnet_queue_boot_string(command);

		gnet_inetaddr_delete(ia);
	}
}


/******************************************************************************
 *
 *  extract_contact_links
 *
 *  Takes a page with ed2k://|contact|-links and extracts them so we can
 *   boot into the overnet network.
 *
 *  returns number of contacts found
 *
 ***/
 
gint
extract_contact_links (const gchar *pagebuf)
{
	guint32	port, num=0, invalid=0;
	gchar	**linkarray, **link;

	if ( pagebuf == NULL )
		return 0;

	linkarray = g_strsplit (pagebuf, "|contact|", 0);

	if (!linkarray)
		return 0;

	link = linkarray;
	while (*link)
	{
		gboolean      foundlink;
		gchar        *div1;

		foundlink = FALSE;

		div1 = strchr(*link, '|');

		if (div1) 
		{
			GInetAddr *ia;
			gchar     *hostname = *link;

			*div1 = 0x00;
	
			if ( sscanf(div1+1, "%u|", &port) == 1 )
			{
				ia = gnet_inetaddr_new_nonblock(hostname,port);

				if ( ia == NULL )
				{
					(void) gnet_inetaddr_new_async (hostname, port, http_get_onContactIPResolved, NULL);
					foundlink = TRUE;
				}
				else if (gnet_inetaddr_is_internet(ia))
				{
					gchar *command = g_strdup_printf("boot %s %u", gnet_inetaddr_get_canonical_name(ia), port);
					misc_boot_overnet_queue_boot_string(command);
//					g_print ("send: %s\n", command);
					gnet_inetaddr_delete(ia);
					foundlink = TRUE;
				}
			}

			*div1 = '|';

		}


		if ( foundlink == FALSE )
		{
			/* last+first line may be invalid */
			if (*(link+1)!=NULL && link!=linkarray)
				invalid++;
		}
		else
		{
			num++;
		}

		link++;
	}

	status_msg (_("Fetched %u contacts to bootstrap from. %u were invalid.\n"), num, invalid);

	g_strfreev(linkarray);

	return num;
}

/******************************************************************************
 *
 *    onTimeoutSetDownloadPriorityFromLink
 *
 ******************************************************************************/

static gboolean
onTimeoutSetDownloadPriorityFromLink (guint8 *hash)
{
	GuiDownload  *dl;
	guint8        counter, prio;

	g_assert (hash != NULL);

	counter = hash[16];
	prio    = hash[17];

	/* enough is enough */
	if (counter >= 25)
	{
		g_free(hash);
		g_return_val_if_reached(FALSE);
	}

	dl = gui_download_list_get_download_from_hash (download_list, hash);

	if (dl != NULL)
	{
		status_msg (_("Setting priority of download received via ed2k-link to '%s'.\n"), misc_get_prio_string (prio));
		gui_core_conn_send_set_download_priority (core, hash, prio);
		dl->priority = prio;
		/* force row to be re-drawn */
		gui_download_list_download_change_status (download_list, dl, dl->status);
		g_free(hash);
		return FALSE;
	}

	hash[16] = counter+1;

	return TRUE;
}


/******************************************************************************
 *
 *    onTimeoutPauseDownloadFromLink
 *
 ******************************************************************************/

static gboolean
onTimeoutPauseDownloadFromLink (guint8 *hash)
{
	GuiDownload  *dl;
	guint8        counter;

	g_assert(hash != NULL);

	/* enough is enough */
	counter = hash[16];

	if (counter >= 25)
	{
		g_free(hash);
		g_return_val_if_reached(FALSE);
	}

	dl = gui_download_list_get_download_from_hash (download_list, hash);

	if (dl != NULL)
	{
		gui_core_conn_send_pause_download(core, hash); /* pause download */
		status_msg("Pausing download received via ed2k-link.\n");
		g_free(hash);
		return FALSE;
	}

	hash[16] = counter+1;

	return TRUE;
}


/******************************************************************************
 *
 *    extract_ed2k_links
 *
 *    (call misc_string_unescape_url() before if necessary)
 *
 ******************************************************************************/

gint
extract_ed2k_links (const gchar *page_utf8)
{
	gchar     *n;
	gint       num = 0;
	Search    *search = NULL;

	g_return_val_if_fail ( page_utf8 != NULL, -1);

	if (!opt_get_bool(OPT_GUI_DOWNLOAD_ED2K_LINK_IMMEDIATELY))
		search = search_get_ed2k_link_search();

	n = strstr (page_utf8, "ed2k:");
	while (n)
	{
		gboolean  addpaused = FALSE;
		gchar	   *fn, *hashstr, hash[16];
		guint	    size, addprio = 1; /* normal priority */

		fn = NULL;
		size = 0;
		hashstr = NULL;

		/* here the page is already in UTF-8, so the link will be too */
		if (misc_check_if_ed2klink_is_valid(n,&fn,&size,&hashstr,&addpaused,&addprio))
		{
			if ((fn) && (hashstr) && size>0)
			{
				memcpy (hash, hash_str_to_hash(hashstr), 16);

				/* remove bad characters that some tossers put into filenames */
				(void) g_strdelimit (fn, "/&\\<>", '_');

				if (!opt_get_bool(OPT_GUI_DOWNLOAD_ED2K_LINK_IMMEDIATELY) && (search)) 
				{
					search_add_or_update_record (search, hash, NULL, size, 0, fn, NULL, NULL, NULL, NULL, NULL, NULL, 0, NULL);
				}
				else
				{
					gui_core_conn_send_new_download(core, hash, size, fn); /* download immediately! */

					if (addpaused)
					{
						guint8 *hashdup;

						hashdup = g_memdup (hash, 16);
						hashdup = g_realloc (hashdup, 16+1);

						hashdup[16] = 0; /* we use that as counter */

						g_timeout_add(500, (GSourceFunc) onTimeoutPauseDownloadFromLink, hashdup);
					}
					
					if (addprio != 1 && addprio <= 3)
					{
						guint8 *hashdup;

						hashdup = g_memdup (hash, 16);
						hashdup = g_realloc (hashdup, 16+2);

						hashdup[16] = 0;                /* we use that as counter             */
						hashdup[17] = (guint8) addprio; /* we use that for the priority value */

						g_timeout_add (500, (GSourceFunc) onTimeoutSetDownloadPriorityFromLink, hashdup);
					}

					if (opt_get_bool(OPT_GUI_LINKHANDLER_ADD_PAUSED_DEFAULT) != addpaused)
					{
						opt_toggle_bool(OPT_GUI_LINKHANDLER_ADD_PAUSED_DEFAULT);
						opt_write_to_disk();
					}
				}

				num++;
				G_FREE(fn);
				G_FREE(hashstr);
			}
		}
		n = strstr (n+15, "ed2k:");
	}

	if (search)
		search_switch_to_search_page(search);

	return num;
}


// http_get_retrieve_serverlist
//
// Asynchroneously (ie. without blocking) retrieves a page from an http-server and
//	passes the result on to a routine to extract the ed2k-server links on that
//	page.

void
http_get_retrieve_serverlist (const gchar *url, guint redirections)
{
	struct http_get_callback_data   *data;
	GConn                           *conn;
	GURI                            *fetchurl, *proxyurl = NULL;
	gchar                           *proxyenv, *firstgetline, *conn_host;
	guint                            conn_port;

	g_return_if_fail ( url != NULL );

	if ( redirections >= MAX_NUMBER_OF_REDIRECTIONS )
	{
		status_warning (_("GUI: Too many redirections while trying to fetch server list! Giving up.\n"));
		return;
	}

	fetchurl = gnet_uri_new (url);
	if (!fetchurl)
	{
		g_printerr (_("GUI: Could not transform URL to GURL structure!\n"));
		return;
	}

	// check server list URL
	if (!misc_gnet_http_url_okay(fetchurl))
	{
		status_msg (_("GUI: malformed http URL\n"));
		return;
	}
	else
	{
		// use standard port 80 if no port is specified
		if ( fetchurl->port == 0 )
			gnet_uri_set_port (fetchurl, 80);
		status_msg (_("GUI: Parsed URL alright (%s)\n"), gnet_uri_get_string(fetchurl));
	}

	// check if the http proxy environment variable is set (and not empty)
	if (!(proxyenv=getenv("http_proxy")))
	{
		proxyenv = getenv("HTTP_PROXY");
	}
	if ((proxyenv) && (*proxyenv!=0x00))
	{
		proxyurl = gnet_uri_new (proxyenv);
	}

	if (proxyurl)	// using http proxy ...
	{
		status_msg (_("GUI: proxy environmental variable (http_proxy) is set ...\n"));

		// check proxy server URL
		if (!misc_gnet_http_url_okay(proxyurl))
		{
			/* maybe the '/' at the end has been forgotten? */
			if ((proxyurl)&&(!proxyurl->path))
				gnet_uri_set_path(proxyurl, "/");

			if (!misc_gnet_http_url_okay(proxyurl))
			{
				status_msg (_("GUI: malformed proxy URL ('%s')\n"), (getenv("http_proxy")!=NULL) ? getenv("http_proxy") : getenv("HTTP_PROXY"));
				status_msg (_("GUI: proxy URL must be in this format: 'http://<host>:<port>/'\n"));
				return;
			}
		}

		status_msg (_("GUI: Parsed proxy URL alright (%s)\n"), gnet_uri_get_string(proxyurl));

		if (proxyurl->port==0)
		{
			gnet_uri_set_port (proxyurl, 8080);
			status_msg (_("GUI: assuming standard port 8080 for http proxy.\n"));
		}

		firstgetline = g_strdup_printf ("GET http://%s:%u%s%s%s%s%s HTTP/1.1\r\n",
		                   fetchurl->hostname, fetchurl->port, fetchurl->path,
		                   (fetchurl->query)    ? "?" : "",
		                   (fetchurl->query)    ? fetchurl->query : "",
		                   (fetchurl->fragment) ? "#" : "",
		                   (fetchurl->fragment) ? fetchurl->fragment : "");

		conn_host = proxyurl->hostname;
		conn_port = proxyurl->port;
	}
	else
	{	/* NOT using http proxy ... */
		gchar *pathelements = g_strdup_printf ("%s%s%s%s%s", fetchurl->path,
		                       (fetchurl->query)    ? "?" : "",
		                       (fetchurl->query)    ? fetchurl->query : "",
		                       (fetchurl->fragment) ? "#" : "",
		                       (fetchurl->fragment) ? fetchurl->fragment : "");

		firstgetline = g_strdup_printf ("GET %s HTTP/1.1\r\n", pathelements);

		conn_host = fetchurl->hostname;
		conn_port = fetchurl->port;

		g_free(pathelements);
	}

	g_return_if_fail (firstgetline!=NULL);

	// make data structure for callback routine (we need that because we can only hand over one user data value
	data = g_new0 (struct http_get_callback_data, 1);

	data->getstring = g_strdup_printf ("%s"
				"Accept: text/html\r\n"
//				"Accept-Encoding: x-compress; x-zip; x-gzip\r\n"
				"Accept-Language: en-us\r\n"
				"User-Agent: Mozilla/4.0 (compatible; ed2k_gui; %s)\r\n"
				"Host: %s\r\n"
				"Connection: close\r\n\r\n", firstgetline, OP_SYSTEM_STRING, fetchurl->hostname);

	if (!data->getstring)
	{
		g_free(data);
		g_return_if_reached();
	}

	g_return_if_fail (data->getstring!=NULL);

	data->buf = NULL;
	data->bufsize = 0;
	data->contentlength = -1;
	data->headerlength = -1;
	data->clearserverlist = (opt_get_bool(OPT_GUI_SERVERS_CLEAR_LIST_BEFORE_FETCHING_NEW_LIST) == TRUE
	                            && strcmp(url, opt_get_str(OPT_GUI_SERVERLIST_URL_1)) == 0);
	data->using_httpproxy = (proxyurl!=NULL);
	data->redirections = redirections;

	// Now make connection, and connect if successful (the callback will handle the rest then)
	// XXX - this may block?! (for the DNS resolution)
	conn = gnet_conn_new (conn_host, conn_port, http_get_connection_callback, (gpointer) data);

	if (!conn)
		status_msg (_("GUI: couldn't resolve hostname '%s'\n"), (proxyurl!=NULL) ? proxyurl->hostname : fetchurl->hostname);
	else
		gnet_conn_connect(conn);

	if (fetchurl)
		gnet_uri_delete (fetchurl);

	if (proxyurl)
		gnet_uri_delete (proxyurl);

	g_free (firstgetline);
	// string in 'get' will be freed by connection callback function
}


/******************************************************************************
 *
 *  http_get_connection_callback
 *
 *  This is the connection callback to the http/proxy server that does all the
 *   reading/writing work
 *
 ***/

static void
http_get_connection_callback (GConn *conn, GConnEvent *event, gpointer datastruct)
{
	struct http_get_callback_data	*data = (struct http_get_callback_data*) datastruct;

	g_return_if_fail ( data  != NULL );
	g_return_if_fail ( event != NULL );

	switch (event->type)
	{
		case GNET_CONN_CONNECT:
		{
			g_return_if_fail ( conn != NULL );

			status_msg (_("GUI: connected to http%s server\n"),
			             (data->using_httpproxy) ? _(" proxy"): "");

			/* we are connected, let's send our GET request and read the answer */
			g_return_if_fail ( data->getstring != NULL );
			gnet_conn_write (conn, data->getstring, strlen(data->getstring));
			/* gnet_conn_write copies buffer in gnet-2.0, so we can delete it here */
			g_free(data->getstring);
			data->getstring = NULL;

			gnet_conn_read(conn);
		}
		break;

		case GNET_CONN_CLOSE:
		{
			status_msg (_("GUI: read list. Connection closed.\n"));

			if (!http_get_check_for_redirection(data->buf, data->redirections))
			{
				if (!gui_core_conn_is_overnet(core) || gui_core_conn_is_hybrid(core))
					http_get_process_serverlist_page (data->buf, data->bufsize, data->clearserverlist);

				if (gui_core_conn_is_overnet(core))
					http_get_process_contactlist_page (data->buf, data->bufsize, data->clearserverlist);
			}

			http_get_connection_cleanup (conn, data);
		}
		return;	/* we're done */

		case GNET_CONN_READABLE:
			g_return_if_reached();

		case GNET_CONN_READ:
		{
			if ( event->length == 0 )	/* does this mean the connection has been closed? */
				return;

			/* so we're called for the next data that arrives as well */
			gnet_conn_read(conn);


			data->buf = g_realloc (data->buf, ( data->bufsize + event->length + 1 ) );
			g_return_if_fail ( data->buf != NULL );
			g_memmove (data->buf + data->bufsize, event->buffer, event->length);
			data->buf[data->bufsize + event->length]=0x00;		// remember: we allocated +1 for terminating 0
			data->bufsize += event->length;
//			g_print ("read %u bytes (total = %8u)\n", length, data->bufsize);

			// check for Content-Length: header line
			if (data->contentlength < 0)
			{
				static const gchar clstr[] = "Content-Length:";
				// we need to search the whole buffer, because the string could be
				//	split across the border of two chunks read
				gchar	*conlen = strstr(data->buf, clstr);
				if (conlen)
				{
					gchar	*termchar = NULL;
					data->contentlength = strtol (conlen+sizeof(clstr), &termchar, 10);
					// in this unlikely case the number has been split across two chunks
					if ((termchar) && *termchar!='\n' && *termchar!='\r')
					{
						data->contentlength = -1;
					} //else g_print ("content-length = %u *****\n", data->contentlength);
				}
				
			}

			if (data->headerlength < 0)
			{
				guint	 endlen = 4;
				gchar	*headerend = strstr(data->buf, "\r\n\r\n");
				if (!headerend)
				{
					endlen = 2;
					headerend = strstr(data->buf, "\n\n");
					if (!headerend) headerend = strstr(data->buf, "\r\r");
				}

				if (headerend)
				{
					data->headerlength = headerend+endlen-data->buf;
				}
			}

			if (data->contentlength > 0  &&  data->headerlength > 0
				&&  data->bufsize >= (data->contentlength + data->headerlength))
			{
				status_msg (_("GUI: read list. Closing connection.\n"));
//				g_print ("---------->\n%s\n<------------\n", data->buf);

				if (!http_get_check_for_redirection(data->buf, data->redirections))
				{
					if (!gui_core_conn_is_overnet(core) || gui_core_conn_is_hybrid(core))
						http_get_process_serverlist_page (data->buf, data->bufsize, data->clearserverlist);

					if (gui_core_conn_is_overnet(core))
						http_get_process_contactlist_page (data->buf, data->bufsize, data->clearserverlist);
				}

				http_get_connection_cleanup (conn, data);

				return;	/*  we're done. */
			}
		}
		return;

		/* we don't set a watch for the 'writable' event on the connection, so
		 * this case should not happen */
		case GNET_CONN_WRITABLE:
			g_return_if_reached();
		break;

		/* called whenever something has been written (which we don't need to act upon) */
		case GNET_CONN_WRITE:
		break;

		case GNET_CONN_TIMEOUT:
		{
			status_msg (_("GUI: Timeout on http connection.\n"));
			http_get_connection_cleanup (conn, data);
		}
		break;

		case GNET_CONN_ERROR:
		{
			const gchar *errconnstr = _("GUI: Error on http connection.");
			if (errno>0)
			{
				status_msg ("%s\n", errconnstr);
			}
			else
			{
				gui_print_error_function (errconnstr);
			}

			http_get_connection_cleanup (conn, data);
		}
		break;
	}
}

// http_get_connection_cleanup
//
// deletes the connection and buffers etc.

static void
http_get_connection_cleanup (GConn *conn, struct http_get_callback_data *data)
{
	if (conn)
	{
		gnet_conn_disconnect (conn);
		gnet_conn_delete (conn);
	}
	if (data)
	{
		G_FREE(data->getstring);
		G_FREE(data->buf);
		g_free (data);
	}
}


/******************************************************************************
 *
 *  http_get_check_for_redirection
 *
 *  if the page is only a redirection header,
 *    download page from new URL, unless max.
 *    number of redirections have been exceeded
 *
 *  returns TRUE if this is a redirection
 *
 ***/

static gboolean
http_get_check_for_redirection (gchar *page, guint redirections)
{
	guint  protomajor, protominor, code = 0;
	gchar *cr, *location, *newurl;

	g_return_val_if_fail ( page != NULL, FALSE);

	if ( sscanf(page,"HTTP/%u.%u %u", &protomajor, &protominor, &code) != 3 )
		return FALSE;

	if (code != 302)
		return FALSE;

	location = strstr(page,"Location: ");

	if (!location)
	{
		status_warning (_("GUI: Redirected, but no new location specified?!\n"));
		return TRUE;
	}

	newurl = location + strlen("Location: ");
	cr = strchr(newurl,'\r');
	if (!cr) cr = strchr(newurl,'\n');
	if (cr) *cr = 0x00;

	status_message_blue (_("GUI: Redirected to new location '%s'.\n"), newurl);

	http_get_retrieve_serverlist (newurl, redirections+1);

	return TRUE;
}

// http_get_process_serverlist_page
//
// Extracts the ed2k-servers from the page, and displays success/failure messages

static void
http_get_process_serverlist_page (gchar *page, guint bufsize, gboolean clearlist)
{
	guint	 num;
	gchar	*data; 

	g_return_if_fail (page!=NULL);	// we should at least have headers!

	data = http_get_beginning_of_data (page);
	if ((data) && *data == 0x0e)
	{
		status_message_blue ("%s\n", _("GUI: This appears to be a server.met file."));
		num = extract_server_links_from_metfile (data, bufsize-(data-page), clearlist);
	}
	else
	{
		num = extract_server_links(page, clearlist);
	}

	if (num == 0)
	{
		gchar	*cr = strchr(page,'\r');
		if (!cr) cr = strchr(page,'\n');
		if (cr)
		{
			cr[1]=0x00;	/* we only want the first line of the server response + new line character */
			cr[0]='\n';	/* we don't want a '\r' */
		}
		status_msg (_("GUI: No ed2k://|server| links found on page :( Server returned this: %s\n"), page);
	}
	else
	{
		gui_core_conn_send_get_serverlist(core);
	}
}



/******************************************************************************
 *
 *  http_get_process_contactlist_page
 *
 *  Extracts the ed2k-contacts from the page, and displays success/failure messages
 *
 ***/

static void
http_get_process_contactlist_page (gchar *page, guint bufsize, gboolean clearlist)
{
	guint	 num;

	g_return_if_fail ( page != NULL );	/* we should at least have headers! */

	num = extract_contact_links (page);

	if (num == 0)
	{
		gchar	*cr = strchr(page,'\r');
		if (!cr) cr = strchr(page,'\n');
		if (cr)
		{
			cr[1]=0x00;	/* we only want the first line of the server response + new line character */
			cr[0]='\n';	/* we don't want a '\r' */
		}
		status_msg (_("GUI: No ed2k://|contact| links found on page :( Server returned this: %s\n"), page);
	} 
}



/* http_get_beginning_of_data
 *
 * Takes a page received via http including headers and skips the headers,
 * pointing to the first byte of data
 *
 */

static gchar *
http_get_beginning_of_data (const gchar *page)
{
	gchar *data;

	g_return_val_if_fail (page!=NULL, NULL);

	data = strstr (page, "\r\n\r\n");
	if (data)
		data += 4;

	return data;
}


/* extract_server_links_from_metfile
 *
 * parses a server.met file
 *
 * returns number of added servers
 *
 */

static gint
extract_server_links_from_metfile (const gchar *data, guint datasize, gboolean clearlist)
{
	gint		 num_servers, num;
	gint		 num_added=0, alreadyknown=0, invalid=0;
	guint8		 header;
	ramfile		*rf;

	g_return_val_if_fail (data!=NULL, 0);
	g_return_val_if_fail (datasize<(1024*1024), 0);	/* a server.met larger than 1MB? */

	rf = ramfile_new (datasize);
	g_return_val_if_fail (rf!=NULL, 0);

	ramfile_write (rf, (guint8*) data, datasize*sizeof(gchar));
	ramfile_set_pos (rf,0);
	header = ramfile_read8(rf);

	if (header != 0x0e)
	{
		status_warning (_("GUI: server.met file: wrong header?!\n"));
		ramfile_free(rf);
		return 0;
	}

	num_servers = ramfile_read32(rf);
	num = num_servers;

	if (num_servers == 0xffffffff)
	{
		status_warning (_("GUI: server.met file: premature end of file?!\n"));
		ramfile_free(rf);
		return 0;
	}

	if (num_servers == 0)
	{
		status_warning (_("GUI: server.met file: claims to have 0 servers?!\n"));
		ramfile_free(rf);
		return 0;
	}

	if (num_servers > 2000)
	{
		status_warning (_("GUI: server.met file: claims to have more than 2000 servers?! (%u)\n"), num_servers);
		ramfile_free(rf);
		return 0;
	}

	while (num_servers > 0)
	{
		guint32		 ip = ramfile_readIP(rf);
		guint16 	 port = ramfile_read16(rf);
		guint8		 n1,n2,n3,n4;
		GPtrArray	*taglist;
		gboolean	 add;
		const gchar	*offender;

		add = TRUE;
		if (ip == 0xffffffff || port == 0xffff)
		{
			status_warning (_("GUI: server.met file: premature end of file?!\n"));
			ramfile_free(rf);
			return num_added;
		}

		n4 = *((guint8*)&ip);
		n3 = *(((guint8*)&ip)+1);
		n2 = *(((guint8*)&ip)+2);
		n1 = *(((guint8*)&ip)+3);

		if (misc_ip_is_local(ip))
		{
			status_message_blue (_("GUI: server.met file: skipped useless server with private IP %u.%u.%u.%u.\n"), n4,n3,n2,n1);
			add = FALSE;
		}

		if (http_get_ip_is_evil (n1, n2, n3, n4, &offender))
		{
			status_message_blue (_("GUI: server.met file: skipped suspicious IP from '%s'.\n"), offender);
			add = FALSE;
		}

/*		g_print ("got IP %8x:%u\n", ip, port); */

		taglist = taglist_read_from_ramfile (rf);
		if (!taglist)
		{
			status_warning (_("GUI: server.met file: premature end of file?!\n"));
			ramfile_free(rf);
			return num_added;
		}

		if (add)
		{
			if (!servers_lookup (ip,port))
			{
				NGS_tag	*priority = taglist_find_tag_from_tagname (taglist, TAGNAME_PREFERENCE_SHORT);

				/* get rid of all servers if wanted */
				if ((clearlist)&& num==0)
				{
					servers_cleanup ();
				}

				gui_core_conn_send_add_server(core, ip, port);
				num_added++;

				if ((priority) && GPOINTER_TO_UINT(priority->data)!=0)
				{
					gui_core_conn_send_set_server_pref(core, ip, port, (guint8) GPOINTER_TO_UINT(priority->data));
				}
			} else alreadyknown++;
		} else invalid++;

		taglist_free(taglist);
		taglist = NULL;

		num_servers--;
	}

	status_msg (_("Fetched %u servers. %u were new. %u links were invalid. Total now: %u servers\n"),
				 num, num_added, invalid, servers_get_count () + num_added);

	ramfile_free(rf);

	return num;
}




/* To add new stuff:
 * { Netmask, Network Address, Type }, // dotted quad
 *
 * To change a dotted quad range to something you want to add, if you want to
 * block an entire netblock, just change the start address to hex and add it.
 * i.e.
 * 172.16.0.0 - 172.31.0.0 == 172.16/12, private address
 * Thus:
 * 172.16.0.0 -> 0xAC100000
 * /12 -> 12
 * Ergo:
 * { 12, 0xAC100000, PRIVATE } // 172.16.0/12
 */

struct net_range net_list[] = {
       {  8, 0x00000000,            T_H_I_S },   //   0.0.0.0/8

       {  4, 0xE0000000,       MULTICAST },   //   244.0.0.0/4

       { 10, 0x0A000000,         PRIVATE },   //   10.0.0.0/8
       {  8, 0x7F000000,         PRIVATE },   //   127.0.0.0/8
       { 12, 0xAC100000,         PRIVATE },   //   172.16.0.0/12
       { 16, 0xC0A80000,         PRIVATE },   //   192.168.0.0/16


       { 24, 0xC0000200,        TEST_NET },   //   192.0.2.0/24
       { 15, 0xC6120000,        TEST_NET },   //   198.18.0.0/15

       { 24, 0xC0586300,    ANYCAST_6TO4 },   //   192.88.99.0/24

       { 16, 0xA9FE0000,      LINK_LOCAL },   //   169.254.0/16

       { 24, 0x042B6000,     MEDIA_FORCE },   //   4.43.96.0/24
       { 24, 0x41C00000,     MEDIA_FORCE },   //   65.192.0.0/24
       { 27, 0x41D9DBC0,     MEDIA_FORCE },   //   65.217.219.192/27
       { 27, 0x41DFFFC0,     MEDIA_FORCE },   //   65.223.255.192/27
       { 26, 0x41F3D73F,     MEDIA_FORCE },   //   65.243.215.0/24
       { 28, 0x41F769F0,     MEDIA_FORCE },   //   65.247.105.240/28
       { 24, 0xD0FB8900,     MEDIA_FORCE },   //   208.251.137.0/24

       { 24, 0x3F400000,     CYVEILLANCE },   //   63.64.0.0/24
       { 24, 0x3F7FFF00,     CYVEILLANCE },   //   63.127.255.0/24
       { 27, 0x3F9463E0,     CYVEILLANCE },   //   63.148.99.224/27
       { 27, 0x417629C0,     CYVEILLANCE },   //   65.118.41.192/27

       { 24, 0x40E1CA00,  MEDIA_DEFENDER },   //   64.225.202.0/24
       { 24, 0x424F0000,  MEDIA_DEFENDER },   //   66.79.0.0/24
       { 24, 0x424F0A00,  MEDIA_DEFENDER },   //   66.79.10.0/24
       { 24, 0x424F5F00,  MEDIA_DEFENDER },   //   66.79.95.0/24

       { 24, 0xD1CC8000,         BAY_TSP },   //   209.204.128.0/24
       { 24, 0xD1CCBF00,         BAY_TSP },   //   209.204.191.0/24
       { 19, 0xD8310000,         BAY_TSP }    //   216.49.0.0/19
};

/*
 * We use ntohl()/htonl() just in case we have to deal with PDP-ENDIAN systems.
 * Also known as VAX-endian.  Yes, Linux runs on VAX. :-)
 */

/* http_get_ip_is_evil
 *
 * returns TRUE if we do not want to contact the given IP (i.e. p2p sniffer companies etc.)
 *
 * IP address is n4.n3.n2.n1
 *
 * if return value is true, 'offender' will be set to name of offender
 */

static gboolean
http_get_ip_is_evil(guint8 n4, guint8 n3, guint8 n2, guint8 n1, const char **offender)
{
	guint32  host;
	gint     i;
	gint     type = OK;
	guint32  andmask;

	for(i = 0; i < (sizeof(net_list) / sizeof(struct net_range)); i++)
	{
		andmask = ~((guint32)0);
		andmask <<= (32U - net_list[i].mask);

 		host = (guint32)n1<<24 | (guint32)n2<<16 | (guint32)n3<<8 | (guint32)n4;

		host &= andmask;

		if(host == net_list[i].addx)
		{
			type = net_list[i].type;
			break;
		}
	}

	switch(type)
	{
		case OK:
		break;

		case PRIVATE:
			if(offender)
				*offender = "Private IP";
		break;

		case MULTICAST:
			if(offender)
				*offender = "Multicast IP";
		break;

		case T_H_I_S:
			if(offender)
				*offender = "\"This\" network IP";
		break;

		case TEST_NET:
			if(offender)
				*offender = "Test network IP";
		break;

		case ANYCAST_6TO4:
			if(offender)
				*offender = "Anycast network IP";
		break;

		case LINK_LOCAL:
			if(offender)
				*offender = "Link local network IP";
		break;

		case CYVEILLANCE:
			if(offender)
				*offender = "Cyveillance (p2p monitoring)";
		break;

		case MEDIA_DEFENDER:
			if(offender)
				*offender = "MediaDefender (p2p fakes/spam/monitoring)";
		break;

		case MEDIA_FORCE:
			if(offender)
				*offender = "Media Force  (p2p monitoring)";
		break;

		case BAY_TSP:
			if(offender)
				*offender = "BayTSP (p2p monitoring)";
		break;

		/* etc... */
	}

	return ( type != OK );
}
