libfetch: merge some features from FreeBSD:

- Supports HTTP/1.1 308 redirect.
- SSLv[23] HTTPS connections are forbidden by default.
- TLS client certificate validation thru OpenSSL.
- Fixes for user/password encoding, misc.

NEWS

@@ -1,5 +1,12 @@
 xbps-0.48 (???):
 
+ * libfetch: merge some features from FreeBSD:
+
+	- Supports HTTP/1.1 308 redirect.
+	- SSLv[23] HTTPS connections are forbidden by default.
+	- TLS client certificate validation thru OpenSSL.
+	- Fixes for user/password encoding, misc.
+
  * lixbps: use a sane umask if the pkgdb file needs to created for the first
    time. Thanks to Wolfgang Draxinger (https://github.com/voidlinux/xbps/pull/108).
 

lib/fetch/common.c

@@ -1,7 +1,9 @@
+/*	$FreeBSD: rev 288217 $	*/
 /*	$NetBSD: common.c,v 1.29 2014/01/08 20:25:34 joerg Exp $	*/
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2014 Dag-Erling Smorgrav
  * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
+ * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,8 +28,6 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $
  */
 
 #include "compat.h"
@@ -58,6 +58,10 @@
 #include <signal.h>
 #endif
 
+#ifdef WITH_SSL
+#include <openssl/x509v3.h>
+#endif
+
 #include <pthread.h>
 
 #include "fetch.h"
@@ -438,6 +442,495 @@ fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
 	pthread_mutex_unlock(&cache_mtx);
 }
 
+
+#ifdef WITH_SSL
+/*
+ * Find the first occurrence of find in s, where the search is limited to the
+ * first slen characters of s.
+ */
+static char *
+strnstr(const char *s, const char *find, size_t slen)
+{
+	char c, sc;
+	size_t len;
+
+	if ((c = *find++) != '\0') {
+		len = strlen(find);
+		do {
+			do {
+				if (slen-- < 1 || (sc = *s++) == '\0')
+					return (NULL);
+			} while (sc != c);
+			if (len > slen)
+				return (NULL);
+		} while (strncmp(s, find, len) != 0);
+		s--;
+	}
+	return ((char *)__UNCONST(s));
+}
+
+/*
+ * Convert characters A-Z to lowercase (intentionally avoid any locale
+ * specific conversions).
+ */
+static char
+fetch_ssl_tolower(char in)
+{
+	if (in >= 'A' && in <= 'Z')
+		return (in + 32);
+	else
+		return (in);
+}
+
+/*
+ * isalpha implementation that intentionally avoids any locale specific
+ * conversions.
+ */
+static int
+fetch_ssl_isalpha(char in)
+{
+	return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z'));
+}
+
+/*
+ * Check if passed hostnames a and b are equal.
+ */
+static int
+fetch_ssl_hname_equal(const char *a, size_t alen, const char *b,
+    size_t blen)
+{
+	size_t i;
+
+	if (alen != blen)
+		return (0);
+	for (i = 0; i < alen; ++i) {
+		if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i]))
+			return (0);
+	}
+	return (1);
+}
+
+/*
+ * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9
+ * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha-
+ * numeric characters. Double hyphens (like they're found in IDN a-labels
+ * 'xn--') are not allowed. Empty labels are invalid.
+ */
+static int
+fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok)
+{
+	size_t i;
+
+	if (!len || l[0] == '-' || l[len-1] == '-')
+		return (0);
+	for (i = 0; i < len; ++i) {
+		if (!isdigit(l[i]) &&
+		    !fetch_ssl_isalpha(l[i]) &&
+		    !(l[i] == '*' && wcok) &&
+		    !(l[i] == '-' && l[i - 1] != '-'))
+			return (0);
+	}
+	return (1);
+}
+
+/*
+ * Check if host name consists only of numbers. This might indicate an IP
+ * address, which is not a good idea for CN wildcard comparison.
+ */
+static int
+fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len)
+{
+	size_t i;
+
+	for (i = 0; i < len; ++i) {
+		if (!((hostname[i] >= '0' && hostname[i] <= '9') ||
+		    hostname[i] == '.'))
+			return (0);
+	}
+	return (1);
+}
+
+/*
+ * Check if the host name h passed matches the pattern passed in m which
+ * is usually part of subjectAltName or CN of a certificate presented to
+ * the client. This includes wildcard matching. The algorithm is based on
+ * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280.
+  */
+static int
+fetch_ssl_hname_match(const char *h, size_t hlen, const char *m,
+    size_t mlen)
+{
+	int delta, hdotidx, mdot1idx, wcidx;
+	const char *hdot, *mdot1, *mdot2;
+	const char *wc; /* wildcard */
+
+	if (!(h && *h && m && *m))
+		return (0);
+	if ((wc = strnstr(m, "*", mlen)) == NULL)
+		return (fetch_ssl_hname_equal(h, hlen, m, mlen));
+	wcidx = wc - m;
+	/* hostname should not be just dots and numbers */
+	if (fetch_ssl_hname_is_only_numbers(h, hlen))
+		return (0);
+	/* only one wildcard allowed in pattern */
+	if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL)
+		return (0);
+	/*
+	 * there must be at least two more domain labels and
+	 * wildcard has to be in the leftmost label (RFC6125)
+	 */
+	mdot1 = strnstr(m, ".", mlen);
+	if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4)
+		return (0);
+	mdot1idx = mdot1 - m;
+		mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1);
+	if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2)
+		return (0);
+	/* hostname must contain a dot and not be the 1st char */
+	hdot = strnstr(h, ".", hlen);
+	if (hdot == NULL || hdot == h)
+		return (0);
+	hdotidx = hdot - h;
+	/*
+	 * host part of hostname must be at least as long as
+	 * pattern it's supposed to match
+	 */
+	if (hdotidx < mdot1idx)
+		return (0);
+	/*
+	 * don't allow wildcards in non-traditional domain names
+	 * (IDN, A-label, U-label...)
+	 */
+	if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) ||
+	    !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1))
+		return (0);
+	/* match domain part (part after first dot) */
+	if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1,
+	    mlen - mdot1idx))
+		return (0);
+	/* match part left of wildcard */
+	if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx))
+		return (0);
+	/* match part right of wildcard */
+	delta = mdot1idx - wcidx - 1;
+	if (!fetch_ssl_hname_equal(hdot - delta, delta,
+	    mdot1 - delta, delta))
+		return (0);
+	/* all tests succeded, it's a match */
+	return (1);
+}
+
+/*
+ * Get numeric host address info - returns NULL if host was not an IP
+ * address. The caller is responsible for deallocation using
+ * freeaddrinfo(3).
+ */
+static struct addrinfo *
+fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len)
+{
+	struct addrinfo hints, *res;
+	char *host;
+
+	host = (char *)malloc(len + 1);
+	memcpy(host, hostname, len);
+	host[len] = '\0';
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = PF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = 0;
+	hints.ai_flags = AI_NUMERICHOST;
+	/* port is not relevant for this purpose */
+	if (getaddrinfo(host, "443", &hints, &res) != 0)
+		return NULL;
+	free(host);
+	return res;
+}
+
+/*
+ * Compare ip address in addrinfo with address passes.
+ */
+static int
+fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost,
+    size_t rhostlen)
+{
+	const void *left;
+
+	if (lhost->ai_family == AF_INET && rhostlen == 4) {
+		left = (void *)&((struct sockaddr_in*)(void *)
+		lhost->ai_addr)->sin_addr.s_addr;
+#ifdef INET6
+	} else if (lhost->ai_family == AF_INET6 && rhostlen == 16) {
+		left = (void *)&((struct sockaddr_in6 *)(void *)
+		lhost->ai_addr)->sin6_addr;
+#endif
+	} else
+		return (0);
+	return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0);
+}
+
+/*
+ * Compare ip address in addrinfo with host passed. If host is not an IP
+ * address, comparison will fail.
+ */
+static int
+fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r,
+    size_t rlen)
+{
+	struct addrinfo *raddr;
+	int ret;
+	char *rip;
+
+	ret = 0;
+	if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL)
+		return 0; /* not a numeric host */
+
+	if (laddr->ai_family == raddr->ai_family) {
+		if (laddr->ai_family == AF_INET) {
+			rip = (char *)&((struct sockaddr_in *)(void *)
+			raddr->ai_addr)->sin_addr.s_addr;
+			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4);
+#ifdef INET6
+		} else if (laddr->ai_family == AF_INET6) {
+			rip = (char *)&((struct sockaddr_in6 *)(void *)
+			raddr->ai_addr)->sin6_addr;
+			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16);
+#endif
+		}
+	}
+	freeaddrinfo(raddr);
+	return (ret);
+}
+
+/*
+ * Verify server certificate by subjectAltName.
+ */
+static int
+fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames,
+    const char *host, struct addrinfo *ip)
+{
+	const GENERAL_NAME *name;
+	size_t nslen;
+	int i;
+	const char *ns;
+
+	for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) {
+		name = sk_GENERAL_NAME_value(altnames, i);
+		ns = (const char *)ASN1_STRING_data(name->d.ia5);
+		nslen = (size_t)ASN1_STRING_length(name->d.ia5);
+
+		if (name->type == GEN_DNS && ip == NULL &&
+		    fetch_ssl_hname_match(host, strlen(host), ns, nslen))
+			return (1);
+		else if (name->type == GEN_IPADD && ip != NULL &&
+		    fetch_ssl_ipaddr_match_bin(ip, ns, nslen))
+			return (1);
+	}
+	return (0);
+}
+
+/*
+ * Verify server certificate by CN.
+ */
+static int
+fetch_ssl_verify_cn(X509_NAME *subject, const char *host,
+    struct addrinfo *ip)
+{
+	ASN1_STRING *namedata;
+	X509_NAME_ENTRY *nameentry;
+	int cnlen, lastpos, loc, ret;
+	unsigned char *cn;
+
+	ret = 0;
+	lastpos = -1;
+	loc = -1;
+	cn = NULL;
+	/* get most specific CN (last entry in list) and compare */
+	while ((lastpos = X509_NAME_get_index_by_NID(subject,
+		    NID_commonName, lastpos)) != -1)
+		loc = lastpos;
+
+		if (loc > -1) {
+			nameentry = X509_NAME_get_entry(subject, loc);
+			namedata = X509_NAME_ENTRY_get_data(nameentry);
+			cnlen = ASN1_STRING_to_UTF8(&cn, namedata);
+			if (ip == NULL &&
+			    fetch_ssl_hname_match(host, strlen(host), (const char *)cn, cnlen))
+				ret = 1;
+			else if (ip != NULL && fetch_ssl_ipaddr_match(ip, (const char *)cn, cnlen))
+				ret = 1;
+			OPENSSL_free(cn);
+		}
+	return (ret);
+}
+
+/*
+ * Verify that server certificate subjectAltName/CN matches
+ * hostname. First check, if there are alternative subject names. If yes,
+ * those have to match. Only if those don't exist it falls back to
+ * checking the subject's CN.
+ */
+static int
+fetch_ssl_verify_hname(X509 *cert, const char *host)
+{
+	struct addrinfo *ip;
+	STACK_OF(GENERAL_NAME) *altnames;
+	X509_NAME *subject;
+	int ret;
+
+	ret = 0;
+	ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host));
+	altnames = X509_get_ext_d2i(cert, NID_subject_alt_name,
+				    NULL, NULL);
+
+	if (altnames != NULL) {
+		ret = fetch_ssl_verify_altname(altnames, host, ip);
+	} else {
+		subject = X509_get_subject_name(cert);
+		if (subject != NULL)
+			ret = fetch_ssl_verify_cn(subject, host, ip);
+	}
+
+	if (ip != NULL)
+		freeaddrinfo(ip);
+	if (altnames != NULL)
+		GENERAL_NAMES_free(altnames);
+	return (ret);
+}
+
+/*
+ * Configure transport security layer based on environment.
+ */
+static void
+fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
+{
+	long ssl_ctx_options;
+
+	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_TICKET;
+	if (getenv("SSL_ALLOW_SSL3") == NULL)
+		ssl_ctx_options |= SSL_OP_NO_SSLv3;
+	if (getenv("SSL_NO_TLS1") != NULL)
+		ssl_ctx_options |= SSL_OP_NO_TLSv1;
+	if (getenv("SSL_NO_TLS1_1") != NULL)
+		ssl_ctx_options |= SSL_OP_NO_TLSv1_1;
+	if (getenv("SSL_NO_TLS1_2") != NULL)
+		ssl_ctx_options |= SSL_OP_NO_TLSv1_2;
+	if (verbose)
+		fetch_info("SSL options: %lx", ssl_ctx_options);
+	SSL_CTX_set_options(ctx, ssl_ctx_options);
+}
+
+
+/*
+ * Configure peer verification based on environment.
+ */
+static int
+fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
+{
+	X509_LOOKUP *crl_lookup;
+	X509_STORE *crl_store;
+	const char *ca_cert_file, *ca_cert_path, *crl_file;
+
+	if (getenv("SSL_NO_VERIFY_PEER") == NULL) {
+		ca_cert_file = getenv("SSL_CA_CERT_FILE") != NULL ?
+		    getenv("SSL_CA_CERT_FILE") : "/etc/ssl/cert.pem";
+		ca_cert_path = getenv("SSL_CA_CERT_PATH");
+		if (verbose) {
+			fetch_info("Peer verification enabled");
+			if (ca_cert_file != NULL)
+				fetch_info("Using CA cert file: %s",
+				    ca_cert_file);
+			if (ca_cert_path != NULL)
+				fetch_info("Using CA cert path: %s",
+				    ca_cert_path);
+		}
+		SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER,
+		    fetch_ssl_cb_verify_crt);
+		SSL_CTX_load_verify_locations(ctx, ca_cert_file,
+		    ca_cert_path);
+		if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) {
+			if (verbose)
+				fetch_info("Using CRL file: %s", crl_file);
+
+			crl_store = SSL_CTX_get_cert_store(ctx);
+			crl_lookup = X509_STORE_add_lookup(crl_store,
+			    X509_LOOKUP_file());
+			if (crl_lookup == NULL ||
+			    !X509_load_crl_file(crl_lookup, crl_file,
+			    X509_FILETYPE_PEM)) {
+				fprintf(stderr,
+				    "Could not load CRL file %s\n",
+				    crl_file);
+				return (0);
+			}
+			X509_STORE_set_flags(crl_store,
+			    X509_V_FLAG_CRL_CHECK |
+			    X509_V_FLAG_CRL_CHECK_ALL);
+		}
+	}
+	return (1);
+}
+
+/*
+ * Configure client certificate based on environment.
+ */
+static int
+fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose)
+{
+	const char *client_cert_file, *client_key_file;
+
+	if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) {
+		client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ?
+		    getenv("SSL_CLIENT_KEY_FILE") : client_cert_file;
+		if (verbose) {
+			fetch_info("Using client cert file: %s",
+			    client_cert_file);
+			fetch_info("Using client key file: %s",
+			    client_key_file);
+		}
+		if (SSL_CTX_use_certificate_chain_file(ctx,
+			client_cert_file) != 1) {
+			fprintf(stderr,
+			    "Could not load client certificate %s\n",
+			    client_cert_file);
+			return (0);
+		}
+		if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file,
+			SSL_FILETYPE_PEM) != 1) {
+			fprintf(stderr,
+			    "Could not load client key %s\n",
+			    client_key_file);
+			return (0);
+		}
+	}
+	return (1);
+}
+
+/*
+ * Callback for SSL certificate verification, this is called on server
+ * cert verification. It takes no decision, but informs the user in case
+ * verification failed.
+ */
+int
+fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx)
+{
+	X509 *crt;
+	X509_NAME *name;
+	char *str;
+
+	str = NULL;
+	if (!verified) {
+		if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL &&
+		    (name = X509_get_subject_name(crt)) != NULL)
+			str = X509_NAME_oneline(name, 0, 0);
+		fprintf(stderr, "Certificate verification failed for %s\n",
+		    str != NULL ? str : "no relevant certificate");
+		OPENSSL_free(str);
+	}
+	return (verified);
+}
+
+#endif
+
 /*
  * Enable SSL on a connection.
  */
@@ -447,6 +940,8 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
 
 #ifdef WITH_SSL
 	int ret;
+	X509_NAME *name;
+	char *str;
 
 	/* Init the SSL library and context */
 	if (!SSL_library_init()){
@@ -460,8 +955,14 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
 	conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
 	SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
 
+	fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose);
+	if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose))
+		return (-1);
+	if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose))
+		return (-1);
+
 	conn->ssl = SSL_new(conn->ssl_ctx);
-	if (conn->ssl == NULL){
+	if (conn->ssl == NULL) {
 		fprintf(stderr, "SSL context creation failed\n");
 		return (-1);
 	}
@@ -483,21 +984,36 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
 		return (-1);
 	}
 
-	if (verbose) {
+	conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
-		X509_NAME *name;
-		char *str;
 
-		fprintf(stderr, "SSL connection established using %s\n",
+	if (conn->ssl_cert == NULL) {
-		    SSL_get_cipher(conn->ssl));
+		fprintf(stderr, "No server SSL certificate\n");
+		return (-1);
+	}
+
+        if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) {
+		if (verbose)
+			fetch_info("Verify hostname");
+		if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) {
+			fprintf(stderr,
+				"SSL certificate subject doesn't match host %s\n",
+				URL->host);
+			return (-1);
+		}
+	}
+
+	if (verbose) {
+		fetch_info("%s connection established using %s",
+		    SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl));
 		conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
 		name = X509_get_subject_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
-		printf("Certificate subject: %s\n", str);
+		fetch_info("Certificate subject: %s", str);
-		free(str);
+		OPENSSL_free(str);
 		name = X509_get_issuer_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
-		printf("Certificate issuer: %s\n", str);
+		fetch_info("Certificate issuer: %s", str);
-		free(str);
+		OPENSSL_free(str);
 	}
 
 	return (0);
@@ -744,7 +1260,17 @@ fetch_close(conn_t *conn)
 #ifdef WITH_SSL
 	if (conn->ssl) {
 		SSL_shutdown(conn->ssl);
+		SSL_set_connect_state(conn->ssl);
 		SSL_free(conn->ssl);
+		conn->ssl = NULL;
+	}
+	if (conn->ssl_ctx) {
+		SSL_CTX_free(conn->ssl_ctx);
+		conn->ssl_ctx = NULL;
+	}
+	if (conn->ssl_cert) {
+		X509_free(conn->ssl_cert);
+		conn->ssl_cert = NULL;
 	}
 #endif
 	ret = close(conn->sd);

lib/fetch/common.h

@@ -1,6 +1,7 @@
+/*	$FreeBSD: rev 267133 $	*/
 /*	$NetBSD: common.h,v 1.23 2014/01/08 20:25:34 joerg Exp $	*/
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2014 Dag-Erling Smorgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,8 +26,6 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD: common.h,v 1.30 2007/12/18 11:03:07 des Exp $
  */
 
 #ifndef _COMMON_H_INCLUDED
@@ -106,6 +105,9 @@ conn_t		*fetch_cache_get(const struct url *, int);
 void		 fetch_cache_put(conn_t *, int (*)(conn_t *));
 conn_t		*fetch_connect(struct url *, int, int);
 conn_t		*fetch_reopen(int);
+#ifdef WITH_SSL
+int		fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*);
+#endif
 int		 fetch_ssl(conn_t *, const struct url *, int);
 ssize_t		 fetch_read(conn_t *, char *, size_t);
 int		 fetch_getln(conn_t *);
@@ -144,4 +146,8 @@ fetchIO		*ftp_request(struct url *, const char *, const char *,
  */
 #define CHECK_FLAG(x)	(flags && strchr(flags, (x)))
 
+#ifndef __UNCONST
+#define __UNCONST(a)    ((void *)(unsigned long)(const void *)(a))
+#endif
+
 #endif

lib/fetch/fetch.c

@@ -1,3 +1,4 @@
+/*	$FreeBSD: rev 252375 $ */
 /*	$NetBSD: fetch.c,v 1.19 2009/08/11 20:48:06 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
@@ -26,8 +27,6 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $
  */
 
 #include "compat.h"
@@ -292,6 +291,48 @@ fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
 	return (u);
 }
 
+/*
+ * Return value of the given hex digit.
+ */
+static int
+fetch_hexval(char ch)
+{
+	if (ch >= '0' && ch <= '9')
+		return (ch - '0');
+	else if (ch >= 'a' && ch <= 'f')
+		return (ch - 'a' + 10);
+	else if (ch >= 'A' && ch <= 'F')
+		return (ch - 'A' + 10);
+	return (-1);
+}
+
+/*
+ * Decode percent-encoded URL component from src into dst, stopping at end
+ * of string, or at @ or : separators.  Returns a pointer to the unhandled
+ * part of the input string (null terminator, @, or :).  No terminator is
+ * written to dst (it is the caller's responsibility).
+ */
+static const char *
+fetch_pctdecode(char *dst, const char *src, size_t dlen)
+{
+	int d1, d2;
+	char c;
+	const char *s;
+
+	for (s = src; *s != '\0' && *s != '@' && *s != ':'; s++) {
+		if (s[0] == '%' && (d1 = fetch_hexval(s[1])) >= 0 &&
+		    (d2 = fetch_hexval(s[2])) >= 0 && (d1 > 0 || d2 > 0)) {
+			c = d1 << 4 | d2;
+			s += 2;
+		} else {
+			c = *s;
+		}
+		if (dlen-- > 0)
+			*dst++ = c;
+	}
+	return (s);
+}
+
 int
 fetch_urlpath_safe(char x)
 {
@@ -426,17 +467,10 @@ find_user:
 	p = strpbrk(URL, "/@");
 	if (p != NULL && *p == '@') {
 		/* username */
-		for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) {
+		q = fetch_pctdecode(u->user, URL, URL_USERLEN);
-			if (i < URL_USERLEN)
-				u->user[i++] = *q;
-		}
-
 		/* password */
-		if (*q == ':') {
+		if (*q == ':')
-			for (q++, i = 0; (*q != '@'); q++)
+			q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN);
-				if (i < URL_PWDLEN)
-					u->pwd[i++] = *q;
-		}
 
 		p++;
 	} else {

lib/fetch/fetch.h

@@ -1,6 +1,6 @@
 /*	$NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $	*/
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2014 Dag-Erling Smorgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,8 +25,6 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD: fetch.h,v 1.26 2004/09/21 18:35:20 des Exp $
  */
 
 #ifndef _FETCH_H_INCLUDED
@@ -36,7 +34,7 @@
 #include <limits.h>
 #include <stdio.h>
 
-#define _LIBFETCH_VER "libfetch/2.0"
+#define _LIBFETCH_VER "xbps/2.0"
 
 #define URL_HOSTLEN 255
 #define URL_SCHEMELEN 16

lib/fetch/ftp.c

@@ -1,6 +1,6 @@
 /*	$NetBSD: ftp.c,v 1.46 2014/06/11 13:12:12 joerg Exp $	*/
 /*-
- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 1998-2004 Dag-Erling Smorgrav
  * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
  *
@@ -42,7 +42,7 @@
  *
  * Major Changelog:
  *
- * Dag-Erling CoýÅan Smgrav
+ * Dag-Erling Smograv
  * 9 Jun 1998
  *
  * Incorporated into libfetch

lib/fetch/http.c

@@ -1,6 +1,7 @@
+/*	$FreeBSD: rev 267127 $	*/
 /*	$NetBSD: http.c,v 1.37 2014/06/11 13:12:12 joerg Exp $	*/
 /*-
- * Copyright (c) 2000-2004 Dag-Erling CoýÅan Smgrav
+ * Copyright (c) 2000-2014 Dag-Erling Smorgrav
  * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
  * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
@@ -27,8 +28,6 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD: http.c,v 1.83 2008/02/06 11:39:55 des Exp $
  */
 
 /*
@@ -63,12 +62,7 @@
  * SUCH DAMAGE.
  */
 
-#if defined(__linux__) || defined(__MINT__) || defined(__FreeBSD_kernel__)
+#if defined(__linux__)
-/* Keep this down to Linux or MiNT, it can create surprises elsewhere. */
-/*
-   __FreeBSD_kernel__ is defined for GNU/kFreeBSD.
-   See http://glibc-bsd.alioth.debian.org/porting/PORTING .
-*/
 #define _GNU_SOURCE
 #endif
 
@@ -106,7 +100,9 @@
 #define HTTP_MOVED_TEMP		302
 #define HTTP_SEE_OTHER		303
 #define HTTP_NOT_MODIFIED	304
+#define HTTP_USE_PROXY		305
 #define HTTP_TEMP_REDIRECT	307
+#define HTTP_PERM_REDIRECT	308
 #define HTTP_NEED_AUTH		401
 #define HTTP_NEED_PROXY_AUTH	407
 #define HTTP_BAD_RANGE		416
@@ -115,6 +111,7 @@
 #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
 			    || (xyz) == HTTP_MOVED_TEMP \
 			    || (xyz) == HTTP_TEMP_REDIRECT \
+			    || (xyz) == HTTP_USE_PROXY \
 			    || (xyz) == HTTP_SEE_OTHER)
 
 #define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599)
@@ -517,6 +514,12 @@ http_parse_mtime(const char *p, time_t *mtime)
 	locale[sizeof(locale)-1] = '\0';
 	setlocale(LC_TIME, "C");
 	r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+	/*
+	 * Some proxies use UTC in response, but it should still be
+	 * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP.
+	 */
+	if (r == NULL)
+		r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm);
 	/* XXX should add support for date-2 and date-3 */
 	setlocale(LC_TIME, locale);
 	if (r == NULL)
@@ -698,6 +701,7 @@ http_authorize(conn_t *conn, const char *hdr, const char *p)
 static conn_t *
 http_connect(struct url *URL, struct url *purl, const char *flags, int *cached)
 {
+	struct url *curl;
 	conn_t *conn;
 	int af, verbose;
 #ifdef TCP_NOPUSH
@@ -718,22 +722,25 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached)
 		af = AF_INET6;
 #endif
 
-	if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) {
+	curl = (purl != NULL) ? purl : URL;
-		URL = purl;
-	} else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) {
-		/* can't talk http to an ftp server */
-		/* XXX should set an error code */
-		return (NULL);
-	}
 
-	if ((conn = fetch_cache_get(URL, af)) != NULL) {
+	if ((conn = fetch_cache_get(curl, af)) != NULL) {
 		*cached = 1;
 		return (conn);
 	}
 
-	if ((conn = fetch_connect(URL, af, verbose)) == NULL)
+	if ((conn = fetch_connect(curl, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
+	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
+		http_cmd(conn, "CONNECT %s:%d HTTP/1.1",
+		    URL->host, URL->port);
+		if (http_get_reply(conn) != HTTP_OK) {
+			fetch_close(conn);
+			return (NULL);
+		}
+		http_get_reply(conn);
+	}
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
 		fetch_close(conn);
@@ -888,7 +895,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		if (verbose)
 			fetch_info("requesting %s://%s%s",
 			    url->scheme, host, url->doc);
-		if (purl) {
+		if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) {
 			http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n",
 			    op, url->scheme, host, url->doc);
 		} else {
@@ -933,10 +940,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			else
 				http_cmd(conn, "Referer: %s\r\n", p);
 		}
-		if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')
+		if ((p = getenv("HTTP_USER_AGENT")) != NULL) {
-			http_cmd(conn, "User-Agent: %s\r\n", p);
+			/* no User-Agent if defined but empty */
-		else
+			if (*p != '\0')
+				http_cmd(conn, "User-Agent: %s\r\n", p);
+		} else {
+			/* default User-Agent */
 			http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER);
+		}
 
 		/*
 		 * Some servers returns 406 (Not Acceptable) if the Accept field is not
@@ -975,6 +986,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		case HTTP_MOVED_PERM:
 		case HTTP_MOVED_TEMP:
 		case HTTP_SEE_OTHER:
+		case HTTP_USE_PROXY:
 			/*
 			 * Not so fine, but we still have to read the
 			 * headers to get the new location.

lib/fetch/http.errors

@@ -1,5 +1,4 @@
-# $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $
+# $FreeBSD$ revision 241840
-# $NetBSD: http.errors,v 1.3 2009/02/05 16:59:45 joerg Exp $
 #
 # This list is taken from RFC 2068.
 #
@@ -19,6 +18,7 @@
 304 UNCHANGED	Not Modified
 305 INFO	Use Proxy
 307 MOVED	Temporary Redirect
+308 MOVED	Permanent Redirect
 400 PROTO	Bad Request
 401 AUTH	Unauthorized
 402 AUTH	Payment Required