Implemented support for working with remote repositories.

libfetch from NetBSD's pkgsrc has been imported into lib/fetch, but
the objects are embedded into libxbps. Only a public function to fetch
files has been implemented: xbps_fetch_file().

The library now is built with -fvisibility=hidden by default, and
exported symbols are the ones that use the SYMEXPORT macro.

The code works well enough, but will need many more cleanups.

--HG--
extra : convert_revision : xtraeme%40gmail.com-20091027004600-0lq9aao67lisbzxv

bin/Makefile

@@ -3,6 +3,7 @@ include ../vars.mk
 SUBDIRS =	xbps-bin
 SUBDIRS +=	xbps-cmpver
 SUBDIRS +=	xbps-digest
+SUBDIRS +=	xbps-fetch
 SUBDIRS +=	xbps-pkgdb
 SUBDIRS +=	xbps-repo
 

bin/xbps-bin/install.c

@@ -108,6 +108,91 @@ check_pkg_hashes(prop_object_iterator_t iter)
 	return 0;
 }
 
+static int
+download_package_list(prop_object_iterator_t iter)
+{
+	prop_object_t obj;
+	const char *pkgname, *version, *repoloc, *filename, *arch;
+	char *savedir, *binfile, *lbinfile, *repoloc_trans;
+	int rv = 0;
+
+	printf("Downloading binary package file(s)...\n");
+	while ((obj = prop_object_iterator_next(iter)) != NULL) {
+		prop_dictionary_get_cstring_nocopy(obj, "repository", &repoloc);
+		/*
+		 * Skip packages in local repositories.
+		 */
+		if ((strncmp(repoloc, "http://", 7)) &&
+		    (strncmp(repoloc, "ftp://", 6)))
+			continue;
+
+		prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
+		prop_dictionary_get_cstring_nocopy(obj, "version", &version);
+		prop_dictionary_get_cstring_nocopy(obj, "filename", &filename);
+		prop_dictionary_get_cstring_nocopy(obj, "architecture", &arch);
+
+		repoloc_trans = xbps_get_remote_repo_string(repoloc);
+		if (repoloc_trans == NULL)
+			return errno;
+
+		savedir = xbps_xasprintf("%s/%s/repo/%s/%s",
+		    xbps_get_rootdir(), XBPS_META_PATH, repoloc_trans, arch);
+		if (savedir == NULL) {
+			free(repoloc_trans);
+			return errno;
+		}
+
+		lbinfile = xbps_xasprintf("%s/%s", savedir, filename);
+		if (lbinfile == NULL) {
+			free(repoloc_trans);
+			free(savedir);
+			return errno;
+		}
+
+		if (access(lbinfile, R_OK) == 0) {
+			free(savedir);
+			free(lbinfile);
+			goto change_repodir;
+		}
+		free(lbinfile);
+
+		binfile = xbps_xasprintf("%s/%s/%s", repoloc, arch, filename);
+		if (binfile == NULL) {
+			free(repoloc_trans);
+			free(savedir);
+			return errno;
+		}
+		printf("Downloading %s-%s binary package ...\n",
+		    pkgname, version);
+		rv = xbps_fetch_file(binfile, savedir);
+		free(savedir);
+		free(binfile);
+		if (rv != 0) {
+			printf("Couldn't download %s from %s (%s)\n",
+			    filename, repoloc, strerror(rv));
+			free(repoloc_trans);
+			return errno;
+		}
+
+change_repodir:
+		/*
+		 * If it was downloaded successfully, override repository
+		 * path in transaction dictionary.
+		 */
+		savedir = xbps_xasprintf("%s/%s/repo/%s",
+		    xbps_get_rootdir(), XBPS_META_PATH, repoloc_trans);
+		free(repoloc_trans);
+		if (savedir == NULL)
+		       return errno;
+
+		prop_dictionary_set_cstring(obj, "repository", savedir);
+		free(savedir);
+	}
+	prop_object_iterator_reset(iter);
+
+	return 0;
+}
+
 static void
 show_package_list(prop_object_iterator_t iter, const char *match)
 {
@@ -409,6 +494,12 @@ exec_transaction(struct transaction *trans)
 		}
 	}
 
+	/*
+	 * Download binary packages if they are in a remote repository.
+	 */
+	if ((rv = download_package_list(trans->iter)) != 0)
+		return rv;
+
 	/*
 	 * Check the SHA256 hash for all required packages.
 	 */

bin/xbps-digest/main.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008 Juan Romero Pardines.
+ * Copyright (c) 2008-2009 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,6 @@
 #include <string.h>
 
 #include <xbps_api.h>
-#include "sha256.h"
 
 static void
 usage(void)
@@ -42,35 +41,21 @@ usage(void)
 int
 main(int argc, char **argv)
 {
-	SHA256_CTX ctx;
+	char *hash;
-	uint8_t buffer[BUFSIZ * 20], *digest;
+	int i;
-	ssize_t bytes;
-	int i, fd;
 
 	if (argc < 2)
 		usage();
 
 	for (i = 1; i < argc; i++) {
-		if ((fd = open(argv[i], O_RDONLY)) == -1) {
+		hash = xbps_get_file_hash(argv[i]);
-			printf("xbps-digest: cannot open %s (%s)\n", argv[i],
+		if (hash == NULL) {
-			    strerror(errno));
+			printf("Couldn't get hash for %s (%s)\n",
+			    argv[i], strerror(errno));
 			exit(EXIT_FAILURE);
 		}
-
+		printf("%s\n", hash);
-		digest = malloc(SHA256_DIGEST_STRING_LENGTH);
+		free(hash);
-		if (digest == NULL) {
-			printf("xbps-digest: malloc failed (%s)\n",
-			    strerror(errno));
-			exit(EXIT_FAILURE);
-		}
-
-		SHA256_Init(&ctx);
-		while ((bytes = read(fd, buffer, sizeof(buffer))) > 0)
-			SHA256_Update(&ctx, buffer, (size_t)bytes);
-
-		printf("%s\n", SHA256_End(&ctx, digest));
-		free(digest);
-		close(fd);
 	}
 
 	exit(EXIT_SUCCESS);

bin/xbps-fetch/Makefile

@@ -0,0 +1,6 @@
+TOPDIR = ../..
+include $(TOPDIR)/vars.mk
+
+BIN = xbps-fetch
+
+include $(TOPDIR)/prog.mk

bin/xbps-fetch/main.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <xbps_api.h>
+
+int
+main(int argc, char **argv)
+{
+	if (argc != 2) {
+		printf("Usage: xbps-fetch [options] URL\n");
+		exit(EXIT_FAILURE);
+	}
+
+	return xbps_fetch_file(argv[1], ".");
+}

bin/xbps-repo/index.c

@@ -63,8 +63,6 @@ repoidx_getdict(const char *pkgdir)
 
 		prop_dictionary_set(dict, "packages", array);
 		prop_object_release(array);
-		prop_dictionary_set_cstring_nocopy(dict,
-		    "location-local", pkgdir);
 		prop_dictionary_set_cstring_nocopy(dict,
 		    "pkgindex-version", XBPS_PKGINDEX_VERSION);
 	}

bin/xbps-repo/main.c

@@ -37,8 +37,6 @@
 
 typedef struct repository_info {
 	const char *index_version;
-	const char *location_local;
-	const char *location_remote;
 	uint64_t total_pkgs;
 } repo_info_t;
 
@@ -84,14 +82,6 @@ pkgindex_getinfo(prop_dictionary_t dict, repo_info_t *ri)
 	    "pkgindex-version", &ri->index_version))
 		return false;
 
-	if (!prop_dictionary_get_cstring_nocopy(dict,
-	    "location-local", &ri->location_local))
-		return false;
-
-	/* This one is optional, thus don't panic */
-	prop_dictionary_get_cstring_nocopy(dict, "location-remote",
-	    &ri->location_remote);
-
 	if (!prop_dictionary_get_uint64(dict, "total-pkgs",
 	    &ri->total_pkgs))
 		return false;
@@ -144,13 +134,79 @@ out:
 	return rv;
 }
 
+static int
+add_repository(const char *uri, bool remote)
+{
+	prop_dictionary_t dict;
+	repo_info_t *rinfo;
+	char *plist, idxstr[PATH_MAX];
+	int rv = 0;
+
+	if (remote) {
+		rv = xbps_sync_repository_pkg_index(uri);
+		if (rv != 0)
+			return rv;
+		plist = xbps_get_pkg_index_plist(uri);
+	} else {
+		if (!sanitize_localpath(idxstr, uri))
+			return errno;
+		plist = xbps_get_pkg_index_plist(idxstr);
+	}
+
+	if (plist == NULL)
+		return errno;
+
+	dict = prop_dictionary_internalize_from_file(plist);
+	if (dict == NULL) {
+		printf("Repository %s does not contain any "
+		    "xbps pkgindex file.\n", idxstr);
+		rv = errno;
+		goto out;
+	}
+
+	rinfo = malloc(sizeof(*rinfo));
+	if (rinfo == NULL) {
+		rv = errno;
+		goto out;
+	}
+
+	if (!pkgindex_getinfo(dict, rinfo)) {
+		printf("'%s' is incomplete.\n", plist);
+		rv = EINVAL;
+		goto out;
+	}
+
+	if (remote)
+		rv = xbps_register_repository(uri);
+	else
+		rv = xbps_register_repository(idxstr);
+
+	if (rv != 0) {
+		printf("ERROR: couldn't register repository (%s)\n",
+		    strerror(rv));
+		goto out;
+	}
+	
+	printf("Added repository at %s (%s) with %ju packages.\n",
+	       uri, rinfo->index_version, rinfo->total_pkgs);
+
+out:
+	if (dict != NULL)
+		prop_object_release(dict);
+	if (rinfo != NULL)
+		free(rinfo);
+	if (plist != NULL)
+		free(plist);
+
+	return rv;
+}
+
 int
 main(int argc, char **argv)
 {
-	prop_dictionary_t dict;
+	char dpkgidx[PATH_MAX], *root = NULL;
-	repo_info_t *rinfo = NULL;
-	char dpkgidx[PATH_MAX], *plist, *root = NULL;
 	int c, rv = 0;
+	bool remote_repo = false;
 
 	while ((c = getopt(argc, argv, "Vr:")) != -1) {
 		switch (c) {
@@ -179,53 +235,11 @@ main(int argc, char **argv)
 		if (argc != 2)
 			usage();
 
-		if (!sanitize_localpath(dpkgidx, argv[1]))
+		if ((strncmp(argv[1], "http://", 7) == 0) ||
-			exit(EXIT_FAILURE);
+		    (strncmp(argv[1], "ftp://", 6) == 0))
+			remote_repo = true;
 
-		/* Temp buffer to verify pkgindex file. */
+		rv = add_repository(argv[1], remote_repo);
-		plist = xbps_get_pkg_index_plist(dpkgidx);
-		if (plist == NULL)
-			exit(EXIT_FAILURE);
-
-		dict = prop_dictionary_internalize_from_file(plist);
-		if (dict == NULL) {
-			printf("Directory %s does not contain any "
-			    "xbps pkgindex file.\n", dpkgidx);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		rinfo = malloc(sizeof(*rinfo));
-		if (rinfo == NULL) {
-			prop_object_release(dict);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		if (!pkgindex_getinfo(dict, rinfo)) {
-			printf("'%s' is incomplete.\n", plist);
-			prop_object_release(dict);
-			free(rinfo);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		if ((rv = xbps_register_repository(dpkgidx)) != 0) {
-			printf("ERROR: couldn't register repository (%s)\n",
-			    strerror(rv));
-			prop_object_release(dict);
-			free(rinfo);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		printf("Added repository at %s (%s) with %ju packages.\n",
-		       rinfo->location_local, rinfo->index_version,
-		       rinfo->total_pkgs);
-
-		prop_object_release(dict);
-		free(rinfo);
-		free(plist);
 
 	} else if (strcasecmp(argv[0], "list") == 0) {
 		/* Lists all repositories registered in pool. */

bin/xbps-repo/util.c

@@ -248,10 +248,8 @@ int
 show_pkg_info_from_repolist(prop_object_t obj, void *arg, bool *loop_done)
 {
 	prop_dictionary_t dict, pkgdict;
-	prop_string_t oloc;
+	const char *repofile;
-	const char *repofile, *repoloc;
 	char *plist;
-
 	assert(prop_object_type(obj) == PROP_TYPE_STRING);
 
 	/* Get the location */
@@ -276,19 +274,7 @@ show_pkg_info_from_repolist(prop_object_t obj, void *arg, bool *loop_done)
 		return 0;
 	}
 
-	oloc = prop_dictionary_get(dict, "location-remote");
+	printf("Repository: %s\n", repofile);
-	if (oloc == NULL)
-		oloc = prop_dictionary_get(dict, "location-local");
-
-	if (oloc && prop_object_type(oloc) == PROP_TYPE_STRING)
-		repoloc = prop_string_cstring_nocopy(oloc);
-	else {
-		prop_object_release(dict);
-		free(plist);
-		return EINVAL;
-	}
-
-	printf("Repository: %s\n", repoloc);
 	show_pkg_info(pkgdict);
 	*loop_done = true;
 	prop_object_release(dict);

doc/BINPKG_REPOSITORY

@@ -11,13 +11,8 @@ a "dictionary per package". Additional objects are added into the
 main dictionary to specify more info, like:
 
 	- pkgindex-version: version used to build the index.
-	- location-local: local path to the repository.
-	- location-remote: remote URI repository.
 	- total-pkgs: total of number of available packages.
 
-"location-local" will always be created, and it might be exported via
-a remote location specified with "location-remote".
-
 The package dictionary will be the same than the one available in
 package's metadata directory "/var/db/xbps/metadata/$pkgname/props.plist",
 but some additional objects are added to provide enough info for
@@ -32,10 +27,6 @@ Here's how the package index plist file shall look like in a repository:
 <dict>
 	<key>pkgindex-version</key>
 	<string>1.0</string>
-	<key>location-local</key>
-	<string>/xbps/repo/local</string>
-	<key>location-remote</key>
-	<string>http://www.xbps-remote.org/repo/public</string>
 	<key>total-pkgs</key>
 	<integer>666</integer>
 	<key>available-packages</key>

include/fetch.h

@@ -0,0 +1,183 @@
+/*	$NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $	*/
+/*-
+ * Copyright (c) 1998-2004 Dag-Erling Coïdn Smorgrav
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD: fetch.h,v 1.26 2004/09/21 18:35:20 des Exp $
+ */
+
+#ifndef _FETCH_H_INCLUDED
+#define _FETCH_H_INCLUDED
+
+#include <sys/types.h>
+#include <limits.h>
+#include <stdio.h>
+
+#define _LIBFETCH_VER "libfetch/2.0"
+
+#define URL_HOSTLEN 255
+#define URL_SCHEMELEN 16
+#define URL_USERLEN 256
+#define URL_PWDLEN 256
+
+typedef struct fetchIO fetchIO;
+
+struct url {
+	char		 scheme[URL_SCHEMELEN + 1];
+	char		 user[URL_USERLEN + 1];
+	char		 pwd[URL_PWDLEN + 1];
+	char		 host[URL_HOSTLEN + 1];
+	int		 port;
+	char		*doc;
+	off_t		 offset;
+	size_t		 length;
+	time_t		 last_modified;
+};
+
+struct url_stat {
+	off_t		 size;
+	time_t		 atime;
+	time_t		 mtime;
+};
+
+struct url_list {
+	size_t		 length;
+	size_t		 alloc_size;
+	struct url	*urls;
+};
+
+/* Recognized schemes */
+#define SCHEME_FTP	"ftp"
+#define SCHEME_HTTP	"http"
+#define SCHEME_HTTPS	"https"
+#define SCHEME_FILE	"file"
+
+/* Error codes */
+#define	FETCH_ABORT	 1
+#define	FETCH_AUTH	 2
+#define	FETCH_DOWN	 3
+#define	FETCH_EXISTS	 4
+#define	FETCH_FULL	 5
+#define	FETCH_INFO	 6
+#define	FETCH_MEMORY	 7
+#define	FETCH_MOVED	 8
+#define	FETCH_NETWORK	 9
+#define	FETCH_OK	10
+#define	FETCH_PROTO	11
+#define	FETCH_RESOLV	12
+#define	FETCH_SERVER	13
+#define	FETCH_TEMP	14
+#define	FETCH_TIMEOUT	15
+#define	FETCH_UNAVAIL	16
+#define	FETCH_UNKNOWN	17
+#define	FETCH_URL	18
+#define	FETCH_VERBOSE	19
+#define	FETCH_UNCHANGED	20
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+void		fetchIO_close(fetchIO *);
+ssize_t		fetchIO_read(fetchIO *, void *, size_t);
+ssize_t		fetchIO_write(fetchIO *, const void *, size_t);
+
+/* fetchIO-specific functions */
+fetchIO		*fetchXGetFile(struct url *, struct url_stat *, const char *);
+fetchIO		*fetchGetFile(struct url *, const char *);
+fetchIO		*fetchPutFile(struct url *, const char *);
+int		 fetchStatFile(struct url *, struct url_stat *, const char *);
+int		 fetchListFile(struct url_list *, struct url *, const char *,
+		    const char *);
+
+/* HTTP-specific functions */
+fetchIO		*fetchXGetHTTP(struct url *, struct url_stat *, const char *);
+fetchIO		*fetchGetHTTP(struct url *, const char *);
+fetchIO		*fetchPutHTTP(struct url *, const char *);
+int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
+int		 fetchListHTTP(struct url_list *, struct url *, const char *,
+		    const char *);
+
+/* FTP-specific functions */
+fetchIO		*fetchXGetFTP(struct url *, struct url_stat *, const char *);
+fetchIO		*fetchGetFTP(struct url *, const char *);
+fetchIO		*fetchPutFTP(struct url *, const char *);
+int		 fetchStatFTP(struct url *, struct url_stat *, const char *);
+int		 fetchListFTP(struct url_list *, struct url *, const char *,
+		    const char *);
+
+/* Generic functions */
+fetchIO		*fetchXGetURL(const char *, struct url_stat *, const char *);
+fetchIO		*fetchGetURL(const char *, const char *);
+fetchIO		*fetchPutURL(const char *, const char *);
+int		 fetchStatURL(const char *, struct url_stat *, const char *);
+int		 fetchListURL(struct url_list *, const char *, const char *,
+		    const char *);
+fetchIO		*fetchXGet(struct url *, struct url_stat *, const char *);
+fetchIO		*fetchGet(struct url *, const char *);
+fetchIO		*fetchPut(struct url *, const char *);
+int		 fetchStat(struct url *, struct url_stat *, const char *);
+int		 fetchList(struct url_list *, struct url *, const char *,
+		    const char *);
+
+/* URL parsing */
+struct url	*fetchMakeURL(const char *, const char *, int,
+		     const char *, const char *, const char *);
+struct url	*fetchParseURL(const char *);
+struct url	*fetchCopyURL(const struct url *);
+char		*fetchStringifyURL(const struct url *);
+void		 fetchFreeURL(struct url *);
+
+/* URL listening */
+void		 fetchInitURLList(struct url_list *);
+int		 fetchAppendURLList(struct url_list *, const struct url_list *);
+void		 fetchFreeURLList(struct url_list *);
+char		*fetchUnquotePath(struct url *);
+char		*fetchUnquoteFilename(struct url *);
+
+/* Authentication */
+typedef int (*auth_t)(struct url *);
+extern auth_t		 fetchAuthMethod;
+
+/* Last error code */
+extern int		 fetchLastErrCode;
+#define MAXERRSTRING 256
+extern char		 fetchLastErrString[MAXERRSTRING];
+
+/* I/O timeout */
+extern int		 fetchTimeout;
+
+/* Restart interrupted syscalls */
+extern volatile int	 fetchRestartCalls;
+
+/* Extra verbosity */
+extern int		 fetchDebug;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif

include/xbps_api.h

@@ -68,17 +68,29 @@
 #define __UNCONST(a)	((void *)(unsigned long)(const void *)(a))
 #endif
 
+#if __GNUC__ >= 4
+#define SYMEXPORT __attribute__ ((visibility("default")))
+#else
+#define SYMEXPORT
+#endif
+
 /* From lib/configure.c */
-int		xbps_configure_pkg(const char *, const char *, bool);
+int SYMEXPORT	xbps_configure_pkg(const char *, const char *, bool);
-int		xbps_configure_all_pkgs(void);
+int SYMEXPORT	xbps_configure_all_pkgs(void);
 
 /* from lib/cmpver.c */
-int		xbps_cmpver(const char *, const char *);
+int SYMEXPORT	xbps_cmpver(const char *, const char *);
+
+/* From lib/download.c */
+int SYMEXPORT	xbps_fetch_file(const char *, const char *);
+void SYMEXPORT	(*xbps_fetch_start_cb)(const char *, off_t *, off_t *);
+void SYMEXPORT	(*xbps_fetch_update_cb)(off_t *);
+void SYMEXPORT	(*xbps_fetch_end_cb)(void);
 
 /* From lib/fexec.c */
-int		xbps_file_exec(const char *, ...);
+int SYMEXPORT	xbps_file_exec(const char *, ...);
-int		xbps_file_exec_skipempty(const char *, ...);
+int SYMEXPORT	xbps_file_exec_skipempty(const char *, ...);
-int		xbps_file_chdir_exec(const char *, const char *, ...);
+int SYMEXPORT	xbps_file_chdir_exec(const char *, const char *, ...);
 
 /* From lib/humanize_number.c */
 #define HN_DECIMAL		0x01
@@ -88,85 +100,89 @@ int		xbps_file_chdir_exec(const char *, const char *, ...);
 #define HN_GETSCALE		0x10
 #define HN_AUTOSCALE		0x20
 
-int		xbps_humanize_number(char *, size_t, int64_t, const char *,
+int SYMEXPORT	xbps_humanize_number(char *, size_t, int64_t, const char *,
 				     int, int);
 
 /* From lib/findpkg.c */
 struct repository_data {
 	SIMPLEQ_ENTRY(repository_data) chain;
 	prop_dictionary_t rd_repod;
+	char *rd_uri;
 };
-SIMPLEQ_HEAD(, repository_data) repodata_queue;
+SYMEXPORT SIMPLEQ_HEAD(, repository_data) repodata_queue;
 
-int		xbps_prepare_pkg(const char *);
+int SYMEXPORT			xbps_prepare_pkg(const char *);
-int		xbps_find_new_pkg(const char *, prop_dictionary_t);
+int SYMEXPORT			xbps_find_new_pkg(const char *, prop_dictionary_t);
-int		xbps_find_new_packages(void);
+int SYMEXPORT			xbps_find_new_packages(void);
-int		xbps_prepare_repolist_data(void);
+int SYMEXPORT			xbps_prepare_repolist_data(void);
-void		xbps_release_repolist_data(void);
+void SYMEXPORT			xbps_release_repolist_data(void);
-prop_dictionary_t	xbps_get_pkg_props(void);
+prop_dictionary_t SYMEXPORT	xbps_get_pkg_props(void);
 
 /* From lib/depends.c */
-int		xbps_find_deps_in_pkg(prop_dictionary_t, prop_dictionary_t);
+int SYMEXPORT	xbps_find_deps_in_pkg(prop_dictionary_t, prop_dictionary_t);
 
 /* From lib/orphans.c */
-prop_array_t	xbps_find_orphan_packages(void);
+prop_array_t SYMEXPORT		xbps_find_orphan_packages(void);
 
 /* From lib/plist.c */
-bool		xbps_add_obj_to_dict(prop_dictionary_t, prop_object_t,
+bool SYMEXPORT	xbps_add_obj_to_dict(prop_dictionary_t, prop_object_t,
 				     const char *);
-bool		xbps_add_obj_to_array(prop_array_t, prop_object_t);
+bool SYMEXPORT	xbps_add_obj_to_array(prop_array_t, prop_object_t);
 
-int 		xbps_callback_array_iter_in_dict(prop_dictionary_t,
+int SYMEXPORT	xbps_callback_array_iter_in_dict(prop_dictionary_t,
 			const char *,
 			int (*fn)(prop_object_t, void *, bool *),
 			void *);
-int		xbps_callback_array_iter_reverse_in_dict(prop_dictionary_t,
+int SYMEXPORT	xbps_callback_array_iter_reverse_in_dict(prop_dictionary_t,
 			const char *,
 			int (*fn)(prop_object_t, void *, bool *),
 			void *);
-int 		xbps_callback_array_iter_in_repolist(int (*fn)(prop_object_t,
+int SYMEXPORT	xbps_callback_array_iter_in_repolist(int (*fn)(prop_object_t,
 			void *, bool *), void *);
 
-prop_dictionary_t	xbps_find_pkg_in_dict(prop_dictionary_t,
+prop_dictionary_t SYMEXPORT	xbps_find_pkg_in_dict(prop_dictionary_t,
 					      const char *, const char *);
-prop_dictionary_t	xbps_find_pkg_from_plist(const char *, const char *);
+prop_dictionary_t SYMEXPORT	xbps_find_pkg_from_plist(const char *,
-prop_dictionary_t 	xbps_find_pkg_installed_from_plist(const char *);
+						const char *);
-bool 		xbps_find_string_in_array(prop_array_t, const char *);
+prop_dictionary_t SYMEXPORT	xbps_find_pkg_installed_from_plist(const char *);
+bool SYMEXPORT	xbps_find_string_in_array(prop_array_t, const char *);
 
-prop_dictionary_t	xbps_prepare_regpkgdb_dict(void);
+prop_dictionary_t SYMEXPORT	xbps_prepare_regpkgdb_dict(void);
-void			xbps_release_regpkgdb_dict(void);
+void SYMEXPORT			xbps_release_regpkgdb_dict(void);
-prop_object_iterator_t	xbps_get_array_iter_from_dict(prop_dictionary_t,
+prop_object_iterator_t	SYMEXPORT
-						      const char *);
+	xbps_get_array_iter_from_dict(prop_dictionary_t, const char *);
 
-prop_dictionary_t	xbps_read_dict_from_archive_entry(struct archive *,
+prop_dictionary_t SYMEXPORT	xbps_read_dict_from_archive_entry(struct archive *,
-							  struct archive_entry *);
+					struct archive_entry *);
 
-int 		xbps_remove_pkg_dict_from_file(const char *, const char *);
+int SYMEXPORT	xbps_remove_pkg_dict_from_file(const char *, const char *);
-int		xbps_remove_pkg_from_dict(prop_dictionary_t, const char *,
+int SYMEXPORT	xbps_remove_pkg_from_dict(prop_dictionary_t, const char *,
 					  const char *);
-int		xbps_remove_string_from_array(prop_array_t, const char *);
+int SYMEXPORT	xbps_remove_string_from_array(prop_array_t, const char *);
 
 /* From lib/purge.c */
-int		xbps_purge_pkg(const char *, bool);
+int SYMEXPORT	xbps_purge_pkg(const char *, bool);
-int		xbps_purge_all_pkgs(void);
+int SYMEXPORT	xbps_purge_all_pkgs(void);
 
 /* From lib/register.c */
-int		xbps_register_pkg(prop_dictionary_t, bool);
+int SYMEXPORT	xbps_register_pkg(prop_dictionary_t, bool);
-int		xbps_unregister_pkg(const char *);
+int SYMEXPORT	xbps_unregister_pkg(const char *);
 
 /* From lib/remove.c */
-int		xbps_remove_pkg(const char *, const char *, bool);
+int SYMEXPORT	xbps_remove_pkg(const char *, const char *, bool);
 
 /* From lib/repository.c */
-int		xbps_register_repository(const char *);
+int SYMEXPORT	xbps_register_repository(const char *);
-int		xbps_unregister_repository(const char *);
+int SYMEXPORT	xbps_unregister_repository(const char *);
+int SYMEXPORT	xbps_sync_repository_pkg_index(const char *);
+char SYMEXPORT	*xbps_get_remote_repo_string(const char *);
 
 /* From lib/requiredby.c */
-int		xbps_requiredby_pkg_add(prop_array_t, prop_dictionary_t);
+int SYMEXPORT	xbps_requiredby_pkg_add(prop_array_t, prop_dictionary_t);
-int		xbps_requiredby_pkg_remove(const char *);
+int SYMEXPORT	xbps_requiredby_pkg_remove(const char *);
 
 /* From lib/sortdeps.c */
-int		xbps_sort_pkg_deps(prop_dictionary_t);
+int SYMEXPORT	xbps_sort_pkg_deps(prop_dictionary_t);
 
 /* From lib/state.c */
 typedef enum pkg_state {
@@ -176,31 +192,31 @@ typedef enum pkg_state {
 	XBPS_PKG_STATE_CONFIG_FILES,
 	XBPS_PKG_STATE_NOT_INSTALLED
 } pkg_state_t;
-int		xbps_get_pkg_state_installed(const char *, pkg_state_t *);
+int SYMEXPORT	xbps_get_pkg_state_installed(const char *, pkg_state_t *);
-int		xbps_get_pkg_state_dictionary(prop_dictionary_t, pkg_state_t *);
+int SYMEXPORT	xbps_get_pkg_state_dictionary(prop_dictionary_t, pkg_state_t *);
-int		xbps_set_pkg_state_installed(const char *, pkg_state_t);
+int SYMEXPORT	xbps_set_pkg_state_installed(const char *, pkg_state_t);
-int		xbps_set_pkg_state_dictionary(prop_dictionary_t, pkg_state_t);
+int SYMEXPORT	xbps_set_pkg_state_dictionary(prop_dictionary_t, pkg_state_t);
 
 /* From lib/unpack.c */
-int		xbps_unpack_binary_pkg(prop_dictionary_t, bool);
+int SYMEXPORT	xbps_unpack_binary_pkg(prop_dictionary_t, bool);
 
 /* From lib/util.c */
-char *		xbps_xasprintf(const char *, ...);
+char SYMEXPORT		*xbps_xasprintf(const char *, ...);
-char *		xbps_get_file_hash(const char *);
+char SYMEXPORT		*xbps_get_file_hash(const char *);
-int		xbps_check_file_hash(const char *, const char *);
+int SYMEXPORT		xbps_check_file_hash(const char *, const char *);
-int		xbps_check_pkg_file_hash(prop_dictionary_t, const char *);
+int SYMEXPORT		xbps_check_pkg_file_hash(prop_dictionary_t, const char *);
-int		xbps_check_is_installed_pkg(const char *);
+int SYMEXPORT		xbps_check_is_installed_pkg(const char *);
-bool		xbps_check_is_installed_pkgname(const char *);
+bool SYMEXPORT		xbps_check_is_installed_pkgname(const char *);
-char *		xbps_get_pkg_index_plist(const char *);
+char SYMEXPORT		*xbps_get_pkg_index_plist(const char *);
-char *		xbps_get_pkg_name(const char *);
+char SYMEXPORT		*xbps_get_pkg_name(const char *);
-const char *	xbps_get_pkg_version(const char *);
+const char SYMEXPORT	*xbps_get_pkg_version(const char *);
-const char *	xbps_get_pkg_revision(const char *);
+const char SYMEXPORT	*xbps_get_pkg_revision(const char *);
-bool		xbps_pkg_has_rundeps(prop_dictionary_t);
+bool SYMEXPORT		xbps_pkg_has_rundeps(prop_dictionary_t);
-void		xbps_set_rootdir(const char *);
+void SYMEXPORT		xbps_set_rootdir(const char *);
-const char *	xbps_get_rootdir(void);
+const char SYMEXPORT	*xbps_get_rootdir(void);
-void		xbps_set_flags(int);
+void SYMEXPORT		xbps_set_flags(int);
-int		xbps_get_flags(void);
+int SYMEXPORT		xbps_get_flags(void);
-bool		xbps_yesno(const char *, ...);
+bool SYMEXPORT		xbps_yesno(const char *, ...);
-bool		xbps_noyes(const char *, ...);
+bool SYMEXPORT		xbps_noyes(const char *, ...);
 
 #endif /* !_XBPS_API_H_ */

lib/Makefile

@@ -1,49 +1,65 @@
 include ../vars.mk
 
-# The shared library.
+INET6	?= yes
-MAJOR		= 0
+SSL	?= yes
-MINOR		= 0
+LIBS	= -larchive -lprop
-MICRO		= 0
-LIBXBPS_SO	= $(LIBXBPS).$(MAJOR).$(MINOR).$(MICRO)
-LIBXBPS		= libxbps.so
-LIBXBPS_STATIC 	= libxbps.a
-LIBXBPS_LDFLAGS	= -larchive -lprop -shared -Wl,-soname,$(LIBXBPS).$(MAJOR)
 
-OBJECTS	=	configure.o cmpver.o depends.o fexec.o findpkg.o
+ifdef SSL
-OBJECTS	+=	humanize_number.o orphans.o plist.o purge.o register.o remove.o
+CPPFLAGS += -DWITH_SSL
-OBJECTS	+=	repository.o requiredby.o sha256.o sortdeps.o state.o unpack.o
+LIBS += -lssl -lcrypto
-OBJECTS	+=	util.o
+endif
 
-all: $(LIBXBPS) $(LIBXBPS_STATIC)
+ifdef INET6
+CPPFLAGS += -DINET6
+endif
+
+ifdef DEBUG
+CPPFLAGS += -DDEBUG
+endif
+
+LIBMAJOR	= 0
+LIBMINOR	= 0
+LIBMICRO	= 0
+LIBXBPS_SHLIB	= libxbps.so.$(LIBMAJOR).$(LIBMINOR).$(LIBMICRO)
+LIBXBPS_LDFLAGS	= $(LIBS) -shared -Wl,-soname,libxbps.so.$(LIBMAJOR)
+
+# libfetch
+OBJS = fetch/common.o fetch/fetch.o fetch/file.o fetch/ftp.o fetch/http.o
+
+# libxbps
+OBJS +=	configure.o cmpver.o depends.o download.o fexec.o findpkg.o
+OBJS +=	humanize_number.o orphans.o plist.o purge.o register.o remove.o
+OBJS +=	repository.o requiredby.o sha256.o sortdeps.o state.o unpack.o
+OBJS +=	util.o
+
+all: libfetch libxbps.so libxbps.a
 .PHONY: all
 
-$(LIBXBPS): $(OBJECTS)
+libfetch:
-	$(CC) $(LIBXBPS_LDFLAGS) $^ -o $(LIBXBPS_SO)
+	$(MAKE) -C fetch
-	-ln -sf $(LIBXBPS_SO) $(LIBXBPS).$(MAJOR)
-	-ln -sf $(LIBXBPS_SO) $(LIBXBPS)
 
-$(LIBXBPS_STATIC): $(OBJECTS)
+libxbps.so: $(OBJS)
+	$(CC) $(LIBXBPS_LDFLAGS) $^ -o $(LIBXBPS_SHLIB)
+	-ln -sf $(LIBXBPS_SHLIB) libxbps.so.$(LIBMAJOR)
+	-ln -sf $(LIBXBPS_SHLIB) libxbps.so
+
+libxbps.a: $(OBJS)
 	$(AR) rcs $@ $^
 	ranlib $@
 
 .PHONY: install
-install: $(LIBXBPS) $(LIBXBPS_STATIC)
+install: all
 	install -d $(LIBDIR)
-	install -m 644 $(LIBXBPS_STATIC) $(LIBDIR)
+	install -m 644 libxbps.a $(LIBDIR)
-	install -m 644 $(LIBXBPS_SO) $(LIBDIR)
+	install -m 644 $(LIBXBPS_SHLIB) $(LIBDIR)
-	cp -a $(LIBXBPS) $(LIBDIR)
+	cp -a libxbps.so $(LIBDIR)
-	cp -a $(LIBXBPS).$(MAJOR) $(LIBDIR)
+	cp -a libxbps.so.$(LIBMAJOR) $(LIBDIR)
 
 .PHONY: uninstall
 uninstall:
 	-rm -f $(LIBDIR)/libxbps.*
 
 .PHONY: clean
-clean: clean-lib clean-objs
+clean:
-
+	-rm -f libxbps* $(OBJS)
-clean-lib:
+	$(MAKE) -C fetch clean
-	-rm -f $(LIBXBPS)*
-	-rm -f $(LIBXBPS_STATIC)
-
-clean-objs:
-	-rm -f *.o

lib/cmpver.c

@@ -223,7 +223,7 @@ get_component(const char *position, version_component *component)
  * of the version should conform to the porting guidelines. It can contain
  * multiple components, separated by a period, including letters.
  */
-int
+int SYMEXPORT
 xbps_cmpver(const char *pkg1, const char *pkg2)
 {
     const char *v1, *v2, *ve1, *ve2;

lib/configure.c

@@ -33,7 +33,7 @@
 /*
  * Configure all packages currently in unpacked state.
  */
-int
+int SYMEXPORT
 xbps_configure_all_pkgs(void)
 {
 	prop_dictionary_t d;
@@ -71,7 +71,7 @@ xbps_configure_all_pkgs(void)
  * post INSTALL action if required and updates package state to
  * to installed.
  */
-int
+int SYMEXPORT
 xbps_configure_pkg(const char *pkgname, const char *version, bool check_state)
 {
 	prop_dictionary_t pkgd;

lib/depends.c

@@ -31,27 +31,28 @@
 #include <xbps_api.h>
 
 static int add_missing_reqdep(prop_dictionary_t, const char *, const char *);
-static int find_repo_deps(prop_dictionary_t, prop_dictionary_t, prop_array_t);
+static int find_repo_deps(prop_dictionary_t, prop_dictionary_t,
-static int find_repo_missing_deps(prop_dictionary_t, prop_dictionary_t);
+			  const char *, prop_array_t);
+static int find_repo_missing_deps(prop_dictionary_t, prop_dictionary_t,
+				  const char *);
 
 static int
 store_dependency(prop_dictionary_t master, prop_dictionary_t depd,
-		 prop_dictionary_t repod)
+		 const char *repoloc)
 {
 	prop_dictionary_t dict;
 	prop_array_t array;
-	const char *repoloc, *pkgname;
+	const char *pkgname;
 	int rv = 0;
 	pkg_state_t state = 0;
 
 	assert(master != NULL);
 	assert(depd != NULL);
-	assert(repod != NULL);
+	assert(repoloc != NULL);
 	/*
 	 * Get some info about dependencies and current repository.
 	 */
 	prop_dictionary_get_cstring_nocopy(depd, "pkgname", &pkgname);
-	prop_dictionary_get_cstring_nocopy(repod, "location-local", &repoloc);
 
 	dict = prop_dictionary_copy(depd);
 	if (dict == NULL)
@@ -136,7 +137,7 @@ add_missing_reqdep(prop_dictionary_t master, const char *pkgname,
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_find_deps_in_pkg(prop_dictionary_t master, prop_dictionary_t pkg)
 {
 	prop_array_t pkg_rdeps, missing_rdeps;
@@ -161,7 +162,8 @@ xbps_find_deps_in_pkg(prop_dictionary_t master, prop_dictionary_t pkg)
 		 * if any of them is not there it will be added
 		 * into the missing_deps array.
 		 */
-		rv = find_repo_deps(master, rdata->rd_repod, pkg_rdeps);
+		rv = find_repo_deps(master, rdata->rd_repod,
+		    rdata->rd_uri, pkg_rdeps);
 		if (rv != 0) {
 			if (rv == ENOENT) {
 				rv = 0;
@@ -181,7 +183,8 @@ xbps_find_deps_in_pkg(prop_dictionary_t master, prop_dictionary_t pkg)
 	 * just in case that indirect deps weren't found.
 	 */
 	SIMPLEQ_FOREACH(rdata, &repodata_queue, chain) {
-		rv = find_repo_missing_deps(master, rdata->rd_repod);
+		rv = find_repo_missing_deps(master, rdata->rd_repod,
+		    rdata->rd_uri);
 		if (rv != 0 && rv != ENOENT)
 			return rv;
 	}
@@ -193,7 +196,8 @@ xbps_find_deps_in_pkg(prop_dictionary_t master, prop_dictionary_t pkg)
 }
 
 static int
-find_repo_missing_deps(prop_dictionary_t master, prop_dictionary_t repo)
+find_repo_missing_deps(prop_dictionary_t master, prop_dictionary_t repo,
+		       const char *repoloc)
 {
 	prop_array_t array;
 	prop_dictionary_t curpkgd;
@@ -233,7 +237,7 @@ find_repo_missing_deps(prop_dictionary_t master, prop_dictionary_t repo)
 		/*
 		 * Package is on repo, add it into the dictionary.
 		 */
-		if ((rv = store_dependency(master, curpkgd, repo)) != 0)
+		if ((rv = store_dependency(master, curpkgd, repoloc)) != 0)
 			break;
 		/*
 		 * Remove package from missing_deps array now.
@@ -252,7 +256,7 @@ find_repo_missing_deps(prop_dictionary_t master, prop_dictionary_t repo)
 
 static int
 find_repo_deps(prop_dictionary_t master, prop_dictionary_t repo,
-	       prop_array_t pkg_rdeps)
+	       const char *repoloc, prop_array_t pkg_rdeps)
 {
 	prop_dictionary_t curpkgd, tmpd = NULL;
 	prop_array_t curpkg_rdeps;
@@ -321,7 +325,7 @@ find_repo_deps(prop_dictionary_t master, prop_dictionary_t repo,
 		/*
 		 * Package is on repo, add it into the dictionary.
 		 */
-		if ((rv = store_dependency(master, curpkgd, repo)) != 0) {
+		if ((rv = store_dependency(master, curpkgd, repoloc)) != 0) {
 			free(pkgname);
 			break;
 		}
@@ -344,7 +348,7 @@ find_repo_deps(prop_dictionary_t master, prop_dictionary_t repo,
 		/*
 		 * Iterate on required pkg to find more deps.
 		 */
-		if (!find_repo_deps(master, repo, curpkg_rdeps))
+		if (!find_repo_deps(master, repo, repoloc, curpkg_rdeps))
 			continue;
 	}
 	prop_object_iterator_release(iter);

lib/download.c

@@ -0,0 +1,300 @@
+/*-
+ * Copyright (c) 2009 Juan Romero Pardines
+ * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * From FreeBSD fetch(8):
+ * $FreeBSD: src/usr.bin/fetch/fetch.c,v 1.84.2.1 2009/08/03 08:13:06 kensmith Exp $
+ */
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <xbps_api.h>
+#include "fetch.h"
+
+struct xferstat {
+	struct timeval	 start;
+	struct timeval	 last;
+	off_t		 size;
+	off_t		 offset;
+	off_t		 rcvd;
+	const char 	 *name;
+};
+
+/*
+ * Compute and display ETA
+ */
+static const char *
+stat_eta(struct xferstat *xsp)
+{
+	static char str[16];
+	long elapsed, eta;
+	off_t received, expected;
+
+	elapsed = xsp->last.tv_sec - xsp->start.tv_sec;
+	received = xsp->rcvd - xsp->offset;
+	expected = xsp->size - xsp->rcvd;
+	eta = (long)(elapsed * expected / received);
+	if (eta > 3600)
+		snprintf(str, sizeof str, "%02ldh%02ldm",
+		    eta / 3600, (eta % 3600) / 60);
+	else
+		snprintf(str, sizeof str, "%02ldm%02lds",
+		    eta / 60, eta % 60);
+	return str;
+}
+
+/*
+ * Compute and display transfer rate
+ */
+static const char *
+stat_bps(struct xferstat *xsp)
+{
+	static char str[16];
+	char size[32];
+	double delta, bps;
+
+	delta = (xsp->last.tv_sec + (xsp->last.tv_usec / 1.e6))
+	    - (xsp->start.tv_sec + (xsp->start.tv_usec / 1.e6));
+	if (delta == 0.0) {
+		snprintf(str, sizeof str, "stalled");
+	} else {
+		bps = ((double)(xsp->rcvd - xsp->offset) / delta);
+		(void)xbps_humanize_number(size, 6, (int64_t)bps, "",
+		    HN_AUTOSCALE, HN_DECIMAL);
+		snprintf(str, sizeof str, "%sB/s", size);
+	}
+	return str;
+}
+
+/*
+ * Update the stats display
+ */
+static void
+stat_display(struct xferstat *xsp)
+{
+	struct timeval now;
+	char totsize[32], recvsize[32];
+
+	gettimeofday(&now, NULL);
+	if (now.tv_sec <= xsp->last.tv_sec)
+		return;
+	xsp->last = now;
+
+	printf("Downloading %s ... ", xsp->name);
+	(void)xbps_humanize_number(totsize, 8, (int64_t)xsp->size, "",
+	    HN_AUTOSCALE, HN_DECIMAL);
+	(void)xbps_humanize_number(recvsize, 8, (int64_t)xsp->rcvd, "",
+	    HN_AUTOSCALE, HN_DECIMAL);
+	printf("%sB [%d%% of %sB]", recvsize,
+	    (int)((double)(100.0 * (double)xsp->rcvd) / (double)xsp->size), totsize);
+
+	printf(" %s", stat_bps(xsp));
+	if (xsp->size > 0 && xsp->rcvd > 0 &&
+	    xsp->last.tv_sec >= xsp->start.tv_sec + 10)
+		printf(" ETA: %s", stat_eta(xsp));
+	printf("\n\033[1A\033[K");
+}
+
+/*
+ * Initialize the transfer statistics
+ */
+static void
+stat_start(struct xferstat *xsp, const char *name, off_t *size, off_t *offset)
+{
+	gettimeofday(&xsp->start, NULL);
+	xsp->last.tv_sec = xsp->last.tv_usec = 0;
+	xsp->name = name;
+	xsp->size = *size;
+	xsp->offset = *offset;
+	xsp->rcvd = *offset;
+}
+
+/*
+ * Update the transfer statistics
+ */
+static void
+stat_update(struct xferstat *xsp, off_t rcvd)
+{
+	xsp->rcvd = rcvd + xsp->offset;
+	stat_display(xsp);
+}
+
+/*
+ * Finalize the transfer statistics
+ */
+static void
+stat_end(struct xferstat *xsp)
+{
+	char size[32];
+
+	(void)xbps_humanize_number(size, 8, (int64_t)xsp->size, "",
+	    HN_AUTOSCALE, HN_DECIMAL);
+	printf("Downloaded %s successfully (%sB at %s)\n",
+	    xsp->name, size, stat_bps(xsp));
+}
+
+int SYMEXPORT
+xbps_fetch_file(const char *uri, const char *outputdir)
+{
+	struct stat st;
+	struct xferstat xs;
+	struct url *url = NULL;
+	struct url_stat url_st;
+	struct fetchIO *fio = NULL;
+	struct timeval tv[2];
+	ssize_t bytes_read, bytes_written;
+	off_t bytes_dld = -1;
+	char buf[32768], *filename, *destfile = NULL;
+	int fd = -1, rv = 0;
+	bool restart = false;
+
+	bytes_read = bytes_written = -1;
+
+	/*
+	 * Get the filename specified in URI argument.
+	 */
+	filename = strrchr(uri, '/');
+	if (filename == NULL)
+		return EINVAL;
+	filename++;
+
+	/*
+	 * Compute destination file path.
+	 */
+	destfile = xbps_xasprintf("%s/%s", outputdir, filename);
+	if (destfile == NULL) {
+		rv = errno;
+		goto out;
+	}
+
+	if (stat(destfile, &st) == 0)
+		restart = true;
+	else {
+		if (errno != ENOENT) {
+			rv = errno;
+			goto out;
+		}
+	}
+
+	/*
+	 * Prepare stuff for libfetch.
+	 */
+	if ((url = fetchParseURL(uri)) == NULL) {
+		rv = fetchLastErrCode;
+		goto out;
+
+	}
+	if ((rv = fetchStat(url, &url_st, "i")) == -1) {
+		rv = fetchLastErrCode;
+		goto out;
+	}
+	/*
+	 * Nothing to do if both files do have same size/mtime.
+	 */
+	if (url_st.size == st.st_size && url_st.mtime == st.st_mtime)
+		goto out;
+
+	if (restart)
+		url->offset = st.st_size;
+
+	fio = fetchXGet(url, &url_st, "i");
+	if (fio == NULL) {
+		rv = fetchLastErrCode;
+		goto out;
+	}
+
+	printf("Connected to %s.\n", url->host);
+
+	/*
+	 * If restarting, open the file for appending otherwise create it.
+	 */
+	if (restart)
+		fd = open(destfile, O_WRONLY|O_APPEND);
+	else
+		fd = open(destfile, O_WRONLY|O_CREAT, 0644);
+
+	if (fd == -1) {
+		rv = errno;
+		goto out;
+	}
+
+	/*
+	 * Start fetching requested file.
+	 */
+	if (xbps_fetch_start_cb != NULL)
+		(*xbps_fetch_start_cb)(filename, &url_st.size, &url->offset);
+	else
+		stat_start(&xs, filename, &url_st.size, &url->offset);
+
+	while ((bytes_read = fetchIO_read(fio, buf, sizeof(buf))) > 0) {
+		bytes_written = write(fd, buf, (size_t)bytes_read);
+		if (bytes_written != bytes_read) {
+			rv = errno;
+			goto out;
+		}
+		bytes_dld += bytes_read;
+		if (xbps_fetch_update_cb != NULL)
+			(*xbps_fetch_update_cb)(&bytes_dld);
+		else
+			stat_update(&xs, bytes_dld);
+	}
+	if (bytes_read == -1) {
+		rv = EINVAL;
+		goto out;
+	}
+	if (xbps_fetch_end_cb != NULL)
+		(*xbps_fetch_end_cb)();
+	else
+		stat_end(&xs);
+
+	/*
+	 * Update mtime to match remote file if download was successful.
+	 */
+	tv[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime;
+	tv[1].tv_sec = url_st.mtime;
+	tv[0].tv_usec = tv[1].tv_usec = 0;
+	rv = utimes(destfile, tv);
+
+out:
+	if (fd != -1)
+		(void)close(fd);
+	if (fio)
+		fetchIO_close(fio);
+	if (url)
+		fetchFreeURL(url);
+	if (destfile)
+		free(destfile);
+
+	return rv;
+}

lib/fetch/Makefile

@@ -0,0 +1,25 @@
+TOPDIR = ../..
+include $(TOPDIR)/vars.mk
+
+CFLAGS += -Wno-unused-macros -Wno-conversion
+CPPFLAGS += -DFTP_COMBINE_CWDS -DNETBSD -I$(TOPDIR)/include
+
+OBJS= fetch.o common.o ftp.o http.o file.o
+INCS= common.h
+GEN = ftperr.h httperr.h
+
+.PHONY: all
+all: $(OBJS)
+
+%.o: %.c $(INCS) $(GEN)
+	$(CC) $(CPPFLAGS) $(CFLAGS) -c $<
+
+ftperr.h: ftp.errors
+	./errlist.sh ftp_errlist FTP ftp.errors > $@
+
+httperr.h: http.errors
+	./errlist.sh http_errlist HTTP http.errors > $@
+
+.PHONY: clean
+clean:
+	-rm -f $(GEN) $(OBJS)

lib/fetch/common.c

@@ -0,0 +1,985 @@
+/*	$NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $	*/
+/*-
+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
+ * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#ifndef NETBSD
+#include <nbcompat.h>
+#endif
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/uio.h>
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <ctype.h>
+#include <errno.h>
+#if defined(HAVE_INTTYPES_H) || defined(NETBSD)
+#include <inttypes.h>
+#endif
+#ifndef NETBSD
+#include <nbcompat/netdb.h>
+#else
+#include <netdb.h>
+#endif
+#include <pwd.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "fetch.h"
+#include "common.h"
+
+#define DECONST(x,y) ((x)(uintptr_t)(y))
+
+
+/*** Local data **************************************************************/
+
+/*
+ * Error messages for resolver errors
+ */
+static struct fetcherr netdb_errlist[] = {
+#ifdef EAI_NODATA
+	{ EAI_NODATA,	FETCH_RESOLV,	"Host not found" },
+#endif
+	{ EAI_AGAIN,	FETCH_TEMP,	"Transient resolver failure" },
+	{ EAI_FAIL,	FETCH_RESOLV,	"Non-recoverable resolver failure" },
+	{ EAI_NONAME,	FETCH_RESOLV,	"No address record" },
+	{ -1,		FETCH_UNKNOWN,	"Unknown resolver error" }
+};
+
+/* End-of-Line */
+static const char ENDL[2] = "\r\n";
+
+
+/*** Error-reporting functions ***********************************************/
+
+/*
+ * Map error code to string
+ */
+static struct fetcherr *
+fetch_finderr(struct fetcherr *p, int e)
+{
+	while (p->num != -1 && p->num != e)
+		p++;
+	return (p);
+}
+
+/*
+ * Set error code
+ */
+void
+fetch_seterr(struct fetcherr *p, int e)
+{
+	p = fetch_finderr(p, e);
+	fetchLastErrCode = p->cat;
+	snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
+}
+
+/*
+ * Set error code according to errno
+ */
+void
+fetch_syserr(void)
+{
+	switch (errno) {
+	case 0:
+		fetchLastErrCode = FETCH_OK;
+		break;
+	case EPERM:
+	case EACCES:
+	case EROFS:
+#ifdef EAUTH
+	case EAUTH:
+#endif
+#ifdef ENEEDAUTH
+	case ENEEDAUTH:
+#endif
+		fetchLastErrCode = FETCH_AUTH;
+		break;
+	case ENOENT:
+	case EISDIR: /* XXX */
+		fetchLastErrCode = FETCH_UNAVAIL;
+		break;
+	case ENOMEM:
+		fetchLastErrCode = FETCH_MEMORY;
+		break;
+	case EBUSY:
+	case EAGAIN:
+		fetchLastErrCode = FETCH_TEMP;
+		break;
+	case EEXIST:
+		fetchLastErrCode = FETCH_EXISTS;
+		break;
+	case ENOSPC:
+		fetchLastErrCode = FETCH_FULL;
+		break;
+	case EADDRINUSE:
+	case EADDRNOTAVAIL:
+	case ENETDOWN:
+	case ENETUNREACH:
+	case ENETRESET:
+	case EHOSTUNREACH:
+		fetchLastErrCode = FETCH_NETWORK;
+		break;
+	case ECONNABORTED:
+	case ECONNRESET:
+		fetchLastErrCode = FETCH_ABORT;
+		break;
+	case ETIMEDOUT:
+		fetchLastErrCode = FETCH_TIMEOUT;
+		break;
+	case ECONNREFUSED:
+	case EHOSTDOWN:
+		fetchLastErrCode = FETCH_DOWN;
+		break;
+default:
+		fetchLastErrCode = FETCH_UNKNOWN;
+	}
+	snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
+}
+
+
+/*
+ * Emit status message
+ */
+void
+fetch_info(const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+	fputc('\n', stderr);
+}
+
+
+/*** Network-related utility functions ***************************************/
+
+/*
+ * Return the default port for a scheme
+ */
+int
+fetch_default_port(const char *scheme)
+{
+	struct servent *se;
+
+	if ((se = getservbyname(scheme, "tcp")) != NULL)
+		return (ntohs(se->s_port));
+	if (strcasecmp(scheme, SCHEME_FTP) == 0)
+		return (FTP_DEFAULT_PORT);
+	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
+		return (HTTP_DEFAULT_PORT);
+	return (0);
+}
+
+/*
+ * Return the default proxy port for a scheme
+ */
+int
+fetch_default_proxy_port(const char *scheme)
+{
+	if (strcasecmp(scheme, SCHEME_FTP) == 0)
+		return (FTP_DEFAULT_PROXY_PORT);
+	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
+		return (HTTP_DEFAULT_PROXY_PORT);
+	return (0);
+}
+
+
+/*
+ * Create a connection for an existing descriptor.
+ */
+conn_t *
+fetch_reopen(int sd)
+{
+	conn_t *conn;
+
+	/* allocate and fill connection structure */
+	if ((conn = calloc(1, sizeof(*conn))) == NULL)
+		return (NULL);
+	conn->next_buf = NULL;
+	conn->next_len = 0;
+	conn->sd = sd;
+	conn->is_active = 0;
+	++conn->ref;
+	return (conn);
+}
+
+
+/*
+ * Bump a connection's reference count.
+ */
+conn_t *
+fetch_ref(conn_t *conn)
+{
+
+	++conn->ref;
+	return (conn);
+}
+
+
+/*
+ * Bind a socket to a specific local address
+ */
+int
+fetch_bind(int sd, int af, const char *addr)
+{
+	struct addrinfo hints, *res, *res0;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = af;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = 0;
+	if (getaddrinfo(addr, NULL, &hints, &res0))
+		return (-1);
+	for (res = res0; res; res = res->ai_next) {
+		if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
+			return (0);
+	}
+	return (-1);
+}
+
+
+/*
+ * Establish a TCP connection to the specified port on the specified host.
+ */
+conn_t *
+fetch_connect(const char *host, int port, int af, int verbose)
+{
+	conn_t *conn;
+	char pbuf[10];
+	const char *bindaddr;
+	struct addrinfo hints, *res, *res0;
+	int sd, error;
+
+	if (verbose)
+		fetch_info("looking up %s", host);
+
+	/* look up host name and set up socket address structure */
+	snprintf(pbuf, sizeof(pbuf), "%d", port);
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = af;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = 0;
+	if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
+		netdb_seterr(error);
+		return (NULL);
+	}
+	bindaddr = getenv("FETCH_BIND_ADDRESS");
+
+	if (verbose)
+		fetch_info("connecting to %s:%d", host, port);
+
+	/* try to connect */
+	for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
+		if ((sd = socket(res->ai_family, res->ai_socktype,
+			 res->ai_protocol)) == -1)
+			continue;
+		if (bindaddr != NULL && *bindaddr != '\0' &&
+		    fetch_bind(sd, res->ai_family, bindaddr) != 0) {
+			fetch_info("failed to bind to '%s'", bindaddr);
+			close(sd);
+			continue;
+		}
+		if (connect(sd, res->ai_addr, res->ai_addrlen) == 0)
+			break;
+		close(sd);
+	}
+	freeaddrinfo(res0);
+	if (sd == -1) {
+		fetch_syserr();
+		return (NULL);
+	}
+
+	if ((conn = fetch_reopen(sd)) == NULL) {
+		fetch_syserr();
+		close(sd);
+	}
+	return (conn);
+}
+
+
+/*
+ * Enable SSL on a connection.
+ */
+int
+fetch_ssl(conn_t *conn, int verbose)
+{
+
+#ifdef WITH_SSL
+	/* Init the SSL library and context */
+	if (!SSL_library_init()){
+		fprintf(stderr, "SSL library init failed\n");
+		return (-1);
+	}
+
+	SSL_load_error_strings();
+
+	conn->ssl_meth = SSLv23_client_method();
+	conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
+	SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
+
+	conn->ssl = SSL_new(conn->ssl_ctx);
+	if (conn->ssl == NULL){
+		fprintf(stderr, "SSL context creation failed\n");
+		return (-1);
+	}
+	SSL_set_fd(conn->ssl, conn->sd);
+	if (SSL_connect(conn->ssl) == -1){
+		ERR_print_errors_fp(stderr);
+		return (-1);
+	}
+
+	if (verbose) {
+		X509_NAME *name;
+		char *str;
+
+		fprintf(stderr, "SSL connection established using %s\n",
+		    SSL_get_cipher(conn->ssl));
+		conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
+		name = X509_get_subject_name(conn->ssl_cert);
+		str = X509_NAME_oneline(name, 0, 0);
+		printf("Certificate subject: %s\n", str);
+		free(str);
+		name = X509_get_issuer_name(conn->ssl_cert);
+		str = X509_NAME_oneline(name, 0, 0);
+		printf("Certificate issuer: %s\n", str);
+		free(str);
+	}
+
+	return (0);
+#else
+	(void)conn;
+	(void)verbose;
+	fprintf(stderr, "SSL support disabled\n");
+	return (-1);
+#endif
+}
+
+
+/*
+ * Read a character from a connection w/ timeout
+ */
+ssize_t
+fetch_read(conn_t *conn, char *buf, size_t len)
+{
+	struct timeval now, timeout, waittv;
+	fd_set readfds;
+	ssize_t rlen;
+	int r;
+
+	if (len == 0)
+		return 0;
+
+	if (conn->next_len != 0) {
+		if (conn->next_len < len)
+			len = conn->next_len;
+		memmove(buf, conn->next_buf, len);
+		conn->next_len -= len;
+		conn->next_buf += len;
+		return len;
+	}
+
+	if (fetchTimeout) {
+		FD_ZERO(&readfds);
+		gettimeofday(&timeout, NULL);
+		timeout.tv_sec += fetchTimeout;
+	}
+
+	for (;;) {
+		while (fetchTimeout && !FD_ISSET(conn->sd, &readfds)) {
+			FD_SET(conn->sd, &readfds);
+			gettimeofday(&now, NULL);
+			waittv.tv_sec = timeout.tv_sec - now.tv_sec;
+			waittv.tv_usec = timeout.tv_usec - now.tv_usec;
+			if (waittv.tv_usec < 0) {
+				waittv.tv_usec += 1000000;
+				waittv.tv_sec--;
+			}
+			if (waittv.tv_sec < 0) {
+				errno = ETIMEDOUT;
+				fetch_syserr();
+				return (-1);
+			}
+			errno = 0;
+			r = select(conn->sd + 1, &readfds, NULL, NULL, &waittv);
+			if (r == -1) {
+				if (errno == EINTR && fetchRestartCalls)
+					continue;
+				fetch_syserr();
+				return (-1);
+			}
+		}
+#ifdef WITH_SSL
+		if (conn->ssl != NULL)
+			rlen = SSL_read(conn->ssl, buf, len);
+		else
+#endif
+			rlen = read(conn->sd, buf, len);
+		if (rlen >= 0)
+			break;
+	
+		if (errno != EINTR || !fetchRestartCalls)
+			return (-1);
+	}
+	return (rlen);
+}
+
+
+/*
+ * Read a line of text from a connection w/ timeout
+ */
+#define MIN_BUF_SIZE 1024
+
+int
+fetch_getln(conn_t *conn)
+{
+	char *tmp, *next;
+	size_t tmpsize;
+	ssize_t len;
+
+	if (conn->buf == NULL) {
+		if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		conn->bufsize = MIN_BUF_SIZE;
+	}
+
+	conn->buflen = 0;
+	next = NULL;
+
+	do {
+		/*
+		 * conn->bufsize != conn->buflen at this point,
+		 * so the buffer can be NUL-terminated below for
+		 * the case of len == 0.
+		 */
+		len = fetch_read(conn, conn->buf + conn->buflen,
+		    conn->bufsize - conn->buflen);
+		if (len == -1)
+			return (-1);
+		if (len == 0)
+			break;
+		next = memchr(conn->buf + conn->buflen, '\n', len);
+		conn->buflen += len;
+		if (conn->buflen == conn->bufsize && next == NULL) {
+			tmp = conn->buf;
+			tmpsize = conn->bufsize * 2;
+			if (tmpsize < conn->bufsize) {
+				errno = ENOMEM;
+				return (-1);
+			}
+			if ((tmp = realloc(tmp, tmpsize)) == NULL) {
+				errno = ENOMEM;
+				return (-1);
+			}
+			conn->buf = tmp;
+			conn->bufsize = tmpsize;
+		}
+	} while (next == NULL);
+
+	if (next != NULL) {
+		*next = '\0';
+		conn->next_buf = next + 1;
+		conn->next_len = conn->buflen - (conn->next_buf - conn->buf);
+		conn->buflen = next - conn->buf;
+	} else {
+		conn->buf[conn->buflen] = '\0';
+		conn->next_len = 0;
+	}
+	return (0);
+}
+
+
+/*
+ * Write to a connection w/ timeout
+ */
+ssize_t
+fetch_write(conn_t *conn, const char *buf, size_t len)
+{
+	struct iovec iov;
+
+	iov.iov_base = DECONST(char *, buf);
+	iov.iov_len = len;
+	return fetch_writev(conn, &iov, 1);
+}
+
+/*
+ * Write a vector to a connection w/ timeout
+ * Note: can modify the iovec.
+ */
+ssize_t
+fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
+{
+	struct timeval now, timeout, waittv;
+	fd_set writefds;
+	ssize_t wlen, total;
+	int r;
+
+	if (fetchTimeout) {
+		FD_ZERO(&writefds);
+		gettimeofday(&timeout, NULL);
+		timeout.tv_sec += fetchTimeout;
+	}
+
+	total = 0;
+	while (iovcnt > 0) {
+		while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
+			FD_SET(conn->sd, &writefds);
+			gettimeofday(&now, NULL);
+			waittv.tv_sec = timeout.tv_sec - now.tv_sec;
+			waittv.tv_usec = timeout.tv_usec - now.tv_usec;
+			if (waittv.tv_usec < 0) {
+				waittv.tv_usec += 1000000;
+				waittv.tv_sec--;
+			}
+			if (waittv.tv_sec < 0) {
+				errno = ETIMEDOUT;
+				fetch_syserr();
+				return (-1);
+			}
+			errno = 0;
+			r = select(conn->sd + 1, NULL, &writefds, NULL, &waittv);
+			if (r == -1) {
+				if (errno == EINTR && fetchRestartCalls)
+					continue;
+				return (-1);
+			}
+		}
+		errno = 0;
+#ifdef WITH_SSL
+		if (conn->ssl != NULL)
+			wlen = SSL_write(conn->ssl,
+			    iov->iov_base, iov->iov_len);
+		else
+#endif
+			wlen = writev(conn->sd, iov, iovcnt);
+		if (wlen == 0) {
+			/* we consider a short write a failure */
+			errno = EPIPE;
+			fetch_syserr();
+			return (-1);
+		}
+		if (wlen < 0) {
+			if (errno == EINTR && fetchRestartCalls)
+				continue;
+			return (-1);
+		}
+		total += wlen;
+		while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
+			wlen -= iov->iov_len;
+			iov++;
+			iovcnt--;
+		}
+		if (iovcnt > 0) {
+			iov->iov_len -= wlen;
+			iov->iov_base = DECONST(char *, iov->iov_base) + wlen;
+		}
+	}
+	return (total);
+}
+
+
+/*
+ * Write a line of text to a connection w/ timeout
+ */
+int
+fetch_putln(conn_t *conn, const char *str, size_t len)
+{
+	struct iovec iov[2];
+	ssize_t ret;
+
+	iov[0].iov_base = DECONST(char *, str);
+	iov[0].iov_len = len;
+	iov[1].iov_base = DECONST(char *, ENDL);
+	iov[1].iov_len = sizeof(ENDL);
+	if (len == 0)
+		ret = fetch_writev(conn, &iov[1], 1);
+	else
+		ret = fetch_writev(conn, iov, 2);
+	if (ret == -1)
+		return (-1);
+	return (0);
+}
+
+
+/*
+ * Close connection
+ */
+int
+fetch_close(conn_t *conn)
+{
+	int ret;
+
+	if (--conn->ref > 0)
+		return (0);
+	ret = close(conn->sd);
+	free(conn->buf);
+	free(conn);
+	return (ret);
+}
+
+
+/*** Directory-related utility functions *************************************/
+
+int
+fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
+    int pre_quoted)
+{
+	struct url *tmp;
+	char *tmp_name;
+	size_t base_doc_len, name_len, i;
+	unsigned char c;
+
+	if (strchr(name, '/') != NULL ||
+	    strcmp(name, "..") == 0 ||
+	    strcmp(name, ".") == 0)
+		return 0;
+
+	if (strcmp(base->doc, "/") == 0)
+		base_doc_len = 0;
+	else
+		base_doc_len = strlen(base->doc);
+
+	name_len = 1;
+	for (i = 0; name[i] != '\0'; ++i) {
+		if ((!pre_quoted && name[i] == '%') ||
+		    !fetch_urlpath_safe(name[i]))
+			name_len += 3;
+		else
+			++name_len;
+	}
+
+	tmp_name = malloc( base_doc_len + name_len + 1);
+	if (tmp_name == NULL) {
+		errno = ENOMEM;
+		fetch_syserr();
+		return (-1);
+	}
+
+	if (ue->length + 1 >= ue->alloc_size) {
+		tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp));
+		if (tmp == NULL) {
+			free(tmp_name);
+			errno = ENOMEM;
+			fetch_syserr();
+			return (-1);
+		}
+		ue->alloc_size = ue->alloc_size * 2 + 1;
+		ue->urls = tmp;
+	}
+
+	tmp = ue->urls + ue->length;
+	strcpy(tmp->scheme, base->scheme);
+	strcpy(tmp->user, base->user);
+	strcpy(tmp->pwd, base->pwd);
+	strcpy(tmp->host, base->host);
+	tmp->port = base->port;
+	tmp->doc = tmp_name;
+	memcpy(tmp->doc, base->doc, base_doc_len);
+	tmp->doc[base_doc_len] = '/';
+
+	for (i = base_doc_len + 1; *name != '\0'; ++name) {
+		if ((!pre_quoted && *name == '%') ||
+		    !fetch_urlpath_safe(*name)) {
+			tmp->doc[i++] = '%';
+			c = (unsigned char)*name / 16;
+			if (c < 10)
+				tmp->doc[i++] = '0' + c;
+			else
+				tmp->doc[i++] = 'a' - 10 + c;
+			c = (unsigned char)*name % 16;
+			if (c < 10)
+				tmp->doc[i++] = '0' + c;
+			else
+				tmp->doc[i++] = 'a' - 10 + c;
+		} else {
+			tmp->doc[i++] = *name;
+		}
+	}
+	tmp->doc[i] = '\0';
+
+	tmp->offset = 0;
+	tmp->length = 0;
+	tmp->last_modified = -1;
+
+	++ue->length;
+
+	return (0);
+}
+
+void
+fetchInitURLList(struct url_list *ue)
+{
+	ue->length = ue->alloc_size = 0;
+	ue->urls = NULL;
+}
+
+int
+fetchAppendURLList(struct url_list *dst, const struct url_list *src)
+{
+	size_t i, j, len;
+
+	len = dst->length + src->length;
+	if (len > dst->alloc_size) {
+		struct url *tmp;
+
+		tmp = realloc(dst->urls, len * sizeof(*tmp));
+		if (tmp == NULL) {
+			errno = ENOMEM;
+			fetch_syserr();
+			return (-1);
+		}
+		dst->alloc_size = len;
+		dst->urls = tmp;
+	}
+
+	for (i = 0, j = dst->length; i < src->length; ++i, ++j) {
+		dst->urls[j] = src->urls[i];
+		dst->urls[j].doc = strdup(src->urls[i].doc);
+		if (dst->urls[j].doc == NULL) {
+			while (i-- > 0)
+				free(dst->urls[j].doc);
+			fetch_syserr();
+			return -1;
+		}
+	}
+	dst->length = len;
+
+	return 0;
+}
+
+void
+fetchFreeURLList(struct url_list *ue)
+{
+	size_t i;
+
+	for (i = 0; i < ue->length; ++i)
+		free(ue->urls[i].doc);
+	free(ue->urls);
+	ue->length = ue->alloc_size = 0;
+}
+
+
+/*** Authentication-related utility functions ********************************/
+
+static const char *
+fetch_read_word(FILE *f)
+{
+	static char word[1024];
+
+	if (fscanf(f, " %1023s ", word) != 1)
+		return (NULL);
+	return (word);
+}
+
+/*
+ * Get authentication data for a URL from .netrc
+ */
+int
+fetch_netrc_auth(struct url *url)
+{
+	char fn[PATH_MAX];
+	const char *word;
+	char *p;
+	FILE *f;
+
+	if ((p = getenv("NETRC")) != NULL) {
+		if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
+			fetch_info("$NETRC specifies a file name "
+			    "longer than PATH_MAX");
+			return (-1);
+		}
+	} else {
+		if ((p = getenv("HOME")) != NULL) {
+			struct passwd *pwd;
+
+			if ((pwd = getpwuid(getuid())) == NULL ||
+			    (p = pwd->pw_dir) == NULL)
+				return (-1);
+		}
+		if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
+			return (-1);
+	}
+
+	if ((f = fopen(fn, "r")) == NULL)
+		return (-1);
+	while ((word = fetch_read_word(f)) != NULL) {
+		if (strcmp(word, "default") == 0)
+			break;
+		if (strcmp(word, "machine") == 0 &&
+		    (word = fetch_read_word(f)) != NULL &&
+		    strcasecmp(word, url->host) == 0) {
+			break;
+		}
+	}
+	if (word == NULL)
+		goto ferr;
+	while ((word = fetch_read_word(f)) != NULL) {
+		if (strcmp(word, "login") == 0) {
+			if ((word = fetch_read_word(f)) == NULL)
+				goto ferr;
+			if (snprintf(url->user, sizeof(url->user),
+				"%s", word) > (int)sizeof(url->user)) {
+				fetch_info("login name in .netrc is too long");
+				url->user[0] = '\0';
+			}
+		} else if (strcmp(word, "password") == 0) {
+			if ((word = fetch_read_word(f)) == NULL)
+				goto ferr;
+			if (snprintf(url->pwd, sizeof(url->pwd),
+				"%s", word) > (int)sizeof(url->pwd)) {
+				fetch_info("password in .netrc is too long");
+				url->pwd[0] = '\0';
+			}
+		} else if (strcmp(word, "account") == 0) {
+			if ((word = fetch_read_word(f)) == NULL)
+				goto ferr;
+			/* XXX not supported! */
+		} else {
+			break;
+		}
+	}
+	fclose(f);
+	return (0);
+ ferr:
+	fclose(f);
+	return (-1);
+}
+
+/*
+ * The no_proxy environment variable specifies a set of domains for
+ * which the proxy should not be consulted; the contents is a comma-,
+ * or space-separated list of domain names.  A single asterisk will
+ * override all proxy variables and no transactions will be proxied
+ * (for compatability with lynx and curl, see the discussion at
+ * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
+ */
+int
+fetch_no_proxy_match(const char *host)
+{
+	const char *no_proxy, *p, *q;
+	size_t h_len, d_len;
+
+	if ((no_proxy = getenv("NO_PROXY")) == NULL &&
+	    (no_proxy = getenv("no_proxy")) == NULL)
+		return (0);
+
+	/* asterisk matches any hostname */
+	if (strcmp(no_proxy, "*") == 0)
+		return (1);
+
+	h_len = strlen(host);
+	p = no_proxy;
+	do {
+		/* position p at the beginning of a domain suffix */
+		while (*p == ',' || isspace((unsigned char)*p))
+			p++;
+
+		/* position q at the first separator character */
+		for (q = p; *q; ++q)
+			if (*q == ',' || isspace((unsigned char)*q))
+				break;
+
+		d_len = q - p;
+		if (d_len > 0 && h_len > d_len &&
+		    strncasecmp(host + h_len - d_len,
+			p, d_len) == 0) {
+			/* domain name matches */
+			return (1);
+		}
+
+		p = q + 1;
+	} while (*q);
+
+	return (0);
+}
+
+struct fetchIO {
+	void *io_cookie;
+	ssize_t (*io_read)(void *, void *, size_t);
+	ssize_t (*io_write)(void *, const void *, size_t);
+	void (*io_close)(void *);
+};
+
+void
+fetchIO_close(fetchIO *f)
+{
+	if (f->io_close != NULL)
+		(*f->io_close)(f->io_cookie);
+
+	free(f);
+}
+
+fetchIO *
+fetchIO_unopen(void *io_cookie, ssize_t (*io_read)(void *, void *, size_t),
+    ssize_t (*io_write)(void *, const void *, size_t),
+    void (*io_close)(void *))
+{
+	fetchIO *f;
+
+	f = malloc(sizeof(*f));
+	if (f == NULL)
+		return f;
+
+	f->io_cookie = io_cookie;
+	f->io_read = io_read;
+	f->io_write = io_write;
+	f->io_close = io_close;
+
+	return f;
+}
+
+ssize_t
+fetchIO_read(fetchIO *f, void *buf, size_t len)
+{
+	if (f->io_read == NULL)
+		return EBADF;
+	return (*f->io_read)(f->io_cookie, buf, len);
+}
+
+ssize_t
+fetchIO_write(fetchIO *f, const void *buf, size_t len)
+{
+	if (f->io_read == NULL)
+		return EBADF;
+	return (*f->io_write)(f->io_cookie, buf, len);
+}

lib/fetch/common.h

@@ -0,0 +1,137 @@
+/*	$NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $	*/
+/*-
+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD: common.h,v 1.30 2007/12/18 11:03:07 des Exp $
+ */
+
+#ifndef _COMMON_H_INCLUDED
+#define _COMMON_H_INCLUDED
+
+#define FTP_DEFAULT_PORT	21
+#define HTTP_DEFAULT_PORT	80
+#define FTP_DEFAULT_PROXY_PORT	21
+#define HTTP_DEFAULT_PROXY_PORT	3128
+
+#ifdef WITH_SSL
+#include <openssl/crypto.h>
+#include <openssl/x509.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#endif
+
+#if !defined(__sun) && !defined(__hpux) && !defined(__INTERIX) && \
+    !defined(__digital__) && !defined(__linux) && !defined(__MINT__) && \
+    !defined(__sgi)
+#define HAVE_SA_LEN
+#endif
+
+/* Connection */
+typedef struct fetchconn conn_t;
+struct fetchconn {
+	int		 sd;		/* socket descriptor */
+	char		*buf;		/* buffer */
+	size_t		 bufsize;	/* buffer size */
+	size_t		 buflen;	/* length of buffer contents */
+	char		*next_buf;	/* pending buffer, e.g. after getln */
+	size_t		 next_len;	/* size of pending buffer */
+	int		 err;		/* last protocol reply code */
+#ifdef WITH_SSL
+	SSL		*ssl;		/* SSL handle */
+	SSL_CTX		*ssl_ctx;	/* SSL context */
+	X509		*ssl_cert;	/* server certificate */
+#  if OPENSSL_VERSION_NUMBER < 0x00909000L
+	SSL_METHOD *ssl_meth;		/* SSL method */
+#  else
+	const SSL_METHOD *ssl_meth;	/* SSL method */
+#  endif
+#endif
+	int		 ref;		/* reference count */
+	int		 is_active;
+};
+
+/* Structure used for error message lists */
+struct fetcherr {
+	const int	 num;
+	const int	 cat;
+	const char	*string;
+};
+
+/* for fetch_writev */
+struct iovec;
+
+void		 fetch_seterr(struct fetcherr *, int);
+void		 fetch_syserr(void);
+void		 fetch_info(const char *, ...);
+int		 fetch_default_port(const char *);
+int		 fetch_default_proxy_port(const char *);
+int		 fetch_bind(int, int, const char *);
+conn_t		*fetch_connect(const char *, int, int, int);
+conn_t		*fetch_reopen(int);
+conn_t		*fetch_ref(conn_t *);
+int		 fetch_ssl(conn_t *, int);
+ssize_t		 fetch_read(conn_t *, char *, size_t);
+int		 fetch_getln(conn_t *);
+ssize_t		 fetch_write(conn_t *, const char *, size_t);
+ssize_t		 fetch_writev(conn_t *, struct iovec *, int);
+int		 fetch_putln(conn_t *, const char *, size_t);
+int		 fetch_close(conn_t *);
+int		 fetch_add_entry(struct url_list *, struct url *, const char *, int);
+int		 fetch_netrc_auth(struct url *url);
+int		 fetch_no_proxy_match(const char *);
+int		 fetch_urlpath_safe(char);
+
+#define ftp_seterr(n)	 fetch_seterr(ftp_errlist, n)
+#define http_seterr(n)	 fetch_seterr(http_errlist, n)
+#define netdb_seterr(n)	 fetch_seterr(netdb_errlist, n)
+#define url_seterr(n)	 fetch_seterr(url_errlist, n)
+
+fetchIO		*fetchIO_unopen(void *, ssize_t (*)(void *, void *, size_t),
+    ssize_t (*)(void *, const void *, size_t), void (*)(void *));
+
+/*
+ * I don't really like exporting http_request() and ftp_request(),
+ * but the HTTP and FTP code occasionally needs to cross-call
+ * eachother, and this saves me from adding a lot of special-case code
+ * to handle those cases.
+ *
+ * Note that _*_request() free purl, which is way ugly but saves us a
+ * whole lot of trouble.
+ */
+fetchIO		*http_request(struct url *, const char *,
+		     struct url_stat *, struct url *, const char *);
+fetchIO		*ftp_request(struct url *, const char *, const char *,
+		     struct url_stat *, struct url *, const char *);
+
+
+/*
+ * Check whether a particular flag is set
+ */
+#define CHECK_FLAG(x)	(flags && strchr(flags, (x)))
+
+#endif

lib/fetch/errlist.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# $NetBSD: errlist.sh,v 1.2 2008/10/06 12:58:29 joerg Exp $
+
+printf "static struct fetcherr $1[] = {\n"
+while read code type msg; do
+	[ "${code}" = "#" ] && continue
+	printf "\t{ ${code}, FETCH_${type}, \"${msg}\" },\n"
+done < $3
+
+printf "\t{ -1, FETCH_UNKNOWN, \"Unknown $2 error\" }\n"
+printf "};\n"

lib/fetch/fetch.c

@@ -0,0 +1,627 @@
+/*	$NetBSD: fetch.c,v 1.19 2009/08/11 20:48:06 joerg Exp $	*/
+/*-
+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
+ * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#ifndef NETBSD
+#include <nbcompat.h>
+#endif
+
+#include <ctype.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fetch.h"
+#include "common.h"
+
+auth_t	 fetchAuthMethod;
+int	 fetchLastErrCode;
+char	 fetchLastErrString[MAXERRSTRING];
+int	 fetchTimeout;
+volatile int	 fetchRestartCalls = 1;
+int	 fetchDebug;
+
+
+/*** Local data **************************************************************/
+
+/*
+ * Error messages for parser errors
+ */
+#define URL_MALFORMED		1
+#define URL_BAD_SCHEME		2
+#define URL_BAD_PORT		3
+static struct fetcherr url_errlist[] = {
+	{ URL_MALFORMED,	FETCH_URL,	"Malformed URL" },
+	{ URL_BAD_SCHEME,	FETCH_URL,	"Invalid URL scheme" },
+	{ URL_BAD_PORT,		FETCH_URL,	"Invalid server port" },
+	{ -1,			FETCH_UNKNOWN,	"Unknown parser error" }
+};
+
+
+/*** Public API **************************************************************/
+
+/*
+ * Select the appropriate protocol for the URL scheme, and return a
+ * read-only stream connected to the document referenced by the URL.
+ * Also fill out the struct url_stat.
+ */
+fetchIO *
+fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
+{
+
+	if (us != NULL) {
+		us->size = -1;
+		us->atime = us->mtime = 0;
+	}
+	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+		return (fetchXGetFile(URL, us, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+		return (fetchXGetFTP(URL, us, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+		return (fetchXGetHTTP(URL, us, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+		return (fetchXGetHTTP(URL, us, flags));
+	url_seterr(URL_BAD_SCHEME);
+	return (NULL);
+}
+
+/*
+ * Select the appropriate protocol for the URL scheme, and return a
+ * read-only stream connected to the document referenced by the URL.
+ */
+fetchIO *
+fetchGet(struct url *URL, const char *flags)
+{
+	return (fetchXGet(URL, NULL, flags));
+}
+
+/*
+ * Select the appropriate protocol for the URL scheme, and return a
+ * write-only stream connected to the document referenced by the URL.
+ */
+fetchIO *
+fetchPut(struct url *URL, const char *flags)
+{
+
+	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+		return (fetchPutFile(URL, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+		return (fetchPutFTP(URL, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+		return (fetchPutHTTP(URL, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+		return (fetchPutHTTP(URL, flags));
+	url_seterr(URL_BAD_SCHEME);
+	return (NULL);
+}
+
+/*
+ * Select the appropriate protocol for the URL scheme, and return the
+ * size of the document referenced by the URL if it exists.
+ */
+int
+fetchStat(struct url *URL, struct url_stat *us, const char *flags)
+{
+
+	if (us != NULL) {
+		us->size = -1;
+		us->atime = us->mtime = 0;
+	}
+	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+		return (fetchStatFile(URL, us, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+		return (fetchStatFTP(URL, us, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+		return (fetchStatHTTP(URL, us, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+		return (fetchStatHTTP(URL, us, flags));
+	url_seterr(URL_BAD_SCHEME);
+	return (-1);
+}
+
+/*
+ * Select the appropriate protocol for the URL scheme, and return a
+ * list of files in the directory pointed to by the URL.
+ */
+int
+fetchList(struct url_list *ue, struct url *URL, const char *pattern,
+    const char *flags)
+{
+
+	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+		return (fetchListFile(ue, URL, pattern, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+		return (fetchListFTP(ue, URL, pattern, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+		return (fetchListHTTP(ue, URL, pattern, flags));
+	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+		return (fetchListHTTP(ue, URL, pattern, flags));
+	url_seterr(URL_BAD_SCHEME);
+	return -1;
+}
+
+/*
+ * Attempt to parse the given URL; if successful, call fetchXGet().
+ */
+fetchIO *
+fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
+{
+	struct url *u;
+	fetchIO *f;
+
+	if ((u = fetchParseURL(URL)) == NULL)
+		return (NULL);
+
+	f = fetchXGet(u, us, flags);
+
+	fetchFreeURL(u);
+	return (f);
+}
+
+/*
+ * Attempt to parse the given URL; if successful, call fetchGet().
+ */
+fetchIO *
+fetchGetURL(const char *URL, const char *flags)
+{
+	return (fetchXGetURL(URL, NULL, flags));
+}
+
+/*
+ * Attempt to parse the given URL; if successful, call fetchPut().
+ */
+fetchIO *
+fetchPutURL(const char *URL, const char *flags)
+{
+	struct url *u;
+	fetchIO *f;
+
+	if ((u = fetchParseURL(URL)) == NULL)
+		return (NULL);
+
+	f = fetchPut(u, flags);
+
+	fetchFreeURL(u);
+	return (f);
+}
+
+/*
+ * Attempt to parse the given URL; if successful, call fetchStat().
+ */
+int
+fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
+{
+	struct url *u;
+	int s;
+
+	if ((u = fetchParseURL(URL)) == NULL)
+		return (-1);
+
+	s = fetchStat(u, us, flags);
+
+	fetchFreeURL(u);
+	return (s);
+}
+
+/*
+ * Attempt to parse the given URL; if successful, call fetchList().
+ */
+int
+fetchListURL(struct url_list *ue, const char *URL, const char *pattern,
+    const char *flags)
+{
+	struct url *u;
+	int rv;
+
+	if ((u = fetchParseURL(URL)) == NULL)
+		return -1;
+
+	rv = fetchList(ue, u, pattern, flags);
+
+	fetchFreeURL(u);
+	return rv;
+}
+
+/*
+ * Make a URL
+ */
+struct url *
+fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
+    const char *user, const char *pwd)
+{
+	struct url *u;
+
+	if (!scheme || (!host && !doc)) {
+		url_seterr(URL_MALFORMED);
+		return (NULL);
+	}
+
+	if (port < 0 || port > 65535) {
+		url_seterr(URL_BAD_PORT);
+		return (NULL);
+	}
+
+	/* allocate struct url */
+	if ((u = calloc(1, sizeof(*u))) == NULL) {
+		fetch_syserr();
+		return (NULL);
+	}
+
+	if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
+		fetch_syserr();
+		free(u);
+		return (NULL);
+	}
+
+#define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
+	seturl(scheme);
+	seturl(host);
+	seturl(user);
+	seturl(pwd);
+#undef seturl
+	u->port = port;
+
+	return (u);
+}
+
+int
+fetch_urlpath_safe(char x)
+{
+	if ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'Z') ||
+	    (x >= 'a' && x <= 'z'))
+		return 1;
+
+	switch (x) {
+	case '$':
+	case '-':
+	case '_':
+	case '.':
+	case '+':
+	case '!':
+	case '*':
+	case '\'':
+	case '(':
+	case ')':
+	case ',':
+	/* The following are allowed in segment and path components: */
+	case '?':
+	case ':':
+	case '@':
+	case '&':
+	case '=':
+	case '/':
+	case ';':
+	/* If something is already quoted... */
+	case '%':
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * Copy an existing URL.
+ */
+struct url *
+fetchCopyURL(const struct url *src)
+{
+	struct url *dst;
+	char *doc;
+
+	/* allocate struct url */
+	if ((dst = malloc(sizeof(*dst))) == NULL) {
+		fetch_syserr();
+		return (NULL);
+	}
+	if ((doc = strdup(src->doc)) == NULL) {
+		fetch_syserr();
+		free(dst);
+		return (NULL);
+	}
+	*dst = *src;
+	dst->doc = doc;
+
+	return dst;
+}
+
+/*
+ * Split an URL into components. URL syntax is:
+ * [method:/][/[user[:pwd]@]host[:port]/][document]
+ * This almost, but not quite, RFC1738 URL syntax.
+ */
+struct url *
+fetchParseURL(const char *URL)
+{
+	const char *p, *q;
+	struct url *u;
+	size_t i, count;
+	int pre_quoted;
+
+	/* allocate struct url */
+	if ((u = calloc(1, sizeof(*u))) == NULL) {
+		fetch_syserr();
+		return (NULL);
+	}
+
+	if (*URL == '/') {
+		pre_quoted = 0;
+		strcpy(u->scheme, SCHEME_FILE);
+		p = URL;
+		goto quote_doc;
+	}
+	if (strncmp(URL, "file:", 5) == 0) {
+		pre_quoted = 1;
+		strcpy(u->scheme, SCHEME_FILE);
+		URL += 5;
+		if (URL[0] != '/' || URL[1] != '/' || URL[2] != '/') {
+			url_seterr(URL_MALFORMED);
+			goto ouch;
+		}
+		p = URL + 2;
+		goto quote_doc;
+	}
+	if (strncmp(URL, "http:", 5) == 0 ||
+	    strncmp(URL, "https:", 6) == 0) {
+		pre_quoted = 1;
+		if (URL[4] == ':') {
+			strcpy(u->scheme, SCHEME_HTTP);
+			URL += 5;
+		} else {
+			strcpy(u->scheme, SCHEME_HTTPS);
+			URL += 6;
+		}
+
+		if (URL[0] != '/' || URL[1] != '/') {
+			url_seterr(URL_MALFORMED);
+			goto ouch;
+		}
+		URL += 2;
+		p = URL;
+		goto find_user;
+	}
+	if (strncmp(URL, "ftp:", 4) == 0) {
+		pre_quoted = 1;
+		strcpy(u->scheme, SCHEME_FTP);
+		URL += 4;
+		if (URL[0] != '/' || URL[1] != '/') {
+			url_seterr(URL_MALFORMED);
+			goto ouch;
+		}
+		URL += 2;
+		p = URL;
+		goto find_user;			
+	}
+
+	url_seterr(URL_BAD_SCHEME);
+	goto ouch;
+
+find_user:
+	p = strpbrk(URL, "/@");
+	if (p != NULL && *p == '@') {
+		/* username */
+		for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) {
+			if (i < URL_USERLEN)
+				u->user[i++] = *q;
+		}
+
+		/* password */
+		if (*q == ':') {
+			for (q++, i = 0; (*q != '@'); q++)
+				if (i < URL_PWDLEN)
+					u->pwd[i++] = *q;
+		}
+
+		p++;
+	} else {
+		p = URL;
+	}
+
+	/* hostname */
+#ifdef INET6
+	if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&
+	    (*++q == '\0' || *q == '/' || *q == ':')) {
+		if ((i = q - p - 2) > URL_HOSTLEN)
+			i = URL_HOSTLEN;
+		strncpy(u->host, ++p, i);
+		p = q;
+	} else
+#endif
+		for (i = 0; *p && (*p != '/') && (*p != ':'); p++)
+			if (i < URL_HOSTLEN)
+				u->host[i++] = *p;
+
+	/* port */
+	if (*p == ':') {
+		for (q = ++p; *q && (*q != '/'); q++)
+			if (isdigit((unsigned char)*q))
+				u->port = u->port * 10 + (*q - '0');
+			else {
+				/* invalid port */
+				url_seterr(URL_BAD_PORT);
+				goto ouch;
+			}
+		p = q;
+	}
+
+	/* document */
+	if (!*p)
+		p = "/";
+
+quote_doc:
+	count = 1;
+	for (i = 0; p[i] != '\0'; ++i) {
+		if ((!pre_quoted && p[i] == '%') ||
+		    !fetch_urlpath_safe(p[i]))
+			count += 3;
+		else
+			++count;
+	}
+
+	if ((u->doc = malloc(count)) == NULL) {
+		fetch_syserr();
+		goto ouch;
+	}
+	for (i = 0; *p != '\0'; ++p) {
+		if ((!pre_quoted && *p == '%') ||
+		    !fetch_urlpath_safe(*p)) {
+			u->doc[i++] = '%';
+			if ((unsigned char)*p < 160)
+				u->doc[i++] = '0' + ((unsigned char)*p) / 16;
+			else
+				u->doc[i++] = 'a' - 10 + ((unsigned char)*p) / 16;
+			if ((unsigned char)*p % 16 < 10)
+				u->doc[i++] = '0' + ((unsigned char)*p) % 16;
+			else
+				u->doc[i++] = 'a' - 10 + ((unsigned char)*p) % 16;
+		} else
+			u->doc[i++] = *p;
+	}
+	u->doc[i] = '\0';
+
+	return (u);
+
+ouch:
+	free(u);
+	return (NULL);
+}
+
+/*
+ * Free a URL
+ */
+void
+fetchFreeURL(struct url *u)
+{
+	free(u->doc);
+	free(u);
+}
+
+static char
+xdigit2digit(char digit)
+{
+	digit = tolower((unsigned char)digit);
+	if (digit >= 'a' && digit <= 'f')
+		digit = digit - 'a' + 10;
+	else
+		digit = digit - '0';
+
+	return digit;
+}
+
+/*
+ * Unquote whole URL.
+ * Skips optional parts like query or fragment identifier.
+ */ 
+char *
+fetchUnquotePath(struct url *url)
+{
+	char *unquoted;
+	const char *iter;
+	size_t i;
+
+	if ((unquoted = malloc(strlen(url->doc) + 1)) == NULL)
+		return NULL;
+
+	for (i = 0, iter = url->doc; *iter != '\0'; ++iter) {
+		if (*iter == '#' || *iter == '?')
+			break;
+		if (iter[0] != '%' ||
+		    !isxdigit((unsigned char)iter[1]) ||
+		    !isxdigit((unsigned char)iter[2])) {
+			unquoted[i++] = *iter;
+			continue;
+		}
+		unquoted[i++] = xdigit2digit(iter[1]) * 16 +
+		    xdigit2digit(iter[2]);
+		iter += 2;
+	}
+	unquoted[i] = '\0';
+	return unquoted;
+}
+
+
+/*
+ * Extract the file name component of a URL.
+ */
+char *
+fetchUnquoteFilename(struct url *url)
+{
+	char *unquoted, *filename;
+	const char *last_slash;
+
+	if ((unquoted = fetchUnquotePath(url)) == NULL)
+		return NULL;
+
+	if ((last_slash = strrchr(unquoted, '/')) == NULL)
+		return unquoted;
+	filename = strdup(last_slash + 1);
+	free(unquoted);
+	return filename;
+}
+
+char *
+fetchStringifyURL(const struct url *url)
+{
+	size_t total;
+	char *doc;
+
+	/* scheme :// user : pwd @ host :port doc */
+	total = strlen(url->scheme) + 3 + strlen(url->user) + 1 +
+	    strlen(url->pwd) + 1 + strlen(url->host) + 6 + strlen(url->doc) + 1;
+	if ((doc = malloc(total)) == NULL)
+		return NULL;
+	if (url->port != 0)
+		snprintf(doc, total, "%s%s%s%s%s%s%s:%d%s",
+		    url->scheme,
+		    url->scheme[0] != '\0' ? "://" : "",
+		    url->user,
+		    url->pwd[0] != '\0' ? ":" : "",
+		    url->pwd,
+		    url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "",
+		    url->host,
+		    (int)url->port,
+		    url->doc);
+	else {
+		snprintf(doc, total, "%s%s%s%s%s%s%s%s",
+		    url->scheme,
+		    url->scheme[0] != '\0' ? "://" : "",
+		    url->user,
+		    url->pwd[0] != '\0' ? ":" : "",
+		    url->pwd,
+		    url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "",
+		    url->host,
+		    url->doc);
+	}
+	return doc;
+}

lib/fetch/file.c

@@ -0,0 +1,269 @@
+/*	$NetBSD: file.c,v 1.15 2009/10/15 12:36:57 joerg Exp $	*/
+/*-
+ * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
+ * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD: file.c,v 1.18 2007/12/14 10:26:58 des Exp $
+ */
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+#ifndef NETBSD
+#include <nbcompat.h>
+#endif
+
+#include <sys/stat.h>
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <fnmatch.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "fetch.h"
+#include "common.h"
+
+static int	fetch_stat_file(int, struct url_stat *);
+
+static ssize_t
+fetchFile_read(void *cookie, void *buf, size_t len)
+{
+	return read(*(int *)cookie, buf, len);
+}
+
+static ssize_t
+fetchFile_write(void *cookie, const void *buf, size_t len)
+{
+	return write(*(int *)cookie, buf, len);
+}
+
+static void
+fetchFile_close(void *cookie)
+{
+	int fd = *(int *)cookie;
+
+	free(cookie);
+
+	close(fd);
+}
+
+fetchIO *
+fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
+{
+	char *path;
+	fetchIO *f;
+	struct url_stat local_us;
+	int if_modified_since, fd, *cookie;
+
+	if_modified_since = CHECK_FLAG('i');
+	if (if_modified_since && us == NULL)
+		us = &local_us;
+
+	if ((path = fetchUnquotePath(u)) == NULL) {
+		fetch_syserr();
+		return NULL;
+	}
+
+	fd = open(path, O_RDONLY);
+	free(path);
+	if (fd == -1) {
+		fetch_syserr();
+		return NULL;
+	}
+
+	if (us && fetch_stat_file(fd, us) == -1) {
+		close(fd);
+		fetch_syserr();
+		return NULL;
+	}
+
+	if (if_modified_since && u->last_modified > 0 &&
+	    u->last_modified >= us->mtime) {
+		close(fd);
+		fetchLastErrCode = FETCH_UNCHANGED;
+		snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
+		return NULL;
+	}
+
+	if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) {
+		close(fd);
+		fetch_syserr();
+		return NULL;
+	}
+
+	cookie = malloc(sizeof(int));
+	if (cookie == NULL) {
+		close(fd);
+		fetch_syserr();
+		return NULL;
+	}
+
+	*cookie = fd;
+	f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close);
+	if (f == NULL) {
+		close(fd);
+		free(cookie);
+	}
+	return f;
+}
+
+fetchIO *
+fetchGetFile(struct url *u, const char *flags)
+{
+	return (fetchXGetFile(u, NULL, flags));
+}
+
+fetchIO *
+fetchPutFile(struct url *u, const char *flags)
+{
+	char *path;
+	fetchIO *f;
+	int fd, *cookie;
+
+	if ((path = fetchUnquotePath(u)) == NULL) {
+		fetch_syserr();
+		return NULL;
+	}
+
+	if (CHECK_FLAG('a'))
+		fd = open(path, O_WRONLY | O_APPEND);
+	else
+		fd = open(path, O_WRONLY);
+
+	free(path);
+
+	if (fd == -1) {
+		fetch_syserr();
+		return NULL;
+	}
+
+	if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) {
+		close(fd);
+		fetch_syserr();
+		return NULL;
+	}
+
+	cookie = malloc(sizeof(int));
+	if (cookie == NULL) {
+		close(fd);
+		fetch_syserr();
+		return NULL;
+	}
+
+	*cookie = fd;
+	f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close);
+	if (f == NULL) {
+		close(fd);
+		free(cookie);
+	}
+	return f;
+}
+
+static int
+fetch_stat_file(int fd, struct url_stat *us)
+{
+	struct stat sb;
+
+	us->size = -1;
+	us->atime = us->mtime = 0;
+	if (fstat(fd, &sb) == -1) {
+		fetch_syserr();
+		return (-1);
+	}
+	us->size = sb.st_size;
+	us->atime = sb.st_atime;
+	us->mtime = sb.st_mtime;
+	return (0);
+}
+
+int
+fetchStatFile(struct url *u, struct url_stat *us, const char *flags)
+{
+	char *path;
+	int fd, rv;
+
+	(void)flags;
+
+	if ((path = fetchUnquotePath(u)) == NULL) {
+		fetch_syserr();
+		return -1;
+	}
+
+	fd = open(path, O_RDONLY);
+	free(path);
+
+	if (fd == -1) {
+		fetch_syserr();
+		return -1;
+	}
+
+	rv = fetch_stat_file(fd, us);
+	close(fd);
+
+	return rv;
+}
+
+int
+fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const char *flags)
+{
+	char *path;
+	struct dirent *de;
+	DIR *dir;
+	int ret;
+
+	(void)flags;
+
+	if ((path = fetchUnquotePath(u)) == NULL) {
+		fetch_syserr();
+		return -1;
+	}
+		
+	dir = opendir(path);
+	free(path);
+
+	if (dir == NULL) {
+		fetch_syserr();
+		return -1;
+	}
+
+	ret = 0;
+
+	while ((de = readdir(dir)) != NULL) {
+		if (pattern && fnmatch(pattern, de->d_name, 0) != 0)
+			continue;
+		ret = fetch_add_entry(ue, u, de->d_name, 0);
+		if (ret)
+			break;
+	}
+
+	closedir(dir);
+
+	return ret;
+}

lib/fetch/ftp.errors

@@ -0,0 +1,48 @@
+# $NetBSD: ftp.errors,v 1.2 2008/10/06 12:58:29 joerg Exp $
+# $FreeBSD: ftp.errors,v 1.6 2002/10/30 06:06:16 des Exp $
+#
+# This list is taken from RFC 959.
+# It probably needs a going over.
+#
+110 OK		Restart marker reply
+120 TEMP	Service ready in a few minutes
+125 OK		Data connection already open; transfer starting
+150 OK		File status okay; about to open data connection
+200 OK		Command okay
+202 PROTO	Command not implemented, superfluous at this site
+211 INFO	System status, or system help reply
+212 INFO	Directory status
+213 INFO	File status
+214 INFO	Help message
+215 INFO	Set system type
+220 OK		Service ready for new user
+221 OK		Service closing control connection
+225 OK		Data connection open; no transfer in progress
+226 OK		Requested file action successful
+227 OK		Entering Passive Mode
+229 OK		Entering Extended Passive Mode
+230 OK		User logged in, proceed
+250 OK		Requested file action okay, completed
+257 OK		File/directory created
+331 AUTH	User name okay, need password
+332 AUTH	Need account for login
+350 OK		Requested file action pending further information
+421 DOWN	Service not available, closing control connection
+425 NETWORK	Can't open data connection
+426 ABORT	Connection closed; transfer aborted
+450 UNAVAIL	File unavailable (e.g., file busy)
+451 SERVER	Requested action aborted: local error in processing
+452 FULL	Insufficient storage space in system
+500 PROTO	Syntax error, command unrecognized
+501 PROTO	Syntax error in parameters or arguments
+502 PROTO	Command not implemented
+503 PROTO	Bad sequence of commands
+504 PROTO	Command not implemented for that parameter
+530 AUTH	Not logged in
+532 AUTH	Need account for storing files
+535 PROTO	Bug in MediaHawk Video Kernel FTP server
+550 UNAVAIL	File unavailable (e.g., file not found, no access)
+551 PROTO	Requested action aborted. Page type unknown
+552 FULL	Exceeded storage allocation
+553 EXISTS	File name not allowed
+999 PROTO	Protocol error

lib/fetch/http.errors

@@ -0,0 +1,46 @@
+# $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $
+# $NetBSD: http.errors,v 1.3 2009/02/05 16:59:45 joerg Exp $
+#
+# This list is taken from RFC 2068.
+#
+100 OK		Continue
+101 OK		Switching Protocols
+200 OK		OK
+201 OK		Created
+202 OK		Accepted
+203 INFO	Non-Authoritative Information
+204 OK		No Content
+205 OK		Reset Content
+206 OK		Partial Content
+300 MOVED	Multiple Choices
+301 MOVED	Moved Permanently
+302 MOVED	Moved Temporarily
+303 MOVED	See Other
+304 UNCHANGED	Not Modified
+305 INFO	Use Proxy
+307 MOVED	Temporary Redirect
+400 PROTO	Bad Request
+401 AUTH	Unauthorized
+402 AUTH	Payment Required
+403 AUTH	Forbidden
+404 UNAVAIL	Not Found
+405 PROTO	Method Not Allowed
+406 PROTO	Not Acceptable
+407 AUTH	Proxy Authentication Required
+408 TIMEOUT	Request Time-out
+409 EXISTS	Conflict
+410 UNAVAIL	Gone
+411 PROTO	Length Required
+412 SERVER	Precondition Failed
+413 PROTO	Request Entity Too Large
+414 PROTO	Request-URI Too Large
+415 PROTO	Unsupported Media Type
+416 UNAVAIL	Requested Range Not Satisfiable
+417 SERVER	Expectation Failed
+500 SERVER	Internal Server Error
+501 PROTO	Not Implemented
+502 SERVER	Bad Gateway
+503 TEMP	Service Unavailable
+504 TIMEOUT	Gateway Time-out
+505 PROTO	HTTP Version not supported
+999 PROTO	Protocol error

lib/fexec.c

@@ -128,7 +128,7 @@ vfcexec(const char *path, int skipempty, const char *arg, va_list ap)
 	return retval;
 }
 
-int
+int SYMEXPORT
 xbps_file_exec(const char *arg, ...)
 {
 	va_list	ap;
@@ -141,7 +141,7 @@ xbps_file_exec(const char *arg, ...)
 	return result;
 }
 
-int
+int SYMEXPORT
 xbps_file_exec_skipempty(const char *arg, ...)
 {
 	va_list	ap;
@@ -154,7 +154,7 @@ xbps_file_exec_skipempty(const char *arg, ...)
 	return result;
 }
 
-int
+int SYMEXPORT
 xbps_file_chdir_exec(const char *path, const char *arg, ...)
 {
 	va_list	ap;

lib/findpkg.c

@@ -84,7 +84,7 @@ fail:
         return rv;
 }
 
-prop_dictionary_t
+prop_dictionary_t SYMEXPORT
 xbps_get_pkg_props(void)
 {
 	if (pkg_props_initialized == false)
@@ -93,7 +93,7 @@ xbps_get_pkg_props(void)
 	return pkg_props;
 }
 
-int
+int SYMEXPORT
 xbps_prepare_repolist_data(void)
 {
 	prop_dictionary_t dict = NULL;
@@ -160,6 +160,7 @@ xbps_prepare_repolist_data(void)
 			goto out2;
 		}
 
+		rdata->rd_uri = prop_string_cstring(obj);
 		rdata->rd_repod = prop_dictionary_internalize_from_file(plist);
 		if (rdata->rd_repod == NULL) {
 			free(plist);
@@ -184,7 +185,7 @@ out:
 
 }
 
-void
+void SYMEXPORT
 xbps_release_repolist_data(void)
 {
 	struct repository_data *rdata;
@@ -192,11 +193,12 @@ xbps_release_repolist_data(void)
 	while ((rdata = SIMPLEQ_FIRST(&repodata_queue)) != NULL) {
 		SIMPLEQ_REMOVE(&repodata_queue, rdata, repository_data, chain);
 		prop_object_release(rdata->rd_repod);
+		free(rdata->rd_uri);
 		free(rdata);
 	}
 }
 
-int
+int SYMEXPORT
 xbps_find_new_packages(void)
 {
 	prop_dictionary_t dict;
@@ -249,13 +251,13 @@ xbps_find_new_packages(void)
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_find_new_pkg(const char *pkgname, prop_dictionary_t instpkg)
 {
 	prop_dictionary_t pkgrd = NULL;
 	prop_array_t unsorted;
 	struct repository_data *rdata;
-	const char *repoloc, *repover, *instver;
+	const char *repover, *instver;
 	int rv = 0;
 	bool newpkg_found = false;
 
@@ -304,12 +306,7 @@ xbps_find_new_pkg(const char *pkgname, prop_dictionary_t instpkg)
 	/*
 	 * Set repository in pkg dictionary.
 	 */
-	if (!prop_dictionary_get_cstring_nocopy(rdata->rd_repod,
+	prop_dictionary_set_cstring(pkgrd, "repository", rdata->rd_uri);
-	    "location-local", &repoloc)) {
-		rv = EINVAL;
-		goto out;
-	}
-	prop_dictionary_set_cstring(pkgrd, "repository", repoloc);
 
 	/*
 	 * Construct the dependency chain for this package.
@@ -370,13 +367,12 @@ set_pkg_state(prop_dictionary_t pkgd, const char *pkgname)
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_prepare_pkg(const char *pkgname)
 {
 	prop_dictionary_t pkgrd = NULL;
 	prop_array_t pkgs_array;
 	struct repository_data *rdata;
-	const char *repoloc;
 	int rv = 0;
 
 	assert(pkgname != NULL);
@@ -408,12 +404,7 @@ xbps_prepare_pkg(const char *pkgname)
 	/*
 	 * Set repository in pkg dictionary.
 	 */
-	if (!prop_dictionary_get_cstring_nocopy(rdata->rd_repod,
+	prop_dictionary_set_cstring(pkgrd, "repository", rdata->rd_uri);
-	    "location-local", &repoloc)) {
-		rv = EINVAL;
-		goto out;
-	}
-	prop_dictionary_set_cstring(pkgrd, "repository", repoloc);
 	prop_dictionary_set_cstring(pkg_props, "origin", pkgname);
 
 	/*

lib/humanize_number.c

@@ -39,7 +39,7 @@
 
 #include <xbps_api.h>
 
-int
+int SYMEXPORT
 xbps_humanize_number(char *buf, size_t len, int64_t bytes,
     const char *suffix, int scale, int flags)
 {

lib/orphans.c

@@ -120,7 +120,7 @@ cleanup(void)
 	}
 }
 
-prop_array_t
+prop_array_t SYMEXPORT
 xbps_find_orphan_packages(void)
 {
 	prop_array_t array;

lib/plist.c

@@ -34,7 +34,7 @@
 static prop_dictionary_t regpkgdb_dict;
 static bool regpkgdb_initialized;
 
-bool
+bool SYMEXPORT
 xbps_add_obj_to_dict(prop_dictionary_t dict, prop_object_t obj,
 		       const char *key)
 {
@@ -51,7 +51,7 @@ xbps_add_obj_to_dict(prop_dictionary_t dict, prop_object_t obj,
 	return true;
 }
 
-bool
+bool SYMEXPORT
 xbps_add_obj_to_array(prop_array_t array, prop_object_t obj)
 {
 	assert(array != NULL);
@@ -66,7 +66,7 @@ xbps_add_obj_to_array(prop_array_t array, prop_object_t obj)
 	return true;
 }
 
-int
+int SYMEXPORT
 xbps_callback_array_iter_in_repolist(int (*fn)(prop_object_t, void *, bool *),
 				     void *arg)
 {
@@ -103,7 +103,7 @@ xbps_callback_array_iter_in_repolist(int (*fn)(prop_object_t, void *, bool *),
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_callback_array_iter_in_dict(prop_dictionary_t dict, const char *key,
 				 int (*fn)(prop_object_t, void *, bool *),
 				 void *arg)
@@ -131,7 +131,7 @@ xbps_callback_array_iter_in_dict(prop_dictionary_t dict, const char *key,
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_callback_array_iter_reverse_in_dict(prop_dictionary_t dict,
     const char *key, int (*fn)(prop_object_t, void *, bool *), void *arg)
 {
@@ -162,7 +162,7 @@ xbps_callback_array_iter_reverse_in_dict(prop_dictionary_t dict,
 	return rv;
 }
 
-prop_dictionary_t
+prop_dictionary_t SYMEXPORT
 xbps_find_pkg_from_plist(const char *plist, const char *pkgname)
 {
 	prop_dictionary_t dict, obj, res;
@@ -189,7 +189,7 @@ xbps_find_pkg_from_plist(const char *plist, const char *pkgname)
 	return res;
 }
 
-prop_dictionary_t
+prop_dictionary_t SYMEXPORT
 xbps_find_pkg_installed_from_plist(const char *pkgname)
 {
 	prop_dictionary_t d, pkgd;
@@ -215,7 +215,7 @@ xbps_find_pkg_installed_from_plist(const char *pkgname)
 	}
 }
 
-prop_dictionary_t
+prop_dictionary_t SYMEXPORT
 xbps_find_pkg_in_dict(prop_dictionary_t dict, const char *key,
 		      const char *pkgname)
 {
@@ -241,7 +241,7 @@ xbps_find_pkg_in_dict(prop_dictionary_t dict, const char *key,
 	return obj;
 }
 
-prop_dictionary_t
+prop_dictionary_t SYMEXPORT
 xbps_prepare_regpkgdb_dict(void)
 {
 	const char *rootdir;
@@ -266,7 +266,7 @@ xbps_prepare_regpkgdb_dict(void)
 	return regpkgdb_dict;
 }
 
-void
+void SYMEXPORT
 xbps_release_regpkgdb_dict(void)
 {
 	if (regpkgdb_initialized == false)
@@ -277,7 +277,7 @@ xbps_release_regpkgdb_dict(void)
 	regpkgdb_initialized = false;
 }
 
-bool
+bool SYMEXPORT
 xbps_find_string_in_array(prop_array_t array, const char *val)
 {
 	prop_object_iterator_t iter;
@@ -303,7 +303,7 @@ xbps_find_string_in_array(prop_array_t array, const char *val)
 	return false;
 }
 
-prop_object_iterator_t
+prop_object_iterator_t SYMEXPORT
 xbps_get_array_iter_from_dict(prop_dictionary_t dict, const char *key)
 {
 	prop_array_t array;
@@ -318,7 +318,7 @@ xbps_get_array_iter_from_dict(prop_dictionary_t dict, const char *key)
 	return prop_array_iterator(array);
 }
 
-int
+int SYMEXPORT
 xbps_remove_string_from_array(prop_array_t array, const char *str)
 {
 	prop_object_t obj;
@@ -351,7 +351,7 @@ xbps_remove_string_from_array(prop_array_t array, const char *str)
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_remove_pkg_from_dict(prop_dictionary_t dict, const char *key,
 			  const char *pkgname)
 {
@@ -393,7 +393,7 @@ xbps_remove_pkg_from_dict(prop_dictionary_t dict, const char *key,
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_remove_pkg_dict_from_file(const char *pkg, const char *plist)
 {
 	prop_dictionary_t pdict;
@@ -422,7 +422,7 @@ xbps_remove_pkg_dict_from_file(const char *pkg, const char *plist)
 	return 0;
 }
 
-prop_dictionary_t
+prop_dictionary_t SYMEXPORT
 xbps_read_dict_from_archive_entry(struct archive *ar,
 				  struct archive_entry *entry)
 {

lib/purge.c

@@ -33,7 +33,7 @@
 
 static int	remove_pkg_metadata(const char *);
 
-int
+int SYMEXPORT
 xbps_purge_all_pkgs(void)
 {
 
@@ -71,7 +71,7 @@ xbps_purge_all_pkgs(void)
  * This removes configuration files if they weren't modified,
  * removes metadata files and fully unregisters the package.
  */
-int
+int SYMEXPORT
 xbps_purge_pkg(const char *pkgname, bool check_state)
 {
 	prop_dictionary_t dict;

lib/register.c

@@ -30,7 +30,7 @@
 
 #include <xbps_api.h>
 
-int
+int SYMEXPORT
 xbps_register_pkg(prop_dictionary_t pkgrd, bool automatic)
 {
 	prop_dictionary_t dict, pkgd;
@@ -92,7 +92,7 @@ out:
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_unregister_pkg(const char *pkgname)
 {
 	const char *rootdir;

lib/remove.c

@@ -183,7 +183,7 @@ dirs:
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_remove_pkg(const char *pkgname, const char *version, bool update)
 {
 	prop_dictionary_t dict;

lib/repository.c

@@ -23,14 +23,18 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <sys/utsname.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
 
 #include <xbps_api.h>
+#include "fetch.h"
 
-int
+static int	mkpath(char *, mode_t);
+
+int SYMEXPORT
 xbps_register_repository(const char *uri)
 {
 	prop_dictionary_t dict;
@@ -54,8 +58,8 @@ xbps_register_repository(const char *uri)
 		/* Looks like not, create it. */
 		dict = prop_dictionary_create();
 		if (dict == NULL) {
-			rv = errno;
+			free(plist);
-			goto out2;
+			return errno;
 		}
 		/* Create the array and add the repository URI on it. */
 		array = prop_array_create();
@@ -98,16 +102,17 @@ xbps_register_repository(const char *uri)
 		if (obj)
 			prop_object_release(obj);
 		rv = errno;
+		goto out;
 	}
+
 out:
 	prop_object_release(dict);
-out2:
 	free(plist);
 
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_unregister_repository(const char *uri)
 {
 	prop_dictionary_t dict;
@@ -149,3 +154,204 @@ out:
 
 	return rv;
 }
+
+char SYMEXPORT *
+xbps_get_remote_repo_string(const char *uri)
+{
+	struct url *url;
+	size_t i;
+	char *p;
+
+	if ((url = fetchParseURL(uri)) == NULL)
+		return NULL;
+
+	/*
+	 * Replace dots and slashes with underscores, so that
+	 * provided URL:
+	 *
+	 * 	www.foo.org/blah/xbps/binpkg-repo
+	 *
+	 * becomes:
+	 *
+	 * 	www_foo_org_blah_xbps_binpkg_repo
+	 * 	
+	 */
+	p = xbps_xasprintf("%s%s", url->host, url->doc);
+	fetchFreeURL(url);
+	if (p == NULL)
+		return NULL;
+
+	for (i = 0; i < strlen(p); i++) {
+		if (p[i] == '.' || p[i] == '/')
+			p[i] = '_';
+	}
+
+	return p;
+}
+
+int SYMEXPORT
+xbps_sync_repository_pkg_index(const char *uri)
+{
+	struct url *url;
+	struct utsname un;
+	const char *rootdir = xbps_get_rootdir();
+	char *rpidx, *dir, *lrepodir, *uri_fixedp = NULL;
+	int rv = 0;
+
+	if (uname(&un) == -1)
+		return errno;
+
+	if ((url = fetchParseURL(uri)) == NULL)
+		return errno;
+
+	uri_fixedp = xbps_get_remote_repo_string(uri);
+	if (uri_fixedp == NULL) {
+		fetchFreeURL(url);
+		return errno;
+	}
+
+	/*
+	 * Create local arch repodir:
+	 *
+	 * 	<rootdir>/var/db/xbps/repo/<url_path_blah>/<arch>
+	 */
+	lrepodir = xbps_xasprintf("%s/%s/repo/%s/%s",
+	    rootdir, XBPS_META_PATH, uri_fixedp, un.machine);
+	if (lrepodir == NULL) {
+		fetchFreeURL(url);
+		free(uri_fixedp);
+		return errno;
+	}
+	if (mkpath(lrepodir, 0755) == -1) {
+		free(lrepodir);
+		free(uri_fixedp);
+		fetchFreeURL(url);
+		return errno;
+	}
+	/*
+	 * Create local noarch repodir:
+	 *
+	 * 	<rootdir>/var/db/xbps/repo/<url_path_blah>/noarch
+	 */
+	dir = xbps_xasprintf("%s/%s/repo/%s/noarch",
+	    rootdir, XBPS_META_PATH, uri_fixedp);
+	free(uri_fixedp);
+	fetchFreeURL(url);
+	if (dir == NULL) {
+		free(lrepodir);
+		return errno;
+	}
+	if (mkpath(dir, 0755) == -1) {
+		free(dir);
+		free(lrepodir);
+		return errno;
+	}
+	free(dir);
+	/*
+	 * Download pkg-index.plist file from repository.
+	 */
+	rpidx = xbps_xasprintf("%s/%s/%s", uri, un.machine, XBPS_PKGINDEX);
+	if (rpidx == NULL) {
+		free(lrepodir);
+		return errno;
+	}
+	rv = xbps_fetch_file(rpidx, lrepodir);
+
+	free(rpidx);
+	free(lrepodir);
+
+	return rv;
+}
+
+/*
+ * The following is a modified function from NetBSD's src/bin/mkdir/mkdir.c
+ */
+
+/*
+ * Copyright (c) 1983, 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * mkpath -- create directories.
+ *	path     - path
+ *	mode     - file mode of terminal directory
+ */
+static int
+mkpath(char *path, mode_t mode)
+{
+	struct stat sb;
+	char *slash = path;
+	int done = 0, rv;
+	mode_t dir_mode;
+
+	/*
+	 * The default file mode is a=rwx (0777) with selected permissions
+	 * removed in accordance with the file mode creation mask.  For
+	 * intermediate path name components, the mode is the default modified
+	 * by u+wx so that the subdirectories can always be created.
+	 */
+	if (mode == 0)
+		mode = (S_IRWXU | S_IRWXG | S_IRWXO) & ~umask(0);
+
+	dir_mode = mode | S_IWUSR | S_IXUSR;
+
+	for (;;) {
+		slash += strspn(slash, "/");
+		slash += strcspn(slash, "/");
+
+		done = (*slash == '\0');
+		*slash = '\0';
+
+		rv = mkdir(path, done ? mode : dir_mode);
+		if (rv < 0) {
+			/*
+			 * Can't create; path exists or no perms.
+			 * stat() path to determine what's there now.
+			 */
+			int	sverrno;
+
+			sverrno = errno;
+			if (stat(path, &sb) < 0) {
+					/* Not there; use mkdir()s error */
+				errno = sverrno;
+				return -1;
+			}
+			if (!S_ISDIR(sb.st_mode)) {
+					/* Is there, but isn't a directory */
+				errno = ENOTDIR;
+				return -1;
+			}
+		}
+		if (done)
+			break;
+
+		*slash = '/';
+	}
+
+	return 0;
+}

lib/requiredby.c

@@ -116,7 +116,7 @@ remove_pkg_from_reqby(prop_object_t obj, void *arg, bool *loop_done)
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_requiredby_pkg_remove(const char *pkgname)
 {
 	prop_dictionary_t dict;
@@ -149,7 +149,7 @@ xbps_requiredby_pkg_remove(const char *pkgname)
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_requiredby_pkg_add(prop_array_t regar, prop_dictionary_t pkg)
 {
 	prop_array_t rdeps;

lib/sortdeps.c

@@ -54,7 +54,7 @@ find_sorteddep_by_name(const char *pkgname)
 	return sdep;
 }
 
-int
+int SYMEXPORT
 xbps_sort_pkg_deps(prop_dictionary_t chaindeps)
 {
 	prop_array_t sorted, unsorted, rundeps;

lib/state.c

@@ -92,7 +92,7 @@ get_state(prop_dictionary_t dict)
 	return state;
 }
 
-int
+int SYMEXPORT
 xbps_get_pkg_state_installed(const char *pkgname, pkg_state_t *state)
 {
 	prop_dictionary_t dict, pkgd;
@@ -128,7 +128,7 @@ xbps_get_pkg_state_installed(const char *pkgname, pkg_state_t *state)
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_get_pkg_state_dictionary(prop_dictionary_t dict, pkg_state_t *state)
 {
 	assert(dict != NULL);
@@ -139,7 +139,7 @@ xbps_get_pkg_state_dictionary(prop_dictionary_t dict, pkg_state_t *state)
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_set_pkg_state_dictionary(prop_dictionary_t dict, pkg_state_t state)
 {
 	assert(dict != NULL);
@@ -147,7 +147,7 @@ xbps_set_pkg_state_dictionary(prop_dictionary_t dict, pkg_state_t state)
 	return set_new_state(dict, state);
 }
 
-int
+int SYMEXPORT
 xbps_set_pkg_state_installed(const char *pkgname, pkg_state_t state)
 {
 	prop_dictionary_t dict, pkgd;

lib/unpack.c

@@ -35,7 +35,7 @@
 static int unpack_archive_fini(struct archive *, prop_dictionary_t, bool);
 static void set_extract_flags(int *);
 
-int
+int SYMEXPORT
 xbps_unpack_binary_pkg(prop_dictionary_t pkg, bool essential)
 {
 	prop_string_t filename, repoloc, arch;

lib/util.c

@@ -41,7 +41,7 @@ static bool	question(bool, const char *, va_list);
 static const char *rootdir;
 static int flags;
 
-char *
+char SYMEXPORT *
 xbps_get_file_hash(const char *file)
 {
 	SHA256_CTX ctx;
@@ -62,7 +62,7 @@ xbps_get_file_hash(const char *file)
 	return hash;
 }
 
-int
+int SYMEXPORT
 xbps_check_file_hash(const char *path, const char *sha256)
 {
 	char *res;
@@ -80,7 +80,7 @@ xbps_check_file_hash(const char *path, const char *sha256)
 	return 0;
 }
 
-int
+int SYMEXPORT
 xbps_check_pkg_file_hash(prop_dictionary_t pkgd, const char *repoloc)
 {
 	const char *sha256, *arch, *filename;
@@ -109,7 +109,7 @@ xbps_check_pkg_file_hash(prop_dictionary_t pkgd, const char *repoloc)
 	return rv;
 }
 
-int
+int SYMEXPORT
 xbps_check_is_installed_pkg(const char *pkg)
 {
 	prop_dictionary_t dict;
@@ -156,7 +156,7 @@ xbps_check_is_installed_pkg(const char *pkg)
 	return rv;
 }
 
-bool
+bool SYMEXPORT
 xbps_check_is_installed_pkgname(const char *pkgname)
 {
 	prop_dictionary_t pkgd;
@@ -172,7 +172,7 @@ xbps_check_is_installed_pkgname(const char *pkgname)
 	return false;
 }
 
-const char *
+const char SYMEXPORT *
 xbps_get_pkg_version(const char *pkg)
 {
 	const char *tmp;
@@ -187,7 +187,7 @@ xbps_get_pkg_version(const char *pkg)
 	return tmp + 1; /* skip first '-' */
 }
 
-const char *
+const char SYMEXPORT *
 xbps_get_pkg_revision(const char *pkg)
 {
 	const char *tmp;
@@ -202,7 +202,7 @@ xbps_get_pkg_revision(const char *pkg)
 	return tmp + 1; /* skip first '_' */
 }
 
-char *
+char SYMEXPORT *
 xbps_get_pkg_name(const char *pkg)
 {
 	const char *tmp;
@@ -225,20 +225,44 @@ xbps_get_pkg_name(const char *pkg)
 	return pkgname;
 }
 
-char *
+static char *
-xbps_get_pkg_index_plist(const char *path)
+get_pkg_index_remote_plist(const char *uri, const char *machine)
+{
+	char *uri_fixed, *repodir;
+
+	uri_fixed = xbps_get_remote_repo_string(uri);
+	if (uri_fixed == NULL)
+		return NULL;
+
+	repodir = xbps_xasprintf("%s/%s/repo/%s/%s/%s",
+	    xbps_get_rootdir(), XBPS_META_PATH, uri_fixed,
+	    machine, XBPS_PKGINDEX);
+	if (repodir == NULL) {
+		free(uri_fixed);
+		return NULL;
+	}
+		
+	return repodir;
+}
+
+char SYMEXPORT *
+xbps_get_pkg_index_plist(const char *uri)
 {
 	struct utsname un;
 
-	assert(path != NULL);
+	assert(uri != NULL);
 
 	if (uname(&un) == -1)
 		return NULL;
 
-	return xbps_xasprintf("%s/%s/%s", path, un.machine, XBPS_PKGINDEX);
+	if ((strncmp(uri, "http://", 7) == 0) ||
+	    (strncmp(uri, "ftp://", 6) == 0))
+		return get_pkg_index_remote_plist(uri, un.machine);
+
+	return xbps_xasprintf("%s/%s/%s", uri, un.machine, XBPS_PKGINDEX);
 }
 
-bool
+bool SYMEXPORT
 xbps_pkg_has_rundeps(prop_dictionary_t pkg)
 {
 	prop_array_t array;
@@ -251,14 +275,14 @@ xbps_pkg_has_rundeps(prop_dictionary_t pkg)
 	return false;
 }
 
-void
+void SYMEXPORT
 xbps_set_rootdir(const char *dir)
 {
 	assert(dir != NULL);
 	rootdir = dir;
 }
 
-const char *
+const char SYMEXPORT *
 xbps_get_rootdir(void)
 {
 	if (rootdir == NULL)
@@ -267,19 +291,19 @@ xbps_get_rootdir(void)
 	return rootdir;
 }
 
-void
+void SYMEXPORT
 xbps_set_flags(int lflags)
 {
 	flags = lflags;
 }
 
-int
+int SYMEXPORT
 xbps_get_flags(void)
 {
 	return flags;
 }
 
-char *
+char SYMEXPORT *
 xbps_xasprintf(const char *fmt, ...)
 {
 	va_list ap;
@@ -293,11 +317,7 @@ xbps_xasprintf(const char *fmt, ...)
 	return buf;
 }
 
-/*
+bool SYMEXPORT
- * The following functions were taken from pacman (src/pacman/util.c)
- * Copyright (c) 2002-2007 by Judd Vinet <jvinet@zeroflux.org>
- */
-bool
 xbps_yesno(const char *fmt, ...)
 {
 	va_list ap;
@@ -310,7 +330,7 @@ xbps_yesno(const char *fmt, ...)
 	return res;
 }
 
-bool
+bool SYMEXPORT
 xbps_noyes(const char *fmt, ...)
 {
 	va_list ap;

vars.mk

@@ -14,7 +14,9 @@ endif
 STATIC_LIBS ?= -lprop -lpthread -larchive
 LDFLAGS +=  -L$(TOPDIR)/lib -L$(PREFIX)/lib -lxbps
 CPPFLAGS += -I$(TOPDIR)/include -D_XOPEN_SOURCE=600 -D_GNU_SOURCE
+CPPFLAGS += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGE_FILES
 WARNFLAGS ?= -pedantic -std=c99 -Wall -Wextra -Werror -Wshadow -Wformat=2
 WARNFLAGS += -Wmissing-declarations -Wcomment -Wunused-macros -Wendif-labels
-WARNFLAGS += -Wcast-qual -Wcast-align -Wconversion -Wstack-protector
+WARNFLAGS += -Wcast-qual -Wcast-align -Wstack-protector
 CFLAGS = $(DEBUG_FLAGS) $(WARNFLAGS) -fstack-protector-all -O2 -fPIC -DPIC
+CFLAGS += -fvisibility=hidden