emo/xbps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge libfetch-2.30 from NetBSD's pkgsrc.

Browse Source 
--HG--
extra : convert_revision : xtraeme%40gmail.com-20100124144753-ni487x8m7r05847b
This commit is contained in:
Juan RP 2010-01-24 15:47:53 +01:00
parent 489f6baaa6
commit 03e8fa53d9
5 changed files with 277 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/fetch.h
View File

@ -1,6 +1,6 @@
/* $NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $ */ /* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */
/*- /*-
* Copyright (c) 1998-2004 Dag-Erling Coïdn Smorgrav * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -158,6 +158,10 @@ void fetchFreeURLList(struct url_list *);
char *fetchUnquotePath(struct url *); char *fetchUnquotePath(struct url *);
char *fetchUnquoteFilename(struct url *); char *fetchUnquoteFilename(struct url *);
/* Connection caching */
void fetchConnectionCacheInit(int, int);
void fetchConnectionCacheClose(void);
/* Authentication */ /* Authentication */
typedef int (*auth_t)(struct url *); typedef int (*auth_t)(struct url *);
extern auth_t fetchAuthMethod; extern auth_t fetchAuthMethod;

204
lib/fetch/common.c
View File

@ -1,7 +1,7 @@
/* $NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $ */ /* $NetBSD: common.c,v 1.24 2010/01/23 14:25:26 joerg Exp $ */
/*- /*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -65,9 +65,6 @@
#include "fetch.h" #include "fetch.h"
#include "common.h" #include "common.h"
#define DECONST(x,y) ((x)(uintptr_t)(y))
/*** Local data **************************************************************/ /*** Local data **************************************************************/
/* /*
@ -83,10 +80,6 @@ static struct fetcherr netdb_errlist[] = {
{ -1, FETCH_UNKNOWN, "Unknown resolver error" } { -1, FETCH_UNKNOWN, "Unknown resolver error" }
}; };
/* End-of-Line */
static const char ENDL[2] = "\r\n";
/*** Error-reporting functions ***********************************************/ /*** Error-reporting functions ***********************************************/
/* /*
@ -234,23 +227,10 @@ fetch_reopen(int sd)
/* allocate and fill connection structure */ /* allocate and fill connection structure */
if ((conn = calloc(1, sizeof(*conn))) == NULL) if ((conn = calloc(1, sizeof(*conn))) == NULL)
return (NULL); return (NULL);
conn->cache_url = NULL;
conn->next_buf = NULL; conn->next_buf = NULL;
conn->next_len = 0; conn->next_len = 0;
conn->sd = sd; conn->sd = sd;
conn->is_active = 0;
++conn->ref;
return (conn);
}
/*
* Bump a connection's reference count.
*/
conn_t *
fetch_ref(conn_t *conn)
{
++conn->ref;
return (conn); return (conn);
} }
@ -281,7 +261,7 @@ fetch_bind(int sd, int af, const char *addr)
* Establish a TCP connection to the specified port on the specified host. * Establish a TCP connection to the specified port on the specified host.
*/ */
conn_t * conn_t *
fetch_connect(const char *host, int port, int af, int verbose) fetch_connect(struct url *url, int af, int verbose)
{ {
conn_t *conn; conn_t *conn;
char pbuf[10]; char pbuf[10];
@ -290,22 +270,22 @@ fetch_connect(const char *host, int port, int af, int verbose)
int sd, error; int sd, error;
if (verbose) if (verbose)
fetch_info("looking up %s", host); fetch_info("looking up %s", url->host);
/* look up host name and set up socket address structure */ /* look up host name and set up socket address structure */
snprintf(pbuf, sizeof(pbuf), "%d", port); snprintf(pbuf, sizeof(pbuf), "%d", url->port);
memset(&hints, 0, sizeof(hints)); memset(&hints, 0, sizeof(hints));
hints.ai_family = af; hints.ai_family = af;
hints.ai_socktype = SOCK_STREAM; hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = 0; hints.ai_protocol = 0;
if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) { if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) {
netdb_seterr(error); netdb_seterr(error);
return (NULL); return (NULL);
} }
bindaddr = getenv("FETCH_BIND_ADDRESS"); bindaddr = getenv("FETCH_BIND_ADDRESS");
if (verbose) if (verbose)
fetch_info("connecting to %s:%d", host, port); fetch_info("connecting to %s:%d", url->host, url->port);
/* try to connect */ /* try to connect */
for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) { for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
@ -332,9 +312,114 @@ fetch_connect(const char *host, int port, int af, int verbose)
fetch_syserr(); fetch_syserr();
close(sd); close(sd);
} }
conn->cache_url = fetchCopyURL(url);
conn->cache_af = af;
return (conn); return (conn);
} }
static conn_t *connection_cache;
static int cache_global_limit = 0;
static int cache_per_host_limit = 0;
/*
* Initialise cache with the given limits.
*/
void
fetchConnectionCacheInit(int global_limit, int per_host_limit)
{
if (global_limit < 0)
cache_global_limit = INT_MAX;
else if (per_host_limit > global_limit)
cache_global_limit = per_host_limit;
else
cache_global_limit = global_limit;
if (per_host_limit < 0)
cache_per_host_limit = INT_MAX;
else
cache_per_host_limit = per_host_limit;
}
/*
* Flush cache and free all associated resources.
*/
void
fetchConnectionCacheClose(void)
{
conn_t *conn;
while ((conn = connection_cache) != NULL) {
connection_cache = conn->next_cached;
(*conn->cache_close)(conn);
}
}
/*
* Check connection cache for an existing entry matching
* protocol/host/port/user/password/family.
*/
conn_t *
fetch_cache_get(const struct url *url, int af)
{
conn_t *conn, *last_conn = NULL;
for (conn = connection_cache; conn; conn = conn->next_cached) {
if (conn->cache_url->port == url->port &&
strcmp(conn->cache_url->scheme, url->scheme) == 0 &&
strcmp(conn->cache_url->host, url->host) == 0 &&
strcmp(conn->cache_url->user, url->user) == 0 &&
strcmp(conn->cache_url->pwd, url->pwd) == 0 &&
(conn->cache_af == AF_UNSPEC || af == AF_UNSPEC ||
conn->cache_af == af)) {
if (last_conn != NULL)
last_conn->next_cached = conn->next_cached;
else
connection_cache = conn->next_cached;
return conn;
}
}
return NULL;
}
/*
* Put the connection back into the cache for reuse.
* If the connection is freed due to LRU or if the cache
* is explicitly closed, the given callback is called.
*/
void
fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
{
conn_t *iter, *last;
int global_count, host_count;
if (conn->cache_url == NULL || cache_global_limit == 0) {
(*closecb)(conn);
return;
}
global_count = host_count = 0;
last = NULL;
for (iter = connection_cache; iter;
last = iter, iter = iter->next_cached) {
++global_count;
if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0)
++host_count;
if (global_count < cache_global_limit &&
host_count < cache_per_host_limit)
continue;
--global_count;
if (last != NULL)
last->next_cached = iter->next_cached;
else
connection_cache = iter->next_cached;
(*iter->cache_close)(iter);
}
conn->cache_close = closecb;
conn->next_cached = connection_cache;
connection_cache = conn;
}
/* /*
* Enable SSL on a connection. * Enable SSL on a connection.
@ -528,26 +613,12 @@ fetch_getln(conn_t *conn)
return (0); return (0);
} }
/*
* Write to a connection w/ timeout
*/
ssize_t
fetch_write(conn_t *conn, const char *buf, size_t len)
{
struct iovec iov;
iov.iov_base = DECONST(char *, buf);
iov.iov_len = len;
return fetch_writev(conn, &iov, 1);
}
/* /*
* Write a vector to a connection w/ timeout * Write a vector to a connection w/ timeout
* Note: can modify the iovec. * Note: can modify the iovec.
*/ */
ssize_t ssize_t
fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) fetch_write(conn_t *conn, const void *buf, size_t len)
{ {
struct timeval now, timeout, waittv; struct timeval now, timeout, waittv;
fd_set writefds; fd_set writefds;
@ -561,7 +632,7 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
} }
total = 0; total = 0;
while (iovcnt > 0) { while (len) {
while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) { while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
FD_SET(conn->sd, &writefds); FD_SET(conn->sd, &writefds);
gettimeofday(&now, NULL); gettimeofday(&now, NULL);
@ -587,11 +658,10 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
errno = 0; errno = 0;
#ifdef WITH_SSL #ifdef WITH_SSL
if (conn->ssl != NULL) if (conn->ssl != NULL)
wlen = SSL_write(conn->ssl, wlen = SSL_write(conn->ssl, buf, len);
iov->iov_base, iov->iov_len);
else else
#endif #endif
wlen = writev(conn->sd, iov, iovcnt); wlen = send(conn->sd, buf, len, MSG_NOSIGNAL);
if (wlen == 0) { if (wlen == 0) {
/* we consider a short write a failure */ /* we consider a short write a failure */
errno = EPIPE; errno = EPIPE;
@ -604,43 +674,13 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
return (-1); return (-1);
} }
total += wlen; total += wlen;
while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) { buf = (const char *)buf + wlen;
wlen -= iov->iov_len; len -= wlen;
iov++;
iovcnt--;
}
if (iovcnt > 0) {
iov->iov_len -= wlen;
iov->iov_base = DECONST(char *, iov->iov_base) + wlen;
}
} }
return (total); return (total);
} }
/*
* Write a line of text to a connection w/ timeout
*/
int
fetch_putln(conn_t *conn, const char *str, size_t len)
{
struct iovec iov[2];
ssize_t ret;
iov[0].iov_base = DECONST(char *, str);
iov[0].iov_len = len;
iov[1].iov_base = DECONST(char *, ENDL);
iov[1].iov_len = sizeof(ENDL);
if (len == 0)
ret = fetch_writev(conn, &iov[1], 1);
else
ret = fetch_writev(conn, iov, 2);
if (ret == -1)
return (-1);
return (0);
}
/* /*
* Close connection * Close connection
*/ */
@ -649,9 +689,9 @@ fetch_close(conn_t *conn)
{ {
int ret; int ret;
if (--conn->ref > 0)
return (0);
ret = close(conn->sd); ret = close(conn->sd);
if (conn->cache_url)
fetchFreeURL(conn->cache_url);
free(conn->buf); free(conn->buf);
free(conn); free(conn);
return (ret); return (ret);

24
lib/fetch/common.h
View File

@ -1,6 +1,6 @@
/* $NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $ */ /* $NetBSD: common.h,v 1.15 2010/01/23 14:25:26 joerg Exp $ */
/*- /*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -53,6 +53,7 @@
/* Connection */ /* Connection */
typedef struct fetchconn conn_t; typedef struct fetchconn conn_t;
struct fetchconn { struct fetchconn {
int sd; /* socket descriptor */ int sd; /* socket descriptor */
char *buf; /* buffer */ char *buf; /* buffer */
@ -71,8 +72,11 @@ struct fetchconn {
const SSL_METHOD *ssl_meth; /* SSL method */ const SSL_METHOD *ssl_meth; /* SSL method */
# endif # endif
#endif #endif
int ref; /* reference count */
int is_active; struct url *cache_url;
int cache_af;
int (*cache_close)(conn_t *);
conn_t *next_cached;
}; };
/* Structure used for error message lists */ /* Structure used for error message lists */
@ -82,24 +86,20 @@ struct fetcherr {
const char *string; const char *string;
}; };
/* for fetch_writev */
struct iovec;
void fetch_seterr(struct fetcherr *, int); void fetch_seterr(struct fetcherr *, int);
void fetch_syserr(void); void fetch_syserr(void);
void fetch_info(const char *, ...); void fetch_info(const char *, ...);
int fetch_default_port(const char *); int fetch_default_port(const char *);
int fetch_default_proxy_port(const char *); int fetch_default_proxy_port(const char *);
int fetch_bind(int, int, const char *); int fetch_bind(int, int, const char *);
conn_t *fetch_connect(const char *, int, int, int); conn_t *fetch_cache_get(const struct url *, int);
void fetch_cache_put(conn_t *, int (*)(conn_t *));
conn_t *fetch_connect(struct url *, int, int);
conn_t *fetch_reopen(int); conn_t *fetch_reopen(int);
conn_t *fetch_ref(conn_t *);
int fetch_ssl(conn_t *, int); int fetch_ssl(conn_t *, int);
ssize_t fetch_read(conn_t *, char *, size_t); ssize_t fetch_read(conn_t *, char *, size_t);
int fetch_getln(conn_t *); int fetch_getln(conn_t *);
ssize_t fetch_write(conn_t *, const char *, size_t); ssize_t fetch_write(conn_t *, const void *, size_t);
ssize_t fetch_writev(conn_t *, struct iovec *, int);
int fetch_putln(conn_t *, const char *, size_t);
int fetch_close(conn_t *); int fetch_close(conn_t *);
int fetch_add_entry(struct url_list *, struct url *, const char *, int); int fetch_add_entry(struct url_list *, struct url *, const char *, int);
int fetch_netrc_auth(struct url *url); int fetch_netrc_auth(struct url *url);

161
lib/fetch/ftp.c
View File

@ -1,7 +1,7 @@
/* $NetBSD: ftp.c,v 1.30 2009/10/15 12:36:57 joerg Exp $ */ /* $NetBSD: ftp.c,v 1.34 2010/01/23 14:25:26 joerg Exp $ */
/*- /*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
@ -115,9 +115,6 @@
#define FTP_SYNTAX_ERROR 500 #define FTP_SYNTAX_ERROR 500
#define FTP_PROTOCOL_ERROR 999 #define FTP_PROTOCOL_ERROR 999
static struct url cached_host;
static conn_t *cached_connection;
#define isftpreply(foo) \ #define isftpreply(foo) \
(isdigit((unsigned char)foo[0]) && \ (isdigit((unsigned char)foo[0]) && \
isdigit((unsigned char)foo[1]) && \ isdigit((unsigned char)foo[1]) && \
@ -212,7 +209,7 @@ ftp_cmd(conn_t *conn, const char *fmt, ...)
return (-1); return (-1);
} }
r = fetch_putln(conn, msg, len); r = fetch_write(conn, msg, len);
free(msg); free(msg);
if (r == -1) { if (r == -1) {
@ -294,7 +291,7 @@ ftp_cwd(conn_t *conn, const char *file, int subdir)
end = file + strlen(file); end = file + strlen(file);
else if ((end = strrchr(file, '/')) == NULL) else if ((end = strrchr(file, '/')) == NULL)
return (0); return (0);
if ((e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || if ((e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY ||
(e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {
ftp_seterr(e); ftp_seterr(e);
return (-1); return (-1);
@ -311,8 +308,8 @@ ftp_cwd(conn_t *conn, const char *file, int subdir)
break; break;
if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/')) if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/'))
break; break;
if ((e = ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK || if ((e = ftp_cmd(conn, "CDUP\r\n")) != FTP_FILE_ACTION_OK ||
(e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || (e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY ||
(e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {
ftp_seterr(e); ftp_seterr(e);
return (-1); return (-1);
@ -329,7 +326,7 @@ ftp_cwd(conn_t *conn, const char *file, int subdir)
return (0); return (0);
/* Change to the directory all in one chunk (e.g., foo/bar/baz). */ /* Change to the directory all in one chunk (e.g., foo/bar/baz). */
e = ftp_cmd(conn, "CWD %.*s", (int)(end - beg), beg); e = ftp_cmd(conn, "CWD %.*s\r\n", (int)(end - beg), beg);
if (e == FTP_FILE_ACTION_OK) if (e == FTP_FILE_ACTION_OK)
return (0); return (0);
#endif /* FTP_COMBINE_CWDS */ #endif /* FTP_COMBINE_CWDS */
@ -340,7 +337,7 @@ ftp_cwd(conn_t *conn, const char *file, int subdir)
++beg, ++i; ++beg, ++i;
for (++i; file + i < end && file[i] != '/'; ++i) for (++i; file + i < end && file[i] != '/'; ++i)
/* nothing */ ; /* nothing */ ;
e = ftp_cmd(conn, "CWD %.*s", file + i - beg, beg); e = ftp_cmd(conn, "CWD %.*s\r\n", file + i - beg, beg);
if (e != FTP_FILE_ACTION_OK) { if (e != FTP_FILE_ACTION_OK) {
ftp_seterr(e); ftp_seterr(e);
return (-1); return (-1);
@ -366,7 +363,7 @@ ftp_mode_type(conn_t *conn, int mode, int type)
default: default:
return (FTP_PROTOCOL_ERROR); return (FTP_PROTOCOL_ERROR);
} }
if ((e = ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) { if ((e = ftp_cmd(conn, "MODE %c\r\n", mode)) != FTP_OK) {
if (mode == 'S') { if (mode == 'S') {
/* /*
* Stream mode is supposed to be the default - so * Stream mode is supposed to be the default - so
@ -402,7 +399,7 @@ ftp_mode_type(conn_t *conn, int mode, int type)
default: default:
return (FTP_PROTOCOL_ERROR); return (FTP_PROTOCOL_ERROR);
} }
if ((e = ftp_cmd(conn, "TYPE %c", type)) != FTP_OK) if ((e = ftp_cmd(conn, "TYPE %c\r\n", type)) != FTP_OK)
return (e); return (e);
return (FTP_OK); return (FTP_OK);
@ -431,7 +428,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
return (-1); return (-1);
} }
e = ftp_cmd(conn, "SIZE %.*s", filenamelen, filename); e = ftp_cmd(conn, "SIZE %.*s\r\n", filenamelen, filename);
if (e != FTP_FILE_STATUS) { if (e != FTP_FILE_STATUS) {
ftp_seterr(e); ftp_seterr(e);
return (-1); return (-1);
@ -448,7 +445,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
if (us->size == 0) if (us->size == 0)
us->size = -1; us->size = -1;
e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename); e = ftp_cmd(conn, "MDTM %.*s\r\n", filenamelen, filename);
if (e != FTP_FILE_STATUS) { if (e != FTP_FILE_STATUS) {
ftp_seterr(e); ftp_seterr(e);
return (-1); return (-1);
@ -560,6 +557,13 @@ ftp_writefn(void *v, const void *buf, size_t len)
return (-1); return (-1);
} }
static int
ftp_disconnect(conn_t *conn)
{
ftp_cmd(conn, "QUIT\r\n");
return fetch_close(conn);
}
static void static void
ftp_closefn(void *v) ftp_closefn(void *v)
{ {
@ -578,15 +582,10 @@ ftp_closefn(void *v)
return; return;
} }
fetch_close(io->dconn); fetch_close(io->dconn);
io->dir = -1;
io->dconn->is_active = 0;
io->dconn = NULL; io->dconn = NULL;
io->dir = -1;
r = ftp_chkerr(io->cconn); r = ftp_chkerr(io->cconn);
if (io->cconn == cached_connection && io->cconn->ref == 1) { fetch_cache_put(io->cconn, ftp_disconnect);
free(cached_host.doc);
cached_connection = NULL;
}
fetch_close(io->cconn);
free(io); free(io);
return; return;
} }
@ -605,7 +604,6 @@ ftp_setup(conn_t *cconn, conn_t *dconn, int mode)
io->dconn = dconn; io->dconn = dconn;
io->dir = mode; io->dir = mode;
io->eof = io->err = 0; io->eof = io->err = 0;
io->cconn->is_active = 1;
f = fetchIO_unopen(io, ftp_readfn, ftp_writefn, ftp_closefn); f = fetchIO_unopen(io, ftp_readfn, ftp_writefn, ftp_closefn);
if (f == NULL) if (f == NULL)
free(io); free(io);
@ -677,14 +675,14 @@ retry_mode:
fetch_info("setting passive mode"); fetch_info("setting passive mode");
switch (u.ss.ss_family) { switch (u.ss.ss_family) {
case AF_INET: case AF_INET:
if ((e = ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE) if ((e = ftp_cmd(conn, "PASV\r\n")) != FTP_PASSIVE_MODE)
goto ouch; goto ouch;
break; break;
case AF_INET6: case AF_INET6:
if ((e = ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) { if ((e = ftp_cmd(conn, "EPSV\r\n")) != FTP_EPASSIVE_MODE) {
if (e == -1) if (e == -1)
goto ouch; goto ouch;
if ((e = ftp_cmd(conn, "LPSV")) != if ((e = ftp_cmd(conn, "LPSV\r\n")) !=
FTP_LPASSIVE_MODE) FTP_LPASSIVE_MODE)
goto ouch; goto ouch;
} }
@ -744,7 +742,7 @@ retry_mode:
/* seek to required offset */ /* seek to required offset */
if (offset) if (offset)
if (ftp_cmd(conn, "REST %lu", (unsigned long)offset) != FTP_FILE_OK) if (ftp_cmd(conn, "REST %lu\r\n", (unsigned long)offset) != FTP_FILE_OK)
goto sysouch; goto sysouch;
/* construct sockaddr for data socket */ /* construct sockaddr for data socket */
@ -789,9 +787,9 @@ retry_mode:
if (verbose) if (verbose)
fetch_info("initiating transfer"); fetch_info("initiating transfer");
if (op_arg) if (op_arg)
e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg); e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg);
else else
e = ftp_cmd(conn, "%s %.*s", oper, e = ftp_cmd(conn, "%s %.*s\r\n", oper,
filenamelen, filename); filenamelen, filename);
if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)
goto ouch; goto ouch;
@ -840,7 +838,7 @@ retry_mode:
case AF_INET: case AF_INET:
a = ntohl(u.sin4.sin_addr.s_addr); a = ntohl(u.sin4.sin_addr.s_addr);
p = ntohs(u.sin4.sin_port); p = ntohs(u.sin4.sin_port);
e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d", e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d\r\n",
(a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 24) & 0xff, (a >> 16) & 0xff,
(a >> 8) & 0xff, a & 0xff, (a >> 8) & 0xff, a & 0xff,
(p >> 8) & 0xff, p & 0xff); (p >> 8) & 0xff, p & 0xff);
@ -852,7 +850,7 @@ retry_mode:
if (getnameinfo(&u.sa, l, if (getnameinfo(&u.sa, l,
hname, sizeof(hname), hname, sizeof(hname),
NULL, 0, NI_NUMERICHOST) == 0) { NULL, 0, NI_NUMERICHOST) == 0) {
e = ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname, e = ftp_cmd(conn, "EPRT |%d|%s|%d|\r\n", 2, hname,
htons(u.sin6.sin6_port)); htons(u.sin6.sin6_port));
if (e == -1) if (e == -1)
goto ouch; goto ouch;
@ -860,7 +858,7 @@ retry_mode:
if (e != FTP_OK) { if (e != FTP_OK) {
ap = (char *)&u.sin6.sin6_addr; ap = (char *)&u.sin6.sin6_addr;
e = ftp_cmd(conn, e = ftp_cmd(conn,
"LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\r\n",
6, 16, 6, 16,
UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]), UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]),
UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]), UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]),
@ -880,16 +878,16 @@ retry_mode:
/* seek to required offset */ /* seek to required offset */
if (offset) if (offset)
if (ftp_cmd(conn, "REST %llu", (unsigned long long)offset) != FTP_FILE_OK) if (ftp_cmd(conn, "REST %llu\r\n", (unsigned long long)offset) != FTP_FILE_OK)
goto sysouch; goto sysouch;
/* make the server initiate the transfer */ /* make the server initiate the transfer */
if (verbose) if (verbose)
fetch_info("initiating transfer"); fetch_info("initiating transfer");
if (op_arg) if (op_arg)
e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg); e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg);
else else
e = ftp_cmd(conn, "%s %.*s", oper, e = ftp_cmd(conn, "%s %.*s\r\n", oper,
filenamelen, filename); filenamelen, filename);
if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)
goto ouch; goto ouch;
@ -940,11 +938,11 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl)
if (user == NULL || *user == '\0') if (user == NULL || *user == '\0')
user = FTP_ANONYMOUS_USER; user = FTP_ANONYMOUS_USER;
if (purl && url->port == fetch_default_port(url->scheme)) if (purl && url->port == fetch_default_port(url->scheme))
e = ftp_cmd(conn, "USER %s@%s", user, url->host); e = ftp_cmd(conn, "USER %s@%s\r\n", user, url->host);
else if (purl) else if (purl)
e = ftp_cmd(conn, "USER %s@%s@%d", user, url->host, url->port); e = ftp_cmd(conn, "USER %s@%s@%d\r\n", user, url->host, url->port);
else else
e = ftp_cmd(conn, "USER %s", user); e = ftp_cmd(conn, "USER %s\r\n", user);
/* did the server request a password? */ /* did the server request a password? */
if (e == FTP_NEED_PASSWORD) { if (e == FTP_NEED_PASSWORD) {
@ -963,7 +961,7 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl)
pbuf[sizeof(pbuf) - 1] = '\0'; pbuf[sizeof(pbuf) - 1] = '\0';
pwd = pbuf; pwd = pbuf;
} }
e = ftp_cmd(conn, "PASS %s", pwd); e = ftp_cmd(conn, "PASS %s\r\n", pwd);
} }
return (e); return (e);
@ -996,10 +994,23 @@ ftp_connect(struct url *url, struct url *purl, const char *flags)
/* check for proxy */ /* check for proxy */
if (purl) { if (purl) {
/* XXX proxy authentication! */ /* XXX proxy authentication! */
conn = fetch_connect(purl->host, purl->port, af, verbose); /* XXX connetion caching */
if (!purl->port)
purl->port = fetch_default_port(purl->scheme);
conn = fetch_connect(purl, af, verbose);
} else { } else {
/* no proxy, go straight to target */ /* no proxy, go straight to target */
conn = fetch_connect(url->host, url->port, af, verbose); if (!url->port)
url->port = fetch_default_port(url->scheme);
while ((conn = fetch_cache_get(url, af)) != NULL) {
e = ftp_cmd(conn, "NOOP\r\n");
if (e == FTP_OK)
return conn;
fetch_close(conn);
}
conn = fetch_connect(url, af, verbose);
purl = NULL; purl = NULL;
} }
@ -1028,70 +1039,6 @@ fouch:
return (NULL); return (NULL);
} }
/*
* Disconnect from server
*/
static void
ftp_disconnect(conn_t *conn)
{
(void)ftp_cmd(conn, "QUIT");
if (conn == cached_connection && conn->ref == 1) {
free(cached_host.doc);
cached_host.doc = NULL;
cached_connection = NULL;
}
fetch_close(conn);
}
/*
* Check if we're already connected
*/
static int
ftp_isconnected(struct url *url)
{
return (cached_connection
&& (cached_connection->is_active == 0)
&& (strcmp(url->host, cached_host.host) == 0)
&& (strcmp(url->user, cached_host.user) == 0)
&& (strcmp(url->pwd, cached_host.pwd) == 0)
&& (url->port == cached_host.port));
}
/*
* Check the cache, reconnect if no luck
*/
static conn_t *
ftp_cached_connect(struct url *url, struct url *purl, const char *flags)
{
char *doc;
conn_t *conn;
int e;
/* set default port */
if (!url->port)
url->port = fetch_default_port(url->scheme);
/* try to use previously cached connection */
if (ftp_isconnected(url)) {
e = ftp_cmd(cached_connection, "NOOP");
if (e == FTP_OK || e == FTP_SYNTAX_ERROR)
return (fetch_ref(cached_connection));
}
/* connect to server */
if ((conn = ftp_connect(url, purl, flags)) == NULL)
return (NULL);
doc = strdup(url->doc);
if (doc != NULL) {
if (cached_connection && !cached_connection->is_active)
ftp_disconnect(cached_connection);
cached_connection = fetch_ref(conn);
memcpy(&cached_host, url, sizeof(*url));
cached_host.doc = doc;
}
return (conn);
}
/* /*
* Check the proxy settings * Check the proxy settings
*/ */
@ -1150,7 +1097,7 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
} }
/* connect to server */ /* connect to server */
conn = ftp_cached_connect(url, purl, flags); conn = ftp_connect(url, purl, flags);
if (purl) if (purl)
fetchFreeURL(purl); fetchFreeURL(purl);
if (conn == NULL) if (conn == NULL)

103
lib/fetch/http.c
View File

@ -1,6 +1,6 @@
/* $NetBSD: http.c,v 1.25 2009/10/15 12:36:57 joerg Exp $ */ /* $NetBSD: http.c,v 1.28 2010/01/23 14:53:08 joerg Exp $ */
/*- /*-
* Copyright (c) 2000-2004 Dag-Erling Coïdan Smørav * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
* Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org> * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved. * All rights reserved.
@ -137,6 +137,7 @@ struct httpio
{ {
conn_t *conn; /* connection */ conn_t *conn; /* connection */
int chunked; /* chunked mode */ int chunked; /* chunked mode */
int keep_alive; /* keep-alive mode */
char *buf; /* chunk buffer */ char *buf; /* chunk buffer */
size_t bufsize; /* size of chunk buffer */ size_t bufsize; /* size of chunk buffer */
ssize_t buflen; /* amount of data currently in buffer */ ssize_t buflen; /* amount of data currently in buffer */
@ -144,6 +145,7 @@ struct httpio
int eof; /* end-of-file flag */ int eof; /* end-of-file flag */
int error; /* error flag */ int error; /* error flag */
size_t chunksize; /* remaining size of current chunk */ size_t chunksize; /* remaining size of current chunk */
off_t contentlength; /* remaining size of the content */
}; };
/* /*
@ -206,6 +208,9 @@ http_fillbuf(struct httpio *io, size_t len)
if (io->eof) if (io->eof)
return (0); return (0);
if (io->contentlength >= 0 && (off_t)len > io->contentlength)
len = io->contentlength;
if (io->chunked == 0) { if (io->chunked == 0) {
if (http_growbuf(io, len) == -1) if (http_growbuf(io, len) == -1)
return (-1); return (-1);
@ -213,6 +218,8 @@ http_fillbuf(struct httpio *io, size_t len)
io->error = 1; io->error = 1;
return (-1); return (-1);
} }
if (io->contentlength)
io->contentlength -= io->buflen;
io->bufpos = 0; io->bufpos = 0;
return (io->buflen); return (io->buflen);
} }
@ -224,6 +231,8 @@ http_fillbuf(struct httpio *io, size_t len)
return (-1); return (-1);
case 0: case 0:
io->eof = 1; io->eof = 1;
if (fetch_getln(io->conn) == -1)
return (-1);
return (0); return (0);
} }
} }
@ -237,6 +246,8 @@ http_fillbuf(struct httpio *io, size_t len)
return (-1); return (-1);
} }
io->chunksize -= io->buflen; io->chunksize -= io->buflen;
if (io->contentlength >= 0)
io->contentlength -= io->buflen;
if (io->chunksize == 0) { if (io->chunksize == 0) {
char endl[2]; char endl[2];
@ -304,8 +315,22 @@ http_closefn(void *v)
{ {
struct httpio *io = (struct httpio *)v; struct httpio *io = (struct httpio *)v;
if (io->keep_alive) {
int val;
val = 0;
setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,
sizeof(val));
fetch_cache_put(io->conn, fetch_close);
#ifdef TCP_NOPUSH
val = 1;
setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,
sizeof(val));
#endif
} else {
fetch_close(io->conn); fetch_close(io->conn);
if (io->buf) }
free(io->buf); free(io->buf);
free(io); free(io);
} }
@ -314,7 +339,7 @@ http_closefn(void *v)
* Wrap a file descriptor up * Wrap a file descriptor up
*/ */
static fetchIO * static fetchIO *
http_funopen(conn_t *conn, int chunked) http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength)
{ {
struct httpio *io; struct httpio *io;
fetchIO *f; fetchIO *f;
@ -325,6 +350,8 @@ http_funopen(conn_t *conn, int chunked)
} }
io->conn = conn; io->conn = conn;
io->chunked = chunked; io->chunked = chunked;
io->contentlength = clength;
io->keep_alive = keep_alive;
f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn); f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn);
if (f == NULL) { if (f == NULL) {
fetch_syserr(); fetch_syserr();
@ -345,6 +372,7 @@ typedef enum {
hdr_error = -1, hdr_error = -1,
hdr_end = 0, hdr_end = 0,
hdr_unknown = 1, hdr_unknown = 1,
hdr_connection,
hdr_content_length, hdr_content_length,
hdr_content_range, hdr_content_range,
hdr_last_modified, hdr_last_modified,
@ -358,6 +386,7 @@ static struct {
hdr_t num; hdr_t num;
const char *name; const char *name;
} hdr_names[] = { } hdr_names[] = {
{ hdr_connection, "Connection" },
{ hdr_content_length, "Content-Length" }, { hdr_content_length, "Content-Length" },
{ hdr_content_range, "Content-Range" }, { hdr_content_range, "Content-Range" },
{ hdr_last_modified, "Last-Modified" }, { hdr_last_modified, "Last-Modified" },
@ -388,7 +417,7 @@ http_cmd(conn_t *conn, const char *fmt, ...)
return (-1); return (-1);
} }
r = fetch_putln(conn, msg, len); r = fetch_write(conn, msg, len);
free(msg); free(msg);
if (r == -1) { if (r == -1) {
@ -629,7 +658,7 @@ http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
free(upw); free(upw);
if (auth == NULL) if (auth == NULL)
return (-1); return (-1);
r = http_cmd(conn, "%s: Basic %s", hdr, auth); r = http_cmd(conn, "%s: Basic %s\r\n", hdr, auth);
free(auth); free(auth);
return (r); return (r);
} }
@ -671,7 +700,7 @@ http_authorize(conn_t *conn, const char *hdr, const char *p)
* Connect to the correct HTTP server or proxy. * Connect to the correct HTTP server or proxy.
*/ */
static conn_t * static conn_t *
http_connect(struct url *URL, struct url *purl, const char *flags) http_connect(struct url *URL, struct url *purl, const char *flags, int *cached)
{ {
conn_t *conn; conn_t *conn;
int af, verbose; int af, verbose;
@ -679,6 +708,8 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
int val; int val;
#endif #endif
*cached = 1;
#ifdef INET6 #ifdef INET6
af = AF_UNSPEC; af = AF_UNSPEC;
#else #else
@ -701,7 +732,12 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
return (NULL); return (NULL);
} }
if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL) if ((conn = fetch_cache_get(URL, af)) != NULL) {
*cached = 1;
return (conn);
}
if ((conn = fetch_connect(URL, af, verbose)) == NULL)
/* fetch_connect() has already set an error code */ /* fetch_connect() has already set an error code */
return (NULL); return (NULL);
if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
@ -759,7 +795,7 @@ set_if_modified_since(conn_t *conn, time_t last_modified)
snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT", snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT",
weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3, weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3,
tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec); tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec);
http_cmd(conn, "If-Modified-Since: %s", buf); http_cmd(conn, "If-Modified-Since: %s\r\n", buf);
} }
@ -779,7 +815,8 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
{ {
conn_t *conn; conn_t *conn;
struct url *url, *new; struct url *url, *new;
int chunked, direct, if_modified_since, need_auth, noredirect, verbose; int chunked, direct, if_modified_since, need_auth, noredirect;
int keep_alive, verbose, cached;
int e, i, n, val; int e, i, n, val;
off_t offset, clength, length, size; off_t offset, clength, length, size;
time_t mtime; time_t mtime;
@ -792,6 +829,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
noredirect = CHECK_FLAG('A'); noredirect = CHECK_FLAG('A');
verbose = CHECK_FLAG('v'); verbose = CHECK_FLAG('v');
if_modified_since = CHECK_FLAG('i'); if_modified_since = CHECK_FLAG('i');
keep_alive = 0;
if (direct && purl) { if (direct && purl) {
fetchFreeURL(purl); fetchFreeURL(purl);
@ -829,7 +867,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
} }
/* connect to server or proxy */ /* connect to server or proxy */
if ((conn = http_connect(url, purl, flags)) == NULL) if ((conn = http_connect(url, purl, flags, &cached)) == NULL)
goto ouch; goto ouch;
host = url->host; host = url->host;
@ -853,10 +891,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
fetch_info("requesting %s://%s%s", fetch_info("requesting %s://%s%s",
url->scheme, host, url->doc); url->scheme, host, url->doc);
if (purl) { if (purl) {
http_cmd(conn, "%s %s://%s%s HTTP/1.1", http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n",
op, url->scheme, host, url->doc); op, url->scheme, host, url->doc);
} else { } else {
http_cmd(conn, "%s %s HTTP/1.1", http_cmd(conn, "%s %s HTTP/1.1\r\n",
op, url->doc); op, url->doc);
} }
@ -864,7 +902,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
set_if_modified_since(conn, url->last_modified); set_if_modified_since(conn, url->last_modified);
/* virtual host */ /* virtual host */
http_cmd(conn, "Host: %s", host); http_cmd(conn, "Host: %s\r\n", host);
/* proxy authorization */ /* proxy authorization */
if (purl) { if (purl) {
@ -892,19 +930,18 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
/* other headers */ /* other headers */
if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') { if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') {
if (strcasecmp(p, "auto") == 0) if (strcasecmp(p, "auto") == 0)
http_cmd(conn, "Referer: %s://%s%s", http_cmd(conn, "Referer: %s://%s%s\r\n",
url->scheme, host, url->doc); url->scheme, host, url->doc);
else else
http_cmd(conn, "Referer: %s", p); http_cmd(conn, "Referer: %s\r\n", p);
} }
if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0') if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')
http_cmd(conn, "User-Agent: %s", p); http_cmd(conn, "User-Agent: %s\r\n", p);
else else
http_cmd(conn, "User-Agent: %s ", _LIBFETCH_VER); http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER);
if (url->offset > 0) if (url->offset > 0)
http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset); http_cmd(conn, "Range: bytes=%lld-\r\n", (long long)url->offset);
http_cmd(conn, "Connection: close"); http_cmd(conn, "\r\n");
http_cmd(conn, "");
/* /*
* Force the queued request to be dispatched. Normally, one * Force the queued request to be dispatched. Normally, one
@ -968,6 +1005,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
case HTTP_PROTOCOL_ERROR: case HTTP_PROTOCOL_ERROR:
/* fall through */ /* fall through */
case -1: case -1:
--i;
if (cached)
continue;
fetch_syserr(); fetch_syserr();
goto ouch; goto ouch;
default: default:
@ -986,6 +1026,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
case hdr_error: case hdr_error:
http_seterr(HTTP_PROTOCOL_ERROR); http_seterr(HTTP_PROTOCOL_ERROR);
goto ouch; goto ouch;
case hdr_connection:
/* XXX too weak? */
keep_alive = (strcasecmp(p, "keep-alive") == 0);
break;
case hdr_content_length: case hdr_content_length:
http_parse_length(p, &clength); http_parse_length(p, &clength);
break; break;
@ -1115,13 +1159,20 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
URL->offset = offset; URL->offset = offset;
URL->length = clength; URL->length = clength;
if (clength == -1 && !chunked)
keep_alive = 0;
if (conn->err == HTTP_NOT_MODIFIED) { if (conn->err == HTTP_NOT_MODIFIED) {
http_seterr(HTTP_NOT_MODIFIED); http_seterr(HTTP_NOT_MODIFIED);
if (keep_alive) {
fetch_cache_put(conn, fetch_close);
conn = NULL;
}
goto ouch; goto ouch;
} }
/* wrap it up in a fetchIO */ /* wrap it up in a fetchIO */
if ((f = http_funopen(conn, chunked)) == NULL) { if ((f = http_funopen(conn, chunked, keep_alive, clength)) == NULL) {
fetch_syserr(); fetch_syserr();
goto ouch; goto ouch;
} }
@ -1132,6 +1183,13 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
fetchFreeURL(purl); fetchFreeURL(purl);
if (HTTP_ERROR(conn->err)) { if (HTTP_ERROR(conn->err)) {
if (keep_alive) {
char buf[512];
do {
} while (fetchIO_read(f, buf, sizeof(buf)) > 0);
}
fetchIO_close(f); fetchIO_close(f);
f = NULL; f = NULL;
} }
@ -1354,6 +1412,7 @@ parse_index(struct index_parser *parser, const char *buf, size_t len)
return -1; return -1;
return end_attr + 1 - buf; return end_attr + 1 - buf;
} }
/* NOTREACHED */
abort(); abort();
} }