diff --git a/include/fetch.h b/include/fetch.h index 986d06d5..30754864 100644 --- a/include/fetch.h +++ b/include/fetch.h @@ -1,6 +1,6 @@ -/* $NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdn Smorgrav + * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -158,6 +158,10 @@ void fetchFreeURLList(struct url_list *); char *fetchUnquotePath(struct url *); char *fetchUnquoteFilename(struct url *); +/* Connection caching */ +void fetchConnectionCacheInit(int, int); +void fetchConnectionCacheClose(void); + /* Authentication */ typedef int (*auth_t)(struct url *); extern auth_t fetchAuthMethod; diff --git a/lib/fetch/common.c b/lib/fetch/common.c index a057ae27..81cb54a5 100644 --- a/lib/fetch/common.c +++ b/lib/fetch/common.c @@ -1,7 +1,7 @@ -/* $NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: common.c,v 1.24 2010/01/23 14:25:26 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav - * Copyright (c) 2008 Joerg Sonnenberger + * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav + * Copyright (c) 2008, 2010 Joerg Sonnenberger * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -65,9 +65,6 @@ #include "fetch.h" #include "common.h" -#define DECONST(x,y) ((x)(uintptr_t)(y)) - - /*** Local data **************************************************************/ /* @@ -83,10 +80,6 @@ static struct fetcherr netdb_errlist[] = { { -1, FETCH_UNKNOWN, "Unknown resolver error" } }; -/* End-of-Line */ -static const char ENDL[2] = "\r\n"; - - /*** Error-reporting functions ***********************************************/ /* @@ -234,23 +227,10 @@ fetch_reopen(int sd) /* allocate and fill connection structure */ if ((conn = calloc(1, sizeof(*conn))) == NULL) return (NULL); + conn->cache_url = NULL; conn->next_buf = NULL; conn->next_len = 0; conn->sd = sd; - conn->is_active = 0; - ++conn->ref; - return (conn); -} - - -/* - * Bump a connection's reference count. - */ -conn_t * -fetch_ref(conn_t *conn) -{ - - ++conn->ref; return (conn); } @@ -281,7 +261,7 @@ fetch_bind(int sd, int af, const char *addr) * Establish a TCP connection to the specified port on the specified host. */ conn_t * -fetch_connect(const char *host, int port, int af, int verbose) +fetch_connect(struct url *url, int af, int verbose) { conn_t *conn; char pbuf[10]; @@ -290,22 +270,22 @@ fetch_connect(const char *host, int port, int af, int verbose) int sd, error; if (verbose) - fetch_info("looking up %s", host); + fetch_info("looking up %s", url->host); /* look up host name and set up socket address structure */ - snprintf(pbuf, sizeof(pbuf), "%d", port); + snprintf(pbuf, sizeof(pbuf), "%d", url->port); memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; - if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) { + if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) { netdb_seterr(error); return (NULL); } bindaddr = getenv("FETCH_BIND_ADDRESS"); if (verbose) - fetch_info("connecting to %s:%d", host, port); + fetch_info("connecting to %s:%d", url->host, url->port); /* try to connect */ for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) { @@ -332,9 +312,114 @@ fetch_connect(const char *host, int port, int af, int verbose) fetch_syserr(); close(sd); } + conn->cache_url = fetchCopyURL(url); + conn->cache_af = af; return (conn); } +static conn_t *connection_cache; +static int cache_global_limit = 0; +static int cache_per_host_limit = 0; + +/* + * Initialise cache with the given limits. + */ +void +fetchConnectionCacheInit(int global_limit, int per_host_limit) +{ + + if (global_limit < 0) + cache_global_limit = INT_MAX; + else if (per_host_limit > global_limit) + cache_global_limit = per_host_limit; + else + cache_global_limit = global_limit; + if (per_host_limit < 0) + cache_per_host_limit = INT_MAX; + else + cache_per_host_limit = per_host_limit; +} + +/* + * Flush cache and free all associated resources. + */ +void +fetchConnectionCacheClose(void) +{ + conn_t *conn; + + while ((conn = connection_cache) != NULL) { + connection_cache = conn->next_cached; + (*conn->cache_close)(conn); + } +} + +/* + * Check connection cache for an existing entry matching + * protocol/host/port/user/password/family. + */ +conn_t * +fetch_cache_get(const struct url *url, int af) +{ + conn_t *conn, *last_conn = NULL; + + for (conn = connection_cache; conn; conn = conn->next_cached) { + if (conn->cache_url->port == url->port && + strcmp(conn->cache_url->scheme, url->scheme) == 0 && + strcmp(conn->cache_url->host, url->host) == 0 && + strcmp(conn->cache_url->user, url->user) == 0 && + strcmp(conn->cache_url->pwd, url->pwd) == 0 && + (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC || + conn->cache_af == af)) { + if (last_conn != NULL) + last_conn->next_cached = conn->next_cached; + else + connection_cache = conn->next_cached; + return conn; + } + } + + return NULL; +} + +/* + * Put the connection back into the cache for reuse. + * If the connection is freed due to LRU or if the cache + * is explicitly closed, the given callback is called. + */ +void +fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) +{ + conn_t *iter, *last; + int global_count, host_count; + + if (conn->cache_url == NULL || cache_global_limit == 0) { + (*closecb)(conn); + return; + } + + global_count = host_count = 0; + last = NULL; + for (iter = connection_cache; iter; + last = iter, iter = iter->next_cached) { + ++global_count; + if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0) + ++host_count; + if (global_count < cache_global_limit && + host_count < cache_per_host_limit) + continue; + --global_count; + if (last != NULL) + last->next_cached = iter->next_cached; + else + connection_cache = iter->next_cached; + (*iter->cache_close)(iter); + } + + conn->cache_close = closecb; + conn->next_cached = connection_cache; + connection_cache = conn; +} /* * Enable SSL on a connection. @@ -528,26 +613,12 @@ fetch_getln(conn_t *conn) return (0); } - -/* - * Write to a connection w/ timeout - */ -ssize_t -fetch_write(conn_t *conn, const char *buf, size_t len) -{ - struct iovec iov; - - iov.iov_base = DECONST(char *, buf); - iov.iov_len = len; - return fetch_writev(conn, &iov, 1); -} - /* * Write a vector to a connection w/ timeout * Note: can modify the iovec. */ ssize_t -fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) +fetch_write(conn_t *conn, const void *buf, size_t len) { struct timeval now, timeout, waittv; fd_set writefds; @@ -561,7 +632,7 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) } total = 0; - while (iovcnt > 0) { + while (len) { while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) { FD_SET(conn->sd, &writefds); gettimeofday(&now, NULL); @@ -587,11 +658,10 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) errno = 0; #ifdef WITH_SSL if (conn->ssl != NULL) - wlen = SSL_write(conn->ssl, - iov->iov_base, iov->iov_len); + wlen = SSL_write(conn->ssl, buf, len); else #endif - wlen = writev(conn->sd, iov, iovcnt); + wlen = send(conn->sd, buf, len, MSG_NOSIGNAL); if (wlen == 0) { /* we consider a short write a failure */ errno = EPIPE; @@ -604,43 +674,13 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) return (-1); } total += wlen; - while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) { - wlen -= iov->iov_len; - iov++; - iovcnt--; - } - if (iovcnt > 0) { - iov->iov_len -= wlen; - iov->iov_base = DECONST(char *, iov->iov_base) + wlen; - } + buf = (const char *)buf + wlen; + len -= wlen; } return (total); } -/* - * Write a line of text to a connection w/ timeout - */ -int -fetch_putln(conn_t *conn, const char *str, size_t len) -{ - struct iovec iov[2]; - ssize_t ret; - - iov[0].iov_base = DECONST(char *, str); - iov[0].iov_len = len; - iov[1].iov_base = DECONST(char *, ENDL); - iov[1].iov_len = sizeof(ENDL); - if (len == 0) - ret = fetch_writev(conn, &iov[1], 1); - else - ret = fetch_writev(conn, iov, 2); - if (ret == -1) - return (-1); - return (0); -} - - /* * Close connection */ @@ -649,9 +689,9 @@ fetch_close(conn_t *conn) { int ret; - if (--conn->ref > 0) - return (0); ret = close(conn->sd); + if (conn->cache_url) + fetchFreeURL(conn->cache_url); free(conn->buf); free(conn); return (ret); diff --git a/lib/fetch/common.h b/lib/fetch/common.h index 93c415bc..2039aedc 100644 --- a/lib/fetch/common.h +++ b/lib/fetch/common.h @@ -1,6 +1,6 @@ -/* $NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $ */ +/* $NetBSD: common.h,v 1.15 2010/01/23 14:25:26 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav + * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -53,6 +53,7 @@ /* Connection */ typedef struct fetchconn conn_t; + struct fetchconn { int sd; /* socket descriptor */ char *buf; /* buffer */ @@ -71,8 +72,11 @@ struct fetchconn { const SSL_METHOD *ssl_meth; /* SSL method */ # endif #endif - int ref; /* reference count */ - int is_active; + + struct url *cache_url; + int cache_af; + int (*cache_close)(conn_t *); + conn_t *next_cached; }; /* Structure used for error message lists */ @@ -82,24 +86,20 @@ struct fetcherr { const char *string; }; -/* for fetch_writev */ -struct iovec; - void fetch_seterr(struct fetcherr *, int); void fetch_syserr(void); void fetch_info(const char *, ...); int fetch_default_port(const char *); int fetch_default_proxy_port(const char *); int fetch_bind(int, int, const char *); -conn_t *fetch_connect(const char *, int, int, int); +conn_t *fetch_cache_get(const struct url *, int); +void fetch_cache_put(conn_t *, int (*)(conn_t *)); +conn_t *fetch_connect(struct url *, int, int); conn_t *fetch_reopen(int); -conn_t *fetch_ref(conn_t *); int fetch_ssl(conn_t *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); -ssize_t fetch_write(conn_t *, const char *, size_t); -ssize_t fetch_writev(conn_t *, struct iovec *, int); -int fetch_putln(conn_t *, const char *, size_t); +ssize_t fetch_write(conn_t *, const void *, size_t); int fetch_close(conn_t *); int fetch_add_entry(struct url_list *, struct url *, const char *, int); int fetch_netrc_auth(struct url *url); diff --git a/lib/fetch/ftp.c b/lib/fetch/ftp.c index e2b54eaf..6773572b 100644 --- a/lib/fetch/ftp.c +++ b/lib/fetch/ftp.c @@ -1,7 +1,7 @@ -/* $NetBSD: ftp.c,v 1.30 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: ftp.c,v 1.34 2010/01/23 14:25:26 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav - * Copyright (c) 2008, 2009 Joerg Sonnenberger + * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav + * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -115,9 +115,6 @@ #define FTP_SYNTAX_ERROR 500 #define FTP_PROTOCOL_ERROR 999 -static struct url cached_host; -static conn_t *cached_connection; - #define isftpreply(foo) \ (isdigit((unsigned char)foo[0]) && \ isdigit((unsigned char)foo[1]) && \ @@ -212,7 +209,7 @@ ftp_cmd(conn_t *conn, const char *fmt, ...) return (-1); } - r = fetch_putln(conn, msg, len); + r = fetch_write(conn, msg, len); free(msg); if (r == -1) { @@ -294,7 +291,7 @@ ftp_cwd(conn_t *conn, const char *file, int subdir) end = file + strlen(file); else if ((end = strrchr(file, '/')) == NULL) return (0); - if ((e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || + if ((e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY || (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { ftp_seterr(e); return (-1); @@ -311,8 +308,8 @@ ftp_cwd(conn_t *conn, const char *file, int subdir) break; if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/')) break; - if ((e = ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK || - (e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || + if ((e = ftp_cmd(conn, "CDUP\r\n")) != FTP_FILE_ACTION_OK || + (e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY || (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { ftp_seterr(e); return (-1); @@ -329,7 +326,7 @@ ftp_cwd(conn_t *conn, const char *file, int subdir) return (0); /* Change to the directory all in one chunk (e.g., foo/bar/baz). */ - e = ftp_cmd(conn, "CWD %.*s", (int)(end - beg), beg); + e = ftp_cmd(conn, "CWD %.*s\r\n", (int)(end - beg), beg); if (e == FTP_FILE_ACTION_OK) return (0); #endif /* FTP_COMBINE_CWDS */ @@ -340,7 +337,7 @@ ftp_cwd(conn_t *conn, const char *file, int subdir) ++beg, ++i; for (++i; file + i < end && file[i] != '/'; ++i) /* nothing */ ; - e = ftp_cmd(conn, "CWD %.*s", file + i - beg, beg); + e = ftp_cmd(conn, "CWD %.*s\r\n", file + i - beg, beg); if (e != FTP_FILE_ACTION_OK) { ftp_seterr(e); return (-1); @@ -366,7 +363,7 @@ ftp_mode_type(conn_t *conn, int mode, int type) default: return (FTP_PROTOCOL_ERROR); } - if ((e = ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) { + if ((e = ftp_cmd(conn, "MODE %c\r\n", mode)) != FTP_OK) { if (mode == 'S') { /* * Stream mode is supposed to be the default - so @@ -402,7 +399,7 @@ ftp_mode_type(conn_t *conn, int mode, int type) default: return (FTP_PROTOCOL_ERROR); } - if ((e = ftp_cmd(conn, "TYPE %c", type)) != FTP_OK) + if ((e = ftp_cmd(conn, "TYPE %c\r\n", type)) != FTP_OK) return (e); return (FTP_OK); @@ -431,7 +428,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us) return (-1); } - e = ftp_cmd(conn, "SIZE %.*s", filenamelen, filename); + e = ftp_cmd(conn, "SIZE %.*s\r\n", filenamelen, filename); if (e != FTP_FILE_STATUS) { ftp_seterr(e); return (-1); @@ -448,7 +445,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us) if (us->size == 0) us->size = -1; - e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename); + e = ftp_cmd(conn, "MDTM %.*s\r\n", filenamelen, filename); if (e != FTP_FILE_STATUS) { ftp_seterr(e); return (-1); @@ -560,6 +557,13 @@ ftp_writefn(void *v, const void *buf, size_t len) return (-1); } +static int +ftp_disconnect(conn_t *conn) +{ + ftp_cmd(conn, "QUIT\r\n"); + return fetch_close(conn); +} + static void ftp_closefn(void *v) { @@ -578,15 +582,10 @@ ftp_closefn(void *v) return; } fetch_close(io->dconn); - io->dir = -1; - io->dconn->is_active = 0; io->dconn = NULL; + io->dir = -1; r = ftp_chkerr(io->cconn); - if (io->cconn == cached_connection && io->cconn->ref == 1) { - free(cached_host.doc); - cached_connection = NULL; - } - fetch_close(io->cconn); + fetch_cache_put(io->cconn, ftp_disconnect); free(io); return; } @@ -605,7 +604,6 @@ ftp_setup(conn_t *cconn, conn_t *dconn, int mode) io->dconn = dconn; io->dir = mode; io->eof = io->err = 0; - io->cconn->is_active = 1; f = fetchIO_unopen(io, ftp_readfn, ftp_writefn, ftp_closefn); if (f == NULL) free(io); @@ -677,14 +675,14 @@ retry_mode: fetch_info("setting passive mode"); switch (u.ss.ss_family) { case AF_INET: - if ((e = ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE) + if ((e = ftp_cmd(conn, "PASV\r\n")) != FTP_PASSIVE_MODE) goto ouch; break; case AF_INET6: - if ((e = ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) { + if ((e = ftp_cmd(conn, "EPSV\r\n")) != FTP_EPASSIVE_MODE) { if (e == -1) goto ouch; - if ((e = ftp_cmd(conn, "LPSV")) != + if ((e = ftp_cmd(conn, "LPSV\r\n")) != FTP_LPASSIVE_MODE) goto ouch; } @@ -744,7 +742,7 @@ retry_mode: /* seek to required offset */ if (offset) - if (ftp_cmd(conn, "REST %lu", (unsigned long)offset) != FTP_FILE_OK) + if (ftp_cmd(conn, "REST %lu\r\n", (unsigned long)offset) != FTP_FILE_OK) goto sysouch; /* construct sockaddr for data socket */ @@ -789,9 +787,9 @@ retry_mode: if (verbose) fetch_info("initiating transfer"); if (op_arg) - e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg); + e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg); else - e = ftp_cmd(conn, "%s %.*s", oper, + e = ftp_cmd(conn, "%s %.*s\r\n", oper, filenamelen, filename); if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -840,7 +838,7 @@ retry_mode: case AF_INET: a = ntohl(u.sin4.sin_addr.s_addr); p = ntohs(u.sin4.sin_port); - e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d", + e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d\r\n", (a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff, (p >> 8) & 0xff, p & 0xff); @@ -852,7 +850,7 @@ retry_mode: if (getnameinfo(&u.sa, l, hname, sizeof(hname), NULL, 0, NI_NUMERICHOST) == 0) { - e = ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname, + e = ftp_cmd(conn, "EPRT |%d|%s|%d|\r\n", 2, hname, htons(u.sin6.sin6_port)); if (e == -1) goto ouch; @@ -860,7 +858,7 @@ retry_mode: if (e != FTP_OK) { ap = (char *)&u.sin6.sin6_addr; e = ftp_cmd(conn, - "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", + "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\r\n", 6, 16, UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]), UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]), @@ -880,16 +878,16 @@ retry_mode: /* seek to required offset */ if (offset) - if (ftp_cmd(conn, "REST %llu", (unsigned long long)offset) != FTP_FILE_OK) + if (ftp_cmd(conn, "REST %llu\r\n", (unsigned long long)offset) != FTP_FILE_OK) goto sysouch; /* make the server initiate the transfer */ if (verbose) fetch_info("initiating transfer"); if (op_arg) - e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg); + e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg); else - e = ftp_cmd(conn, "%s %.*s", oper, + e = ftp_cmd(conn, "%s %.*s\r\n", oper, filenamelen, filename); if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -940,11 +938,11 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl) if (user == NULL || *user == '\0') user = FTP_ANONYMOUS_USER; if (purl && url->port == fetch_default_port(url->scheme)) - e = ftp_cmd(conn, "USER %s@%s", user, url->host); + e = ftp_cmd(conn, "USER %s@%s\r\n", user, url->host); else if (purl) - e = ftp_cmd(conn, "USER %s@%s@%d", user, url->host, url->port); + e = ftp_cmd(conn, "USER %s@%s@%d\r\n", user, url->host, url->port); else - e = ftp_cmd(conn, "USER %s", user); + e = ftp_cmd(conn, "USER %s\r\n", user); /* did the server request a password? */ if (e == FTP_NEED_PASSWORD) { @@ -963,7 +961,7 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl) pbuf[sizeof(pbuf) - 1] = '\0'; pwd = pbuf; } - e = ftp_cmd(conn, "PASS %s", pwd); + e = ftp_cmd(conn, "PASS %s\r\n", pwd); } return (e); @@ -996,10 +994,23 @@ ftp_connect(struct url *url, struct url *purl, const char *flags) /* check for proxy */ if (purl) { /* XXX proxy authentication! */ - conn = fetch_connect(purl->host, purl->port, af, verbose); + /* XXX connetion caching */ + if (!purl->port) + purl->port = fetch_default_port(purl->scheme); + + conn = fetch_connect(purl, af, verbose); } else { /* no proxy, go straight to target */ - conn = fetch_connect(url->host, url->port, af, verbose); + if (!url->port) + url->port = fetch_default_port(url->scheme); + + while ((conn = fetch_cache_get(url, af)) != NULL) { + e = ftp_cmd(conn, "NOOP\r\n"); + if (e == FTP_OK) + return conn; + fetch_close(conn); + } + conn = fetch_connect(url, af, verbose); purl = NULL; } @@ -1028,70 +1039,6 @@ fouch: return (NULL); } -/* - * Disconnect from server - */ -static void -ftp_disconnect(conn_t *conn) -{ - (void)ftp_cmd(conn, "QUIT"); - if (conn == cached_connection && conn->ref == 1) { - free(cached_host.doc); - cached_host.doc = NULL; - cached_connection = NULL; - } - fetch_close(conn); -} - -/* - * Check if we're already connected - */ -static int -ftp_isconnected(struct url *url) -{ - return (cached_connection - && (cached_connection->is_active == 0) - && (strcmp(url->host, cached_host.host) == 0) - && (strcmp(url->user, cached_host.user) == 0) - && (strcmp(url->pwd, cached_host.pwd) == 0) - && (url->port == cached_host.port)); -} - -/* - * Check the cache, reconnect if no luck - */ -static conn_t * -ftp_cached_connect(struct url *url, struct url *purl, const char *flags) -{ - char *doc; - conn_t *conn; - int e; - - /* set default port */ - if (!url->port) - url->port = fetch_default_port(url->scheme); - - /* try to use previously cached connection */ - if (ftp_isconnected(url)) { - e = ftp_cmd(cached_connection, "NOOP"); - if (e == FTP_OK || e == FTP_SYNTAX_ERROR) - return (fetch_ref(cached_connection)); - } - - /* connect to server */ - if ((conn = ftp_connect(url, purl, flags)) == NULL) - return (NULL); - doc = strdup(url->doc); - if (doc != NULL) { - if (cached_connection && !cached_connection->is_active) - ftp_disconnect(cached_connection); - cached_connection = fetch_ref(conn); - memcpy(&cached_host, url, sizeof(*url)); - cached_host.doc = doc; - } - return (conn); -} - /* * Check the proxy settings */ @@ -1150,7 +1097,7 @@ ftp_request(struct url *url, const char *op, const char *op_arg, } /* connect to server */ - conn = ftp_cached_connect(url, purl, flags); + conn = ftp_connect(url, purl, flags); if (purl) fetchFreeURL(purl); if (conn == NULL) diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 82e48675..38e8e8b7 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1,6 +1,6 @@ -/* $NetBSD: http.c,v 1.25 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: http.c,v 1.28 2010/01/23 14:53:08 joerg Exp $ */ /*- - * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørav + * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2003 Thomas Klausner * Copyright (c) 2008, 2009 Joerg Sonnenberger * All rights reserved. @@ -137,6 +137,7 @@ struct httpio { conn_t *conn; /* connection */ int chunked; /* chunked mode */ + int keep_alive; /* keep-alive mode */ char *buf; /* chunk buffer */ size_t bufsize; /* size of chunk buffer */ ssize_t buflen; /* amount of data currently in buffer */ @@ -144,6 +145,7 @@ struct httpio int eof; /* end-of-file flag */ int error; /* error flag */ size_t chunksize; /* remaining size of current chunk */ + off_t contentlength; /* remaining size of the content */ }; /* @@ -206,6 +208,9 @@ http_fillbuf(struct httpio *io, size_t len) if (io->eof) return (0); + if (io->contentlength >= 0 && (off_t)len > io->contentlength) + len = io->contentlength; + if (io->chunked == 0) { if (http_growbuf(io, len) == -1) return (-1); @@ -213,6 +218,8 @@ http_fillbuf(struct httpio *io, size_t len) io->error = 1; return (-1); } + if (io->contentlength) + io->contentlength -= io->buflen; io->bufpos = 0; return (io->buflen); } @@ -224,6 +231,8 @@ http_fillbuf(struct httpio *io, size_t len) return (-1); case 0: io->eof = 1; + if (fetch_getln(io->conn) == -1) + return (-1); return (0); } } @@ -237,6 +246,8 @@ http_fillbuf(struct httpio *io, size_t len) return (-1); } io->chunksize -= io->buflen; + if (io->contentlength >= 0) + io->contentlength -= io->buflen; if (io->chunksize == 0) { char endl[2]; @@ -304,9 +315,23 @@ http_closefn(void *v) { struct httpio *io = (struct httpio *)v; - fetch_close(io->conn); - if (io->buf) - free(io->buf); + if (io->keep_alive) { + int val; + + val = 0; + setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val, + sizeof(val)); + fetch_cache_put(io->conn, fetch_close); +#ifdef TCP_NOPUSH + val = 1; + setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, + sizeof(val)); +#endif + } else { + fetch_close(io->conn); + } + + free(io->buf); free(io); } @@ -314,7 +339,7 @@ http_closefn(void *v) * Wrap a file descriptor up */ static fetchIO * -http_funopen(conn_t *conn, int chunked) +http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength) { struct httpio *io; fetchIO *f; @@ -325,6 +350,8 @@ http_funopen(conn_t *conn, int chunked) } io->conn = conn; io->chunked = chunked; + io->contentlength = clength; + io->keep_alive = keep_alive; f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn); if (f == NULL) { fetch_syserr(); @@ -345,6 +372,7 @@ typedef enum { hdr_error = -1, hdr_end = 0, hdr_unknown = 1, + hdr_connection, hdr_content_length, hdr_content_range, hdr_last_modified, @@ -358,6 +386,7 @@ static struct { hdr_t num; const char *name; } hdr_names[] = { + { hdr_connection, "Connection" }, { hdr_content_length, "Content-Length" }, { hdr_content_range, "Content-Range" }, { hdr_last_modified, "Last-Modified" }, @@ -388,7 +417,7 @@ http_cmd(conn_t *conn, const char *fmt, ...) return (-1); } - r = fetch_putln(conn, msg, len); + r = fetch_write(conn, msg, len); free(msg); if (r == -1) { @@ -629,7 +658,7 @@ http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd) free(upw); if (auth == NULL) return (-1); - r = http_cmd(conn, "%s: Basic %s", hdr, auth); + r = http_cmd(conn, "%s: Basic %s\r\n", hdr, auth); free(auth); return (r); } @@ -671,7 +700,7 @@ http_authorize(conn_t *conn, const char *hdr, const char *p) * Connect to the correct HTTP server or proxy. */ static conn_t * -http_connect(struct url *URL, struct url *purl, const char *flags) +http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) { conn_t *conn; int af, verbose; @@ -679,6 +708,8 @@ http_connect(struct url *URL, struct url *purl, const char *flags) int val; #endif + *cached = 1; + #ifdef INET6 af = AF_UNSPEC; #else @@ -701,7 +732,12 @@ http_connect(struct url *URL, struct url *purl, const char *flags) return (NULL); } - if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL) + if ((conn = fetch_cache_get(URL, af)) != NULL) { + *cached = 1; + return (conn); + } + + if ((conn = fetch_connect(URL, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ return (NULL); if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && @@ -759,7 +795,7 @@ set_if_modified_since(conn_t *conn, time_t last_modified) snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT", weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3, tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec); - http_cmd(conn, "If-Modified-Since: %s", buf); + http_cmd(conn, "If-Modified-Since: %s\r\n", buf); } @@ -779,7 +815,8 @@ http_request(struct url *URL, const char *op, struct url_stat *us, { conn_t *conn; struct url *url, *new; - int chunked, direct, if_modified_since, need_auth, noredirect, verbose; + int chunked, direct, if_modified_since, need_auth, noredirect; + int keep_alive, verbose, cached; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; @@ -792,6 +829,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); if_modified_since = CHECK_FLAG('i'); + keep_alive = 0; if (direct && purl) { fetchFreeURL(purl); @@ -829,7 +867,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, } /* connect to server or proxy */ - if ((conn = http_connect(url, purl, flags)) == NULL) + if ((conn = http_connect(url, purl, flags, &cached)) == NULL) goto ouch; host = url->host; @@ -853,10 +891,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us, fetch_info("requesting %s://%s%s", url->scheme, host, url->doc); if (purl) { - http_cmd(conn, "%s %s://%s%s HTTP/1.1", + http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n", op, url->scheme, host, url->doc); } else { - http_cmd(conn, "%s %s HTTP/1.1", + http_cmd(conn, "%s %s HTTP/1.1\r\n", op, url->doc); } @@ -864,7 +902,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, set_if_modified_since(conn, url->last_modified); /* virtual host */ - http_cmd(conn, "Host: %s", host); + http_cmd(conn, "Host: %s\r\n", host); /* proxy authorization */ if (purl) { @@ -892,19 +930,18 @@ http_request(struct url *URL, const char *op, struct url_stat *us, /* other headers */ if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') { if (strcasecmp(p, "auto") == 0) - http_cmd(conn, "Referer: %s://%s%s", + http_cmd(conn, "Referer: %s://%s%s\r\n", url->scheme, host, url->doc); else - http_cmd(conn, "Referer: %s", p); + http_cmd(conn, "Referer: %s\r\n", p); } if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0') - http_cmd(conn, "User-Agent: %s", p); + http_cmd(conn, "User-Agent: %s\r\n", p); else - http_cmd(conn, "User-Agent: %s ", _LIBFETCH_VER); + http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER); if (url->offset > 0) - http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset); - http_cmd(conn, "Connection: close"); - http_cmd(conn, ""); + http_cmd(conn, "Range: bytes=%lld-\r\n", (long long)url->offset); + http_cmd(conn, "\r\n"); /* * Force the queued request to be dispatched. Normally, one @@ -968,6 +1005,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us, case HTTP_PROTOCOL_ERROR: /* fall through */ case -1: + --i; + if (cached) + continue; fetch_syserr(); goto ouch; default: @@ -986,6 +1026,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us, case hdr_error: http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; + case hdr_connection: + /* XXX too weak? */ + keep_alive = (strcasecmp(p, "keep-alive") == 0); + break; case hdr_content_length: http_parse_length(p, &clength); break; @@ -1115,13 +1159,20 @@ http_request(struct url *URL, const char *op, struct url_stat *us, URL->offset = offset; URL->length = clength; + if (clength == -1 && !chunked) + keep_alive = 0; + if (conn->err == HTTP_NOT_MODIFIED) { http_seterr(HTTP_NOT_MODIFIED); + if (keep_alive) { + fetch_cache_put(conn, fetch_close); + conn = NULL; + } goto ouch; } /* wrap it up in a fetchIO */ - if ((f = http_funopen(conn, chunked)) == NULL) { + if ((f = http_funopen(conn, chunked, keep_alive, clength)) == NULL) { fetch_syserr(); goto ouch; } @@ -1132,6 +1183,13 @@ http_request(struct url *URL, const char *op, struct url_stat *us, fetchFreeURL(purl); if (HTTP_ERROR(conn->err)) { + + if (keep_alive) { + char buf[512]; + do { + } while (fetchIO_read(f, buf, sizeof(buf)) > 0); + } + fetchIO_close(f); f = NULL; } @@ -1354,6 +1412,7 @@ parse_index(struct index_parser *parser, const char *buf, size_t len) return -1; return end_attr + 1 - buf; } + /* NOTREACHED */ abort(); }