Hi i have one more problem and question :-( i tested tproxy4 in debian linux and succeed in transparent proxying function of the apache proxy server (a tproxy patched) but when i tested the same apache version in Cento OS (5.4) , the apache proxy did not operate correctly. listen () function did not return when a client tried to connect a web server. then the apache did not do any action. no response but the usage count of filter in the mangle table was increased. and then.. i found something strange like below. there are two listening daemon with same port but the daemons are same process root@waf:~/tproxy4-20091127/httpd-2.2.9-tproxy-v1# netstat -ap | grep http tcp 0 0 *:squid *:* LISTEN 7397/httpd tcp 0 0 *:squid *:* LISTEN 7397/httpd i don't know why the apache proxy which succeed in doing tproxy in the debian linux did not operate correctly in CentOS (5.4) but squid proxy succeed in tproxy in both the debian and the CentOS box do you have any idea ? previously thanks ~ Cheers, JeHo Park [Environment] - bridged network Client <---> [eth2 apache tproxy eth1] <---> WebServer +-------- br0 ---------+ - CPU: AMD Phenom(tm) II X4 955 Processor - OS: CentOS release 5.4 (Final) - Kernel: vannilla 2.6.31 - Kernel config: kconfig-2.6.31 [attached] - apache server: apache 2.2.9 + http-2.2.9-tproxy4.patch [attached] - iptables: 1.4.3 - gcc 4.1.2 *- iptables & ebroute & ip rules* ${IPTABLES} -t mangle -F ${IPTABLES} -t mangle -X DIVERT ${IPTABLES} -t nat -F #ip rule del fwmark 0x1/0x1 lookup 100 ip rule del fwmark 1 lookup 100 ip route del local 0.0.0.0/0 dev lo table 100 ebtables -t broute -F ${IPTABLES} -t mangle -N DIVERT ${IPTABLES} -t mangle -A PREROUTING -p tcp -m socket -j DIVERT ${IPTABLES} -t mangle -A DIVERT -j MARK --set-mark 1 ${IPTABLES} -t mangle -A DIVERT -j ACCEPT ${IPTABLES} -t mangle -A PREROUTING -p tcp --dport 80 -j TPROXY --tproxy-mark 0x1/0x1 --on-port ${PORT} ip rule add fwmark 1 lookup 100 ip route add local 0.0.0.0/0 dev lo table 100 ebtables -t broute -A BROUTING -i $CLIENT_IFACE -p ipv4 --ip-proto tcp --ip-dport 80 -j redirect --redirect-target ACCEPT ebtables -t broute -A BROUTING -i $INET_IFACE -p ipv4 --ip-proto tcp --ip-sport 80 -j redirect --redirect-target ACCEPT cd /proc/sys/net/bridge/ for i in * do echo 0 > $i done unset i sysctl net.ipv4.ip_nonlocal_bind=1 sysctl net.ipv4.ip_forward=1 KOVACS Krisztian 쓴 글:
Hi,
On Mon, 2009-11-23 at 15:51 +0900, 박제호 wrote:
i have a problem in my transparent proxy test, i recently made up the testbed as below to run the tproxy patched apache proxy [mod_proxy], and i applied all iptables and routing rules with referencing the readme file [http://www.balabit.com/downloads/files/tproxy/README.txt, http://www.mjmwired.net/kernel/Documentation/networking/tproxy.txt] the proxy server listening the port 3128 and i checked there were no problem. but when the client tried to connect the web server, the packets reached to the box and i found the usage counts of filter rules in the mangle table incresed but my tproxy server could not receive any corresponding packet from the socket
I want to know why my proxy server can't receive any packet through the socket, Do i need some more DNAT rules ?
Would you mind testing the setup with an unpatched upstream kernel, that has tproxy built-in? (2.6.31, for example)
Also, please download the latest iptables from netfilter.org and try using that. (No need for patching, tproxy support is in upstream.)
That would help a lot in identifying the source of the issue. Thanks in advance.
Cheers, Krisztian
diff -uNr httpd-2.2.9/modules/proxy/mod_proxy.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.c --- httpd-2.2.9/modules/proxy/mod_proxy.c 2008-05-29 06:11:24.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.c 2009-12-02 11:26:25.000000000 +0900 @@ -1870,6 +1870,18 @@ return NULL; } +static const char * +set_tproxy(cmd_parms *parms, void *dummy, int flag) +{ + proxy_server_conf *psf = + ap_get_module_config(parms->server->module_config, &proxy_module); + + psf->tproxy = flag; + psf->tproxy_set = 1; + + return NULL; +} + static void ap_add_per_proxy_conf(server_rec *s, ap_conf_vector_t *dir_config) { proxy_server_conf *sconf = ap_get_module_config(s->module_config, @@ -2082,6 +2094,8 @@ "A balancer or worker name with list of params"), AP_INIT_TAKE1("ProxyFtpDirCharset", set_ftp_directory_charset, NULL, RSRC_CONF|ACCESS_CONF, "Define the character set for proxied FTP listings"), + AP_INIT_FLAG("ProxyTProxy", set_tproxy, NULL, RSRC_CONF, + "on if the proxy should use client IP to connect to the origin server"), {NULL} }; diff -uNr httpd-2.2.9/modules/proxy/mod_proxy.h httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.h --- httpd-2.2.9/modules/proxy/mod_proxy.h 2008-05-29 06:24:04.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy.h 2009-12-02 11:26:25.000000000 +0900 @@ -193,6 +193,9 @@ } proxy_status; /* Status display options */ char proxy_status_set; apr_pool_t *pool; /* Pool used for allocating this struct */ + int tproxy; /* true if tproxy is enabled */ + char tproxy_set; + } proxy_server_conf; @@ -711,6 +714,7 @@ * @param conn acquired connection * @param worker connection worker * @param s current server record + * @param remote_addr client addr * @return OK or HTTP_XXX error * @note In case the socket already exists for conn, just check the link * status. @@ -718,7 +722,8 @@ PROXY_DECLARE(int) ap_proxy_connect_backend(const char *proxy_function, proxy_conn_rec *conn, proxy_worker *worker, - server_rec *s); + server_rec *s, + apr_sockaddr_t *remote_addr); /** * Make a connection record for backend connection * @param proxy_function calling proxy scheme (http, ajp, ...) diff -uNr httpd-2.2.9/modules/proxy/mod_proxy_ajp.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ajp.c --- httpd-2.2.9/modules/proxy/mod_proxy_ajp.c 2008-06-05 21:46:43.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ajp.c 2009-12-02 11:26:25.000000000 +0900 @@ -605,11 +605,11 @@ goto cleanup; /* Step Two: Make the Connection */ - if (ap_proxy_connect_backend(scheme, backend, worker, r->server)) { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, - "proxy: AJP: failed to make connection to backend: %s", - backend->hostname); - status = HTTP_SERVICE_UNAVAILABLE; + if (ap_proxy_connect_backend(scheme, backend, worker, r->server, r->connection->remote_addr)) { + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "proxy: AJP: failed to make connection to backend: %s", + backend->hostname); + status = HTTP_SERVICE_UNAVAILABLE; goto cleanup; } diff -uNr httpd-2.2.9/modules/proxy/mod_proxy_ftp.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ftp.c --- httpd-2.2.9/modules/proxy/mod_proxy_ftp.c 2008-05-18 04:42:03.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_ftp.c 2009-12-02 11:26:25.000000000 +0900 @@ -973,11 +973,11 @@ */ - if (ap_proxy_connect_backend("FTP", backend, worker, r->server)) { - ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, - "proxy: FTP: an error occurred creating a new connection to %pI (%s)", - connect_addr, connectname); - proxy_ftp_cleanup(r, backend); + if (ap_proxy_connect_backend("FTP", backend, worker, r->server, c->remote_addr)) { + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "proxy: FTP: an error occurred creating a new connection to %pI (%s)", + connect_addr, connectname); + proxy_ftp_cleanup(r, backend); return HTTP_SERVICE_UNAVAILABLE; } diff -uNr httpd-2.2.9/modules/proxy/mod_proxy_http.c httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_http.c --- httpd-2.2.9/modules/proxy/mod_proxy_http.c 2008-06-11 03:49:19.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/modules/proxy/mod_proxy_http.c 2009-12-02 11:26:25.000000000 +0900 @@ -1871,6 +1871,10 @@ * connection ID of the current upstream connection is the same as that * of the connection when the socket was opened. */ + + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "--> linuxpark: %s/%s/%d %s\n", + __FILE__, __FUNCTION__, __LINE__, url); apr_pool_t *p = r->connection->pool; conn_rec *c = r->connection; apr_uri_t *uri = apr_palloc(r->connection->pool, sizeof(*uri)); @@ -1929,7 +1933,7 @@ goto cleanup; /* Step Two: Make the Connection */ - if (ap_proxy_connect_backend(proxy_function, backend, worker, r->server)) { + if (ap_proxy_connect_backend(proxy_function, backend, worker, r->server, c->remote_addr)) { if (r->proxyreq == PROXYREQ_PROXY) status = HTTP_NOT_FOUND; else diff -uNr httpd-2.2.9/modules/proxy/proxy_util.c httpd-2.2.9-tproxy-v1/modules/proxy/proxy_util.c --- httpd-2.2.9/modules/proxy/proxy_util.c 2008-05-29 06:11:24.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/modules/proxy/proxy_util.c 2009-12-02 11:26:25.000000000 +0900 @@ -1596,8 +1596,8 @@ } ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, - "proxy: %s: fam %d socket created to connect to %s", - proxy_function, backend_addr->family, backend_name); + "%s/%d proxy: %s: fam %d socket created to connect to %s", + __FUNCTION__, __LINE__, proxy_function, backend_addr->family, backend_name); /* make the connection out of the socket */ rv = apr_socket_connect(*newsock, backend_addr); @@ -2269,7 +2269,8 @@ PROXY_DECLARE(int) ap_proxy_connect_backend(const char *proxy_function, proxy_conn_rec *conn, proxy_worker *worker, - server_rec *s) + server_rec *s, + apr_sockaddr_t *remote_addr) { apr_status_t rv; int connected = 0; @@ -2342,14 +2343,31 @@ ap_log_error(APLOG_MARK, APLOG_ERR, rv, s, "apr_socket_opt_set(SO_KEEPALIVE): Failed to set" " Keepalive"); - } - } + } + } + + /* TProxy support */ + if (conf->tproxy) { +#if 1 + //if ((rv = apr_socket_opt_set(newsock, APR_IP_TRANSPARENT, 1)) != APR_SUCCESS) { + if ((rv = apr_socket_opt_set(newsock, 32, 1)) != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_ERR, rv, s, + "apr_socket_opt_set(APR_IP_TRANSPARENT): Failed to set"); + } +#endif + if ((rv = apr_socket_bind(newsock, remote_addr)) != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_ERR, rv, s, + "proxy: error binding to socket %pI", remote_addr); + return HTTP_INTERNAL_SERVER_ERROR; + } + } + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, - "proxy: %s: fam %d socket created to connect to %s", - proxy_function, backend_addr->family, worker->hostname); + "%s/%d proxy: %s: fam %d socket created to connect to %s", + __FUNCTION__, __LINE__, proxy_function, backend_addr->family, worker->hostname); - /* make the connection out of the socket */ - rv = apr_socket_connect(newsock, backend_addr); + /* make the connection out of the socket */ + rv = apr_socket_connect(newsock, backend_addr); /* if an error occurred, loop round and try again */ if (rv != APR_SUCCESS) { diff -uNr httpd-2.2.9/os/unix/unixd.c httpd-2.2.9-tproxy-v1/os/unix/unixd.c --- httpd-2.2.9/os/unix/unixd.c 2006-07-12 12:38:44.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/os/unix/unixd.c 2009-12-02 11:26:26.000000000 +0900 @@ -49,8 +49,10 @@ #endif #ifdef HAVE_SYS_PRCTL_H #include <sys/prctl.h> +#include <sys/capability.h> #endif + unixd_config_rec unixd_config; /* Set group privileges. @@ -60,6 +62,36 @@ * with different sets of groups for each. */ +void +keep_capabilities(void) +{ + if (prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) != 0) { + ap_log_error(APLOG_MARK, APLOG_ERR, errno, NULL, "prctl Failed"); + } +} + +static void +set_capabilities(int capability) +{ + cap_user_header_t head = (cap_user_header_t) calloc(1, sizeof(*head)); + cap_user_data_t cap = (cap_user_data_t) calloc(1, sizeof(*cap)); + + head->version = _LINUX_CAPABILITY_VERSION; + head->pid = 0; + cap->effective = (1 << capability); + cap->inheritable = cap->effective; + cap->permitted = cap->effective; + + if (capset(head, cap) != 0) { + ap_log_error(APLOG_MARK, APLOG_ERR, errno, NULL, "capset Failed"); + } + + free(head); + free(cap); +} + + + static int set_group_privs(void) { if (!geteuid()) { @@ -117,6 +149,7 @@ if (set_group_privs()) { return -1; } + keep_capabilities(); #ifdef MPE /* Only try to switch if we're running as MANAGER.SYS */ if (geteuid() == 1 && unixd_config.user_id > 1) { @@ -142,6 +175,10 @@ (long) unixd_config.user_id); return -1; } + + /* Needed for tproxy */ + set_capabilities(CAP_NET_ADMIN); + #if defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE) /* this applies to Linux 2.4+ */ #ifdef AP_MPM_WANT_SET_COREDUMPDIR @@ -461,7 +498,6 @@ #ifdef _OSD_POSIX int sockdes; #endif - *accepted = NULL; status = apr_socket_accept(&csd, lr->sd, ptrans); if (status == APR_SUCCESS) { diff -uNr httpd-2.2.9/srclib/apr/include/apr_network_io.h httpd-2.2.9-tproxy-v1/srclib/apr/include/apr_network_io.h --- httpd-2.2.9/srclib/apr/include/apr_network_io.h 2007-12-15 13:10:38.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/srclib/apr/include/apr_network_io.h 2009-12-02 11:26:30.000000000 +0900 @@ -65,6 +65,7 @@ #define APR_SO_DEBUG 4 /**< Debug */ #define APR_SO_NONBLOCK 8 /**< Non-blocking IO */ #define APR_SO_REUSEADDR 16 /**< Reuse addresses */ +#define APR_IP_TRANSPARENT 32 /**< TProxy support */ #define APR_SO_SNDBUF 64 /**< Send buffer */ #define APR_SO_RCVBUF 128 /**< Receive buffer */ #define APR_SO_DISCONNECTED 256 /**< Disconnected */ diff -uNr httpd-2.2.9/srclib/apr/network_io/unix/sockets.c httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockets.c --- httpd-2.2.9/srclib/apr/network_io/unix/sockets.c 2009-12-02 14:18:45.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockets.c 2009-12-02 11:26:30.000000000 +0900 @@ -151,6 +151,9 @@ apr_status_t apr_socket_bind(apr_socket_t *sock, apr_sockaddr_t *sa) { + int val = 1; + + setsockopt (sock->socketdes, SOL_IP, 19, &val, sizeof (val)); if (bind(sock->socketdes, (struct sockaddr *)&sa->sa, sa->salen) == -1) { return errno; } diff -uNr httpd-2.2.9/srclib/apr/network_io/unix/sockopt.c httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockopt.c --- httpd-2.2.9/srclib/apr/network_io/unix/sockopt.c 2006-08-03 19:55:31.000000000 +0900 +++ httpd-2.2.9-tproxy-v1/srclib/apr/network_io/unix/sockopt.c 2009-12-02 11:26:30.000000000 +0900 @@ -17,6 +17,7 @@ #include "apr_arch_networkio.h" #include "apr_strings.h" +#define IP_TRANSPARENT 19 static apr_status_t soblock(int sd) { @@ -318,6 +319,15 @@ return APR_ENOTIMPL; #endif break; + case APR_IP_TRANSPARENT: +#ifdef IP_TRANSPARENT + if (setsockopt(sock->socketdes, SOL_IP, IP_TRANSPARENT, (void *)&one, sizeof(int)) == -1) { + return errno; + } +#else + return APR_ENOTIMPL; +#endif + break; default: return APR_EINVAL; }