head 1.3; access; symbols pkgsrc-2013Q2:1.3.0.4 pkgsrc-2013Q2-base:1.3 pkgsrc-2012Q4:1.3.0.2 pkgsrc-2012Q4-base:1.3 pkgsrc-2012Q3:1.2.0.8 pkgsrc-2012Q3-base:1.2 pkgsrc-2012Q2:1.2.0.6 pkgsrc-2012Q2-base:1.2 pkgsrc-2012Q1:1.2.0.4 pkgsrc-2012Q1-base:1.2 pkgsrc-2011Q4:1.2.0.2 pkgsrc-2011Q4-base:1.2; locks; strict; comment @# @; 1.3 date 2012.10.19.04.15.21; author manu; state dead; branches; next 1.2; 1.2 date 2011.12.16.05.40.46; author manu; state Exp; branches; next 1.1; 1.1 date 2011.12.09.16.57.44; author manu; state Exp; branches; next ; desc @@ 1.3 log @Bump to glusterfs-3.3.1, which brings - unified file and object storage - storage for Hadoop (not tested here) - proactive self-healing - much better performance @ text @$NetBSD: ssl.patch,v 1.2 2011/12/16 05:40:46 manu Exp $ SSL support pulled from not yet committed upstream patch http://review.gluster.com/#change,362 --- ./xlators/performance/write-behind/src/write-behind.c.orig 2011-11-14 14:46:02.000000000 +0100 +++ ./xlators/performance/write-behind/src/write-behind.c 2011-12-15 03:22:43.000000000 +0100 @@@@ -371,8 +371,13 @@@@ local = frame->local; winds = &local->winds; + /* + * I don't know how we get here without "local" being valid, but we + * do sometimes during disconnect processing. + */ + GF_VALIDATE_OR_GOTO (this->name, local, out); file = local->file; GF_VALIDATE_OR_GOTO (this->name, file, out); LOCK (&file->lock); --- ./xlators/mgmt/glusterd/src/glusterd-volgen.c.orig 2011-11-14 14:46:02.000000000 +0100 +++ ./xlators/mgmt/glusterd/src/glusterd-volgen.c 2011-12-14 20:47:10.000000000 +0100 @@@@ -187,8 +187,12 @@@@ {VKEY_FEATURES_QUOTA, "features/marker", "quota", "off", NO_DOC, OPT_FLAG_FORCE}, {VKEY_FEATURES_LIMIT_USAGE, "features/quota", "limit-set", NULL, NO_DOC, 0}, {"features.quota-timeout", "features/quota", "timeout", "0", DOC, 0}, + { "server.ssl", "protocol/server", +"transport.socket.ssl-enabled", NULL, NO_DOC, 0}, + { "client.ssl", "protocol/client", +"transport.socket.ssl-enabled", NULL, NO_DOC, 0}, {NULL, } }; --- ./xlators/mgmt/glusterd/src/glusterd.c.orig 2011-12-15 03:16:25.000000000 +0100 +++ ./xlators/mgmt/glusterd/src/glusterd.c 2011-12-15 03:17:31.000000000 +0100 @@@@ -691,16 +691,21 @@@@ ret = configure_syncdaemon (conf); if (ret) goto out; - ret = glusterd_restore (); - if (ret < 0) - goto out; glusterd_friend_sm_init (); glusterd_op_sm_init (); glusterd_opinfo_init (); + /* + * This uses some of the fields initialized in glusterd_*_init, so + * do it *after* those. + */ + ret = glusterd_restore (); + if (ret < 0) + goto out; + ret = glusterd_handle_upgrade_downgrade (this->options, conf); if (ret) goto out; --- ./xlators/protocol/server/src/server3_1-fops.c.orig 2011-11-14 14:46:03.000000000 +0100 +++ ./xlators/protocol/server/src/server3_1-fops.c 2011-12-14 20:47:10.000000000 +0100 @@@@ -3129,8 +3129,13 @@@@ goto out; } conn = req->trans->xl_private; + if (!conn) { + /* Handshake is not complete yet. */ + req->rpc_err = SYSTEM_ERR; + goto out; + } gf_fd_put (conn->fdtable, args.fd); server_submit_reply (NULL, req, &rsp, NULL, 0, NULL, xdr_serialize_common_rsp); --- ./rpc/rpc-transport/socket/src/Makefile.am.orig 2011-11-14 14:46:00.000000000 +0100 +++ ./rpc/rpc-transport/socket/src/Makefile.am 2011-12-14 20:47:10.000000000 +0100 @@@@ -2,9 +2,9 @@@@ rpctransport_LTLIBRARIES = socket.la rpctransportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport -socket_la_LDFLAGS = -module -avoidversion +socket_la_LDFLAGS = -module -avoidversion -lssl socket_la_SOURCES = socket.c name.c socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la --- ./rpc/rpc-transport/socket/src/Makefile.in.orig 2011-11-14 14:46:14.000000000 +0100 +++ ./rpc/rpc-transport/socket/src/Makefile.in 2011-12-14 20:47:10.000000000 +0100 @@@@ -226,9 +226,9 @@@@ target_alias = @@target_alias@@ noinst_HEADERS = socket.h name.h rpctransport_LTLIBRARIES = socket.la rpctransportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport -socket_la_LDFLAGS = -module -avoidversion +socket_la_LDFLAGS = -module -avoidversion -lssl socket_la_SOURCES = socket.c name.c socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\ -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src/ \ --- ./rpc/rpc-transport/socket/src/socket.c.orig 2011-12-14 20:46:23.000000000 +0100 +++ ./rpc/rpc-transport/socket/src/socket.c 2011-12-15 03:31:03.000000000 +0100 @@@@ -42,12 +42,35 @@@@ #include #include #include #include - +#include #define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR) #define SA(ptr) ((struct sockaddr *)ptr) +#define SSL_ENABLED_OPT "transport.socket.ssl-enabled" +#define SSL_OWN_CERT_OPT "transport.socket.ssl-own-cert" +#define SSL_PRIVATE_KEY_OPT "transport.socket.ssl-private-key" +#define SSL_CA_LIST_OPT "transport.socket.ssl-ca-list" +#define OWN_THREAD_OPT "transport.socket.own-thread" + +/* TBD: do automake substitutions etc. (ick) to set these. */ +#if !defined(DEFAULT_CERT_PATH) +#define DEFAULT_CERT_PATH "/etc/ssl/glusterfs.pem" +#endif +#if !defined(DEFAULT_KEY_PATH) +#define DEFAULT_KEY_PATH "/etc/ssl/glusterfs.key" +#endif +#if !defined(DEFAULT_CA_PATH) +#define DEFAULT_CA_PATH "/etc/ssl/glusterfs.ca" +#endif + +#define POLL_MASK_INPUT (POLLIN | POLLPRI) +#define POLL_MASK_OUTPUT (POLLOUT) +#define POLL_MASK_ERROR (POLLERR | POLLHUP | POLLNVAL) + +typedef int SSL_unary_func (SSL *); +typedef int SSL_trinary_func (SSL *, void *, int); #define __socket_proto_reset_pending(priv) do { \ memset (&priv->incoming.frag.vector, 0, \ sizeof (priv->incoming.frag.vector)); \ @@@@ -132,11 +155,158 @@@@ } \ __socket_proto_update_priv_after_read (priv, ret, bytes_read); \ } - int socket_init (rpc_transport_t *this); +void +ssl_dump_error_stack (const char *caller) +{ + unsigned long errnum = 0; + char errbuf[120] = {0,}; + + /* OpenSSL docs explicitly give 120 as the error-string length. */ + + while ((errnum = ERR_get_error())) { + ERR_error_string(errnum,errbuf); + gf_log(caller,GF_LOG_ERROR," %s",errbuf); + } +} + +int +ssl_do (rpc_transport_t *this, void *buf, size_t len, SSL_trinary_func *func) +{ + int r = (-1); + struct pollfd pfd = {-1,}; + socket_private_t *priv = NULL; + + GF_VALIDATE_OR_GOTO(this->name,this->private,out); + priv = this->private; + + for (;;) { + if (buf) { + if (priv->connected == -1) { + /* + * Fields in the SSL structure (especially + * the BIO pointers) are not valid at this + * point, so we'll segfault if we pass them + * to SSL_read/SSL_write. + */ + gf_log(this->name,GF_LOG_INFO, + "lost connection in %s", __func__); + break; + } + r = func(priv->ssl_ssl,buf,len); + } + else { + /* + * We actually need these functions to get to + * priv->connected == 1. + */ + r = ((SSL_unary_func *)func)(priv->ssl_ssl); + } + switch (SSL_get_error(priv->ssl_ssl,r)) { + case SSL_ERROR_NONE: + return r; + case SSL_ERROR_WANT_READ: + pfd.fd = priv->sock; + pfd.events = POLLIN; + if (poll(&pfd,1,-1) < 0) { + gf_log(this->name,GF_LOG_ERROR,"poll error %d", + errno); + } + break; + case SSL_ERROR_WANT_WRITE: + pfd.fd = priv->sock; + pfd.events = POLLOUT; + if (poll(&pfd,1,-1) < 0) { + gf_log(this->name,GF_LOG_ERROR,"poll error %d", + errno); + } + break; + case SSL_ERROR_SYSCALL: + /* This is what we get when remote disconnects. */ + gf_log(this->name,GF_LOG_DEBUG, + "syscall error (probably remote disconnect)"); + errno = ENODATA; + goto out; + default: + errno = EIO; + goto out; /* "break" would just loop again */ + } + } +out: + return -1; +} + +#define ssl_connect_one(t) ssl_do((t),NULL,0,(SSL_trinary_func *)SSL_connect) +#define ssl_accept_one(t) ssl_do((t),NULL,0,(SSL_trinary_func *)SSL_accept) +#define ssl_read_one(t,b,l) ssl_do((t),(b),(l),(SSL_trinary_func *)SSL_read) +#define ssl_write_one(t,b,l) ssl_do((t),(b),(l),(SSL_trinary_func *)SSL_write) + +int +ssl_setup_connection (rpc_transport_t *this, int server) +{ + X509 *peer = NULL; + char peer_CN[256] = ""; + int ret = -1; + socket_private_t *priv = NULL; + + GF_VALIDATE_OR_GOTO(this->name,this->private,done); + priv = this->private; + + priv->ssl_ssl = SSL_new(priv->ssl_ctx); + if (!priv->ssl_ssl) { + gf_log(this->name,GF_LOG_ERROR,"SSL_new failed"); + ssl_dump_error_stack(this->name); + goto done; + } + priv->ssl_sbio = BIO_new_socket(priv->sock,BIO_NOCLOSE); + if (!priv->ssl_sbio) { + gf_log(this->name,GF_LOG_ERROR,"BIO_new_socket failed"); + ssl_dump_error_stack(this->name); + goto free_ssl; + } + SSL_set_bio(priv->ssl_ssl,priv->ssl_sbio,priv->ssl_sbio); + + if (server) { + ret = ssl_accept_one(this); + } + else { + ret = ssl_connect_one(this); + } + + /* Make sure _the call_ succeeded. */ + if (ret < 0) { + goto ssl_error; + } + + /* Make sure _SSL verification_ succeeded, yielding an identity. */ + if (SSL_get_verify_result(priv->ssl_ssl) != X509_V_OK) { + goto ssl_error; + } + peer = SSL_get_peer_certificate(priv->ssl_ssl); + if (!peer) { + goto ssl_error; + } + + /* Finally, everything seems OK. */ + X509_NAME_get_text_by_NID(X509_get_subject_name(peer), + NID_commonName, peer_CN, sizeof(peer_CN)-1); + peer_CN[sizeof(peer_CN)-1] = '\0'; + gf_log(this->name,GF_LOG_INFO,"peer CN = %s", peer_CN); + return 0; + + /* Error paths. */ +ssl_error: + gf_log(this->name,GF_LOG_ERROR,"SSL connect error"); + ssl_dump_error_stack(this->name); +free_ssl: + SSL_free(priv->ssl_ssl); +done: + return ret; +} + /* * return value: * 0 = success (completed) * -1 = error @@@@ -167,19 +337,43 @@@@ if (bytes != NULL) { *bytes = 0; } - while (opcount) { + while (opcount > 0) { + if (opvector->iov_len == 0) { + gf_log(this->name,GF_LOG_DEBUG, + "would have passed zero length to read/write"); + ++opvector; + --opcount; + continue; + } if (write) { - ret = writev (sock, opvector, opcount); + if (priv->use_ssl) { + ret = ssl_write_one(this, + opvector->iov_base, opvector->iov_len); + } + else { + ret = writev (sock, opvector, opcount); + } if (ret == 0 || (ret == -1 && errno == EAGAIN)) { /* done for now */ break; } this->total_bytes_write += ret; } else { - ret = readv (sock, opvector, opcount); + if (priv->use_ssl) { + ret = ssl_read_one(this, + opvector->iov_base, opvector->iov_len); + } + else { + ret = readv (sock, opvector, opcount); + } + if (ret == 0) { + gf_log(this->name,GF_LOG_DEBUG,"EOF on socket"); + errno = ENODATA; + ret = -1; + } if (ret == -1 && errno == EAGAIN) { /* done for now */ break; } @@@@ -201,8 +395,11 @@@@ gf_log (this->name, GF_LOG_WARNING, "%s failed (%s)", write ? "writev" : "readv", strerror (errno)); + if (priv->use_ssl) { + ssl_dump_error_stack(this->name); + } opcount = -1; break; } @@@@ -212,8 +409,19 @@@@ moved = 0; while (moved < ret) { + if (!opcount) { + gf_log(this->name,GF_LOG_DEBUG, + "ran out of iov, moved %d/%d", + moved, ret); + goto ran_out; + } + if (!opvector[0].iov_len) { + opvector++; + opcount--; + continue; + } if ((ret - moved) >= opvector[0].iov_len) { moved += opvector[0].iov_len; opvector++; opcount--; @@@@ -221,15 +429,13 @@@@ opvector[0].iov_len -= (ret - moved); opvector[0].iov_base += (ret - moved); moved += (ret - moved); } - while (opcount && !opvector[0].iov_len) { - opvector++; - opcount--; - } } } +ran_out: + if (pending_vector) *pending_vector = opvector; if (pending_count) @@@@ -287,8 +493,22 @@@@ gf_log (this->name, GF_LOG_DEBUG, "shutdown() returned %d. %s", ret, strerror (errno)); } + if (priv->use_ssl) { + SSL_shutdown(priv->ssl_ssl); + SSL_clear(priv->ssl_ssl); + SSL_free(priv->ssl_ssl); + } + if (priv->own_thread) { + /* + * Without this, reconnect (= disconnect + connect) + * won't work except by accident. + */ + close(priv->sock); + priv->sock = -1; + ++(priv->socket_gen); + } } out: return ret; @@@@ -364,9 +584,8 @@@@ return ret; } - int __socket_nodelay (int fd) { int on = 1; @@@@ -620,11 +839,13 @@@@ } int -__socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry) +__socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry, int direct) { - int ret = -1; + int ret = -1; + socket_private_t *priv = NULL; + char a_byte = 0; ret = __socket_writev (this, entry->pending_vector, entry->pending_count, &entry->pending_vector, @@@@ -633,8 +854,20 @@@@ if (ret == 0) { /* current entry was completely written */ GF_ASSERT (entry->pending_count == 0); __socket_ioq_entry_free (entry); + priv = this->private; + if (priv->own_thread) { + /* + * The pipe should only remain readable if there are + * more entries after this, so drain the byte + * representing this entry. + */ + if (!direct && read(priv->pipe[0],&a_byte,1) < 1) { + gf_log(this->name,GF_LOG_WARNING, + "read error on pipe"); + } + } } return ret; } @@@@ -655,15 +888,15 @@@@ while (!list_empty (&priv->ioq)) { /* pick next entry */ entry = priv->ioq_next; - ret = __socket_ioq_churn_entry (this, entry); + ret = __socket_ioq_churn_entry (this, entry, 0); if (ret != 0) break; } - if (list_empty (&priv->ioq)) { + if (!priv->own_thread && list_empty (&priv->ioq)) { /* all pending writes done, not interested in POLLOUT */ priv->idx = event_select_on (this->ctx->event_pool, priv->sock, priv->idx, -1, 0); } @@@@ -1645,9 +1878,8 @@@@ if (pollin != NULL) { ret = rpc_transport_notify (this, RPC_TRANSPORT_MSG_RECEIVED, pollin); - rpc_transport_pollin_destroy (pollin); } return ret; @@@@ -1668,9 +1900,9 @@@@ priv = this->private; pthread_mutex_lock (&priv->lock); { - if (priv->connected) + if (priv->connected != 0) goto unlock; ret = __socket_connect_finish (priv->sock); @@@@ -1730,11 +1962,11 @@@@ int socket_event_handler (int fd, int idx, void *data, int poll_in, int poll_out, int poll_err) { - rpc_transport_t *this = NULL; + rpc_transport_t *this = NULL; socket_private_t *priv = NULL; - int ret = 0; + int ret = -1; this = data; GF_VALIDATE_OR_GOTO ("socket", this, out); GF_VALIDATE_OR_GOTO ("socket", this->private, out); @@@@ -1742,18 +1974,15 @@@@ THIS = this->xl; priv = this->private; - pthread_mutex_lock (&priv->lock); { priv->idx = idx; } pthread_mutex_unlock (&priv->lock); - if (!priv->connected) { - ret = socket_connect_finish (this); - } + ret = (priv->connected == 1) ? 0 : socket_connect_finish(this); if (!ret && poll_out) { ret = socket_event_poll_out (this); } @@@@ -1767,15 +1996,114 @@@@ gf_log ("transport", ((ret >= 0) ? GF_LOG_INFO : GF_LOG_DEBUG), "disconnecting now"); socket_event_poll_err (this); rpc_transport_unref (this); - } + } out: - return 0; + return ret; +} + + +void * +socket_poller (void *ctx) +{ + rpc_transport_t *this = ctx; + socket_private_t *priv = this->private; + struct pollfd pfd[2] = {{0,},}; + gf_boolean_t to_write = _gf_false; + int ret = 0; + int orig_gen; + + orig_gen = ++(priv->socket_gen); + + if (priv->connected == 0) { + THIS = this->xl; + ret = socket_connect_finish (this); + } + + for (;;) { + if (priv->socket_gen != orig_gen) { + gf_log(this->name,GF_LOG_DEBUG, + "redundant poller exiting"); + return NULL; + } + pthread_mutex_lock(&priv->lock); + to_write = !list_empty(&priv->ioq); + pthread_mutex_unlock(&priv->lock); + pfd[0].fd = priv->pipe[0]; + pfd[0].events = POLL_MASK_ERROR; + pfd[0].revents = 0; + pfd[1].fd = priv->sock; + pfd[1].events = POLL_MASK_INPUT | POLL_MASK_ERROR; + pfd[1].revents = 0; + if (to_write) { + pfd[1].events |= POLL_MASK_OUTPUT; + } + else { + pfd[0].events |= POLL_MASK_INPUT; + } + if (poll(pfd,2,-1) < 0) { + gf_log(this->name,GF_LOG_ERROR,"poll failed"); + break; + } + if (pfd[0].revents & POLL_MASK_ERROR) { + gf_log(this->name,GF_LOG_ERROR, + "poll error on pipe"); + break; + } + /* Only glusterd actually seems to need this. */ + THIS = this->xl; + if (pfd[1].revents & POLL_MASK_INPUT) { + ret = socket_event_poll_in(this); + if (ret >= 0) { + /* Suppress errors while making progress. */ + pfd[1].revents &= ~POLL_MASK_ERROR; + } + else if (errno == ENOTCONN) { + ret = 0; + } + } + else if (pfd[1].revents & POLL_MASK_OUTPUT) { + ret = socket_event_poll_out(this); + if (ret >= 0) { + /* Suppress errors while making progress. */ + pfd[1].revents &= ~POLL_MASK_ERROR; + } + else if (errno == ENOTCONN) { + ret = 0; + } + } + else { + /* + * This usually means that we left poll() because + * somebody pushed a byte onto our pipe. That wakeup + * is why the pipe is there, but once awake we can do + * all the checking we need on the next iteration. + */ + ret = 0; + } + if (pfd[1].revents & POLL_MASK_ERROR) { + gf_log(this->name,GF_LOG_ERROR, + "poll error on socket"); + break; + } + if (ret < 0) { + gf_log(this->name,GF_LOG_ERROR, + "error in polling loop"); + break; + } + } + + /* All (and only) I/O errors should come here. */ + __socket_disconnect (this); + rpc_transport_notify (this, RPC_TRANSPORT_DISCONNECT, this); + rpc_transport_unref (this); + return NULL; } + int socket_server_event_handler (int fd, int idx, void *data, int poll_in, int poll_out, int poll_err) { @@@@ -1812,21 +2140,8 @@@@ priv->sock, strerror (errno)); goto unlock; } - if (!priv->bio) { - ret = __socket_nonblock (new_sock); - - if (ret == -1) { - gf_log (this->name, GF_LOG_WARNING, - "NBIO on %d failed (%s)", - new_sock, strerror (errno)); - - close (new_sock); - goto unlock; - } - } - if (priv->nodelay) { ret = __socket_nodelay (new_sock); if (ret == -1) { gf_log (this->name, GF_LOG_WARNING, @@@@ -1871,9 +2186,13 @@@@ goto unlock; } get_transport_identifiers (new_trans); - socket_init (new_trans); + ret = socket_init (new_trans); + if (ret != 0) { + close(new_sock); + goto unlock; + } new_trans->ops = this->ops; new_trans->init = this->init; new_trans->fini = this->fini; new_trans->ctx = ctx; @@@@ -1882,22 +2201,63 @@@@ new_trans->notify = this->notify; new_trans->listener = this; new_priv = new_trans->private; + new_priv->use_ssl = priv->use_ssl; + new_priv->sock = new_sock; + new_priv->own_thread = priv->own_thread; + + if (priv->use_ssl) { + new_priv->ssl_ctx = priv->ssl_ctx; + if (ssl_setup_connection(new_trans,1) < 0) { + gf_log(this->name,GF_LOG_ERROR, + "server setup failed"); + close(new_sock); + goto unlock; + } + } + + if (!priv->bio) { + ret = __socket_nonblock (new_sock); + + if (ret == -1) { + gf_log (this->name, GF_LOG_WARNING, + "NBIO on %d failed (%s)", + new_sock, strerror (errno)); + + close (new_sock); + goto unlock; + } + } + pthread_mutex_lock (&new_priv->lock); { - new_priv->sock = new_sock; new_priv->connected = 1; rpc_transport_ref (new_trans); - new_priv->idx = - event_register (ctx->event_pool, - new_sock, - socket_event_handler, - new_trans, 1, 0); + if (new_priv->own_thread) { + if (pipe(new_priv->pipe) < 0) { + gf_log(this->name,GF_LOG_ERROR, + "could not create pipe"); + } + if (pthread_create(&new_priv->thread, + NULL, socket_poller, + new_trans) != 0) { + gf_log(this->name,GF_LOG_ERROR, + "could not create poll thread"); + } + } + else { + new_priv->idx = + event_register (ctx->event_pool, + new_sock, + socket_event_handler, + new_trans, + 1, 0); + if (new_priv->idx == -1) + ret = -1; + } - if (new_priv->idx == -1) - ret = -1; } pthread_mutex_unlock (&new_priv->lock); if (ret == -1) { gf_log ("", GF_LOG_WARNING, @@@@ -1984,8 +2344,24 @@@@ if (port > 0) ((struct sockaddr_in *) (&sockaddr))->sin_port = htons (port); + if (ntohs(((struct sockaddr_in *) (&sockaddr))->sin_port) == + GF_DEFAULT_SOCKET_LISTEN_PORT) { + if (priv->use_ssl) { + gf_log(this->name,GF_LOG_DEBUG, + "disabling SSL for portmapper connection"); + priv->use_ssl = _gf_false; + } + } + else { + if (priv->ssl_enabled && !priv->use_ssl) { + gf_log(this->name,GF_LOG_DEBUG, + "re-enabling SSL for I/O connection"); + priv->use_ssl = _gf_true; + } + } + pthread_mutex_lock (&priv->lock); { if (priv->sock != -1) { gf_log (this->name, GF_LOG_TRACE, @@@@ -2034,21 +2410,8 @@@@ strerror (errno)); } } - if (!priv->bio) { - ret = __socket_nonblock (priv->sock); - - if (ret == -1) { - gf_log (this->name, GF_LOG_ERROR, - "NBIO on %d failed (%s)", - priv->sock, strerror (errno)); - close (priv->sock); - priv->sock = -1; - goto unlock; - } - } - if (priv->keepalive) { ret = __socket_keepalive (priv->sock, priv->keepaliveintvl, priv->keepaliveidle); @@@@ -2082,19 +2445,58 @@@@ priv->sock = -1; goto unlock; } - priv->connected = 0; + if (priv->use_ssl) { + ret = ssl_setup_connection(this,0); + if (ret < 0) { + gf_log(this->name,GF_LOG_ERROR, + "client setup failed"); + close(priv->sock); + priv->sock = -1; + goto unlock; + } + } + + if (!priv->bio) { + ret = __socket_nonblock (priv->sock); + + if (ret == -1) { + gf_log (this->name, GF_LOG_ERROR, + "NBIO on %d failed (%s)", + priv->sock, strerror (errno)); + close (priv->sock); + priv->sock = -1; + goto unlock; + } + } + + priv->connected = 0; + rpc_transport_ref (this); + + if (priv->own_thread) { + if (pipe(priv->pipe) < 0) { + gf_log(this->name,GF_LOG_ERROR, + "could not create pipe"); + } + + if (pthread_create(&priv->thread,NULL, + socket_poller, this) != 0) { + gf_log(this->name,GF_LOG_ERROR, + "could not create poll thread"); + } + } + else { + priv->idx = event_register (ctx->event_pool, priv->sock, + socket_event_handler, + this, 1, 1); + if (priv->idx == -1) { + gf_log ("", GF_LOG_WARNING, + "failed to register the event"); + ret = -1; + } + } - rpc_transport_ref (this); - - priv->idx = event_register (ctx->event_pool, priv->sock, - socket_event_handler, this, 1, 1); - if (priv->idx == -1) { - gf_log ("", GF_LOG_WARNING, - "failed to register the event"); - ret = -1; - } } unlock: pthread_mutex_unlock (&priv->lock); @@@@ -2258,8 +2660,9 @@@@ char need_poll_out = 0; char need_append = 1; struct ioq *entry = NULL; glusterfs_ctx_t *ctx = NULL; + char a_byte = 'j'; GF_VALIDATE_OR_GOTO ("socket", this, out); GF_VALIDATE_OR_GOTO ("socket", this->private, out); @@@@ -2283,23 +2686,33 @@@@ if (!entry) goto unlock; if (list_empty (&priv->ioq)) { - ret = __socket_ioq_churn_entry (this, entry); + ret = __socket_ioq_churn_entry (this, entry, 1); - if (ret == 0) + if (ret == 0) { need_append = 0; - - if (ret > 0) + } + if (ret > 0) { need_poll_out = 1; + } } if (need_append) { list_add_tail (&entry->list, &priv->ioq); + if (priv->own_thread) { + /* + * Make sure the polling thread wakes up, by + * writing a byte to represent this entry. + */ + if (write(priv->pipe[1],&a_byte,1) < 1) { + gf_log(this->name,GF_LOG_WARNING, + "write error on pipe"); + } + } ret = 0; } - - if (need_poll_out) { + if (!priv->own_thread && need_poll_out) { /* first entry to wait. continue writing on POLLOUT */ priv->idx = event_select_on (ctx->event_pool, priv->sock, priv->idx, -1, 1); @@@@ -2321,8 +2734,9 @@@@ char need_poll_out = 0; char need_append = 1; struct ioq *entry = NULL; glusterfs_ctx_t *ctx = NULL; + char a_byte = 'd'; GF_VALIDATE_OR_GOTO ("socket", this, out); GF_VALIDATE_OR_GOTO ("socket", this->private, out); @@@@ -2339,35 +2753,46 @@@@ priv->submit_log = 1; } goto unlock; } + priv->submit_log = 0; entry = __socket_ioq_new (this, &reply->msg); if (!entry) goto unlock; + if (list_empty (&priv->ioq)) { - ret = __socket_ioq_churn_entry (this, entry); + ret = __socket_ioq_churn_entry (this, entry, 1); - if (ret == 0) + if (ret == 0) { need_append = 0; - - if (ret > 0) + } + if (ret > 0) { need_poll_out = 1; + } } if (need_append) { list_add_tail (&entry->list, &priv->ioq); + if (priv->own_thread) { + /* + * Make sure the polling thread wakes up, by + * writing a byte to represent this entry. + */ + if (write(priv->pipe[1],&a_byte,1) < 1) { + gf_log(this->name,GF_LOG_WARNING, + "write error on pipe"); + } + } ret = 0; } - - if (need_poll_out) { + if (!priv->own_thread && need_poll_out) { /* first entry to wait. continue writing on POLLOUT */ priv->idx = event_select_on (ctx->event_pool, priv->sock, priv->idx, -1, 1); } } - unlock: pthread_mutex_unlock (&priv->lock); out: @@@@ -2513,8 +2938,9 @@@@ uint64_t windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE; char *optstr = NULL; uint32_t keepalive = 0; uint32_t backlog = 0; + int session_id = 0; if (this->private) { gf_log_callingfn (this->name, GF_LOG_ERROR, "double init attempted"); @@@@ -2627,12 +3053,131 @@@@ priv->backlog = backlog; } priv->windowsize = (int)windowsize; + + priv->ssl_enabled = _gf_false; + if (dict_get_str(this->options,SSL_ENABLED_OPT,&optstr) == 0) { + if (gf_string2boolean (optstr, &priv->ssl_enabled) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "invalid value given for ssl-enabled boolean"); + } + } + + priv->ssl_own_cert = DEFAULT_CERT_PATH; + if (dict_get_str(this->options,SSL_OWN_CERT_OPT,&optstr) == 0) { + if (!priv->ssl_enabled) { + gf_log(this->name,GF_LOG_WARNING, + "%s specified without %s (ignored)", + SSL_OWN_CERT_OPT, SSL_ENABLED_OPT); + } + priv->ssl_own_cert = optstr; + } + priv->ssl_own_cert = gf_strdup(priv->ssl_own_cert); + + priv->ssl_private_key = DEFAULT_KEY_PATH; + if (dict_get_str(this->options,SSL_PRIVATE_KEY_OPT,&optstr) == 0) { + if (!priv->ssl_enabled) { + gf_log(this->name,GF_LOG_WARNING, + "%s specified without %s (ignored)", + SSL_PRIVATE_KEY_OPT, SSL_ENABLED_OPT); + } + priv->ssl_private_key = optstr; + } + priv->ssl_private_key = gf_strdup(priv->ssl_private_key); + + priv->ssl_ca_list = DEFAULT_CA_PATH; + if (dict_get_str(this->options,SSL_CA_LIST_OPT,&optstr) == 0) { + if (!priv->ssl_enabled) { + gf_log(this->name,GF_LOG_WARNING, + "%s specified without %s (ignored)", + SSL_CA_LIST_OPT, SSL_ENABLED_OPT); + } + priv->ssl_ca_list = optstr; + } + priv->ssl_ca_list = gf_strdup(priv->ssl_ca_list); + + gf_log(this->name,GF_LOG_INFO,"SSL support is %s", + priv->ssl_enabled ? "ENABLED" : "NOT enabled"); + /* + * This might get overridden temporarily in socket_connect (q.v.) + * if we're using the glusterd portmapper. + */ + priv->use_ssl = priv->ssl_enabled; + + priv->own_thread = priv->use_ssl; + if (dict_get_str(this->options,OWN_THREAD_OPT,&optstr) == 0) { + if (gf_string2boolean (optstr, &priv->own_thread) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "invalid value given for own-thread boolean"); + } + } + gf_log(this->name,GF_LOG_INFO,"using %s polling thread", + priv->own_thread ? "private" : "system"); + + if (priv->use_ssl) { + SSL_library_init(); + SSL_load_error_strings(); + priv->ssl_meth = (SSL_METHOD *)TLSv1_method(); + priv->ssl_ctx = SSL_CTX_new(priv->ssl_meth); + + if (SSL_CTX_set_cipher_list(priv->ssl_ctx, + "HIGH:-SSLv2") == 0) { + gf_log(this->name,GF_LOG_ERROR, + "failed to find any valid ciphers"); + goto err; + } + + if (!SSL_CTX_use_certificate_chain_file(priv->ssl_ctx, + priv->ssl_own_cert)) { + gf_log(this->name,GF_LOG_ERROR, + "could not load our cert"); + goto err; + } + + if (!SSL_CTX_use_PrivateKey_file(priv->ssl_ctx, + priv->ssl_private_key, + SSL_FILETYPE_PEM)) { + gf_log(this->name,GF_LOG_ERROR, + "could not load private key"); + goto err; + } + + if (!SSL_CTX_load_verify_locations(priv->ssl_ctx, + priv->ssl_ca_list,0)) { + gf_log(this->name,GF_LOG_ERROR, + "could not load CA list"); + goto err; + } + +#if (OPENSSL_VERSION_NUMBER < 0x00905100L) + SSL_CTX_set_verify_depth(ctx,1); +#endif + + priv->ssl_session_id = ++session_id; + SSL_CTX_set_session_id_context(priv->ssl_ctx, + (void *)&priv->ssl_session_id, + sizeof(priv->ssl_session_id)); + + SSL_CTX_set_verify(priv->ssl_ctx,SSL_VERIFY_PEER,0); + } + out: this->private = priv; - return 0; + +err: + if (priv->ssl_own_cert) { + GF_FREE(priv->ssl_own_cert); + } + if (priv->ssl_private_key) { + GF_FREE(priv->ssl_private_key); + } + if (priv->ssl_ca_list) { + GF_FREE(priv->ssl_ca_list); + } + GF_FREE(priv); + return -1; } void @@@@ -2656,8 +3201,17 @@@@ gf_log (this->name, GF_LOG_TRACE, "transport %p destroyed", this); pthread_mutex_destroy (&priv->lock); + if (priv->ssl_private_key) { + GF_FREE(priv->ssl_private_key); + } + if (priv->ssl_own_cert) { + GF_FREE(priv->ssl_own_cert); + } + if (priv->ssl_ca_list) { + GF_FREE(priv->ssl_ca_list); + } GF_FREE (priv); } this->private = NULL; @@@@ -2731,6 +3285,21 @@@@ }, { .key = {"transport.socket.listen-backlog"}, .type = GF_OPTION_TYPE_INT }, + { .key = {SSL_ENABLED_OPT}, + .type = GF_OPTION_TYPE_BOOL + }, + { .key = {SSL_OWN_CERT_OPT}, + .type = GF_OPTION_TYPE_STR + }, + { .key = {SSL_PRIVATE_KEY_OPT}, + .type = GF_OPTION_TYPE_STR + }, + { .key = {SSL_CA_LIST_OPT}, + .type = GF_OPTION_TYPE_STR + }, + { .key = {OWN_THREAD_OPT}, + .type = GF_OPTION_TYPE_BOOL + }, { .key = {NULL} } }; --- ./rpc/rpc-transport/socket/src/socket.h.orig 2011-11-14 14:46:00.000000000 +0100 +++ ./rpc/rpc-transport/socket/src/socket.h 2011-12-14 20:47:10.000000000 +0100 @@@@ -19,8 +19,10 @@@@ #ifndef _SOCKET_H #define _SOCKET_H +#include +#include #ifndef _CONFIG_H #define _CONFIG_H #include "config.h" @@@@ -143,9 +145,10 @@@@ typedef struct { int32_t sock; int32_t idx; - unsigned char connected; // -1 = not connected. 0 = in progress. 1 = connected + // -1 = not connected. 0 = in progress. 1 = connected + unsigned char connected; char bio; char connect_finish_log; char submit_log; union { @@@@ -192,8 +195,22 @@@@ int keepalive; int keepaliveidle; int keepaliveintvl; uint32_t backlog; + gf_boolean_t ssl_enabled; + gf_boolean_t use_ssl; + SSL_METHOD *ssl_meth; + SSL_CTX *ssl_ctx; + int ssl_session_id; + BIO *ssl_sbio; + SSL *ssl_ssl; + char *ssl_own_cert; + char *ssl_private_key; + char *ssl_ca_list; + pthread_t thread; + int pipe[2]; + gf_boolean_t own_thread; + volatile int socket_gen; } socket_private_t; #endif @ 1.2 log @- SSL bug fixes - Use secondary groups @ text @d1 1 a1 1 $NetBSD$ @ 1.1 log @- Add experimental support for SSL - Ignore again .attribute (a patch part that was lost in last upgrade) @ text @d6 76 a81 2 --- rpc/rpc-transport/socket/src/Makefile.am.orig 2011-11-14 14:46:00.000000000 +0100 +++ rpc/rpc-transport/socket/src/Makefile.am 2011-12-03 06:46:39.000000000 +0100 d93 2 a94 2 --- rpc/rpc-transport/socket/src/Makefile.in.orig 2011-11-14 14:46:14.000000000 +0100 +++ rpc/rpc-transport/socket/src/Makefile.in 2011-12-03 06:46:39.000000000 +0100 d106 3 a108 3 --- rpc/rpc-transport/socket/src/socket.c.orig 2011-12-07 02:20:01.000000000 +0100 +++ rpc/rpc-transport/socket/src/socket.c 2011-12-07 02:19:21.000000000 +0100 @@@@ -42,12 +42,36 @@@@ d113 1 a114 1 d124 1 a140 1 + d145 1 a145 1 @@@@ -132,11 +156,143 @@@@ d179 11 d193 4 d305 1 a305 1 @@@@ -167,19 +323,43 @@@@ d313 1 a313 1 + gf_log(this->name,GF_LOG_WARNING, d317 1 a317 1 + break; d352 1 a352 1 @@@@ -201,8 +381,11 @@@@ d364 1 a364 1 @@@@ -212,8 +395,19 @@@@ d370 1 a370 1 + gf_log(this->name,GF_LOG_WARNING, d384 1 a384 1 @@@@ -221,15 +415,13 @@@@ d402 1 a402 1 @@@@ -287,8 +479,22 @@@@ d425 1 a425 1 @@@@ -364,9 +570,8 @@@@ d435 1 a435 1 @@@@ -620,11 +825,13 @@@@ d451 1 a451 1 @@@@ -633,8 +840,20 @@@@ d472 1 a472 1 @@@@ -655,15 +874,15 @@@@ d490 1 a490 1 @@@@ -1645,9 +1864,8 @@@@ d500 12 a511 1 @@@@ -1730,11 +1948,11 @@@@ d525 1 a525 1 @@@@ -1742,18 +1960,15 @@@@ d540 1 a540 1 + ret = priv->connected ? 0 : socket_connect_finish(this); d545 1 a545 1 @@@@ -1767,15 +1982,114 @@@@ d571 1 a571 1 + if (!priv->connected) { d662 1 a662 1 @@@@ -1812,21 +2126,8 @@@@ d684 1 a684 1 @@@@ -1871,9 +2172,13 @@@@ d691 4 a694 4 + if (ret != 0) { + close(new_sock); + goto unlock; + } d699 1 a699 1 @@@@ -1882,22 +2187,63 @@@@ d771 1 a771 1 @@@@ -1984,8 +2330,24 @@@@ d776 15 a790 15 + if (ntohs(((struct sockaddr_in *) (&sockaddr))->sin_port) == + GF_DEFAULT_SOCKET_LISTEN_PORT) { + if (priv->use_ssl) { + gf_log(this->name,GF_LOG_DEBUG, + "disabling SSL for portmapper connection"); + priv->use_ssl = _gf_false; + } + } + else { + if (priv->ssl_enabled && !priv->use_ssl) { + gf_log(this->name,GF_LOG_DEBUG, + "re-enabling SSL for I/O connection"); + priv->use_ssl = _gf_true; + } + } d796 1 a796 1 @@@@ -2034,21 +2396,8 @@@@ d818 1 a818 1 @@@@ -2082,19 +2431,58 @@@@ a823 2 - - rpc_transport_ref (this); d874 2 d887 1 a887 1 @@@@ -2258,8 +2646,9 @@@@ d897 1 a897 1 @@@@ -2283,23 +2672,33 @@@@ d937 1 a937 1 @@@@ -2321,8 +2720,9 @@@@ d947 1 a947 1 @@@@ -2339,35 +2739,46 @@@@ d1001 1 a1001 1 @@@@ -2513,8 +2924,9 @@@@ d1011 1 a1011 1 @@@@ -2627,12 +3039,131 @@@@ d1144 1 a1144 1 @@@@ -2656,8 +3187,17 @@@@ d1162 1 a1162 1 @@@@ -2731,6 +3271,18 @@@@ d1167 3 d1184 2 a1185 2 --- rpc/rpc-transport/socket/src/socket.h.orig 2011-11-14 14:46:00.000000000 +0100 +++ rpc/rpc-transport/socket/src/socket.h 2011-12-03 06:46:39.000000000 +0100 d1197 13 a1209 1 @@@@ -192,8 +194,22 @@@@ a1231 15 --- xlators/mgmt/glusterd/src/glusterd-volgen.c.orig 2011-11-14 14:46:02.000000000 +0100 +++ xlators/mgmt/glusterd/src/glusterd-volgen.c 2011-12-03 06:46:39.000000000 +0100 @@@@ -187,8 +187,12 @@@@ {VKEY_FEATURES_QUOTA, "features/marker", "quota", "off", NO_DOC, OPT_FLAG_FORCE}, {VKEY_FEATURES_LIMIT_USAGE, "features/quota", "limit-set", NULL, NO_DOC, 0}, {"features.quota-timeout", "features/quota", "timeout", "0", DOC, 0}, + { "server.ssl", "protocol/server", +"transport.socket.ssl-enabled", NULL, NO_DOC, 0}, + { "client.ssl", "protocol/client", +"transport.socket.ssl-enabled", NULL, NO_DOC, 0}, {NULL, } }; @