diff --git a/include/gfarm/error.h b/include/gfarm/error.h index 99a171d..f03eb0c 100644 --- a/include/gfarm/error.h +++ b/include/gfarm/error.h @@ -137,7 +137,7 @@ enum gfarm_errcode { GFARM_ERR_INVALID_CREDENTIAL, GFARM_ERR_NO_FILESYSTEM_NODE, GFARM_ERR_DIRECTORY_QUOTA_EXISTS, - + GFARM_ERR_TOO_SLOW_CONNECTION_IS_DROPPED, GFARM_ERR_NUMBER }; diff --git a/lib/libgfarm/gfarm/Makefile b/lib/libgfarm/gfarm/Makefile index 9278168..db321c5 100644 --- a/lib/libgfarm/gfarm/Makefile +++ b/lib/libgfarm/gfarm/Makefile @@ -234,7 +234,7 @@ gfp_xdr_server.lo: $(GFUTIL_SRCDIR)/id_table.h $(GFUTIL_SRCDIR)/thrsubr.h liberr gfs_acl.lo: gfs_chmod.lo: $(GFUTIL_SRCDIR)/gfutil.h $(GFUTIL_SRCDIR)/timer.h context.h gfs_profile.h gfm_client.h lookup.h gfs_chown.lo: $(GFUTIL_SRCDIR)/gfutil.h $(GFUTIL_SRCDIR)/timer.h context.h gfs_profile.h gfm_client.h lookup.h -gfs_client.lo: $(GFUTIL_SRCDIR)/gfutil.h $(GFUTIL_SRCDIR)/gfevent.h $(GFUTIL_SRCDIR)/hash.h $(GFUTIL_SRCDIR)/lru_cache.h context.h liberror.h sockutil.h iobuffer.h gfp_xdr.h io_fd.h host.h sockopt.h auth.h config.h conn_cache.h gfs_proto.h gfs_client.h gfm_client.h iostat.h filesystem.h gfs_failover.h +gfs_client.lo: $(GFUTIL_SRCDIR)/gfutil.h $(GFUTIL_SRCDIR)/gfevent.h $(GFUTIL_SRCDIR)/gfnetdb.h $(GFUTIL_SRCDIR)/hash.h $(GFUTIL_SRCDIR)/lru_cache.h context.h liberror.h sockutil.h iobuffer.h gfp_xdr.h io_fd.h host.h sockopt.h auth.h config.h conn_cache.h gfs_proto.h gfs_client.h gfm_client.h iostat.h filesystem.h gfs_failover.h gfs_dir.lo: $(GFUTIL_SRCDIR)/timer.h $(GFUTIL_SRCDIR)/gfutil.h gfs_profile.h gfm_client.h config.h lookup.h gfs_io.h gfs_dir.h gfs_failover.h gfs_dirplus.lo: $(GFUTIL_SRCDIR)/gfutil.h config.h gfm_client.h lookup.h gfs_io.h gfs_failover.h gfs_dirplusxattr.lo: $(GFUTIL_SRCDIR)/gfutil.h config.h gfm_client.h gfs_io.h gfs_dirplusxattr.h gfs_failover.h diff --git a/lib/libgfarm/gfarm/config.c b/lib/libgfarm/gfarm/config.c index 7669437..3c24549 100644 --- a/lib/libgfarm/gfarm/config.c +++ b/lib/libgfarm/gfarm/config.c @@ -1057,6 +1057,13 @@ int gfarm_iostat_max_client = GFARM_CONFIG_MISC_DEFAULT; #define GFARM_METADB_SERVER_LONG_TERM_LOCK_TYPE_DEFAULT \ GFARM_LOCK_TYPE_TICKETLOCK #define GFARM_NETWORK_RECEIVE_TIMEOUT_DEFAULT 60 /* 60 seconds */ +#define NETWORK_RECEIVE_BANDWIDTH_DROP_RATIO_DEFAULT 0 +#define NETWORK_RECEIVE_BANDWIDTH_MEASUREMENT_DURATION_DEFAULT 20 + /* should be different from GFARM_CONFIG_MISC_DEFAULT */ +#define NETWORK_SEND_BANDWIDTH_DROP_RATIO_DEFAULT 0 +#define NETWORK_SEND_BANDWIDTH_MEASUREMENT_DURATION_DEFAULT 20 + /* should be different from GFARM_CONFIG_MISC_DEFAULT */ + #define GFARM_FILE_TRACE_DEFAULT 0 /* disable */ #define GFARM_FATAL_ACTION_DEFAULT GFLOG_FATAL_ACTION_ABORT_BACKTRACE #ifdef HAVE_INFINIBAND @@ -3625,6 +3632,22 @@ parse_one_line(const char *s, char *p, e = parse_set_misc_enabled(p, &metadb_server_force_slave); } else if (strcmp(s, o = "network_receive_timeout") == 0) { e = parse_set_misc_int(p, &gfarm_ctxp->network_receive_timeout); + } else if (strcmp(s, o = "network_receive_bandwidth_drop_ratio") == 0 + ) { + e = parse_set_misc_int(p, + &gfarm_ctxp->network_receive_bandwidth_drop_ratio); + } else if (strcmp(s, + o = "network_receive_bandwidth_measurement_duration") == 0) { + e = parse_set_misc_int(p, + &gfarm_ctxp->network_receive_bandwidth_measurement_duration); + } else if (strcmp(s, o = "network_send_bandwidth_drop_ratio") == 0 + ) { + e = parse_set_misc_int(p, + &gfarm_ctxp->network_send_bandwidth_drop_ratio); + } else if (strcmp(s, + o = "network_send_bandwidth_measurement_duration") == 0) { + e = parse_set_misc_int(p, + &gfarm_ctxp->network_send_bandwidth_measurement_duration); } else if (strcmp(s, o = "file_trace") == 0) { e = parse_set_misc_enabled(p, &gfarm_ctxp->file_trace); } else if (strcmp(s, o = "debug_command") == 0) { @@ -4034,6 +4057,22 @@ gfarm_config_set_default_misc(void) if (gfarm_ctxp->network_receive_timeout == GFARM_CONFIG_MISC_DEFAULT) gfarm_ctxp->network_receive_timeout = GFARM_NETWORK_RECEIVE_TIMEOUT_DEFAULT; + if (gfarm_ctxp->network_receive_bandwidth_drop_ratio == + GFARM_CONFIG_MISC_DEFAULT) + gfarm_ctxp->network_receive_bandwidth_drop_ratio = + NETWORK_RECEIVE_BANDWIDTH_DROP_RATIO_DEFAULT; + if (gfarm_ctxp->network_receive_bandwidth_measurement_duration == + GFARM_CONFIG_MISC_DEFAULT) + gfarm_ctxp->network_receive_bandwidth_measurement_duration = + NETWORK_RECEIVE_BANDWIDTH_MEASUREMENT_DURATION_DEFAULT; + if (gfarm_ctxp->network_send_bandwidth_drop_ratio == + GFARM_CONFIG_MISC_DEFAULT) + gfarm_ctxp->network_send_bandwidth_drop_ratio = + NETWORK_SEND_BANDWIDTH_DROP_RATIO_DEFAULT; + if (gfarm_ctxp->network_send_bandwidth_measurement_duration == + GFARM_CONFIG_MISC_DEFAULT) + gfarm_ctxp->network_send_bandwidth_measurement_duration = + NETWORK_SEND_BANDWIDTH_MEASUREMENT_DURATION_DEFAULT; if (gfarm_ctxp->file_trace == GFARM_CONFIG_MISC_DEFAULT) gfarm_ctxp->file_trace = GFARM_FILE_TRACE_DEFAULT; if (gfarm_ctxp->fatal_action == GFARM_CONFIG_MISC_DEFAULT) diff --git a/lib/libgfarm/gfarm/context.c b/lib/libgfarm/gfarm/context.c index 5b8272e..3e381d3 100644 --- a/lib/libgfarm/gfarm/context.c +++ b/lib/libgfarm/gfarm/context.c @@ -156,6 +156,12 @@ gfarm_context_init(void) ctxp->client_parallel_copy = GFARM_CONFIG_MISC_DEFAULT; ctxp->client_parallel_max = GFARM_CONFIG_MISC_DEFAULT; ctxp->network_receive_timeout = GFARM_CONFIG_MISC_DEFAULT; + ctxp->network_receive_bandwidth_drop_ratio = GFARM_CONFIG_MISC_DEFAULT; + ctxp->network_receive_bandwidth_measurement_duration = + GFARM_CONFIG_MISC_DEFAULT; + ctxp->network_send_bandwidth_drop_ratio = GFARM_CONFIG_MISC_DEFAULT; + ctxp->network_send_bandwidth_measurement_duration = + GFARM_CONFIG_MISC_DEFAULT; ctxp->file_trace = GFARM_CONFIG_MISC_DEFAULT; ctxp->ib_rdma = GFARM_CONFIG_MISC_DEFAULT; ctxp->rdma_min_size = GFARM_CONFIG_MISC_DEFAULT; diff --git a/lib/libgfarm/gfarm/context.h b/lib/libgfarm/gfarm/context.h index c085c8d..57ca3f3 100644 --- a/lib/libgfarm/gfarm/context.h +++ b/lib/libgfarm/gfarm/context.h @@ -39,6 +39,10 @@ struct gfarm_context { int on_demand_replication; int call_rpc_instead_syscall; int network_receive_timeout; + int network_receive_bandwidth_drop_ratio; + int network_receive_bandwidth_measurement_duration; + int network_send_bandwidth_drop_ratio; + int network_send_bandwidth_measurement_duration; int file_trace; int fatal_action; int ib_rdma; diff --git a/lib/libgfarm/gfarm/gfp_xdr.h b/lib/libgfarm/gfarm/gfp_xdr.h index e6b7a8a..d31deb0 100644 --- a/lib/libgfarm/gfarm/gfp_xdr.h +++ b/lib/libgfarm/gfarm/gfp_xdr.h @@ -32,7 +32,8 @@ struct gfp_xdr; (e) == GFARM_ERR_SOCKET_IS_NOT_CONNECTED || \ (e) == GFARM_ERR_OPERATION_TIMED_OUT || \ (e) == GFARM_ERR_CONNECTION_REFUSED || \ - (e) == GFARM_ERR_NO_ROUTE_TO_HOST) + (e) == GFARM_ERR_NO_ROUTE_TO_HOST || \ + (e) == GFARM_ERR_TOO_SLOW_CONNECTION_IS_DROPPED /* software error */ ) gfarm_error_t gfp_xdr_new(struct gfp_iobuffer_ops *, void *, int, int, struct gfp_xdr **); diff --git a/lib/libgfarm/gfarm/gfs_client.c b/lib/libgfarm/gfarm/gfs_client.c index dcc207b..b5420ef 100644 --- a/lib/libgfarm/gfarm/gfs_client.c +++ b/lib/libgfarm/gfarm/gfs_client.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include "gfutil.h" #include "gfevent.h" +#include "gfnetdb.h" #include "hash.h" #include "lru_cache.h" #include "thrsubr.h" @@ -2363,6 +2365,40 @@ gfs_ib_rdma_result_multiplexed(struct gfs_ib_rdma_state *state) return (e); } /*---------------------------------------------*/ + +static void +drop_connection(struct gfp_xdr *conn, + double speed, double max_speed, int ratio, const char *msg) +{ + int gai_error, fd = gfp_xdr_fd(conn); + const char *hostaddr_prefix, *hostaddr; + char hostbuf[NI_MAXHOST]; + struct sockaddr_storage sa; + socklen_t sa_len = sizeof(sa); + + if (getpeername(fd, (struct sockaddr *)&sa, &sa_len) == -1) { + hostaddr = strerror(errno); + hostaddr_prefix = "cannot get peer address: "; + } else if ((gai_error = gfarm_getnameinfo( + (struct sockaddr *)&sa, sa_len, + hostbuf, sizeof(hostbuf), NULL, 0, + NI_NUMERICHOST | NI_NUMERICSERV)) != 0) { + hostaddr = gai_strerror(gai_error); + hostaddr_prefix = "cannot convert peer address to string: "; + } else { + hostaddr = hostbuf; + hostaddr_prefix = ""; + } + + gfp_xdr_shutdown(conn); /* to avoid a protocol error */ + + gflog_error(GFARM_MSG_UNFIXED, + "%s: too slow connection is dropped: " + "current speed (%g byte/sec) is more than %d times slower than " + "max speed (%g bytes/sec) against host %s%s", + msg, speed, ratio, max_speed, hostaddr_prefix, hostaddr); +} + #ifndef __KERNEL__ #define GFS_STACK_BUFSIZE GFS_PROTO_MAX_IOSIZE #else /* __KERNEL__ */ @@ -2393,6 +2429,14 @@ gfs_sendfile_common(struct gfp_xdr *conn, gfarm_int32_t *src_errp, off_t sent = 0; int mode_unknown = 1, mode_thread_safe = 1, until_eof = len < 0; char buffer[GFS_STACK_BUFSIZE]; + + int do_measure = 0, + ratio = gfarm_ctxp->network_send_bandwidth_drop_ratio, + duration = gfarm_ctxp->network_send_bandwidth_measurement_duration; + off_t measured_size; + struct timeval measured_time, target_time, now, dt; + double speed, max_speed; + #if 0 /* not yet in gfarm v2 */ struct gfs_client_rep_rate_info *rinfo = NULL; @@ -2402,6 +2446,15 @@ gfs_sendfile_common(struct gfp_xdr *conn, gfarm_int32_t *src_errp, fatal("%s: rate_info_alloc: no_memory", diag); } #endif + if (ratio > 0) { + do_measure = 1; + max_speed = 0.0; + measured_size = 0; + gettimeofday(&measured_time, NULL); + target_time = measured_time; + target_time.tv_sec += duration; + } + if (until_eof || len > 0) { for (;;) { to_read = until_eof ? GFS_STACK_BUFSIZE : @@ -2448,6 +2501,37 @@ gfs_sendfile_common(struct gfp_xdr *conn, gfarm_int32_t *src_errp, if (md_ctx != NULL) EVP_DigestUpdate(md_ctx, buffer, rv); + if (do_measure) { + measured_size += rv; + gettimeofday(&now, NULL); + if (gfarm_timeval_cmp(&now, &target_time) + >= 0) { + dt = now; + gfarm_timeval_sub(&dt, &measured_time); + /* avoid division-by-0 */ + if (dt.tv_sec > 0 || dt.tv_usec > 0) { + speed = measured_size / + (dt.tv_sec + + (double)dt.tv_sec / + GFARM_SECOND_BY_MICROSEC); + if (speed < max_speed/ratio) { + e = GFARM_ERR_TOO_SLOW_CONNECTION_IS_DROPPED; + e_conn = e; + drop_connection(conn, + speed, max_speed, + ratio, "sendfile"); + break; + } + if (max_speed < speed) + max_speed = speed; + measured_size = 0; + measured_time = now; + } + target_time = now; + target_time.tv_sec += duration; + } + } + #if 0 /* not yet in gfarm v2 */ if (rate_limit != 0) gfs_client_rep_rate_control(rinfo, rv); @@ -2459,8 +2543,10 @@ gfs_sendfile_common(struct gfp_xdr *conn, gfarm_int32_t *src_errp, gfs_client_rep_rate_info_free(rinfo); #endif - /* send EOF mark */ - e = gfp_xdr_send(conn, "b", 0, buffer); + if (e_conn != GFARM_ERR_TOO_SLOW_CONNECTION_IS_DROPPED) { + /* send EOF mark */ + e = gfp_xdr_send(conn, "b", 0, buffer); + } if (e_conn == GFARM_ERR_NO_ERROR) e_conn = e; if (src_errp != NULL) @@ -2490,6 +2576,23 @@ gfs_recvfile_common(struct gfp_xdr *conn, gfarm_int32_t *dst_errp, int md_aborted = 0; int mode_unknown = 1, mode_thread_safe = 1; + int do_measure = 0, + ratio = gfarm_ctxp->network_receive_bandwidth_drop_ratio, + duration = + gfarm_ctxp->network_receive_bandwidth_measurement_duration; + off_t measured_size; + struct timeval measured_time, target_time, now, dt; + double speed, max_speed; + + if (ratio > 0) { + do_measure = 1; + max_speed = 0.0; + measured_size = 0; + gettimeofday(&measured_time, NULL); + target_time = measured_time; + target_time.tv_sec += duration; + } + if (append_mode) { mode_unknown = 0; mode_thread_safe = 0; @@ -2523,6 +2626,37 @@ gfs_recvfile_common(struct gfp_xdr *conn, gfarm_int32_t *dst_errp, break; } size -= partial; + + if (do_measure) { + measured_size += partial; + gettimeofday(&now, NULL); + if (gfarm_timeval_cmp(&now, &target_time) + >= 0) { + dt = now; + gfarm_timeval_sub(&dt, &measured_time); + /* avoid division-by-0 */ + if (dt.tv_sec > 0 || dt.tv_usec > 0) { + speed = measured_size / + (dt.tv_sec + + (double)dt.tv_sec / + GFARM_SECOND_BY_MICROSEC); + if (speed < max_speed/ratio) { + e = GFARM_ERR_TOO_SLOW_CONNECTION_IS_DROPPED; + drop_connection(conn, + speed, max_speed, + ratio, "recvfile"); + break; + } + if (max_speed < speed) + max_speed = speed; + measured_size = 0; + measured_time = now; + } + target_time = now; + target_time.tv_sec += duration; + } + } + if (e_write != GFARM_ERR_NO_ERROR) { /* * write(2) returned an error. @@ -2588,6 +2722,7 @@ gfs_recvfile_common(struct gfp_xdr *conn, gfarm_int32_t *dst_errp, if (md_ctx != NULL && !md_aborted) EVP_DigestUpdate( md_ctx, buffer, partial); + } while (size > 0); if (e != GFARM_ERR_NO_ERROR) break; diff --git a/lib/libgfarm/gfarm/io_fd.c b/lib/libgfarm/gfarm/io_fd.c index 1513f0d..c6d501b 100644 --- a/lib/libgfarm/gfarm/io_fd.c +++ b/lib/libgfarm/gfarm/io_fd.c @@ -41,10 +41,11 @@ gfarm_iobuffer_blocking_read_timeout_fd_op(struct gfarm_iobuffer *b, void *cookie, int fd, void *data, int length) { ssize_t rv; - int save_errno, avail, timeout = gfarm_ctxp->network_receive_timeout; - char hostbuf[NI_MAXHOST], *hostaddr_prefix, *hostaddr; - struct sockaddr_in sin; - socklen_t slen = sizeof(sin); + int gai_error, avail, timeout = gfarm_ctxp->network_receive_timeout; + const char *hostaddr_prefix, *hostaddr; + char hostbuf[NI_MAXHOST]; + struct sockaddr_storage sa; + socklen_t sa_len = sizeof(sa); for (;;) { #ifdef HAVE_POLL @@ -66,15 +67,15 @@ gfarm_iobuffer_blocking_read_timeout_fd_op(struct gfarm_iobuffer *b, if (avail == 0) { gfarm_iobuffer_set_error(b, GFARM_ERR_OPERATION_TIMED_OUT); - if (getpeername(fd, (struct sockaddr *)&sin, &slen) + if (getpeername(fd, (struct sockaddr *)&sa, &sa_len) == -1) { hostaddr = strerror(errno); hostaddr_prefix = "cannot get peer address: "; - } else if ((save_errno = gfarm_getnameinfo( - (struct sockaddr *)&sin, slen, + } else if ((gai_error = gfarm_getnameinfo( + (struct sockaddr *)&sa, sa_len, hostbuf, sizeof(hostbuf), NULL, 0, - NI_NUMERICHOST | NI_NUMERICSERV) != 0)) { - hostaddr = strerror(save_errno); + NI_NUMERICHOST | NI_NUMERICSERV)) != 0) { + hostaddr = gai_strerror(gai_error); hostaddr_prefix = "cannot convert peer address to string: "; } else { diff --git a/lib/libgfarm/gfarm/liberror.c b/lib/libgfarm/gfarm/liberror.c index e4fae92..ede6518 100644 --- a/lib/libgfarm/gfarm/liberror.c +++ b/lib/libgfarm/gfarm/liberror.c @@ -147,6 +147,7 @@ static const char *errcode_string[GFARM_ERR_NUMBER] = { "invalid credential", "no filesystem node", "directory quota exists", + "too slow connection is dropped", }; static const char *errmsg_string[GFARM_ERRMSG_END - GFARM_ERRMSG_BEGIN] = {