From c66f5528f8850366811259672476523e2421fd3d Mon Sep 17 00:00:00 2001 From: Jeff Raubitschek Date: Wed, 17 Aug 2022 09:22:28 -0700 Subject: [PATCH 1/3] merge --- wandb/sdk/lib/sock_client.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/wandb/sdk/lib/sock_client.py b/wandb/sdk/lib/sock_client.py index 13f409981e7..f75e66f0651 100644 --- a/wandb/sdk/lib/sock_client.py +++ b/wandb/sdk/lib/sock_client.py @@ -25,6 +25,8 @@ class SockClient: _sockid: str _retry_delay: float _lock: "threading.Lock" + _buffer: SockBuffer + _bufsize: int # current header is magic byte "W" followed by 4 byte length of the message HEADLEN = 1 + 4 @@ -35,11 +37,19 @@ def __init__(self) -> None: self._sockid = uuid.uuid4().hex self._retry_delay = 0.1 self._lock = threading.Lock() + self._buffer = SockBuffer() + self._bufsize = 4096 def connect(self, port: int) -> None: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("localhost", port)) self._sock = s + self._detect_bufsize() + + def _detect_bufsize(self) -> None: + sndbuf_size = self._sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + rcvbuf_size = self._sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) + self._bufsize = min(sndbuf_size, rcvbuf_size, 131072) def close(self) -> None: self._sock.close() @@ -49,6 +59,7 @@ def shutdown(self, val: int) -> None: def set_socket(self, sock: socket.socket) -> None: self._sock = sock + self._detect_bufsize() def _sendall_with_error_handle(self, data: bytes) -> None: # This is a helper function for sending data in a retry fashion. @@ -183,7 +194,7 @@ def _read_packet_bytes(self, timeout: int = None) -> Optional[bytes]: if timeout: self._sock.settimeout(timeout) try: - data = self._sock.recv(4096) + data = self._sock.recv(self._bufsize) except socket.timeout: break except ConnectionResetError: From ea87c8283ace951d49c744e0fe165a3215e26df5 Mon Sep 17 00:00:00 2001 From: Jeff Raubitschek Date: Wed, 17 Aug 2022 09:26:53 -0700 Subject: [PATCH 2/3] remove sockbuffer --- wandb/sdk/lib/sock_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wandb/sdk/lib/sock_client.py b/wandb/sdk/lib/sock_client.py index f75e66f0651..f8ada717f3b 100644 --- a/wandb/sdk/lib/sock_client.py +++ b/wandb/sdk/lib/sock_client.py @@ -25,7 +25,6 @@ class SockClient: _sockid: str _retry_delay: float _lock: "threading.Lock" - _buffer: SockBuffer _bufsize: int # current header is magic byte "W" followed by 4 byte length of the message @@ -37,7 +36,6 @@ def __init__(self) -> None: self._sockid = uuid.uuid4().hex self._retry_delay = 0.1 self._lock = threading.Lock() - self._buffer = SockBuffer() self._bufsize = 4096 def connect(self, port: int) -> None: From a689bd23ef7b3e82cbb3c53bcbf859851a1f0a6c Mon Sep 17 00:00:00 2001 From: Jeff Raubitschek Date: Wed, 17 Aug 2022 23:00:14 -0700 Subject: [PATCH 3/3] drop to 64k --- wandb/sdk/lib/sock_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wandb/sdk/lib/sock_client.py b/wandb/sdk/lib/sock_client.py index 1e03f948d6b..a4fa334cbec 100644 --- a/wandb/sdk/lib/sock_client.py +++ b/wandb/sdk/lib/sock_client.py @@ -106,7 +106,7 @@ def connect(self, port: int) -> None: def _detect_bufsize(self) -> None: sndbuf_size = self._sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) rcvbuf_size = self._sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) - self._bufsize = min(sndbuf_size, rcvbuf_size, 131072) + self._bufsize = min(sndbuf_size, rcvbuf_size, 65536) def close(self) -> None: self._sock.close()