Skip to content

Commit

Permalink
Drop File limits
Browse files Browse the repository at this point in the history
  • Loading branch information
shnela committed Oct 28, 2022
1 parent f1b17bc commit 606ffb3
Show file tree
Hide file tree
Showing 3 changed files with 1 addition and 58 deletions.
10 changes: 0 additions & 10 deletions src/neptune/new/internal/utils/images.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
from pandas import DataFrame

from neptune.new.exceptions import PlotlyIncompatibilityException
from neptune.new.internal.utils import limits
from neptune.new.internal.utils.logger import logger

_logger = logging.getLogger(__name__)
Expand All @@ -55,27 +54,18 @@ def pilimage_fromarray():
def get_image_content(image) -> Optional[bytes]:
content = _image_to_bytes(image)

if limits.image_size_exceeds_limit(len(content)):
return None

return content


def get_html_content(chart) -> Optional[str]:
content = _to_html(chart)

if limits.file_size_exceeds_limit(len(content)):
return None

return content


def get_pickle_content(obj) -> Optional[bytes]:
content = _export_pickle(obj)

if limits.file_size_exceeds_limit(len(content)):
return None

return content


Expand Down
42 changes: 0 additions & 42 deletions src/neptune/new/internal/utils/limits.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,10 @@


_CUSTOM_RUN_ID_LENGTH = 36
_IMAGE_SIZE_LIMIT_MB = 15
_IN_MEMORY_SIZE_LIMIT_MB = 32
_STREAM_SIZE_LIMIT_MB = 32
_LOGGED_IMAGE_SIZE_LIMIT_MB = 15

BYTES_IN_MB = 1024 * 1024

STREAM_SIZE_LIMIT_BYTES = _STREAM_SIZE_LIMIT_MB * BYTES_IN_MB


def custom_run_id_exceeds_length(custom_run_id):
if custom_run_id and len(custom_run_id) > _CUSTOM_RUN_ID_LENGTH:
Expand All @@ -40,19 +35,6 @@ def custom_run_id_exceeds_length(custom_run_id):
return False


def image_size_exceeds_limit(content_size):
if content_size > _IMAGE_SIZE_LIMIT_MB * BYTES_IN_MB:
_logger.warning(
"You are attempting to create an image that is %.2fMB large. "
"Neptune supports logging images smaller than %dMB. "
"Resize or increase compression of this image",
content_size / BYTES_IN_MB,
_IMAGE_SIZE_LIMIT_MB,
)
return True
return False


def image_size_exceeds_limit_for_logging(content_size):
if content_size > _LOGGED_IMAGE_SIZE_LIMIT_MB * BYTES_IN_MB:
warnings.warn(
Expand All @@ -63,27 +45,3 @@ def image_size_exceeds_limit_for_logging(content_size):
)
return True
return False


def file_size_exceeds_limit(content_size):
if content_size > _IN_MEMORY_SIZE_LIMIT_MB * BYTES_IN_MB:
_logger.warning(
"You are attempting to create an in-memory file that is %.1fMB large. "
"Neptune supports logging in-memory file objects smaller than %dMB. "
"Resize or increase compression of this object",
content_size / BYTES_IN_MB,
_IN_MEMORY_SIZE_LIMIT_MB,
)
return True
return False


def stream_size_exceeds_limit(content_size):
if content_size > _STREAM_SIZE_LIMIT_MB * BYTES_IN_MB:
_logger.warning(
"Your stream is larger than %dMB. " "Neptune supports saving files from streams smaller than %dMB.",
_STREAM_SIZE_LIMIT_MB,
_STREAM_SIZE_LIMIT_MB,
)
return True
return False
7 changes: 1 addition & 6 deletions src/neptune/new/types/atoms/file_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,7 @@

from neptune.common.exceptions import NeptuneException
from neptune.new.exceptions import StreamAlreadyUsedException
from neptune.new.internal.utils import (
limits,
verify_type,
)
from neptune.new.internal.utils import verify_type


class _FileType(enum.Enum):
Expand Down Expand Up @@ -95,8 +92,6 @@ class _InMemoryComposite(_FileComposite):
file_type = _FileType.IN_MEMORY

def __init__(self, content: Union[str, bytes], extension: Optional[str] = None):
if limits.file_size_exceeds_limit(len(content)):
content = b""
if isinstance(content, str):
ext = "txt"
content = content.encode("utf-8")
Expand Down

0 comments on commit 606ffb3

Please sign in to comment.