diff --git a/py/Makefile b/py/Makefile index fb8f2f88e6b..94374c30005 100644 --- a/py/Makefile +++ b/py/Makefile @@ -37,6 +37,7 @@ docs: ## Build API docs test: ./venv/bin/python -m tests echo "Testing using BASE_URL" && H2O_WAVE_BASE_URL="/foo/" ./venv/bin/python -m tests + echo "Testing using LOCAL UPLOAD" && H2O_WAVE_WAVED_DIR=".." ./venv/bin/python -m tests purge: ## Purge previous build rm -rf build dist h2o_wave.egg-info diff --git a/py/h2o_wave/core.py b/py/h2o_wave/core.py index b3a50156ec5..05d74ded0cf 100644 --- a/py/h2o_wave/core.py +++ b/py/h2o_wave/core.py @@ -662,14 +662,42 @@ def upload(self, files: List[str]) -> List[str]: Returns: A list of remote URLs for the uploaded files, in order. """ - upload_files = [] + for f in files: + if not os.path.isfile(f): + raise ValueError(f'{f} is not a file.') + + waved_dir = os.environ.get('H2O_WAVE_WAVED_DIR', None) + data_dir = os.environ.get('H2O_WAVE_DATA_DIR', 'data') + skip_local_upload = os.environ.get('H2O_WAVE_SKIP_LOCAL_UPLOAD', None) + + # If we know the path of waved and running app on the same machine, + # we can simply copy the files instead of making an HTTP request. + if not skip_local_upload and waved_dir and _is_loopback_address(_config.hub_address): + try: + is_windows = 'Windows' in platform.system() + cp_command = 'xcopy' if is_windows else 'cp' + uploaded_files = [] + for f in files: + uuid = str(uuid4()) + dst = os.path.join(waved_dir, data_dir, 'f', uuid, os.path.basename(f)) + os.makedirs(os.path.dirname(dst), exist_ok=True) + p = subprocess.Popen([cp_command, f, dst, '/K/O/X' if is_windows else ''], stderr=subprocess.PIPE) + _, err = p.communicate() + if err: + raise ValueError(err.decode()) + uploaded_files.append(f'/_f/{uuid}/{os.path.basename(f)}') + return uploaded_files + except: + pass + + uploaded_files = [] file_handles: List[BufferedReader] = [] for f in files: file_handle = open(f, 'rb') - upload_files.append(('files', (os.path.basename(f), file_handle))) + uploaded_files.append(('files', (os.path.basename(f), file_handle))) file_handles.append(file_handle) - res = self._http.post(f'{_config.hub_address}_f/', files=upload_files) + res = self._http.post(f'{_config.hub_address}_f/', files=uploaded_files) for h in file_handles: h.close() @@ -872,22 +900,33 @@ async def upload(self, files: List[str]) -> List[str]: Returns: A list of remote URLs for the uploaded files, in order. """ + for f in files: + if not os.path.isfile(f): + raise ValueError(f'{f} is not a file.') + waved_dir = os.environ.get('H2O_WAVE_WAVED_DIR', None) data_dir = os.environ.get('H2O_WAVE_DATA_DIR', 'data') + skip_local_upload = os.environ.get('H2O_WAVE_SKIP_LOCAL_UPLOAD', None) + # If we know the path of waved and running app on the same machine, # we can simply copy the files instead of making an HTTP request. - if waved_dir and _is_loopback_address(_config.hub_address): - is_windows = 'Windows' in platform.system() - cp_command = 'xcopy' if is_windows else 'cp' - uuid = str(uuid4()) - for f in files: - if not os.path.isfile(f): - raise ValueError(f'{f} is not a file.') - dst = os.path.join(waved_dir, data_dir, 'f', uuid, os.path.basename(f)) - os.makedirs(os.path.dirname(dst), exist_ok=True) - p = subprocess.Popen([cp_command, f, dst, '/K/O/X' if is_windows else '']) - p.communicate() - return [f'/_f/{uuid}/{os.path.basename(f)}' for f in files] + if not skip_local_upload and waved_dir and _is_loopback_address(_config.hub_address): + try: + is_windows = 'Windows' in platform.system() + cp_command = 'xcopy' if is_windows else 'cp' + uploaded_files = [] + for f in files: + uuid = str(uuid4()) + dst = os.path.join(waved_dir, data_dir, 'f', uuid, os.path.basename(f)) + os.makedirs(os.path.dirname(dst), exist_ok=True) + p = subprocess.Popen([cp_command, f, dst, '/K/O/X' if is_windows else ''], stderr=subprocess.PIPE) + _, err = p.communicate() + if err: + raise ValueError(err.decode()) + uploaded_files.append(f'/_f/{uuid}/{os.path.basename(f)}') + return uploaded_files + except: + pass upload_files = [] file_handles: List[BufferedReader] = [] @@ -918,6 +957,7 @@ async def download(self, url: str, path: str) -> str: path = os.path.abspath(path) # If path is a directory, get basename from url filepath = os.path.join(path, os.path.basename(url)) if os.path.isdir(path) else path + async with self._http.stream('GET', f'{_config.hub_host_address}{url}') as res: if res.status_code != 200: await res.aread() diff --git a/py/tests/test_python_server.py b/py/tests/test_python_server.py index 4cebc90270a..9747a0fb76e 100644 --- a/py/tests/test_python_server.py +++ b/py/tests/test_python_server.py @@ -330,7 +330,6 @@ def test_cyc_buf_write(self): i=2, )))) - def test_proxy(self): # waved -proxy must be set url = 'https://wave.h2o.ai' @@ -342,7 +341,6 @@ def test_proxy(self): assert result.code == 400 assert len(result.headers) > 0 - def test_file_server(self): f1 = 'temp_file1.txt' with open(f1, 'w') as f: @@ -356,14 +354,12 @@ def test_file_server(self): os.remove(f2) assert s1 == s2 - def test_public_dir(self): p = site.download(f'{base_url}assets/brand/h2o.svg', 'h2o.svg') svg = read_file(p) os.remove(p) assert svg.index(' 0 - def test_deleting_files(self): upload_path, = site.upload([os.path.join('tests', 'test_folder', 'test.txt')]) res = httpx.get(f'http://localhost:10101{upload_path}') diff --git a/py/tests/test_python_server_async.py b/py/tests/test_python_server_async.py index 4c7be5bf977..f2d81b26665 100644 --- a/py/tests/test_python_server_async.py +++ b/py/tests/test_python_server_async.py @@ -19,6 +19,7 @@ from .utils import read_file + # TODO: Add cleanup (site.unload) to tests that upload files. class TestPythonServerAsync(unittest.IsolatedAsyncioTestCase): def __init__(self, methodName: str = ...) -> None: @@ -38,7 +39,6 @@ async def test_file_server(self): os.remove(f2) assert s1 == s2 - async def test_public_dir(self): base_url = os.getenv('H2O_WAVE_BASE_URL', '/') p = await self.site.download(f'{base_url}assets/brand/h2o.svg', 'h2o.svg') @@ -46,7 +46,6 @@ async def test_public_dir(self): os.remove(p) assert svg.index('