Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ celerybeat-schedule
# virtualenv
.venv
venv/
uv.lock
ENV/

# Spyder project settings
Expand Down
78 changes: 5 additions & 73 deletions submitit/core/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,9 @@
# LICENSE file in the root directory of this source tree.
#

import io
import os
import shutil
import subprocess
import sys
import tarfile
from pathlib import Path
from typing import Optional

Expand Down Expand Up @@ -71,26 +68,12 @@ def test_slurmpaths_id_independent() -> None:


def test_archive_dev_folders(tmp_path: Path) -> None:
utils.archive_dev_folders([Path(__file__).parent], outfile=tmp_path.with_suffix(".tar.gz"))
shutil.unpack_archive(str(tmp_path.with_suffix(".tar.gz")), extract_dir=tmp_path)
archive = utils.archive_dev_folders([Path(__file__).parent], outfile=tmp_path.with_suffix(".tar.gz"))
assert archive.read_bytes()[:2] == b"\x1f\x8b", "archive is not gzip-compressed"
shutil.unpack_archive(str(archive), extract_dir=tmp_path)
assert (tmp_path / "core").exists()


@pytest.mark.parametrize("default_outfile", [False, True])
def test_archive_dev_folders_is_gzip(tmp_path: Path, monkeypatch, default_outfile: bool) -> None:
monkeypatch.chdir(tmp_path)
folder = tmp_path / "project"
folder.mkdir()
(folder / "data.txt").write_text("archived content")
outfile = None if default_outfile else tmp_path / "custom.tar.gz"
archive = utils.archive_dev_folders([folder], outfile=outfile)

with tarfile.open(archive, mode="r:gz") as reader:
content = reader.extractfile("project/data.txt")
assert content is not None
assert content.read() == b"archived content"


def test_command_function() -> None:
# This will call `submitit.core.test_core.do_nothing`
command = [sys.executable, "-m", "submitit.core.test_core"]
Expand All @@ -103,52 +86,21 @@ def test_command_function() -> None:
utils.CommandFunction(command, verbose=True)(error=True)


@pytest.mark.parametrize("verbose", [False, True])
def test_copy_process_streams_split_utf8(monkeypatch, capsys, verbose: bool) -> None:
code = "import os; os.write(1, 'A\\u20acZ'.encode()); os.write(2, 'B\\U0001f600Y'.encode())"
stdout, stderr = io.StringIO(), io.StringIO()
with subprocess.Popen(
[sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE
) as process:
# A pipe read can end inside a UTF-8 code point. Force that boundary
# without depending on subprocess timing or the OS pipe buffer size.
for stream in (process.stdout, process.stderr):
assert isinstance(stream, io.BufferedReader)
monkeypatch.setattr(stream.raw, "read", lambda size, read=stream.raw.read: read(1))
utils.copy_process_streams(process, stdout, stderr, verbose)
assert process.wait() == 0
assert stdout.getvalue() == "A\u20acZ"
assert stderr.getvalue() == "B\U0001f600Y"
captured = capsys.readouterr()
assert captured.out == (stdout.getvalue() if verbose else "")
assert captured.err == (stderr.getvalue() if verbose else "")


def test_copy_process_streams_incomplete_utf8() -> None:
code = "import os; os.write(1, bytes([0xe2, 0x82]))"
with subprocess.Popen(
[sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE
) as process:
with pytest.raises(UnicodeDecodeError):
utils.copy_process_streams(process, io.StringIO(), io.StringIO())
assert process.wait() == 0


def test_command_function_deadlock(executor) -> None:
code = """
import sys;
print(sys.__stderr__)
# The goal here is to fill up the stderr pipe buffer.
for i in range({n}):
print("-" * 1024, file=sys.stdout)
print("\u20ac" * 1024, file=sys.stdout) # multi-byte chars split across pipe reads
print("printed {n} lines to stderr")
"""
fn1 = utils.CommandFunction([sys.executable, "-c", code.format(n=10)])
executor.update_parameters(timeout_min=2 / 60)
j1 = executor.submit(fn1)
assert "10 lines" in j1.result()

fn2 = utils.CommandFunction(["python", "-c", code.format(n=1000)])
fn2 = utils.CommandFunction([sys.executable, "-c", code.format(n=1000)])
j2 = executor.submit(fn2)
assert "1000 lines" in j2.result()

Expand All @@ -160,23 +112,3 @@ def test_jobpaths(tmp_path: Path) -> None:
utils.JobPaths(tmp_path / "array-%A-index-%a", "456_3").stdout
== tmp_path / "array-456-index-3" / "456_3_0_log.out"
)


def test_command_function_environment(monkeypatch) -> None:
monkeypatch.setenv("SUBMITIT_TEST_PARENT", "inherited")
monkeypatch.delenv("SUBMITIT_TEST_CHILD", raising=False)
command = [
sys.executable,
"-c",
"import os; print(os.getenv('SUBMITIT_TEST_PARENT', 'missing'), "
"os.getenv('SUBMITIT_TEST_CHILD', 'missing'))",
]
assert utils.CommandFunction(command, verbose=False)() == "inherited missing"
replacement = {"SUBMITIT_TEST_CHILD": "value"}
assert utils.CommandFunction(
command, env=replacement, verbose=False
)() == "missing value"
merged = {**os.environ, **replacement}
assert utils.CommandFunction(
command, env=merged, verbose=False
)() == "inherited value"
Loading