diff --git a/tests/conftest.py b/tests/conftest.py index b7391f3..8a9a5f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,12 @@ # pylint: disable=missing-function-docstring # pylint: disable=redefined-outer-name import os +import pathlib import subprocess import time import uuid +import oss2 import pytest import requests @@ -16,29 +18,38 @@ PORT = 5555 AccessKeyId = os.environ.get("OSS_ACCESS_KEY_ID", "") AccessKeySecret = os.environ.get("OSS_SECRET_ACCESS_KEY", "") +LICENSE_PATH = os.path.join( + pathlib.Path(__file__).parent.parent.resolve(), "LICENSE" +) +NUMBERS = b"1234567890\n" test_id = uuid.uuid4() -@pytest.fixture() +@pytest.fixture(scope="session") def emulator_endpoint(): - return "http://127.0.0.1:%s/" % PORT + return f"http://127.0.0.1:{PORT}/" -@pytest.fixture() +@pytest.fixture(scope="session") def endpoint(): return os.environ.get("OSS_ENDPOINT") -@pytest.fixture() +@pytest.fixture(scope="session") def test_bucket_name(): return os.environ.get("OSS_TEST_BUCKET_NAME") -@pytest.fixture() -def test_path(test_bucket_name): - return f"/{test_bucket_name}/ossfs_test/{test_id}" +@pytest.fixture(scope="session") +def test_directory(): + return f"ossfs_test/{test_id}" + + +@pytest.fixture(scope="session") +def test_path(test_bucket_name, test_directory): + return f"/{test_bucket_name}/{test_directory}" @pytest.fixture() @@ -46,7 +57,7 @@ def oss_emulator_server_start(emulator_endpoint): """ Start a local emulator server """ - with subprocess.Popen("ruby bin/emulator -r store -p {}".format(PORT)): + with subprocess.Popen(f"ruby bin/emulator -r store -p {PORT}"): timeout = 5 while timeout > 0: @@ -61,7 +72,7 @@ def oss_emulator_server_start(emulator_endpoint): yield -@pytest.fixture() +@pytest.fixture(scope="session") def init_config(endpoint): result = {} result["key"] = AccessKeyId @@ -73,3 +84,37 @@ def init_config(endpoint): @pytest.fixture() def ossfs(init_config): return OSSFileSystem(**init_config) + + +def put_object(endpoint, bucket_name, filename, contents): + auth = oss2.Auth(AccessKeyId, AccessKeySecret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + bucket.put_object(filename, contents) + + +def put_file(endpoint, bucket_name, key, filename): + auth = oss2.Auth(AccessKeyId, AccessKeySecret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + bucket.put_object_from_file(key, filename) + + +@pytest.fixture(scope="session") +def file_in_anonymous(endpoint, test_directory): + bucket = "dvc-anonymous" + file = f"{test_directory}/file" + put_object(endpoint, bucket, file, "foobar") + return f"/{bucket}/{file}" + + +@pytest.fixture(scope="session") +def number_file(test_bucket_name, endpoint, test_directory): + filename = f"{test_directory}/number" + put_object(endpoint, test_bucket_name, filename, NUMBERS) + return f"/{test_bucket_name}/{filename}" + + +@pytest.fixture(scope="session") +def license_file(test_bucket_name, endpoint, test_directory): + filename = f"{test_directory}/LICENSE" + put_file(endpoint, test_bucket_name, filename, LICENSE_PATH) + return f"/{test_bucket_name}/{filename}" diff --git a/tests/test_file.py b/tests/test_file.py index c7e5986..2725c55 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -8,17 +8,7 @@ import os import pytest - -files = { - "LICENSE": ( - b" Apache License\n" - b" Version 2.0, January 2004\n" - b" http://www.apache.org/licenses/\n" - ), - "number": (b"1234567890\n"), -} - -glob_files = {"file.dat": b"", "filexdat": b""} +from conftest import LICENSE_PATH, NUMBERS def test_simple(ossfs, test_path): @@ -62,35 +52,36 @@ def test_seek(ossfs, test_path): assert f.seek(i) == i -def test_read_small(ossfs, test_bucket_name): - fn = test_bucket_name + "/number" - with ossfs.open(fn, "rb", block_size=3) as f: +def test_read_small(ossfs, number_file): + with ossfs.open(number_file, "rb", block_size=3) as f: out = [] while True: data = f.read(2) if data == b"": break out.append(data) - assert ossfs.cat(fn) == b"".join(out) + assert ossfs.cat(number_file) == b"".join(out) -def test_read_ossfs_block(ossfs, test_bucket_name): - data = files["LICENSE"] +def test_read_ossfs_block(ossfs, license_file, number_file): + with open(LICENSE_PATH, "rb") as f_r: + data = f_r.read() lines = io.BytesIO(data).readlines() - path = test_bucket_name + "/LICENSE" - assert ossfs.read_block(path, 0, 10, b"\n") == lines[0] - assert ossfs.read_block(path, 40, 10, b"\n") == lines[1] - assert ossfs.read_block(path, 0, 80, b"\n") == lines[0] + lines[1] - assert ossfs.read_block(path, 0, 120, b"\n") == data + assert ossfs.read_block(license_file, 0, 10, b"\n") == lines[0] + assert ossfs.read_block(license_file, 40, 10, b"\n") == lines[1] + assert ossfs.read_block(license_file, 0, 80, b"\n") == lines[0] + lines[1] + assert ossfs.read_block(license_file, 0, 120, b"\n") == b"".join( + [lines[0], lines[1], lines[2]] + ) - data = files["number"] - lines = io.BytesIO(data).readlines() - path = test_bucket_name + "/number" - assert len(ossfs.read_block(path, 0, 5)) == 5 - assert len(ossfs.read_block(path, 4, 150)) == len(data) - 4 - assert ossfs.read_block(path, 20, 25) == b"" + lines = io.BytesIO(NUMBERS).readlines() + assert len(ossfs.read_block(number_file, 0, 5)) == 5 + assert len(ossfs.read_block(number_file, 4, 150)) == len(NUMBERS) - 4 + assert ossfs.read_block(number_file, 20, 25) == b"" - assert ossfs.read_block(path, 5, None) == ossfs.read_block(path, 5, 25) + assert ossfs.read_block(number_file, 5, None) == ossfs.read_block( + number_file, 5, 25 + ) @pytest.mark.parametrize("size", [2 ** 10, 2 ** 20, 10 * 2 ** 20]) @@ -134,15 +125,18 @@ def test_write_blocks(ossfs, test_path): assert ossfs.info(file)["Size"] == 15 * 2 ** 20 -def test_readline(ossfs, test_bucket_name): - all_items = files.items() - for k, data in all_items: - with ossfs.open("/".join([test_bucket_name, k]), "rb") as f: - result = f.readline() - expected = data.split(b"\n")[0] + ( - b"\n" if data.count(b"\n") else b"" - ) - assert result == expected +def test_readline(ossfs, number_file, license_file): + with ossfs.open("/".join([number_file]), "rb") as f_r: + result = f_r.readline() + expected = NUMBERS + assert result == expected + + with ossfs.open("/".join([license_file]), "rb") as f_r, open( + LICENSE_PATH, "rb" + ) as f_l: + result = f_r.readline() + expected = f_l.readline() + assert result == expected def test_readline_empty(ossfs, test_path): @@ -174,10 +168,10 @@ def test_readline_blocksize(ossfs, test_path): assert result == expected -def test_next(ossfs, test_bucket_name): - expected = files["LICENSE"].split(b"\n")[0] + b"\n" - with ossfs.open(test_bucket_name + "/LICENSE") as f: - result = next(f) +def test_next(ossfs, license_file): + with open(LICENSE_PATH, "rb") as f_l, ossfs.open(license_file) as f_r: + expected = f_l.readline() + result = next(f_r) assert result == expected @@ -219,7 +213,7 @@ def test_file_status(ossfs, test_path): @pytest.mark.parametrize("data_size", [0, 20, 10 * 2 ** 20]) @pytest.mark.parametrize("append_size", [0, 20, 10 * 2 ** 20]) def test_append(ossfs, test_path, data_size, append_size): - file = test_path + "/test_append/file_{}_{}".format(data_size, append_size) + file = test_path + f"/test_append/file_{data_size}_{append_size}" data = os.urandom(data_size) extra = os.urandom(append_size) with ossfs.open(file, "wb") as f: @@ -230,8 +224,8 @@ def test_append(ossfs, test_path, data_size, append_size): assert ossfs.cat(file) == data + extra -def test_bigger_than_block_read(ossfs, test_bucket_name): - with ossfs.open(test_bucket_name + "/number", "rb", block_size=3) as f: +def test_bigger_than_block_read(ossfs, number_file): + with ossfs.open(number_file, "rb", block_size=3) as f: out = [] while True: data = f.read(4) diff --git a/tests/test_login.py b/tests/test_login.py index 0c88913..d2064f1 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -73,6 +73,6 @@ def test_env_endpoint(endpoint, test_bucket_name, monkeypatch): ossfs.ls(test_bucket_name) -def test_anonymous_login(): - ossfs = OSSFileSystem(endpoint="http://oss-cn-hangzhou.aliyuncs.com") - ossfs.cat("/dvc-test-anonymous/LICENSE", connect_timeout=600) +def test_anonymous_login(file_in_anonymous, endpoint): + ossfs = OSSFileSystem(endpoint=endpoint) + ossfs.cat(f"{file_in_anonymous}") diff --git a/tests/test_object.py b/tests/test_object.py index 8480ea5..ae9ce92 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -132,14 +132,13 @@ def test_bulk_delete(ossfs, test_path): ) -def test_ossfs_file_access(ossfs, test_bucket_name): - fn = test_bucket_name + "/number" +def test_ossfs_file_access(ossfs, number_file): data = b"1234567890\n" - assert ossfs.cat(fn) == data - assert ossfs.head(fn, 3) == data[:3] - assert ossfs.tail(fn, 3) == data[-3:] - assert ossfs.tail(fn, 10000) == data - assert ossfs.info(fn)["Size"] == len(data) + assert ossfs.cat(number_file) == data + assert ossfs.head(number_file, 3) == data[:3] + assert ossfs.tail(number_file, 3) == data[-3:] + assert ossfs.tail(number_file, 10000) == data + assert ossfs.info(number_file)["Size"] == len(data) def test_du(ossfs, test_path): @@ -173,16 +172,16 @@ def test_ossfs_ls(ossfs, test_path): assert all(isinstance(item, dict) for item in L) -def test_ossfs_big_ls(ossfs, test_bucket_name): - path = test_bucket_name + "/test_ossfs_big_ls" +def test_ossfs_big_ls(ossfs, test_path): + path = test_path + "/test_ossfs_big_ls" if not ossfs.exists(path): for x in range(1200): - ossfs.touch(path + "/%i.part" % x) + ossfs.touch(path + f"/{x}.part") files = ossfs.find(path, connect_timeout=600) for x in range(1200): - file = path + "/%i.part" % x + file = path + f"/{x}.part" if file not in files: - ossfs.touch(path + "/%i.part" % x) + ossfs.touch(path + f"/{x}.part") assert len(ossfs.find(path, connect_timeout=600)) == 1200 @@ -210,15 +209,13 @@ def test_ossfs_glob(ossfs, test_path): assert fn2 not in ossfs.glob(path + "/nested/file.*") -def test_copy(ossfs, test_bucket_name, test_path): - number_file = test_bucket_name + "/number" +def test_copy(ossfs, number_file, test_path): new_file = test_path + "/test_copy/file" ossfs.copy(number_file, new_file) assert ossfs.cat(number_file) == ossfs.cat(new_file) -def test_move(ossfs, test_bucket_name, test_path): - number_file = test_bucket_name + "/number" +def test_move(ossfs, number_file, test_path): from_file = test_path + "/test_move/from" to_file = test_path + "/test_move/to" ossfs.copy(number_file, from_file) @@ -378,7 +375,7 @@ def test_get_file_info_with_selector(ossfs, test_path): elif info["name"].rstrip("/").endswith(dir_a): assert info["type"] == "directory" else: - raise ValueError("unexpected path {}".format(info["name"])) + raise ValueError(f"unexpected path {info['name']}") def test_same_name_but_no_exact(ossfs, test_path): @@ -419,8 +416,8 @@ def test_leading_forward_slash(ossfs, test_path): assert ossfs.exists("/" + path + "/some/file") -def test_find_with_prefix(ossfs, test_bucket_name): - path = test_bucket_name + "/test_find_with_prefix" +def test_find_with_prefix(ossfs, test_path): + path = test_path + "/test_find_with_prefix" if not ossfs.exists(path): for cursor in range(100): ossfs.touch(path + f"/prefixes/test_{cursor}") @@ -448,10 +445,10 @@ def test_find_with_prefix(ossfs, test_bucket_name): READ_BLOCK_SIZE = 2 ** 14 # 16KB blocks -def test_get_put_file(ossfs, tmpdir, test_bucket_name): +def test_get_put_file(ossfs, tmpdir, test_path): src_file = str(tmpdir / "source") src2_file = str(tmpdir / "source_2") - dest_file = test_bucket_name + "/get_put_file/dest" + dest_file = test_path + "/get_put_file/dest" data = b"test" * 2 ** 20