Subject: Unbreak various things Origin: upstream, commit 0.4.27-39-ga3ed086 Author: Adam Hupp Date: Tue Feb 18 10:55:05 2025 -0800 * A merge to reduce error spam during loading broke .so loading in at least some (maybe all?) cases, where find_library doesn't return an absolute path. * Prematurely pushed some in-progress test changes that were super broken, all fixed now. --- a/magic/loader.py +++ b/magic/loader.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + def _lib_candidates_linux(): """Yield possible libmagic library names on Linux. @@ -51,7 +52,7 @@ "darwin": _lib_candidates_macos, "linux": _lib_candidates_linux, "win32": _lib_candidates_windows, - "sunos5": _lib_candidates_linux, + "sunos5": _lib_candidates_linux, }.get(sys.platform) if func is None: raise ImportError("python-magic: Unsupported platform: " + sys.platform) @@ -61,17 +62,20 @@ def load_lib(): + exc = [] for lib in _lib_candidates(): # find_library returns None when lib not found if lib is None: continue - if not os.path.exists(lib): - continue try: return ctypes.CDLL(lib) - except OSError: - logger.warning("Failed to load: " + lib, exc_info=True) + except OSError as e: + exc.append(e) + + msg = "\n".join([str(e) for e in exc]) # It is better to raise an ImportError since we are importing magic module - raise ImportError("python-magic: failed to find libmagic. Check your installation") + raise ImportError( + "python-magic: failed to find libmagic. Check your installation: \n" + msg + ) --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -5,6 +5,7 @@ import shutil import sys import tempfile +from typing import List, Union import unittest import pytest @@ -19,140 +20,162 @@ import magic + @dataclass class TestFile: file_name: str - mime_results: list[str] - text_results: list[str] - no_check_elf_results: list[str] | None + mime_results: List[str] + text_results: List[str] + no_check_elf_results: Union[List[str], None] buf_equals_file: bool = True + # magic_descriptor is broken (?) in centos 7, so don't run those tests SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR")) -COMMON_PLAIN = [ - {}, - {"check_soft": True}, - {"check_soft": False}, - {"check_json": True}, - {"check_json": False}, -] - -NO_SOFT = {"check_soft": False} - -COMMON_MIME = [{"mime": True, **k} for k in COMMON_PLAIN] +COMMON_PLAIN = [{}] +NO_SOFT = [{"check_soft": False}] +COMMON_MIME = [{"mime": True}] CASES = { - "magic._pyc_": [ - (COMMON_MIME, [ - "application/octet-stream", - "text/x-bytecode.python", - "application/x-bytecode.python", - ]), + b"magic._pyc_": [ + ( + COMMON_MIME, + [ + "application/octet-stream", + "text/x-bytecode.python", + "application/x-bytecode.python", + ], + ), (COMMON_PLAIN, ["python 2.4 byte-compiled"]), (NO_SOFT, ["data"]), ], - "test.pdf": [ + b"test.pdf": [ (COMMON_MIME, ["application/pdf"]), - (COMMON_PLAIN, [ - "PDF document, version 1.2", - "PDF document, version 1.2, 2 pages", - "PDF document, version 1.2, 2 page(s)", - ]), + ( + COMMON_PLAIN, + [ + "PDF document, version 1.2", + "PDF document, version 1.2, 2 pages", + "PDF document, version 1.2, 2 page(s)", + ], + ), (NO_SOFT, ["ASCII text"]), ], - "test.gz": [ + b"test.gz": [ (COMMON_MIME, ["application/gzip", "application/x-gzip"]), - (COMMON_PLAIN, [ - 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', - 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', - ]), - ({"extension": True}, [ - # some versions return '' for the extensions of a gz file, - # including w/ the command line. Who knows... - "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", - "gz/tgz/tpz/zabw/svgz", - "", - "???", - ]), + ( + COMMON_PLAIN, + [ + 'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15', + 'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated', + ], + ), + ( + [{"extension": True}], + [ + # some versions return '' for the extensions of a gz file, + # including w/ the command line. Who knows... + "gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz", + "gz/tgz/tpz/zabw/svgz", + "", + "???", + ], + ), (NO_SOFT, ["data"]), ], - "test.snappy.parquet": [ + b"test.snappy.parquet": [ (COMMON_MIME, ["application/octet-stream"]), (COMMON_PLAIN, ["Apache Parquet", "Par archive data"]), (NO_SOFT, ["data"]), ], - "test.json": [ - # TODO: soft, no_json + b"test.json": [ (COMMON_MIME, ["application/json"]), (COMMON_PLAIN, ["JSON text data"]), - ({"mime": True, "check_json": False}, [ - "data", - ]), - (NO_SOFT, ["JSON text data"]) + ( + [{"mime": True, "check_json": False}], + [ + "text/plain", + ], + ), + (NO_SOFT, ["JSON text data"]), ], - "elf-NetBSD-x86_64-echo": [ + b"elf-NetBSD-x86_64-echo": [ # TODO: soft, no elf - (COMMON_PLAIN, [ - "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - ]), - (COMMON_MIME, [ - "application/x-pie-executable", - "application/x-sharedlib", - ]), - ({"check_elf": False}, [ - "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", - ]), + ( + COMMON_PLAIN, + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", + ], + ), + ( + COMMON_MIME, + [ + "application/x-pie-executable", + "application/x-sharedlib", + ], + ), + ( + [{"check_elf": False}], + [ + "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)", + ], + ), # TODO: sometimes # "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped", - (NO_SOFT, ["data"]), ], - "test.txt": [ + b"text.txt": [ (COMMON_MIME, ["text/plain"]), (COMMON_PLAIN, ["ASCII text"]), - ({"mime_encoding": True}, [ - "us-ascii", - ]), + ( + [{"mime_encoding": True}], + [ + "us-ascii", + ], + ), (NO_SOFT, ["ASCII text"]), ], - "text-iso8859-1.txt": [ - ({"mime_encoding": True}, [ - "iso-8859-1", - ]), + b"text-iso8859-1.txt": [ + ( + [{"mime_encoding": True}], + [ + "iso-8859-1", + ], + ), ], b"\xce\xbb": [ (COMMON_MIME, ["text/plain"]), ], - "b\xce\xbb".decode("utf-8"): [ - (COMMON_MIME, ["text/plain"]), + b"name_use.jpg": [ + ([{"extension": True}], ["jpeg/jpg/jpe/jfif"]), + ], + b"keep-going.jpg": [ + (COMMON_MIME, ["image/jpeg"]), + ( + [{"mime": True, "keep_going": True}], + [ + "image/jpeg\\012- application/octet-stream", + ], + ), + ], + b"../../magic/loader.py": [ + ( + COMMON_MIME, + [ + "text/x-python", + "text/x-script.python", + ], + ) ], - "name_use.jpg": [ - ({"extension": True}, [ - "jpeg/jpg/jpe/jfif" - ]), - ], - "keep-going.jpg": [ - (COMMON_MIME, [ - "image/jpeg" - ]), - ({"mime": True, "keep_going": True}, [ - "image/jpeg\\012- application/octet-stream", - ]) - ], - "test.py": [ - (COMMON_MIME, [ - "text/x-python", - "text/x-script.python", - ]) - ] } + class MagicTest(unittest.TestCase): TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata")) @@ -165,7 +188,6 @@ def test_fs_encoding(self): self.assertEqual("utf-8", sys.getfilesystemencoding().lower()) - def test_from_file_str_and_bytes(self): filename = os.path.join(self.TESTDATA_DIR, "test.pdf") @@ -174,7 +196,6 @@ "application/pdf", magic.from_file(filename.encode("utf-8"), mime=True) ) - def test_all_cases(self): # TODO: # * MAGIC_EXTENSION not supported @@ -184,21 +205,24 @@ shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest) os.environ["TZ"] = "UTC" try: - for file_name, cases in CASES: - filename = os.path.join(self.TESTDATA_DIR, file_name) - for flags, outputs in cases: - m = magic.Magic(**flags) - with open(filename) as f: - self.assertIn(m.from_descriptor(f.fileno()), outputs) - - self.assertIn(m.from_file(filename), outputs) - - fname_bytes = filename.encode("utf-8") - self.assertIn(m.from_file(fname_bytes), outputs) - - with open(file_name, "rb") as f: - buf_result = m.from_buffer(f.read(1024)) - self.assertIn(buf_result, outputs) + for filename, cases in CASES.items(): + filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) + print("test case ", filename, file=sys.stderr) + for flag_variants, outputs in cases: + for flags in flag_variants: + print("flags", flags, file=sys.stderr) + m = magic.Magic(**flags) + with open(filename) as f: + self.assertIn(m.from_descriptor(f.fileno()), outputs) + + self.assertIn(m.from_file(filename), outputs) + + fname_str = filename.decode("utf-8") + self.assertIn(m.from_file(fname_str), outputs) + + with open(filename, "rb") as f: + buf_result = m.from_buffer(f.read(1024)) + self.assertIn(buf_result, outputs) finally: del os.environ["TZ"] os.unlink(dest) @@ -222,7 +246,6 @@ else: raise unittest.SkipTest("Magic file doesn't return expected type.") - def test_errors(self): m = magic.Magic() self.assertRaises(IOError, m.from_file, "nonexistent") @@ -233,7 +256,6 @@ finally: del os.environ["MAGIC"] - def test_rethrow(self): old = magic.magic_buffer try: