1
0

__init__.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. """
  2. magic is a wrapper around the libmagic file identification library.
  3. See README for more information.
  4. Usage:
  5. >>> import magic
  6. >>> magic.from_file("testdata/test.pdf")
  7. 'PDF document, version 1.2'
  8. >>> magic.from_file("testdata/test.pdf", mime=True)
  9. 'application/pdf'
  10. >>> magic.from_buffer(open("testdata/test.pdf").read(1024))
  11. 'PDF document, version 1.2'
  12. >>>
  13. """
  14. import sys
  15. import glob
  16. import ctypes
  17. import ctypes.util
  18. import threading
  19. import logging
  20. from ctypes import c_char_p, c_int, c_size_t, c_void_p, byref, POINTER
  21. # avoid shadowing the real open with the version from compat.py
  22. _real_open = open
  23. class MagicException(Exception):
  24. def __init__(self, message):
  25. super(Exception, self).__init__(message)
  26. self.message = message
  27. class Magic:
  28. """
  29. Magic is a wrapper around the libmagic C library.
  30. """
  31. def __init__(self, mime=False, magic_file=None, mime_encoding=False,
  32. keep_going=False, uncompress=False, raw=False, extension=False):
  33. """
  34. Create a new libmagic wrapper.
  35. mime - if True, mimetypes are returned instead of textual descriptions
  36. mime_encoding - if True, codec is returned
  37. magic_file - use a mime database other than the system default
  38. keep_going - don't stop at the first match, keep going
  39. uncompress - Try to look inside compressed files.
  40. raw - Do not try to decode "non-printable" chars.
  41. extension - Print a slash-separated list of valid extensions for the file type found.
  42. """
  43. self.flags = MAGIC_NONE
  44. if mime:
  45. self.flags |= MAGIC_MIME_TYPE
  46. if mime_encoding:
  47. self.flags |= MAGIC_MIME_ENCODING
  48. if keep_going:
  49. self.flags |= MAGIC_CONTINUE
  50. if uncompress:
  51. self.flags |= MAGIC_COMPRESS
  52. if raw:
  53. self.flags |= MAGIC_RAW
  54. if extension:
  55. self.flags |= MAGIC_EXTENSION
  56. self.cookie = magic_open(self.flags)
  57. self.lock = threading.Lock()
  58. magic_load(self.cookie, magic_file)
  59. # MAGIC_EXTENSION was added in 523 or 524, so bail if
  60. # it doesn't appear to be available
  61. if extension and (not _has_version or version() < 524):
  62. raise NotImplementedError('MAGIC_EXTENSION is not supported in this version of libmagic')
  63. # For https://github.com/ahupp/python-magic/issues/190
  64. # libmagic has fixed internal limits that some files exceed, causing
  65. # an error. We can avoid this (at least for the sample file given)
  66. # by bumping the limit up. It's not clear if this is a general solution
  67. # or whether other internal limits should be increased, but given
  68. # the lack of other reports I'll assume this is rare.
  69. if _has_param:
  70. try:
  71. self.setparam(MAGIC_PARAM_NAME_MAX, 64)
  72. except MagicException as e:
  73. # some versions of libmagic fail this call,
  74. # so rather than fail hard just use default behavior
  75. pass
  76. def from_buffer(self, buf):
  77. """
  78. Identify the contents of `buf`
  79. """
  80. with self.lock:
  81. try:
  82. # if we're on python3, convert buf to bytes
  83. # otherwise this string is passed as wchar*
  84. # which is not what libmagic expects
  85. # NEXTBREAK: only take bytes
  86. if type(buf) == str and str != bytes:
  87. buf = buf.encode('utf-8', errors='replace')
  88. return maybe_decode(magic_buffer(self.cookie, buf))
  89. except MagicException as e:
  90. return self._handle509Bug(e)
  91. def from_file(self, filename):
  92. # raise FileNotFoundException or IOError if the file does not exist
  93. with _real_open(filename):
  94. pass
  95. with self.lock:
  96. try:
  97. return maybe_decode(magic_file(self.cookie, filename))
  98. except MagicException as e:
  99. return self._handle509Bug(e)
  100. def from_descriptor(self, fd):
  101. with self.lock:
  102. try:
  103. return maybe_decode(magic_descriptor(self.cookie, fd))
  104. except MagicException as e:
  105. return self._handle509Bug(e)
  106. def _handle509Bug(self, e):
  107. # libmagic 5.09 has a bug where it might fail to identify the
  108. # mimetype of a file and returns null from magic_file (and
  109. # likely _buffer), but also does not return an error message.
  110. if e.message is None and (self.flags & MAGIC_MIME_TYPE):
  111. return "application/octet-stream"
  112. else:
  113. raise e
  114. def setparam(self, param, val):
  115. return magic_setparam(self.cookie, param, val)
  116. def getparam(self, param):
  117. return magic_getparam(self.cookie, param)
  118. def __del__(self):
  119. # no _thread_check here because there can be no other
  120. # references to this object at this point.
  121. # during shutdown magic_close may have been cleared already so
  122. # make sure it exists before using it.
  123. # the self.cookie check should be unnecessary and was an
  124. # incorrect fix for a threading problem, however I'm leaving
  125. # it in because it's harmless and I'm slightly afraid to
  126. # remove it.
  127. if hasattr(self, 'cookie') and self.cookie and magic_close:
  128. magic_close(self.cookie)
  129. self.cookie = None
  130. _instances = {}
  131. def _get_magic_type(mime):
  132. i = _instances.get(mime)
  133. if i is None:
  134. i = _instances[mime] = Magic(mime=mime)
  135. return i
  136. def from_file(filename, mime=False):
  137. """"
  138. Accepts a filename and returns the detected filetype. Return
  139. value is the mimetype if mime=True, otherwise a human readable
  140. name.
  141. >>> magic.from_file("testdata/test.pdf", mime=True)
  142. 'application/pdf'
  143. """
  144. m = _get_magic_type(mime)
  145. return m.from_file(filename)
  146. def from_buffer(buffer, mime=False):
  147. """
  148. Accepts a binary string and returns the detected filetype. Return
  149. value is the mimetype if mime=True, otherwise a human readable
  150. name.
  151. >>> magic.from_buffer(open("testdata/test.pdf").read(1024))
  152. 'PDF document, version 1.2'
  153. """
  154. m = _get_magic_type(mime)
  155. return m.from_buffer(buffer)
  156. def from_descriptor(fd, mime=False):
  157. """
  158. Accepts a file descriptor and returns the detected filetype. Return
  159. value is the mimetype if mime=True, otherwise a human readable
  160. name.
  161. >>> f = open("testdata/test.pdf")
  162. >>> magic.from_descriptor(f.fileno())
  163. 'PDF document, version 1.2'
  164. """
  165. m = _get_magic_type(mime)
  166. return m.from_descriptor(fd)
  167. from . import loader
  168. libmagic = loader.load_lib()
  169. magic_t = ctypes.c_void_p
  170. def errorcheck_null(result, func, args):
  171. if result is None:
  172. err = magic_error(args[0])
  173. raise MagicException(err)
  174. else:
  175. return result
  176. def errorcheck_negative_one(result, func, args):
  177. if result == -1:
  178. err = magic_error(args[0])
  179. raise MagicException(err)
  180. else:
  181. return result
  182. # return str on python3. Don't want to unconditionally
  183. # decode because that results in unicode on python2
  184. def maybe_decode(s):
  185. # NEXTBREAK: remove
  186. if str == bytes:
  187. return s
  188. else:
  189. # backslashreplace here because sometimes libmagic will return metadata in the charset
  190. # of the file, which is unknown to us (e.g the title of a Word doc)
  191. return s.decode('utf-8', 'backslashreplace')
  192. try:
  193. from os import PathLike
  194. def unpath(filename):
  195. if isinstance(filename, PathLike):
  196. return filename.__fspath__()
  197. else:
  198. return filename
  199. except ImportError:
  200. def unpath(filename):
  201. return filename
  202. def coerce_filename(filename):
  203. if filename is None:
  204. return None
  205. filename = unpath(filename)
  206. # ctypes will implicitly convert unicode strings to bytes with
  207. # .encode('ascii'). If you use the filesystem encoding
  208. # then you'll get inconsistent behavior (crashes) depending on the user's
  209. # LANG environment variable
  210. # NEXTBREAK: remove
  211. is_unicode = (sys.version_info[0] <= 2 and
  212. isinstance(filename, unicode)) or \
  213. (sys.version_info[0] >= 3 and
  214. isinstance(filename, str))
  215. if is_unicode:
  216. return filename.encode('utf-8', 'surrogateescape')
  217. else:
  218. return filename
  219. magic_open = libmagic.magic_open
  220. magic_open.restype = magic_t
  221. magic_open.argtypes = [c_int]
  222. magic_close = libmagic.magic_close
  223. magic_close.restype = None
  224. magic_close.argtypes = [magic_t]
  225. magic_error = libmagic.magic_error
  226. magic_error.restype = c_char_p
  227. magic_error.argtypes = [magic_t]
  228. magic_errno = libmagic.magic_errno
  229. magic_errno.restype = c_int
  230. magic_errno.argtypes = [magic_t]
  231. _magic_file = libmagic.magic_file
  232. _magic_file.restype = c_char_p
  233. _magic_file.argtypes = [magic_t, c_char_p]
  234. _magic_file.errcheck = errorcheck_null
  235. def magic_file(cookie, filename):
  236. return _magic_file(cookie, coerce_filename(filename))
  237. _magic_buffer = libmagic.magic_buffer
  238. _magic_buffer.restype = c_char_p
  239. _magic_buffer.argtypes = [magic_t, c_void_p, c_size_t]
  240. _magic_buffer.errcheck = errorcheck_null
  241. def magic_buffer(cookie, buf):
  242. return _magic_buffer(cookie, buf, len(buf))
  243. magic_descriptor = libmagic.magic_descriptor
  244. magic_descriptor.restype = c_char_p
  245. magic_descriptor.argtypes = [magic_t, c_int]
  246. magic_descriptor.errcheck = errorcheck_null
  247. _magic_descriptor = libmagic.magic_descriptor
  248. _magic_descriptor.restype = c_char_p
  249. _magic_descriptor.argtypes = [magic_t, c_int]
  250. _magic_descriptor.errcheck = errorcheck_null
  251. def magic_descriptor(cookie, fd):
  252. return _magic_descriptor(cookie, fd)
  253. _magic_load = libmagic.magic_load
  254. _magic_load.restype = c_int
  255. _magic_load.argtypes = [magic_t, c_char_p]
  256. _magic_load.errcheck = errorcheck_negative_one
  257. def magic_load(cookie, filename):
  258. return _magic_load(cookie, coerce_filename(filename))
  259. magic_setflags = libmagic.magic_setflags
  260. magic_setflags.restype = c_int
  261. magic_setflags.argtypes = [magic_t, c_int]
  262. magic_check = libmagic.magic_check
  263. magic_check.restype = c_int
  264. magic_check.argtypes = [magic_t, c_char_p]
  265. magic_compile = libmagic.magic_compile
  266. magic_compile.restype = c_int
  267. magic_compile.argtypes = [magic_t, c_char_p]
  268. _has_param = False
  269. if hasattr(libmagic, 'magic_setparam') and hasattr(libmagic, 'magic_getparam'):
  270. _has_param = True
  271. _magic_setparam = libmagic.magic_setparam
  272. _magic_setparam.restype = c_int
  273. _magic_setparam.argtypes = [magic_t, c_int, POINTER(c_size_t)]
  274. _magic_setparam.errcheck = errorcheck_negative_one
  275. _magic_getparam = libmagic.magic_getparam
  276. _magic_getparam.restype = c_int
  277. _magic_getparam.argtypes = [magic_t, c_int, POINTER(c_size_t)]
  278. _magic_getparam.errcheck = errorcheck_negative_one
  279. def magic_setparam(cookie, param, val):
  280. if not _has_param:
  281. raise NotImplementedError("magic_setparam not implemented")
  282. v = c_size_t(val)
  283. return _magic_setparam(cookie, param, byref(v))
  284. def magic_getparam(cookie, param):
  285. if not _has_param:
  286. raise NotImplementedError("magic_getparam not implemented")
  287. val = c_size_t()
  288. _magic_getparam(cookie, param, byref(val))
  289. return val.value
  290. _has_version = False
  291. if hasattr(libmagic, "magic_version"):
  292. _has_version = True
  293. magic_version = libmagic.magic_version
  294. magic_version.restype = c_int
  295. magic_version.argtypes = []
  296. def version():
  297. if not _has_version:
  298. raise NotImplementedError("magic_version not implemented")
  299. return magic_version()
  300. MAGIC_NONE = 0x000000 # No flags
  301. MAGIC_DEBUG = 0x000001 # Turn on debugging
  302. MAGIC_SYMLINK = 0x000002 # Follow symlinks
  303. MAGIC_COMPRESS = 0x000004 # Check inside compressed files
  304. MAGIC_DEVICES = 0x000008 # Look at the contents of devices
  305. MAGIC_MIME_TYPE = 0x000010 # Return a mime string
  306. MAGIC_MIME_ENCODING = 0x000400 # Return the MIME encoding
  307. # TODO: should be
  308. # MAGIC_MIME = MAGIC_MIME_TYPE | MAGIC_MIME_ENCODING
  309. MAGIC_MIME = 0x000010 # Return a mime string
  310. MAGIC_EXTENSION = 0x1000000 # Return a /-separated list of extensions
  311. MAGIC_CONTINUE = 0x000020 # Return all matches
  312. MAGIC_CHECK = 0x000040 # Print warnings to stderr
  313. MAGIC_PRESERVE_ATIME = 0x000080 # Restore access time on exit
  314. MAGIC_RAW = 0x000100 # Don't translate unprintable chars
  315. MAGIC_ERROR = 0x000200 # Handle ENOENT etc as real errors
  316. MAGIC_NO_CHECK_COMPRESS = 0x001000 # Don't check for compressed files
  317. MAGIC_NO_CHECK_TAR = 0x002000 # Don't check for tar files
  318. MAGIC_NO_CHECK_SOFT = 0x004000 # Don't check magic entries
  319. MAGIC_NO_CHECK_APPTYPE = 0x008000 # Don't check application type
  320. MAGIC_NO_CHECK_ELF = 0x010000 # Don't check for elf details
  321. MAGIC_NO_CHECK_ASCII = 0x020000 # Don't check for ascii files
  322. MAGIC_NO_CHECK_TROFF = 0x040000 # Don't check ascii/troff
  323. MAGIC_NO_CHECK_FORTRAN = 0x080000 # Don't check ascii/fortran
  324. MAGIC_NO_CHECK_TOKENS = 0x100000 # Don't check ascii/tokens
  325. MAGIC_PARAM_INDIR_MAX = 0 # Recursion limit for indirect magic
  326. MAGIC_PARAM_NAME_MAX = 1 # Use count limit for name/use magic
  327. MAGIC_PARAM_ELF_PHNUM_MAX = 2 # Max ELF notes processed
  328. MAGIC_PARAM_ELF_SHNUM_MAX = 3 # Max ELF program sections processed
  329. MAGIC_PARAM_ELF_NOTES_MAX = 4 # # Max ELF sections processed
  330. MAGIC_PARAM_REGEX_MAX = 5 # Length limit for regex searches
  331. MAGIC_PARAM_BYTES_MAX = 6 # Max number of bytes to read from file
  332. # This package name conflicts with the one provided by upstream
  333. # libmagic. This is a common source of confusion for users. To
  334. # resolve, We ship a copy of that module, and expose it's functions
  335. # wrapped in deprecation warnings.
  336. def _add_compat(to_module):
  337. import warnings, re
  338. from magic import compat
  339. def deprecation_wrapper(fn):
  340. def _(*args, **kwargs):
  341. warnings.warn(
  342. "Using compatibility mode with libmagic's python binding. "
  343. "See https://github.com/ahupp/python-magic/blob/master/COMPAT.md for details.",
  344. PendingDeprecationWarning)
  345. return fn(*args, **kwargs)
  346. return _
  347. fn = ['detect_from_filename',
  348. 'detect_from_content',
  349. 'detect_from_fobj',
  350. 'open']
  351. for fname in fn:
  352. to_module[fname] = deprecation_wrapper(compat.__dict__[fname])
  353. # copy constants over, ensuring there's no conflicts
  354. is_const_re = re.compile("^[A-Z_]+$")
  355. allowed_inconsistent = set(['MAGIC_MIME'])
  356. for name, value in compat.__dict__.items():
  357. if is_const_re.match(name):
  358. if name in to_module:
  359. if name in allowed_inconsistent:
  360. continue
  361. if to_module[name] != value:
  362. raise Exception("inconsistent value for " + name)
  363. else:
  364. continue
  365. else:
  366. to_module[name] = value
  367. _add_compat(globals())