Import upstream version 5.20

ChangeLog

@@ -1,3 +1,19 @@
+2014-10-10  15:01  Christos Zoulas <christos@zoulas.com>
+
+	* release 5.20
+
+2014-08-17  10:01  Christos Zoulas <christos@zoulas.com>
+
+	* recognize encrypted CDF documents
+
+2014-08-04   9:18  Christos Zoulas <christos@zoulas.com>
+
+	* add magic_load_buffers from Brooks Davis
+
+2014-07-24  16:40  Christos Zoulas <christos@zoulas.com>
+
+	* add thumbs.db support
+
 2014-06-12  12:28  Christos Zoulas <christos@zoulas.com>
 
 	* release 5.19

config.h.in

@@ -44,6 +44,9 @@
 /* Define to 1 if you have the `fork' function. */
 #undef HAVE_FORK
 
+/* Define to 1 if you have the `freelocale' function. */
+#undef HAVE_FREELOCALE
+
 /* Define to 1 if fseeko (and presumably ftello) exists and is declared. */
 #undef HAVE_FSEEKO
 
@@ -95,6 +98,9 @@
 /* Define to 1 if you have a working `mmap' system call. */
 #undef HAVE_MMAP
 
+/* Define to 1 if you have the `newlocale' function. */
+#undef HAVE_NEWLOCALE
+
 /* Define to 1 if you have the `pread' function. */
 #undef HAVE_PREAD
 
@@ -182,6 +188,9 @@
 /* Define to 1 if you have the <unistd.h> header file. */
 #undef HAVE_UNISTD_H
 
+/* Define to 1 if you have the `uselocale' function. */
+#undef HAVE_USELOCALE
+
 /* Define to 1 if you have the `utime' function. */
 #undef HAVE_UTIME
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for file 5.19.
+# Generated by GNU Autoconf 2.69 for file 5.20.
 #
 # Report bugs to <christos@astron.com>.
 #
@@ -590,8 +590,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='file'
 PACKAGE_TARNAME='file'
-PACKAGE_VERSION='5.19'
-PACKAGE_STRING='file 5.19'
+PACKAGE_VERSION='5.20'
+PACKAGE_STRING='file 5.20'
 PACKAGE_BUGREPORT='christos@astron.com'
 PACKAGE_URL=''
 
@@ -1327,7 +1327,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures file 5.19 to adapt to many kinds of systems.
+\`configure' configures file 5.20 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1397,7 +1397,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of file 5.19:";;
+     short | recursive ) echo "Configuration of file 5.20:";;
    esac
   cat <<\_ACEOF
 
@@ -1507,7 +1507,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-file configure 5.19
+file configure 5.20
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2163,7 +2163,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by file $as_me 5.19, which was
+It was created by file $as_me 5.20, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3029,7 +3029,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='file'
- VERSION='5.19'
+ VERSION='5.20'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -14191,7 +14191,7 @@ fi
 fi
 
 
-for ac_func in strerror strndup strtoul mkstemp mkostemp utimes utime wcwidth strtof
+for ac_func in strerror strndup strtoul mkstemp mkostemp utimes utime wcwidth strtof newlocale uselocale freelocale
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
@@ -14998,7 +14998,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by file $as_me 5.19, which was
+This file was extended by file $as_me 5.20, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -15064,7 +15064,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-file config.status 5.19
+file config.status 5.20
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,5 +1,5 @@
 dnl Process this file with autoconf to produce a configure script.
-AC_INIT([file],[5.19],[christos@astron.com])
+AC_INIT([file],[5.20],[christos@astron.com])
 AM_INIT_AUTOMAKE([subdir-objects foreign])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
@@ -138,7 +138,7 @@ else
 fi])
 
 dnl Checks for functions
-AC_CHECK_FUNCS(strerror strndup strtoul mkstemp mkostemp utimes utime wcwidth strtof)
+AC_CHECK_FUNCS(strerror strndup strtoul mkstemp mkostemp utimes utime wcwidth strtof newlocale uselocale freelocale)
 
 dnl Provide implementation of some required functions if necessary
 AC_REPLACE_FUNCS(getopt_long asprintf vasprintf strlcpy strlcat getline ctime_r asctime_r pread strcasestr fmtcheck)

doc/libmagic.man

@@ -1,4 +1,4 @@
-.\" $File: libmagic.man,v 1.28 2014/03/02 14:47:16 christos Exp $
+.\" $File: libmagic.man,v 1.29 2014/08/04 06:19:44 christos Exp $
 .\"
 .\" Copyright (c) Christos Zoulas 2003.
 .\" All Rights Reserved.
@@ -25,7 +25,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd January 6, 2012
+.Dd August 4, 2014
 .Dt LIBMAGIC 3
 .Os
 .Sh NAME
@@ -71,6 +71,8 @@
 .Ft int
 .Fn magic_load "magic_t cookie" "const char *filename"
 .Ft int
+.Fn magic_load_buffers "magic_t cookie" "void **buffers" "size_t *sizes" "size_t nbuffers"
+.Ft int
 .Fn magic_version "void"
 .Sh DESCRIPTION
 These functions
@@ -253,6 +255,19 @@ adds
 to the database filename as appropriate.
 .Pp
 The
+.Fn magic_load_buffers
+function takes an array of size
+.Fa nbuffers
+of
+.Fa buffers
+with a respective size for each in the array of
+.Fa sizes
+loaded with the contents of the magic databases from the filesystem.
+This function can be used in environment where the magic library does
+not have direct access to the filesystem, but can access the magic
+database via shared memory or other IPC means.
+.Pp
+The
 .Fn magic_version
 command returns the version number of this library which is compiled into
 the shared library using the constant

magic/Magdir/android

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------
-# $File: android,v 1.4 2014/06/03 19:01:34 christos Exp $
+# $File: android,v 1.6 2014/08/04 06:00:36 christos Exp $
 # Various android related magic entries
 #------------------------------------------------------------
 
@@ -98,3 +98,85 @@
 #>>>>>&1	regex/1l .*	\b, PBKDF2 rounds: %s
 #>>>>>>&1	regex/1l .*	\b, IV: %s
 #>>>>>>>&1	regex/1l .*	\b, Key: %s
+
+# *.pit files by Joerg Jenderek
+# http://forum.xda-developers.com/showthread.php?p=9122369
+# http://forum.xda-developers.com/showthread.php?t=816449
+# Partition Information Table for Samsung's smartphone with Android
+# used by flash software Odin
+0		ulelong			0x12349876	
+# 1st pit entry marker
+>0x01C	ulequad&0xFFFFFFFCFFFFFFFC	=0x0000000000000000	
+# minimal 13 and maximal 18 PIT entries found
+>>4		ulelong			<128	Partition Information Table for Samsung smartphone
+>>>4		ulelong			x	\b, %d entries
+# 1. pit entry
+>>>4		ulelong			>0	\b; #1
+>>>0x01C	use				PIT-entry
+>>>4		ulelong			>1	\b; #2
+>>>0x0A0	use				PIT-entry
+>>>4		ulelong			>2	\b; #3
+>>>0x124	use				PIT-entry
+>>>4		ulelong			>3	\b; #4
+>>>0x1A8	use				PIT-entry
+>>>4		ulelong			>4	\b; #5
+>>>0x22C	use				PIT-entry
+>>>4		ulelong			>5	\b; #6
+>>>0x2B0	use				PIT-entry
+>>>4		ulelong			>6	\b; #7
+>>>0x334	use				PIT-entry
+>>>4		ulelong			>7 	\b; #8
+>>>0x3B8	use				PIT-entry
+>>>4		ulelong			>8 	\b; #9
+>>>0x43C	use				PIT-entry
+>>>4		ulelong			>9	\b; #10
+>>>0x4C0	use				PIT-entry
+>>>4		ulelong			>10	\b; #11
+>>>0x544	use				PIT-entry
+>>>4		ulelong			>11	\b; #12
+>>>0x5C8	use				PIT-entry
+>>>4		ulelong			>12	\b; #13
+>>>>0x64C	use				PIT-entry
+# 14. pit entry
+>>>4		ulelong			>13	\b; #14
+>>>>0x6D0	use				PIT-entry
+>>>4		ulelong			>14	\b; #15
+>>>0x754	use				PIT-entry
+>>>4		ulelong			>15	\b; #16
+>>>0x7D8	use				PIT-entry
+>>>4		ulelong			>16	\b; #17
+>>>0x85C	use				PIT-entry
+# 18. pit entry
+>>>4		ulelong			>17	\b; #18
+>>>0x8E0	use				PIT-entry
+
+0	name			PIT-entry
+# garbage value implies end of pit entries
+>0x00		ulequad&0xFFFFFFFCFFFFFFFC	=0x0000000000000000	
+# skip empty partition name
+>>0x24		ubyte				!0			
+# partition name
+>>>0x24		string				>\0			%-.32s
+# flags
+>>>0x0C		ulelong&0x00000002		2			\b+RW
+# partition ID:
+# 0~IPL,MOVINAND,GANG;1~PIT,GPT;2~HIDDEN;3~SBL,HIDDEN;4~SBL2,HIDDEN;5~BOOT;6~KENREl,RECOVER,misc;7~RECOVER
+# ;11~MODEM;20~efs;21~PARAM;22~FACTORY,SYSTEM;23~DBDATAFS,USERDATA;24~CACHE;80~BOOTLOADER;81~TZSW
+>>>0x08	ulelong		x			(0x%x)
+# filename
+>>>0x44		string				>\0			"%-.64s"
+#>>>0x18	ulelong				>0			
+# blocksize in 512 byte units ?
+#>>>>0x18	ulelong				x			\b, %db
+# partition size in blocks ?
+#>>>>0x22	ulelong				x			\b*%d
+
+# Android bootimg format
+# From https://android.googlesource.com/\
+# platform/system/core/+/master/libsparse/sparse_format.h
+0		lelong	0xed26ff3a		Android sparse image
+>4		leshort	x			\b, version: %d
+>6		leshort	x			\b.%d
+>16		lelong	x			\b, Total of %d
+>12		lelong	x			\b %d-byte output blocks in
+>20		lelong	x			\b %d input chunks.

magic/Magdir/animation

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: animation,v 1.53 2014/04/30 21:41:02 christos Exp $
+# $File: animation,v 1.55 2014/09/13 14:29:51 christos Exp $
 # animation:  file(1) magic for animation/movie formats
 #
 # animation formats
@@ -32,43 +32,155 @@
 !:mime	application/x-quicktime-player
 4	string/W	jP		JPEG 2000 image
 !:mime	image/jp2
+# http://www.ftyps.com/ with local additions
 4	string		ftyp		ISO Media
->8	string		isom		\b, MPEG v4 system, version 1
-!:mime	video/mp4
->8	string		iso2		\b, MPEG v4 system, part 12 revision
->8	string		mp41		\b, MPEG v4 system, version 1
-!:mime	video/mp4
->8	string		mp42		\b, MPEG v4 system, version 2
-!:mime	video/mp4
->8	string		mp7t		\b, MPEG v4 system, MPEG v7 XML
->8	string		mp7b		\b, MPEG v4 system, MPEG v7 binary XML
->8	string/W	jp2		\b, JPEG 2000
-!:mime	image/jp2
+>8	string		3g2		\b, MPEG v4 system, 3GPP2
+!:mime	video/3gpp2
+>>11	byte		4		\b v4 (H.263/AMR GSM 6.10)
+>>11	byte		5		\b v5 (H.263/AMR GSM 6.10)
+>>11	byte		6		\b v6 (ITU H.264/AMR GSM 6.10)
+>>11	byte		a		\b C.S0050-0 V1.0
+>>11	byte		b		\b C.S0050-0-A V1.0.0
+>>11	byte		c		\b C.S0050-0-B V1.0
 >8	string		3ge		\b, MPEG v4 system, 3GPP
 !:mime	video/3gpp
+>>11	byte		6		\b, Release 6 MBMS Extended Presentations
+>>11	byte		7		\b, Release 7 MBMS Extended Presentations
 >8	string		3gg		\b, MPEG v4 system, 3GPP
+>11	byte		6		\b, Release 6 General Profile
 !:mime	video/3gpp
 >8	string		3gp		\b, MPEG v4 system, 3GPP
+>11	byte		1		\b, Release %d (non existent)
+>11	byte		2		\b, Release %d (non existent)
+>11	byte		3		\b, Release %d (non existent)
+>11	byte		4		\b, Release %d
+>11	byte		5		\b, Release %d
+>11	byte		6		\b, Release %d
+>11	byte		7		\b, Release %d Streaming Servers
 !:mime	video/3gpp
 >8	string		3gs		\b, MPEG v4 system, 3GPP
+>11	byte		7		\b, Release %d Streaming Servers
 !:mime	video/3gpp
->8	string		3g2		\b, MPEG v4 system, 3GPP2
+>8	string		avc1		\b, MPEG v4 system, 3GPP JVT AVC [ISO 14496-12:2005]
+!:mime	video/mp4
+>8	string/W	qt		\b, Apple QuickTime movie
+!:mime	video/quicktime
+>8	string		CAEP		\b, Canon Digital Camera
+>8	string		caqv		\b, Casio Digital Camera
+>8	string		CDes		\b, Convergent Design
+>8	string		da0a		\b, DMB MAF w/ MPEG Layer II aud, MOT slides, DLS, JPG/PNG/MNG
+>8	string		da0b		\b, DMB MAF, ext DA0A, with 3GPP timed text, DID, TVA, REL, IPMP
+>8	string		da1a		\b, DMB MAF audio with ER-BSAC audio, JPG/PNG/MNG images
+>8	string		da1b		\b, DMB MAF, ext da1a, with 3GPP timed text, DID, TVA, REL, IPMP
+>8	string		da2a		\b, DMB MAF aud w/ HE-AAC v2 aud, MOT slides, DLS, JPG/PNG/MNG
+>8	string		da2b		\b, DMB MAF, ext da2a, with 3GPP timed text, DID, TVA, REL, IPMP
+>8	string		da3a		\b, DMB MAF aud with HE-AAC aud, JPG/PNG/MNG images
+>8	string		da3b		\b, DMB MAF, ext da3a w/ BIFS, 3GPP, DID, TVA, REL, IPMP
+>8	string		dmb1		\b, DMB MAF supporting all the components defined in the spec
+>8	string		dmpf		\b, Digital Media Project
+>8	string		drc1		\b, Dirac (wavelet compression), encap in ISO base media (MP4)
+>8	string		dv1a		\b, DMB MAF vid w/ AVC vid, ER-BSAC aud, BIFS, JPG/PNG/MNG, TS
+>8	string		dv1b		\b, DMB MAF, ext dv1a, with 3GPP timed text, DID, TVA, REL, IPMP
+>8	string		dv2a		\b, DMB MAF vid w/ AVC vid, HE-AAC v2 aud, BIFS, JPG/PNG/MNG, TS
+>8	string		dv2b		\b, DMB MAF, ext dv2a, with 3GPP timed text, DID, TVA, REL, IPMP
+>8	string		dv3a		\b, DMB MAF vid w/ AVC vid, HE-AAC aud, BIFS, JPG/PNG/MNG, TS
+>8	string		dv3b		\b, DMB MAF, ext dv3a, with 3GPP timed text, DID, TVA, REL, IPMP
+>8	string		dvr1		\b, DVB (.DVB) over RTP
+!:mime	video/vnd.dvb.file
+>8	string		dvt1		\b, DVB (.DVB) over MPEG-2 Transport Stream
+!:mime	video/vnd.dvb.file
+>8	string		F4V		\b, Video for Adobe Flash Player 9+ (.F4V)
+!:mime	video/mp4
+>8	string		F4P		\b, Protected Video for Adobe Flash Player 9+ (.F4P)
+!:mime	video/mp4
+>8	string		F4A		\b, Audio for Adobe Flash Player 9+ (.F4A)
+!:mime	audio/mp4
+>8	string		F4B		\b, Audio Book for Adobe Flash Player 9+ (.F4B)
+!:mime	audio/mp4
+>8	string		isc2		\b, ISMACryp 2.0 Encrypted File
+#	?/enc-isoff-generic
+>8	string		iso2		\b, MP4 Base Media v2 [ISO 14496-12:2005]
+!:mime	video/mp4
+>8	string		isom		\b, MP4 Base Media v1 [IS0 14496-12:2003]
+!:mime	video/mp4
+>8	string/W	jp2		\b, JPEG 2000
+!:mime	image/jp2
+>8	string		JP2		\b, JPEG 2000 Image (.JP2) [ISO 15444-1 ?]
+!:mime	image/jp2
+>8	string		JP20		\b, Unknown, from GPAC samples (prob non-existent)
+>8	string		jpm		\b, JPEG 2000 Compound Image (.JPM) [ISO 15444-6]
+!:mime	image/jpm
+>8	string		jpx		\b, JPEG 2000 w/ extensions (.JPX) [ISO 15444-2]
+!:mime	image/jpx
+>8	string		KDDI		\b, 3GPP2 EZmovie for KDDI 3G cellphones
 !:mime	video/3gpp2
->>11	byte		4		\b v4 (H.263/AMR GSM 6.10)
->>11	byte		5		\b v5 (H.263/AMR GSM 6.10)
->>11	byte		6		\b v6 (ITU H.264/AMR GSM 6.10)
+>8	string		M4A 		\b, Apple iTunes AAC-LC (.M4A) Audio
+!:mime	audio/x-m4a
+>8	string		M4B 		\b, Apple iTunes AAC-LC (.M4B) Audio Book
+!:mime	audio/mp4
+>8	string		M4P 		\b, Apple iTunes AAC-LC (.M4P) AES Protected Audio
+!:mime	video/mp4
+>8	string		M4V 		\b, Apple iTunes Video (.M4V) Video
+!:mime	video/x-m4v
+>8	string		M4VH		\b, Apple TV (.M4V)
+!:mime	video/x-m4v
+>8	string		M4VP		\b, Apple iPhone (.M4V)
+!:mime	video/x-m4v
+>8	string		mj2s		\b, Motion JPEG 2000 [ISO 15444-3] Simple Profile
+!:mime	video/mj2
+>8	string		mjp2		\b, Motion JPEG 2000 [ISO 15444-3] General Profile
+!:mime	video/mj2
+>8	string		mmp4		\b, MPEG-4/3GPP Mobile Profile (.MP4 / .3GP) (for NTT)
+!:mime	video/mp4
+>8	string		mobi		\b, MPEG-4, MOBI format
+!:mime	video/mp4
+>8	string		mp21		\b, MPEG-21 [ISO/IEC 21000-9]
+>8	string		mp41		\b, MP4 v1 [ISO 14496-1:ch13]
+!:mime	video/mp4
+>8	string		mp42		\b, MP4 v2 [ISO 14496-14]
+!:mime	video/mp4
+>8	string		mp71		\b, MP4 w/ MPEG-7 Metadata [per ISO 14496-12]
+>8	string		mp7t		\b, MPEG v4 system, MPEG v7 XML
+>8	string		mp7b		\b, MPEG v4 system, MPEG v7 binary XML
 >8	string		mmp4		\b, MPEG v4 system, 3GPP Mobile
 !:mime	video/mp4
->8	string		avc1		\b, MPEG v4 system, 3GPP JVT AVC
-!:mime	video/3gpp
->8	string/W	M4A		\b, MPEG v4 system, iTunes AAC-LC
+>8	string		MPPI		\b, Photo Player, MAF [ISO/IEC 23000-3]
+>8	string		mqt		\b, Sony / Mobile QuickTime (.MQV) US Pat 7,477,830
+!:mime	video/quicktime
+>8	string		MSNV		\b, MPEG-4 (.MP4) for SonyPSP
+!:mime	audio/mp4
+>8	string		NDAS		\b, MP4 v2 [ISO 14496-14] Nero Digital AAC Audio
 !:mime	audio/mp4
->8	string/W	M4V		\b, MPEG v4 system, iTunes AVC-LC
+>8	string		NDSC		\b, MPEG-4 (.MP4) Nero Cinema Profile
 !:mime	video/mp4
->8	string/W	M4P		\b, MPEG v4 system, iTunes AES encrypted
->8	string/W	M4B		\b, MPEG v4 system, iTunes bookmarked
->8	string/W	qt		\b, Apple QuickTime movie
+>8	string		NDSH		\b, MPEG-4 (.MP4) Nero HDTV Profile
+!:mime	video/mp4
+>8	string		NDSM		\b, MPEG-4 (.MP4) Nero Mobile Profile
+!:mime	video/mp4
+>8	string		NDSP		\b, MPEG-4 (.MP4) Nero Portable Profile
+!:mime	video/mp4
+>8	string		NDSS		\b, MPEG-4 (.MP4) Nero Standard Profile
+!:mime	video/mp4
+>8	string		NDXC		\b, H.264/MPEG-4 AVC (.MP4) Nero Cinema Profile
+!:mime	video/mp4
+>8	string		NDXH		\b, H.264/MPEG-4 AVC (.MP4) Nero HDTV Profile
+!:mime	video/mp4
+>8	string		NDXM		\b, H.264/MPEG-4 AVC (.MP4) Nero Mobile Profile
+!:mime	video/mp4
+>8	string		NDXP		\b, H.264/MPEG-4 AVC (.MP4) Nero Portable Profile
+!:mime	video/mp4
+>8	string		NDXS		\b, H.264/MPEG-4 AVC (.MP4) Nero Standard Profile
+!:mime	video/mp4
+>8	string		odcf  		\b, OMA DCF DRM Format 2.0 (OMA-TS-DRM-DCF-V2_0-20060303-A)
+>8	string		opf2 		\b, OMA PDCF DRM Format 2.1 (OMA-TS-DRM-DCF-V2_1-20070724-C)
+>8	string		opx2  		\b, OMA PDCF DRM + XBS ext (OMA-TS-DRM_XBS-V1_0-20070529-C)
+>8	string		pana		\b, Panasonic Digital Camera
+>8	string		qt  		\b, Apple QuickTime (.MOV/QT)
 !:mime	video/quicktime
+>8	string		ROSS		\b, Ross Video
+>8	string		sdv		\b, SD Memory Card Video
+>8	string		ssc1		\b, Samsung stereo, single stream (patent pending)
+>8	string		ssc2		\b, Samsung stereo, dual stream (patent pending)
 
 # MPEG sequences
 # Scans for all common MPEG header start codes

magic/Magdir/archive

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# $File: archive,v 1.87 2014/06/03 19:15:58 christos Exp $
+# $File: archive,v 1.88 2014/08/16 10:42:17 christos Exp $
 # archive:  file(1) magic for archive formats (see also "msdos" for self-
 #           extracting compressed archives)
 #
@@ -954,34 +954,3 @@
 >0xE08	search/7776		\x55\xAA	
 >>&-512	indirect		x		\b; contains 
 
-# Symantec GHOST image by Joerg Jenderek at May 2014
-# http://us.norton.com/ghost/
-# http://www.garykessler.net/library/file_sigs.html
-0		ubelong&0xFFFFf7f0	0xFEEF0100	Norton GHost image
-# *.GHO
->2		ubyte&0x08		0x00		\b, first file
-# *.GHS or *.[0-9] with cns program option
->2		ubyte&0x08		0x08		\b, split file
-# part of split index interesting for *.ghs
->>4		ubyte			x		id=0x%x
-# compression tag minus one equals numeric compression command line switch z[1-9]
->3		ubyte			0		\b, no compression
->3		ubyte			2		\b, fast compression (Z1)
->3		ubyte			3		\b, medium compression (Z2)
->3		ubyte			>3		
->>3		ubyte			<11		\b, compression (Z%d-1)
->2		ubyte&0x08		0x00		
-# ~ 30 byte password field only for *.gho
->>12		ubequad			!0		\b, password protected
->>44		ubyte			!1		
-# 1~Image All, sector-by-sector only for *.gho
->>>10		ubyte			1		\b, sector copy
-# 1~Image Boot track only for *.gho
->>>43		ubyte			1		\b, boot track
-# 1~Image Disc only for *.gho implies Image Boot track and sector copy
->>44		ubyte			1		\b, disc sector copy
-# optional image description only *.gho
->>0xff		string			>\0		"%-.254s"
-# look for DOS sector end sequence
->0xE08	search/7776		\x55\xAA	
->>&-512	indirect		x		\b; contains 

magic/Magdir/blender

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: blender,v 1.5 2009/09/19 16:28:08 christos Exp $
+# $File: blender,v 1.6 2014/08/30 08:34:17 christos Exp $
 # blender: file(1) magic for Blender 3D related files
 #
 # Native format rule v1.2. For questions use the developers list 
@@ -35,5 +35,5 @@
 >>>0x44		string	=GLOB		\b.
 >>>>0x60	beshort	x		\b%.4d
 
-# Scripts that run in the embeded Python interpreter
+# Scripts that run in the embedded Python interpreter
 0		string	#!BPY		Blender3D BPython script

magic/Magdir/commands

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: commands,v 1.50 2014/05/30 16:48:44 christos Exp $
+# $File: commands,v 1.51 2014/09/27 00:12:55 christos Exp $
 # commands:  file(1) magic for various shells and interpreters
 #
 #0	string/w	:			shell archive or script for antique kernel text
@@ -56,7 +56,7 @@
 !:mime	text/x-awk
 0	string/wt	#!\ /usr/bin/awk	awk script text executable
 !:mime	text/x-awk
-0	regex/4096	=^\\s{0,100}BEGIN\\s{0,100}[{]	awk script text
+0	regex/4096	=^\\s{0,100}BEGIN\\s{0,100}[{]	awk or perl script text
 
 # AT&T Bell Labs' Plan 9 shell
 0	string/wt	#!\ /bin/rc	Plan 9 rc shell script text executable

magic/Magdir/compress

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# $File: compress,v 1.58 2014/05/07 19:36:59 christos Exp $
+# $File: compress,v 1.62 2014/09/13 14:27:12 christos Exp $
 # compress:  file(1) magic for pure-compression formats (no archives)
 #
 # compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc.
@@ -251,3 +251,13 @@
 # http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
 0	string	\377\006\0\0sNaPpY	snappy framed data
 !:mime	application/x-snappy-framed
+
+# qpress, http://www.quicklz.com/
+0	string	qpress10	qpress compressed data
+!:mime	application/x-qpress
+
+# Zlib https://www.ietf.org/rfc/rfc6713.txt
+0	beshort%31	=0	
+>0	byte&0xf	=8
+>>0	byte&0x80 	=0	zlib compressed data
+!:mime	application/zlib

magic/Magdir/database

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: database,v 1.41 2014/06/03 19:17:27 christos Exp $
+# $File: database,v 1.42 2014/08/19 14:18:04 christos Exp $
 # database:  file(1) magic for various databases
 #
 # extracted from header/code files by Graeme Wilford (eep2gw@ee.surrey.ac.uk)
@@ -9,9 +9,17 @@
 # GDBM magic numbers
 #  Will be maintained as part of the GDBM distribution in the future.
 #  <downsj@teeny.org>
-0	belong	0x13579ace	GNU dbm 1.x or ndbm database, big endian
+0	belong	0x13579acd	GNU dbm 1.x or ndbm database, big endian, 32-bit
 !:mime	application/x-gdbm
-0	lelong	0x13579ace	GNU dbm 1.x or ndbm database, little endian
+0	belong	0x13579ace	GNU dbm 1.x or ndbm database, big endian, old
+!:mime	application/x-gdbm
+0	belong	0x13579acf	GNU dbm 1.x or ndbm database, big endian, 64-bit
+!:mime	application/x-gdbm
+0	lelong	0x13579acd	GNU dbm 1.x or ndbm database, little endian, 32-bit
+!:mime	application/x-gdbm
+0	lelong	0x13579ace	GNU dbm 1.x or ndbm database, little endian, old
+!:mime	application/x-gdbm
+0	lelong	0x13579acf	GNU dbm 1.x or ndbm database, little endian, 64-bit
 !:mime	application/x-gdbm
 0	string	GDBM		GNU dbm 2.x database
 !:mime	application/x-gdbm

magic/Magdir/elf

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: elf,v 1.67 2014/06/12 13:52:48 christos Exp $
+# $File: elf,v 1.68 2014/09/19 19:05:57 christos Exp $
 # elf:  file(1) magic for ELF executables
 #
 # We have to check the byte order flag to see what byte order all the
@@ -257,6 +257,7 @@
 >18	leshort		216		Cognitive Smart Memory,
 >18	leshort		217		iCelero CoolEngine,
 >18	leshort		218		Nanoradio Optimized RISC,
+>18	leshort		243		UCB RISC-V,
 >18	leshort		0x1057		AVR (unofficial),
 >18	leshort		0x1059		MSP430 (unofficial),
 >18	leshort		0x1223		Adapteva Epiphany (unofficial),

magic/Magdir/filesystems

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# $File: filesystems,v 1.95 2014/06/03 19:17:27 christos Exp $
+# $File: filesystems,v 1.103 2014/09/11 15:09:34 christos Exp $
 # filesystems:  file(1) magic for different filesystems
 #
 0	name	partid  
@@ -254,7 +254,7 @@
 30		search/481	\x55\xAA	
 # to display DOS/MBR boot sector (40) before old one (strength=50+21),Syslinux bootloader (71),SYSLINUX MBR (37+36),NetBSD mbr (110),AdvanceMAME mbr (111)
 # DOS BPB information (70) and after DOS floppy (120) like in previous file version
-!:strength +72
+!:strength +65
 # for sector sizes < 512 Bytes
 >11		uleshort	<512		
 >>(11.s-2)	uleshort	0xAA55		DOS/MBR boot sector
@@ -265,8 +265,8 @@
 0x1FE		leshort		0xAA55		
 #
 # to display information (50) before DOS BPB (strength=70) and after DOS floppy (120) like in old file version
-!:strength +21
->2	string	OSBS			\b, OS/BS MBR
+!:strength +70
+>2	string	OSBS			OS/BS MBR
 # added by Joerg Jenderek at Feb 2013 according to http://thestarman.pcministry.com/asm/mbr/
 # and http://en.wikipedia.org/wiki/Master_Boot_Record
 # test for nearly all MS-DOS Master Boot Record initial program loader (IPL) is now done by 
@@ -1353,18 +1353,18 @@
 # minimal short forward jump found 0x29 for bootloaders or 0x0
 # maximal short forward jump is 0x7f
 # OEM-ID is empty or contain readable bytes
-0		ulelong&0x804000E9	0x000000E9	
+0		ulelong&0x804000E9	0x000000E9
 # mtools-3.9.8/msdos.h
 # usual values are marked with comments to get only informations of strange FAT systems
 # valid sectorsize must be a power of 2 from 32 to 32768
->11		uleshort&0xf001f	0	
+>11		uleshort&0x001f	0	
 >>11		uleshort	<32769		
 >>>11		uleshort	>31		
 >>>>21		ubyte&0xf0	0xF0		
 >>>>>0		ubyte		0xEB		
 >>>>>>1		ubyte		x		\b, code offset 0x%x+2
 >>>>>0		ubyte		0xE9		
->>>>>>1		uleshort	x		\b, code offset 0x%x+2
+>>>>>>1		uleshort	x		\b, code offset 0x%x+3
 >>>>>3		string		>\0		\b, OEM-ID "%-.8s"
 #http://mirror.href.com/thestarman/asm/debug/debug2.htm#IHC
 >>>>>>8		string		IHC		\b cached by Windows 9M
@@ -1373,10 +1373,11 @@
 >>>>>11		uleshort	<512		\b, Bytes/sector %u
 >>>>>13		ubyte		>1		\b, sectors/cluster %u
 #>>>>>13	ubyte		=1		\b, sectors/cluster %u (usual on Floppies)
->>>>>82		string		FAT32		
+# for lazy FAT32 implementation like Transcend digital photo frame PF830
+>>>>>82		string/c	fat32		
 >>>>>>14	uleshort	!32		\b, reserved sectors %u
 #>>>>>>14	uleshort	=32		\b, reserved sectors %u (usual Fat32)
->>>>>82		string		!FAT32		
+>>>>>82		string/c	!fat32		
 >>>>>>14	uleshort	>1		\b, reserved sectors %u
 #>>>>>>14	uleshort	=1		\b, reserved sectors %u (usual FAT12,FAT16)
 #>>>>>>14	uleshort	0		\b, reserved sectors %u (usual NTFS)
@@ -1399,24 +1400,29 @@
 >>>>>26		ubyte		=1		\b, heads %u
 # valid only for sector sizes with more then 32 Bytes
 >>>>>11		uleshort	>32		
-# skip for Digital Research DOS (version 3.41) 1440 kB Bootdisk
->>>>>>38	ubyte		!0x70		
+# http://en.wikipedia.org/wiki/Design_of_the_FAT_file_system#Extended_BIOS_Parameter_Block
+# skip for values 2,2Ah,70h,73h,DFh
+# and continue for extended boot signature values 0,28h,29h,80h
+>>>>>>38	ubyte&0x56	=0		
 >>>>>>>28	ulelong		>0		\b, hidden sectors %u
 #>>>>>>>28	ulelong		=0		\b, hidden sectors %u (usual floppy)
 >>>>>>>32	ulelong		>0		\b, sectors %u (volumes > 32 MB) 
 #>>>>>>>32	ulelong		=0		\b, sectors %u (volumes > 32 MB)
 # FAT<32 bit specific 
->>>>>>>82	string		!FAT32
+>>>>>>>82	string/c	!fat32		
 #>>>>>>>>36	ubyte		0x80		\b, physical drive 0x%x=0x80 (usual harddisk)
 #>>>>>>>>36	ubyte		0		\b, physical drive 0x%x=0 (usual floppy)
 >>>>>>>>36	ubyte		!0x80		
 >>>>>>>>>36	ubyte		!0		\b, physical drive 0x%x
+# VGA-copy CRC or
+# in Windows NT bit 0 is a dirty flag to request chkdsk at boot time. bit 1 requests surface scan too
 >>>>>>>>37	ubyte		>0		\b, reserved 0x%x
 #>>>>>>>>37	ubyte		=0		\b, reserved 0x%x
-# value is 0x80 for NTFS
+# extended boot signatur value is 0x80 for NTFS, 0x28 or 0x29 for others
 >>>>>>>>38	ubyte		!0x29		\b, dos < 4.0 BootSector (0x%x)
->>>>>>>>38	ubyte		=0x29
+>>>>>>>>38	ubyte&0xFE	=0x28
 >>>>>>>>>39	ulelong		x		\b, serial number 0x%x
+>>>>>>>>38	ubyte		=0x29
 >>>>>>>>>43	string		<NO\ NAME	\b, label: "%11.11s"
 >>>>>>>>>43	string		>NO\ NAME	\b, label: "%11.11s"
 >>>>>>>>>43	string		=NO\ NAME	\b, unlabeled
@@ -1426,11 +1432,35 @@
 # if it is small enough FAT is 12 bit, if it is too big enough FAT is 32 bit,
 # otherwise FAT is 16 bit.
 # http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/determining-fat-widths.html
->>>>>>54	string		FAT		\b, FAT
->>>>>>>54	string		FAT12		\b (12 bit)
->>>>>>>54	string		FAT16		\b (16 bit)
+>>>>>82		string/c	!fat32		
+>>>>>>54	string		FAT12		\b, FAT (12 bit)
+>>>>>>54	string		FAT16		\b, FAT (16 bit)
+>>>>>>54	default		x		
+# determinate FAT bit size by media descriptor
+# small floppies implies FAT12
+>>>>>>>21	ubyte		<0xF0		\b, FAT (12 bit by descriptor)
+# with media descriptor F0h floppy or maybe superfloppy with FAT16
+>>>>>>>21	ubyte		=0xF0		
+# superfloppy (many sectors) implies FAT16
+>>>>>>>>32	ulelong		>0xFFFF		\b, FAT (16 bit by descriptor+sectors)
+# no superfloppy with media descriptor F0h implies FAT12
+>>>>>>>>32	default		x		\b, FAT (12 bit by descriptor+sectors)
+# with media descriptor F8h floppy or hard disc with FAT12 or FAT16
+>>>>>>>21	ubyte		=0xF8		
+# 360 KiB with media descriptor F8h, 9 sectors per track ,single sided floppy implies FAT12
+>>>>>>>>19	ubequad	0xd002f80300090001	\b, FAT (12 bit by descriptor+geometry)
+# hard disc with FAT12 or FAT16
+>>>>>>>>19	default		x		\b, FAT (1Y bit by descriptor)
+# with media descriptor FAh floppy, RAM disc with FAT12 or FAT16 or Tandy hard disc
+>>>>>>>21	ubyte		=0xFA		
+# 320 KiB with media descriptor FAh, 8 sectors per track ,single sided floppy implies FAT12
+>>>>>>>>19	ubequad	0x8002fa0200080001	\b, FAT (12 bit by descriptor+geometry)
+# RAM disc with FAT12 or FAT16 or Tandy hard disc
+>>>>>>>>19	default		x		\b, FAT (1Y bit by descriptor)
+# others are floppy
+>>>>>>>21	default		x		\b, FAT (12 bit by descriptor)
 # FAT32 bit specific
->>>>>82		string		FAT32		\b, FAT (32 bit)
+>>>>>82		string/c	fat32		\b, FAT (32 bit)
 >>>>>>36	ulelong		x		\b, sectors/FAT %u
 # http://technet.microsoft.com/en-us/library/cc977221.aspx
 >>>>>>40	uleshort	>0		\b, extension flags 0x%x
@@ -1443,9 +1473,12 @@
 >>>>>>48	uleshort	>1		\b, infoSector %u
 #>>>>>>48	uleshort	=1		\b, infoSector %u (usual)
 >>>>>>48	uleshort	<1		\b, infoSector %u
->>>>>>50	uleshort	>6		\b, Backup boot sector %u
+# 0 or 0xFFFF instead of usual 6 means no backup sector
+>>>>>>50	uleshort	=0xFFFF		\b, no Backup boot sector
+>>>>>>50	uleshort	=0		\b, no Backup boot sector
 #>>>>>>50	uleshort	=6		\b, Backup boot sector %u (usual) 
->>>>>>50	uleshort	<6		\b, Backup boot sector %u
+>>>>>>50	default		x		
+>>>>>>>50	uleshort	x		\b, Backup boot sector %u
 # corrected by Joerg Jenderek at Feb 2011 according to http://thestarman.pcministry.com/asm/mbr/MSWIN41.htm#FSINFO
 >>>>>>52	ulelong		>0		\b, reserved1 0x%x
 >>>>>>56	ulelong		>0		\b, reserved2 0x%x
@@ -1881,11 +1914,19 @@
 #>>>>>0x162	use			2xDOS-filename
 
 # CDROM Filesystems
+# https://en.wikipedia.org/wiki/ISO_9660
 # Modified for UDF by gerardo.cacciari@gmail.com
-32769	string    CD001     #
-!:mime	application/x-iso9660-image
+32769	string    CD001
+# mime line at that position does not work
+# to display CD-ROM (70=81-11) after MBR (113=40+72+1), partition-table (71=50+21) and before Apple Driver Map (51)
+!:strength -11
+# to display CD-ROM (114=81+33) before MBR (113=40+72+1), partition-table (71=50+21) and Apple Driver Map (51)
+# does not work
+#!:strength +33
 >38913	string   !NSR0      ISO 9660 CD-ROM filesystem data
+!:mime	application/x-iso9660-image
 >38913	string    NSR0      UDF filesystem data
+!:mime	application/x-iso9660-image
 >>38917	string    1         (version 1.0)
 >>38917	string    2         (version 1.5)
 >>38917	string    3         (version 2.0)
@@ -2131,7 +2172,6 @@
 # which is mapped to VBN 2 of [000000]INDEXF.SYS;1 - gerardo.cacciari@gmail.com
 #
 1008    string          DECFILE11       Files-11 On-Disk Structure
->525    byte            x               Level %d
 >525    byte            x               (ODS-%d);
 >1017   string          A               RSX-11, VAX/VMS or OpenVMS VAX file system;
 >1017   string          B
@@ -2266,14 +2306,31 @@
 
 # UBIfs
 # Linux kernel sources: fs/ubifs/ubifs-media.h
-0       belong  0x31181006
->0x16   short   0               UBIfs image
->0x08   lequad  x               \b, sequence number %llu
->0x10   leshort x               \b, length %u
->0x04   lelong  x               \b, CRC 0x%08x
-
-0       belong  0x55424923
->0x04   short   <2
->0x05   string  \0\0\0
->0x1c   string  \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0
->0x04   short   x       UBI image, version %u
+0	lelong	0x06101831
+>0x16	leshort	0		UBIfs image
+>0x08	lequad	x		\b, sequence number %llu
+>0x10	leshort x		\b, length %u
+>0x04	lelong	x		\b, CRC 0x%08x
+
+0	lelong	0x23494255
+>0x04	leshort	<2
+>0x05	string	\0\0\0
+>0x1c	string	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0
+>0x04	leshort	x		UBI image, version %u
+
+# NEC PC-88 2D disk image
+# From Fabio R. Schmidlin <sd-snatcher@users.sourceforge.net>
+0x20		ulelong&0xFFFFFEFF	0x2A0
+>0x10		string			\0\0\0\0\0\0\0\0\0\0
+>>0x280		string			\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0
+>>>0x1A		ubyte&0xEF		0
+>>>>0x1B	ubyte&0x8F		0
+>>>>>0x1B	ubyte&70		<0x40
+>>>>>>0x1C	ulelong			>0x21
+>>>>>>>0	regex	[[:print:]]*	NEC PC-88 disk image, name=%s
+>>>>>>>>0x1B	ubyte	0		\b, media=2D
+>>>>>>>>0x1B	ubyte	0x10		\b, media=2DD
+>>>>>>>>0x1B	ubyte	0x20		\b, media=2HD
+>>>>>>>>0x1B	ubyte	0x30		\b, media=1D
+>>>>>>>>0x1B	ubyte	0x40		\b, media=1DD
+>>>>>>>>0x1A	ubyte	0x10		\b, write-protected

magic/Magdir/images

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: images,v 1.91 2014/04/30 21:41:02 christos Exp $
+# $File: images,v 1.95 2014/09/22 21:21:59 christos Exp $
 # images:  file(1) magic for image formats (see also "iff", and "c-lang" for
 # XPM bitmaps)
 #
@@ -115,8 +115,92 @@
 # never changed.  The TIFF specification recommends testing for it.
 0	string		MM\x00\x2a	TIFF image data, big-endian
 !:mime	image/tiff
+>(4.L)	use		tiff_ifd
 0	string		II\x2a\x00	TIFF image data, little-endian
 !:mime	image/tiff
+>(4.l)	use		tiff_ifd
+
+0	name		tiff_ifd
+>0	leshort		x		\b, direntries=%d
+>2	use		tiff_entry
+
+0	name		tiff_entry
+>0	leshort		0x100
+>>4	lelong		1
+>>>12	use		tiff_entry
+>>>8	lelong		x		\b, width=%d
+>0	leshort		0x101
+>>4	lelong		1
+>>>8	lelong		x		\b, height=%d
+>>>12	use		tiff_entry
+>0	leshort		0x102
+>>8	lelong		x		\b, bps=%d
+>>12	use		tiff_entry
+>0	leshort		0x103
+>>4	lelong		1		\b, compression=
+>>>8	lelong		1		\bnone
+>>>8	lelong		2		\bhuffman
+>>>8	lelong		3		\bbi-level group 3
+>>>8	lelong		4		\bbi-level group 4
+>>>8	lelong		5		\bLZW
+>>>8	lelong		6		\bJPEG (old)
+>>>8	lelong		7		\bJPEG
+>>>8	lelong		8		\bdeflate
+>>>8	lelong		9		\bJBIG, ITU-T T.85
+>>>8	lelong		0xa		\bJBIG, ITU-T T.43
+>>>8	lelong		0x7ffe		\bNeXT RLE 2-bit
+>>>8	lelong		0x8005		\bPackBits (Macintosh RLE)
+>>>8	lelong		0x8029		\bThunderscan RLE
+>>>8	lelong		0x807f		\bRasterPadding (CT or MP)
+>>>8	lelong		0x8080		\bRLE (Line Work)
+>>>8	lelong		0x8081		\bRLE (High-Res Cont-Tone)
+>>>8	lelong		0x8082		\bRLE (Binary Line Work)
+>>>8	lelong		0x80b2		\bDeflate (PKZIP)
+>>>8	lelong		0x80b3		\bKodak DCS
+>>>8	lelong		0x8765		\bJBIG
+>>>8	lelong		0x8798		\bJPEG2000
+>>>8	lelong		0x8799		\bNikon NEF Compressed
+>>>8	default		x	
+>>>>8	lelong		x		\b(unknown 0x%x)
+>>>12	use		tiff_entry
+>0	leshort		0x106		\b, PhotometricIntepretation=
+>>8	lelong		0		\bWhiteIsZero
+>>8	lelong		1		\bBlackIsZero
+>>8	lelong		2		\bRGB
+>>8	lelong		3		\bRGB Palette
+>>8	lelong		4		\bTransparency Mask
+>>8	lelong		5		\bCMYK
+>>8	lelong		6		\bYCbCr
+>>8	lelong		8		\bCIELab
+>>>8	lelong		x		\b(unknown=0x%x)
+>>12	use		tiff_entry
+# FillOrder
+>0	leshort		0x10a
+>>4	lelong		1
+>>>12	use		tiff_entry
+# DocumentName
+>0	leshort		0x10d
+>>(8.l)	string		x		\b, name=%s
+>>>12	use		tiff_entry
+# ImageDescription
+>0	leshort		0x10e
+>>(8.l)	string		x		\b, description=%s
+>>>12	use		tiff_entry
+# StripOffsets
+>0	leshort		0x111
+>>12	use		tiff_entry
+# NewSubFileType
+>0	leshort		0xfe
+>>12	use		tiff_entry
+# Datetime
+>0	leshort		0x132
+>>(8.l)	string		x		\b, datetime=%s
+>>>12	use		tiff_entry
+# HostComputer
+>0	leshort		0x13c
+>>(8.l)	string		x		\b, hostcomputer=%s
+>>>12	use		tiff_entry
+#>0	leshort		x		\b, unknown=0x%x
 
 0	string		MM\x00\x2b	Big TIFF image data, big-endian
 !:mime	image/tiff
@@ -892,6 +976,56 @@
 0	string	\x46\x4d\x52\x00	ISO/IEC 19794-2 Format Minutiae Record (FMR)
 
 # WEBP https://developers.google.com/speed/webp/docs/riff_container
-0	string	RIFF
->8	string	WEBP	Web/P image data
->>4	lelong	x	\b, %d bytes
+#0	string	RIFF
+#>8	string	WEBP	Web/P image data
+#>>4	lelong	x	\b, %d bytes
+
+# doc: http://www.shikino.co.jp/eng/products/images/FLOWER.jpg.zip
+# example: http://www.shikino.co.jp/eng/products/images/FLOWER.wdp.zip
+90	bequad		0x574D50484F544F00	JPEG-XR Image
+>98	byte&0x08	=0x08			\b, hard tiling
+>99	byte&0x80	=0x80			\b, tiling present
+>99	byte&0x40	=0x40			\b, codestream present
+>99	byte&0x38	x			\b, spatial xform=
+>99	byte&0x38	0x00			\bTL
+>99	byte&0x38	0x08			\bBL
+>99	byte&0x38	0x10			\bTR
+>99	byte&0x38	0x18			\bBR
+>99	byte&0x38	0x20			\bBT
+>99	byte&0x38	0x28			\bRB
+>99	byte&0x38	0x30			\bLT
+>99	byte&0x38	0x38			\bLB
+>100	byte&0x80	=0x80			\b, short header
+>>102	beshort+1	x			\b, %d
+>>104	beshort+1	x			\bx%d
+>100	byte&0x80	=0x00			\b, long header
+>>102	belong+1	x			\b, %x
+>>106	belong+1	x			\bx%x
+>101	beshort&0xf	x			\b, bitdepth=
+>>101	beshort&0xf	0x0			\b1-WHITE=1
+>>101	beshort&0xf	0x1			\b8
+>>101	beshort&0xf	0x2			\b16
+>>101	beshort&0xf	0x3			\b16-SIGNED
+>>101	beshort&0xf	0x4			\b16-FLOAT
+>>101	beshort&0xf	0x5			\b(reserved 5)
+>>101	beshort&0xf	0x6			\b32-SIGNED
+>>101	beshort&0xf	0x7			\b32-FLOAT
+>>101	beshort&0xf	0x8			\b5
+>>101	beshort&0xf	0x9			\b10
+>>101	beshort&0xf	0xa			\b5-6-5
+>>101	beshort&0xf	0xb			\b(reserved %d)
+>>101	beshort&0xf	0xc			\b(reserved %d)
+>>101	beshort&0xf	0xd			\b(reserved %d)
+>>101	beshort&0xf	0xe			\b(reserved %d)
+>>101	beshort&0xf	0xf			\b1-BLACK=1
+>101	beshort&0xf0	x			\b, colorfmt=
+>>101	beshort&0xf0	0x00			\bYONLY
+>>101	beshort&0xf0	0x10			\bYUV240
+>>101	beshort&0xf0	0x20			\bYWV422
+>>101	beshort&0xf0	0x30			\bYWV444
+>>101	beshort&0xf0	0x40			\bCMYK
+>>101	beshort&0xf0	0x50			\bCMYKDIRECT
+>>101	beshort&0xf0	0x60			\bNCOMPONENT
+>>101	beshort&0xf0	0x70			\bRGB
+>>101	beshort&0xf0	0x80			\bRGBE
+>>101	beshort&0xf0	>0x80			\b(reserved 0x%x)

magic/Magdir/jpeg

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: jpeg,v 1.19 2013/02/04 15:50:03 christos Exp $
+# $File: jpeg,v 1.21 2014/09/12 20:47:00 christos Exp $
 # JPEG images
 # SunOS 5.5.1 had
 #
@@ -22,10 +22,12 @@
 >>11	byte		x		\b %d.
 >>12	byte		x		\b%02d
 # Next, the resolution or aspect ratio of the image:
-#>>13	byte		0		\b, aspect ratio
-#>>13	byte		1		\b, resolution (DPI)
-#>>13	byte		2		\b, resolution (DPCM)
-#>>4	beshort		x		\b, segment length %d
+>>13	byte		0		\b, aspect ratio
+>>13	byte		1		\b, resolution (DPI)
+>>13	byte		2		\b, resolution (DPCM)
+>>14	beshort		x		\b, density %dx
+>>16	beshort		x		\b%d
+>>4	beshort		x		\b, segment length %d
 # Next, show thumbnail info, if it exists:
 >>18	byte		!0		\b, thumbnail %dx
 >>>19	byte		x		\b%d
@@ -121,33 +123,52 @@
 >>>>>(150.L+59)	byte	x		%c
 >>>>>(150.L+60)	byte	x		\b.%c
 >>>>>(150.L+61)	byte	!0x30		\b%c
-# Here things get sticky.  We can do ONE MORE marker segment with
-# indirect addressing, and that's all.  It would be great if we could
-# do pointer arithemetic like in an assembler language.  Christos?
-# And if there was some sort of looping construct to do searches, plus a few
-# named accumulators, it would be even more effective...
-# At least we can show a comment if no other segments got inserted before:
->(4.S+5)	byte		0xFE		\b, comment:
->>(4.S+6)	pstring/HJ	x		"%s"
-# Or, we can show the encoding type (I've included only the three most common)
-# and image dimensions if we are lucky and the SOFn (image segment) is here:
->(4.S+5)	byte		0xC0		\b, baseline
->>(4.S+6)	byte		x		\b, precision %d
->>(4.S+7)	beshort		x		\b, %dx
->>(4.S+9)	beshort		x		\b%d
->(4.S+5)	byte		0xC1		\b, extended sequential
->>(4.S+6)	byte		x		\b, precision %d
->>(4.S+7)	beshort		x		\b, %dx
->>(4.S+9)	beshort		x		\b%d
->(4.S+5)	byte		0xC2		\b, progressive
->>(4.S+6)	byte		x		\b, precision %d
->>(4.S+7)	beshort		x		\b, %dx
->>(4.S+9)	beshort		x		\b%d
-# I've commented-out quantisation table reporting.  I doubt anyone cares yet.
-#>(4.S+5)	byte		0xDB		\b, quantisation table
-#>>(4.S+6)	beshort		x		\b length=%d
-#>14	beshort		x		\b, %d x
-#>16	beshort		x		\b %d
+
+# Jump to the first segment
+>(4.S+4)	use		jpeg_segment
+
+# This uses recursion...
+0		name		jpeg_segment
+>0	beshort		0xFFFE
+>>(2.S+2)	use			jpeg_segment
+>>2	pstring/HJ	x		\b, comment: "%s"
+
+>0	beshort		0xFFC0
+>>(2.S+2)	use			jpeg_segment
+>>4	byte		x		\b, baseline, precision %d
+>>7	beshort		x		\b, %dx
+>>5	beshort		x		\b%d
+>>9	byte		x		\b, frames %d
+
+>0	beshort		0xFFC1		
+>>(2.S+2)	use			jpeg_segment
+>>4	byte		x		\b, extended sequential, precision %d
+>>7	beshort		x		\b, %dx
+>>5	beshort		x		\b%d
+>>9	byte		x		\b, frames %d
+
+>0	beshort		0xFFC2		
+>>(2.S+2)	use			jpeg_segment
+>>4	byte		x		\b, progressive, precision %d
+>>7	beshort		x		\b, %dx
+>>5	beshort		x		\b%d
+>>9	byte		x		\b, frames %d
+
+# Define Huffman Tables
+>0	beshort		0xFFC4
+>>(2.S+2)	use			jpeg_segment
+
+# Application specific markers
+>0	beshort&0xFFE0	=0xFFE0
+>>(2.S+2)	use			jpeg_segment
+
+# DB: Define Quantization tables
+# DD: Define Restart interval [XXX: wrong here, it is 4 bytes]
+# D8: Start of image
+# D9: End of image
+# Dn: Restart
+>0	beshort&0xFFD0	=0xFFD0
+>>(2.S+2)	use			jpeg_segment
 
 # HSI is Handmade Software's proprietary JPEG encoding scheme
 0	string		hsi1		JPEG image data, HSI proprietary

magic/Magdir/linux

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: linux,v 1.57 2014/05/20 20:10:17 christos Exp $
+# $File: linux,v 1.58 2014/08/04 06:21:30 christos Exp $
 # linux:  file(1) magic for Linux files
 #
 # Values for Linux/i386 binaries, from Daniel Quinlan <quinlan@yggdrasil.com>
@@ -413,3 +413,6 @@
 >>>>20		belong		>16
 >>>>>36		belong		x	\b, DT structure block size=%d
 
+# glibc locale archive as defined in glibc locale/locarchive.h
+0		lelong		0xde020109	locale archive
+>24		lelong		x		%d strings

magic/Magdir/macintosh

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: macintosh,v 1.23 2013/11/19 18:47:58 christos Exp $
+# $File: macintosh,v 1.25 2014/09/03 13:34:16 christos Exp $
 # macintosh description
 #
 # BinHex is the Macintosh ASCII-encoded file format (see also "apple")
@@ -165,7 +165,7 @@
 #>65	string		ZSYS		(Pre-System 7 system file)
 #>65	string		acf3		(Aldus FreeHand)
 #>65	string		cdev		(control panel)
-#>65	string		dfil		(Desk Acessory suitcase)
+#>65	string		dfil		(Desk Accessory suitcase)
 #>65	string		libr		(library)
 #>65	string		nX^d		(WriteNow word processor)
 #>65	string		nX^w		(WriteNow dictionary)
@@ -288,20 +288,38 @@
 >0x412	beshort			x		number of blocks: %d,
 >0x424	pstring			x		volume name: %s
 
+# *.hfs updated by Joerg Jenderek
+# http://en.wikipedia.org/wiki/Hierarchical_File_System
 # "BD" gives many false positives
-#0x400	beshort			0x4244		Macintosh HFS data
-#>0	beshort			0x4C4B		(bootable)
-#>0x40a	beshort			&0x8000		(locked)
-#>0x40a	beshort			^0x0100		(mounted)
-#>0x40a	beshort			&0x0200		(spared blocks)
-#>0x40a	beshort			&0x0800		(unclean)
-#>0x47C	beshort			0x482B		(Embedded HFS+ Volume)
-#>0x402	beldate-0x7C25B080	x		created: %s,
-#>0x406	beldate-0x7C25B080	x		last modified: %s,
-#>0x440	beldate-0x7C25B080	>0		last backup: %s,
-#>0x414	belong			x		block size: %d,
-#>0x412	beshort			x		number of blocks: %d,
-#>0x424	pstring			x		volume name: %s
+0x400	beshort			0x4244		
+# ftp://ftp.mars.org/pub/hfs/hfsutils-3.2.6.tar.gz/hfsutils-3.2.6/libhfs/apple.h
+# first block of volume bit map (always 3)
+>0x40e	ubeshort		0x0003		
+# maximal length of volume name is 27
+>>0x424		ubyte			<28	Macintosh HFS data
+#!:mime	application/octet-stream
+# these mime and apple types are not sure
+!:mime	application/x-apple-diskimage
+#!:apple	hfsdINIT
+#!:apple	MACSdisk
+>>>0		beshort			0x4C4B	(bootable)
+#>>>0		beshort			0x0000	(not bootable)
+>>>0x40a	beshort			&0x8000	(locked)
+>>>0x40a	beshort			^0x0100	(mounted)
+>>>0x40a	beshort			&0x0200	(spared blocks)
+>>>0x40a	beshort			&0x0800	(unclean)
+>>>0x47C	beshort			0x482B	(Embedded HFS+ Volume)
+# http://www.epochconverter.com/
+# 0x7C245F00 seconds	~ 2082758400	~ 01 Jan 2036 00:00:00	~ 66 years to 1970
+# 0x7C25B080 seconds	~ 2082844800	~ 02 Jan 2036 00:00:00
+# construct not working
+#>>>0x402	beldate-0x7C25B080	x	created: %s,
+#>>>0x406	beldate-0x7C25B080	x	last modified: %s,
+#>>>0x440	beldate-0x7C25B080	>0	last backup: %s,
+# found block sizes 200h,1200h,2800h
+>>>0x414	belong			x	block size: %d,
+>>>0x412	beshort			x	number of blocks: %d,
+>>>0x424	pstring			x	volume name: %s
 
 0x400	beshort			0x482B		Macintosh HFS Extended
 >&0	beshort			x		version %d data
@@ -322,43 +340,9 @@
 >&42	belong			x		number of blocks: %d,
 >&46	belong			x		free blocks: %d
 
-# I don't think this is really necessary since it doesn't do much and 
-# anything with a valid driver descriptor will also have a valid
-# partition map
-#0		beshort		0x4552		Apple Device Driver data
-#>&24		beshort		=1		\b, MacOS
-
-# Is that the partition type a cstring or a pstring? Well, IM says "strings 
-# shorter than 32 bytes must be terminated with NULL" so I'll treat it as a 
-# cstring. Of course, partitions can contain more than four entries, but 
-# what're you gonna do?
-# GRR: This magic is too weak, it is just "PM"
-#0x200		beshort		0x504D		Apple Partition data
-#>0x2		beshort		x		(block size: %d):
-#>0x230		string		x		first type: %s,
-#>0x210		string		x		name: %s,
-#>0x254		belong		x		number of blocks: %d,
-#>0x400		beshort		0x504D		
-#>>0x430		string		x		second type: %s,
-#>>0x410		string		x		name: %s,
-#>>0x454		belong		x		number of blocks: %d,
-#>>0x600		beshort		0x504D
-#>>>0x630	string		x		third type: %s,
-#>>>0x610	string		x		name: %s,
-#>>>0x654	belong		x		number of blocks: %d,
-#>>0x800		beshort		0x504D		
-#>>>0x830	string		x		fourth type: %s,
-#>>>0x810	string		x		name: %s,
-#>>>0x854	belong		x		number of blocks: %d,
-#>>>0xa00	beshort		0x504D		
-#>>>>0xa30	string		x		fifth type: %s,
-#>>>>0xa10	string		x		name: %s,
-#>>>>0xa54	belong		x		number of blocks: %d
-#>>>0xc00	beshort		0x504D
-#>>>>0xc30	string		x		sixth type: %s,
-#>>>>0xc10	string		x		name: %s,
-#>>>>0xc54	belong		x		number of blocks: %d
 ## AFAIK, only the signature is different
+# same as Apple Partition Map
+# GRR: This magic is too weak, it is just "TS"
 #0x200		beshort		0x5453		Apple Old Partition data
 #>0x2		beshort		x		block size: %d,
 #>0x230		string		x		first type: %s,

magic/Magdir/rinex

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: rinex,v 1.4 2011/05/03 01:44:17 christos Exp $
+# $File: meteorological,v 1.1 2014/08/04 06:26:16 christos Exp $
 # rinex:  file(1) magic for RINEX files
 # http://igscb.jpl.nasa.gov/igscb/data/format/rinex210.txt
 # ftp://cddis.gsfc.nasa.gov/pub/reports/formats/rinex300.pdf
@@ -42,3 +42,8 @@
 >>&32	string		x		\b, date %15.15s
 >>5	string		x		\b, version %6.6s
 !:mime	rinex/observation
+
+# https://en.wikipedia.org/wiki/GRIB
+0	string	GRIB
+>7	byte	=1	Gridded binary (GRIB) version 1
+>7	byte	=2	Gridded binary (GRIB) version 2

magic/Magdir/msooxml

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: msooxml,v 1.4 2014/01/06 18:16:24 rrt Exp $
+# $File: msooxml,v 1.5 2014/08/05 07:38:45 christos Exp $
 # msooxml:  file(1) magic for Microsoft Office XML
 # From: Ralf Brown <ralf.brown@gmail.com>
 
@@ -16,7 +16,7 @@
 0		string		PK\003\004
 !:strength +10
 # make sure the first file is correct
->0x1E		regex		\[Content_Types\]\.xml|_rels/\.rels
+>0x1E		regex		\\[Content_Types\\]\\.xml|_rels/\\.rels
 # skip to the second local file header
 # since some documents include a 520-byte extra field following the file
 # header, we need to scan for the next header

magic/Magdir/pascal

@@ -1,10 +1,10 @@
 #------------------------------------------------------------------------------
-# $File: pascal,v 1.1 2011/12/08 12:12:46 rrt Exp $
+# $File: pascal,v 1.2 2014/07/14 14:21:33 rrt Exp $
 # pascal:  file(1) magic for Pascal source
 #
 0	search/8192	(input,		Pascal source text
 !:mime	text/x-pascal
-0	regex		\^program	Pascal source text
-!:mime	text/x-pascal
-0	regex           	\^record		Pascal source text
-!:mime	text/x-pascal
+#0	regex		\^program	Pascal source text
+#!:mime	text/x-pascal
+#0	regex           	\^record		Pascal source text
+#!:mime	text/x-pascal

magic/Magdir/python

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: python,v 1.25 2014/05/06 16:08:32 christos Exp $
+# $File: python,v 1.26 2014/08/04 05:58:40 christos Exp $
 # python:  file(1) magic for python
 #
 # Outlook puts """ too for urgent messages
@@ -23,6 +23,7 @@
 0	belong		0x4f0c0d0a	python 3.1 byte-compiled
 0	belong		0x6c0c0d0a	python 3.2 byte-compiled
 0	belong		0x9e0c0d0a	python 3.3 byte-compiled
+0	belong		0xee0c0d0a	python 3.4 byte-compiled
 
 0	search/1/w	#!\ /usr/bin/python	Python script text executable
 !:mime text/x-python

magic/Magdir/riff

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: riff,v 1.27 2014/04/30 21:41:02 christos Exp $
+# $File: riff,v 1.30 2014/09/23 17:02:12 christos Exp $
 # riff:  file(1) magic for RIFF format
 # See
 #
@@ -45,6 +45,26 @@
 >>&(4.l+4)  use riff-walk
 >0  string  fact
 >>&(4.l+4)  use riff-walk
+>0  string  VP8
+>>11		byte		0x9d
+>>>12		byte		0x01
+>>>>13		byte		0x2a	\b, VP8 encoding
+>>>>>14		leshort&0x3fff	x	\b, %d
+>>>>>16		leshort&0x3fff	x	\bx%d, Scaling:
+>>>>>14		leshort&0xc000	0x0000	\b [none]
+>>>>>14		leshort&0xc000	0x1000	\b [5/4]
+>>>>>14		leshort&0xc000	0x2000	\b [5/3]
+>>>>>14		leshort&0xc000	0x3000	\b [2]
+>>>>>14		leshort&0xc000	0x0000	\bx[none]
+>>>>>14		leshort&0xc000	0x1000	\bx[5/4]
+>>>>>14		leshort&0xc000	0x2000	\bx[5/3]
+>>>>>14		leshort&0xc000	0x3000	\bx[2]
+>>>>>15		byte&0x80	=0x00	\b, YUV color
+>>>>>15		byte&0x80	=0x80	\b, bad color specification
+>>>>>15		byte&0x40	=0x40	\b, no clamping required
+>>>>>15		byte&0x40	=0x00	\b, decoders should clamp
+#>0  string  x		we got %s
+#>>&(4.l+4)  use riff-walk
 
 # AVI section extended by Patrik Radman <patrik+file-magic@iki.fi>
 #
@@ -209,6 +229,8 @@
 >8	string		4XMV		\b, 4X Movie file 
 # AMV-type AVI file: http://wiki.multimedia.cx/index.php?title=AMV
 >8	string		AMV\040		\b, AMV 
+>8      string          WEBP            \b, Web/P image
+>>12	use		riff-walk
 
 #
 # XXX - some of the below may only appear in little-endian form.
@@ -284,4 +306,3 @@
 >>&6	leshort		=2		\b, stereo
 >>&6	leshort		>2		\b, %d channels
 >>&8	lelong		>0		%d Hz
-

magic/Magdir/sequent

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: sequent,v 1.11 2014/06/02 19:27:54 christos Exp $
+# $File: sequent,v 1.12 2014/08/16 16:07:12 christos Exp $
 # sequent:  file(1) magic for Sequent machines
 #
 # Sequent information updated by Don Dwiggins <atsun!dwiggins>.
@@ -30,9 +30,6 @@
 0	leshort	0x32eb		SYMMETRY i386 executable (invalid @ 0)
 >16	lelong	>0		not stripped
 >124	lelong	>0		version %d
-0	leshort	0x42eb		SYMMETRY i386 standalone executable
->16	lelong	>0		not stripped
->124	lelong	>0		version %d
 # http://en.wikipedia.org/wiki/Sequent_Computer_Systems
 # below test line conflicts with MS-DOS 2.11 floppies and Acronis loader
 #0	leshort	0x42eb		SYMMETRY i386 standalone executable

magic/Magdir/vms

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: vms,v 1.7 2013/03/09 22:36:00 christos Exp $
+# $File: vms,v 1.9 2014/08/17 13:47:59 christos Exp $
 # vms:  file(1) magic for VMS executables (experimental)
 #
 # VMS .exe formats, both VAX and AXP (Greg Roelofs, newt@uchicago.edu)

magic/Magdir/vorbis

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: vorbis,v 1.18 2014/04/30 21:41:02 christos Exp $
+# $File: vorbis,v 1.20 2014/09/23 16:35:08 christos Exp $
 # vorbis:  file(1) magic for Ogg/Vorbis files
 #
 # From Felix von Leitner <leitner@fefe.de>
@@ -23,7 +23,6 @@
 # --- Ogg Framing ---
 #0		search/1000	OggS		Ogg data
 0		string	OggS		Ogg data
-!:mime		application/ogg
 >4		byte		!0		UNKNOWN REVISION %u
 ##>4		byte		0		revision 0
 >4		byte		0
@@ -31,9 +30,12 @@
 # non-Vorbis content: FLAC (Free Lossless Audio Codec, http://flac.sourceforge.net)
 >>28		string		\x7fFLAC	\b, FLAC audio
 # non-Vorbis content: Theora
+!:mime		audio/ogg
 >>28		string		\x80theora	\b, Theora video
+!:mime		video/ogg
 # non-Vorbis content: Kate
->>28		string		\x80kate\0\0\0\0	\b, Kate
+>>28		string		\x80kate\0\0\0\0	\b, Kate (Karaoke and Text)
+!:mime		application/ogg
 >>>37		ubyte		x		v%u
 >>>38		ubyte		x		\b.%u,
 >>>40		byte		0		utf8 encoding,
@@ -44,18 +46,22 @@
 >>>76		string		\0		no category set
 # non-Vorbis content: Skeleton
 >>28		string		fishead\0	\b, Skeleton
+!:mime		video/ogg
 >>>36		short		x		v%u
 >>>40		short		x		\b.%u
 # non-Vorbis content: Speex
 >>28		string		Speex\ \ \ 	\b, Speex audio
+!:mime		audio/ogg
 # non-Vorbis content: OGM
 >>28		string		\x01video\0\0\0	\b, OGM video
+!:mime		video/ogg
 >>>37		string/c	div3		(DivX 3)
 >>>37		string/c	divx		(DivX 4)
 >>>37		string/c	dx50		(DivX 5)
 >>>37		string/c	xvid		(XviD)
 # --- First vorbis packet - general header ---
 >>28		string		\x01vorbis	\b, Vorbis audio,
+!:mime		audio/ogg
 >>>35		lelong		!0		UNKNOWN VERSION %u,
 ##>>>35		lelong		0		version 0,
 >>>35		lelong		0

magic/Magdir/windows

@@ -1,6 +1,6 @@
 
 #------------------------------------------------------------------------------
-# $File: windows,v 1.8 2014/04/30 21:41:02 christos Exp $
+# $File: windows,v 1.10 2014/09/24 19:52:46 christos Exp $
 # windows:  file(1) magic for Microsoft Windows
 #
 # This file is mainly reserved for files where programs
@@ -159,8 +159,9 @@
 >&0	string		Version\ 5.00\r\n\r\n	Windows Registry text (Win2K or above)
 
 # Windows *.INF *.INI files updated by Joerg Jenderek at Apr 2013
-# emtpy ,comment , section , unicode line
-0	regex/s		\\`(\r\n|;|[[]|\xFF\xFE)			
+# empty ,comment , section
+# PR/383: remove unicode BOM because it is not portable across regex impls
+0	regex/s		\\`(\\r\\n|;|[[])
 # left bracket in section line
 >&0	search/8192	[						
 # http://en.wikipedia.org/wiki/Autorun.inf

magic/Makefile.am

@@ -1,5 +1,5 @@
 #
-# $File: Makefile.am,v 1.98 2014/06/03 18:22:34 christos Exp $
+# $File: Makefile.am,v 1.99 2014/08/04 06:26:16 christos Exp $
 #
 MAGIC_FRAGMENT_BASE = Magdir
 MAGIC_DIR = $(top_srcdir)/magic
@@ -142,6 +142,7 @@ $(MAGIC_FRAGMENT_DIR)/matroska \
 $(MAGIC_FRAGMENT_DIR)/mcrypt \
 $(MAGIC_FRAGMENT_DIR)/mercurial \
 $(MAGIC_FRAGMENT_DIR)/metastore \
+$(MAGIC_FRAGMENT_DIR)/meteorological \
 $(MAGIC_FRAGMENT_DIR)/mime \
 $(MAGIC_FRAGMENT_DIR)/mips \
 $(MAGIC_FRAGMENT_DIR)/mirage \
@@ -198,7 +199,6 @@ $(MAGIC_FRAGMENT_DIR)/pyramid \
 $(MAGIC_FRAGMENT_DIR)/python \
 $(MAGIC_FRAGMENT_DIR)/revision \
 $(MAGIC_FRAGMENT_DIR)/riff \
-$(MAGIC_FRAGMENT_DIR)/rinex \
 $(MAGIC_FRAGMENT_DIR)/rpm \
 $(MAGIC_FRAGMENT_DIR)/rtf \
 $(MAGIC_FRAGMENT_DIR)/ruby \

magic/Makefile.in

@@ -262,7 +262,7 @@ top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 
 #
-# $File: Makefile.am,v 1.98 2014/06/03 18:22:34 christos Exp $
+# $File: Makefile.am,v 1.99 2014/08/04 06:26:16 christos Exp $
 #
 MAGIC_FRAGMENT_BASE = Magdir
 MAGIC_DIR = $(top_srcdir)/magic
@@ -403,6 +403,7 @@ $(MAGIC_FRAGMENT_DIR)/matroska \
 $(MAGIC_FRAGMENT_DIR)/mcrypt \
 $(MAGIC_FRAGMENT_DIR)/mercurial \
 $(MAGIC_FRAGMENT_DIR)/metastore \
+$(MAGIC_FRAGMENT_DIR)/meteorological \
 $(MAGIC_FRAGMENT_DIR)/mime \
 $(MAGIC_FRAGMENT_DIR)/mips \
 $(MAGIC_FRAGMENT_DIR)/mirage \
@@ -459,7 +460,6 @@ $(MAGIC_FRAGMENT_DIR)/pyramid \
 $(MAGIC_FRAGMENT_DIR)/python \
 $(MAGIC_FRAGMENT_DIR)/revision \
 $(MAGIC_FRAGMENT_DIR)/riff \
-$(MAGIC_FRAGMENT_DIR)/rinex \
 $(MAGIC_FRAGMENT_DIR)/rpm \
 $(MAGIC_FRAGMENT_DIR)/rtf \
 $(MAGIC_FRAGMENT_DIR)/ruby \

src/Makefile.in

@@ -81,9 +81,9 @@ build_triplet = @build@
 host_triplet = @host@
 bin_PROGRAMS = file$(EXEEXT)
 subdir = src
-DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am pread.c \
-	ctime_r.c getline.c vasprintf.c asprintf.c asctime_r.c \
-	fmtcheck.c strlcpy.c getopt_long.c strcasestr.c strlcat.c \
+DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am ctime_r.c \
+	getline.c strcasestr.c strlcat.c getopt_long.c asctime_r.c \
+	fmtcheck.c vasprintf.c pread.c strlcpy.c asprintf.c \
 	$(top_srcdir)/depcomp $(include_HEADERS)
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \

src/apprentice.c

@@ -32,7 +32,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: apprentice.c,v 1.211 2014/06/03 19:01:34 christos Exp $")
+FILE_RCSID("@(#)$File: apprentice.c,v 1.216 2014/09/24 19:49:07 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -86,6 +86,10 @@ FILE_RCSID("@(#)$File: apprentice.c,v 1.211 2014/06/03 19:01:34 christos Exp $")
 #define ALLOC_CHUNK	(size_t)10
 #define ALLOC_INCR	(size_t)200
 
+#define MAP_TYPE_MMAP	0
+#define MAP_TYPE_MALLOC	1
+#define MAP_TYPE_USER	2
+
 struct magic_entry {
 	struct magic *mp;	
 	uint32_t cont_count;
@@ -101,6 +105,7 @@ struct magic_entry_set {
 struct magic_map {
 	void *p;
 	size_t len;
+	int type;
 	struct magic *magic[MAGIC_SETS];
 	uint32_t nmagic[MAGIC_SETS];
 };
@@ -131,7 +136,10 @@ private uint16_t swap2(uint16_t);
 private uint32_t swap4(uint32_t);
 private uint64_t swap8(uint64_t);
 private char *mkdbname(struct magic_set *, const char *, int);
+private struct magic_map *apprentice_buf(struct magic_set *, struct magic *,
+    size_t);
 private struct magic_map *apprentice_map(struct magic_set *, const char *);
+private int check_buffer(struct magic_set *, struct magic_map *, const char *);
 private void apprentice_unmap(struct magic_map *);
 private int apprentice_compile(struct magic_set *, struct magic_map *,
     const char *);
@@ -416,9 +424,11 @@ add_mlist(struct mlist *mlp, struct magic_map *map, size_t idx)
 private int
 apprentice_1(struct magic_set *ms, const char *fn, int action)
 {
-	struct mlist *ml;
 	struct magic_map *map;
+#ifndef COMPILE_ONLY
+	struct mlist *ml;
 	size_t i;
+#endif
 
 	if (magicsize != FILE_MAGICSIZE) {
 		file_error(ms, 0, "magic element size %lu != %lu",
@@ -454,15 +464,15 @@ apprentice_1(struct magic_set *ms, const char *fn, int action)
 
 	if (action == FILE_LIST) {
 		for (i = 0; i < MAGIC_SETS; i++) {
-			printf("Set %zu:\nBinary patterns:\n", i);
+			printf("Set %" SIZE_T_FORMAT "u:\nBinary patterns:\n",
+			    i);
 			apprentice_list(ms->mlist[i], BINTEST);
 			printf("Text patterns:\n");
 			apprentice_list(ms->mlist[i], TEXTTEST);
 		}
 	}
-	
-	return 0;
 #endif /* COMPILE_ONLY */
+	return 0;
 }
 
 protected void
@@ -517,9 +527,9 @@ apprentice_unmap(struct magic_map *map)
 {
 	if (map == NULL)
 		return;
-	if (map->p != NULL) {
+	if (map->p != NULL && map->type != MAP_TYPE_USER) {
 #ifdef QUICK
-		if (map->len)
+		if (map->type == MAP_TYPE_MMAP)
 			(void)munmap(map->p, map->len);
 		else
 #endif
@@ -561,6 +571,56 @@ mlist_free(struct mlist *mlist)
 	free(ml);
 }
 
+#ifndef COMPILE_ONLY
+/* void **bufs: an array of compiled magic files */
+protected int
+buffer_apprentice(struct magic_set *ms, struct magic **bufs,
+    size_t *sizes, size_t nbufs)
+{
+	size_t i;
+	struct mlist *ml;
+	struct magic_map *map;
+
+	if (nbufs == 0)
+		return -1;
+
+	if (ms->mlist[0] != NULL)
+		file_reset(ms);
+
+	init_file_tables();
+
+	for (i = 0; i < MAGIC_SETS; i++) {
+		mlist_free(ms->mlist[i]);
+		if ((ms->mlist[i] = mlist_alloc()) == NULL) {
+			file_oomem(ms, sizeof(*ms->mlist[i]));
+			if (i != 0) {
+				--i;
+				do
+					mlist_free(ms->mlist[i]);
+				while (i != 0);
+			}
+			return -1;
+		}
+	}
+
+	for (i = 0; i < nbufs; i++) {
+		map = apprentice_buf(ms, bufs[i], sizes[i]);
+		if (map == NULL)
+			return -1;
+
+		for (i = 0; i < MAGIC_SETS; i++) {
+			if (add_mlist(ms->mlist[i], map, i) == -1) {
+				file_oomem(ms, sizeof(*ml));
+				apprentice_unmap(map);
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+#endif
+
 /* const char *fn: list of magic files and directories */
 protected int
 file_apprentice(struct magic_set *ms, const char *fn, int action)
@@ -2065,8 +2125,14 @@ out:
 }
 
 private int
+goodchar(unsigned char x, const char *extra)
+{
+	return (isascii(x) && isalnum(x)) || strchr(extra, x);
+}
+
+private int
 parse_extra(struct magic_set *ms, struct magic_entry *me, const char *line,
-    off_t off, size_t len, const char *name, int nt)
+    off_t off, size_t len, const char *name, const char *extra, int nt)
 {
 	size_t i;
 	const char *l = line;
@@ -2087,9 +2153,7 @@ parse_extra(struct magic_set *ms, struct magic_entry *me, const char *line,
 	}
 
 	EATAB;
-	for (i = 0; *l && ((isascii((unsigned char)*l) &&
-	    isalnum((unsigned char)*l)) || strchr("-+/.", *l)) &&
-	    i < len; buf[i++] = *l++)
+	for (i = 0; *l && i < len && goodchar(*l, extra); buf[i++] = *l++)
 		continue;
 
 	if (i == len && *l) {
@@ -2099,14 +2163,18 @@ parse_extra(struct magic_set *ms, struct magic_entry *me, const char *line,
 			file_magwarn(ms, "%s type `%s' truncated %"
 			    SIZE_T_FORMAT "u", name, line, i);
 	} else {
+		if (!isspace((unsigned char)*l) && !goodchar(*l, extra))
+			file_magwarn(ms, "%s type `%s' has bad char '%c'",
+			    name, line, *l);
 		if (nt)
 			buf[i] = '\0';
 	}
 
 	if (i > 0)
 		return 0;
-	else
-		return -1;
+
+	file_magerror(ms, "Bad magic entry '%s'", line);
+	return -1;
 }
 
 /*
@@ -2119,7 +2187,7 @@ parse_apple(struct magic_set *ms, struct magic_entry *me, const char *line)
 	struct magic *m = &me->mp[0];
 
 	return parse_extra(ms, me, line, offsetof(struct magic, apple),
-	    sizeof(m->apple), "APPLE", 0);
+	    sizeof(m->apple), "APPLE", "!+-./", 0);
 }
 
 /*
@@ -2132,7 +2200,7 @@ parse_mime(struct magic_set *ms, struct magic_entry *me, const char *line)
 	struct magic *m = &me->mp[0];
 
 	return parse_extra(ms, me, line, offsetof(struct magic, mimetype),
-	    sizeof(m->mimetype), "MIME", 1);
+	    sizeof(m->mimetype), "MIME", "+-/.", 1);
 }
 
 private int
@@ -2693,6 +2761,28 @@ eatsize(const char **p)
 }
 
 /*
+ * handle a buffer containging a compiled file.
+ */
+private struct magic_map *
+apprentice_buf(struct magic_set *ms, struct magic *buf, size_t len)
+{
+	struct magic_map *map;
+
+	if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL) {
+		file_oomem(ms, sizeof(*map));
+		return NULL;
+	}
+	map->len = len;
+	map->p = buf;
+	map->type = MAP_TYPE_USER;
+	if (check_buffer(ms, map, "buffer") != 0) {
+		apprentice_unmap(map);
+		return NULL;
+	}
+	return map;
+}
+
+/*
  * handle a compiled file.
  */
 
@@ -2701,12 +2791,8 @@ apprentice_map(struct magic_set *ms, const char *fn)
 {
 	int fd;
 	struct stat st;
-	uint32_t *ptr;
-	uint32_t version, entries, nentries;
-	int needsbyteswap;
 	char *dbname = NULL;
 	struct magic_map *map;
-	size_t i;
 
 	fd = -1;
 	if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL) {
@@ -2738,6 +2824,7 @@ apprentice_map(struct magic_set *ms, const char *fn)
 		file_error(ms, errno, "cannot map `%s'", dbname);
 		goto error;
 	}
+	map->type = MAP_TYPE_MMAP;
 #else
 	if ((map->p = CAST(void *, malloc(map->len))) == NULL) {
 		file_oomem(ms, map->len);
@@ -2747,16 +2834,39 @@ apprentice_map(struct magic_set *ms, const char *fn)
 		file_badread(ms);
 		goto error;
 	}
-	map->len = 0;
+	map->type = MAP_TYPE_MALLOC;
 #define RET	1
 #endif
 	(void)close(fd);
 	fd = -1;
+
+	if (check_buffer(ms, map, dbname) != 0)
+		goto error;
+
+	free(dbname);
+	return map;
+
+error:
+	if (fd != -1)
+		(void)close(fd);
+	apprentice_unmap(map);
+	free(dbname);
+	return NULL;
+}
+
+private int
+check_buffer(struct magic_set *ms, struct magic_map *map, const char *dbname)
+{
+	uint32_t *ptr;
+	uint32_t entries, nentries;
+	uint32_t version;
+	int i, needsbyteswap;
+
 	ptr = CAST(uint32_t *, map->p);
 	if (*ptr != MAGICNO) {
 		if (swap4(*ptr) != MAGICNO) {
 			file_error(ms, 0, "bad magic in `%s'", dbname);
-			goto error;
+			return -1;
 		}
 		needsbyteswap = 1;
 	} else
@@ -2769,15 +2879,14 @@ apprentice_map(struct magic_set *ms, const char *fn)
 		file_error(ms, 0, "File %s supports only version %d magic "
 		    "files. `%s' is version %d", VERSION,
 		    VERSIONNO, dbname, version);
-		goto error;
+		return -1;
 	}
-	entries = (uint32_t)(st.st_size / sizeof(struct magic));
-	if ((off_t)(entries * sizeof(struct magic)) != st.st_size) {
-		file_error(ms, 0, "Size of `%s' %" INT64_T_FORMAT "u is not "
+	entries = (uint32_t)(map->len / sizeof(struct magic));
+	if ((entries * sizeof(struct magic)) != map->len) {
+		file_error(ms, 0, "Size of `%s' %" SIZE_T_FORMAT "u is not "
 		    "a multiple of %" SIZE_T_FORMAT "u",
-		    dbname, (unsigned long long)st.st_size,
-		    sizeof(struct magic));
-		goto error;
+		    dbname, map->len, sizeof(struct magic));
+		return -1;
 	}
 	map->magic[0] = CAST(struct magic *, map->p) + 1;
 	nentries = 0;
@@ -2793,20 +2902,12 @@ apprentice_map(struct magic_set *ms, const char *fn)
 	if (entries != nentries + 1) {
 		file_error(ms, 0, "Inconsistent entries in `%s' %u != %u",
 		    dbname, entries, nentries + 1);
-		goto error;
+		return -1;
 	}
 	if (needsbyteswap)
 		for (i = 0; i < MAGIC_SETS; i++)
 			byteswap(map->magic[i], map->nmagic[i]);
-	free(dbname);
-	return map;
-
-error:
-	if (fd != -1)
-		(void)close(fd);
-	apprentice_unmap(map);
-	free(dbname);
-	return NULL;
+	return 0;
 }
 
 /*

src/cdf.c

@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: cdf.c,v 1.63 2014/06/09 13:04:37 christos Exp $")
+FILE_RCSID("@(#)$File: cdf.c,v 1.67 2014/09/24 19:49:07 christos Exp $")
 #endif
 
 #include <assert.h>
@@ -73,6 +73,8 @@ static union {
 #define CDF_TOLE8(x)	((uint64_t)(NEED_SWAP ? _cdf_tole8(x) : (uint64_t)(x)))
 #define CDF_TOLE4(x)	((uint32_t)(NEED_SWAP ? _cdf_tole4(x) : (uint32_t)(x)))
 #define CDF_TOLE2(x)	((uint16_t)(NEED_SWAP ? _cdf_tole2(x) : (uint16_t)(x)))
+#define CDF_TOLE(x)	(sizeof(x) == 2 ? CDF_TOLE2(x) : (sizeof(x) == 4 ? \
+    CDF_TOLE4(x) : CDF_TOLE8(x)))
 #define CDF_GETUINT32(x, y)	cdf_getuint32(x, y)
 
 
@@ -824,6 +826,10 @@ cdf_read_property_info(const cdf_stream_t *sst, const cdf_header_t *h,
 		q = (const uint8_t *)(const void *)
 		    ((const char *)(const void *)p + ofs
 		    - 2 * sizeof(uint32_t));
+		if (q < p) {
+			DPRINTF(("Wrapped around %p < %p\n", q, p));
+			goto out;
+		}
 		if (q > e) {
 			DPRINTF(("Ran of the end %p > %p\n", q, e));
 			goto out;
@@ -985,6 +991,54 @@ cdf_unpack_summary_info(const cdf_stream_t *sst, const cdf_header_t *h,
 }
 
 
+#define extract_catalog_field(f, l) \
+    memcpy(&ce[i].f, b + (l), sizeof(ce[i].f)); \
+    ce[i].f = CDF_TOLE(ce[i].f)
+
+int
+cdf_unpack_catalog(const cdf_header_t *h, const cdf_stream_t *sst,
+    cdf_catalog_t **cat)
+{
+	size_t ss = sst->sst_dirlen < h->h_min_size_standard_stream ?
+	    CDF_SHORT_SEC_SIZE(h) : CDF_SEC_SIZE(h);
+	const char *b = CAST(const char *, sst->sst_tab);
+	const char *eb = b + ss * sst->sst_len;
+	size_t nr, i, k;
+	cdf_catalog_entry_t *ce;
+	uint16_t reclen;
+	const uint16_t *np;
+
+	for (nr = 0; b < eb; nr++) {
+		memcpy(&reclen, b, sizeof(reclen));
+		reclen = CDF_TOLE2(reclen);
+		if (reclen == 0)
+			break;
+		b += reclen;
+	}
+	*cat = CAST(cdf_catalog_t *,
+	    malloc(sizeof(cdf_catalog_t) + nr * sizeof(*ce)));
+	(*cat)->cat_num = nr;
+	ce = (*cat)->cat_e;
+	b = CAST(const char *, sst->sst_tab);
+	for (i = 0; i < nr; i++) {
+		extract_catalog_field(ce_namlen, 0);
+		extract_catalog_field(ce_num, 2);
+		extract_catalog_field(ce_timestamp, 6);
+		reclen = ce[i].ce_namlen;
+		ce[i].ce_namlen =
+		    sizeof(ce[i].ce_name) / sizeof(ce[i].ce_name[0]) - 1;
+		if (ce[i].ce_namlen > reclen - 14)
+			ce[i].ce_namlen = reclen - 14;
+		np = CAST(const uint16_t *, (b + 16));
+		for (k = 0; k < ce[i].ce_namlen; k++) {
+			ce[i].ce_name[k] = np[k];
+			CDF_TOLE2(ce[i].ce_name[k]);
+		}
+		ce[i].ce_name[ce[i].ce_namlen] = 0;
+		b += reclen;
+	}
+	return 0;
+}
 
 int
 cdf_print_classid(char *buf, size_t buflen, const cdf_classid_t *id)
@@ -1068,6 +1122,15 @@ cdf_print_elapsed_time(char *buf, size_t bufsiz, cdf_timestamp_t ts)
 	return len;
 }
 
+char *
+cdf_u16tos8(char *buf, size_t len, const uint16_t *p)
+{
+	size_t i;
+	for (i = 0; i < len && p[i]; i++)
+		buf[i] = (char)p[i];
+	buf[i] = '\0';
+	return buf;
+}
 
 #ifdef CDF_DEBUG
 void
@@ -1093,7 +1156,7 @@ cdf_dump_header(const cdf_header_t *h)
 	for (i = 0; i < __arraycount(h->h_master_sat); i++) {
 		if (h->h_master_sat[i] == CDF_SECID_FREE)
 			break;
-		(void)fprintf(stderr, "%35.35s[%.3zu] = %d\n",
+		(void)fprintf(stderr, "%35.35s[%.3" SIZE_T_FORMAT "u] = %d\n",
 		    "master_sat", i, h->h_master_sat[i]);
 	}
 }
@@ -1288,7 +1351,7 @@ cdf_dump_summary_info(const cdf_header_t *h, const cdf_stream_t *sst)
 		return;
 	(void)fprintf(stderr, "Endian: %x\n", ssi.si_byte_order);
 	(void)fprintf(stderr, "Os Version %d.%d\n", ssi.si_os_version & 0xff,
-		ssi.si_os_version >> 8);
+	    ssi.si_os_version >> 8);
 	(void)fprintf(stderr, "Os %d\n", ssi.si_os);
 	cdf_print_classid(buf, sizeof(buf), &ssi.si_class);
 	(void)fprintf(stderr, "Class %s\n", buf);
@@ -1297,6 +1360,27 @@ cdf_dump_summary_info(const cdf_header_t *h, const cdf_stream_t *sst)
 	free(info);
 }
 
+
+void
+cdf_dump_catalog(const cdf_header_t *h, const cdf_stream_t *sst)
+{
+	cdf_catalog_t *cat;
+	cdf_unpack_catalog(h, sst, &cat);
+	const cdf_catalog_entry_t *ce = cat->cat_e;
+	struct timespec ts;
+	char tbuf[64], sbuf[256];
+	size_t i;
+
+	printf("Catalog:\n");
+	for (i = 0; i < cat->cat_num; i++) {
+		cdf_timestamp_to_timespec(&ts, ce[i].ce_timestamp);
+		printf("\t%d %s %s", ce[i].ce_num,
+		    cdf_u16tos8(sbuf, ce[i].ce_namlen, ce[i].ce_name),
+		    cdf_ctime(&ts.tv_sec, tbuf));
+	}
+	free(cat);
+}
+
 #endif
 
 #ifdef TEST
@@ -1309,6 +1393,7 @@ main(int argc, char *argv[])
 	cdf_stream_t sst, scn;
 	cdf_dir_t dir;
 	cdf_info_t info;
+	const cdf_directory_t *root;
 
 	if (argc < 2) {
 		(void)fprintf(stderr, "Usage: %s <filename>\n", getprogname());
@@ -1342,7 +1427,8 @@ main(int argc, char *argv[])
 		if (cdf_read_dir(&info, &h, &sat, &dir) == -1)
 			err(1, "Cannot read dir");
 
-		if (cdf_read_short_stream(&info, &h, &sat, &dir, &sst) == -1)
+		if (cdf_read_short_stream(&info, &h, &sat, &dir, &sst, &root)
+		    == -1)
 			err(1, "Cannot read short stream");
 #ifdef CDF_DEBUG
 		cdf_dump_stream(&h, &sst);
@@ -1355,9 +1441,17 @@ main(int argc, char *argv[])
 
 		if (cdf_read_summary_info(&info, &h, &sat, &ssat, &sst, &dir,
 		    &scn) == -1)
-			err(1, "Cannot read summary info");
+			warn("Cannot read summary info");
 #ifdef CDF_DEBUG
-		cdf_dump_summary_info(&h, &scn);
+		else
+			cdf_dump_summary_info(&h, &scn);
+#endif
+		if (cdf_read_catalog(&info, &h, &sat, &ssat, &sst, &dir,
+		    &scn) == -1)
+			warn("Cannot read catalog");
+#ifdef CDF_DEBUG
+		else
+			cdf_dump_catalog(&h, &scn);
 #endif
 
 		(void)close(info.i_fd);

src/cdf.h

@@ -267,6 +267,19 @@ typedef struct {
 	size_t i_len;
 } cdf_info_t;
 
+
+typedef struct {
+	uint16_t ce_namlen;
+	uint32_t ce_num;
+	uint64_t ce_timestamp; 
+	uint16_t ce_name[256];
+} cdf_catalog_entry_t;
+
+typedef struct {
+	size_t cat_num;
+	cdf_catalog_entry_t cat_e[0];
+} cdf_catalog_t;
+
 struct timespec;
 int cdf_timestamp_to_timespec(struct timespec *, cdf_timestamp_t);
 int cdf_timespec_to_timestamp(cdf_timestamp_t *, const struct timespec *);
@@ -301,11 +314,19 @@ int cdf_read_property_info(const cdf_stream_t *, const cdf_header_t *, uint32_t,
 int cdf_read_user_stream(const cdf_info_t *, const cdf_header_t *,
     const cdf_sat_t *, const cdf_sat_t *, const cdf_stream_t *,
     const cdf_dir_t *, const char *, cdf_stream_t *);
+#define cdf_read_catalog(info, header, sat, ssat, stream, dir, scn) \
+    cdf_read_user_stream(info, header, sat, ssat, stream, dir, "Catalog", \
+    scn)
+#define cdf_read_encrypted_package(info, header, sat, ssat, stream, dir, scn) \
+    cdf_read_user_stream(info, header, sat, ssat, stream, dir, \
+    "EncryptedPackage", scn)
 int cdf_read_summary_info(const cdf_info_t *, const cdf_header_t *,
     const cdf_sat_t *, const cdf_sat_t *, const cdf_stream_t *,
     const cdf_dir_t *, cdf_stream_t *);
 int cdf_unpack_summary_info(const cdf_stream_t *, const cdf_header_t *,
     cdf_summary_info_header_t *, cdf_property_info_t **, size_t *);
+int cdf_unpack_catalog(const cdf_header_t *, const cdf_stream_t *,
+    cdf_catalog_t **);
 int cdf_print_classid(char *, size_t, const cdf_classid_t *);
 int cdf_print_property_name(char *, size_t, uint32_t);
 int cdf_print_elapsed_time(char *, size_t, cdf_timestamp_t);
@@ -313,6 +334,7 @@ uint16_t cdf_tole2(uint16_t);
 uint32_t cdf_tole4(uint32_t);
 uint64_t cdf_tole8(uint64_t);
 char *cdf_ctime(const time_t *, char *);
+char *cdf_u16tos8(char *, size_t, const uint16_t *);
 
 #ifdef CDF_DEBUG
 void cdf_dump_header(const cdf_header_t *);
@@ -323,6 +345,7 @@ void cdf_dump_dir(const cdf_info_t *, const cdf_header_t *, const cdf_sat_t *,
     const cdf_sat_t *, const cdf_stream_t *, const cdf_dir_t *);
 void cdf_dump_property_info(const cdf_property_info_t *, size_t);
 void cdf_dump_summary_info(const cdf_header_t *, const cdf_stream_t *);
+void cdf_dump_catalog(const cdf_header_t *, const cdf_stream_t *);
 #endif
 
 

src/encoding.c

@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: encoding.c,v 1.9 2013/11/19 20:45:50 christos Exp $")
+FILE_RCSID("@(#)$File: encoding.c,v 1.10 2014/09/11 12:08:52 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -97,7 +97,6 @@ file_encoding(struct magic_set *ms, const unsigned char *buf, size_t nbytes, uni
 		*code_mime = "utf-8";
 	} else if (file_looks_utf8(buf, nbytes, *ubuf, ulen) > 1) {
 		DPRINTF(("utf8 %" SIZE_T_FORMAT "u\n", *ulen));
-		*code = "UTF-8 Unicode (with BOM)";
 		*code = "UTF-8 Unicode";
 		*code_mime = "utf-8";
 	} else if ((ucs_type = looks_ucs16(buf, nbytes, *ubuf, ulen)) != 0) {

src/file.c

@@ -32,7 +32,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: file.c,v 1.153 2014/02/11 15:41:04 christos Exp $")
+FILE_RCSID("@(#)$File: file.c,v 1.154 2014/09/10 18:41:51 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -54,9 +54,6 @@ FILE_RCSID("@(#)$File: file.c,v 1.153 2014/02/11 15:41:04 christos Exp $")
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>	/* for read() */
 #endif
-#ifdef HAVE_LOCALE_H
-#include <locale.h>
-#endif
 #ifdef HAVE_WCHAR_H
 #include <wchar.h>
 #endif

src/file.h

@@ -27,7 +27,7 @@
  */
 /*
  * file.h - definitions for file(1) program
- * @(#)$File: file.h,v 1.152 2014/06/03 19:01:34 christos Exp $
+ * @(#)$File: file.h,v 1.154 2014/09/10 18:41:51 christos Exp $
  */
 
 #ifndef __file_h__
@@ -442,6 +442,8 @@ protected int file_is_tar(struct magic_set *, const unsigned char *, size_t);
 protected int file_softmagic(struct magic_set *, const unsigned char *, size_t,
     size_t, int, int);
 protected int file_apprentice(struct magic_set *, const char *, int);
+protected int buffer_apprentice(struct magic_set *, struct magic **,
+    size_t *, size_t);
 protected int file_magicfind(struct magic_set *, const char *, struct mlist *);
 protected uint64_t file_signextend(struct magic_set *, struct magic *,
     uint64_t);
@@ -469,9 +471,17 @@ protected int file_os2_apptype(struct magic_set *, const char *, const void *,
     size_t);
 #endif /* __EMX__ */
 
+#if defined(HAVE_LOCALE_H)
+#include <locale.h>
+#endif
+
 typedef struct {
 	const char *pat;
-	char *old_lc_ctype;
+#if defined(HAVE_NEWLOCALE) && defined(HAVE_USELOCALE) && defined(HAVE_FREELOCALE)
+#define USE_C_LOCALE
+	locale_t old_lc_ctype;
+	locale_t c_lc_ctype;
+#endif
 	int rc;
 	regex_t rx;
 } file_regex_t;

src/funcs.c

@@ -27,7 +27,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: funcs.c,v 1.72 2014/05/14 23:15:42 christos Exp $")
+FILE_RCSID("@(#)$File: funcs.c,v 1.73 2014/09/10 18:41:51 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -45,9 +45,6 @@ FILE_RCSID("@(#)$File: funcs.c,v 1.72 2014/05/14 23:15:42 christos Exp $")
 #if defined(HAVE_LIMITS_H)
 #include <limits.h>
 #endif
-#if defined(HAVE_LOCALE_H)
-#include <locale.h>
-#endif
 
 #ifndef SIZE_MAX
 #define SIZE_MAX	((size_t)~0)
@@ -455,13 +452,14 @@ out:
 protected int
 file_regcomp(file_regex_t *rx, const char *pat, int flags)
 {
-	rx->old_lc_ctype = setlocale(LC_CTYPE, NULL);
-	assert(rx->old_lc_ctype != NULL);
-	rx->old_lc_ctype = strdup(rx->old_lc_ctype);
+#ifdef USE_C_LOCALE
+	rx->c_lc_ctype = newlocale(LC_CTYPE_MASK, "C", 0);
+	assert(rx->c_lc_ctype != NULL);
+	rx->old_lc_ctype = uselocale(rx->c_lc_ctype);
 	assert(rx->old_lc_ctype != NULL);
+#endif
 	rx->pat = pat;
 
-	(void)setlocale(LC_CTYPE, "C");
 	return rx->rc = regcomp(&rx->rx, pat, flags);
 }
 
@@ -478,8 +476,10 @@ file_regfree(file_regex_t *rx)
 {
 	if (rx->rc == 0)
 		regfree(&rx->rx);
-	(void)setlocale(LC_CTYPE, rx->old_lc_ctype);
-	free(rx->old_lc_ctype);
+#ifdef USE_C_LOCALE
+	(void)uselocale(rx->old_lc_ctype);
+	freelocale(rx->c_lc_ctype);
+#endif
 }
 
 protected void

src/getline.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: fgetln.c,v 1.9 2008/04/29 06:53:03 martin Exp $	*/
+/*	$NetBSD: getline.c,v 1.2 2014/09/16 17:23:50 christos Exp $	*/
 
 /*-
  * Copyright (c) 2011 The NetBSD Foundation, Inc.
@@ -52,10 +52,14 @@ getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp)
 	for (ptr = *buf, eptr = *buf + *bufsiz;;) {
 		int c = fgetc(fp);
 		if (c == -1) {
-			if (feof(fp))
-				return ptr == *buf ? -1 : ptr - *buf;
-			else
-				return -1;
+			if (feof(fp)) {
+				ssize_t diff = (ssize_t)(ptr - *buf);
+				if (diff != 0) {
+					*ptr = '\0';
+					return diff;
+				}
+			}
+			return -1;
 		}
 		*ptr++ = c;
 		if (c == delimiter) {
@@ -93,7 +97,7 @@ main(int argc, char *argv[])
 	size_t n = 0;
 
 	while ((len = getline(&p, &n, stdin)) != -1)
-		(void)printf("%zd %s", len, p);
+		(void)printf("%" SIZE_T_FORMAT "d %s", len, p);
 	free(p);
 	return 0;
 }

src/magic.c

@@ -33,7 +33,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: magic.c,v 1.84 2014/05/14 23:15:42 christos Exp $")
+FILE_RCSID("@(#)$File: magic.c,v 1.85 2014/08/04 06:19:44 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -257,6 +257,20 @@ magic_load(struct magic_set *ms, const char *magicfile)
 	return file_apprentice(ms, magicfile, FILE_LOAD);
 }
 
+#ifndef COMPILE_ONLY
+/*
+ * Install a set of compiled magic buffers.
+ */
+public int
+magic_load_buffers(struct magic_set *ms, void **bufs, size_t *sizes,
+    size_t nbufs)
+{
+	if (ms == NULL)
+		return -1;
+	return buffer_apprentice(ms, (struct magic **)bufs, sizes, nbufs);
+}
+#endif
+
 public int
 magic_compile(struct magic_set *ms, const char *magicfile)
 {

src/magic.h

@@ -75,7 +75,7 @@
 #define	MAGIC_NO_CHECK_FORTRAN	0x000000 /* Don't check ascii/fortran */
 #define	MAGIC_NO_CHECK_TROFF	0x000000 /* Don't check ascii/troff */
 
-#define MAGIC_VERSION		518	/* This implementation */
+#define MAGIC_VERSION		519	/* This implementation */
 
 
 #ifdef __cplusplus
@@ -96,6 +96,8 @@ int magic_setflags(magic_t, int);
 
 int magic_version(void);
 int magic_load(magic_t, const char *);
+int magic_load_buffers(struct magic_set *, void **, size_t *, size_t);
+
 int magic_compile(magic_t, const char *);
 int magic_check(magic_t, const char *);
 int magic_list(magic_t, const char *);

src/magic.h.in

@@ -96,6 +96,8 @@ int magic_setflags(magic_t, int);
 
 int magic_version(void);
 int magic_load(magic_t, const char *);
+int magic_load_buffers(struct magic_set *, void **, size_t *, size_t);
+
 int magic_compile(magic_t, const char *);
 int magic_check(magic_t, const char *);
 int magic_list(magic_t, const char *);

src/pread.c

@@ -1,14 +1,23 @@
 #include "file.h"
 #ifndef lint
-FILE_RCSID("@(#)$File: pread.c,v 1.2 2013/04/02 16:23:07 christos Exp $")
+FILE_RCSID("@(#)$File: pread.c,v 1.3 2014/09/15 19:11:25 christos Exp $")
 #endif  /* lint */
 #include <fcntl.h>
 #include <unistd.h>
 
 ssize_t
 pread(int fd, void *buf, size_t len, off_t off) {
-	if (lseek(fd, off, SEEK_SET) == (off_t)-1)
+	off_t old;
+	ssize_t rv;
+
+	if ((old = lseek(fd, off, SEEK_SET)) == -1)
+		return -1;
+
+	if ((rv = read(fd, buf, len)) == -1)
+		return -1;
+
+	if (lseek(fd, old, SEEK_SET) == -1)
 		return -1;
 
-	return read(fd, buf, len);
+	return rv;
 }

src/readcdf.c

@@ -26,7 +26,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: readcdf.c,v 1.44 2014/05/14 23:22:48 christos Exp $")
+FILE_RCSID("@(#)$File: readcdf.c,v 1.48 2014/09/10 18:41:51 christos Exp $")
 #endif
 
 #include <assert.h>
@@ -35,9 +35,6 @@ FILE_RCSID("@(#)$File: readcdf.c,v 1.44 2014/05/14 23:22:48 christos Exp $")
 #include <string.h>
 #include <time.h>
 #include <ctype.h>
-#if defined(HAVE_LOCALE_H)
-#include <locale.h>
-#endif
 
 #include "cdf.h"
 #include "magic.h"
@@ -107,20 +104,23 @@ cdf_app_to_mime(const char *vbuf, const struct nv *nv)
 {
 	size_t i;
 	const char *rv = NULL;
-	char *old_lc_ctype;
+#ifdef USE_C_LOCALE
+	locale_t old_lc_ctype, c_lc_ctype;
 
-	old_lc_ctype = setlocale(LC_CTYPE, NULL);
-	assert(old_lc_ctype != NULL);
-	old_lc_ctype = strdup(old_lc_ctype);
+	c_lc_ctype = newlocale(LC_CTYPE_MASK, "C", 0);
+	assert(c_lc_ctype != NULL);
+	old_lc_ctype = uselocale(c_lc_ctype);
 	assert(old_lc_ctype != NULL);
-	(void)setlocale(LC_CTYPE, "C");
+#endif
 	for (i = 0; nv[i].pattern != NULL; i++)
 		if (strcasestr(vbuf, nv[i].pattern) != NULL) {
 			rv = nv[i].mime;
 			break;
 		}
-	(void)setlocale(LC_CTYPE, old_lc_ctype);
-	free(old_lc_ctype);
+#ifdef USE_C_LOCALE
+	(void)uselocale(old_lc_ctype);
+	freelocale(c_lc_ctype);
+#endif
 	return rv;
 }
 
@@ -241,6 +241,37 @@ cdf_file_property_info(struct magic_set *ms, const cdf_property_info_t *info,
 }
 
 private int
+cdf_file_catalog(struct magic_set *ms, const cdf_header_t *h,
+    const cdf_stream_t *sst)
+{
+	cdf_catalog_t *cat;
+	size_t i;
+	char buf[256];
+	cdf_catalog_entry_t *ce;
+
+        if (NOTMIME(ms)) {
+		if (file_printf(ms, "Microsoft Thumbs.db [") == -1)
+			return -1;
+		if (cdf_unpack_catalog(h, sst, &cat) == -1)
+			return -1;
+		ce = cat->cat_e;
+		/* skip first entry since it has a , or paren */
+		for (i = 1; i < cat->cat_num; i++)
+			if (file_printf(ms, "%s%s",
+			    cdf_u16tos8(buf, ce[i].ce_namlen, ce[i].ce_name),
+			    i == cat->cat_num - 1 ? "]" : ", ") == -1) {
+				free(cat);
+				return -1;
+			}
+		free(cat);
+	} else {
+		if (file_printf(ms, "application/CDFV2") == -1)
+			return -1;
+	}
+	return 1;
+}
+
+private int
 cdf_file_summary_info(struct magic_set *ms, const cdf_header_t *h,
     const cdf_stream_t *sst, const cdf_directory_t *root_storage)
 {
@@ -285,11 +316,12 @@ cdf_file_summary_info(struct magic_set *ms, const cdf_header_t *h,
 		if (root_storage) {
 			str = cdf_clsid_to_mime(root_storage->d_storage_uuid,
 			    clsid2desc);
-			if (str)
+			if (str) {
 				if (file_printf(ms, ", %s", str) == -1)
 					return -2;
 			}
 		}
+	}
 
         m = cdf_file_property_info(ms, info, count, root_storage);
         free(info);
@@ -404,8 +436,24 @@ file_trycdf(struct magic_set *ms, int fd, const unsigned char *buf,
         if ((i = cdf_read_summary_info(&info, &h, &sat, &ssat, &sst, &dir,
             &scn)) == -1) {
                 if (errno == ESRCH) {
-                        corrupt = expn;
-                        expn = "No summary info";
+			if ((i = cdf_read_catalog(&info, &h, &sat, &ssat, &sst,
+			    &dir, &scn)) == -1) {
+				corrupt = expn;
+				if ((i = cdf_read_encrypted_package(&info, &h,
+				    &sat, &ssat, &sst, &dir, &scn)) == -1)
+					expn = "No summary info";
+				else {
+					expn = "Encrypted";
+					i = -1;
+				}
+				goto out4;
+			}
+#ifdef CDF_DEBUG
+			cdf_dump_catalog(&h, &scn);
+#endif
+			if ((i = cdf_file_catalog(ms, &h, &scn))
+			    < 0)
+				expn = "Can't expand catalog";
                 } else {
                         expn = "Cannot read summary info";
                 }
@@ -464,7 +512,8 @@ out0:
 		    if (file_printf(ms, ", %s%s", corrupt, expn) == -1)
 			return -1;
 	    } else {
-		if (file_printf(ms, "application/CDFV2-corrupt") == -1)
+		if (file_printf(ms, "application/CDFV2-%s",
+		    *corrupt ? "corrupt" : "encrypted") == -1)
 		    return -1;
 	    }
 	    i = 1;

src/softmagic.c

@@ -32,7 +32,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: softmagic.c,v 1.191 2014/06/04 17:36:34 christos Exp $")
+FILE_RCSID("@(#)$File: softmagic.c,v 1.195 2014/09/24 19:49:07 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -41,10 +41,6 @@ FILE_RCSID("@(#)$File: softmagic.c,v 1.191 2014/06/04 17:36:34 christos Exp $")
 #include <ctype.h>
 #include <stdlib.h>
 #include <time.h>
-#if defined(HAVE_LOCALE_H)
-#include <locale.h>
-#endif
-
 
 private int match(struct magic_set *, struct magic *, uint32_t,
     const unsigned char *, size_t, size_t, int, int, int, int, int *, int *,
@@ -78,6 +74,7 @@ file_softmagic(struct magic_set *ms, const unsigned char *buf, size_t nbytes,
 {
 	struct mlist *ml;
 	int rv, printed_something = 0, need_separator = 0;
+
 	for (ml = ms->mlist[0]->next; ml != ms->mlist[0]; ml = ml->next)
 		if ((rv = match(ms, ml->magic, ml->nmagic, buf, nbytes, 0, mode,
 		    text, 0, level, &printed_something, &need_separator,
@@ -98,8 +95,8 @@ file_fmtcheck(struct magic_set *ms, const struct magic *m, const char *def,
 	const char *ptr = fmtcheck(m->desc, def);
 	if (ptr == def)
 		file_magerror(ms,
-		    "%s, %zu: format `%s' does not match with `%s'",
-		    file, line, m->desc, def);
+		    "%s, %" SIZE_T_FORMAT "u: format `%s' does not match"
+		    " with `%s'", file, line, m->desc, def);
 	return ptr;
 }
 #else
@@ -235,9 +232,9 @@ match(struct magic_set *ms, struct magic *magic, uint32_t nmagic,
 		if (file_check_mem(ms, ++cont_level) == -1)
 			return -1;
 
-		while (++magindex < nmagic &&
-		    magic[magindex].cont_level != 0) {
-			m = &magic[magindex];
+		while (magindex + 1 < nmagic &&
+		    magic[magindex + 1].cont_level != 0) {
+			m = &magic[++magindex];
 			ms->line = m->lineno; /* for messages */
 
 			if (cont_level < m->cont_level)
@@ -1060,7 +1057,7 @@ mconvert(struct magic_set *ms, struct magic *m, int flip)
 private void
 mdebug(uint32_t offset, const char *str, size_t len)
 {
-	(void) fprintf(stderr, "mget/%zu @%d: ", len, offset);
+	(void) fprintf(stderr, "mget/%" SIZE_T_FORMAT "u @%d: ", len, offset);
 	file_showstr(stderr, str, len);
 	(void) fputc('\n', stderr);
 	(void) fputc('\n', stderr);
@@ -1210,8 +1207,9 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
 		return -1;
 
 	if ((ms->flags & MAGIC_DEBUG) != 0) {
-		fprintf(stderr, "mget(type=%d, flag=%x, offset=%u, o=%zu, "
-		    "nbytes=%zu)\n", m->type, m->flag, offset, o, nbytes);
+		fprintf(stderr, "mget(type=%d, flag=%x, offset=%u, o=%"
+		    SIZE_T_FORMAT "u, " "nbytes=%" SIZE_T_FORMAT "u)\n",
+		    m->type, m->flag, offset, o, nbytes);
 		mdebug(offset, (char *)(void *)p, sizeof(union VALUETYPE));
 #ifndef COMPILE_ONLY
 		file_mdump(m);
@@ -1944,6 +1942,7 @@ magiccheck(struct magic_set *ms, struct magic *m)
 	case FILE_REGEX: {
 		int rc;
 		file_regex_t rx;
+		const char *search;
 
 		if (ms->search.s == NULL)
 			return 0;
@@ -1960,19 +1959,31 @@ magiccheck(struct magic_set *ms, struct magic *m)
 			size_t slen = ms->search.s_len;
 #ifndef REG_STARTEND
 #define	REG_STARTEND	0
-			char c;
-			if (slen != 0)
-				slen--;
-			c = ms->search.s[slen];
-			((char *)(intptr_t)ms->search.s)[slen] = '\0';
+			char *copy;
+			if (slen != 0) {
+			    copy = malloc(slen);
+			    if (copy == NULL)  {
+				file_error(ms, errno,
+				    "can't allocate %" SIZE_T_FORMAT "u bytes",
+				    slen);
+				return -1;
+			    }
+			    memcpy(copy, ms->search.s, slen);
+			    copy[--slen] = '\0';
+			    search = copy;
+			} else {
+			    search = ms->search.s;
+			    copy = NULL;
+			}
 #else
+			search = ms->search.s;
 			pmatch[0].rm_so = 0;
 			pmatch[0].rm_eo = slen;
 #endif
-			rc = file_regexec(&rx, (const char *)ms->search.s,
+			rc = file_regexec(&rx, (const char *)search,
 			    1, pmatch, REG_STARTEND);
 #if REG_STARTEND == 0
-			((char *)(intptr_t)ms->search.s)[l] = c;
+			free(copy);
 #endif
 			switch (rc) {
 			case 0: