// This file is part of Deark.
// Copyright (C) 2016 Jason Summers
// See the file COPYING for terms of use.

// Microsoft Compound File Binary File Format
// a.k.a. "OLE Compound Document Format", and a million other names

#include <deark-config.h>
#include <deark-private.h>
#include <deark-fmtutil.h>
DE_DECLARE_MODULE(de_module_cfb);

#define OBJTYPE_EMPTY        0x00
#define OBJTYPE_STORAGE      0x01
#define OBJTYPE_STREAM       0x02
#define OBJTYPE_ROOT_STORAGE 0x05

struct dir_entry_info {
	// Relative order in which to process this entry
	//  1 = For the root storage object (for the mini sector stream)
	//  2 = Other high priority streams
	//  3 = Normal dir entries
	int pass;

	u8 entry_type;

	int is_mini_stream;
	i64 stream_size;
	i64 normal_sec_id; // First SecID, valid if is_mini_stream==0
	i64 minisec_id; // First MiniSecID, valid if is_mini_stream==1
	struct de_stringreaderdata *fname_srd;
	u8 clsid[16];
	struct de_timestamp mod_time;

	const char *entry_type_name;
	i64 name_len_raw;
	u8 node_color;

	i32 child_id;
	i32 sibling_id[2];
	i32 parent_id; // If parent_id==0, entry is in root dir.
	de_ucstring *path; // Full dir path. Used by non-root STORAGE objects.

	u8 is_thumbsdb_catalog;
};

struct thumbsdb_catalog_entry {
	u32 id;
	struct de_stringreaderdata *fname_srd;
	struct de_timestamp mod_time;
};

typedef struct localctx_struct {
#define SUBFMT_AUTO       0
#define SUBFMT_RAW        1
#define SUBFMT_THUMBSDB   2
#define SUBFMT_TIFF37680  3
	int subformat_req;
	int subformat_final;
	int thumbsdb_msrgba_mode;
	u8 extract_raw_streams;
	u8 decode_streams;
	u8 dump_dir_structure;
	i64 minor_ver, major_ver;
	i64 sec_size;
	//i64 num_dir_sectors;
	i64 num_fat_sectors;
	i64 first_dir_sec_id;
	i64 std_stream_min_size;
	i64 first_minifat_sec_id;
	i64 num_minifat_sectors;
	i64 mini_sector_size;
	i64 first_difat_sec_id;
	i64 num_difat_sectors;
	i64 num_fat_entries;
	i64 num_dir_entries;

	// The DIFAT is an array of the secIDs that contain the FAT.
	// It is stored in a linked list of sectors, except that the first
	// 109 array entries are stored in the header.
	// After that, the last 4 bytes of each sector are the SecID of the
	// sector containing the next part of the DIFAT, and the remaining
	// bytes are the payload data.
	dbuf *difat;

	// The FAT is an array of "next sectors". Given a SecID, it will tell you
	// the "next" SecID in the stream that uses that sector, or it may have
	// a special code that means "end of chain", etc.
	// All the bytes of a FAT sector are used for payload data.
	dbuf *fat;

	dbuf *minifat; // mini sector allocation table
	dbuf *dir;
	struct dir_entry_info *dir_entry; // array[num_dir_entries]
	dbuf *mini_sector_stream;

	i64 thumbsdb_catalog_num_entries;
	struct thumbsdb_catalog_entry *thumbsdb_catalog;

	int could_be_thumbsdb;
	int thumbsdb_old_names_found;
	int thumbsdb_new_names_found;
	int thumbsdb_catalog_found;
} lctx;

struct clsid_id_struct {
	const u8 clsid[16];
	u32 mask;
	u32 flags;
	const char *name;
};
static const struct clsid_id_struct known_clsids[] = {
	{{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, 0xffff, 0, "n/a"}, // This must be first.
	{{0x00,0x02,0x08,0x00,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xefff, 0, "Excel?"},
	{{0x00,0x02,0x09,0x00,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xefff, 0, "MS Word?"},
	{{0x00,0x02,0x0d,0x0b,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xffff, 0, "Outlook item?"},
	{{0x00,0x02,0x12,0x01,0x00,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x46}, 0xffff, 0, "MS Publisher?"},
	{{0x00,0x02,0x13,0x03,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xffff, 0, "MS Works WDB?"},
	{{0x00,0x02,0x1a,0x00,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xefff, 0, "Visio?"},
	{{0x00,0x06,0xf0,0x46,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xffff, 0, "Outlook item?"},
	{{0x00,0x0c,0x10,0x84,0x00,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x46}, 0xffff, 0, "MSI?"},
	{{0x1c,0xdd,0x8c,0x7b,0x81,0xc0,0x45,0xa0,0x9f,0xed,0x04,0x14,0x31,0x44,0xcc,0x1e}, 0xffff, 0, "3ds Max?"},
	{{0x56,0x61,0x67,0x00,0xc1,0x54,0x11,0xce,0x85,0x53,0x00,0xaa,0x00,0xa1,0xf9,0x5b}, 0xffff, 0, "FlashPix?"},
	{{0x64,0x81,0x8d,0x10,0x4f,0x9b,0x11,0xcf,0x86,0xea,0x00,0xaa,0x00,0xb9,0x29,0xe8}, 0xffff, 0, "PowerPoint?"}
};
#define EMPTY_CLSID (known_clsids[0].clsid)

static i64 sec_id_to_offset(deark *c, lctx *d, i64 sec_id)
{
	if(sec_id<0) return 0;
	return d->sec_size + sec_id * d->sec_size;
}

static i64 get_next_sec_id(deark *c, lctx *d, i64 cur_sec_id)
{
	i64 next_sec_id;

	if(cur_sec_id < 0) return -2;
	if(!d->fat) return -2;
	next_sec_id = dbuf_geti32le(d->fat, cur_sec_id*4);
	return next_sec_id;
}

static i64 get_next_minisec_id(deark *c, lctx *d, i64 cur_minisec_id)
{
	i64 next_minisec_id;

	if(cur_minisec_id < 0) return -2;
	if(!d->minifat) return -2;
	next_minisec_id = dbuf_geti32le(d->minifat, cur_minisec_id*4);
	return next_minisec_id;
}

static void describe_sec_id(deark *c, lctx *d, i64 sec_id,
	char *buf, size_t buf_len)
{
	i64 sec_offset;

	if(sec_id >= 0) {
		sec_offset = sec_id_to_offset(c, d, sec_id);
		de_snprintf(buf, buf_len, "offs=%d", (int)sec_offset);
	}
	else if(sec_id == -1) {
		de_strlcpy(buf, "free", buf_len);
	}
	else if(sec_id == -2) {
		de_strlcpy(buf, "end of chain", buf_len);
	}
	else if(sec_id == -3) {
		de_strlcpy(buf, "FAT SecID", buf_len);
	}
	else if(sec_id == -4) {
		de_strlcpy(buf, "DIFAT SecID", buf_len);
	}
	else {
		de_strlcpy(buf, "?", buf_len);
	}
}

// Copy a stream (with a known byte size) to a dbuf.
static void copy_normal_stream_to_dbuf(deark *c, lctx *d, i64 first_sec_id,
	i64 stream_startpos, i64 stream_size,
	dbuf *outf)
{
	i64 sec_id;
	i64 bytes_left_to_copy;
	i64 bytes_left_to_skip;

	if(stream_size<=0) return;
	if(stream_startpos+stream_size > c->infile->len) {
		// This is a not-too-strict emergency brake. If the file has been
		// truncated, we might still be able to process some of the data
		// that is there.
		stream_size = c->infile->len - stream_startpos;
	}

	bytes_left_to_copy = stream_size;
	bytes_left_to_skip = stream_startpos;
	sec_id = first_sec_id;
	while(bytes_left_to_copy > 0) {
		i64 sec_offs;
		i64 bytes_to_copy;
		i64 bytes_to_skip;

		if(sec_id<0) break;
		sec_offs = sec_id_to_offset(c, d, sec_id);

		bytes_to_skip = bytes_left_to_skip;
		if(bytes_to_skip > d->sec_size) bytes_to_skip = d->sec_size;

		bytes_to_copy = d->sec_size - bytes_to_skip;
		if(bytes_to_copy > bytes_left_to_copy) bytes_to_copy = bytes_left_to_copy;

		dbuf_copy(c->infile, sec_offs + bytes_to_skip, bytes_to_copy, outf);

		bytes_left_to_copy -= bytes_to_copy;
		bytes_left_to_skip -= bytes_to_skip;
		sec_id = get_next_sec_id(c, d, sec_id);
	}
}

// Same as copy_normal_stream_to_dbuf(), but for mini streams.
static void copy_mini_stream_to_dbuf(deark *c, lctx *d, i64 first_minisec_id,
	i64 stream_startpos, i64 stream_size,
	dbuf *outf)
{
	i64 minisec_id;
	i64 bytes_left_to_copy;
	i64 bytes_left_to_skip;

	if(!d->mini_sector_stream) return;
	if(stream_size<=0 || stream_size>c->infile->len ||
		stream_size>d->mini_sector_stream->len)
	{
		return;
	}

	bytes_left_to_copy = stream_size;
	bytes_left_to_skip = stream_startpos;
	minisec_id = first_minisec_id;
	while(bytes_left_to_copy > 0) {
		i64 minisec_offs;
		i64 bytes_to_copy;
		i64 bytes_to_skip;

		if(minisec_id<0) break;
		minisec_offs = minisec_id * d->mini_sector_size;

		bytes_to_skip = bytes_left_to_skip;
		if(bytes_to_skip > d->mini_sector_size) bytes_to_skip = d->mini_sector_size;

		bytes_to_copy = d->mini_sector_size - bytes_to_skip;
		if(bytes_to_copy > bytes_left_to_copy) bytes_to_copy = bytes_left_to_copy;

		dbuf_copy(d->mini_sector_stream, minisec_offs + bytes_to_skip, bytes_to_copy, outf);

		bytes_left_to_copy -= bytes_to_copy;
		bytes_left_to_skip -= bytes_to_skip;
		minisec_id = get_next_minisec_id(c, d, minisec_id);
	}
}

static void copy_any_stream_to_dbuf(deark *c, lctx *d, struct dir_entry_info *dei,
	i64 stream_startpos, i64 stream_size,
	dbuf *outf)
{
	if(dei->is_mini_stream) {
		copy_mini_stream_to_dbuf(c, d, dei->minisec_id, stream_startpos, stream_size, outf);
	}
	else {
		copy_normal_stream_to_dbuf(c, d, dei->normal_sec_id, stream_startpos, stream_size, outf);
	}
}

static int do_header(deark *c, lctx *d)
{
	i64 pos = 0;
	i64 byte_order_code;
	i64 sector_shift;
	i64 mini_sector_shift;
	char buf[80];
	int retval = 0;

	de_dbg(c, "header at %d", (int)pos);
	de_dbg_indent(c, 1);

	// offset 0-7: signature
	// offset 8-23: CLSID

	d->minor_ver = de_getu16le(pos+24);
	d->major_ver = de_getu16le(pos+26);
	de_dbg(c, "format version: %d.%d", (int)d->major_ver, (int)d->minor_ver);
	if(d->major_ver!=3 && d->major_ver!=4) {
		de_err(c, "Unsupported format version: %d", (int)d->major_ver);
		goto done;
	}

	byte_order_code = de_getu16le(pos+28);
	if(byte_order_code != 0xfffe) {
		de_err(c, "Unsupported byte order code: 0x%04x", (unsigned int)byte_order_code);
		goto done;
	}

	sector_shift = de_getu16le(pos+30); // aka ssz
	d->sec_size = de_pow2(sector_shift);
	de_dbg(c, "sector size: 2^%d (%d bytes)", (int)sector_shift,
		(int)d->sec_size);
	if(d->sec_size!=512 && d->sec_size!=4096) {
		de_err(c, "Unsupported sector size: %d", (int)d->sec_size);
		goto done;
	}

	mini_sector_shift = de_getu16le(pos+32); // aka sssz
	d->mini_sector_size = de_pow2(mini_sector_shift);
	de_dbg(c, "mini sector size: 2^%d (%d bytes)", (int)mini_sector_shift,
		(int)d->mini_sector_size);
	if(d->mini_sector_size!=64) {
		de_err(c, "Unsupported mini sector size: %d", (int)d->mini_sector_size);
		goto done;
	}

	// offset 34: 6 reserved bytes

	//d->num_dir_sectors = de_getu32le(pos+40);
	//de_dbg(c, "number of directory sectors: %u", (unsigned int)d->num_dir_sectors);
	// Should be 0 if major_ver==3

	// Number of sectors used by sector allocation table (FAT)
	d->num_fat_sectors = de_getu32le(pos+44);
	de_dbg(c, "number of FAT sectors: %d", (int)d->num_fat_sectors);

	d->first_dir_sec_id = de_geti32le(pos+48);
	describe_sec_id(c, d, d->first_dir_sec_id, buf, sizeof(buf));
	de_dbg(c, "first directory sector: %d (%s)", (int)d->first_dir_sec_id, buf);

	// offset 52, transaction signature number

	d->std_stream_min_size = de_getu32le(pos+56);
	de_dbg(c, "min size of a standard stream: %d", (int)d->std_stream_min_size);

	// First sector of mini sector allocation table (MiniFAT)
	d->first_minifat_sec_id = de_geti32le(pos+60);
	describe_sec_id(c, d, d->first_minifat_sec_id, buf, sizeof(buf));
	de_dbg(c, "first MiniFAT sector: %d (%s)", (int)d->first_minifat_sec_id, buf);

	// Number of sectors used by MiniFAT
	d->num_minifat_sectors = de_getu32le(pos+64);
	de_dbg(c, "number of MiniFAT sectors: %d", (int)d->num_minifat_sectors);

	// SecID of first (extra?) sector of the DIFAT
	// (also called the Master Sector Allocation Table (MSAT))
	d->first_difat_sec_id = de_geti32le(pos+68);
	describe_sec_id(c, d, d->first_difat_sec_id, buf, sizeof(buf));
	de_dbg(c, "first extended DIFAT sector: %d (%s)", (int)d->first_difat_sec_id, buf);

	// Number of (extra?) sectors used by the DIFAT
	d->num_difat_sectors = de_getu32le(pos+72);
	de_dbg(c, "number of extended DIFAT sectors: %d", (int)d->num_difat_sectors);

	// offset 76: 436 bytes of DIFAT data
	retval = 1;

done:
	de_dbg_indent(c, -1);
	return retval;
}

// Read the locations of the FAT sectors
static void read_difat(deark *c, lctx *d)
{
	i64 num_to_read;
	i64 still_to_read;
	i64 difat_sec_id;
	i64 difat_sec_offs;


	de_dbg(c, "reading DIFAT (total number of entries=%d)", (int)d->num_fat_sectors);
	de_dbg_indent(c, 1);

	if(d->num_fat_sectors > 1000000) {
		// TODO: Decide what limits to enforce.
		d->num_fat_sectors = 1000000;
	}

	// Expecting d->num_fat_sectors in the DIFAT table
	d->difat = dbuf_create_membuf(c, d->num_fat_sectors * 4, 1);

	still_to_read = d->num_fat_sectors;

	// Copy the part of the DIFAT that is in the header
	num_to_read = still_to_read;
	if(num_to_read>109) num_to_read = 109;
	de_dbg(c, "reading %d DIFAT entries from header, at 76", (int)num_to_read);
	dbuf_copy(c->infile, 76, num_to_read*4, d->difat);
	still_to_read -= num_to_read;

	difat_sec_id = d->first_difat_sec_id;
	while(still_to_read>0) {
		if(difat_sec_id<0) break;

		difat_sec_offs = sec_id_to_offset(c, d, difat_sec_id);
		de_dbg(c, "reading DIFAT sector at %d", (int)difat_sec_offs);
		num_to_read = (d->sec_size - 4)/4;

		dbuf_copy(c->infile, difat_sec_offs, num_to_read*4, d->difat);
		still_to_read -= num_to_read;
		difat_sec_id = de_geti32le(difat_sec_offs + num_to_read*4);
	}

	de_dbg_indent(c, -1);
}

static void dump_fat(deark *c, lctx *d)
{
	i64 i;
	i64 sec_id;
	char buf[80];

	if(c->debug_level<2) return;

	de_dbg2(c, "dumping FAT contents (%d entries)", (int)d->num_fat_entries);

	de_dbg_indent(c, 1);
	for(i=0; i<d->num_fat_entries; i++) {
		sec_id = dbuf_geti32le(d->fat, i*4);
		describe_sec_id(c, d, sec_id, buf, sizeof(buf));
		de_dbg2(c, "FAT[%d]: next_SecID=%d (%s)", (int)i, (int)sec_id, buf);
	}
	de_dbg_indent(c, -1);
}

// Read the contents of the FAT sectors
static void read_fat(deark *c, lctx *d)
{
	i64 i;
	i64 sec_id;
	i64 sec_offset;
	char buf[80];

	d->fat = dbuf_create_membuf(c, d->num_fat_sectors * d->sec_size, 1);

	de_dbg(c, "reading FAT contents (%d sectors)", (int)d->num_fat_sectors);
	de_dbg_indent(c, 1);
	for(i=0; i<d->num_fat_sectors; i++) {
		sec_id = dbuf_geti32le(d->difat, i*4);
		sec_offset = sec_id_to_offset(c, d, sec_id);
		describe_sec_id(c, d, sec_id, buf, sizeof(buf));
		de_dbg(c, "reading sector: DIFAT_idx=%d, SecID=%d (%s)",
			(int)i, (int)sec_id, buf);
		dbuf_copy(c->infile, sec_offset, d->sec_size, d->fat);
	}
	de_dbg_indent(c, -1);

	d->num_fat_entries = d->fat->len/4;
	dump_fat(c, d);
}

static void dump_minifat(deark *c, lctx *d)
{
	i64 i;
	i64 sec_id;
	i64 num_minifat_entries;

	if(c->debug_level<2) return;
	if(!d->minifat) return;

	num_minifat_entries = d->minifat->len / 4;
	de_dbg2(c, "dumping MiniFAT contents (%d entries)", (int)num_minifat_entries);

	de_dbg_indent(c, 1);
	for(i=0; i<num_minifat_entries; i++) {
		sec_id = dbuf_geti32le(d->minifat, i*4);
		de_dbg2(c, "MiniFAT[%d]: next_MiniSecID=%d", (int)i, (int)sec_id);
	}
	de_dbg_indent(c, -1);
}

// Read the contents of the MiniFAT sectors into d->minifat
static void read_minifat(deark *c, lctx *d)
{
	i64 i;
	i64 sec_id;
	i64 sec_offset;
	char buf[80];

	if(d->num_minifat_sectors > 1000000) {
		// TODO: Decide what limits to enforce.
		d->num_minifat_sectors = 1000000;
	}

	d->minifat = dbuf_create_membuf(c, d->num_minifat_sectors * d->sec_size, 1);

	// TODO: Use copy_normal_stream_to_dbuf
	de_dbg(c, "reading MiniFAT contents (%d sectors)", (int)d->num_minifat_sectors);
	de_dbg_indent(c, 1);

	sec_id = d->first_minifat_sec_id;

	for(i=0; i<d->num_minifat_sectors; i++) {
		if(sec_id<0) break;

		sec_offset = sec_id_to_offset(c, d, sec_id);
		describe_sec_id(c, d, sec_id, buf, sizeof(buf));
		de_dbg(c, "reading MiniFAT sector #%d, SecID=%d (%s), MiniSecIDs %d-%d",
			(int)i, (int)sec_id, buf,
			(int)(i*(d->sec_size/4)), (int)((i+1)*(d->sec_size/4)-1));
		dbuf_copy(c->infile, sec_offset, d->sec_size, d->minifat);

		sec_id = get_next_sec_id(c, d, sec_id);
	}
	de_dbg_indent(c, -1);

	dump_minifat(c, d);
}

// Returns -1 if not a valid name
static i64 stream_name_to_catalog_id(deark *c, lctx *d, struct dir_entry_info *dei)
{
	char buf[16];
	size_t nlen;
	size_t i;

	nlen = dei->fname_srd->sz_utf8_strlen;
	if(nlen>sizeof(buf)-1) return -1;

	for(i=0; i<nlen; i++) {
		// Name should contain only digits
		if(dei->fname_srd->sz_utf8[i]<'0' || dei->fname_srd->sz_utf8[i]>'9') return -1;

		// The stream name is the *reversed* string form of the ID number.
		// (I assume this is to try to keep the directory tree structure balanced.)
		buf[nlen-1-i] = dei->fname_srd->sz_utf8[i];
	}
	buf[nlen] = '\0';

	return de_atoi64(buf);
}

// Returns an index into d->thumbsdb_catalog.
// Returns -1 if not found.
static i64 lookup_thumbsdb_catalog_entry(deark *c, lctx *d, struct dir_entry_info *dei)
{
	i64 i;
	i64 id;

	if(d->thumbsdb_catalog_num_entries<1 || !d->thumbsdb_catalog) return -1;
	if(!dei->fname_srd || !dei->fname_srd->str) return -1;

	id = stream_name_to_catalog_id(c, d, dei);
	if(id<0) return -1;

	for(i=0; i<d->thumbsdb_catalog_num_entries; i++) {
		if(d->thumbsdb_catalog[i].id == id)
			return i;
	}
	return -1;
}

static void thumbsdb_msrgba_write_dqt(deark *c, dbuf *outf)
{
	size_t i;
	u8 tbl[64];

	fmtutil_get_std_jpeg_qtable(0, tbl);

	for(i=0; i<64; i++) {
		tbl[i] = (u8)(((UI)tbl[i]+1)/2);
	}

	dbuf_write(outf, (const u8*)"\xff\xdb\x00\x43\x00", 5);
	dbuf_write(outf, tbl, 64);
}

// This function tries to better handle a special nonstandard JPEG thumbnail format
// that I'm calling MSRGBA.
// We can't *really* handle it, because Deark doesn't decompress lossy formats, and
// AFAIK there is no standard format that we can losslessly convert it to.
// What we can do is add the missing quantization and Huffman tables, and add a
// custom segment to help identify the format.
// This should allow most JPEG viewers to decode the image, though most will guess
// it is CMYK, and display the colors all wrong (often all black).
// Note that the component ID numbers are ASCII 'R','G','B','A'.
// Based on my (possibly wrong) analysis, the 'R' channel is blue(!), 'G' is green,
// 'B' is red, and 'A' can be either opacity, or unused (dunno how to tell which).
//
// hdrsize is the length of just the first header.
// Returns 0 if nothing was extracted.
static int thumbsdb_msrgba_special_extract(deark *c, lctx *d, struct dir_entry_info *dei,
	i64 hdrsize, dbuf *outf)
{
	i64 inf_pos;
	i64 idseg_len;

	if(dei->stream_size<hdrsize+16+2+22) return 0;
	inf_pos = hdrsize+16; // Also skip past the 16-byte extra header

	// SOI
	copy_any_stream_to_dbuf(c, d, dei, inf_pos, 2, outf);
	inf_pos += 2;

	// Special APP1 segment to record both headers, and identify the format.
	dbuf_write(outf, (const u8*)"\xff\xe1", 2);
	idseg_len = 2 + 12 + 1 + hdrsize + 16;
	dbuf_writeu16be(outf, idseg_len);
	dbuf_write(outf, (const u8*)"Deark_MSRGBA\0", 13);
	copy_any_stream_to_dbuf(c, d, dei, 0, hdrsize+16, outf);

	// DQT
	thumbsdb_msrgba_write_dqt(c, outf);
	// TODO: Do we ever need another quantization table?

	// SOF0
	// TODO?: This code is fragile. We could parse the JPEG data, instead of
	// just hoping it is laid out like we expect.
	copy_any_stream_to_dbuf(c, d, dei, inf_pos, 22, outf);
	inf_pos += 22;

	// DHT
	dbuf_write(outf, (const u8*)"\xff\xc4\x00\xd2\x00", 5);
	fmtutil_write_std_jpeg_dht(outf, 0);
	dbuf_writebyte(outf, 0x10);
	fmtutil_write_std_jpeg_dht(outf, 1);

	// The rest of the file
	copy_any_stream_to_dbuf(c, d, dei, inf_pos, dei->stream_size-inf_pos, outf);

	return 1;
}

// Special handling of Thumbs.db files.
// Caller sets fi and tmpfn to default values. This function may modify them.
// firstpart = caller-supplied dbuf containing the first 256 or so bytes of the stream
static void do_extract_stream_to_file_thumbsdb(deark *c, lctx *d, struct dir_entry_info *dei,
	de_finfo *fi, de_ucstring *tmpfn, dbuf *firstpart)
{
	i64 hdrsize;
	i64 catalog_idx;
	dbuf *outf = NULL;
	i64 ver;
	i64 reported_size;
	i64 startpos;
	i64 final_streamsize;
	int is_msrgba = 0;
	struct fmtutil_fmtid_ctx *idctx = NULL;

	if(dei->is_thumbsdb_catalog) {
		// We've already read the catalog.
		goto done;
	}

	de_dbg(c, "reading Thumbs.db stream");
	de_dbg_indent(c, 1);

	startpos = 0;
	final_streamsize = dei->stream_size;

	// A Thumbs.db stream typically has a header, followed by an embedded JPEG
	// (or something) file.

	catalog_idx = lookup_thumbsdb_catalog_entry(c, d, dei);

	if(catalog_idx>=0) {
		if(d->thumbsdb_catalog[catalog_idx].mod_time.is_valid) {
			fi->timestamp[DE_TIMESTAMPIDX_MODIFY] = d->thumbsdb_catalog[catalog_idx].mod_time; // struct copy
		}
	}

	hdrsize = dbuf_getu32le(firstpart, 0);
	de_dbg(c, "header size: %d", (int)hdrsize);

	ver = dbuf_getu32le(firstpart, 4);
	de_dbg(c, "version: %d", (int)ver);

	// 0x0c = "Original format" Thumbs.db
	// 0x18 = "Windows 7 format"

	if((hdrsize==0x0c || hdrsize==0x18) && dei->stream_size>hdrsize) {
		const char *ext;

		reported_size = dbuf_getu32le(firstpart, 8);
		de_dbg(c, "reported size: %d", (int)reported_size);

		startpos = hdrsize;
		final_streamsize -= hdrsize;
		de_dbg(c, "calculated size: %d", (int)final_streamsize);

		if(catalog_idx>=0 && c->filenames_from_file) {
			de_dbg(c, "name from catalog: \"%s\"",
				ucstring_getpsz(d->thumbsdb_catalog[catalog_idx].fname_srd->str));

			// Replace the default name with the name from the catalog.
			ucstring_empty(tmpfn);

			if(!de_strcasecmp(d->thumbsdb_catalog[catalog_idx].fname_srd->sz_utf8,
				"{A42CD7B6-E9B9-4D02-B7A6-288B71AD28BA}"))
			{
				ucstring_append_sz(tmpfn, "_folder", DE_ENCODING_LATIN1);
			}
			else {
				ucstring_append_ucstring(tmpfn, d->thumbsdb_catalog[catalog_idx].fname_srd->str);
			}
		}

		idctx = de_malloc(c, sizeof(struct fmtutil_fmtid_ctx));
		dbuf_read(firstpart, idctx->bof64bytes, hdrsize, sizeof(idctx->bof64bytes));
		idctx->have_bof64bytes = 1;
		// Expecting JPEG, PNG, or special MS-JPEG.
		idctx->mode = FMTUTIL_FMTIDMODE_ALL_IMG;
		fmtutil_fmtid(c, idctx);
		ext = idctx->ext_sz;

		if(idctx->fmtid==0 &&
			idctx->bof64bytes[0]==0x01 && idctx->bof64bytes[1]==0x00 &&
			idctx->bof64bytes[16]==0xff && idctx->bof64bytes[17]==0xd8)
		{
			// Looks like a nonstandard Microsoft RGBA JPEG.
			ext = "msrgbajpg";
			is_msrgba = 1;
		}

		ucstring_printf(tmpfn, DE_ENCODING_LATIN1, ".thumb.%s", ext);
	}
	else {
		de_warn(c, "Unidentified Thumbs.db stream \"%s\"",
			ucstring_getpsz(dei->fname_srd->str));
	}

	de_dbg_indent(c, -1);

	de_finfo_set_name_from_ucstring(c, fi, tmpfn, 0);
	fi->original_filename_flag = 1;

	outf = dbuf_create_output_file(c, NULL, fi, 0);

	if(is_msrgba) {
		if(d->thumbsdb_msrgba_mode) {
			int ok = 0;
			ok = thumbsdb_msrgba_special_extract(c, d, dei, hdrsize, outf);
			if(ok) goto done;
		}

		// "MSRGBA" thumbnails seem to have an additional 16-byte header,
		// before the JPEG data starts. In this mode, we just ignore it.
		startpos += 16;
		final_streamsize -= 16;
	}

	copy_any_stream_to_dbuf(c, d, dei, startpos, final_streamsize, outf);

done:
	dbuf_close(outf);
	de_free(c, idctx);
}

static void do_OfficeArtStream(deark *c, lctx *d, struct dir_entry_info *dei)
{
	dbuf *tmpstream = NULL;

	de_dbg(c, "OfficeArt stream, len=%"I64_FMT, dei->stream_size);
	de_dbg_indent(c, 1);
	tmpstream = dbuf_create_membuf(c, dei->stream_size, 0x1);
	copy_any_stream_to_dbuf(c, d, dei, 0, dei->stream_size, tmpstream);
	if(tmpstream->len < dei->stream_size) {
		de_warn(c, "OfficeArt stream might have been truncated");
	}

	de_run_module_by_id_on_slice2(c, "officeart", NULL, tmpstream, 0, tmpstream->len);
	de_dbg_indent(c, -1);
	dbuf_close(tmpstream);
}

static void do_Corel_simple_image(deark *c, lctx *d, struct dir_entry_info *dei,
	dbuf *f, i64 pos1)
{
	i64 pos = pos1;
	i64 w, h;
	i64 i, j;
	u8 b;
	de_bitmap *img = NULL;

	w = dbuf_getu32le_p(f, &pos);
	h = dbuf_getu32le_p(f, &pos);
	de_dbg_dimensions(c, w, h);
	if(!de_good_image_dimensions(c, w, h)) goto done;

	img = de_bitmap_create(c, w, h, 1);

	// TODO: I don't know whether this is the right way to interpret this
	// image type.
	for(j=0; j<img->height; j++) {
		for(i=0; i<img->width; i++) {
			b = dbuf_getbyte(f, pos + j*w + i);
			de_bitmap_setpixel_gray(img, i, j, b);
		}
	}

	de_bitmap_write_to_file(img, NULL, DE_CREATEFLAG_FLIP_IMAGE);

done:
	de_bitmap_destroy(img);
}

// This is an object found in Corel Print House (.CPH) and similar files.
// This decoder is based on reverse engineering. It may be incorrect.
static void do_Corel_UIformat(deark *c, lctx *d, struct dir_entry_info *dei,
	dbuf *f, i64 pos1, i64 len, int is_thumb)
{
	i64 pos = pos1;
	i64 hdr_len;
	i64 ri_pos;
	i64 ri_len;
	i64 w, h;
	i64 pal_offs, img_offs;
	i64 rowspan;
	i64 pixels_size;
	i64 npalent;
	int bpp;
	int ok = 0;
	int saved_indent_level;
	de_bitmap *img = NULL;
	u32 pal[256];

	de_dbg_indent_save(c, &saved_indent_level);

	if(dbuf_memcmp(f, pos, "UI\x00\x00", 4)) goto done;

	de_dbg(c, "CorelUI at [%"I64_FMT"], len=%"I64_FMT, pos1, len);
	de_dbg_indent(c, 1);
	pos += 2; // "UI"
	pos += 2; // ?
	pos += 4; // The size of the "RI" segment? Redundant?
	pos += 4; // ?

	hdr_len = dbuf_getu32le_p(f, &pos);
	// Apparently the size of the "UI" segment
	if(hdr_len != 32) goto done;
	// TODO: More fields here

	ri_pos = pos1+hdr_len;
	pos = ri_pos;
	if(dbuf_memcmp(f, pos, "RI", 2)) goto done;
	pos += 2;
	ri_len = dbuf_getu32le_p(f, &pos);
	if(ri_pos + ri_len > f->len) goto done;

	pos += 16;
	w = dbuf_getu32le_p(f, &pos);
	h = dbuf_getu32le_p(f, &pos);
	de_dbg_dimensions(c, w, h);

	pos += 4; // ? (observed 1)

	bpp = (int)dbuf_getu32le_p(f, &pos);
	de_dbg(c, "bits/pixel?: %d", bpp);
	rowspan = dbuf_getu32le_p(f, &pos);
	de_dbg(c, "bytes/row?: %d", (int)rowspan);
	pixels_size = dbuf_getu32le_p(f, &pos);
	de_dbg(c, "pixels size: %"I64_FMT, pixels_size);
	pos += 4; // ?
	pos += 8; // ? (density?)

	pal_offs = dbuf_getu32le_p(f, &pos);
	de_dbg(c, "pal offs: %"I64_FMT, pal_offs);

	img_offs = dbuf_getu32le_p(f, &pos);
	de_dbg(c, "img offs: %"I64_FMT, img_offs);

	pos += 12; // ?

	if(bpp!=8 && bpp!=24) goto done;

	// == palette ==
	de_make_grayscale_palette(pal, 256, 0);
	if(pal_offs!=0) {
		// This formula doesn't make sense to me, but seems to work.
		pos = ri_pos+14+pal_offs;
		de_dbg(c, "palette at [%"I64_FMT"]", pos);
		de_dbg_indent(c, 1);
		pos += 2; // ? (observed 4, 5)

		npalent = dbuf_getu16le_p(f, &pos);
		de_dbg(c, "num pal entries: %d", (int)npalent);
		if(npalent>256) goto done;

		de_read_palette_rgb(f, pos, npalent, 3, pal, 256, DE_GETRGBFLAG_BGR);
		de_dbg_indent(c, -1);
	}

	// == image ==
	if(!de_good_image_dimensions(c, w, h)) goto done;

	img = de_bitmap_create(c, w, h, (bpp<24 && pal_offs==0)?1:3);
	pos = ri_pos+14+img_offs;
	de_dbg(c, "bitmap at [%"I64_FMT"]", pos);
	if(bpp<24) {
		de_convert_image_paletted(f, pos, 8, rowspan, pal, img, 0);
	}
	else {
		de_convert_image_rgb(f, pos, rowspan, 3, img, DE_GETRGBFLAG_BGR);
	}

	de_bitmap_write_to_file(img, is_thumb?"thumb":NULL, DE_CREATEFLAG_FLIP_IMAGE);

	ok = 1;

done:
	if(!ok) {
		de_dbg(c, "[unsupported image type]");
	}
	de_bitmap_destroy(img);
	de_dbg_indent_restore(c, saved_indent_level);
}

static void do_CorelImages_internal(deark *c, lctx *d, struct dir_entry_info *dei,
	dbuf *f)
{
	i64 pos = 0;
	int saved_indent_level;

	de_dbg_indent_save(c, &saved_indent_level);

	while(1) {
		unsigned int imgtype1_or_size;
		unsigned int imgtype1;
		i64 size1 = 0;

		if(pos >= f->len-8) break;

		de_dbg(c, "image at [%"I64_FMT"]", pos);
		de_dbg_indent(c, 1);

		// Seems like sometimes this first 'type' field is present, and
		// sometimes it isn't (and we treat it like it's 0).
		imgtype1_or_size = (unsigned int)dbuf_getu32le_p(f, &pos);
		if(imgtype1_or_size<8) {
			imgtype1 = imgtype1_or_size;

			if(imgtype1==0) {
				size1 = dbuf_getu32le_p(f, &pos);
			}
		}
		else {
			imgtype1 = 0;
			size1 = (i64)imgtype1_or_size;
		}

		de_dbg(c, "low level imgtype: %u", imgtype1);

		if(imgtype1==0) {
			unsigned int imgtype2;

			imgtype2 = (unsigned int)dbuf_getu32le_p(f, &pos);
			de_dbg(c, "high level imgtype: %u", imgtype2);
			de_dbg(c, "len: %"I64_FMT, size1);

			if(pos+size1 > f->len) break;

			if(imgtype2==0) { // "Uncompressed Image"?
				do_Corel_UIformat(c, d, dei, f, pos, size1, 0);
			}
			else if(imgtype2==1) { // JPEG?
				dbuf_create_file_from_slice(f, pos, size1, "jpg", NULL, 0);
			}
			else {
				de_dbg(c, "[unsupported image type: %u:%u]", imgtype1, imgtype2);
			}
		}
		else if(imgtype1==1) {
			size1 = dbuf_getu32le(f, pos+8);
			do_Corel_simple_image(c, d, dei, f, pos);
			pos += 8 + 4;
		}
		else {
			de_dbg(c, "[unsupported image type (%u), can't continue]", imgtype1);
			goto done;
		}

		pos += size1;
		de_dbg_indent(c, -1);
	}

done:
	de_dbg_indent_restore(c, saved_indent_level);
}

static void do_StreamNamedThumbnail(deark *c, lctx *d, struct dir_entry_info *dei)
{
	dbuf *f = NULL;
	i64 size1;

	if(dei->stream_size<32 || dei->stream_size>DE_MAX_SANE_OBJECT_SIZE) {
		goto done;
	}
	f = dbuf_create_membuf(c, 0, 0);

	// Start by reading just a little, to figure out the data type
	copy_any_stream_to_dbuf(c, d, dei, 0, 16, f);
	size1 = dbuf_getu32le(f, 0);
	if(size1+4 != dei->stream_size) goto done;
	if(dbuf_memcmp(f, 4, "UI\x00\x00", 4)) goto done;

	copy_any_stream_to_dbuf(c, d, dei, 16, dei->stream_size-16, f);
	do_Corel_UIformat(c, d, dei, f, 4, size1-4, 1);

done:
	dbuf_close(f);
}

static void do_StreamNamedImages(deark *c, lctx *d, struct dir_entry_info *dei)
{
	dbuf *f = NULL;

	if(dei->stream_size<32 || dei->stream_size>DE_MAX_SANE_OBJECT_SIZE) {
		goto done;
	}
	f = dbuf_create_membuf(c, 0, 0);

	// Start by reading just a little, to figure out the data type
	copy_any_stream_to_dbuf(c, d, dei, 0, 16, f);
	if(dbuf_memcmp(f, 4, "\x01\x00\x00\x00\xff\xd8\xff", 7) &&
		dbuf_memcmp(f, 4, "\x00\x00\x00\x00\x55\x49\x00\x00", 8))
	{
		// Not an "Images" stream we recognize.
		// TODO: Can we detect this if the first image is in "simple"
		// format?
		goto done;
	}

	// This is an object found in Corel Print House (.CPH) and similar files.
	copy_any_stream_to_dbuf(c, d, dei, 16, dei->stream_size-16, f);
	do_CorelImages_internal(c, d, dei, f);

done:
	dbuf_close(f);
}

static void dbg_timestamp(deark *c, struct de_timestamp *ts, const char *field_name)
{
	char timestamp_buf[64];

	if(ts->is_valid) {
		de_timestamp_to_string(ts, timestamp_buf, sizeof(timestamp_buf), 0);
		de_dbg(c, "%s: %s", field_name, timestamp_buf);
	}
}

static void read_and_cvt_timestamp(deark *c, dbuf *f, i64 pos,
	struct de_timestamp *ts)
{
	i64 ts_as_FILETIME;

	de_zeromem(ts, sizeof(struct de_timestamp));
	ts_as_FILETIME = dbuf_geti64le(f, pos);
	if(ts_as_FILETIME!=0) {
		de_FILETIME_to_timestamp(ts_as_FILETIME, ts, 0x1);
	}
}

static int read_thumbsdb_catalog(deark *c, lctx *d, struct dir_entry_info *dei)
{
	i64 item_len;
	i64 n;
	i64 i;
	i64 pos;
	int retval = 0;
	dbuf *catf = NULL;

	if(d->thumbsdb_catalog) return 0; // Already read a catalog

	de_dbg(c, "reading thumbsdb catalog");
	de_dbg_indent(c, 1);

	catf = dbuf_create_membuf(c, dei->stream_size, 0);
	copy_any_stream_to_dbuf(c, d, dei, 0, dei->stream_size, catf);

	item_len = dbuf_getu16le(catf, 0);
	de_dbg(c, "header size: %d", (int)item_len); // (?)
	if(item_len!=16) goto done;

	n = dbuf_getu16le(catf, 2);
	de_dbg(c, "catalog version: %d", (int)n); // (?)
	if(n!=5 && n!=6 && n!=7) {
		de_warn(c, "Unsupported Catalog version: %d", (int)n);
		goto done;
	}

	d->thumbsdb_catalog_num_entries = dbuf_getu16le(catf, 4); // This might really be a 4 byte int.
	de_dbg(c, "num entries: %d", (int)d->thumbsdb_catalog_num_entries);
	if(d->thumbsdb_catalog_num_entries>2048)
		d->thumbsdb_catalog_num_entries = 2048;

	d->thumbsdb_catalog = de_mallocarray(c, d->thumbsdb_catalog_num_entries,
		sizeof(struct thumbsdb_catalog_entry));

	pos = item_len;

	for(i=0; i<d->thumbsdb_catalog_num_entries; i++) {
		i64 name_len;

		if(pos >= catf->len) goto done;
		item_len = dbuf_getu32le(catf, pos);
		de_dbg(c, "catalog entry #%d, len=%d", (int)i, (int)item_len);
		if(item_len<20) goto done;
		if(pos+item_len > catf->len) goto done;

		de_dbg_indent(c, 1);

		d->thumbsdb_catalog[i].id = (u32)dbuf_getu32le(catf, pos+4);
		de_dbg(c, "id: %u", (unsigned int)d->thumbsdb_catalog[i].id);

		read_and_cvt_timestamp(c, catf, pos+8, &d->thumbsdb_catalog[i].mod_time);
		dbg_timestamp(c, &d->thumbsdb_catalog[i].mod_time, "timestamp");

		name_len = de_min_int(item_len-20, 65536);
		d->thumbsdb_catalog[i].fname_srd = dbuf_read_string(catf, pos+16, name_len, name_len,
			DE_CONVFLAG_WANT_UTF8, DE_ENCODING_UTF16LE);
		de_dbg(c, "name: \"%s\"", ucstring_getpsz(d->thumbsdb_catalog[i].fname_srd->str));

		de_dbg_indent(c, -1);

		pos += item_len;
	}

	retval = 1;
done:
	de_dbg_indent(c, -1);
	dbuf_close(catf);
	if(!retval) {
		d->thumbsdb_catalog_num_entries = 0; // Make sure we don't use a bad catalog.
	}
	return retval;
}

static void do_cfb_olepropertyset(deark *c, lctx *d, struct dir_entry_info *dei,
	int is_summaryinfo, int is_root)
{
	dbuf *f = NULL;
	int saved_indent_level;

	if(dei->stream_size>1000000) goto done;
	f = dbuf_create_membuf(c, dei->stream_size, 1);
	copy_any_stream_to_dbuf(c, d, dei, 0, dei->stream_size, f);

	de_dbg_indent_save(c, &saved_indent_level);
	if(is_summaryinfo) {
		de_dbg(c, "SummaryInformation (%s)", is_root?"root":"non-root");
	}
	else {
		de_dbg(c, "property set stream");
	}

	de_dbg_indent(c, 1);
	de_run_module_by_id_on_slice(c, "olepropset", NULL, f, 0, f->len);
	de_dbg_indent(c, -1);

done:
	dbuf_close(f);
}

static void read_mini_sector_stream(deark *c, lctx *d, i64 first_sec_id, i64 stream_size)
{
	if(d->mini_sector_stream) return; // Already done

	de_dbg(c, "reading mini sector stream (%d bytes)", (int)stream_size);
	d->mini_sector_stream = dbuf_create_membuf(c, 0, 0);
	copy_normal_stream_to_dbuf(c, d, first_sec_id, 0, stream_size, d->mini_sector_stream);
}

// Reads the directory stream into d->dir, and sets d->num_dir_entries.
static void read_directory_stream(deark *c, lctx *d)
{
	i64 dir_sec_id;
	i64 dir_sector_offs;
	i64 num_entries_per_sector;
	i64 dir_sector_count = 0;

	de_dbg(c, "reading directory stream");
	de_dbg_indent(c, 1);

	d->dir = dbuf_create_membuf(c, 0, 0);

	dir_sec_id = d->first_dir_sec_id;

	num_entries_per_sector = d->sec_size / 128;
	d->num_dir_entries = 0;

	// TODO: Use copy_normal_stream_to_dbuf
	while(1) {
		if(dir_sec_id<0) break;
		if(d->dir->len > c->infile->len) break;

		dir_sector_offs = sec_id_to_offset(c, d, dir_sec_id);

		de_dbg(c, "directory sector #%d SecID=%d (offs=%d), entries %d-%d",
			(int)dir_sector_count,
			(int)dir_sec_id, (int)dir_sector_offs,
			(int)d->num_dir_entries, (int)(d->num_dir_entries + num_entries_per_sector - 1));

		dbuf_copy(c->infile, dir_sector_offs, d->sec_size, d->dir);

		d->num_dir_entries += num_entries_per_sector;

		dir_sec_id = get_next_sec_id(c, d, dir_sec_id);
		dir_sector_count++;
	}

	de_dbg(c, "number of directory entries: %d", (int)d->num_dir_entries);

	de_dbg_indent(c, -1);
}

static void do_init_format_detection(deark *c, lctx *d)
{
	if(d->subformat_req!=SUBFMT_AUTO) return;
	d->could_be_thumbsdb = 1;
}

static void do_finalize_format_detection(deark *c, lctx *d)
{
	d->subformat_final = SUBFMT_RAW; // default

	if(d->subformat_req!=SUBFMT_AUTO) {
		d->subformat_final = d->subformat_req;
		goto done;
	}

	if(!d->could_be_thumbsdb) goto done;

	if(d->thumbsdb_old_names_found>0 && !d->thumbsdb_catalog_found)
	{
		d->could_be_thumbsdb = 0;
	}
	else if(d->thumbsdb_old_names_found + d->thumbsdb_new_names_found +
		d->thumbsdb_catalog_found < 1)
	{
		d->could_be_thumbsdb = 0;
	}

	if(d->could_be_thumbsdb) {
		d->subformat_final = SUBFMT_THUMBSDB;
	}

done:
	switch(d->subformat_final) {
	case SUBFMT_THUMBSDB:
		de_declare_fmt(c, "Thumbs.db");
		d->thumbsdb_msrgba_mode = de_get_ext_option_bool(c, "cfb:msrgbamode", 1);
		break;
	}
}

static void do_dump_dir_structure(deark *c, lctx *d)
{
	i64 i;

	de_dbg(c, "dir structure:");
	de_dbg_indent(c, 1);
	for(i=0; i<d->num_dir_entries; i++) {
		de_dbg(c, "[%d] t=%d p=%d c=%d s=%d,%d", (int)i,
			(int)d->dir_entry[i].entry_type,
			(int)d->dir_entry[i].parent_id,
			(int)d->dir_entry[i].child_id,
			(int)d->dir_entry[i].sibling_id[0],
			(int)d->dir_entry[i].sibling_id[1]);
		de_dbg_indent(c, 1);
		if(d->dir_entry[i].fname_srd && d->dir_entry[i].fname_srd->str) {
			de_dbg(c, "fname: \"%s\"",
				ucstring_getpsz(d->dir_entry[i].fname_srd->str));
		}
		if(d->dir_entry[i].path) {
			de_dbg(c, "path: \"%s\"",
				ucstring_getpsz(d->dir_entry[i].path));
		}
		de_dbg_indent(c, -1);
	}
	de_dbg_indent(c, -1);
}

static void do_mark_dir_entries_recursively(deark *c, lctx *d, i32 parent_id,
	i32 dir_entry_idx, int level)
{
	struct dir_entry_info *dei;
	int k;

	if(dir_entry_idx<0 || (i64)dir_entry_idx>=d->num_dir_entries) return;

	dei = &d->dir_entry[dir_entry_idx];

	if(dei->entry_type!=OBJTYPE_STORAGE && dei->entry_type!=OBJTYPE_STREAM) return;

	dei->parent_id = parent_id;

	if(dei->entry_type==OBJTYPE_STORAGE && dei->fname_srd && dei->fname_srd->str && !dei->path) {
		// Set the full pathname
		dei->path = ucstring_create(c);
		if(parent_id>0 && d->dir_entry[parent_id].path) {
			ucstring_append_ucstring(dei->path, d->dir_entry[parent_id].path);
			ucstring_append_sz(dei->path, "/", DE_ENCODING_LATIN1);
		}
		ucstring_append_ucstring(dei->path, dei->fname_srd->str);
	}

	if(level>50) return;
	for(k=0; k<2; k++) {
		do_mark_dir_entries_recursively(c, d, parent_id, dei->sibling_id[k], level+1);
	}

	if(dei->entry_type==OBJTYPE_STORAGE) {
		// This is a "subdirectory" entry, so examine its children (starting with the
		// one that we know about).
		do_mark_dir_entries_recursively(c, d, dir_entry_idx, dei->child_id, level+1);
	}
}

// Figure out which entries are in the root directory.
static void do_analyze_dir_structure(deark *c, lctx *d)
{
	de_dbg_indent(c, 1);

	if(d->dump_dir_structure) do_dump_dir_structure(c, d);

	if(d->num_dir_entries<1) goto done;

	// The first entry should be the root entry.
	if(d->dir_entry[0].entry_type!=OBJTYPE_ROOT_STORAGE) goto done;

	// Its child is one of the entries in the root directory. Start with it.
	do_mark_dir_entries_recursively(c, d, 0, d->dir_entry[0].child_id, 0);

	if(d->dump_dir_structure) do_dump_dir_structure(c, d);
done:
	de_dbg_indent(c, -1);
}

// Things to do after we've read the directory stream into memory, and
// know how many entries there are.
static void do_before_reading_directory_entries(deark *c, lctx *d)
{
	i64 i;

	// Stores some extra information for each directory entry, and a copy of
	// some information for convenience.
	// (The original entry is still available at d->dir[128*n].)
	d->dir_entry = de_mallocarray(c, d->num_dir_entries, sizeof(struct dir_entry_info));

	// Set defaults for each entry
	for(i=0; i<d->num_dir_entries; i++) {
		d->dir_entry[i].child_id = -1;
		d->dir_entry[i].sibling_id[0] = -1;
		d->dir_entry[i].sibling_id[1] = -1;
	}
}

static int is_thumbsdb_orig_name(deark *c, lctx *d, const char *name, size_t nlen)
{
	size_t i;

	if(nlen<1 || nlen>6) return 0;
	for(i=0; i<nlen; i++) {
		if(name[i]<'0' || name[i]>'9') return 0;
	}
	return 1;
}

static int is_thumbsdb_new_name(deark *c, lctx *d, const char *name, size_t nlen)
{
	size_t i;
	int count1 = 0;
	int found_underscore = 0;
	int count2 = 0;

	if(nlen<4 || nlen>22) return 0;
	for(i=0; i<nlen; i++) {
		if(!found_underscore && name[i]=='_') {
			found_underscore = 1;
		}
		else if(!found_underscore) {
			// pre-underscore (pixel dimension)
			if(name[i]>='0' && name[i]<='9')
				count1++;
			else
				return 0;
		}
		else {
			// post-underscore (hash?)
			if((name[i]>='0' && name[i]<='9') ||
				(name[i]>='a' && name[i]<='f'))
			{
				count2++;
			}
			else
			{
				return 0;
			}
		}
	}
	if(!found_underscore) return 0;
	if(count1<1 || count1>5) return 0;
	if(count2<1 || count2>16) return 0;
	return 1;
}

static void do_per_dir_entry_format_detection(deark *c, lctx *d, struct dir_entry_info *dei)
{
	size_t nlen;

	if(dei->entry_type==OBJTYPE_EMPTY) return;
	if(d->subformat_req!=SUBFMT_AUTO) return;
	if(!d->could_be_thumbsdb) return;

	if(dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		if(de_memcmp(dei->clsid, EMPTY_CLSID, 16)) {
			d->could_be_thumbsdb = 0;
			return;
		}
	}

	if(dei->entry_type==OBJTYPE_STORAGE) {
		// Thumbs.db files aren't expected to have any Storage objects.
		d->could_be_thumbsdb = 0;
		return;
	}
	if(dei->entry_type!=OBJTYPE_STREAM) {
		return;
	}

	nlen = dei->fname_srd->sz_utf8_strlen;
	if(nlen<1 || nlen>21) {
		d->could_be_thumbsdb = 0;
		return;
	}

	if(dei->is_thumbsdb_catalog) {
		d->thumbsdb_catalog_found++;
		return;
	}

	if(is_thumbsdb_orig_name(c, d, dei->fname_srd->sz_utf8, nlen)) {
		d->thumbsdb_old_names_found++;
		return;
	}

	if(is_thumbsdb_new_name(c, d, dei->fname_srd->sz_utf8, nlen)) {
		d->thumbsdb_new_names_found++;
		return;
	}
}

// Caller supplies and initializes buf
static void identify_clsid(deark *c, lctx *d, const u8 *clsid, char *buf, size_t buflen)
{
	const char *name = "?";
	size_t i;

	for(i=0; i<DE_ARRAYCOUNT(known_clsids); i++) {
		u8 tmpclsid[16];
		unsigned int k;
		const struct clsid_id_struct *ci = &known_clsids[i];

		de_memcpy(tmpclsid, clsid, 16);
		for(k=0; k<16; k++) {
			if((ci->mask & (1<<(15-k)))==0) {
				tmpclsid[k] = 0x00;
			}
		}
		if(!de_memcmp(tmpclsid, ci->clsid, 16)) {
			name = ci->name;
			break;
		}
	}
	de_snprintf(buf, buflen, " (%s)", name);
}

static void do_process_stream(deark *c, lctx *d, struct dir_entry_info *dei)
{
	int saved_indent_level;
	de_finfo *fi_raw = NULL; // Use this if we extract the raw stream
	de_finfo *fi_tmp = NULL; // Can be used by format-specific code
	de_ucstring *fn_raw = NULL; // Use this if we extract the raw stream
	de_ucstring *fn_tmp = NULL; // Can be used by format-specific code
	dbuf *firstpart = NULL;
	int is_thumbsdb_stream = 0;
	int is_OfficeArtStream = 0;
	int is_summaryinfo = 0;
	int is_propset = 0;
	int is_namedThumbnail = 0;
	int is_namedImages = 0;
	int is_root = (dei->parent_id==0);

	de_dbg_indent_save(c, &saved_indent_level);

	// By default, use the "stream name" as the filename.
	fn_raw = ucstring_create(c);

	if(dei->parent_id>0 && d->dir_entry[dei->parent_id].path) {
		ucstring_append_ucstring(fn_raw, d->dir_entry[dei->parent_id].path);
		ucstring_append_sz(fn_raw, "/", DE_ENCODING_LATIN1);
	}

	ucstring_append_ucstring(fn_raw, dei->fname_srd->str);
	fn_tmp = ucstring_clone(fn_raw);

	fi_raw = de_finfo_create(c);
	fi_tmp = de_finfo_create(c);

	// By default, use the mod time from the directory entry.
	if(dei->mod_time.is_valid) {
		fi_raw->timestamp[DE_TIMESTAMPIDX_MODIFY] = dei->mod_time; // struct copy
		fi_tmp->timestamp[DE_TIMESTAMPIDX_MODIFY] = dei->mod_time; // struct copy
	}

	if(d->extract_raw_streams) {
		dbuf *outf = NULL;

		de_finfo_set_name_from_ucstring(c, fi_raw, fn_raw, DE_SNFLAG_FULLPATH);
		fi_raw->original_filename_flag = 1;

		outf = dbuf_create_output_file(c, NULL, fi_raw, 0);
		copy_any_stream_to_dbuf(c, d, dei, 0, dei->stream_size, outf);
		dbuf_close(outf);
	}

	if(!d->decode_streams) goto done;

	// Read the first part of the stream, to use for format detection.
	firstpart = dbuf_create_membuf(c, 256, 0x1);
	copy_any_stream_to_dbuf(c, d, dei, 0,
		(dei->stream_size>256)?256:dei->stream_size, firstpart);

	// Stream type detection

	// FIXME? The stream detection happens even if d->subformat_req==SUBFMT_RAW.
	// We probably should have different detection logic in that case.

	if(!de_strcasecmp(dei->fname_srd->sz_utf8, "\x05" "SummaryInformation")) {
		is_propset = 1;
		is_summaryinfo = 1;
	}
	else if(!de_strncmp(dei->fname_srd->sz_utf8, "\x05", 1)) {
		// TODO: Is there a good way to tell whether a stream is a property set?
		is_propset = 1;
	}
	else if(d->subformat_final==SUBFMT_TIFF37680 &&
		!de_strcasecmp(dei->fname_srd->sz_utf8, "CONTENTS"))
	{
		// TODO: This is not the only place to find a "CONTENTS" stream.
		is_propset = 1;
	}
	else if(d->subformat_final==SUBFMT_THUMBSDB) {
		is_thumbsdb_stream = 1;
	}
	else if(!de_strcasecmp(dei->fname_srd->sz_utf8, "Pictures")) {
		// This stream often appears in PPT documents.
		is_OfficeArtStream = 1;
	}
	else if(!de_strcasecmp(dei->fname_srd->sz_utf8, "EscherStm")) {
		is_OfficeArtStream = 1;
	}
	else if(!de_strcasecmp(dei->fname_srd->sz_utf8, "EscherDelayStm")) {
		// Found in MS Publisher, and probably other formats.
		is_OfficeArtStream = 1;
	}
	else if(!de_strcasecmp(dei->fname_srd->sz_utf8, "Thumbnail")) {
		is_namedThumbnail = 1;
	}
	else if(!de_strcasecmp(dei->fname_srd->sz_utf8, "Images")) {
		is_namedImages = 1;
	}

	if(is_OfficeArtStream) {
		unsigned int rectype;
		rectype = (unsigned int)dbuf_getu16le(firstpart, 2);
		if((rectype&0xf000)!=0xf000) {
			is_OfficeArtStream = 0;
		}
	}

	// End of stream type detection

	if(is_propset) {
		do_cfb_olepropertyset(c, d, dei, is_summaryinfo, is_root);
	}
	else if(is_thumbsdb_stream) {
		do_extract_stream_to_file_thumbsdb(c, d, dei, fi_tmp, fn_tmp, firstpart);
	}
	else if(is_OfficeArtStream) {
		do_OfficeArtStream(c, d, dei);
	}
	else if(is_namedThumbnail) {
		do_StreamNamedThumbnail(c, d, dei);
	}
	else if(is_namedImages) {
		do_StreamNamedImages(c, d, dei);
	}

done:
	de_dbg_indent_restore(c, saved_indent_level);
	dbuf_close(firstpart);
	ucstring_destroy(fn_raw);
	ucstring_destroy(fn_tmp);
	de_finfo_destroy(c, fi_raw);
	de_finfo_destroy(c, fi_tmp);
}


// Read information about a directory entry. Do not print anything about it.
static void do_read_dir_entry(deark *c, lctx *d, i64 dir_entry_idx, i64 dir_entry_offs)
{
	i64 name_len_bytes;
	i64 raw_sec_id;
	struct dir_entry_info *dei = NULL;

	if(!d->dir_entry) goto done; // error
	dei = &d->dir_entry[dir_entry_idx];

	dei->pass = 3; // Default pass in which to process this entry

	dei->entry_type = dbuf_getbyte(d->dir, dir_entry_offs+66);
	switch(dei->entry_type) {
	case OBJTYPE_EMPTY: dei->entry_type_name="empty"; break;
	case OBJTYPE_STORAGE: dei->entry_type_name="storage object"; break;
	case OBJTYPE_STREAM: dei->entry_type_name="stream"; break;
	case OBJTYPE_ROOT_STORAGE: dei->entry_type_name="root storage object"; break;
	default: dei->entry_type_name="?";
	}

	if(dei->entry_type==OBJTYPE_EMPTY) goto done;

	dei->name_len_raw = dbuf_getu16le(d->dir, dir_entry_offs+64);

	name_len_bytes = dei->name_len_raw-2; // Ignore the trailing U+0000
	if(name_len_bytes<0) name_len_bytes = 0;

	dei->fname_srd = dbuf_read_string(d->dir, dir_entry_offs, name_len_bytes, name_len_bytes,
		DE_CONVFLAG_WANT_UTF8, DE_ENCODING_UTF16LE);

	dei->node_color = dbuf_getbyte(d->dir, dir_entry_offs+67);

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_STREAM) {
		dei->sibling_id[0] = (i32)dbuf_geti32le(d->dir, dir_entry_offs+68);
		dei->sibling_id[1] = (i32)dbuf_geti32le(d->dir, dir_entry_offs+72);
	}

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		dei->child_id = (i32)dbuf_geti32le(d->dir, dir_entry_offs+76);
	}

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		dbuf_read(d->dir, dei->clsid, dir_entry_offs+80, 16);
		fmtutil_guid_to_uuid(dei->clsid);
	}

	read_and_cvt_timestamp(c, d->dir, dir_entry_offs+108, &dei->mod_time);

	raw_sec_id = dbuf_geti32le(d->dir, dir_entry_offs+116);

	if(d->major_ver<=3) {
		dei->stream_size = dbuf_getu32le(d->dir, dir_entry_offs+120);
	}
	else {
		dei->stream_size = dbuf_geti64le(d->dir, dir_entry_offs+120);
		de_sanitize_length(&dei->stream_size);
	}

	dei->is_mini_stream = (dei->entry_type==OBJTYPE_STREAM) && (dei->stream_size < d->std_stream_min_size);

	if(dei->is_mini_stream) {
		dei->minisec_id = raw_sec_id;
	}
	else {
		dei->normal_sec_id = raw_sec_id;
	}

	if((d->subformat_req==SUBFMT_THUMBSDB || d->subformat_req==SUBFMT_AUTO) &&
		!de_strcmp(dei->fname_srd->sz_utf8, "Catalog"))
	{
		dei->is_thumbsdb_catalog = 1;
		if(d->decode_streams) dei->pass = 2;
	}

	if(dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		dei->pass = 1;
	}

	do_per_dir_entry_format_detection(c, d, dei);

done:
	;
}

// Process an directory entry from the d->dir stream, that has previously been
// read into the d->dir_entry array.
static void do_process_dir_entry(deark *c, lctx *d, i64 dir_entry_idx)
{
	struct dir_entry_info *dei = NULL;
	char clsid_string[50];
	char buf[80];

	if(!d->dir_entry) return; // error
	dei = &d->dir_entry[dir_entry_idx];

	de_dbg(c, "type: 0x%02x (%s)", (unsigned int)dei->entry_type, dei->entry_type_name);
	if(dei->entry_type==OBJTYPE_EMPTY) goto done;

	de_dbg2(c, "name len: %d bytes", (int)dei->name_len_raw);
	de_dbg(c, "name: \"%s\"", ucstring_getpsz(dei->fname_srd->str));
	de_dbg(c, "node color: %u", (unsigned int)dei->node_color);

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_STREAM) {
		de_dbg(c, "sibling StreamIDs: %d, %d", (int)dei->sibling_id[0], (int)dei->sibling_id[1]);
	}

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		de_dbg(c, "child StreamID: %d", (int)dei->child_id);
	}

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_STREAM) {
		de_dbg(c, "parent: %d", (int)dei->parent_id);
	}

	if(dei->entry_type==OBJTYPE_STORAGE || dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		buf[0] = '\0';
		if(dei->entry_type==OBJTYPE_ROOT_STORAGE) {
			identify_clsid(c, d, dei->clsid, buf, sizeof(buf));
		}

		fmtutil_render_uuid(c, dei->clsid, clsid_string, sizeof(clsid_string));
		de_dbg(c, "%sclsid: {%s}%s", (dei->entry_type==OBJTYPE_ROOT_STORAGE)?"root ":"",
			clsid_string, buf);
	}

	dbg_timestamp(c, &dei->mod_time, "mod time");

	de_dbg(c, "stream size: %"I64_FMT, dei->stream_size);

	if(dei->is_mini_stream) {
		de_dbg(c, "first MiniSecID: %d", (int)dei->minisec_id);
	}
	else {
		describe_sec_id(c, d, dei->normal_sec_id, buf, sizeof(buf));
		de_dbg(c, "first SecID: %d (%s)", (int)dei->normal_sec_id, buf);
	}

	if(dei->entry_type==OBJTYPE_ROOT_STORAGE) {
		read_mini_sector_stream(c, d, dei->normal_sec_id, dei->stream_size);
	}
	else if(dei->is_thumbsdb_catalog && d->decode_streams) {
		// TODO: Move this to do_process_stream()?
		read_thumbsdb_catalog(c, d, dei);
	}
	else if(dei->entry_type==OBJTYPE_STREAM) {
		do_process_stream(c, d, dei);
	}

done:
	;
}

static void do_directory(deark *c, lctx *d)
{
	i64 i;
	int pass;
	int saved_indent_level;

	de_dbg_indent_save(c, &saved_indent_level);

	de_dbg(c, "reading directory entries");
	do_before_reading_directory_entries(c, d);
	for(i=0; i<d->num_dir_entries; i++) {
		i64 dir_entry_offs = 128*i;
		do_read_dir_entry(c, d, i, dir_entry_offs);
	}

	de_dbg(c, "decoding directory structure");
	do_analyze_dir_structure(c, d);

	de_dbg(c, "detecting format");
	do_finalize_format_detection(c, d);

	de_dbg(c, "processing directory entries");
	de_dbg_indent(c, 1);
	for(pass=1; pass<=3; pass++) {
		de_dbg2(c, "[pass %d]", pass);
		for(i=0; i<d->num_dir_entries; i++) {
			if(d->dir_entry[i].pass == pass) {
				de_dbg(c, "directory entry, StreamID=%d", (int)i);
				de_dbg_indent(c, 1);
				do_process_dir_entry(c, d, i);
				de_dbg_indent(c, -1);
			}
		}
	}

	de_dbg_indent_restore(c, saved_indent_level);
}

static void de_run_cfb_internal(deark *c, lctx *d)
{
	do_init_format_detection(c, d);

	if(!do_header(c, d)) {
		goto done;
	}

	read_difat(c, d);

	read_fat(c, d);

	read_minifat(c, d);

	read_directory_stream(c, d);

	do_directory(c, d);

done:
	dbuf_close(d->difat);
	dbuf_close(d->fat);
	dbuf_close(d->minifat);
	dbuf_close(d->dir);
	if(d->dir_entry) {
		i64 k;
		for(k=0; k<d->num_dir_entries; k++) {
			de_destroy_stringreaderdata(c, d->dir_entry[k].fname_srd);
			ucstring_destroy(d->dir_entry[k].path);
		}
		de_free(c, d->dir_entry);
	}
	dbuf_close(d->mini_sector_stream);
	if(d->thumbsdb_catalog) {
		i64 k;
		for(k=0; k<d->thumbsdb_catalog_num_entries; k++) {
			de_destroy_stringreaderdata(c, d->thumbsdb_catalog[k].fname_srd);
		}
		de_free(c, d->thumbsdb_catalog);
		d->thumbsdb_catalog = NULL;
	}
}

static void de_run_cfb(deark *c, de_module_params *mparams)
{
	lctx *d = NULL;
	const char *cfbfmt_opt;

	d = de_malloc(c, sizeof(lctx));
	d->decode_streams = 1;
	d->subformat_req = SUBFMT_AUTO;

	if(de_get_ext_option(c, "cfb:extractstreams")) {
		d->extract_raw_streams = 1;
		d->decode_streams = 0;
	}
	if(de_get_ext_option(c, "cfb:dumpdir")) {
		d->dump_dir_structure = 1; // A low-level debugging feature
	}

	if(de_havemodcode(c, mparams, 'T')) {
		// TIFF tag 37680 mode
		// TODO: Handle 'OLE Property Set Storage' more generally.
		d->subformat_req = SUBFMT_TIFF37680;
	}

	if(d->subformat_req == SUBFMT_AUTO) {
		// If we haven't set subformat_req yet, look at the command-line option

		cfbfmt_opt = de_get_ext_option(c, "cfb:fmt");
		if(cfbfmt_opt) {
			if(!de_strcmp(cfbfmt_opt, "auto")) {
				d->subformat_req = SUBFMT_AUTO;
			}
			else if(!de_strcmp(cfbfmt_opt, "raw")) {
				d->subformat_req = SUBFMT_RAW;
			}
			else if(!de_strcmp(cfbfmt_opt, "thumbsdb")) {
				d->subformat_req = SUBFMT_THUMBSDB;
			}
		}
	}

	de_run_cfb_internal(c, d);

	de_free(c, d);
}

static int de_identify_cfb(deark *c)
{
	if(!dbuf_memcmp(c->infile, 0, "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", 8))
		return 100;
	return 0;
}

static void de_help_cfb(deark *c)
{
	de_msg(c, "-opt cfb:extractstreams : Extract raw streams, instead of decoding");
	de_msg(c, "-opt cfb:fmt=raw : Do not try to detect the document type");
	de_msg(c, "-opt cfb:fmt=thumbsdb : Assume Thumbs.db format");
	de_msg(c, "-opt cfb:msrgbamode=0 : Disable special processing of nonstandard-"
		"JPEG Thumbs.db thumbnails");
}

void de_module_cfb(deark *c, struct deark_module_info *mi)
{
	mi->id = "cfb";
	mi->desc = "Microsoft Compound File Binary File";
	mi->run_fn = de_run_cfb;
	mi->identify_fn = de_identify_cfb;
	mi->help_fn = de_help_cfb;
}
