// This file is part of Deark.
// Copyright (C) 2019 Jason Summers
// See the file COPYING for terms of use.

// ARC compressed archive
// Spark
// ArcMac

#include <deark-config.h>
#include <deark-private.h>
#include <deark-fmtutil.h>
DE_DECLARE_MODULE(de_module_arc);
DE_DECLARE_MODULE(de_module_spark);
DE_DECLARE_MODULE(de_module_arcmac);

#define FMT_ARC 1
#define FMT_SPARK 2
#define FMT_ARCMAC 3
#define FMT_PAK16SFX 4

#define MAX_NESTING_LEVEL 24

struct localctx_struct;
typedef struct localctx_struct lctx;
struct member_data;
typedef void (*decompressor_fn)(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri, struct de_dfilter_out_params *dcmpro,
	struct de_dfilter_results *dres);

struct cmpr_meth_info {
	u8 cmpr_meth;
	unsigned int flags;
	const char *name;
	decompressor_fn decompressor;
};

struct persistent_member_data {
	de_ucstring *comment;
	de_ucstring *path;
};

struct member_data {
	deark *c;
	lctx *d;

	u8 cmpr_meth;
	u8 cmpr_meth_masked;
	u8 has_spark_attribs;
	const struct cmpr_meth_info *cmi;
	const char *cmpr_meth_name;
	i64 orig_size;
	i64 cmpr_data_pos;
	i64 cmpr_size;
	u32 crc_reported;
	u32 crc_calc;
	de_ucstring *fn;
	struct de_timestamp arc_timestamp;
	struct de_riscos_file_attrs rfa;
	int is_dir;

	i64 arcmac_dforklen;
	i64 arcmac_rforklen;
	struct de_stringreaderdata *arcmac_fn;
	struct de_advfile *arcmac_advf;
};

struct localctx_struct {
	int fmt;
	const char *fmtname;
	de_ext_encoding input_encoding_for_filenames;
	de_ext_encoding input_encoding_for_comments;
	de_ext_encoding input_encoding_for_arcmac_fn;
	u8 method10; // 1=trimmed, 2=crushed
	int recurse_subdirs;
	u8 sig_byte;
	u8 prescan_found_eoa;
	u8 has_trailer_data;
	u8 has_pak_trailer;
	u8 has_arc_extensions;
	i64 prescan_pos_after_eoa;
	i64 num_top_level_members; // Not including EOA marker
	i64 end_of_known_data;
	struct de_crcobj *crco;
	struct de_strarray *curpath;
	struct persistent_member_data *persistent_md; // optional array[num_top_level_members]
	dbuf *pak16sfx_outf;
};

struct member_parser_data {
	int nesting_level;
	int member_idx;
	u8 magic;
	u8 cmpr_meth, cmpr_meth_masked;
	i64 member_pos;
	i64 member_len;
	i64 cmpr_data_pos;
	i64 cmpr_data_len;
};

typedef void (*member_cb_type)(deark *c, lctx *d, struct member_parser_data *mpd);

// Look for any known ARC cmpr meth, including end-of-archive marker.
// Not for Spark format
static int is_known_cmpr_meth(u8 m)
{
	if(m<=11 || m==20 || m==21 || m==22 || m==30) {
		return 1;
	}
	return 0;
}

// Is this a plausible cmpr meth for the first archive member
// (including end-of-archive marker)?
// (Not for Spark format.)
static int is_known_first_cmpr_meth(u8 m)
{
	if((m<=11) || m==20) {
		return 1;
	}
	return 0;
}

// For ARC, not for Spark or ArcMac.
// It's assumed that the byte at pos1 is known to be 0x1a.
// Will validate the first member, and (unless it is the end marker,
// or strictness==0), the first two bytes of the second member.
static int is_valid_file_at(dbuf *f, i64 pos1, i64 endpos, UI strictness)
{
	u8 marker;
	u8 cmpr_meth;
	i64 cmpr_len;
	i64 pos = pos1;

	pos++;

	cmpr_meth = dbuf_getbyte_p(f, &pos);
	if(!is_known_first_cmpr_meth(cmpr_meth)) {
		return 0;
	}
	if(cmpr_meth==0) return 1; // End marker

	if(cmpr_meth<20 && strictness>=2) {
		u8 f1;

		// test 1st char of filename
		f1 = dbuf_getbyte(f, pos);
		if(f1<0x20) return 0;
	}
	pos += 13;

	if(strictness==0) return 1;

	cmpr_len = dbuf_getu32le_p(f, &pos);
	pos += 6;
	if(cmpr_meth!=1) pos += 4;
	pos += cmpr_len;
	if(pos==endpos) return 1;
	if(pos>endpos-2) return 0;

	marker = dbuf_getbyte_p(f, &pos);
	if(marker!=0x1a) return 0;
	cmpr_meth = dbuf_getbyte_p(f, &pos);
	if(!is_known_cmpr_meth(cmpr_meth)) return 0;
	return 1;
}

static void mark_end_of_known_data(lctx *d, i64 pos)
{
	if(pos > d->end_of_known_data) {
		d->end_of_known_data = pos;
	}
}

// Calls the supplied callback function for each ARC member found.
// Also called for end-of-archive/directory markers.
// Also called if unexpected data is encountered (with mpd->magic != 0x1a).
static void parse_member_sequence(deark *c, lctx *d, i64 pos1, i64 len, int nesting_level,
	member_cb_type member_cbfn)
{
	struct member_parser_data *mpd = NULL;
	int member_idx = 0;
	i64 pos = pos1;

	mpd = de_malloc(c, sizeof(struct member_parser_data));

	while(1) {
		if(pos+2 > pos1+len) break;
		de_zeromem(mpd, sizeof(struct member_parser_data));
		mpd->nesting_level = nesting_level;
		mpd->member_idx = member_idx++;
		mpd->member_pos = pos;

		mpd->magic = de_getbyte_p(&pos);

		if(mpd->magic==0xfe && d->fmt==FMT_PAK16SFX) {
			// Some hacks here. We're not setting all the fields correctly,
			// just the ones we need.
			mpd->cmpr_data_len = (i64)de_getbyte_p(&pos);
			mpd->cmpr_data_pos = pos;
			mpd->member_len = 2+mpd->cmpr_data_len;
			member_cbfn(c, d, mpd);
			pos = mpd->member_pos + mpd->member_len;
			continue;
		}
		else if(mpd->magic!=d->sig_byte) {
			mpd->member_len = 1;
			mpd->cmpr_data_pos = mpd->member_pos; // dummy value
			member_cbfn(c, d, mpd);
			break;
		}
		mpd->cmpr_meth = de_getbyte_p(&pos);
		mpd->cmpr_meth_masked = mpd->cmpr_meth & 0x7f;

		if(mpd->cmpr_meth_masked==0x00 || mpd->cmpr_meth==0x1f) { // end of archive/dir
			mpd->member_len = 2;
			mpd->cmpr_data_pos = mpd->member_pos+2; // dummy value
			member_cbfn(c, d, mpd);
			break;
		}

		if(d->fmt==FMT_ARCMAC) {
			u8 magic2;

			// TODO: Check for EOF?
			pos += 57; // Skip remainder of 59-byte ArcMac preheader
			magic2 = de_getbyte_p(&pos);
			if(magic2 != 0x1a) { // Error
				// TODO: Call member_cbfn()?
				break;
			}

			// Read the "real" compression method field (should be the same?).
			mpd->cmpr_meth = de_getbyte_p(&pos);
			mpd->cmpr_meth_masked = mpd->cmpr_meth & 0x7f;
		}

		pos += 13;
		mpd->cmpr_data_len = de_getu32le_p(&pos);
		pos += 2+2+2;
		if(mpd->cmpr_meth_masked!=0x01) {
			pos += 4; // original size
		}
		if(mpd->cmpr_meth & 0x80) {
			pos += 12; // Spark-specific data
		}

		mpd->cmpr_data_pos = pos;
		mpd->member_len = mpd->cmpr_data_pos + mpd->cmpr_data_len - mpd->member_pos;
		member_cbfn(c, d, mpd);

		pos = mpd->member_pos + mpd->member_len;
	}

	de_free(c, mpd);
}

static void decompressor_stored(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri, struct de_dfilter_out_params *dcmpro,
	struct de_dfilter_results *dres)
{
	fmtutil_decompress_uncompressed(c, dcmpri, dcmpro, dres, 0);
}

static void decompressor_spark_compressed(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri, struct de_dfilter_out_params *dcmpro,
	struct de_dfilter_results *dres)
{
	struct de_lzw_params delzwp;

	de_zeromem(&delzwp, sizeof(struct de_lzw_params));
	delzwp.fmt = DE_LZWFMT_UNIXCOMPRESS;
	delzwp.flags |= DE_LZWFLAG_HAS1BYTEHEADER;
	fmtutil_decompress_lzw(c, dcmpri, dcmpro, dres, &delzwp);
}

static void decompressor_squashed(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri, struct de_dfilter_out_params *dcmpro,
	struct de_dfilter_results *dres)
{
	struct de_lzw_params delzwp;

	de_zeromem(&delzwp, sizeof(struct de_lzw_params));
	delzwp.fmt = DE_LZWFMT_UNIXCOMPRESS;
	delzwp.max_code_size = 13;
	fmtutil_decompress_lzw(c, dcmpri, dcmpro, dres, &delzwp);
}

static void decompressor_packed(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	fmtutil_decompress_rle90_ex(c, dcmpri, dcmpro, dres, 0);
}

static void decompressor_squeezed(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	struct de_dcmpr_two_layer_params tlp;

	de_zeromem(&tlp, sizeof(struct de_dcmpr_two_layer_params));
	tlp.codec1_type1 = fmtutil_huff_squeeze_codectype1;
	tlp.codec2 = dfilter_rle90_codec;
	tlp.dcmpri = dcmpri;
	tlp.dcmpro = dcmpro;
	tlp.dres = dres;
	de_dfilter_decompress_two_layer(c, &tlp);
}

static void decompressor_crunched5(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	struct de_lzw_params delzwp;

	de_zeromem(&delzwp, sizeof(struct de_lzw_params));
	delzwp.fmt = DE_LZWFMT_ARC5;
	fmtutil_decompress_lzw(c, dcmpri, dcmpro, dres, &delzwp);
}

static void decompressor_crunched6(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	struct de_dcmpr_two_layer_params tlp;
	struct de_lzw_params delzwp;

	de_zeromem(&delzwp, sizeof(struct de_lzw_params));
	delzwp.fmt = DE_LZWFMT_ARC5;

	de_zeromem(&tlp, sizeof(struct de_dcmpr_two_layer_params));
	tlp.codec1_pushable = dfilter_lzw_codec;
	tlp.codec1_private_params = (void*)&delzwp;

	tlp.codec2 = dfilter_rle90_codec;

	tlp.dcmpri = dcmpri;
	tlp.dcmpro = dcmpro;
	tlp.dres = dres;

	de_dfilter_decompress_two_layer(c, &tlp);
}

static void decompressor_crunched8(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	struct de_dcmpr_two_layer_params tlp;
	struct de_lzw_params delzwp;

	// "Crunched" means "packed", then "compressed".
	// So we have to "uncompress" (LZW), then "unpack" (RLE90).

	de_zeromem(&delzwp, sizeof(struct de_lzw_params));
	delzwp.fmt = DE_LZWFMT_UNIXCOMPRESS;
	delzwp.flags |= DE_LZWFLAG_HAS1BYTEHEADER;

	de_zeromem(&tlp, sizeof(struct de_dcmpr_two_layer_params));
	tlp.codec1_pushable = dfilter_lzw_codec;
	tlp.codec1_private_params = (void*)&delzwp;

	tlp.codec2 = dfilter_rle90_codec;

	tlp.dcmpri = dcmpri;
	tlp.dcmpro = dcmpro;
	tlp.dres = dres;

	de_dfilter_decompress_two_layer(c, &tlp);
}

static void decompressor_trimmed(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	struct de_dcmpr_two_layer_params tlp;
	struct de_lh1_params lh1p;

	de_zeromem(&lh1p, sizeof(struct de_lh1_params));
	lh1p.is_arc_trimmed = 1;
	lh1p.history_fill_val = 0x00;

	de_zeromem(&tlp, sizeof(struct de_dcmpr_two_layer_params));
	tlp.codec1_pushable = dfilter_lh1_codec;
	tlp.codec1_private_params = (void*)&lh1p;

	tlp.codec2 = dfilter_rle90_codec;

	tlp.dcmpri = dcmpri;
	tlp.dcmpro = dcmpro;
	tlp.dres = dres;

	de_dfilter_decompress_two_layer(c, &tlp);
}

static void decompressor_distilled(deark *c, lctx *d, struct member_data *md,
	struct de_dfilter_in_params *dcmpri,
	struct de_dfilter_out_params *dcmpro, struct de_dfilter_results *dres)
{
	fmtutil_distilled_codectype1(c, dcmpri, dcmpro, dres, NULL);
}

// Flags:
//  0x01 = valid in ARC
//  0x02 = valid in Spark
//  0x80 = assume high bit of cmpr_meth is set for Spark format
//  0x100, 0x200 = special
static const struct cmpr_meth_info cmpr_meth_info_arr[] = {
	{ 0x00, 0x03, "end of archive marker", NULL },
	{ 0x01, 0x83, "stored (old format)", decompressor_stored },
	{ 0x02, 0x83, "stored", decompressor_stored },
	{ 0x03, 0x83, "packed (RLE)", decompressor_packed },
	{ 0x04, 0x83, "squeezed (RLE + Huffman)", decompressor_squeezed },
	{ 0x05, 0x83, "crunched5 (static LZW)", decompressor_crunched5 },
	{ 0x06, 0x83, "crunched6 (RLE + static LZW)", decompressor_crunched6 },
	{ 0x07, 0x83, "crunched7 (ARC 4.6)", NULL },
	{ 0x08, 0x83, "crunched8 (RLE + dynamic LZW)", decompressor_crunched8 },
	{ 0x09, 0x83, "squashed (dynamic LZW)", decompressor_squashed },
	{ 10,  0x101, "trimmed", decompressor_trimmed },
	{ 10,  0x201, "crushed", NULL },
	{ 10,   0x01, "trimmed or crushed", NULL },
	{ 0x0b, 0x01, "distilled", decompressor_distilled },
	{ 20,   0x01, "archive info", NULL },
	{ 21,   0x01, "extended file info", NULL },
	{ 22,   0x01, "OS info", NULL },
	{ 0x1e, 0x01, "subdir", NULL },
	{ 0x1f, 0x01, "end of subdir marker", NULL },
	{ 0x80, 0x02, "end of archive marker", NULL },
	{ 0xff, 0x02, "compressed", decompressor_spark_compressed }
};

static const struct cmpr_meth_info *get_cmpr_meth_info(lctx *d, u8 cmpr_meth)
{
	size_t k;
	const struct cmpr_meth_info *p;

	for(k=0; k<DE_ARRAYCOUNT(cmpr_meth_info_arr); k++) {
		u8 meth_adjusted;

		p = &cmpr_meth_info_arr[k];
		if(d->fmt==FMT_ARC && !(p->flags & 0x1)) continue;
		if(d->fmt==FMT_SPARK && !(p->flags & 0x2)) continue;
		meth_adjusted = p->cmpr_meth;
		if(d->fmt==FMT_SPARK && (p->flags & 0x80)) {
			meth_adjusted |= 0x80;
		}
		if(meth_adjusted != cmpr_meth) continue;

		if(p->cmpr_meth==10) {
			// Method 10 has a conflict -- it could be either Trimmed (ARC7)
			// or Crushed (PAK).
			if(p->flags&0x100) { // Skip this unless we're sure it's Trimmed
				if(d->method10!=1) {
					if(d->has_pak_trailer || !d->has_arc_extensions) continue;
				}
			}
			else if(p->flags&0x200) { // Skip this unless we're sure it's Crushed
				if(d->method10!=2) {
					if(!d->has_pak_trailer || d->has_arc_extensions) continue;
				}
			}
		}
		return p;
	}
	return NULL;
}

static void read_one_pk_comment(deark *c, lctx *d, i64 pos, de_ucstring *s)
{
	dbuf_read_to_ucstring(c->infile, pos, 32, s, 0, d->input_encoding_for_comments);
	ucstring_strip_trailing_spaces(s);
}

static void init_trailer_data(deark *c, lctx *d)
{
	d->has_trailer_data = 1;
	if(!d->persistent_md) {
		d->persistent_md = de_mallocarray(c, d->num_top_level_members,
			sizeof(struct persistent_member_data));
	}
}

static int is_pkcomment_sig_at(deark *c, lctx *d, i64 pos)
{
	i64 comments_descr_pos;

	if(!d->prescan_found_eoa) return 0;
	if(pos < d->prescan_pos_after_eoa) return 0;
	if(pos > c->infile->len-8) return 0;
	if((UI)de_getu32be(pos) != 0x504baa55U) {
		return 0;
	}
	comments_descr_pos = de_getu32le(pos+4);
	if((comments_descr_pos < d->prescan_pos_after_eoa) ||
		(comments_descr_pos > pos-4))
	{
		return 0;
	}
	return 1;
}

static void do_pk_comments(deark *c, lctx *d)
{
	int found = 0;
	i64 sig_pos;
	i64 comments_descr_pos;
	int has_file_comments = 0;
	int has_archive_comment = 0;
	i64 file_comments_pos = 0;
	de_ucstring *archive_comment = NULL;
	u8 dscr[4];

	if(!d->prescan_found_eoa) return;
	sig_pos = c->infile->len-8;
	if(is_pkcomment_sig_at(c, d, sig_pos)){
		found = 1;
	}

	if(!found) {
		i64 count;

		// The PK signature normally appears 8 bytes from the end of the file,
		// but we'll scan for it, in case the file has padding appended to it.
		// The assumption is that it appears after a whole number of 32-byte
		// records. In case of multiple signatures, we use the earliest one that
		// looks valid.
		sig_pos = d->prescan_pos_after_eoa+32;
		count = 0;

		while(1) {
			if(sig_pos >= c->infile->len-8) break;
			if(count > d->num_top_level_members+10) break;
			if(is_pkcomment_sig_at(c, d, sig_pos)){
				found = 1;
				break;
			}
			sig_pos += 32;
			count++;
		}
	}

	if(!found) return;

	de_dbg(c, "PKARC/PKPAK comment block found, ID at %"I64_FMT, sig_pos);
	de_dbg_indent(c, 1);
	mark_end_of_known_data(d, sig_pos+8);
	init_trailer_data(c, d);

	// Note: This logic is based on reverse engineering, and could be wrong.
	comments_descr_pos = de_getu32le(sig_pos+4);
	de_dbg(c, "descriptor pos: %"I64_FMT, comments_descr_pos);
	// comments_descr_pos has already been validated, by is_pkcomment_sig_at().

	de_read(dscr, comments_descr_pos, 4);
	if(dscr[0]==0x20 && dscr[1]==0x20 && dscr[2]==0x20 && dscr[3]==0x00) {
		has_file_comments = 0;
		has_archive_comment = 1;
	}
	else if(dscr[0]==0x01 && dscr[3]==0x20) {
		has_file_comments = 1;
		has_archive_comment = 0;
	}
	else if(dscr[0]==0x01 && dscr[3]==0x00) {
		has_file_comments = 1;
		has_archive_comment = 1;
	}
	else {
		de_dbg(c, "[unrecognized comments descriptor]");
	}

	if(has_file_comments) {
		file_comments_pos = comments_descr_pos + 32;
		if(sig_pos - file_comments_pos < 32) {
			has_file_comments = 0;
		}
	}

	if(has_archive_comment) {
		archive_comment = ucstring_create(c);
		read_one_pk_comment(c, d, comments_descr_pos-32, archive_comment);
		de_dbg(c, "archive comment: \"%s\"", ucstring_getpsz_d(archive_comment));
	}

	if(has_file_comments) {
		i64 num_file_comments;
		i64 i;

		num_file_comments = (sig_pos - file_comments_pos)/32;
		de_dbg(c, "apparent number of file comments: %d", (int)num_file_comments);

		for(i=0; i<num_file_comments && i<d->num_top_level_members; i++) {
			if(!d->persistent_md[i].comment) {
				d->persistent_md[i].comment = ucstring_create(c);
			}
			if(ucstring_isnonempty(d->persistent_md[i].comment)) continue;
			read_one_pk_comment(c, d,file_comments_pos + i*32, d->persistent_md[i].comment);
		}
	}

	ucstring_destroy(archive_comment);
	de_dbg_indent(c, -1);
}

// Always sets *pbytes_consumed.
// Returns 0 if there are no more records after this.
static int do_pak_ext_record(deark *c, lctx *d, i64 pos1, i64 *pbytes_consumed)
{
	i64 pos = pos1;
	u8 rectype;
	const char *rtname = "?";
	int retval = 0;
	i64 filenum;
	i64 filenum_adj = 0;
	i64 dlen;
	de_ucstring *archive_comment = NULL;
	struct persistent_member_data *pmd = NULL;
	int saved_indent_level;

	de_dbg_indent_save(c, &saved_indent_level);
	*pbytes_consumed = 0;
	if(de_getbyte_p(&pos) != 0xfe) goto done;
	de_dbg(c, "record at %"I64_FMT, pos1);
	de_dbg_indent(c, 1);

	rectype = de_getbyte_p(&pos);
	switch(rectype) {
	case 0: rtname = "end"; break;
	case 1: rtname = "remark"; break;
	case 2: rtname = "path"; break;
	case 3: rtname = "security envelope"; break;
	}
	de_dbg(c, "rectype: %d (%s)", (int)rectype, rtname);
	if(rectype==0) {
		*pbytes_consumed = 2;
		goto done;
	}

	filenum = de_getu16le_p(&pos);
	de_dbg(c, "file num: %d", (int)filenum);
	dlen = de_getu32le_p(&pos);
	de_dbg(c, "dlen: %"I64_FMT, dlen);
	if(pos+dlen > c->infile->len) goto done;

	*pbytes_consumed = 8 + dlen;
	retval = 1;

	if(filenum > 0) {
		filenum_adj = filenum - 1;
		if(filenum_adj < d->num_top_level_members) {
			pmd = &d->persistent_md[filenum_adj];
		}
	}

	if(rectype==1) { // remark
		if(filenum==0) { // archive comment
			archive_comment = ucstring_create(c);
			dbuf_read_to_ucstring_n(c->infile, pos, dlen, 16384, archive_comment,
				0, d->input_encoding_for_comments);
			de_dbg(c, "archive comment: \"%s\"", ucstring_getpsz_d(archive_comment));
		}
		else { // file comment
			if(!pmd) goto done;

			if(!pmd->comment) {
				pmd->comment = ucstring_create(c);
			}
			if(ucstring_isnonempty(pmd->comment)) goto done;
			dbuf_read_to_ucstring_n(c->infile, pos, dlen, 2048, pmd->comment,
				0, d->input_encoding_for_comments);
		}
	}
	else if(rectype==2) {
		if(!pmd) goto done;
		if(!pmd->path) {
			pmd->path = ucstring_create(c);
		}
		if(ucstring_isnonempty(pmd->path)) goto done;
		dbuf_read_to_ucstring_n(c->infile, pos, dlen, 512, pmd->path,
			0, d->input_encoding_for_comments);
	}

done:
	if(archive_comment) ucstring_destroy(archive_comment);
	de_dbg_indent_restore(c, saved_indent_level);
	return retval;
}

static void do_pak_trailer(deark *c, lctx *d)
{
	u8 b;
	i64 pos;

	if(!d->prescan_found_eoa) return;
	if(c->infile->len - d->prescan_pos_after_eoa < 2) return;
	if(de_getbyte(d->prescan_pos_after_eoa) != 0xfe) return;
	b = de_getbyte(d->prescan_pos_after_eoa+1);
	if(b>4) return;

	pos = d->prescan_pos_after_eoa;
	de_dbg(c, "PAK extended records at %"I64_FMT, pos);
	de_dbg_indent(c, 1);
	d->has_pak_trailer = 1;
	init_trailer_data(c, d);

	while(1) {
		int ret;
		i64 bytes_consumed = 0;

		if(pos > c->infile->len-2) break;
		ret = do_pak_ext_record(c, d, pos, &bytes_consumed);
		pos += bytes_consumed;
		mark_end_of_known_data(d, pos);
		if(!ret || bytes_consumed<8) break;
	}

	de_dbg_indent(c, -1);
}

static void dbg_timestamp(deark *c, struct de_timestamp *ts, const char *name)
{
	char timestamp_buf[64];

	de_timestamp_to_string(ts, timestamp_buf, sizeof(timestamp_buf), 0);
	de_dbg(c, "%s: %s", name, timestamp_buf);
}

// Convert backslashes to slashes, and make sure the string ends with a /.
static void fixup_path(deark *c, lctx *d, de_ucstring *s)
{
	i64 i;

	if(s->len<1) return;

	for(i=0; i<s->len; i++) {
		if(s->str[i]=='\\') {
			s->str[i] = '/';
		}
	}

	if(s->str[s->len-1]!='/') {
		ucstring_append_char(s, '/');
	}
}

static void do_decompress_fork_arcmac(struct member_data *md,
	dbuf *outf, const char *fork_name)
{
	struct de_dfilter_in_params dcmpri;
	struct de_dfilter_out_params dcmpro;
	struct de_dfilter_results dres;
	deark *c = md->c;
	int saved_indent_level;

	de_dbg_indent_save(c, &saved_indent_level);
	if(md->orig_size==0) goto done;

	de_dbg(c, "decompressing %s fork", fork_name);
	de_dbg_indent(c, 1);

	de_dfilter_init_objects(c, &dcmpri, &dcmpro, &dres);
	dcmpri.f = c->infile;
	dcmpri.pos = md->cmpr_data_pos;
	dcmpri.len = md->cmpr_size;
	dcmpro.f = outf;
	dcmpro.len_known = 1;
	dcmpro.expected_len = md->orig_size;

	if(dcmpri.pos + dcmpri.len > dcmpri.f->len) {
		de_err(c, "%s: Data goes beyond end of file", ucstring_getpsz_d(md->arcmac_fn->str));
		goto done;
	}

	md->cmi->decompressor(c, md->d, md, &dcmpri, &dcmpro, &dres);
	dbuf_flush(dcmpro.f);
	if(dres.errcode) {
		de_err(c, "Decompression failed for file %s[%s fork]: %s",
			ucstring_getpsz_d(md->arcmac_fn->str),
			fork_name, de_dfilter_get_errmsg(c, &dres));
		goto done;
	}

	md->crc_calc = de_crcobj_getval(md->d->crco);
	de_dbg(c, "crc (calculated): 0x%04x", (unsigned int)md->crc_calc);

	if(md->crc_calc!=md->crc_reported) {
		de_err(c, "%s: CRC check failed", ucstring_getpsz_d(md->arcmac_fn->str));
	}

done:
	de_dbg_indent_restore(c, saved_indent_level);
}

static int my_advfile_cbfn(deark *c, struct de_advfile *advf,
	struct de_advfile_cbparams *afp)
{
	struct member_data *md = (struct member_data*)advf->userdata;

	if(afp->whattodo == DE_ADVFILE_WRITEMAIN) {
		do_decompress_fork_arcmac(md, afp->outf, "data");
	}
	else if(afp->whattodo == DE_ADVFILE_WRITERSRC) {
		do_decompress_fork_arcmac(md, afp->outf, "rsrc");
	}

	return 1;
}

// TODO: Reduce code duplication with do_extract_member_file(), etc.
// Retrofitting the arc module for arcmac format made some things messy.
// It could be made somewhat cleaner by using the "advfile" system unconditionally
// -- there are pros and cons of doing that.
static void do_extract_member_file_arcmac(deark *c, lctx *d, struct member_data *md,
	de_finfo *fi)
{
	int saved_indent_level;

	de_dbg_indent_save(c, &saved_indent_level);

	if(!md->cmi || !md->cmi->decompressor) {
		de_err(c, "%s: Compression type 0x%02x (%s) is not supported.",
			ucstring_getpsz_d(md->fn), (unsigned int)md->cmpr_meth, md->cmpr_meth_name);
		goto done;
	}

	if(md->arcmac_dforklen && md->arcmac_rforklen) {
		// This seems to be allowed, but I need sample files.
		de_err(c, "Can't handle multi-fork ArcMac file");
		goto done;
	}
	if(md->arcmac_dforklen + md->arcmac_rforklen != md->orig_size) {
		de_err(c, "Inconsistent ArcMac fork size");
		goto done;
	}

	if(md->arcmac_fn && ucstring_isnonempty(md->arcmac_fn->str)) {
		ucstring_append_ucstring(md->arcmac_advf->filename, md->arcmac_fn->str);
	}
	else {
		ucstring_append_ucstring(md->arcmac_advf->filename, md->fn);
	}
	md->arcmac_advf->original_filename_flag = 1;

	md->arcmac_advf->mainfork.fi->timestamp[DE_TIMESTAMPIDX_MODIFY] =
		fi->timestamp[DE_TIMESTAMPIDX_MODIFY];

	md->arcmac_advf->userdata = (void*)md;
	md->arcmac_advf->writefork_cbfn = my_advfile_cbfn;

	md->arcmac_advf->mainfork.writelistener_cb = de_writelistener_for_crc;
	md->arcmac_advf->mainfork.userdata_for_writelistener = (void*)d->crco;
	md->arcmac_advf->rsrcfork.writelistener_cb = de_writelistener_for_crc;
	md->arcmac_advf->rsrcfork.userdata_for_writelistener = (void*)d->crco;
	de_crcobj_reset(d->crco);

	de_advfile_run(md->arcmac_advf);

done:
	de_dbg_indent_restore(c, saved_indent_level);
}

static void do_extract_member_file(deark *c, lctx *d, struct member_data *md,
	struct persistent_member_data *pmd, de_finfo *fi, i64 pos)
{
	de_ucstring *fullfn = NULL;
	dbuf *outf = NULL;
	int ignore_failed_crc = 0;
	int saved_indent_level;
	struct de_dfilter_in_params dcmpri;
	struct de_dfilter_out_params dcmpro;
	struct de_dfilter_results dres;

	de_dbg_indent_save(c, &saved_indent_level);
	fullfn = ucstring_create(c);

	if(pmd && ucstring_isnonempty(pmd->path)) {
		// For PAK-style paths.
		// (Pretty useless, until we support cmpr. meth. #11.)
		// Note that PAK-style paths, and directory recursion, are not expected to
		// be possible in the same file.
		ucstring_append_ucstring(fullfn, pmd->path);
		fixup_path(c, d, fullfn);
	}

	de_strarray_make_path(d->curpath, fullfn, DE_MPFLAG_NOTRAILINGSLASH);

	if(md->rfa.file_type_known) {
		fmtutil_riscos_append_type_to_filename(c, fi, fullfn, &md->rfa, md->is_dir, 0);
	}
	de_finfo_set_name_from_ucstring(c, fi, fullfn, DE_SNFLAG_FULLPATH);

	de_dbg_indent(c, 1);

	if(!md->cmi || !md->cmi->decompressor) {
		const char *mname;

		// A quick hack because we don't want the error message claiming that
		// "trimmed or crushed" isn't supported. We do support trimmed, so
		// if we got here, we must have decided it's crushed.
		mname = md->cmpr_meth==10 ? "crushed" : md->cmpr_meth_name;

		de_err(c, "%s: Compression type 0x%02x (%s) is not supported.",
			ucstring_getpsz_d(md->fn), (UI)md->cmpr_meth, mname);
		goto done;
	}

	outf = dbuf_create_output_file(c, NULL, fi, 0x0);
	dbuf_enable_wbuffer(outf);

	dbuf_set_writelistener(outf, de_writelistener_for_crc, (void*)d->crco);
	de_crcobj_reset(d->crco);

	de_dfilter_init_objects(c, &dcmpri, &dcmpro, &dres);
	dcmpri.f = c->infile;
	dcmpri.pos = pos;
	dcmpri.len = md->cmpr_size;
	dcmpro.f = outf;
	dcmpro.len_known = 1;
	dcmpro.expected_len = md->orig_size;

	if(dcmpri.pos + dcmpri.len > dcmpri.f->len) {
		de_err(c, "%s: Data goes beyond end of file", ucstring_getpsz_d(md->fn));
		goto done;
	}

	md->cmi->decompressor(c, d, md, &dcmpri, &dcmpro, &dres);
	dbuf_flush(dcmpro.f);
	if(dres.errcode) {
		de_err(c, "%s: Decompression failed: %s", ucstring_getpsz_d(md->fn),
			de_dfilter_get_errmsg(c, &dres));
		goto done;
	}

	md->crc_calc = de_crcobj_getval(d->crco);
	de_dbg(c, "crc (calculated): 0x%04x", (unsigned int)md->crc_calc);
	if(md->crc_reported==0 && !d->recurse_subdirs && md->rfa.file_type_known &&
		md->rfa.file_type==0xddc && md->cmpr_meth==0x82)
	{
		ignore_failed_crc = 1;
	}
	if((md->crc_calc!=md->crc_reported) && !ignore_failed_crc) {
		de_err(c, "%s: CRC check failed", ucstring_getpsz_d(md->fn));
	}

done:
	dbuf_close(outf);
	ucstring_destroy(fullfn);
	de_dbg_indent_restore(c, saved_indent_level);
}

// "Extract" a directory entry
static void do_extract_member_dir(deark *c, lctx *d, struct member_data *md,
	de_finfo *fi)
{
	dbuf *outf = NULL;
	de_ucstring *fullfn = NULL;

	fullfn = ucstring_create(c);
	de_strarray_make_path(d->curpath, fullfn, DE_MPFLAG_NOTRAILINGSLASH);

	fi->is_directory = 1;
	de_finfo_set_name_from_ucstring(c, fi, fullfn, DE_SNFLAG_FULLPATH);

	outf = dbuf_create_output_file(c, NULL, fi, 0x0);
	dbuf_close(outf);
	ucstring_destroy(fullfn);
}

struct extinfo_item_info {
	u8 cmprmeth;
	u8 rectype;
	unsigned int flags; // 0x1 = string
	const char *name;
	void *reserved;
};

static void do_info_record_string(deark *c, lctx *d, i64 pos, i64 len, const char *name)
{
	de_ucstring *s = NULL;

	s = ucstring_create(c);
	dbuf_read_to_ucstring_n(c->infile, pos, len, 2048, s, DE_CONVFLAG_STOP_AT_NUL,
		d->input_encoding_for_comments);
	de_dbg(c, "%s: \"%s\"", name, ucstring_getpsz_d(s));
	ucstring_destroy(s);
}

static const struct extinfo_item_info extinfo_arr[] = {
	{ 20, 0, 0x01, "archive description", NULL },
	{ 20, 1, 0x01, "archive created by", NULL },
	{ 20, 2, 0x01, "archive last modified by", NULL },
	{ 21, 0, 0x01, "file description", NULL },
	{ 21, 1, 0x01, "long name", NULL },
	{ 21, 2, 0x00, "timestamps", NULL },
	{ 21, 3, 0x00, "icon", NULL },
	{ 21, 4, 0x01, "attributes", NULL },
	{ 21, 5, 0x01, "full path", NULL }
};

static const struct extinfo_item_info *find_extinfo_item(u8 cmprmeth, u8 rectype)
{
	size_t k;

	for(k=0; k<DE_ARRAYCOUNT(extinfo_arr); k++) {
		if(extinfo_arr[k].cmprmeth==cmprmeth && extinfo_arr[k].rectype==rectype) {
			return &extinfo_arr[k];
		}
	}
	return NULL;
}

static void do_info_item(deark *c, lctx *d, struct member_data *md)
{
	int saved_indent_level;
	i64 pos = md->cmpr_data_pos;
	i64 endpos = md->cmpr_data_pos+md->cmpr_size;

	de_dbg_indent_save(c, &saved_indent_level);
	de_dbg(c, "info item data (meth=%d) at %"I64_FMT" len=%"I64_FMT, (int)md->cmpr_meth,
		md->cmpr_data_pos, md->cmpr_size);
	de_dbg_indent(c, 1);

	while(1) {
		i64 reclen;
		i64 recpos;
		i64 dpos;
		i64 dlen;
		u8 rectype;
		const struct extinfo_item_info *ei;
		const char *ei_name;

		recpos = pos;
		if(pos+3 > endpos) goto done;
		reclen = de_getu16le_p(&pos);
		if(reclen<3 || recpos+reclen > endpos) goto done;
		rectype = de_getbyte_p(&pos);
		ei = find_extinfo_item(md->cmpr_meth, rectype);
		if(ei && ei->name) ei_name = ei->name;
		else ei_name = "?";

		dpos = recpos + 3;
		dlen = reclen - 3;
		de_dbg(c, "record type %d (%s) at %"I64_FMT", dpos=%"I64_FMT", dlen=%"I64_FMT,
			(int)rectype, ei_name, recpos, dpos, dlen);
		de_dbg_indent(c, 1);
		if(ei && (ei->flags & 0x01)) {
			do_info_record_string(c, d, dpos, dlen, ei_name);
		}
		else {
			de_dbg_hexdump(c, c->infile, dpos, dlen, 256, NULL, 0x1);
		}
		de_dbg_indent(c, -1);
		pos = recpos + reclen;
	}

done:
	de_dbg_indent_restore(c, saved_indent_level);
}

static void do_sequence_of_members(deark *c, lctx *d, i64 pos1, i64 len, int nesting_level);

static void do_arcmac_preheader(deark *c, lctx *d, struct member_data *md, i64 pos1)
{
	i64 pos = pos1;
	u16 finder_flags;
	struct de_fourcc filetype;
	struct de_fourcc creator;

	if(md->arcmac_advf) return;
	md->arcmac_advf = de_advfile_create(c);
	md->arcmac_advf->enable_wbuffer = 1;

	pos += 2; // magic / cmprtype

	md->arcmac_fn = dbuf_read_string(c->infile, pos, 31, 31, DE_CONVFLAG_STOP_AT_NUL,
		d->input_encoding_for_arcmac_fn);
	de_dbg(c, "ArcMac filename: \"%s\"", ucstring_getpsz_d(md->arcmac_fn->str));
	if(md->arcmac_fn->sz_strlen>0) {
		md->arcmac_advf->original_filename_flag = 1;
		de_advfile_set_orig_filename(md->arcmac_advf, md->arcmac_fn->sz, md->arcmac_fn->sz_strlen);
	}
	pos += 32;

	dbuf_read_fourcc(c->infile, pos, &filetype, 4, 0x0);
	de_dbg(c, "filetype: '%s'", filetype.id_dbgstr);
	de_memcpy(md->arcmac_advf->typecode, filetype.bytes, 4);
	md->arcmac_advf->has_typecode = 1;
	pos += 4;

	dbuf_read_fourcc(c->infile, pos, &creator, 4, 0x0);
	de_dbg(c, "creator: '%s'", creator.id_dbgstr);
	de_memcpy(md->arcmac_advf->creatorcode, creator.bytes, 4);
	md->arcmac_advf->has_creatorcode = 1;
	pos += 4;

	finder_flags = (u16)de_getu16be_p(&pos);
	de_dbg(c, "finder flags: 0x%04x", finder_flags);
	md->arcmac_advf->finderflags = finder_flags;
	md->arcmac_advf->has_finderflags = 1;
	pos += 6; // remainder of finfo

	md->arcmac_dforklen = de_getu32le_p(&pos);
	de_dbg(c, "data fork len: %"I64_FMT, md->arcmac_dforklen);
	md->arcmac_rforklen = de_getu32le_p(&pos);
	de_dbg(c, "rsrc fork len: %"I64_FMT, md->arcmac_rforklen);

	md->arcmac_advf->mainfork.fork_exists = (md->arcmac_dforklen!=0);
	md->arcmac_advf->mainfork.fork_len = md->arcmac_dforklen;
	md->arcmac_advf->rsrcfork.fork_exists = (md->arcmac_rforklen!=0);
	md->arcmac_advf->rsrcfork.fork_len = md->arcmac_rforklen;
}

// The main per-member processing function
static void member_cb_main(deark *c, lctx *d, struct member_parser_data *mpd)
{
	int saved_indent_level;
	i64 pos1 = mpd->member_pos;
	i64 pos = pos1;
	i64 hdrsize;
	i64 mod_time_raw, mod_date_raw;
	de_finfo *fi = NULL;
	struct member_data *md = NULL;
	int need_curpath_pop = 0;
	struct persistent_member_data *pmd = NULL;

	de_dbg_indent_save(c, &saved_indent_level);
	de_dbg(c, "member at %"I64_FMT, pos1);
	de_dbg_indent(c, 1);
	md = de_malloc(c, sizeof(struct member_data));
	md->c = c;
	md->d = d;

	if(mpd->nesting_level==0 && d->persistent_md && (mpd->member_idx < d->num_top_level_members)) {
		pmd = &d->persistent_md[mpd->member_idx];
		if(ucstring_isnonempty(pmd->comment)) {
			de_dbg(c, "file comment: \"%s\"", ucstring_getpsz_d(pmd->comment));
		}
		if(ucstring_isnonempty(pmd->path)) {
			de_dbg(c, "path: \"%s\"", ucstring_getpsz_d(pmd->path));
		}
	}

	pos++; // 'magic' byte, already read by the parser
	if(mpd->magic != d->sig_byte) {
		de_err(c, "Failed to find %s member at %"I64_FMT, d->fmtname, pos1);
		goto done;
	}

	if(d->fmt==FMT_ARCMAC && mpd->cmpr_meth_masked!=0) {
		do_arcmac_preheader(c, d, md, mpd->member_pos);
		pos += 59;
	}

	pos++; // compression ID, already read by the parser
	md->cmpr_meth = mpd->cmpr_meth;
	md->cmpr_meth_masked = mpd->cmpr_meth_masked;

	md->cmi = get_cmpr_meth_info(d, md->cmpr_meth);
	if(md->cmi && md->cmi->name) {
		md->cmpr_meth_name = md->cmi->name;
	}
	else {
		md->cmpr_meth_name = "?";
	}

	de_dbg(c, "cmpr meth: 0x%02x (%s)", (unsigned int)md->cmpr_meth, md->cmpr_meth_name);

	if(md->cmpr_meth_masked==0x00 || md->cmpr_meth==0x1f) {
		hdrsize = 2;
	}
	else {
		if(md->cmpr_meth_masked==0x01) {
			hdrsize = 25;
		}
		else {
			hdrsize = 29;
		}
		if(md->cmpr_meth>=128) {
			hdrsize += 12;
		}
	}
	if(mpd->member_len<hdrsize) {
		de_err(c, "Insufficient data for archive member at %"I64_FMT, pos1);
		goto done;
	}

	if(md->cmpr_meth_masked==0x00 || md->cmpr_meth==0x1f) { // end of archive/dir marker
		goto done;
	}

	md->fn = ucstring_create(c);
	dbuf_read_to_ucstring(c->infile, pos, 13, md->fn, DE_CONVFLAG_STOP_AT_NUL,
		d->input_encoding_for_filenames);
	de_dbg(c, "filename: \"%s\"", ucstring_getpsz_d(md->fn));
	pos += 13;

	pos += 4; // cmpr_size, already read by the parser
	md->cmpr_size = mpd->cmpr_data_len;
	de_dbg(c, "cmpr size: %"I64_FMT, md->cmpr_size);

	mod_date_raw = de_getu16le_p(&pos);
	mod_time_raw = de_getu16le_p(&pos);
	de_dos_datetime_to_timestamp(&md->arc_timestamp, mod_date_raw, mod_time_raw);
	md->arc_timestamp.tzcode = DE_TZCODE_LOCAL;
	dbg_timestamp(c, &md->arc_timestamp, ((d->fmt==FMT_SPARK) ? "timestamp (ARC)":"timestamp"));

	md->crc_reported = (u32)de_getu16le_p(&pos);
	de_dbg(c, "crc (reported): 0x%04x", (unsigned int)md->crc_reported);
	if((md->cmpr_meth_masked)==0x01) {
		md->orig_size = md->cmpr_size;
	}
	else {
		md->orig_size = de_getu32le_p(&pos);
		de_dbg(c, "orig size: %"I64_FMT, md->orig_size);
	}

	if(d->fmt == FMT_SPARK) {
		md->has_spark_attribs = 1;
		fmtutil_riscos_read_load_exec(c, c->infile, &md->rfa, pos);
		pos += 8;
		fmtutil_riscos_read_attribs_field(c, c->infile, &md->rfa, pos, 0);
		pos += 4;
	}

	md->cmpr_data_pos = mpd->cmpr_data_pos;

	de_strarray_push(d->curpath, md->fn);
	need_curpath_pop = 1;

	// TODO: Is it possible to distinguish between a subdirectory, and a Spark
	// member file that should always be extracted? Does a nonzero CRC mean
	// we should not recurse?
	if(d->fmt==FMT_SPARK && d->recurse_subdirs && md->rfa.file_type_known &&
		(md->rfa.file_type==0xddc) && md->cmpr_meth==0x82)
	{
		md->is_dir = 1;
	}
	else if(d->fmt==FMT_ARC && d->recurse_subdirs && md->cmpr_meth==0x1e) {
		md->is_dir = 1;
	}

	if(d->recurse_subdirs) {
		de_dbg(c, "is directory: %d", md->is_dir);
	}

	de_dbg(c, "file data at %"I64_FMT", len=%"I64_FMT, md->cmpr_data_pos, md->cmpr_size);

	// Extract...
	fi = de_finfo_create(c);
	fi->original_filename_flag = 1;

	if(md->rfa.mod_time.is_valid) {
		fi->timestamp[DE_TIMESTAMPIDX_MODIFY] = md->rfa.mod_time;
	}
	else if(md->arc_timestamp.is_valid) {
		fi->timestamp[DE_TIMESTAMPIDX_MODIFY] = md->arc_timestamp;
	}

	if(md->has_spark_attribs) {
		fi->has_riscos_data = 1;
		fi->riscos_attribs = md->rfa.attribs;
		fi->load_addr = md->rfa.load_addr;
		fi->exec_addr = md->rfa.exec_addr;
	}

	if(md->is_dir) {
		fi->is_directory = 1;
	}

	if(md->is_dir) {
		do_extract_member_dir(c, d, md, fi);

		// Nested subdirectory archives (ARC 6 "z" option, or Spark) have both a known
		// length (md->cmpr_size), and an end-of-archive marker. So there are two
		// ways to parse them:
		// 1) Recursively, meaning we trust the md->cmpr_size field (or maybe we should
		//    use orig_size instead?).
		// 2) As a flat sequence of members, meaning we trust that a nested archive
		//    will not have extra data after the end-of-archive marker.
		// Here, we use the recursive method.
		do_sequence_of_members(c, d, md->cmpr_data_pos, md->cmpr_size, mpd->nesting_level+1);
	}
	else if(md->cmpr_meth>=30 && md->cmpr_meth<=39) {
		de_warn(c, "Unknown control item type %d at %"I64_FMT, (int)md->cmpr_meth, pos1);
		goto done;
	}
	else if(md->cmpr_meth>=20 && md->cmpr_meth<=29) {
		do_info_item(c, d, md);
	}
	else if(d->fmt==FMT_ARCMAC && md->arcmac_advf) {
		do_extract_member_file_arcmac(c, d, md, fi);
	}
	else {
		do_extract_member_file(c, d, md, pmd, fi, md->cmpr_data_pos);
	}

done:
	if(need_curpath_pop) {
		de_strarray_pop(d->curpath);
	}
	if(fi) de_finfo_destroy(c, fi);
	if(md) {
		ucstring_destroy(md->fn);
		if(md->arcmac_fn) de_destroy_stringreaderdata(c, md->arcmac_fn);
		if(md->arcmac_advf) de_advfile_destroy(md->arcmac_advf);
		de_free(c, md);
	}
	de_dbg_indent_restore(c, saved_indent_level);
}

static void do_sequence_of_members(deark *c, lctx *d, i64 pos1, i64 len, int nesting_level)
{
	if(nesting_level >= MAX_NESTING_LEVEL) {
		de_err(c, "Max subdir nesting level exceeded");
		return;
	}

	de_dbg(c, "archive at %"I64_FMT, pos1);
	de_dbg_indent(c, 1);
	parse_member_sequence(c, d, pos1, len, nesting_level, member_cb_main);
	de_dbg_indent(c, -1);
}

static void member_cb_for_prescan(deark *c, lctx *d, struct member_parser_data *mpd)
{
	if(mpd->magic!=d->sig_byte) return;
	if(mpd->cmpr_meth_masked==0x00) { // end of archive
		d->prescan_found_eoa = 1;
		d->prescan_pos_after_eoa = mpd->member_pos + mpd->member_len;
		de_dbg2(c, "end of member sequence at %"I64_FMT, d->prescan_pos_after_eoa);
		return;
	}
	if(mpd->cmpr_meth==20 || mpd->cmpr_meth==21 || mpd->cmpr_meth==30) {
		// Features we're pretty sure aren't used by PAK.
		d->has_arc_extensions = 1;
	}
	d->num_top_level_members++;
	de_dbg2(c, "member at %"I64_FMT, mpd->member_pos);
}

// Unfortunately, a pre-pass is necessary for robust handling of some ARC format
// extensions. The main issue is member-file comments, which we want to be
// available when we process that member file, but can only be found after we've
// read through the whole ARC file.
static void do_prescan_file(deark *c, lctx *d, i64 startpos)
{
	de_dbg2(c, "prescan");
	d->num_top_level_members = 0;
	de_dbg_indent(c, 1);
	parse_member_sequence(c, d, startpos, c->infile->len-startpos, 0, member_cb_for_prescan);
	de_dbg2(c, "number of members: %"I64_FMT, d->num_top_level_members);
	de_dbg_indent(c, -1);
}

static int find_arc_marker(deark *c, const u8 *buf, size_t buflen, i64 *ppos)
{
	size_t i;

	for(i=0; i<buflen; i++) {
		if(buf[i]==0x1a) {
			*ppos = (i64)i;
			return 1;
		}
	}
	return 0;
}

static void destroy_lctx(deark *c, lctx *d)
{
	if(!d) return;
	de_crcobj_destroy(d->crco);
	de_strarray_destroy(d->curpath);
	if(d->pak16sfx_outf) {
		dbuf_close(d->pak16sfx_outf);
	}
	if(d->persistent_md) {
		i64 i;

		for(i=0; i<d->num_top_level_members; i++) {
			ucstring_destroy(d->persistent_md[i].comment);
			ucstring_destroy(d->persistent_md[i].path);
		}
		de_free(c, d->persistent_md);
	}
	de_free(c, d);
}

static int is_btspk(deark *c, lctx *d)
{
	if(d->fmt!=FMT_ARC && d->fmt!=FMT_ARCMAC) return 0;

	if(dbuf_memcmp(c->infile, c->infile->len-31,
		(const void*)"\x1b\x03" "MASTR\xff" "END!", 12))
	{
		return 0;
	}
	return 1;
}

static void do_run_arc_spark_internal(deark *c, lctx *d)
{
	i64 members_endpos;
	i64 num_extra_bytes;
	i64 pos = 0;

	if(is_btspk(c, d)) {
		de_warn(c, "This looks like BTSPK format, which is not correctly supported.");
	}

	d->sig_byte = (d->fmt==FMT_ARCMAC) ? 0x1b : 0x1a;

	if(d->sig_byte==0x1a) {
		u8 buf[33];

		// Tolerate up to sizeof(buf)-1 bytes of initial junk
		de_read(buf, 0, sizeof(buf));
		if(!find_arc_marker(c, buf, sizeof(buf), &pos)) {
			de_err(c, "Not a(n) %s file", d->fmtname);
			goto done;
		}
	}

	de_declare_fmt(c, d->fmtname);
	d->curpath = de_strarray_create(c, MAX_NESTING_LEVEL+10);
	d->crco = de_crcobj_create(c, DE_CRCOBJ_CRC16_ARC);

	do_prescan_file(c, d, pos);
	if(d->prescan_found_eoa) {
		members_endpos = d->prescan_pos_after_eoa;
	}
	else {
		members_endpos = c->infile->len;
	}
	mark_end_of_known_data(d, members_endpos);

	if(d->fmt==FMT_ARC) {
		do_pk_comments(c, d);
		do_pak_trailer(c, d);
	}

	do_sequence_of_members(c, d, pos, members_endpos, 0);

	num_extra_bytes = c->infile->len - d->end_of_known_data;
	if(num_extra_bytes>0) {
		de_dbg(c, "extra bytes at end of archive: %"I64_FMT" (at %"I64_FMT")",
			num_extra_bytes, d->end_of_known_data);
	}

done:
	;
}

////////// Special converter for PAK v1.6 SFX archives

static void member_cb_for_pak16sfx(deark *c, lctx *d, struct member_parser_data *mpd)
{
	de_dbg2(c, "pak16sfx member at %"I64_FMT", type=0x%02x", mpd->member_pos,
		(UI)mpd->magic);
	de_dbg_indent(c, 1);

	// This is not perfect. We throw away the extended info, instead of converting
	// it to extended records.
	if(mpd->magic==0x1a) {
		if(!d->pak16sfx_outf) {
			d->pak16sfx_outf = dbuf_create_output_file(c, "pak", NULL, 0);
		}
		dbuf_copy(c->infile, mpd->member_pos, mpd->member_len, d->pak16sfx_outf);
	}
	de_dbg_indent(c, -1);
}

static void do_convert_pak16sfx(deark *c, lctx *d)
{
	d->fmt = FMT_PAK16SFX;
	d->recurse_subdirs = 0;
	d->sig_byte = 0x1a;
	parse_member_sequence(c, d, 0, c->infile->len, 0,member_cb_for_pak16sfx);
	if(d->pak16sfx_outf) {
		dbuf_writeu16be(d->pak16sfx_outf, 0xfe00);
	}
}

/////////////////////// ARC (core ARC-only functions)

static void de_run_arc(deark *c, de_module_params *mparams)
{
	lctx *d = NULL;
	const char *s;

	d = de_malloc(c, sizeof(lctx));
	d->fmt = FMT_ARC;
	d->fmtname = "ARC";
	// TODO: Make 'recurse' configurable. Would require us to make the embedded
	// archives end with the correct marker.
	d->recurse_subdirs = 1;
	d->input_encoding_for_filenames = de_get_input_encoding(c, NULL, DE_ENCODING_CP437);
	d->input_encoding_for_comments = DE_EXTENC_MAKE(d->input_encoding_for_filenames,
		DE_ENCSUBTYPE_HYBRID);

	// TODO: It would probably be worth it to have a separate module for PAK, so we
	// can take the .PAK file extension into account when guessing what method #10
	// is. It's complicated, though, and not very useful until we support Crushed
	// decompression.

	s = de_get_ext_option(c, "arc:method10");
	if(s) {
		if(!de_strcmp(s, "trimmed")) {
			d->method10 = 1;
		}
		else if(!de_strcmp(s, "crushed")) {
			d->method10 = 2;
		}
	}

	if(de_havemodcode(c, mparams, '6')) {
		do_convert_pak16sfx(c, d);
		goto done;
	}

	do_run_arc_spark_internal(c, d);

done:
	destroy_lctx(c, d);
}

static int de_identify_arc(deark *c)
{
	static const char *exts[] = {"arc", "ark", "pak", "spk", "sdn", "com"};
	u8 has_ext = 0;
	UI ext_idx = 0;
	UI strictness;
	int ends_with_trailer = 0;
	int ends_with_comments = 0;
	int starts_with_trailer = 0;
	i64 arc_start = 0;
	size_t k;
	u8 cmpr_meth;
	u8 buf[5];

	de_read(buf, 0, sizeof(buf));

	// Look for 0x1a in the first 4 bytes. Some .COM-style self-extracting
	// archives start with 1-3 bytes of code before the ARC marker.
	if(!find_arc_marker(c, buf, sizeof(buf)-1, &arc_start)) {
		return 0;
	}

	cmpr_meth = buf[arc_start+1];

	if(cmpr_meth==0) {
		// Don't tolerate empty archives that don't start at the beginning of file.
		if(arc_start!=0) return 0;

		starts_with_trailer = 1;
	}

	// Get info about file extension
	for(k=0; k<DE_ARRAYCOUNT(exts); k++) {
		if(de_input_file_has_ext(c, exts[k])) {
			has_ext = 1;
			ext_idx = (UI)k;
			break;
		}
	}

	// Only tolerate leading junk for a few file extensions
	if(arc_start>0) {
		if(ext_idx==0 || ext_idx==1 || ext_idx==5) { // .arc, .ark, .com
			;
		}
		else {
			return 0;
		}
	}

	// Look at some of the file, to see if it seems ok.
	if(has_ext && ext_idx<=1 && arc_start==0) {
		strictness = 0;
	}
	else if(has_ext && ext_idx<=3 && arc_start==0) {
		strictness = 1;
	}
	else {
		strictness = 2;
	}
	if(!is_valid_file_at(c->infile, arc_start, c->infile->len, strictness)) {
		return 0;
	}

	// Handle 2-byte files
	if(starts_with_trailer && c->infile->len==2) {
		if(!has_ext) return 0;
		if(ext_idx==0) return 100;
		if(ext_idx<=3) return 15;
		return 0;
	}

	if(de_getu16be(c->infile->len-2) == 0x1a00) {
		// Standard ARC trailer
		ends_with_trailer = 1;
	}

	if(de_getu32be(c->infile->len-8) == 0x504baa55) {
		// PKARC trailer, for files with comments
		ends_with_comments = 1;
	}

	if(!ends_with_trailer && !ends_with_comments) {
		// PAK-style extensions
		if(de_getu16be(c->infile->len-2) == 0xfe00) {
			ends_with_comments = 1;
		}
	}

	if(starts_with_trailer) {
		if(ends_with_comments) return 25;
		else return 0;
	}
	if(has_ext && (ends_with_trailer || ends_with_comments)) return 90;
	if(ends_with_trailer || ends_with_comments) return 35;
	if(has_ext) return 24;
	if(arc_start==0) return 19;
	return 0;
}

static void de_help_arc(deark *c)
{
	de_msg(c, "-opt arc:method10=<trimmed|crushed|auto> : How to interpret compression "
		"method #10");
}

void de_module_arc(deark *c, struct deark_module_info *mi)
{
	mi->id = "arc";
	mi->desc = "ARC compressed archive";
	mi->run_fn = de_run_arc;
	mi->identify_fn = de_identify_arc;
	mi->help_fn = de_help_arc;
}

/////////////////////// Spark

static void de_run_spark(deark *c, de_module_params *mparams)
{
	lctx *d = NULL;

	d = de_malloc(c, sizeof(lctx));
	d->fmt = FMT_SPARK;
	d->fmtname = "Spark";
	d->input_encoding_for_filenames = de_get_input_encoding(c, NULL, DE_ENCODING_RISCOS);
	d->input_encoding_for_comments = DE_EXTENC_MAKE(d->input_encoding_for_filenames,
		DE_ENCSUBTYPE_HYBRID);
	d->recurse_subdirs = de_get_ext_option_bool(c, "spark:recurse", 1);

	do_run_arc_spark_internal(c, d);
	destroy_lctx(c, d);
}

static int de_identify_spark(deark *c)
{
	u8 b;
	u32 load_addr;
	int ldaddrcheck = 0;
	int has_trailer = 0;

	if(de_getbyte(0) != 0x1a) return 0;
	b = de_getbyte(1); // compression method
	if(b==0x82 || b==0x83 || b==0x88 || b==0x89 || b==0xff) {
		;
	}
	else if(b==0x81 || b==0x84 || b==0x85 || b==0x86) {
		; // TODO: Verify that these are possible in Spark.
	}
	else {
		return 0;
	}

	load_addr = (u32)de_getu32le(29);
	if((load_addr & 0xfff00000) == 0xfff00000) {
		ldaddrcheck = 1;
	}

	if(de_getu16be(c->infile->len-2) == 0x1a80) {
		has_trailer = 1;
	}

	if(has_trailer && ldaddrcheck) return 85;
	if(ldaddrcheck) return 30;
	if(has_trailer) return 10;
	return 0;
}

static void de_help_spark(deark *c)
{
	de_msg(c, "-opt spark:recurse=0 : Extract subdirs as Spark files");
}

void de_module_spark(deark *c, struct deark_module_info *mi)
{
	mi->id = "spark";
	mi->desc = "Spark archive";
	mi->run_fn = de_run_spark;
	mi->identify_fn = de_identify_spark;
	mi->help_fn = de_help_spark;
}

/////////////////////// ArcMac

static void de_run_arcmac(deark *c, de_module_params *mparams)
{
	lctx *d = NULL;

	d = de_malloc(c, sizeof(lctx));
	d->fmt = FMT_ARCMAC;
	d->fmtname = "ArcMac";
	d->recurse_subdirs = 1;
	d->input_encoding_for_arcmac_fn = de_get_input_encoding(c, NULL, DE_ENCODING_MACROMAN);
	d->input_encoding_for_filenames = DE_ENCODING_CP437;
	d->input_encoding_for_comments = DE_EXTENC_MAKE(d->input_encoding_for_filenames,
		DE_ENCSUBTYPE_HYBRID);

	do_run_arc_spark_internal(c, d);
	destroy_lctx(c, d);
}

static int de_identify_arcmac(deark *c)
{
	u8 buf1[2];
	u8 buf2[2];

	de_read(buf1, 0, 2);
	if(buf1[0]!=0x1b) return 0;
	if(!(buf1[1]>=1 && buf1[1]<=9)) return 0;
	de_read(buf2, 59, 2);
	if(buf2[0]!=0x1a) return 0;
	if(buf2[1]!=buf1[1]) return 0;
	return 80;
}

void de_module_arcmac(deark *c, struct deark_module_info *mi)
{
	mi->id = "arcmac";
	mi->desc = "ArcMac compressed archive";
	mi->run_fn = de_run_arcmac;
	mi->identify_fn = de_identify_arcmac;
}
