/*
 *	dmem.c -- dynamic memory allocator
 *	Copyright (C) 2000-2005 Fred Barnes  <frmb@kent.ac.uk>
 *	Modifications (C) 2002 Brian Vinter
 *	Based on Per Brinch Hansen's parallel allocator as adapted by David Wood
 *
 *	This program is free software; you can redistribute it and/or modify
 *	it under the terms of the GNU General Public License as published by
 *	the Free Software Foundation; either version 2 of the License, or
 *	(at your option) any later version.
 *
 *	This program is distributed in the hope that it will be useful,
 *	but WITHOUT ANY WARRANTY; without even the implied warranty of
 *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *	GNU General Public License for more details.
 *
 *	You should have received a copy of the GNU General Public License
 *	along with this program; if not, write to the Free Software
 *	Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */


#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*{{{  includes*/
#if defined(OOS_BUILD)
	#include <oos_funcs.h>

	#define MESSAGE(fmt,args...) oos_printk((fmt),##args)
	#define MESSAGE0(fmt) oos_printk((fmt))
	#define EXIT(code) oos_exit(1)

#else	/* !OOS_BUILD */
	#include <stdio.h>
	#if defined(HAVE_STDLIB_H)
		#include <stdlib.h>
	#endif
	#if defined(HAVE_MALLOC_H)
		#include <malloc.h>
	#endif
	#if defined(DMEM_USE_MMAP)
		#include <sys/types.h>
		#include <sys/mman.h>
		#include <unistd.h>
		#include <fcntl.h>
	#endif
	#include <errno.h>


	#define MESSAGE(fmt,args...) fprintf(stderr,(fmt),##args)
	#define MESSAGE0(fmt) fprintf(stderr,(fmt))
	#define EXIT(code) exit((code))
#endif	/* !OOS_BUILD */

#include <sched_consts.h>
#include <arch/sched_types.h>
#include <arch/occam_io.h>
#include <arch/timer.h>
#include <rts.h>

#include <dmem_if.h>
#include <typedesc.h>
#include <arch/dmem.h>


/*}}}*/
/*{{{  memory lock for locked malloc*/
#if defined(BLOCKING_SYSCALLS) && !defined(OOS_BUILD)
	#if defined(USE_PTHREADS)
		#include <pthread.h>
		static pthread_mutex_t memory_lock;
	#else	/* !defined(USE_PTHREADS) */
		#include <arch/atomics.h>
		static spl_t memory_lock;
	#endif	/* !defined(USE_PTHREADS) */
#endif	/* defined(BLOCKING_SYSCALLS) && !defined(OOS_BUILD) */


/*}}}*/
/*{{{  default allocator*/

/* forwards */
static void *std_dmem_alloc (int bytes);
static void std_dmem_alloc2 (void **pptr);
static void std_dmem_release (void *ptr);
static void *std_dmem_realloc (void *ptr, int oldsize, int newsize);


#if !defined(OOS_BUILD)
static void *mlc_dmem_alloc (int bytes);
static void mlc_dmem_alloc2 (void **pptr);
static void mlc_dmem_release (void *ptr);
static void *mlc_dmem_realloc (void *ptr, int oldsize, int newsize);

static void *dbg_dmem_alloc (int bytes);
static void dbg_dmem_alloc2 (void **pptr);
static void dbg_dmem_release (void *ptr);
static void *dbg_dmem_realloc (void *ptr, int oldsize, int newsize);
#endif	/* !defined(OOS_BUILD) */


static dmem_allocator_t std_allocator = {
	name: "default",
	priv: NULL,
	alloc: std_dmem_alloc,
	alloc2: std_dmem_alloc2,
	release: std_dmem_release,
	realloc: std_dmem_realloc
};


#if !defined(OOS_BUILD)
static dmem_allocator_t mallocator = {
	name: "mallocator",
	priv: NULL,
	alloc: mlc_dmem_alloc,
	alloc2: mlc_dmem_alloc2,
	release: mlc_dmem_release,
	realloc: mlc_dmem_realloc
};

static dmem_allocator_t debug_allocator = {
	name: "debug",
	priv: NULL,
	alloc: dbg_dmem_alloc,
	alloc2: dbg_dmem_alloc2,
	release: dbg_dmem_release,
	realloc: dbg_dmem_realloc
};
#endif	/* !defined(OOS_BUILD) */


dmem_allocator_t *default_allocator = &std_allocator;
/*}}}*/
/*{{{  memory allocator state*/

/*{{{  memory map*/
/*
 *	the stock 4GB VM is chopped into 2048 2M "chunks"
 */
#define NUM_CHUNKS	2048

#define CHUNK_FREE		0x00		/* available chunk */
#define CHUNK_ADDRMAPPED	0x01		/* chunk is address-mapped */
#define CHUNK_HEAP		0x02		/* chunk is a heap-chunk */
#define CHUNK_RESERVED		0x04		/* chunk is reserved for something */

#define CHUNK_SHIFT 21
#define CHUNK_SIZE (1UL << CHUNK_SHIFT)		/* 2 meg chunks */
#define CHUNK_AMAP_OF(ADDR) ((ADDR >> 16) & 0x1f)

static unsigned char memmap[NUM_CHUNKS];

#define ChunkType(A) (memmap[((unsigned long)(A) >> CHUNK_SHIFT) & 0x000007ff])
#define ChunkNum(A) (((unsigned long)(A) >> CHUNK_SHIFT) & 0x000007ff)


/*}}}*/
/*{{{  free memory heap*/
static unsigned long mheap_addr;
static unsigned long mheap_left;


/*}}}*/
/*{{{  free-memory pools*/


#define POOL_COUNTERS

#define FIRST_POOL 4
/* #define N_POOLS 23 */
#define N_POOLS 45			/* 32-bytes to 128-MB */
#define FIRST_BIG_POOL 20		/* 32 kb - 128 MB pools are "big" */


/*
 *	memory-pools are organised by half-powers of 2,
 *	a variation on Per Brinch-Hansen's allocator.
 */

#ifdef POOL_COUNTERS

static unsigned long dmempools[N_POOLS << 2];
#define DMAddrSlot(X) dmempools[(X) << 2]
#define DMSizeSlot(X) dmempools[((X) << 2) | 0x1]
#define DMAvailSlot(X) dmempools[((X) << 2) | 0x2]
#define DMCountSlot(X) dmempools[((X) << 2) | 0x3]

#else	/* !POOL_COUNTERS */

static unsigned long dmempools[N_POOLS];
#define DMAddrSlot(X) dmempools[(X)]
#define DMSizeSlot(X) (-1)
#define DMAvailSlot(X) (-1)
#define DMCountSlot(X) (-1)

#endif	/* !POOL_COUNTERS */

#if !defined(TARGET_CPU_SPARC) && !defined(TARGET_CPU_PPC64)
#define SlotToSize(N)	((2 << ((((N) + FIRST_POOL) >> 1) + 2)) + (((N) & 1) ? (2 << ((((N) + FIRST_POOL) >> 1) + 1)) : 0))
#define SizeToSlot(X)	((((X) <= 32) ? 4 : (((bsr((X) - 1) - 2) << 1)) - (((X) < (3 << (bsr((X) - 1) - 1))) ? 1 : 0)) - FIRST_POOL)
#endif

static void *zero_block;		/* return this for 0 sized allocations */

/*}}}*/
/*{{{  address-mapped pool allocation*/
static int pooladdrmap[32] = {
	0, 1, 2, 3, 4, 5, 6, 7,
	8, 9, 10, 11, 11, 12, 13, 13,
	13, 14, 14, 15, 15, 15, 16, 16,
	17, 17, 17, 18, 18, 19, 19, 19
};

#define SlotOfAddr(A) pooladdrmap[CHUNK_AMAP_OF((unsigned long)(A))]
#define SizeOfAddr(A) SlotToSize(SlotOfAddr((A)))

static int pool_totalloc = 0;


/*}}}*/
#if defined(DMEM_USE_MMAP) || defined(OOS_BUILD)
/*{{{  pool variables when mapping chunks of anonymous memory*/

/* the pool can go up to 1 gig.. */
#if defined(OOS_BUILD)
/* static void *pool_baseaddr = (void *)0x10000000;
static void *pool_limit = (void *)0x40000000; */
#elif defined(DMEM_MEMBASE)
static void *pool_baseaddr = (void *)DMEM_MEMBASE;
static void *pool_limit = (void *)(DMEM_MEMBASE + 0x40000000);
#else
static void *pool_baseaddr = (void *)0x80000000;
static void *pool_limit = (void *)0xc0000000;
#endif


static void *pool_nextchunkaddr;
static int pool_mapfd = -1;		/* descriptor for /dev/zero */

/*}}}*/
#endif	/* defined(DMEM_USE_MMAP) || defined(OOS_BUILD) */

/*}}}*/
/*{{{  static void init_memmap (void)*/
/*
 *	initialises the memory-map
 */
static void init_memmap (void)
{
	int i;
#if !defined(OOS_BUILD)
	unsigned long heapbrk;

	heapbrk = ((unsigned long)sbrk (0) >> CHUNK_SHIFT);
#endif

	for (i=0; i<NUM_CHUNKS; i++) {
#if defined(OOS_BUILD)
		memmap[i] = CHUNK_FREE;
#else	/* !defined(OOS_BUILD) */
		memmap[i] = ((i <= heapbrk) ? CHUNK_RESERVED : CHUNK_FREE);
#endif	/* !defined(OOS_BUILD) */
	}

	return;
}
/*}}}*/


#if defined(DM_DEBUG) && (DM_DEBUG==1) && !defined(OOS_BUILD)

#if defined(DM_DEBUG_LIBELF) && (DM_DEBUG_LIBELF==1)
	#define USING_LIBELF

	#include <sys/types.h>
	#include <unistd.h>
	#include <fcntl.h>
	#include <gelf.h>
	#include <libelf.h>
#endif	/* defined(DM_DEBUG_LIBELF) && (DM_DEBUG_LIBELF==1) */


/*{{{  memory debugging routines*/
/*{{{  constants/structures for memory debugger*/
#define DM_MAGIC ((void *)0xffffffff)

typedef struct TAG_ord_mem_block {
	void *ptr;
	int size;
	struct TAG_ord_mem_block *next;
	void *spare;
	int age;
	int mode;
	int mdparams[6];
} ord_mem_block;
/*}}}*/
/*{{{  static vars*/
static ord_mem_block *memout = NULL;
/*}}}*/
/*{{{  static void add_ord_mem (void *ptr, int size)*/
/*
 *	adds a block to the allocator.  this is called locally and
 *	gets READ/WRITE/BALLOC flags
 */
static void add_ord_mem (void *ptr, int size)
{
	ord_mem_block *tmp = (ord_mem_block *)locked_malloc (sizeof (ord_mem_block));
	ord_mem_block *walk, *prev;

	if (!tmp) {
		MESSAGE0 ("add_ord_mem: malloc() failed!\n");
		occam_exit (1, 0);
	}
	tmp->ptr = ptr;
	tmp->size = size;
	tmp->next = NULL;
	tmp->spare = DM_MAGIC;
	tmp->age = 0;
	tmp->mode = MODE_READ | MODE_WRITE | MODE_BALLOC;
	tmp->mdparams[0] = mdparam1;
	tmp->mdparams[1] = mdparam2;
	tmp->mdparams[2] = mdparam3;
	tmp->mdparams[3] = mdparam4;
	tmp->mdparams[4] = mdparam5;
	tmp->mdparams[5] = mdparam6;
	/* insert into ordered list */
	for (walk=memout, prev=NULL; walk && ((unsigned int)ptr > (unsigned int)walk->ptr); prev=walk, walk=walk->next);
	if (!prev) {
		tmp->next = memout;
		memout = tmp;
	} else {
		tmp->next = prev->next;
		prev->next = tmp;
	}
	if (!size) {
		MESSAGE0 ("add_ord_mem: warning: 0 sized allocation.\n");
	}
	return;
}
/*}}}*/
/*{{{  void extadd_ord_mem (void *ptr, int size, int mode)*/
/*
 *	this is used to add memory blocks that are allocated
 *	through other means -- code sections, malloc()'d bits, etc.
 */
void extadd_ord_mem (void *ptr, int size, int mode)
{
	ord_mem_block *tmp = (ord_mem_block *)locked_malloc (sizeof (ord_mem_block));
	ord_mem_block *walk, *prev;

	if (!tmp) {
		MESSAGE0 ("add_ord_mem: malloc() failed!\n");
		occam_exit (1, 0);
	}
	tmp->ptr = ptr;
	tmp->size = size;
	tmp->next = NULL;
	tmp->spare = DM_MAGIC;
	tmp->age = 0;
	tmp->mode = mode;
	tmp->mdparams[0] = mdparam1;
	tmp->mdparams[1] = mdparam2;
	tmp->mdparams[2] = mdparam3;
	tmp->mdparams[3] = mdparam4;
	tmp->mdparams[4] = mdparam5;
	tmp->mdparams[5] = mdparam6;
	/* insert into ordered list */
	for (walk=memout, prev=NULL; walk && ((unsigned int)ptr > (unsigned int)walk->ptr); prev=walk, walk=walk->next);
	if (!prev) {
		tmp->next = memout;
		memout = tmp;
	} else {
		tmp->next = prev->next;
		prev->next = tmp;
	}
	if (!size) {
		MESSAGE0 ("extadd_ord_mem: warning: 0 sized allocation.\n");
	}
	return;
}
/*}}}*/
/*{{{  void extdel_ord_mem (void *ptr)*/
/*
 *	called to remove externally allocated memory from the memory debugger
 */
void extdel_ord_mem (void *ptr)
{
	ord_mem_block *walk, *last, *tmp;

	tmp = NULL;
	for (walk=memout, last=NULL; walk && ((unsigned int)ptr > (unsigned int)walk->ptr); last=walk, walk=walk->next);
	if (!walk) {
		MESSAGE ("extdel_ord_mem: attempt to release %p, ", ptr);
		if (last && ((unsigned int)ptr < ((unsigned int)last->ptr + last->size))) {
			MESSAGE ("%d bytes into block at %p (%d bytes)\n", (unsigned int)ptr - (unsigned int)last->ptr, last->ptr, last->size);
		} else if (last) {
			MESSAGE ("%d bytes past end of block %p (%d bytes)\n", (unsigned int)ptr - ((unsigned int)last->ptr + last->size), last->ptr, last->size);
		} else {
			MESSAGE0 ("no allocations ?\n");
		}
		show_last_debug_insert ();
		occam_exit (1, 0);
	} else if ((unsigned int)ptr < (unsigned int)walk->ptr) {
		MESSAGE ("extdel_ord_mem: attempt to release %p, %d bytes before block at %p (%d bytes)\n", ptr, (unsigned int)walk->ptr - (unsigned int)ptr, walk->ptr, walk->size);
		show_last_debug_insert ();
		occam_exit (1, 0);
	} else if ((unsigned int)ptr > (unsigned int)walk->ptr) {
		MESSAGE ("extdel_ord_mem: attempt to release %p, ", ptr);
		if ((unsigned int)ptr >= ((unsigned int)walk->ptr + walk->size)) {
			MESSAGE ("%d bytes past end of block at %p (%d bytes)\n", (unsigned int)ptr - ((unsigned int)walk->ptr + walk->size), walk->ptr, walk->size);
		} else {
			MESSAGE ("%d bytes into block at %p (%d bytes)\n", (unsigned int)ptr - (unsigned int)walk->ptr, walk->ptr, walk->size);
		}
	}	/* else good! */
	if ((walk->mode & MODE_BALLOC)) {
		MESSAGE ("extdel_ord_mem: block at %p was allocated by internal allocator (%d bytes)\n", ptr, walk->size);
	}
	if (walk == memout) {
		tmp = walk;
		memout = walk->next;
	} else {
		tmp = walk;
		last->next = walk->next;
	}
	if (tmp) {
		locked_free (tmp);
	}
}
/*}}}*/
#if defined(DM_DEBUG_LIBELF) && (DM_DEBUG_LIBELF==1)
/*{{{  void extadd_elf_mem (char *argvz)*/
/*
 *	scans the named file for data and code sections,
 *	then adds those to the memory debugger.
 *	.text and .rodata are marked as read-only
 *	.data is marked read-write
 */
void extadd_elf_mem (char *argvz)
{
	int fd;
	Elf *elf;

	/* oki, ELF here we come..! (trying to extract address/size of the constant data area) */
	elf_version (EV_CURRENT);
	fd = open (argvz, O_RDONLY);
	if (fd < 0) {
		MESSAGE ("KRoC: unable to open executable (%s) for dynamic memory debugger\n", argvz);
		return;
	}
	elf = elf_begin (fd, ELF_C_READ_MMAP, NULL);
	if (elf) {
		Elf_Scn *scn;
		GElf_Shdr shdr;
		size_t shstrndx;

		if (elf_kind (elf) != ELF_K_ELF) {
			MESSAGE ("KRoC: dynamic memory debugger cannot handle archive type files yet! (%s)\n", argvz);
			elf_end (elf);
			close (fd);
			return;
		}
		/* get section header string table */
		if (elf_getshstrndx (elf, &shstrndx) < 0) {
			MESSAGE ("KRoC: cannot get ELF section header string table from %s\n", argvz);
			elf_end (elf);
			close (fd);
			return;
		}
		scn = NULL;
		while ((scn = elf_nextscn (elf, scn)) != NULL) {
			GElf_Shdr *pshdr = gelf_getshdr (scn, &shdr);

			if ((pshdr->sh_flags & SHF_ALLOC)) {
				const char *secstr = elf_strptr (elf, shstrndx, pshdr->sh_name);
				void *secaddr = ((void *)((unsigned int)pshdr->sh_addr));
				int seclen = (int)pshdr->sh_size;
				int addmode = 0;

				if (!strcmp (secstr, ".rodata") || !strcmp (secstr, ".text")) {
					/* rodata and code are read-only */
					addmode = MODE_READ;
				} else if (!strcmp (secstr, ".data")) {
					/* allow data to be read/write */
					addmode = MODE_READ | MODE_WRITE;
				}
				if (addmode) {
#if 0
fprintf (stderr, "extadd_elf_mem: adding section [%s] at %p for %d bytes\n", secstr, (void *)((unsigned int)pshdr->sh_addr), (int)pshdr->sh_size);
#endif
					extadd_ord_mem (secaddr, seclen, addmode);
				}
			}
		}
		elf_end (elf);
	}
	close (fd);
	return;
}
/*}}}*/
/*{{{  void extdel_elf_mem (char *argvz)*/
/*
 *	this is used to delete memory blocks that came from ELF files,
 *	see extadd_elf_mem() above
 */
void extdel_elf_mem (char *argvz)
{
	int fd;
	Elf *elf;

	/* oki, ELF here we come..! (trying to extract address/size of the constant data area) */
	elf_version (EV_CURRENT);
	fd = open (argvz, O_RDONLY);
	if (fd < 0) {
		MESSAGE ("KRoC: unable to open executable (%s) for dynamic memory debugger\n", argvz);
		return;
	}
	elf = elf_begin (fd, ELF_C_READ_MMAP, NULL);
	if (elf) {
		Elf_Scn *scn;
		GElf_Shdr shdr;
		size_t shstrndx;

		if (elf_kind (elf) != ELF_K_ELF) {
			MESSAGE ("KRoC: dynamic memory debugger cannot handle archive type files yet! (%s)\n", argvz);
			elf_end (elf);
			close (fd);
			return;
		}
		/* get section header string table */
		if (elf_getshstrndx (elf, &shstrndx) < 0) {
			MESSAGE ("KRoC: cannot get ELF section header string table from %s\n", argvz);
			elf_end (elf);
			close (fd);
			return;
		}
		scn = NULL;
		while ((scn = elf_nextscn (elf, scn)) != NULL) {
			GElf_Shdr *pshdr = gelf_getshdr (scn, &shdr);

			if ((pshdr->sh_flags & SHF_ALLOC)) {
				const char *secstr = elf_strptr (elf, shstrndx, pshdr->sh_name);
				void *secaddr = ((void *)((unsigned int)pshdr->sh_addr));

				if (!strcmp (secstr, ".rodata") || !strcmp (secstr, ".text") || !strcmp (secstr, ".data")) {
					extdel_ord_mem (secaddr);
				}
			}
		}
		elf_end (elf);
	}
	close (fd);
	return;

}
/*}}}*/
#else
/*{{{  dummies for extadd_elf_mem/extdel_elf_mem*/
void extadd_elf_mem (char *argvz)
{
	return;
}
void extdel_elf_mem (char *argvz)
{
	return;
}
/*}}}*/
#endif
/*{{{  static int del_ord_mem (void *ptr)*/
/*
 *	removes a block from the memory debugger.  this is called locally.
 *	if this returns non-zero, the pointer passed should be freed proper (onto a free-list, etc.)
 *	if zero, the block should stay allocated -- the memory debugger will deal with its eventual release
 */
static int del_ord_mem (void *ptr)
{
	ord_mem_block *walk, *last, *tmp;

	tmp = NULL;
	for (walk=memout, last=NULL; walk && ((unsigned int)ptr > (unsigned int)walk->ptr); last=walk, walk=walk->next);
	if (!walk) {
		MESSAGE ("del_ord_mem: attempt to release %p, ", ptr);
		if (last && ((unsigned int)ptr < ((unsigned int)last->ptr + last->size))) {
			MESSAGE ("%d bytes into block at %p (%d bytes)\n", (unsigned int)ptr - (unsigned int)last->ptr, last->ptr, last->size);
		} else if (last) {
			MESSAGE ("%d bytes past end of block %p (%d bytes)\n", (unsigned int)ptr - ((unsigned int)last->ptr + last->size), last->ptr, last->size);
		} else {
			MESSAGE0 ("no allocations ?\n");
		}
		show_last_debug_insert ();
		occam_exit (1, 0);
	} else if ((unsigned int)ptr < (unsigned int)walk->ptr) {
		MESSAGE ("del_ord_mem: attempt to release %p, %d bytes before block at %p (%d bytes)\n", ptr, (unsigned int)walk->ptr - (unsigned int)ptr, walk->ptr, walk->size);
		show_last_debug_insert ();
		occam_exit (1, 0);
	} else if ((unsigned int)ptr > (unsigned int)walk->ptr) {
		MESSAGE ("del_ord_mem: attempt to release %p, ", ptr);
		if ((unsigned int)ptr >= ((unsigned int)walk->ptr + walk->size)) {
			MESSAGE ("%d bytes past end of block at %p (%d bytes)\n", (unsigned int)ptr - ((unsigned int)walk->ptr + walk->size), walk->ptr, walk->size);
		} else {
			MESSAGE ("%d bytes into block at %p (%d bytes)\n", (unsigned int)ptr - (unsigned int)walk->ptr, walk->ptr, walk->size);
		}
	}	/* else good! */
	if (*((unsigned int *)ptr + 1) != 0xffffffff) {
		int smds[6] = {mdparam1, mdparam2, mdparam3, mdparam4, mdparam5, mdparam6};

		/* magic has been trashed.. :( */
		MESSAGE ("del_ord_mem: block at %p has trashed magic (0x%8.8x, idx=0x%8.8x), %d bytes\n", ptr, *(unsigned int *)((char *)ptr + sizeof (int)), *(unsigned int *)(ptr), walk->size);
		/* find out where it came from.. :) */
		mdparam1 = walk->mdparams[0];
		mdparam2 = walk->mdparams[1];
		mdparam3 = walk->mdparams[2];
		mdparam4 = walk->mdparams[3];
		mdparam5 = walk->mdparams[4];
		mdparam6 = walk->mdparams[5];
		show_last_debug_insert ();
		mdparam1 = smds[0];
		mdparam2 = smds[1];
		mdparam3 = smds[2];
		mdparam4 = smds[3];
		mdparam5 = smds[4];
		mdparam6 = smds[5];
	}
	if ((walk->mode & MODE_BALLOC) == 0) {
		MESSAGE ("del_ord_mem: attempt to release %p, but externally allocated.\n", ptr);
		show_last_debug_insert ();
		occam_exit (1, 0);
	} else if ((walk->mode & MODE_TRASHED)) {
		MESSAGE ("del_ord_mem: attempt to release %p, but links were trashed.\n", ptr);
		show_last_debug_insert ();
		occam_exit (1, 0);
	}
	if (walk == memout) {
		tmp = walk;
		memout = walk->next;
	} else {
		tmp = walk;
		last->next = walk->next;
	}
	if (tmp) {
		locked_free (tmp);
	}
	return 1;
}
/*}}}*/
/*{{{  void verify_ord_mem (void *ptr, int dmcount)*/
/*
 *	this is called at run-time to verify a particular address for read/write
 */
void verify_ord_mem (void *ptr, int dmcount)
{
	ord_mem_block *walk, *prev;
	int mode = dmcount & 0x0f;

	/* dmcount is shifted.. */
	dmcount >>= 4;
#if 0
fprintf (stderr, "verify_ord_mem (%p, %d) in.\n", ptr, dmcount);
#endif
	for (walk=memout, prev=NULL; walk && ((unsigned int)ptr >= (unsigned int)(walk->ptr)); prev=walk, walk=walk->next);
	walk = prev;
	if (!walk && !prev) {
		/* strange.. */
		MESSAGE ("verify_ord_mem [%d]: %p invalid (no blocks?)\n", dmcount, ptr);
		show_last_debug_insert ();
	} else if (!walk) {
		/* beyond last allocated block */
		MESSAGE ("verify_ord_mem [%d]: %p is %d bytes beyond last known block\n", dmcount, ptr, (unsigned int)ptr - (((unsigned int)prev->ptr) + prev->size));
		show_last_debug_insert ();
	} else if (((unsigned int)ptr >= (unsigned int)walk->ptr)) {
		/* in this block */
		if ((unsigned int)ptr >= (((unsigned int)walk->ptr) + walk->size)) {
			/* trampling off the block-end.. */
			MESSAGE ("verify_ord_mem [%d]: %p is %d bytes past the end of %p (%d bytes)", dmcount, ptr,
					(((unsigned int)ptr) - ((unsigned int)(walk->ptr) + walk->size)), walk->ptr, walk->size);
			if (walk->next) {
				MESSAGE (", next block at %p (%d bytes)\n", walk->next->ptr, walk->next->size);
			} else {
				MESSAGE0 ("\n");
			}
			show_last_debug_insert ();
		}
		if ((mode & ADDR_TARGET)) {
			/* check target */
			if ((walk->mode & MODE_BALLOC) && ((unsigned int)ptr <= (((unsigned int)walk->ptr) + sizeof(int)))) {
				/* trampling on memory-allocator bits.. */
				MESSAGE ("verify_ord_mem [%d] (write): %p is in the first 2 words of block at %p (%d bytes)\n", dmcount, ptr, walk->ptr, walk->size);
				walk->mode |= MODE_TRASHED;
				show_last_debug_insert ();
			}
			if (!(walk->mode & MODE_WRITE)) {
				MESSAGE ("verify_ord_mem [%d] (write): %p is in a read-only block at %p (%d bytes)\n", dmcount, ptr, walk->ptr, walk->size);
				show_last_debug_insert ();
			}
		} else if ((mode & ADDR_SOURCE)) {
			/* check source */
			if (!(walk->mode & MODE_READ)) {
				MESSAGE ("verify_ord_mem [%d] (read): %p is in a non-readable block at %p (%d bytes)\n", dmcount, ptr, walk->ptr, walk->size);
				show_last_debug_insert ();
			}
		}
	}
#if 0
fprintf (stderr, "verify_ord_mem (%p, %d) out.\n", ptr, dmcount);
#endif
	return;
}
/*}}}*/
/*{{{  void dm_debug_dump (void)*/
/*
 *	dumps the `live' memory map (on stderr)
 */
void dm_debug_dump (void)
{
	ord_mem_block *walk;

	for (walk = memout; walk; walk = walk->next) {
		MESSAGE ("%c%c%c%c 0x%8.8x 0x%-6x %d\n",
				(walk->mode & MODE_READ) ? 'r' : '-',
				(walk->mode & MODE_WRITE) ? 'w' : '-',
				(walk->mode & MODE_BALLOC) ? 'b' : '-',
				(walk->mode & MODE_TRASHED) ? 't' : '-',
				(unsigned int)(walk->ptr), (unsigned int)(walk->size),
				walk->age);
	}
	return;
}
/*}}}*/


/*}}}*/
#endif	/* defined(DM_DEBUG) && !defined(OOS_BUILD)*/


/*{{{  static void pool_smashaddr (unsigned long memstart, unsigned long memsize)*/
/*
 *	takes a chunk of memory and breaks it up in the address-mapped allocator
 */
static void pool_smashaddr (unsigned long memstart, unsigned long memsize)
{
	unsigned long addr = 0UL;
	int pool, psize;

#if defined(OOS_BUILD)
MESSAGE ("pool_smashaddr(): smashing 0x%8.8x (%d) bytes at 0x%8.8x\n", (unsigned int)memsize, (int)memsize, (unsigned int)memstart);
#endif
	if (addr & 0x1f) {
		/* start on a 32-byte boundary */
		addr = (memstart & 0xffffffe0UL) + 32;
	} else {
		addr = memstart;
	}
	pool = SlotOfAddr (addr);
	psize = SlotToSize (pool);

	while (addr < (memstart + memsize)) {
		if (ChunkType (addr) == CHUNK_FREE) {
			ChunkType (addr) = CHUNK_ADDRMAPPED;
		} else if (ChunkType (addr) != CHUNK_ADDRMAPPED) {
			MESSAGE ("KRoC: warning: not sharing non-addr-mapped chunk! (%lu bytes lost)\n", CHUNK_SIZE - (addr & 0x001fffff));
			addr = (addr & 0xffe00000UL) + CHUNK_SIZE;
			ChunkType (addr) = CHUNK_ADDRMAPPED;
		}

		if (!(addr & 0xffff) && (SlotOfAddr (addr + (psize - 1)) != pool)) {
			/* natural next pool */
			pool = SlotOfAddr (addr);
			psize = SlotToSize (pool);
		} else if (SlotOfAddr (addr + (psize - 1)) != pool) {
			/* won't fit, move onto next pool -- 64k boundaries */
			unsigned long newaddr = (addr & 0xffff) ? (addr & 0xffff0000) + 0x10000 : (addr);

#if 0
fprintf (stderr, "pool_smashaddr(): new pool, can't fit %d bytes at addr = 0x%8.8x (pool = %d).  newaddr = 0x%8.8x\n", psize, (unsigned int)addr, pool, (unsigned int)newaddr);
fprintf (stderr, "pool_smashaddr():         , SlotOfAddr(newaddr) = %d\n", SlotOfAddr (newaddr));
#endif
			addr = newaddr;
			pool = SlotOfAddr (addr);
			psize = SlotToSize (pool);
			continue;
		} else if ((addr + psize) > (memstart + memsize)) {
			/* won't fit full stop */
			break;
		}
		/* otherwise add addr to pool */
		((unsigned long *)addr)[0] = DMAddrSlot (pool);
		((unsigned long *)addr)[1] = 0xdeadbeef;

		DMAddrSlot (pool) = addr;
#if defined(POOL_COUNTERS)
		DMCountSlot (pool) = DMCountSlot (pool) + 1;
		DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif
		addr += psize;
	}

	return;
}
/*}}}*/
/*{{{  static void pool_smashheap (int plimit)*/
/*
 *	smashes any free heap left at mheap_addr for mheap_left.
 *	splits down to and including the pool specified by plimit
 */
static void pool_smashheap (int plimit)
{
	int pool = 31;		/* this is the ~1.5 meg pool */
	int psize = SlotToSize (pool);
	int minbytes = SlotToSize (plimit);

	while ((mheap_left >= minbytes) && (pool >= plimit)) {
		if (mheap_left < psize) {
			pool--;
			psize = SlotToSize (pool);
		} else {
			/* fits here */
			((unsigned long *)mheap_addr)[0] = DMAddrSlot (pool);
			((unsigned long *)mheap_addr)[1] = 0xdeadbeef;

			DMAddrSlot (pool) = mheap_addr;
#if defined(POOL_COUNTERS)
			DMCountSlot (pool) = DMCountSlot (pool) + 1;
			DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif

			mheap_addr += psize;
			mheap_left -= psize;
		}
	}

	mheap_addr = 0x00000000UL;
	mheap_left = 0x00000000UL;
	return;
}
/*}}}*/

#if defined(DMEM_USE_MMAP) || defined(OOS_BUILD)
/*{{{  static unsigned long chunk_getnextchunk (int fd, int nchunks)*/
/*
 *	allocates another chunk for the pool allocator
 */
static unsigned long chunk_getnextchunk (int fd, int nchunks)
{
	void *addr = (void *)0;
#if !defined(OOS_BUILD)
	int i;

	if (pool_nextchunkaddr >= pool_limit) {
		MESSAGE0 ("KRoC: out of memory in chunk_getnextchunk()");
		EXIT (EXIT_FAILURE);
	}
#endif

#if defined(OOS_BUILD)

	MESSAGE ("KRoC: oos_kmalloc()'ing 0x%8.8x (%d) bytes (%d chunks)\n", (unsigned int)(CHUNK_SIZE * nchunks), (int)(CHUNK_SIZE * nchunks), nchunks);

	/* allocate a chunk too much to avoid worrying about alignment problems */
	addr = oos_kmalloc (CHUNK_SIZE * (nchunks + 1), OOS_KM_CONTIG);
	if ((unsigned long)addr & (CHUNK_SIZE - 1)) {
		MESSAGE ("KRoC: warning: chunk-unaligned oos_kmalloc to 0x%8.8x\n", (unsigned int)addr);
		addr = (void *)(((unsigned long)addr & ~(CHUNK_SIZE - 1)) + CHUNK_SIZE);
	}
	/*	faddr = (void *)do_brk ((unsigned long)pool_nextchunkaddr, CHUNK_SIZE); */

#else	/* !defined(OOS_BUILD) */

	for (i=0; i<nchunks; i++) {
		void *faddr;

		int mapflags = MAP_PRIVATE | MAP_ANON;
#if defined(DMEM_MMAP_FIXED)
		mapflags |= MAP_FIXED;
#endif
#if defined(DMEM_MMAP_INVFD)
		faddr = mmap (pool_nextchunkaddr, CHUNK_SIZE, PROT_READ | PROT_WRITE, mapflags, -1, 0);
#else
		faddr = mmap (pool_nextchunkaddr, CHUNK_SIZE, PROT_READ | PROT_WRITE, mapflags, fd, 0);
#endif

		if (faddr == ((void *)-1)) {
			MESSAGE ("KRoC: failed to map chunk: %s\n", strerror (errno));
			EXIT (EXIT_FAILURE);
		} else if (faddr != pool_nextchunkaddr) {
			MESSAGE ("KRoC: memory mapped in the wrong place! at: %8.8x (tried 0x%8.8x)\n", (unsigned int)faddr, (unsigned int)pool_nextchunkaddr);
			EXIT (EXIT_FAILURE);
		}

		/* we've got a chunk.. */
		pool_nextchunkaddr += CHUNK_SIZE;
		if (!i) {
			addr = faddr;
		}
	}
#endif	/* !defined(OOS_BUILD) */


	return (unsigned long)addr;
}
/*}}}*/
#endif	/* defined(DMEM_USE_MMAP) || defined(OOS_BUILD) */
#if !defined(OOS_BUILD)
/*{{{  static unsigned long chunk_getsystem (int bytes)*/
/*
 *	allocates some bytes using the system memory-allocator (libc)
 */
static unsigned long chunk_getsystem (int bytes)
{
	unsigned long addr;

	addr = (unsigned long)malloc (bytes);
	if (addr == (unsigned long)NULL) {
		MESSAGE ("KRoC: failed to malloc %d bytes\n", bytes);
		EXIT (EXIT_FAILURE);
	}

	return addr;
}
/*}}}*/
#endif	/* !defined(OOS_BUILD) */
/*{{{  static void mlc_dmem_moremem (int pool)*/
/*
 *	called when we need some more memory.  pool index is given.
 */
static void mlc_dmem_moremem (int pool)
{
	unsigned long bytes;

	bytes = SlotToSize (pool);
	if (bytes < mheap_left) {
		/* easy */
		((unsigned int *)mheap_addr)[0] = DMAddrSlot (pool);
		((unsigned int *)mheap_addr)[1] = 0xdeadbeef;
		DMAddrSlot (pool) = (unsigned int)mheap_addr;
#if defined(POOL_COUNTERS)
		DMCountSlot (pool) = DMCountSlot (pool) + 1;
		DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif

		mheap_addr += bytes;
		mheap_left -= bytes;
	} else {
		unsigned long caddr;
		int nchunks;

		/* can't use what's left here, smash it */
		pool_smashheap (0);

		if (bytes <= CHUNK_SIZE) {
			nchunks = 1;
		} else {
			nchunks = (bytes >> CHUNK_SHIFT);
			if (bytes & (CHUNK_SIZE - 1)) {
				nchunks++;
			}
		}

		/* allocate required amount */
#if defined(OOS_BUILD)
		caddr = (unsigned long)oos_kmalloc (nchunks << CHUNK_SHIFT, OOS_KM_CONTIG);
#else
		caddr = (unsigned long)malloc (nchunks << CHUNK_SHIFT);
#endif
		if (caddr == (unsigned long)NULL) {
			MESSAGE ("KRoC: out of memory in mlc_dmem_moremem, wanted %d/%d bytes\n", (int)bytes, nchunks << CHUNK_SHIFT);
			EXIT (EXIT_FAILURE);
		}

		((unsigned int *)caddr)[0] = DMAddrSlot (pool);
		((unsigned int *)caddr)[1] = 0xdeadbeef;
		DMAddrSlot (pool) = (unsigned int)caddr;
#if defined(POOL_COUNTERS)
		DMCountSlot (pool) = DMCountSlot (pool) + 1;
		DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif
		mheap_addr = caddr + bytes;
		mheap_left = (nchunks << CHUNK_SHIFT) - bytes;
	}
	return;
}
/*}}}*/
/*{{{  static void std_dmem_moremem (int pool)*/
/*
 *	called when we need some more memory.  pool index is given.
 */
static void std_dmem_moremem (int pool)
{
	unsigned long caddr;
	unsigned long bytes;

	if (pool < FIRST_BIG_POOL) {
		/* address-mapped bits, grab another chunk */
#if defined(OOS_BUILD)
		caddr = chunk_getnextchunk (0, 1);
		bytes = CHUNK_SIZE;
#elif defined(DMEM_USE_MMAP)
		if (pool_mapfd < 0) {
			bytes = (1UL << 22);		/* 4-meg lump */
			caddr = chunk_getsystem (bytes);
		} else {
			caddr = chunk_getnextchunk (pool_mapfd, 1);
			bytes = CHUNK_SIZE;
		}
#else
		bytes = (3UL << 20);		/* 3-meg lump */
		caddr = chunk_getsystem (bytes);
#endif
		pool_smashaddr (caddr, bytes);
	} else {
		int nchunks;

		bytes = SlotToSize (pool);

		if (bytes < mheap_left) {
			/* easy */
			((unsigned int *)mheap_addr)[0] = DMAddrSlot (pool);
			((unsigned int *)mheap_addr)[1] = 0xdeadbeef;
			DMAddrSlot (pool) = (unsigned int)mheap_addr;
#if defined(POOL_COUNTERS)
			DMCountSlot (pool) = DMCountSlot (pool) + 1;
			DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif

			mheap_addr += bytes;
			mheap_left -= bytes;
		} else {
			/* can't use what's left here, smash it */
			pool_smashheap (FIRST_BIG_POOL);

			if (bytes <= CHUNK_SIZE) {
				/* fairly simple, fits in a chunk for sure */
				nchunks = 1;
			} else {
				/* need multiple chunks for this */
				nchunks = (bytes >> CHUNK_SHIFT);
				if (bytes & (CHUNK_SIZE - 1)) {
					nchunks++;
				}
			}
			/* allocate chunks */

#if defined(OOS_BUILD)
			{
				int cnk;

				caddr = chunk_getnextchunk (pool_mapfd, nchunks);
				ChunkType (caddr) = CHUNK_HEAP;
				for (cnk = ChunkNum(caddr); cnk < (ChunkNum(caddr) + nchunks); cnk++) {
					ChunkType (cnk << CHUNK_SHIFT) = CHUNK_HEAP;
				}

				((unsigned int *)caddr)[0] = DMAddrSlot (pool);
				((unsigned int *)caddr)[1] = 0xdeadbeef;
				DMAddrSlot (pool) = (unsigned int)caddr;
#if defined(POOL_COUNTERS)
				DMCountSlot (pool) = DMCountSlot (pool) + 1;
				DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif

				mheap_addr = caddr + bytes;
				mheap_left = (nchunks << CHUNK_SHIFT) - bytes;
			}
#elif defined(DMEM_USE_MMAP)
			if (pool_mapfd < 0) {
				/* exact allocation */
				caddr = chunk_getsystem (bytes);

				((unsigned int *)caddr)[0] = DMAddrSlot (pool);
				((unsigned int *)caddr)[1] = 0xdeadbeef;
				DMAddrSlot (pool) = (unsigned int)caddr;
#if defined(POOL_COUNTERS)
				DMCountSlot (pool) = DMCountSlot (pool) + 1;
				DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif
				if (ChunkType (caddr) == CHUNK_FREE) {
					ChunkType (caddr) = CHUNK_HEAP;
				} else if (ChunkType (caddr) != CHUNK_HEAP) {
					MESSAGE ("KRoC: fatal: chunk %d is not heap! (0x%2.2x)\n", (int)ChunkNum (caddr), ChunkType (caddr));
				}
			} else {
				int cnk;

				caddr = chunk_getnextchunk (pool_mapfd, nchunks);
				for (cnk = ChunkNum(caddr); cnk < (ChunkNum(caddr) + nchunks); cnk++) {
					ChunkType (cnk << CHUNK_SHIFT) = CHUNK_HEAP;
				}

				((unsigned int *)caddr)[0] = DMAddrSlot (pool);
				((unsigned int *)caddr)[1] = 0xdeadbeef;
				DMAddrSlot (pool) = (unsigned int)caddr;
#if defined(POOL_COUNTERS)
				DMCountSlot (pool) = DMCountSlot (pool) + 1;
				DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif

				mheap_addr = caddr + bytes;
				mheap_left = (nchunks << CHUNK_SHIFT) - bytes;
			}
#else	/* !defined(DMEM_USE_MMAP) */
			/* exact allocation */

			/* crummy -- keep allocating until we hit a free chunk */
			do {
				caddr = chunk_getsystem (bytes);
			} while ((ChunkType (caddr) != CHUNK_FREE) && (ChunkType (caddr) != CHUNK_HEAP));

			((unsigned int *)caddr)[0] = DMAddrSlot (pool);
			((unsigned int *)caddr)[1] = 0xdeadbeef;
			DMAddrSlot (pool) = (unsigned int)caddr;
#if defined(POOL_COUNTERS)
			DMCountSlot (pool) = DMCountSlot (pool) + 1;
			DMAvailSlot (pool) = DMAvailSlot (pool) + 1;
#endif

			if (ChunkType (caddr) == CHUNK_FREE) {
				ChunkType (caddr) = CHUNK_HEAP;
			} else if (ChunkType (caddr) != CHUNK_HEAP) {
				MESSAGE ("KRoC: fatal: chunk %d is not heap! (0x%2.2x)\n", (int)ChunkNum (caddr), ChunkType (caddr));
			}
#endif	/* !defined(DMEM_USE_MMAP) */
		}
	}
	return;
}
/*}}}*/
/*{{{  static unsigned long dmem_new (int p, void (*moremem)(int))*/
/*
 *	allocates a block of memory from the specified pool
 */
static unsigned long dmem_new (int p, void (*moremem)(int))
{
	unsigned long thisblk = 0UL;
	unsigned long nextblk;

	if ((p < 0) || (p >= N_POOLS)) {
		return (unsigned long)NULL;
	}
	if (!DMAddrSlot (p)) {
		/* need some more for this slot */
		moremem (p);
	}

	thisblk = DMAddrSlot (p);
	nextblk = ((unsigned long *)thisblk)[0];

	DMAddrSlot (p) = nextblk;

	if (nextblk && ((unsigned long *)thisblk)[1] != 0xdeadbeefUL) {
		MESSAGE ("dmem_new(): bad block magic 0x%8.8lx for block at 0x%8.8lx in pool %d", ((unsigned long *)thisblk)[1], thisblk, p);
		EXIT (EXIT_FAILURE);
	}

#ifdef POOL_COUNTERS
	DMAvailSlot (p) = DMAvailSlot (p) - 1;
#endif	/* POOL_COUNTERS */

	pool_totalloc++;

	return thisblk;
}
/*}}}*/


/*{{{  void dmem_init (void)*/
/*
 *	initialises the allocator
 */
void dmem_init (void)
{
	int i;
#if !defined(OOS_BUILD)
	char *envstr;
#endif

	zero_block = (void *)&zero_block;

#if defined(OOS_BUILD)
	MESSAGE0 ("dmem_init(): initialising allocator\n");
#endif
	init_memmap ();
#if defined(OOS_BUILD)
	MESSAGE0 ("dmem_init(): mapped out VM\n");
#endif

#if defined(BLOCKING_SYSCALLS) && defined(USE_PTHREADS) && !defined(OOS_BUILD)
	pthread_mutex_init (&memory_lock, NULL);
#elif defined(BLOCKING_SYSCALLS) && !defined(OOS_BUILD)
	spl_init (&memory_lock);
#endif


#if !defined(OOS_BUILD)
	/* allow environment to override memory allocator used */
	envstr = getenv ("DMEM_ALLOCATOR");
	if (envstr) {
		if (!strcmp (envstr, "standard") || !strcmp (envstr, "default")) {
			/* skip, default allocator */
		} else if (!strcmp (envstr, "malloc")) {
			default_allocator = &mallocator;
		} else if (!strcmp (envstr, "debug")) {
			MESSAGE0 ("KRoC: dmem_init(): using debugging allocator (flat malloc/free)\n");
			default_allocator = &debug_allocator;
		} else {
			MESSAGE ("KRoC: dmem_init(): unknown DMEM_ALLOCATOR: %s\n", envstr);
		}
	}
#endif

#if defined(OOS_BUILD)
	/* pool_nextchunkaddr = pool_baseaddr; */
	pool_mapfd = -1;

	/* MESSAGE ("dmem_init(): first chunk at 0x%8.8x\n", (unsigned int)pool_nextchunkaddr); */

#elif defined(DMEM_USE_MMAP)
	if (default_allocator == &std_allocator) {
		/* environment might have a base-address set */
		envstr = getenv ("DMEM_POOLBASE");
		if (envstr) {
			if (!strncmp (envstr, "0x", 2)) {
				if (sscanf (envstr + 2, "%x", (unsigned int *)(&pool_baseaddr)) != 1) {
					MESSAGE ("KRoC: dmem_init(): mangled DMEM_POOLBASE environment ?: %s", envstr);
#if defined(DMEM_MEMBASE)
					pool_baseaddr = (void *)DMEM_MEMBASE;
#else
					pool_baseaddr = (void *)0x80000000;
#endif
				}
				pool_limit = pool_baseaddr + 0x40000000;
			}
		}
		envstr = getenv ("DMEM_POOLLIMIT");
		if (envstr) {
			if (!strncmp (envstr, "0x", 2)) {
				if (sscanf (envstr + 2, "%x", (unsigned int *)&pool_limit) != 1) {
					MESSAGE ("KRoC: dmem_init(): mangled DMEM_POOLLIMIT environment ?: %s", envstr);
					pool_limit = pool_baseaddr + 0x40000000;
				}
			}
		}
		if (pool_baseaddr >= pool_limit) {
			MESSAGE ("KRoC: dmem_init(): memory pool base >= limit! (0x%8.8x, 0x%8.8x)", (unsigned int)pool_baseaddr, (unsigned int)pool_limit);
			EXIT (EXIT_FAILURE);
		}

		pool_nextchunkaddr = pool_baseaddr;

		pool_mapfd = open ("/dev/zero", O_RDWR);
		if (pool_mapfd < 0) {
			MESSAGE ("KRoC: dmem_init(): failed to open /dev/zero: %s", strerror (errno));
			EXIT (EXIT_FAILURE);
		}
	} else {
		mheap_left = 0x00000000UL;
		mheap_addr = 0x00000000UL;
	}
#endif	/* defined(DMEM_USE_MMAP) && !defined(OOS_BUILD) */

	mheap_left = 0x00000000UL;
	mheap_addr = 0x00000000UL;

#ifdef DEBUG
	MESSAGE0 ("initialising pool allocator..\n");
#endif

	for (i=0; i<N_POOLS; i++) {
		DMAddrSlot(i) = (unsigned int)NULL;
#ifdef POOL_COUNTERS
		DMSizeSlot(i) = SlotToSize (i);
		DMAvailSlot(i) = 0;
		DMCountSlot(i) = 0;
#endif	/* POOL_COUNTERS */
	}

	/* do an allocation and free from pool 0 -- forces chunk allocation */
	{
		void *ptr = (void *)dmem_alloc (SlotToSize (0));

#if defined(OOS_BUILD)
		MESSAGE ("dmem_init(): forced chunk allocation via pool 0, got ptr = 0x%8.8x\n", (unsigned int)ptr);
#endif

#if defined(DEBUG)
		MESSAGE ("KRoC: dummy allocation from pool 0 = 0x%8.8x (ChunkType (ptr) = 0x%2.2x)\n", (unsigned int)ptr, ChunkType (ptr));
#endif
		dmem_release (ptr);
	}

	return;
}
/*}}}*/
/*{{{  void dmem_shutdown (void)*/
/*
 *	shuts down the pool allocator
 */
void dmem_shutdown (void)
{
	int i;

#if defined(DMEM_USE_MMAP) && !defined(OOS_BUILD)
	void *qaddr;

	if (pool_mapfd >= 0) {
		for (qaddr = (void *)pool_baseaddr; qaddr < pool_nextchunkaddr; qaddr += CHUNK_SIZE) {
			munmap (qaddr, CHUNK_SIZE);
		}
		close (pool_mapfd);
		pool_mapfd = -1;
	}
#endif	/* defined(DMEM_USE_MMAP) && !defined(OOS_BUILD) */

	for (i=0; i<N_POOLS; i++) {
		DMAddrSlot(i) = (unsigned int)NULL;
#ifdef POOL_COUNTERS
		DMSizeSlot(i) = SlotToSize (i);
		DMAvailSlot(i) = 0;
		DMCountSlot(i) = 0;
#endif	/* POOL_COUNTERS */
	}
	return;
}
/*}}}*/

/*{{{  static void *std_dmem_alloc (int bytes)*/
/*
 *	allocates a specified amount of memory
 */
static void *std_dmem_alloc (int bytes)
{
	void *ptr;
	int pool = SizeToSlot (bytes);

	if (!bytes) {
		return (void *)zero_block;
	}

#if defined(DM_DEBUG)
	pool = SizeToSlot (bytes + 16);			/* always grab 16 extra here */
#else
	if (pool >= FIRST_BIG_POOL) {
		pool = SizeToSlot (bytes + 32);		/* only want to store a word, but grab a cache-line's worth extra */
	}
#endif

	ptr = (void *)dmem_new (pool, std_dmem_moremem);

#if defined(DM_DEBUG)
	/* put in debugging markers */
	((int *)ptr)[0] = pool;
	((int *)ptr)[1] = 0xffffffff;
	((int *)ptr)[2 + (bytes >> 2)] = 0xffffffff;
	((int *)ptr)[3 + (bytes >> 2)] = pool;
	ptr += 8;

	/* and do checks as below */
	if ((pool >= FIRST_BIG_POOL) && (ChunkType (ptr) != CHUNK_HEAP)) {
		MESSAGE ("KRoC: fatal: chunk type of heap'd block is 0x%2.2x\n", ChunkType (ptr));
		EXIT (EXIT_FAILURE);
	} else if ((pool < FIRST_BIG_POOL) && (ChunkType (ptr) != CHUNK_ADDRMAPPED)) {
		MESSAGE ("KRoC: fatal: chunk type of addr-mapped block (pool %d) is 0x%2.2x\n", pool, ChunkType (ptr));
		EXIT (EXIT_FAILURE);
	}
#else
	if (pool >= FIRST_BIG_POOL) {
		/* save pool number and adjust pointer */
		*(void **)ptr = (void *)pool;
		ptr += 32;
		if (ChunkType (ptr) != CHUNK_HEAP) {
			MESSAGE ("KRoC: fatal: chunk type of heap'd block is 0x%2.2x\n", ChunkType (ptr));
			EXIT (EXIT_FAILURE);
		}
	} else if (ChunkType (ptr) != CHUNK_ADDRMAPPED) {
		MESSAGE ("KRoC: fatal: chunk type of addr-mapped block is 0x%2.2x\n", ChunkType (ptr));
		EXIT (EXIT_FAILURE);
	}
#endif

#if 0
	if (!(bytes & 0x03)) {
		/* experiment.. */
		int i;

		for (i=0; i<(bytes >> 2); i++) {
			((unsigned int *)ptr)[i] = 0x80000000;
		}
	} else
#endif
	{
		memset (ptr, 0, bytes);
	}

#if defined(DM_DEBUG)
	add_ord_mem (ptr - 8, bytes + 8);
#endif
	return ptr;
}
/*}}}*/
/*{{{  static void std_dmem_release (void *ptr)*/
/*
 *	releases a block of memory
 */
static void std_dmem_release (void *ptr)
{
	int slot = -1;

	switch (ChunkType (ptr)) {
	default:
		MESSAGE ("KRoC: fatal: tried to free 0x%8.8x from chunk %d (type = %2.2x)\n", (unsigned int)ptr, (int)(ChunkNum (ptr)), ChunkType (ptr));
#if defined(DM_DEBUG)
		/* free it early from ordered memory lists */
		del_ord_mem (ptr);
#endif
		EXIT (EXIT_FAILURE);
		break;
#if defined(DM_DEBUG)
	case CHUNK_HEAP:
		ptr -= 8;
		slot = (int)(*(void **)ptr);
		if ((slot > N_POOLS) || (slot < 0)) {
			MESSAGE ("KRoC: fatal: tried to free 0x%8.8x from heap chunk, but slot = %d\n", (unsigned int)ptr, slot);
			EXIT (EXIT_FAILURE);
		}
		break;
	case CHUNK_ADDRMAPPED:
		ptr -= 8;
		slot = (int)(*(void **)ptr);
		if ((slot > N_POOLS) || (slot < 0)) {
			MESSAGE ("KRoC: fatal: tried to free 0x%8.8x from address-mapped chunk, but slot = %d\n", (unsigned int)ptr, slot);
			EXIT (EXIT_FAILURE);
		}
		break;
#else
	case CHUNK_HEAP:
		/* 32-bytes before this has a pool index */
		ptr -= 32;
		slot = (int)(*(void **)ptr);
		if ((slot > N_POOLS) || (slot < FIRST_BIG_POOL)) {
			MESSAGE ("KRoC: fatal: tried to free 0x%8.8x from heap chunk, but slot = %d\n", (unsigned int)(ptr + 32), slot);
			EXIT (EXIT_FAILURE);
		}
		break;
	case CHUNK_ADDRMAPPED:
		slot = SlotOfAddr (ptr);
		break;
#endif
	}

#if defined(DM_DEBUG) && (DM_DEBUG == 1)
	if (!del_ord_mem (ptr)) {
		/* don't free */
		return;
	}
#endif

	((unsigned int *)ptr)[0] = DMAddrSlot (slot);
	((unsigned int *)ptr)[1] = 0xdeadbeef;
	DMAddrSlot (slot) = (unsigned int)ptr;
#ifdef POOL_COUNTERS
	DMAvailSlot (slot) = DMAvailSlot (slot) + 1;
#endif	/* POOL_COUNTERS */

	return;
}
/*}}}*/
/*{{{  static void std_dmem_alloc2 (void **ptr)*/
/*
 *	allocates arbitary sized memory block (single param size/result)
 */
static void std_dmem_alloc2 (void **ptr)
{
	int size = *(int *)ptr;

	*ptr = std_dmem_alloc (size);
	return;
}
/*}}}*/
/*{{{  static void *std_dmem_realloc (void *ptr, int obytes, int nbytes)*/
/*
 *	reallocates a block of memory
 */
static void *std_dmem_realloc (void *ptr, int obytes, int nbytes)
{
	void *nptr;

	if (SizeToSlot (obytes) == SizeToSlot (nbytes)) {
		/* same pool, -- easy :) */
		nptr = ptr;
	} else {
		nptr = dmem_alloc (nbytes);
		/* copy data */
		memcpy (nptr, ptr, (nbytes < obytes) ? nbytes : obytes);
		dmem_release (ptr);
	}

#if 0
fprintf (stderr, "dmem_realloc (0x%8.8x, %d) -> (0x%8.8x, %d)\n", (unsigned int)ptr, obytes, (unsigned int)nptr, nbytes);
#endif
	return nptr;
}
/*}}}*/

#if !defined(OOS_BUILD)
/*{{{  static void *mlc_dmem_alloc (int bytes)*/
/*
 *	allocates a specified amount of memory
 */
static void *mlc_dmem_alloc (int bytes)
{
	void *ptr;
	int pool;

#if defined(DM_DEBUG)
	pool = SizeToSlot (bytes + 16);			/* 16 extra (4 words) */
#else
	pool = SizeToSlot (bytes + 8);			/* just 8 extra (2 words) */
#endif

	ptr = (void *)dmem_new (pool, mlc_dmem_moremem);

	((int *)ptr)[0] = pool;
	((int *)ptr)[1] = 0xffffffff;
#if defined(DM_DEBUG)
	/* put in debugging markers */
	((int *)ptr)[2 + (bytes >> 2)] = 0xffffffff;
	((int *)ptr)[3 + (bytes >> 2)] = pool;
#endif
	ptr += 8;
	memset (ptr, 0, bytes);

#if defined(DM_DEBUG)
	add_ord_mem (ptr - 8, bytes + 8);
#endif

	return ptr;
}
/*}}}*/
/*{{{  static void mlc_dmem_release (void *ptr)*/
/*
 *	releases the memory block associated with the given pointer
 */
static void mlc_dmem_release (void *ptr)
{
	int slot;

	if (!ptr) {
		MESSAGE0 ("KRoC: fatal: attempt to free NULL!\n");
		EXIT (EXIT_FAILURE);
	}
	ptr -= 8;
	slot = (int)(*(void **)ptr);
	if ((slot > N_POOLS) || (slot < 0)) {
		MESSAGE ("KRoC: fatal: tried to free 0x%8.8x (+8), apparently in slot %d\n", (unsigned int)ptr, slot);
		EXIT (EXIT_FAILURE);
	}

#if defined(DM_DEBUG)
	if (!del_ord_mem (ptr)) {
		return;			/* don't free */
	}
#endif

	/* put back on free-list */
	((unsigned int *)ptr)[0] = DMAddrSlot (slot);
	((unsigned int *)ptr)[1] = 0xdeadbeef;
	DMAddrSlot (slot) = (unsigned int)ptr;
#if defined(POOL_COUNTERS)
	DMAvailSlot (slot) = DMAvailSlot (slot) + 1;
#endif

	return;
}
/*}}}*/
/*{{{  static void mlc_dmem_alloc2 (void **ptr)*/
/*
 *	allocates arbitary sized memory block (single param size/result)
 */
static void mlc_dmem_alloc2 (void **ptr)
{
	int size = *(int *)ptr;

	*ptr = mlc_dmem_alloc (size);
	return;
}
/*}}}*/
/*{{{  static void *mlc_dmem_realloc (void *ptr, int obytes, int nbytes)*/
/*
 *	reallocates the specified block of memory
 */
static void *mlc_dmem_realloc (void *ptr, int obytes, int nbytes)
{
	void *nptr;

	if (SizeToSlot (obytes) == SizeToSlot (nbytes)) {
		/* same pool, -- easy :) */
		nptr = ptr;
	} else {
		nptr = mlc_dmem_alloc (nbytes);
		/* copy data */
		memcpy (nptr, ptr, (nbytes < obytes) ? nbytes : obytes);
		mlc_dmem_release (ptr);
	}

#if 0
fprintf (stderr, "mlc_dmem_realloc (0x%8.8x, %d) -> (0x%8.8x, %d)\n", (unsigned int)ptr, obytes, (unsigned int)nptr, nbytes);
#endif
	return nptr;
}
/*}}}*/

/*{{{  static void *dbg_dmem_alloc (int bytes)*/
/*
 *	allocates the specified amount of memory
 */
static void *dbg_dmem_alloc (int bytes)
{
	void *ptr;
	
	if (!bytes) {
		return (void *)zero_block;
	}
#if 0
fprintf (stderr, "allocating %d bytes ...\n", bytes);
#endif
#ifdef OOS_BUILD
	ptr = oos_kmalloc (bytes, 0);
#else
	ptr = malloc (bytes);
#endif
	if (!ptr) {
		MESSAGE ("KRoC: fatal: failed to malloc(%d) -- continuing for fault\n", bytes);
		*(int *)0 = 0;
	} else {
		memset (ptr, 0, bytes);
	}
#if 0
fprintf (stderr, "allocated %d bytes at 0x%8.8x\n", bytes, (unsigned int)ptr);
#endif

	return ptr;
}
/*}}}*/
/*{{{  static void dbg_dmem_release (void *ptr)*/
/*
 *	frees the specified block of memory
 */
static void dbg_dmem_release (void *ptr)
{
#ifdef OOS_BUILD
	oos_kfree (ptr);
#else
	free (ptr);
#endif
	return;
}
/*}}}*/
/*{{{  static void dbg_dmem_alloc2 (void **ptr)*/
/*
 *	allocates the specified amount of memory, single size/result parameter
 */
static void dbg_dmem_alloc2 (void **ptr)
{
	int size = *(int *)ptr;

	*ptr = dbg_dmem_alloc (size);
	return;
}
/*}}}*/
/*{{{  static void *dbg_dmem_realloc (void *ptr, int obytes, int nbytes)*/
/*
 *	re-allocates the specified block of memory
 */
static void *dbg_dmem_realloc (void *ptr, int obytes, int nbytes)
{
	void *nptr = dbg_dmem_alloc (nbytes);

	memcpy (nptr, ptr, (nbytes < obytes) ? nbytes : obytes);
	dbg_dmem_release (ptr);

	return nptr;
}
/*}}}*/
#endif	/* !defined(OOS_BUILD) */

/*{{{  int dmem_slot_to_size (int idx)*/
/*
 *	int dmem_slot_to_size (int idx)
 *	wrapper for SlotToSize
 */
int dmem_slot_to_size (int idx)
{
	return SlotToSize (idx);
}
/*}}}*/
/*{{{  void dmem_usagedump (void)*/
/*
 *	shows a dump of the pool allocator
 */
void dmem_usagedump (void)
{
#if defined(DMEM_USE_MMAP) && !defined(OOS_BUILD)
	MESSAGE ("memory pool at 0x%8.8x -> 0x%8.8x (%d bytes, >= %d M), limit 0x%8.8x\n", (unsigned int)pool_baseaddr, (unsigned int)pool_nextchunkaddr - 1, ((unsigned int)pool_nextchunkaddr - (unsigned int)pool_baseaddr), ((unsigned int)pool_nextchunkaddr - (unsigned int)pool_baseaddr) >> 20, (unsigned int)pool_limit);
#endif
	MESSAGE ("pool contents (%d allocations):\n", pool_totalloc);
	{
		int i;

		for (i=0; i<N_POOLS; i++) {
			if (!DMCountSlot (i)) {
				continue;
			}
			MESSAGE ("\t%d\t%-10ld\t0x%8.8lx\t%ld\t%ld\n", i, DMSizeSlot(i),
					DMAddrSlot (i), DMAvailSlot(i), DMCountSlot(i));
		}
	}

	return;
}
/*}}}*/
/*{{{  void dmem_chunkdump (void)*/
/*
 *	shows a dump of the pool allocator's chunks
 */
void dmem_chunkdump (void)
{
	unsigned long i;
	static char msgbuf[88];

	MESSAGE0 ("virtual address chunks:\n");

	for (i=0UL; i<NUM_CHUNKS; i+=8) {
		int offs, j;

		offs = sprintf (msgbuf, "0x%8.8x  ", (unsigned int)(i << CHUNK_SHIFT));
		for (j=i; j<(i+8); j++) {
			switch (ChunkType (j << CHUNK_SHIFT)) {
			case CHUNK_FREE:
				offs += sprintf (msgbuf + offs, "--  ");
				break;
			case CHUNK_ADDRMAPPED:
				offs += sprintf (msgbuf + offs, "AM  ");
				break;
			case CHUNK_RESERVED:
				offs += sprintf (msgbuf + offs, "RES ");
				break;
			case CHUNK_HEAP:
				offs += sprintf (msgbuf + offs, "HP  ");
				break;
			default:
				offs += sprintf (msgbuf + offs, "?%2.2x ", ChunkType (j << CHUNK_SHIFT));
				break;
			}
		}
		MESSAGE ("%s\n", msgbuf);
	}
	return;
}
/*}}}*/

/*{{{  static void dmem_getnpools (int *target)*/
/*
 *	void dmem_getnpools (int *target)
 *	PROC C.ccsp.dmem.getnpools (INT count)
 *	provides the number of memory pools for an occam program
 */
static void dmem_getnpools (int *target)
{
	*target = N_POOLS;
	return;
}
void _ccsp_dmem_getnpools (int *ws) { dmem_getnpools ((int *)(ws[0])); }
/*}}}*/
/*{{{  static void dmem_getpoolinfo (int pool, int *size, int *addr, int *avail, int *count)*/
/*
 *	void dmem_getpoolinfo (int pool, int *size, int *addr, int *avail, int *count)
 *	PROC C.ccsp.dmem.getpoolinfo (VAL INT pool, INT size, addr, avail, count)
 *	provides information about a specific pool for an occam program
 */
static void dmem_getpoolinfo (int pool, int *size, int *addr, int *avail, int *count)
{
	if ((pool < 0) || (pool >= N_POOLS)) {
		*size = 0;
		*addr = 0;
		*avail = 0;
		*count = 0;
	} else {
		*size = DMSizeSlot (pool);
		*addr = DMAddrSlot (pool);
		*avail = DMAvailSlot (pool);
		*count = DMCountSlot (pool);
	}
	return;
}
void _ccsp_dmem_getpoolinfo (int *ws) { dmem_getpoolinfo ((int)(ws[0]), (int *)(ws[1]), (int *)(ws[2]), (int *)(ws[3]), (int *)(ws[4])); }
/*}}}*/


#if defined(BLOCKING_SYSCALLS) && !defined(OOS_BUILD)
/*{{{  void *dmem_locked_malloc (const int nbytes)*/
void *dmem_locked_malloc (const int nbytes)
{
	void *blk = NULL;

#ifdef USE_PTHREADS
	pthread_mutex_lock (&memory_lock);
	blk = malloc (nbytes);
	pthread_mutex_unlock (&memory_lock);
#else
	spl_lock (&memory_lock);
#if defined(OOS_BUILD)
	blk = oos_kmalloc (nbytes, 0);
#else
	blk = malloc (nbytes);
#endif
	spl_unlock (&memory_lock);
#endif

	return blk;
}
/*}}}*/
/*{{{  void dmem_locked_free (const void *ptr)*/
void dmem_locked_free (const void *ptr)
{
#ifdef USE_PTHREADS
	pthread_mutex_lock (&memory_lock);
	free ((void *)ptr);
	pthread_mutex_unlock (&memory_lock);
#else
	spl_lock (&memory_lock);
#if defined(OOS_BUILD)
	oos_kfree ((void *)ptr);
#else
	free ((void *)ptr);
#endif
	spl_unlock (&memory_lock);
#endif
	return;
}
/*}}}*/
#endif	/* defined(BLOCKING_SYSCALLS) && !defined(OOS_BUILD) */


/*{{{  void *dmem_mtnew (unsigned int *typedesc)*/
/*
 *	allocates a new mobile type from a type description
 */
void *dmem_mtnew (unsigned int *typedesc)
{
	return NULL;
}
/*}}}*/
/*{{{  void dmem_mtfree (unsigned int *typedesc, void *ptr)*/
/*
 *	frees a mobile type, uses type description to free deep stuff
 */
void dmem_mtfree (unsigned int *typedesc, void *ptr)
{
	return;
}
/*}}}*/
/*{{{  void *dmem_mtclone (unsigned int *typedesc, void *ptr)*/
/*
 *	clones a mobile type, uses type description
 */
void *dmem_mtclone (unsigned int *typedesc, void *ptr)
{
	return NULL;
}
/*}}}*/


