arch/powerpc/oprofile/cell/spu_task_sync.c - kernel/windcharger - Git at Google

 /*
  * Cell Broadband Engine OProfile Support
  *
  * (C) Copyright IBM Corporation 2006
  *
  * Author: Maynard Johnson <maynardj@us.ibm.com>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */

 /* The purpose of this file is to handle SPU event task switching
  * and to record SPU context information into the OProfile
  * event buffer.
  *
  * Additionally, the spu_sync_buffer function is provided as a helper
  * for recoding actual SPU program counter samples to the event buffer.
  */
 #include <linux/dcookies.h>
 #include <linux/kref.h>
 #include <linux/mm.h>
 #include <linux/fs.h>
 #include <linux/module.h>
 #include <linux/notifier.h>
 #include <linux/numa.h>
 #include <linux/oprofile.h>
 #include <linux/spinlock.h>
 #include "pr_util.h"

 #define RELEASE_ALL 9999

 static DEFINE_SPINLOCK(buffer_lock);
 static DEFINE_SPINLOCK(cache_lock);
 static int num_spu_nodes;
 int spu_prof_num_nodes;

 struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
 struct delayed_work spu_work;
 static unsigned max_spu_buff;

 static void spu_buff_add(unsigned long int value, int spu)
 {
 	/* spu buff is a circular buffer.  Add entries to the
 	 * head.  Head is the index to store the next value.
 	 * The buffer is full when there is one available entry
 	 * in the queue, i.e. head and tail can't be equal.
 	 * That way we can tell the difference between the
 	 * buffer being full versus empty.
 	 *
 	 *  ASSUPTION: the buffer_lock is held when this function
 	 *             is called to lock the buffer, head and tail.
 	 */
 	int full = 1;

 	if (spu_buff[spu].head >= spu_buff[spu].tail) {
 		if ((spu_buff[spu].head - spu_buff[spu].tail)
 		    <  (max_spu_buff - 1))
 			full = 0;

 	} else if (spu_buff[spu].tail > spu_buff[spu].head) {
 		if ((spu_buff[spu].tail - spu_buff[spu].head)
 		    > 1)
 			full = 0;
 	}

 	if (!full) {
 		spu_buff[spu].buff[spu_buff[spu].head] = value;
 		spu_buff[spu].head++;

 		if (spu_buff[spu].head >= max_spu_buff)
 			spu_buff[spu].head = 0;
 	} else {
 		/* From the user's perspective make the SPU buffer
 		 * size management/overflow look like we are using
 		 * per cpu buffers.  The user uses the same
 		 * per cpu parameter to adjust the SPU buffer size.
 		 * Increment the sample_lost_overflow to inform
 		 * the user the buffer size needs to be increased.
 		 */
 		oprofile_cpu_buffer_inc_smpl_lost();
 	}
 }

 /* This function copies the per SPU buffers to the
  * OProfile kernel buffer.
  */
 void sync_spu_buff(void)
 {
 	int spu;
 	unsigned long flags;
 	int curr_head;

 	for (spu = 0; spu < num_spu_nodes; spu++) {
 		/* In case there was an issue and the buffer didn't
 		 * get created skip it.
 		 */
 		if (spu_buff[spu].buff == NULL)
 			continue;

 		/* Hold the lock to make sure the head/tail
 		 * doesn't change while spu_buff_add() is
 		 * deciding if the buffer is full or not.
 		 * Being a little paranoid.
 		 */
 		spin_lock_irqsave(&buffer_lock, flags);
 		curr_head = spu_buff[spu].head;
 		spin_unlock_irqrestore(&buffer_lock, flags);

 		/* Transfer the current contents to the kernel buffer.
 		 * data can still be added to the head of the buffer.
 		 */
 		oprofile_put_buff(spu_buff[spu].buff,
 				  spu_buff[spu].tail,
 				  curr_head, max_spu_buff);

 		spin_lock_irqsave(&buffer_lock, flags);
 		spu_buff[spu].tail = curr_head;
 		spin_unlock_irqrestore(&buffer_lock, flags);
 	}

 }

 static void wq_sync_spu_buff(struct work_struct *work)
 {
 	/* move data from spu buffers to kernel buffer */
 	sync_spu_buff();

 	/* only reschedule if profiling is not done */
 	if (spu_prof_running)
 		schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
 }

 /* Container for caching information about an active SPU task. */
 struct cached_info {
 	struct vma_to_fileoffset_map *map;
 	struct spu *the_spu;	/* needed to access pointer to local_store */
 	struct kref cache_ref;
 };

 static struct cached_info *spu_info[MAX_NUMNODES * 8];

 static void destroy_cached_info(struct kref *kref)
 {
 	struct cached_info *info;

 	info = container_of(kref, struct cached_info, cache_ref);
 	vma_map_free(info->map);
 	kfree(info);
 	module_put(THIS_MODULE);
 }

 /* Return the cached_info for the passed SPU number.
  * ATTENTION:  Callers are responsible for obtaining the
  *	       cache_lock if needed prior to invoking this function.
  */
 static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
 {
 	struct kref *ref;
 	struct cached_info *ret_info;

 	if (spu_num >= num_spu_nodes) {
 		printk(KERN_ERR "SPU_PROF: "
 		       "%s, line %d: Invalid index %d into spu info cache\n",
 		       __func__, __LINE__, spu_num);
 		ret_info = NULL;
 		goto out;
 	}
 	if (!spu_info[spu_num] && the_spu) {
 		ref = spu_get_profile_private_kref(the_spu->ctx);
 		if (ref) {
 			spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
 			kref_get(&spu_info[spu_num]->cache_ref);
 		}
 	}

 	ret_info = spu_info[spu_num];
  out:
 	return ret_info;
 }


 /* Looks for cached info for the passed spu.  If not found, the
  * cached info is created for the passed spu.
  * Returns 0 for success; otherwise, -1 for error.
  */
 static int
 prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
 {
 	unsigned long flags;
 	struct vma_to_fileoffset_map *new_map;
 	int retval = 0;
 	struct cached_info *info;

 	/* We won't bother getting cache_lock here since
 	 * don't do anything with the cached_info that's returned.
 	 */
 	info = get_cached_info(spu, spu->number);

 	if (info) {
 		pr_debug("Found cached SPU info.\n");
 		goto out;
 	}

 	/* Create cached_info and set spu_info[spu->number] to point to it.
 	 * spu->number is a system-wide value, not a per-node value.
 	 */
 	info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
 	if (!info) {
 		printk(KERN_ERR "SPU_PROF: "
 		       "%s, line %d: create vma_map failed\n",
 		       __func__, __LINE__);
 		retval = -ENOMEM;
 		goto err_alloc;
 	}
 	new_map = create_vma_map(spu, objectId);
 	if (!new_map) {
 		printk(KERN_ERR "SPU_PROF: "
 		       "%s, line %d: create vma_map failed\n",
 		       __func__, __LINE__);
 		retval = -ENOMEM;
 		goto err_alloc;
 	}

 	pr_debug("Created vma_map\n");
 	info->map = new_map;
 	info->the_spu = spu;
 	kref_init(&info->cache_ref);
 	spin_lock_irqsave(&cache_lock, flags);
 	spu_info[spu->number] = info;
 	/* Increment count before passing off ref to SPUFS. */
 	kref_get(&info->cache_ref);

 	/* We increment the module refcount here since SPUFS is
 	 * responsible for the final destruction of the cached_info,
 	 * and it must be able to access the destroy_cached_info()
 	 * function defined in the OProfile module.  We decrement
 	 * the module refcount in destroy_cached_info.
 	 */
 	try_module_get(THIS_MODULE);
 	spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
 				destroy_cached_info);
 	spin_unlock_irqrestore(&cache_lock, flags);
 	goto out;

 err_alloc:
 	kfree(info);
 out:
 	return retval;
 }

 /*
  * NOTE:  The caller is responsible for locking the
  *	  cache_lock prior to calling this function.
  */
 static int release_cached_info(int spu_index)
 {
 	int index, end;

 	if (spu_index == RELEASE_ALL) {
 		end = num_spu_nodes;
 		index = 0;
 	} else {
 		if (spu_index >= num_spu_nodes) {
 			printk(KERN_ERR "SPU_PROF: "
 				"%s, line %d: "
 				"Invalid index %d into spu info cache\n",
 				__func__, __LINE__, spu_index);
 			goto out;
 		}
 		end = spu_index + 1;
 		index = spu_index;
 	}
 	for (; index < end; index++) {
 		if (spu_info[index]) {
 			kref_put(&spu_info[index]->cache_ref,
 				 destroy_cached_info);
 			spu_info[index] = NULL;
 		}
 	}

 out:
 	return 0;
 }

 /* The source code for fast_get_dcookie was "borrowed"
  * from drivers/oprofile/buffer_sync.c.
  */

 /* Optimisation. We can manage without taking the dcookie sem
  * because we cannot reach this code without at least one
  * dcookie user still being registered (namely, the reader
  * of the event buffer).
  */
 static inline unsigned long fast_get_dcookie(struct path *path)
 {
 	unsigned long cookie;

 	if (path->dentry->d_flags & DCACHE_COOKIE)
 		return (unsigned long)path->dentry;
 	get_dcookie(path, &cookie);
 	return cookie;
 }

 /* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
  * which corresponds loosely to "application name". Also, determine
  * the offset for the SPU ELF object.  If computed offset is
  * non-zero, it implies an embedded SPU object; otherwise, it's a
  * separate SPU binary, in which case we retrieve it's dcookie.
  * For the embedded case, we must determine if SPU ELF is embedded
  * in the executable application or another file (i.e., shared lib).
  * If embedded in a shared lib, we must get the dcookie and return
  * that to the caller.
  */
 static unsigned long
 get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
 			    unsigned long *spu_bin_dcookie,
 			    unsigned long spu_ref)
 {
 	unsigned long app_cookie = 0;
 	unsigned int my_offset = 0;
 	struct file *app = NULL;
 	struct vm_area_struct *vma;
 	struct mm_struct *mm = spu->mm;

 	if (!mm)
 		goto out;

 	down_read(&mm->mmap_sem);

 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 		if (!vma->vm_file)
 			continue;
 		if (!(vma->vm_flags & VM_EXECUTABLE))
 			continue;
 		app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
 		pr_debug("got dcookie for %s\n",
 			 vma->vm_file->f_dentry->d_name.name);
 		app = vma->vm_file;
 		break;
 	}

 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 		if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
 			continue;
 		my_offset = spu_ref - vma->vm_start;
 		if (!vma->vm_file)
 			goto fail_no_image_cookie;

 		pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
 			 my_offset, spu_ref,
 			 vma->vm_file->f_dentry->d_name.name);
 		*offsetp = my_offset;
 		break;
 	}

 	*spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
 	pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);

 	up_read(&mm->mmap_sem);

 out:
 	return app_cookie;

 fail_no_image_cookie:
 	up_read(&mm->mmap_sem);

 	printk(KERN_ERR "SPU_PROF: "
 		"%s, line %d: Cannot find dcookie for SPU binary\n",
 		__func__, __LINE__);
 	goto out;
 }


 /* This function finds or creates cached context information for the
  * passed SPU and records SPU context information into the OProfile
  * event buffer.
  */
 static int process_context_switch(struct spu *spu, unsigned long objectId)
 {
 	unsigned long flags;
 	int retval;
 	unsigned int offset = 0;
 	unsigned long spu_cookie = 0, app_dcookie;

 	retval = prepare_cached_spu_info(spu, objectId);
 	if (retval)
 		goto out;

 	/* Get dcookie first because a mutex_lock is taken in that
 	 * code path, so interrupts must not be disabled.
 	 */
 	app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
 	if (!app_dcookie || !spu_cookie) {
 		retval  = -ENOENT;
 		goto out;
 	}

 	/* Record context info in event buffer */
 	spin_lock_irqsave(&buffer_lock, flags);
 	spu_buff_add(ESCAPE_CODE, spu->number);
 	spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
 	spu_buff_add(spu->number, spu->number);
 	spu_buff_add(spu->pid, spu->number);
 	spu_buff_add(spu->tgid, spu->number);
 	spu_buff_add(app_dcookie, spu->number);
 	spu_buff_add(spu_cookie, spu->number);
 	spu_buff_add(offset, spu->number);

 	/* Set flag to indicate SPU PC data can now be written out.  If
 	 * the SPU program counter data is seen before an SPU context
 	 * record is seen, the postprocessing will fail.
 	 */
 	spu_buff[spu->number].ctx_sw_seen = 1;

 	spin_unlock_irqrestore(&buffer_lock, flags);
 	smp_wmb();	/* insure spu event buffer updates are written */
 			/* don't want entries intermingled... */
 out:
 	return retval;
 }

 /*
  * This function is invoked on either a bind_context or unbind_context.
  * If called for an unbind_context, the val arg is 0; otherwise,
  * it is the object-id value for the spu context.
  * The data arg is of type 'struct spu *'.
  */
 static int spu_active_notify(struct notifier_block *self, unsigned long val,
 				void *data)
 {
 	int retval;
 	unsigned long flags;
 	struct spu *the_spu = data;

 	pr_debug("SPU event notification arrived\n");
 	if (!val) {
 		spin_lock_irqsave(&cache_lock, flags);
 		retval = release_cached_info(the_spu->number);
 		spin_unlock_irqrestore(&cache_lock, flags);
 	} else {
 		retval = process_context_switch(the_spu, val);
 	}
 	return retval;
 }

 static struct notifier_block spu_active = {
 	.notifier_call = spu_active_notify,
 };

 static int number_of_online_nodes(void)
 {
         u32 cpu; u32 tmp;
         int nodes = 0;
         for_each_online_cpu(cpu) {
                 tmp = cbe_cpu_to_node(cpu) + 1;
                 if (tmp > nodes)
                         nodes++;
         }
         return nodes;
 }

 static int oprofile_spu_buff_create(void)
 {
 	int spu;

 	max_spu_buff = oprofile_get_cpu_buffer_size();

 	for (spu = 0; spu < num_spu_nodes; spu++) {
 		/* create circular buffers to store the data in.
 		 * use locks to manage accessing the buffers
 		 */
 		spu_buff[spu].head = 0;
 		spu_buff[spu].tail = 0;

 		/*
 		 * Create a buffer for each SPU.  Can't reliably
 		 * create a single buffer for all spus due to not
 		 * enough contiguous kernel memory.
 		 */

 		spu_buff[spu].buff = kzalloc((max_spu_buff
 					      * sizeof(unsigned long)),
 					     GFP_KERNEL);

 		if (!spu_buff[spu].buff) {
 			printk(KERN_ERR "SPU_PROF: "
 			       "%s, line %d:  oprofile_spu_buff_create "
 		       "failed to allocate spu buffer %d.\n",
 			       __func__, __LINE__, spu);

 			/* release the spu buffers that have been allocated */
 			while (spu >= 0) {
 				kfree(spu_buff[spu].buff);
 				spu_buff[spu].buff = 0;
 				spu--;
 			}
 			return -ENOMEM;
 		}
 	}
 	return 0;
 }

 /* The main purpose of this function is to synchronize
  * OProfile with SPUFS by registering to be notified of
  * SPU task switches.
  *
  * NOTE: When profiling SPUs, we must ensure that only
  * spu_sync_start is invoked and not the generic sync_start
  * in drivers/oprofile/oprof.c.	 A return value of
  * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
  * accomplish this.
  */
 int spu_sync_start(void)
 {
 	int spu;
 	int ret = SKIP_GENERIC_SYNC;
 	int register_ret;
 	unsigned long flags = 0;

 	spu_prof_num_nodes = number_of_online_nodes();
 	num_spu_nodes = spu_prof_num_nodes * 8;
 	INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);

 	/* create buffer for storing the SPU data to put in
 	 * the kernel buffer.
 	 */
 	ret = oprofile_spu_buff_create();
 	if (ret)
 		goto out;

 	spin_lock_irqsave(&buffer_lock, flags);
 	for (spu = 0; spu < num_spu_nodes; spu++) {
 		spu_buff_add(ESCAPE_CODE, spu);
 		spu_buff_add(SPU_PROFILING_CODE, spu);
 		spu_buff_add(num_spu_nodes, spu);
 	}
 	spin_unlock_irqrestore(&buffer_lock, flags);

 	for (spu = 0; spu < num_spu_nodes; spu++) {
 		spu_buff[spu].ctx_sw_seen = 0;
 		spu_buff[spu].last_guard_val = 0;
 	}

 	/* Register for SPU events  */
 	register_ret = spu_switch_event_register(&spu_active);
 	if (register_ret) {
 		ret = SYNC_START_ERROR;
 		goto out;
 	}

 	pr_debug("spu_sync_start -- running.\n");
 out:
 	return ret;
 }

 /* Record SPU program counter samples to the oprofile event buffer. */
 void spu_sync_buffer(int spu_num, unsigned int *samples,
 		     int num_samples)
 {
 	unsigned long long file_offset;
 	unsigned long flags;
 	int i;
 	struct vma_to_fileoffset_map *map;
 	struct spu *the_spu;
 	unsigned long long spu_num_ll = spu_num;
 	unsigned long long spu_num_shifted = spu_num_ll << 32;
 	struct cached_info *c_info;

 	/* We need to obtain the cache_lock here because it's
 	 * possible that after getting the cached_info, the SPU job
 	 * corresponding to this cached_info may end, thus resulting
 	 * in the destruction of the cached_info.
 	 */
 	spin_lock_irqsave(&cache_lock, flags);
 	c_info = get_cached_info(NULL, spu_num);
 	if (!c_info) {
 		/* This legitimately happens when the SPU task ends before all
 		 * samples are recorded.
 		 * No big deal -- so we just drop a few samples.
 		 */
 		pr_debug("SPU_PROF: No cached SPU contex "
 			  "for SPU #%d. Dropping samples.\n", spu_num);
 		goto out;
 	}

 	map = c_info->map;
 	the_spu = c_info->the_spu;
 	spin_lock(&buffer_lock);
 	for (i = 0; i < num_samples; i++) {
 		unsigned int sample = *(samples+i);
 		int grd_val = 0;
 		file_offset = 0;
 		if (sample == 0)
 			continue;
 		file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);

 		/* If overlays are used by this SPU application, the guard
 		 * value is non-zero, indicating which overlay section is in
 		 * use.	 We need to discard samples taken during the time
 		 * period which an overlay occurs (i.e., guard value changes).
 		 */
 		if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
 			spu_buff[spu_num].last_guard_val = grd_val;
 			/* Drop the rest of the samples. */
 			break;
 		}

 		/* We must ensure that the SPU context switch has been written
 		 * out before samples for the SPU.  Otherwise, the SPU context
 		 * information is not available and the postprocessing of the
 		 * SPU PC will fail with no available anonymous map information.
 		 */
 		if (spu_buff[spu_num].ctx_sw_seen)
 			spu_buff_add((file_offset | spu_num_shifted),
 					 spu_num);
 	}
 	spin_unlock(&buffer_lock);
 out:
 	spin_unlock_irqrestore(&cache_lock, flags);
 }


 int spu_sync_stop(void)
 {
 	unsigned long flags = 0;
 	int ret;
 	int k;

 	ret = spu_switch_event_unregister(&spu_active);

 	if (ret)
 		printk(KERN_ERR "SPU_PROF: "
 		       "%s, line %d: spu_switch_event_unregister "	\
 		       "returned %d\n",
 		       __func__, __LINE__, ret);

 	/* flush any remaining data in the per SPU buffers */
 	sync_spu_buff();

 	spin_lock_irqsave(&cache_lock, flags);
 	ret = release_cached_info(RELEASE_ALL);
 	spin_unlock_irqrestore(&cache_lock, flags);

 	/* remove scheduled work queue item rather then waiting
 	 * for every queued entry to execute.  Then flush pending
 	 * system wide buffer to event buffer.
 	 */
 	cancel_delayed_work(&spu_work);

 	for (k = 0; k < num_spu_nodes; k++) {
 		spu_buff[k].ctx_sw_seen = 0;

 		/*
 		 * spu_sys_buff will be null if there was a problem
 		 * allocating the buffer.  Only delete if it exists.
 		 */
 		kfree(spu_buff[k].buff);
 		spu_buff[k].buff = 0;
 	}
 	pr_debug("spu_sync_stop -- done.\n");
 	return ret;
 }
	/*
	* Cell Broadband Engine OProfile Support
	*
	* (C) Copyright IBM Corporation 2006
	*
	* Author: Maynard Johnson <maynardj@us.ibm.com>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public License
	* as published by the Free Software Foundation; either version
	* 2 of the License, or (at your option) any later version.
	*/

	/* The purpose of this file is to handle SPU event task switching
	* and to record SPU context information into the OProfile
	* event buffer.
	*
	* Additionally, the spu_sync_buffer function is provided as a helper
	* for recoding actual SPU program counter samples to the event buffer.
	*/
	#include <linux/dcookies.h>
	#include <linux/kref.h>
	#include <linux/mm.h>
	#include <linux/fs.h>
	#include <linux/module.h>
	#include <linux/notifier.h>
	#include <linux/numa.h>
	#include <linux/oprofile.h>
	#include <linux/spinlock.h>
	#include "pr_util.h"

	#define RELEASE_ALL 9999

	static DEFINE_SPINLOCK(buffer_lock);
	static DEFINE_SPINLOCK(cache_lock);
	static int num_spu_nodes;
	int spu_prof_num_nodes;

	struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
	struct delayed_work spu_work;
	static unsigned max_spu_buff;

	static void spu_buff_add(unsigned long int value, int spu)
	{
	/* spu buff is a circular buffer. Add entries to the
	* head. Head is the index to store the next value.
	* The buffer is full when there is one available entry
	* in the queue, i.e. head and tail can't be equal.
	* That way we can tell the difference between the
	* buffer being full versus empty.
	*
	* ASSUPTION: the buffer_lock is held when this function
	* is called to lock the buffer, head and tail.
	*/
	int full = 1;

	if (spu_buff[spu].head >= spu_buff[spu].tail) {
	if ((spu_buff[spu].head - spu_buff[spu].tail)
	< (max_spu_buff - 1))
	full = 0;

	} else if (spu_buff[spu].tail > spu_buff[spu].head) {
	if ((spu_buff[spu].tail - spu_buff[spu].head)
	> 1)
	full = 0;
	}

	if (!full) {
	spu_buff[spu].buff[spu_buff[spu].head] = value;
	spu_buff[spu].head++;

	if (spu_buff[spu].head >= max_spu_buff)
	spu_buff[spu].head = 0;
	} else {
	/* From the user's perspective make the SPU buffer
	* size management/overflow look like we are using
	* per cpu buffers. The user uses the same
	* per cpu parameter to adjust the SPU buffer size.
	* Increment the sample_lost_overflow to inform
	* the user the buffer size needs to be increased.
	*/
	oprofile_cpu_buffer_inc_smpl_lost();
	}
	}

	/* This function copies the per SPU buffers to the
	* OProfile kernel buffer.
	*/
	void sync_spu_buff(void)
	{
	int spu;
	unsigned long flags;
	int curr_head;

	for (spu = 0; spu < num_spu_nodes; spu++) {
	/* In case there was an issue and the buffer didn't
	* get created skip it.
	*/
	if (spu_buff[spu].buff == NULL)
	continue;

	/* Hold the lock to make sure the head/tail
	* doesn't change while spu_buff_add() is
	* deciding if the buffer is full or not.
	* Being a little paranoid.
	*/
	spin_lock_irqsave(&buffer_lock, flags);
	curr_head = spu_buff[spu].head;
	spin_unlock_irqrestore(&buffer_lock, flags);

	/* Transfer the current contents to the kernel buffer.
	* data can still be added to the head of the buffer.
	*/
	oprofile_put_buff(spu_buff[spu].buff,
	spu_buff[spu].tail,
	curr_head, max_spu_buff);

	spin_lock_irqsave(&buffer_lock, flags);
	spu_buff[spu].tail = curr_head;
	spin_unlock_irqrestore(&buffer_lock, flags);
	}

	}

	static void wq_sync_spu_buff(struct work_struct *work)
	{
	/* move data from spu buffers to kernel buffer */
	sync_spu_buff();

	/* only reschedule if profiling is not done */
	if (spu_prof_running)
	schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
	}

	/* Container for caching information about an active SPU task. */
	struct cached_info {
	struct vma_to_fileoffset_map *map;
	struct spu the_spu; / needed to access pointer to local_store */
	struct kref cache_ref;
	};

	static struct cached_info spu_info[MAX_NUMNODES 8];

	static void destroy_cached_info(struct kref *kref)
	{
	struct cached_info *info;

	info = container_of(kref, struct cached_info, cache_ref);
	vma_map_free(info->map);
	kfree(info);
	module_put(THIS_MODULE);
	}

	/* Return the cached_info for the passed SPU number.
	* ATTENTION: Callers are responsible for obtaining the
	* cache_lock if needed prior to invoking this function.
	*/
	static struct cached_info get_cached_info(struct spu the_spu, int spu_num)
	{
	struct kref *ref;
	struct cached_info *ret_info;

	if (spu_num >= num_spu_nodes) {
	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: Invalid index %d into spu info cache\n",
	__func__, __LINE__, spu_num);
	ret_info = NULL;
	goto out;
	}
	if (!spu_info[spu_num] && the_spu) {
	ref = spu_get_profile_private_kref(the_spu->ctx);
	if (ref) {
	spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
	kref_get(&spu_info[spu_num]->cache_ref);
	}
	}

	ret_info = spu_info[spu_num];
	out:
	return ret_info;
	}


	/* Looks for cached info for the passed spu. If not found, the
	* cached info is created for the passed spu.
	* Returns 0 for success; otherwise, -1 for error.
	*/
	static int
	prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
	{
	unsigned long flags;
	struct vma_to_fileoffset_map *new_map;
	int retval = 0;
	struct cached_info *info;

	/* We won't bother getting cache_lock here since
	* don't do anything with the cached_info that's returned.
	*/
	info = get_cached_info(spu, spu->number);

	if (info) {
	pr_debug("Found cached SPU info.\n");
	goto out;
	}

	/* Create cached_info and set spu_info[spu->number] to point to it.
	* spu->number is a system-wide value, not a per-node value.
	*/
	info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
	if (!info) {
	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: create vma_map failed\n",
	__func__, __LINE__);
	retval = -ENOMEM;
	goto err_alloc;
	}
	new_map = create_vma_map(spu, objectId);
	if (!new_map) {
	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: create vma_map failed\n",
	__func__, __LINE__);
	retval = -ENOMEM;
	goto err_alloc;
	}

	pr_debug("Created vma_map\n");
	info->map = new_map;
	info->the_spu = spu;
	kref_init(&info->cache_ref);
	spin_lock_irqsave(&cache_lock, flags);
	spu_info[spu->number] = info;
	/* Increment count before passing off ref to SPUFS. */
	kref_get(&info->cache_ref);

	/* We increment the module refcount here since SPUFS is
	* responsible for the final destruction of the cached_info,
	* and it must be able to access the destroy_cached_info()
	* function defined in the OProfile module. We decrement
	* the module refcount in destroy_cached_info.
	*/
	try_module_get(THIS_MODULE);
	spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
	destroy_cached_info);
	spin_unlock_irqrestore(&cache_lock, flags);
	goto out;

	err_alloc:
	kfree(info);
	out:
	return retval;
	}

	/*
	* NOTE: The caller is responsible for locking the
	* cache_lock prior to calling this function.
	*/
	static int release_cached_info(int spu_index)
	{
	int index, end;

	if (spu_index == RELEASE_ALL) {
	end = num_spu_nodes;
	index = 0;
	} else {
	if (spu_index >= num_spu_nodes) {
	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: "
	"Invalid index %d into spu info cache\n",
	__func__, __LINE__, spu_index);
	goto out;
	}
	end = spu_index + 1;
	index = spu_index;
	}
	for (; index < end; index++) {
	if (spu_info[index]) {
	kref_put(&spu_info[index]->cache_ref,
	destroy_cached_info);
	spu_info[index] = NULL;
	}
	}

	out:
	return 0;
	}

	/* The source code for fast_get_dcookie was "borrowed"
	* from drivers/oprofile/buffer_sync.c.
	*/

	/* Optimisation. We can manage without taking the dcookie sem
	* because we cannot reach this code without at least one
	* dcookie user still being registered (namely, the reader
	* of the event buffer).
	*/
	static inline unsigned long fast_get_dcookie(struct path *path)
	{
	unsigned long cookie;

	if (path->dentry->d_flags & DCACHE_COOKIE)
	return (unsigned long)path->dentry;
	get_dcookie(path, &cookie);
	return cookie;
	}

	/* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
	* which corresponds loosely to "application name". Also, determine
	* the offset for the SPU ELF object. If computed offset is
	* non-zero, it implies an embedded SPU object; otherwise, it's a
	* separate SPU binary, in which case we retrieve it's dcookie.
	* For the embedded case, we must determine if SPU ELF is embedded
	* in the executable application or another file (i.e., shared lib).
	* If embedded in a shared lib, we must get the dcookie and return
	* that to the caller.
	*/
	static unsigned long
	get_exec_dcookie_and_offset(struct spu spu, unsigned int offsetp,
	unsigned long *spu_bin_dcookie,
	unsigned long spu_ref)
	{
	unsigned long app_cookie = 0;
	unsigned int my_offset = 0;
	struct file *app = NULL;
	struct vm_area_struct *vma;
	struct mm_struct *mm = spu->mm;

	if (!mm)
	goto out;

	down_read(&mm->mmap_sem);

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
	if (!vma->vm_file)
	continue;
	if (!(vma->vm_flags & VM_EXECUTABLE))
	continue;
	app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
	pr_debug("got dcookie for %s\n",
	vma->vm_file->f_dentry->d_name.name);
	app = vma->vm_file;
	break;
	}

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
	if (vma->vm_start > spu_ref \|\| vma->vm_end <= spu_ref)
	continue;
	my_offset = spu_ref - vma->vm_start;
	if (!vma->vm_file)
	goto fail_no_image_cookie;

	pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
	my_offset, spu_ref,
	vma->vm_file->f_dentry->d_name.name);
	*offsetp = my_offset;
	break;
	}

	*spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
	pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);

	up_read(&mm->mmap_sem);

	out:
	return app_cookie;

	fail_no_image_cookie:
	up_read(&mm->mmap_sem);

	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: Cannot find dcookie for SPU binary\n",
	__func__, __LINE__);
	goto out;
	}



	/* This function finds or creates cached context information for the
	* passed SPU and records SPU context information into the OProfile
	* event buffer.
	*/
	static int process_context_switch(struct spu *spu, unsigned long objectId)
	{
	unsigned long flags;
	int retval;
	unsigned int offset = 0;
	unsigned long spu_cookie = 0, app_dcookie;

	retval = prepare_cached_spu_info(spu, objectId);
	if (retval)
	goto out;

	/* Get dcookie first because a mutex_lock is taken in that
	* code path, so interrupts must not be disabled.
	*/
	app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
	if (!app_dcookie \|\| !spu_cookie) {
	retval = -ENOENT;
	goto out;
	}

	/* Record context info in event buffer */
	spin_lock_irqsave(&buffer_lock, flags);
	spu_buff_add(ESCAPE_CODE, spu->number);
	spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
	spu_buff_add(spu->number, spu->number);
	spu_buff_add(spu->pid, spu->number);
	spu_buff_add(spu->tgid, spu->number);
	spu_buff_add(app_dcookie, spu->number);
	spu_buff_add(spu_cookie, spu->number);
	spu_buff_add(offset, spu->number);

	/* Set flag to indicate SPU PC data can now be written out. If
	* the SPU program counter data is seen before an SPU context
	* record is seen, the postprocessing will fail.
	*/
	spu_buff[spu->number].ctx_sw_seen = 1;

	spin_unlock_irqrestore(&buffer_lock, flags);
	smp_wmb(); /* insure spu event buffer updates are written */
	/* don't want entries intermingled... */
	out:
	return retval;
	}

	/*
	* This function is invoked on either a bind_context or unbind_context.
	* If called for an unbind_context, the val arg is 0; otherwise,
	* it is the object-id value for the spu context.
	* The data arg is of type 'struct spu *'.
	*/
	static int spu_active_notify(struct notifier_block *self, unsigned long val,
	void *data)
	{
	int retval;
	unsigned long flags;
	struct spu *the_spu = data;

	pr_debug("SPU event notification arrived\n");
	if (!val) {
	spin_lock_irqsave(&cache_lock, flags);
	retval = release_cached_info(the_spu->number);
	spin_unlock_irqrestore(&cache_lock, flags);
	} else {
	retval = process_context_switch(the_spu, val);
	}
	return retval;
	}

	static struct notifier_block spu_active = {
	.notifier_call = spu_active_notify,
	};

	static int number_of_online_nodes(void)
	{
	u32 cpu; u32 tmp;
	int nodes = 0;
	for_each_online_cpu(cpu) {
	tmp = cbe_cpu_to_node(cpu) + 1;
	if (tmp > nodes)
	nodes++;
	}
	return nodes;
	}

	static int oprofile_spu_buff_create(void)
	{
	int spu;

	max_spu_buff = oprofile_get_cpu_buffer_size();

	for (spu = 0; spu < num_spu_nodes; spu++) {
	/* create circular buffers to store the data in.
	* use locks to manage accessing the buffers
	*/
	spu_buff[spu].head = 0;
	spu_buff[spu].tail = 0;

	/*
	* Create a buffer for each SPU. Can't reliably
	* create a single buffer for all spus due to not
	* enough contiguous kernel memory.
	*/

	spu_buff[spu].buff = kzalloc((max_spu_buff
	* sizeof(unsigned long)),
	GFP_KERNEL);

	if (!spu_buff[spu].buff) {
	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: oprofile_spu_buff_create "
	"failed to allocate spu buffer %d.\n",
	__func__, __LINE__, spu);

	/* release the spu buffers that have been allocated */
	while (spu >= 0) {
	kfree(spu_buff[spu].buff);
	spu_buff[spu].buff = 0;
	spu--;
	}
	return -ENOMEM;
	}
	}
	return 0;
	}

	/* The main purpose of this function is to synchronize
	* OProfile with SPUFS by registering to be notified of
	* SPU task switches.
	*
	* NOTE: When profiling SPUs, we must ensure that only
	* spu_sync_start is invoked and not the generic sync_start
	* in drivers/oprofile/oprof.c. A return value of
	* SKIP_GENERIC_SYNC or SYNC_START_ERROR will
	* accomplish this.
	*/
	int spu_sync_start(void)
	{
	int spu;
	int ret = SKIP_GENERIC_SYNC;
	int register_ret;
	unsigned long flags = 0;

	spu_prof_num_nodes = number_of_online_nodes();
	num_spu_nodes = spu_prof_num_nodes * 8;
	INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);

	/* create buffer for storing the SPU data to put in
	* the kernel buffer.
	*/
	ret = oprofile_spu_buff_create();
	if (ret)
	goto out;

	spin_lock_irqsave(&buffer_lock, flags);
	for (spu = 0; spu < num_spu_nodes; spu++) {
	spu_buff_add(ESCAPE_CODE, spu);
	spu_buff_add(SPU_PROFILING_CODE, spu);
	spu_buff_add(num_spu_nodes, spu);
	}
	spin_unlock_irqrestore(&buffer_lock, flags);

	for (spu = 0; spu < num_spu_nodes; spu++) {
	spu_buff[spu].ctx_sw_seen = 0;
	spu_buff[spu].last_guard_val = 0;
	}

	/* Register for SPU events */
	register_ret = spu_switch_event_register(&spu_active);
	if (register_ret) {
	ret = SYNC_START_ERROR;
	goto out;
	}

	pr_debug("spu_sync_start -- running.\n");
	out:
	return ret;
	}

	/* Record SPU program counter samples to the oprofile event buffer. */
	void spu_sync_buffer(int spu_num, unsigned int *samples,
	int num_samples)
	{
	unsigned long long file_offset;
	unsigned long flags;
	int i;
	struct vma_to_fileoffset_map *map;
	struct spu *the_spu;
	unsigned long long spu_num_ll = spu_num;
	unsigned long long spu_num_shifted = spu_num_ll << 32;
	struct cached_info *c_info;

	/* We need to obtain the cache_lock here because it's
	* possible that after getting the cached_info, the SPU job
	* corresponding to this cached_info may end, thus resulting
	* in the destruction of the cached_info.
	*/
	spin_lock_irqsave(&cache_lock, flags);
	c_info = get_cached_info(NULL, spu_num);
	if (!c_info) {
	/* This legitimately happens when the SPU task ends before all
	* samples are recorded.
	* No big deal -- so we just drop a few samples.
	*/
	pr_debug("SPU_PROF: No cached SPU contex "
	"for SPU #%d. Dropping samples.\n", spu_num);
	goto out;
	}

	map = c_info->map;
	the_spu = c_info->the_spu;
	spin_lock(&buffer_lock);
	for (i = 0; i < num_samples; i++) {
	unsigned int sample = *(samples+i);
	int grd_val = 0;
	file_offset = 0;
	if (sample == 0)
	continue;
	file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);

	/* If overlays are used by this SPU application, the guard
	* value is non-zero, indicating which overlay section is in
	* use. We need to discard samples taken during the time
	* period which an overlay occurs (i.e., guard value changes).
	*/
	if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
	spu_buff[spu_num].last_guard_val = grd_val;
	/* Drop the rest of the samples. */
	break;
	}

	/* We must ensure that the SPU context switch has been written
	* out before samples for the SPU. Otherwise, the SPU context
	* information is not available and the postprocessing of the
	* SPU PC will fail with no available anonymous map information.
	*/
	if (spu_buff[spu_num].ctx_sw_seen)
	spu_buff_add((file_offset \| spu_num_shifted),
	spu_num);
	}
	spin_unlock(&buffer_lock);
	out:
	spin_unlock_irqrestore(&cache_lock, flags);
	}


	int spu_sync_stop(void)
	{
	unsigned long flags = 0;
	int ret;
	int k;

	ret = spu_switch_event_unregister(&spu_active);

	if (ret)
	printk(KERN_ERR "SPU_PROF: "
	"%s, line %d: spu_switch_event_unregister " \
	"returned %d\n",
	__func__, __LINE__, ret);

	/* flush any remaining data in the per SPU buffers */
	sync_spu_buff();

	spin_lock_irqsave(&cache_lock, flags);
	ret = release_cached_info(RELEASE_ALL);
	spin_unlock_irqrestore(&cache_lock, flags);

	/* remove scheduled work queue item rather then waiting
	* for every queued entry to execute. Then flush pending
	* system wide buffer to event buffer.
	*/
	cancel_delayed_work(&spu_work);

	for (k = 0; k < num_spu_nodes; k++) {
	spu_buff[k].ctx_sw_seen = 0;

	/*
	* spu_sys_buff will be null if there was a problem
	* allocating the buffer. Only delete if it exists.
	*/
	kfree(spu_buff[k].buff);
	spu_buff[k].buff = 0;
	}
	pr_debug("spu_sync_stop -- done.\n");
	return ret;
	}