[prev in list] [next in list] [prev in thread] [next in thread] 

List:       linux-smp
Subject:    Problems with mmaping shared memory...
From:       Manuel Eduardo Correia <mcc () ncc ! up ! pt>
Date:       1999-07-28 14:51:36
[Download RAW message or body]


Hi,

I have been having for some time problems with mmaped shared memory in
SMP systems. The problem does not manifest itself if when instead of
mmap I use SysV shm.

To exemplify please test the following code in a SMP machine (it works
ok if SMP is not enabled). I have tested it with 2.2.10 and it still
does not work with mmap !!

Compile with "gcc -O -o lock_test lock_test.c"

Test with for ex) lock_test shm 20 1000 ; lock_test mmap 20 1000

With mmap sometimes we deadlock, some other times when we arrive at
the end, the result is wrong most of the times... Everything works
fine in without SMP or in SMP if SysV shm is used instead of mmap to
allocate shared memory.

Any explanation or bug fix would be tremendously helpfull !!

Thanks,

------------------------------------------------------------------------
Manuel Eduardo C. D. Correia 
------------------------------------------------------------------------
LIACC,
Rua do Campo Alegre, 823, 4150 Porto, PORTUGAL
Tel: (351-2) 607 8830, Ext: 114, Fax: (351-2) 600 3654,
Internet: mcc@ncc.up.pt 
------------------------------------------------------------------------


########### parallel.h #####################

#ifndef LOCK

/*
 * Some hacks to defeat gcc over-optimizations..
 */
struct __dummy { unsigned long a[100]; };
#define ADDR (*(struct __dummy *) addr)
#define CONST_ADDR (*(const struct __dummy *) addr)

#define MAX_NUMBER_OF_WORKERS 32

typedef volatile int slock_t;


extern inline int set_bit(int nr, volatile void * addr)
{
	int oldbit;

	asm volatile ("
                lock ;
		btsl %2,%1\n\tsbbl %0,%0"
		:"=r" (oldbit),"=m" (ADDR)
		:"ir" (nr));
	return oldbit;
}


extern inline int clear_bit(int nr, volatile void * addr)
{
	int oldbit;

	asm volatile(" 
                lock ;
		btrl %2,%1\n\tsbbl %0,%0"
		:"=r" (oldbit),"=m" (ADDR)
		:"ir" (nr));
	return oldbit;
}

#define S_LOCK(p)\
{\
  while(set_bit(0,(p)) ) ;\
}
	
#define S_UNLOCK(p) (clear_bit(0,(p)))

#define S_CLOCK(p) (!set_bit(0,(p)))

#define S_INIT_LOCK(p)	S_UNLOCK(p)


typedef slock_t LOCKTYPE;
#define LOCK(X) S_LOCK(&(X))
#define UNLOCK(X) S_UNLOCK(&(X))
#define INIT_LOCK(X) S_INIT_LOCK(&(X))
#define COND_LOCK(X) S_CLOCK(&(X))


#endif LOCK


########### parallel.h #####################

########### lock_test.c #####################


#include <stdio.h>
#include "parallel.h"
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/ipc.h>
#include <sys/shm.h>


#define SHM 1
#define MMAP 2

int  shared_mem = MMAP;

typedef struct {
  int last_worker;
  slock_t lock;
  volatile long counter;
  volatile int n_workers;
} COUNTER;


long n_times;

pid_t *worker_pid;

work(int worker, COUNTER *c)
{
  int i;

  if( worker > 0 )
    {
      if( (i = fork()) ) 
	{
	  return;
	}
      else 
	if( i == -1 )
	  return -1;
      else 
	if(  worker_pid[worker] == -1 )
	  {
	    printf("Could not initiate worker[%d]", worker );
	    perror("");
	  }
    }
  
  worker_pid[worker] = getpid();

  printf("Worker[pid=%-4d,n=%-4d] started\n", worker_pid[worker],worker);
  fflush(stdout);

  for( i = 0 ; i < n_times; i++ ) 
    {
      
#ifdef ATOMIC

      ATOMIC_INCR(&(c->counter));
#else
      LOCK(c->lock);

#ifdef MCC_DEBUG      
      if( worker != c->last_worker && c->last_worker != -1 ) 
	{
	  fprintf(stderr,"worker[pid=%-4d,n=%-2d] grabbed lock from \
worker[pid=%-4d,n=%-2d]\t",   worker_pid[worker], worker, 
		 worker_pid[c->last_worker],
		 c->last_worker );
	  fprintf(stderr,"counter=%d\n", c->counter );
	  fflush(stdout);
	}

      c->last_worker = worker;
#endif
      c->counter ++;
      
      UNLOCK(c->lock);
#endif

    }
  
  printf("Worker[%d] ended\n", worker);
  fflush(stdout);
  

  LOCK(c->lock);
    --c->n_workers;
  UNLOCK(c->lock);

  printf("Worker[%d] exiting with counter=%d\n", worker, c->counter );


if( worker != 0 )  exit(0);

}


main(int argc, char *argv[])
{
  COUNTER *counter;
  int n_workers,i;
  int fd_counter, fd_pid;
  char *c;

  int shmid;

  umask(077);

  if( argc < 3 ) 
    {
      printf("Usage : %s mmap|shm <n_workers> <number>\n", *argv );
      exit(1);
    }
  else c = argv[1];

      

  if( strcmp(c,"mmap" ) == 0 ) 
    shared_mem = MMAP;
  else if(  strcmp(c,"shm") == 0 ) 
    shared_mem = SHM;
  else
    {
      printf("Usage : %s -mmap|shm <n_workers> <number>\n", *argv );
      exit(1);
    }

  if( shared_mem == MMAP )
    {

      printf("Using BSD mmap\n");
      fflush(stdout);

      if( (fd_counter = open("./foo", O_RDWR, 0700 ))  == -1  )
	{
	  perror("Could not open ./foo\n");
	  exit(3);
	}

      if( (long) ( counter = (COUNTER *) 
		   mmap( 0, sizeof(COUNTER), 
			 PROT_READ | PROT_WRITE, MAP_SHARED ,
			 fd_counter , 0 ) ) == -1 )
       {

	 perror("Allocation of shared memory failed");
	 exit(2);
       }
    }
  else 
    {
      printf("Using SYSV SHM  \n");
      fflush(stdout);

      if( ( shmid = shmget(IPC_PRIVATE, sizeof(COUNTER), 0600)) == -1 ) 
	{
	  perror("shmget failed :");
	  exit(1);
	}

      if( ( counter = (COUNTER *) shmat(shmid,0,0) ) == (void *) -1 )
	{
	  perror("shmat failed");
	}
    }
  
  printf("\nCounter allocated at %lx\n\n", (long) counter );
  fflush(stdout);

  n_workers = atoi(argv[2]);

  if( (fd_pid = open("./pid_t", O_RDWR, 0700 ))  == -1  )
    {
      perror("Could not open ./pid_t\n");
      exit(3);
    }
  if( (  worker_pid = (pid_t *) mmap( 0,sizeof(pid_t) * n_workers,  PROT_READ | \
PROT_WRITE, MAP_SHARED, fd_pid , 0 ) ) == (pid_t *) -1 )  {
      printf("Could not allocate pid_t space, Aborting --> ");
      perror(NULL);
    }

  worker_pid[0] = getpid();
  n_times = atoi(argv[3]);

  INIT_LOCK(counter->lock);
  counter -> counter = 0;
  counter -> last_worker = -1;
  counter -> n_workers = n_workers;

  for( i = 1 ; i < n_workers ; i++ ) work(i, counter );

  work(0, counter);


  while( counter->n_workers  ) ; /* Wait for everyone to finish */


  puts("\n\n");

  for( i = 1 ; i < n_workers  ; i++ ) 
    {
      printf("Going to wait for process %d to finish\n", worker_pid[i]);
      waitpid(worker_pid[i], NULL, 0 );
      printf("Worker[%d] finished\n", i  );
    }

  puts("Worker[0] finished\n");
  
  
  printf("\n\n -----> %d * %d == %d <----- \n\n", n_workers, n_times , counter -> \
counter );


  if( shared_mem == SHM ) 
    {
      shmdt((void *)counter);
      shmctl(shmid,IPC_RMID,NULL);
    }


}

########### lock_test.c #####################


-
Linux SMP list: FIRST see FAQ at http://www.irisa.fr/prive/mentre/smp-faq/
To Unsubscribe: send "unsubscribe linux-smp" to majordomo@vger.rutgers.edu


[prev in list] [next in list] [prev in thread] [next in thread]