Attachment #128306 for bug #189130

Lines 105-110 Link Here

(-)lzop-1.01/config.hin (+3 lines)
105	/* Define to 1 if you have the `mktime' function. */	105	/* Define to 1 if you have the `mktime' function. */
106	#undef HAVE_MKTIME	106	#undef HAVE_MKTIME
107		107
		108	/* Define to 1 if you have the `pthread_join' function. */
		109	#undef HAVE_PTHREAD_JOIN
		110
108	/* Define to 1 if you have the <share.h> header file. */	111	/* Define to 1 if you have the <share.h> header file. */
109	#undef HAVE_SHARE_H	112	#undef HAVE_SHARE_H
110		113




done


# Begin PTHREADS mods by James Lemley, 2003
echo "$as_me:$LINENO: checking for library containing pthread_join" >&5
echo $ECHO_N "checking for library containing pthread_join... $ECHO_C" >&6
if test "${ac_cv_search_pthread_join+set}" = set; then
  echo $ECHO_N "(cached) $ECHO_C" >&6
else
  ac_func_search_save_LIBS=$LIBS
ac_cv_search_pthread_join=no
cat >conftest.$ac_ext <<_ACEOF
#line $LINENO "configure"
/* confdefs.h.  */
_ACEOF
cat confdefs.h >>conftest.$ac_ext
cat >>conftest.$ac_ext <<_ACEOF
/* end confdefs.h.  */

/* Override any gcc2 internal prototype to avoid an error.  */
#ifdef __cplusplus
extern "C"
#endif
/* We use char because int might match the return type of a gcc2
   builtin and then its argument prototype would still apply.  */
char pthread_join ();
int
main ()
{
pthread_join ();
  ;
  return 0;
}
_ACEOF
rm -f conftest.$ac_objext conftest$ac_exeext
if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
  (eval $ac_link) 2>&5
  ac_status=$?
  echo "$as_me:$LINENO: \$? = $ac_status" >&5
  (exit $ac_status); } &&
         { ac_try='test -s conftest$ac_exeext'
  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
  (eval $ac_try) 2>&5
  ac_status=$?
  echo "$as_me:$LINENO: \$? = $ac_status" >&5
  (exit $ac_status); }; }; then
  ac_cv_search_pthread_join="none required"
else
  echo "$as_me: failed program was:" >&5
sed 's/^/| /' conftest.$ac_ext >&5

fi
rm -f conftest.$ac_objext conftest$ac_exeext conftest.$ac_ext
if test "$ac_cv_search_pthread_join" = no; then
  for ac_lib in pthread pthreads; do
    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
    cat >conftest.$ac_ext <<_ACEOF
#line $LINENO "configure"
/* confdefs.h.  */
_ACEOF
cat confdefs.h >>conftest.$ac_ext
cat >>conftest.$ac_ext <<_ACEOF
/* end confdefs.h.  */

/* Override any gcc2 internal prototype to avoid an error.  */
#ifdef __cplusplus
extern "C"
#endif
/* We use char because int might match the return type of a gcc2
   builtin and then its argument prototype would still apply.  */
char pthread_join ();
int
main ()
{
pthread_join ();
  ;
  return 0;
}
_ACEOF
rm -f conftest.$ac_objext conftest$ac_exeext
if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
  (eval $ac_link) 2>&5
  ac_status=$?
  echo "$as_me:$LINENO: \$? = $ac_status" >&5
  (exit $ac_status); } &&
         { ac_try='test -s conftest$ac_exeext'
  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
  (eval $ac_try) 2>&5
  ac_status=$?
  echo "$as_me:$LINENO: \$? = $ac_status" >&5
  (exit $ac_status); }; }; then
  ac_cv_search_pthread_join="-l$ac_lib"
break
else
  echo "$as_me: failed program was:" >&5
sed 's/^/| /' conftest.$ac_ext >&5

fi
rm -f conftest.$ac_objext conftest$ac_exeext conftest.$ac_ext
  done
fi
LIBS=$ac_func_search_save_LIBS
fi
echo "$as_me:$LINENO: result: $ac_cv_search_pthread_join" >&5
echo "${ECHO_T}$ac_cv_search_pthread_join" >&6
if test "$ac_cv_search_pthread_join" != no; then
  test "$ac_cv_search_pthread_join" = "none required" || LIBS="$ac_cv_search_pthread_join $LIBS"

else

        echo "Can't find a POSIX threads library.  If one is actually installed,"
        echo "please fix configure.in and submit a patch back to the maintainer."
        echo "Thanks!"
        exit 1

fi


for ac_func in pthread_join
do
as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
echo "$as_me:$LINENO: checking for $ac_func" >&5
echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
if eval "test \"\${$as_ac_var+set}\" = set"; then
  echo $ECHO_N "(cached) $ECHO_C" >&6
else
  cat >conftest.$ac_ext <<_ACEOF
#line $LINENO "configure"
/* confdefs.h.  */
_ACEOF
cat confdefs.h >>conftest.$ac_ext
cat >>conftest.$ac_ext <<_ACEOF
/* end confdefs.h.  */
/* System header to define __stub macros and hopefully few prototypes,
    which can conflict with char $ac_func (); below.
    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
    <limits.h> exists even on freestanding compilers.  */
#ifdef __STDC__
# include <limits.h>
#else
# include <assert.h>
#endif
/* Override any gcc2 internal prototype to avoid an error.  */
#ifdef __cplusplus
extern "C"
{
#endif
/* We use char because int might match the return type of a gcc2
   builtin and then its argument prototype would still apply.  */
char $ac_func ();
/* The GNU C library defines this for functions which it implements
    to always fail with ENOSYS.  Some functions are actually named
    something starting with __ and the normal name is an alias.  */
#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
choke me
#else
char (*f) () = $ac_func;
#endif
#ifdef __cplusplus
}
#endif

int
main ()
{
return f != $ac_func;
  ;
  return 0;
}
_ACEOF
rm -f conftest.$ac_objext conftest$ac_exeext
if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
  (eval $ac_link) 2>&5
  ac_status=$?
  echo "$as_me:$LINENO: \$? = $ac_status" >&5
  (exit $ac_status); } &&
         { ac_try='test -s conftest$ac_exeext'
  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
  (eval $ac_try) 2>&5
  ac_status=$?
  echo "$as_me:$LINENO: \$? = $ac_status" >&5
  (exit $ac_status); }; }; then
  eval "$as_ac_var=yes"
else
  echo "$as_me: failed program was:" >&5
sed 's/^/| /' conftest.$ac_ext >&5

eval "$as_ac_var=no"
fi
rm -f conftest.$ac_objext conftest$ac_exeext conftest.$ac_ext
fi
echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
if test `eval echo '${'$as_ac_var'}'` = yes; then
  cat >>confdefs.h <<_ACEOF
#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
_ACEOF

fi
done

# End PTHREADS mods by James Lemley, 2003

# /***********************************************************************
# // Write output files

Lines 130-135 Link Here

(-)lzop-1.01/configure.ac (+9 lines)
130	AC_CHECK_FUNCS(access alloca atoi atol chmod chown ctime difftime fstat gettimeofday gmtime localtime lstat memcmp memcpy memmove memset mktime snprintf strchr strdup strerror strftime strrchr umask utime vsnprintf)	130	AC_CHECK_FUNCS(access alloca atoi atol chmod chown ctime difftime fstat gettimeofday gmtime localtime lstat memcmp memcpy memmove memset mktime snprintf strchr strdup strerror strftime strrchr umask utime vsnprintf)
131	AC_CHECK_FUNCS(fchmod stricmp strnicmp)	131	AC_CHECK_FUNCS(fchmod stricmp strnicmp)
132		132
		133	# Begin PTHREADS mods by James Lemley, 2003
		134	AC_SEARCH_LIBS(pthread_join,pthread pthreads,,[
		135	echo "Can't find a POSIX threads library. If one is actually installed,"
		136	echo "please fix configure.in and submit a patch back to the maintainer."
		137	echo "Thanks!"
		138	exit 1
		139	])
		140	AC_CHECK_FUNCS(pthread_join)
		141	# End PTHREADS mods by James Lemley, 2003
133		142
134	# /***********************************************************************	143	# /***********************************************************************
135	# // Write output files	144	# // Write output files

Lines 79-84 Link Here

(-)lzop-1.01/src/conf.h (+36 lines)
79	# error	79	# error
80	#endif	80	#endif
81		81
		82	/* -- James Lemley -- adding support for POSIX threads via autoconf. */
		83	#ifdef HAVE_PTHREAD_JOIN
		84	#define USE_PTHREADS
		85	#endif
		86
		87	#ifdef USE_PTHREADS
		88	#include <pthread.h>
		89	#endif
82		90
83	/*************************************************************************	91	/*************************************************************************
84	//	92	//
Lines 596-601 Link Here
596	/*************************************************************************	604	/*************************************************************************
597	// mblock.c	605	// mblock.c
598	**************************************************************************/	606	**************************************************************************/
		607	#ifdef USE_PTHREADS
		608
		609	/* define the states buffers can be in for parallel operation: */
		610	/* input buffers go from empty to filled to busy to empty again */
		611	/* output buffers go from empty to busy to filled to empty again */
		612	/* worker threads shut down when they find an EOF flag on an input buffer */
		613
		614	#define BUFFER_EMPTY 0x00000100L
		615	#define BUFFER_FILLED 0x00000200L
		616	#define BUFFER_BUSY 0x00000400L
		617	#define BUFFER_EOF 0x00000800L
		618
		619	/* More defines for parallel operation: */
		620	#ifndef MAX_THREADS
		621	#define MAX_THREADS 128
		622	#endif
		623
		624	#define DEFAULT_THREADS 8
		625
		626	#endif /* USE_PTHREADS */
599		627
600	typedef struct	628	typedef struct
601	{	629	{
Lines 605-610 Link Here
605	lzo_uint32 size;	633	lzo_uint32 size;
606	lzo_uint32 align;	634	lzo_uint32 align;
607	lzo_uint32 flags;	635	lzo_uint32 flags;
		636	#ifdef USE_PTHREADS
		637	/* flags is initially set to 0x00000001 in mblock.c */
		638	/* 0x00000001 means "memory allocated"; otherwise unused. */
		639	/* I will use this field for parallel buffer status. */
		640	lzo_uint32 adler32; /* moved from local storage in lzo_compress to buffer areas */
		641	lzo_uint32 crc32; /* for parallel processing */
		642	lzo_uint32 len; /* for parallel processing */
		643	#endif /* USE_PTHREADS */
608	}	644	}
609	memblock_t;	645	memblock_t;
610	#define memblock_p memblock_t *	646	#define memblock_p memblock_t *

Lines 50-55 Link Here

(-)lzop-1.01/src/help.c (+10 lines)
50	" Laetor Zonas Obteri\n"	50	" Laetor Zonas Obteri\n"
51	" Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003\n"	51	" Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003\n"
52	"lzop v%-11s Markus Franz Xaver Johannes Oberhumer %20s\n"	52	"lzop v%-11s Markus Franz Xaver Johannes Oberhumer %20s\n"
		53	#ifdef USE_PTHREADS
		54	"\n##POSIX thread enabled patch 05 contributed by James Lemley, 2002, 2003##\n"
		55	"For information on this version only, see http://lemley.net/lzop_patches/\n"
		56	#endif
53	"\n",	57	"\n",
54	LZOP_VERSION_STRING, LZOP_VERSION_DATE);	58	LZOP_VERSION_STRING, LZOP_VERSION_DATE);
55	fg = con_fg(f,fg);	59	fg = con_fg(f,fg);
Lines 129-136 Link Here
129	" -S.suf use suffix .suf on compressed files\n"	133	" -S.suf use suffix .suf on compressed files\n"
130	" -F do not store or verify checksum of files (faster)\n"	134	" -F do not store or verify checksum of files (faster)\n"
131	" -U delete input files after successful operation (like gzip and bzip2)\n"	135	" -U delete input files after successful operation (like gzip and bzip2)\n"
		136	#ifdef USE_PTHREADS
		137	" -TN use N parallel compression threads (default %d max %d)\n"
		138	" file.. files to (de)compress. If none given, try standard input.\n"
		139	, dn, dN, DEFAULT_THREADS, MAX_THREADS);
		140	#else
132	" file.. files to (de)compress. If none given, try standard input.\n"	141	" file.. files to (de)compress. If none given, try standard input.\n"
133	, dn, dN);	142	, dn, dN);
		143	#endif /* USE_PTHREADS */
134	}	144	}
135		145
136		146

Lines 103-108 Link Here

(-)lzop-1.01/src/lzop.c (+24 lines)
103	static unsigned long total_bytes_written = 0;	103	static unsigned long total_bytes_written = 0;
104	static unsigned long total_bytes_read = 0;	104	static unsigned long total_bytes_read = 0;
105		105
		106	#ifdef USE_PTHREADS
		107	extern int num_threads;
		108	#endif
106		109
107	/*************************************************************************	110	/*************************************************************************
108	// exit handlers	111	// exit handlers
Lines 2549-2554 Link Here
2549	case 't':	2552	case 't':
2550	set_cmd(CMD_TEST);	2553	set_cmd(CMD_TEST);
2551	break;	2554	break;
		2555	#ifdef USE_PTHREADS
		2556	case 'T': /* set number of compression threads for parallel compression */
		2557	if (mfx_optarg && isdigit(mfx_optarg[0]))
		2558	{
		2559	num_threads = atoi(mfx_optarg);
		2560	if (num_threads > MAX_THREADS)
		2561	{
		2562	fprintf(stderr, "%s: %d threads requested; using maximum of %d\n", argv0, num_threads, MAX_THREADS);
		2563	num_threads = MAX_THREADS;
		2564	}
		2565	if (num_threads < 1)
		2566	num_threads = 1;
		2567	}
		2568	break;
		2569	#endif
2552	case 'U':	2570	case 'U':
2553	opt_unlink = 1;	2571	opt_unlink = 1;
2554	break;	2572	break;
Lines 2749-2754 Link Here
2749	{"path", 1, 0, 'p'+256},	2767	{"path", 1, 0, 'p'+256},
2750	{"quiet", 0, 0, 'q'}, /* quiet mode */	2768	{"quiet", 0, 0, 'q'}, /* quiet mode */
2751	{"silent", 0, 0, 'q'}, /* quiet mode */	2769	{"silent", 0, 0, 'q'}, /* quiet mode */
		2770	#ifdef USE_PTHREADS
		2771	{"threads", 1, 0, 'T'}, /* set number of threads for parallel operation */
		2772	#endif
2752	{"stdout", 0, 0, 'c'}, /* write output on standard output */	2773	{"stdout", 0, 0, 'c'}, /* write output on standard output */
2753	{"suffix", 1, 0, 'S'}, /* use given suffix instead of .lzo */	2774	{"suffix", 1, 0, 'S'}, /* use given suffix instead of .lzo */
2754	{"to-stdout", 0, 0, 'c'}, /* write output on standard output */	2775	{"to-stdout", 0, 0, 'c'}, /* write output on standard output */
Lines 2811-2816 Link Here
2811	{"no-warn", 0, 0, 525}, /* do not display any warnings */	2832	{"no-warn", 0, 0, 525}, /* do not display any warnings */
2812	{"quiet", 0, 0, 'q'}, /* quiet mode */	2833	{"quiet", 0, 0, 'q'}, /* quiet mode */
2813	{"silent", 0, 0, 'q'}, /* quiet mode */	2834	{"silent", 0, 0, 'q'}, /* quiet mode */
		2835	#ifdef USE_PTHREADS
		2836	{"threads", 1, 0, 'T'}, /* set number of threads for parallel operation */
		2837	#endif
2814	{"unlink", 0, 0, 'U'},	2838	{"unlink", 0, 0, 'U'},
2815	{"verbose", 0, 0, 'v'}, /* verbose mode */	2839	{"verbose", 0, 0, 'v'}, /* verbose mode */
2816		2840

Lines 28-33 Link Here

(-)lzop-1.01/src/p_lzo.c (-1 / +391 lines)
28		28
29	#include "conf.h"	29	#include "conf.h"
30		30
		31	#ifdef USE_PTHREADS
		32	/* POSIX threads code added by James Lemley, 2002-2003 */
		33
		34	/* define some things for parallel processing */
		35	#define MAX_BUFFERS (MAX_THREADS * 4)
		36	int num_threads = DEFAULT_THREADS;
		37	#define num_buffers (num_threads * 4 ) /* must be at least num_threads * 2, and must be even number */
		38	pthread_mutex_t buffer_mutex; /* to lock buffer arrays for checking && setting flags */
		39	#define IO_SLEEP_USEC 20 /* how long do I/O threads sleep in microseconds if they can't get a buffer. */
		40	#define COMP_SLEEP_USEC 10 /* how long do worker threads sleep in microseconds if they can't get a buffer. */
		41
		42	/* define a macro for easy debugging print */
		43	#define D(x, y) if(0) {fprintf(stderr, x, y); fflush(stderr);}
		44	#endif /* USE_PTHREADS */
31		45
32	#if defined(WITH_LZO)	46	#if defined(WITH_LZO)
33		47
Lines 172-194 Link Here
172	#endif	186	#endif
173		187
174		188
		189	#ifdef USE_PTHREADS
		190	static memblock_t block1[MAX_BUFFERS]; /* define buffers in array instead of discreetly */
		191	#else
175	static memblock_t block1;	192	static memblock_t block1;
176	static memblock_t block2;	193	static memblock_t block2;
177	static memblock_t wrkmem;	194	static memblock_t wrkmem;
		195	#endif /* USE_PTHREADS */
178		196
179		197
180	static void free_mem(void)	198	static void free_mem(void)
181	{	199	{
		200	#ifdef USE_PTHREADS
		201	int i;
		202	for (i = 0; i < num_buffers; i++)
		203	{
		204	mb_free(block1+i);
		205	}
		206	#else
182	mb_free(&wrkmem);	207	mb_free(&wrkmem);
183	mb_free(&block2);	208	mb_free(&block2);
184	mb_free(&block1);	209	mb_free(&block1);
		210	#endif /* USE_PTHREADS */
185	}	211	}
186		212
		213	#ifdef USE_PTHREADS
		214	static lzo_uint32 W; /* holds work_len for worker memory size, in each thread. */
		215	#endif /* USE_PTHREADS */
187		216
188	static lzo_bool alloc_mem(lzo_uint32 s1, lzo_uint32 s2, lzo_uint32 w)	217	static lzo_bool alloc_mem(lzo_uint32 s1, lzo_uint32 s2, lzo_uint32 w)
189	{	218	{
190	lzo_bool r = 1;	219	lzo_bool r = 1;
191		220
		221	#ifdef USE_PTHREADS
		222	#if defined(USE_MALLOC)
		223	int i;
		224	for (i = 0; r && i < num_buffers; i+=2)
		225	r &= mb_alloc(block1+i, s1, ALIGN_SIZE);
		226	for (i = 0; r && i < num_buffers; i+=2)
		227	r &= mb_alloc(block1+i+1, s2, ALIGN_SIZE);
		228
		229	if (pthread_mutex_init(&buffer_mutex, NULL))
		230	{
		231	fprintf(stderr, "pthread_mutex_init failed\n");
		232	r = 0;
		233	}
		234	/* save W for allocating workmem in each worker thread */
		235	W = w;
		236	#else
		237	#error "parallel version requires use of malloc()"
		238	#endif /* USE_MALLOC */
		239	#else /* if not defined USE_PTHREADS */
192	#if defined(USE_MALLOC)	240	#if defined(USE_MALLOC)
193	r &= mb_alloc(&block1, s1, ALIGN_SIZE);	241	r &= mb_alloc(&block1, s1, ALIGN_SIZE);
194	r &= mb_alloc(&block2, s2, ALIGN_SIZE);	242	r &= mb_alloc(&block2, s2, ALIGN_SIZE);
Lines 197-203 Link Here
197	r &= mb_init(&block1, s1, ALIGN_SIZE, heap_block1, sizeof(heap_block1));	245	r &= mb_init(&block1, s1, ALIGN_SIZE, heap_block1, sizeof(heap_block1));
198	r &= mb_init(&block2, s2, ALIGN_SIZE, heap_block2, sizeof(heap_block2));	246	r &= mb_init(&block2, s2, ALIGN_SIZE, heap_block2, sizeof(heap_block2));
199	r &= mb_init(&wrkmem, w, ALIGN_SIZE, heap_wrkmem, sizeof(heap_wrkmem));	247	r &= mb_init(&wrkmem, w, ALIGN_SIZE, heap_wrkmem, sizeof(heap_wrkmem));
200	#endif	248	#endif /* defined(USE_MALLOC) */
		249	#endif /* USE_PTHREADS */
201	if (!r)	250	if (!r)
202	free_mem();	251	free_mem();
203	return r;	252	return r;
Lines 282-287 Link Here
282	}	331	}
283		332
284		333
		334	#ifdef USE_PTHREADS
		335	/*************************************************************************
		336	// compress a file using POSIX threads
		337	**************************************************************************/
		338
		339	lzo_bool lzo_compress_worker(const header_t *h);
		340	lzo_bool lzo_compress_reader(file_t *fip);
		341
		342	lzo_bool lzo_compress_parallel(file_t fip, file_t fop, const header_t *h)
		343	{
		344	int i;
		345	int r = 1; /* changed 2002-12-23 */
		346	lzo_uint32 output_sequence = 0;
		347	pthread_t worker_threads[MAX_THREADS];
		348	pthread_t reader_thread;
		349	void *thread_ret;
		350
		351	/* set initial state to BUFFER_EMPTY on all input and output buffers. */
		352	for (i = 0; i < num_buffers; i++)
		353	{
		354	block1[i].flags &= ~(BUFFER_BUSY \| BUFFER_FILLED \| BUFFER_EOF);
		355	block1[i].flags \|= BUFFER_EMPTY;
		356	}
		357
		358	/* start dedicated reader thread */
		359	if (pthread_create(&reader_thread, NULL, (void )(void )lzo_compress_reader, (void *) fip))
		360	{
		361	fprintf(stderr, "Unable to create reader thread\n");
		362	/* print errno here too */
		363	exit(1);
		364	}
		365
		366	/* start worker threads */
		367	for (i = 0; i < num_threads; i++)
		368	{
		369	if (pthread_create(&worker_threads[i], NULL, (void )(void )lzo_compress_worker, (void *) h))
		370	{
		371	fprintf(stderr, "Unable to create worker thread %d\n", i);
		372	/* print errno here too */
		373	exit(1);
		374	}
		375	}
		376
		377	for (;;)
		378	{
		379	/* troll for THE ready output buffer - identified by output_sequence and BUFFER_FILLED */
		380
		381	D("lzo_compress: %s\n", "trolling for finished output buffer...");
		382	pthread_mutex_lock(&buffer_mutex);
		383	for (i = 0; i < num_buffers; i+=2)
		384	{
		385	if (block1[i+1].flags & BUFFER_FILLED) /* an output buffer that is ready */
		386	if ((block1[i+1].flags >> 16) == output_sequence) /* and it is the one I am waiting on */
		387	break;
		388	}
		389	if (i != num_buffers) /* found work */
		390	{
		391	D("lzo_compress: %s\n", "found it! writing block.");
		392	block1[i+1].flags &= ~BUFFER_FILLED; /* mark output buffer as busy */
		393	block1[i+1].flags \|= BUFFER_BUSY;
		394
		395	pthread_mutex_unlock(&buffer_mutex); /* unlock buffers for file write that may block */
		396
		397	/* write uncompressed block size */
398	write32(fop,block1[i].len);
399	D("lzo_compress: uncompressed size: %d\n", block1[i].len);
400	D("lzo_compress: compressed size: %d\n", block1[i+1].len);
401
402	/* exit if last block */
403	if (block1[i].len == 0)
404	break;
405
406	/* write compressed block size */
407	if (block1[i+1].len < block1[i].len)
408	write32(fop,block1[i+1].len);
409	else
410	write32(fop,block1[i].len);
411
412	/* write checksum of uncompressed block */
413	if (h->flags & F_ADLER32_D)
414	write32(fop,block1[i].adler32);
415	if (h->flags & F_CRC32_D)
416	write32(fop,block1[i].crc32);
417
418	/* write checksum of compressed block */
419	if (block1[i+1].len < block1[i].len && (h->flags & F_ADLER32_C))
420	{
421	write32(fop,block1[i+1].adler32);
422	}
423	if (block1[i+1].len < block1[i].len && (h->flags & F_CRC32_C))
424	{
425	write32(fop,block1[i+1].crc32);
426	}
427
428	/* write compressed block data */
429	if (block1[i+1].len < block1[i].len)
430	write_buf(fop,block1[i+1].mem,block1[i+1].len); /* this line had an error */
431	else
432	write_buf(fop,block1[i].mem,block1[i].len);
433
434	/* mark both input and output buffers as free */
435	pthread_mutex_lock(&buffer_mutex);
436	block1[i+1].flags &= ~BUFFER_BUSY;
437	block1[i+1].flags \|= BUFFER_EMPTY;
438	block1[i].flags &= ~BUFFER_BUSY; /* clear busy flag that the worker left set on the input buffer */
439	block1[i].flags \|= BUFFER_EMPTY;
440	pthread_mutex_unlock(&buffer_mutex); /* unlock buffers */
441
442	output_sequence++;
443	if (output_sequence > 0x0FFF)
444	output_sequence = 0; /* do same with input_sequence on input to keep in sync */
445	}
446	else /* no output buffer found ready to go */
447	{
448	D("lzo_compress: %s\n", "No ouput ready. Sleeping.");
449	pthread_mutex_unlock(&buffer_mutex); /* unlock buffers */
450	usleep (IO_SLEEP_USEC); /* give up my CPU */
451	}
452	}
453
454	D("lzo_compress: %s\n", "Waiting on reader to join... ");
455	pthread_join(reader_thread, &thread_ret);
456	D("lzo_compress: %s\n", "Waiting on workers to join... ");
457	/* wait on workers */
458	for (i = 0; i < num_threads; i++)
459	{
460	pthread_join(worker_threads[i], &thread_ret);
461	/FIXME -- dumps core. r &= ((lzo_bool ) thread_ret); /
462	}
463	D("lzo_compress: %s\n", "Workers joined. Exiting... ");
464
465	return r;
466	}
467
468	lzo_bool lzo_compress_worker(const header_t *h)
469	{
470	int i;
471	int r = LZO_E_OK;
472	lzo_bool input_eof = 0;
473	memblock_t wrkmem;
474	r &= mb_alloc(&wrkmem, W, ALIGN_SIZE); /* allocate work memory on a per-thread basis */
475	D(" lzo_compress_worker: %s\n", "startup!");
476
477	while (1)
478	{
479	/* troll for full input buffer to compress */
480	D(" lzo_compress_worker: %s\n", "trolling for filled input buffer...");
481	pthread_mutex_lock(&buffer_mutex);
482	for (i = 0; i < num_buffers; i+=2)
483	{
484	if (block1[i].flags & BUFFER_EOF)
485	input_eof = 1;
486	if (block1[i].flags & BUFFER_FILLED)
487	break;
488	}
489	if (i == num_buffers)
490	{
491	pthread_mutex_unlock(&buffer_mutex);
492	/* no work for me... */
493	if (input_eof)
494	{
495	D(" lzo_compress_worker: %s\n", "Found EOF in input buffer and there are no other filled inputs. ");
496	break;
497	}
498
499	D(" lzo_compress_worker: %s\n", "sleeping.");
500	usleep(COMP_SLEEP_USEC); /* give up my CPU... */
501	continue; /* go back and try again... */
502	}
503
504	D(" lzo_compress_worker: %s\n", "Got input buffer. ");
505	block1[i].flags &= ~BUFFER_FILLED; /* clear buffer full flag so no other worker grabs this one too */
506	block1[i].flags \|= BUFFER_BUSY; /* set busy flag */
507	/* work for this thread to do! now to find a free output buffer... */
508	D(" lzo_compress_worker: %s\n", "Got output buffer. Compressing... ");
509	block1[i+1].flags &= ~BUFFER_EMPTY; /* unset output buffer free flag */
510	block1[i+1].flags \|= BUFFER_BUSY; /* set output buffer busy flag */
511	block1[i+1].flags &= 0x0000FFFF; /* clear any old sequence number */
512	block1[i+1].flags \|= (block1[i].flags & 0xFFFF0000); /copy input sequence to output buffer /
513	if (block1[i].flags & BUFFER_EOF) /* if this is the EOF buffer, don't really compress. */
514	{
515	D(" lzo_compress_worker: %s\n", "I've got the EOF buffer.");
516	/* but first set EOF flag on ouptut buffer, and zero length. */
517	block1[i+1].flags &= ~BUFFER_BUSY; /* set output buffer busy flag */
518	block1[i+1].flags \|= BUFFER_FILLED;
519	block1[i+1].len = 0;
520	pthread_mutex_unlock(&buffer_mutex);
521	/* go back and make sure there isn't more work to do... */
522	continue;
523	}
524
525	pthread_mutex_unlock(&buffer_mutex); /* unlock buffers now that I have the ones I want */
526
527	/* compute checksum of uncompressed block */
528
529	if (h->flags & F_ADLER32_D)
530	block1[i].adler32 = lzo_adler32(ADLER32_INIT_VALUE,block1[i].mem,block1[i].len);
531	if (h->flags & F_CRC32_D)
532	block1[i].crc32 = lzo_crc32(CRC32_INIT_VALUE,block1[i].mem,block1[i].len);
533
534	x_filter(block1[i].mem,block1[i].len,h);
535
536	/* compress */
537	if (h->method == M_LZO1X_1)
538	r = lzo1x_1_compress(block1[i].mem, block1[i].len, block1[i+1].mem, &block1[i+1].len, wrkmem.mem);
539	#if defined(USE_LZO1X_1_15)
540	else if (h->method == M_LZO1X_1_15)
541	r = lzo1x_1_15_compress(block1[i].mem, block1[i].len,
542	block1[i+1].mem, &block1[i+1].len, wrkmem.mem);
543	#endif
544	#if defined(USE_LZO1X_999)
545	else if (h->method == M_LZO1X_999)
546	r = lzo1x_999_compress_level(block1[i].mem, block1[i].len,
547	block1[i+1].mem, &block1[i+1].len, wrkmem.mem,
548	NULL, 0, 0, h->level);
549	#endif
550	else
551	fatal(NULL,"Internal error");
552
553	#if 0
554	fprintf(stderr, "%ld %ld %ld\n", (long)src_len, (long)dst_len, (long)block2.size);
555	#endif
556	assert(block1[i+1].len <= block1[i+1].size);
557	if (r != LZO_E_OK)
558	fatal(NULL,"Internal error - compression failed");
559
560	/* optimize */
561	if (opt_optimize && block1[i+1].len < block1[i].len)
562	{
563	lzo_uint32 new_len = block1[i].len;
564	r = lzo1x_optimize(block1[i+1].mem, block1[i+1].len, block1[i].mem, &new_len, NULL);
565	if (r != LZO_E_OK \|\| new_len != block1[i].len)
566	fatal(NULL,"Internal error - optimization failed");
567	}
568	/* compute checksum of compressed block */
569	if (block1[i+1].len < block1[i].len && (h->flags & F_ADLER32_C))
570	{
571	block1[i+1].adler32 = lzo_adler32(ADLER32_INIT_VALUE,block1[i+1].mem,block1[i+1].len);
572	}
573	if (block1[i+1].len < block1[i].len && (h->flags & F_CRC32_C))
574	{
575	block1[i+1].crc32 = lzo_crc32(CRC32_INIT_VALUE,block1[i+1].mem,block1[i+1].len);
576	}
577	D(" lzo_compress_worker: %s\n", "setting ouput buffer filled... ");
578	pthread_mutex_lock(&buffer_mutex);
579	/* set flags on output buffer to full - I/O thread will reset flags on input buffer */
580
581	block1[i+1].flags &= ~BUFFER_BUSY;
582	block1[i+1].flags \|= BUFFER_FILLED;
583
584	pthread_mutex_unlock(&buffer_mutex);
585	}
586	mb_free(&wrkmem);
587	D(" *****lzo_compress_worker: %s\n", "exiting!");
588	return r;
589	}
590
591	lzo_bool lzo_compress_reader(file_t *fip)
592	{
593	int i;
594	lzo_uint32 input_sequence = 0;
595	lzo_int l;
596
597	for (;;)
598	{
599	/* troll for an empty input buffer */
600	D(" lzo_compress_reader: %s\n", "trolling for empty input buffer...");
601	pthread_mutex_lock(&buffer_mutex);
602	for (i = 0; i < num_buffers; i+=2)
603	if (block1[i].flags & BUFFER_EMPTY)
604	break;
605	if (i != num_buffers) /* got one */
606	{
607	D(" lzo_compress_reader: %s\n", "Got one. Reading block.");
608	block1[i].flags &= ~BUFFER_EMPTY;
609	block1[i].flags \|= BUFFER_BUSY;
610	/* save buffer sequence so we can keep proper order on output */
611	block1[i].flags ^= (block1[i].flags & 0xFFFF0000); /* clear old buffer sequence */
612	block1[i].flags \|= (input_sequence << 16); /* store new buffer sequence */
613	input_sequence++;
614	if (input_sequence > 0x0FFF)
615	input_sequence = 0; /* do same with output_sequence on output to keep in sync */
616
617	pthread_mutex_unlock(&buffer_mutex); /* unlock buffers for the read - if we block, others can work */
618
619	/* read a block */
620	l = read_buf(fip, block1[i].mem, block_size);
621	block1[i].len = (l > 0 ? l : 0);
622
623	pthread_mutex_lock(&buffer_mutex);
624	block1[i].flags &= ~BUFFER_BUSY;
625	block1[i].flags \|= BUFFER_FILLED;
626	if (block1[i].len == 0)
627	{
628	D(" lzo_compress_reader: %s\n", "EOF.");
629	block1[i].flags \|= BUFFER_EOF; /* signal all waiting threads to give up */
630	}
631	}
632	pthread_mutex_unlock(&buffer_mutex);
633	if (i == num_buffers)
634	{
635	D(" lzo_compress_reader: %s\n", " no empty input buffers - sleeping.");
636	usleep(IO_SLEEP_USEC);
637	}
638	else if (block1[i].len == 0)
639	{
640	D(" lzo_compress_reader: %s\n", " bailing..");
641	break;
642	}
643	}
644	return 1;
645	}
646
647	#endif /* ifdef USE_PTHREADS */
648
285	/*************************************************************************	649	/*************************************************************************
286	// compress a file	650	// compress a file
287	**************************************************************************/	651	**************************************************************************/
Lines 289-302 Link Here
289	lzo_bool lzo_compress(file_t fip, file_t fop, const header_t *h)	653	lzo_bool lzo_compress(file_t fip, file_t fop, const header_t *h)
290	{	654	{
291	int r = LZO_E_OK;	655	int r = LZO_E_OK;
		656	#ifdef USE_PTHREADS
		657	/* this still needs to work for calling the thread enabled version with only 1 thread */
		658	lzo_byte * const b1 = block1[0].mem;
		659	lzo_byte * const b2 = block1[1].mem;
		660	memblock_t wrkmem;
		661	#else
292	lzo_byte * const b1 = block1.mem;	662	lzo_byte * const b1 = block1.mem;
293	lzo_byte * const b2 = block2.mem;	663	lzo_byte * const b2 = block2.mem;
		664	#endif /* USE_PTHREADS */
294	lzo_uint32 src_len = 0, dst_len = 0;	665	lzo_uint32 src_len = 0, dst_len = 0;
295	lzo_uint32 c_adler32 = ADLER32_INIT_VALUE, d_adler32 = ADLER32_INIT_VALUE;	666	lzo_uint32 c_adler32 = ADLER32_INIT_VALUE, d_adler32 = ADLER32_INIT_VALUE;
296	lzo_uint32 c_crc32 = CRC32_INIT_VALUE, d_crc32 = CRC32_INIT_VALUE;	667	lzo_uint32 c_crc32 = CRC32_INIT_VALUE, d_crc32 = CRC32_INIT_VALUE;
297	lzo_int l;	668	lzo_int l;
298	lzo_bool ok = 1;	669	lzo_bool ok = 1;
299		670
		671	#ifdef USE_PTHREADS
		672	/* if parallelism is requested, call lzo_compress_parallel, else fall through to original code */
		673	if (num_threads > 1)
		674	return lzo_compress_parallel(fip, fop, h);
		675	r &= mb_alloc(&wrkmem, W, ALIGN_SIZE); /* allocate work memory on a per-thread basis */
		676	#endif
300	for (;;)	677	for (;;)
301	{	678	{
302	/* read a block */	679	/* read a block */
Lines 338-344 Link Here
338	#if 0	715	#if 0
339	fprintf(stderr, "%ld %ld %ld\n", (long)src_len, (long)dst_len, (long)block2.size);	716	fprintf(stderr, "%ld %ld %ld\n", (long)src_len, (long)dst_len, (long)block2.size);
340	#endif	717	#endif
		718	#ifdef USE_PTHREADS
		719	assert(dst_len <= block1[1].size);
		720	#else
341	assert(dst_len <= block2.size);	721	assert(dst_len <= block2.size);
		722	#endif /* USE_PTHREADS */
342	if (r != LZO_E_OK)	723	if (r != LZO_E_OK)
343	fatal(fip,"Internal error - compression failed");	724	fatal(fip,"Internal error - compression failed");
344		725
Lines 382-387 Link Here
382	write_buf(fop,b1,src_len);	763	write_buf(fop,b1,src_len);
383	}	764	}
384		765
		766	#ifdef USE_PTHREADS
		767	mb_free(&wrkmem);
		768	#endif
385	return ok;	769	return ok;
386	}	770	}
387		771
Lines 399-405 Link Here
399	lzo_uint32 c_crc32 = CRC32_INIT_VALUE, d_crc32 = CRC32_INIT_VALUE;	783	lzo_uint32 c_crc32 = CRC32_INIT_VALUE, d_crc32 = CRC32_INIT_VALUE;
400	lzo_bool ok = 1;	784	lzo_bool ok = 1;
401	lzo_bool use_seek;	785	lzo_bool use_seek;
		786	#ifdef USE_PTHREADS
		787	/* the below was &block2 (an output block) but now block1 is an array */
		788	/* of memblock_t, allocated in input-output-input-output... order. */
		789	memblock_t * const block = block1+1;
		790	#else
402	memblock_t * const block = &block2;	791	memblock_t * const block = &block2;
		792	#endif /* USE_PTHREADS */
403	lzo_byte * b1;	793	lzo_byte * b1;
404	lzo_byte * const b2 = block->mem;	794	lzo_byte * const b2 = block->mem;
405		795

Return to bug 189130

Lines 13815-13820 Link Here

(-)lzop-1.01/configure (+199 lines)
13815	done	13815	done
13816		13816
13817		13817
		13818	# Begin PTHREADS mods by James Lemley, 2003
		13819	echo "$as_me:$LINENO: checking for library containing pthread_join" >&5
		13820	echo $ECHO_N "checking for library containing pthread_join... $ECHO_C" >&6
		13821	if test "${ac_cv_search_pthread_join+set}" = set; then
		13822	echo $ECHO_N "(cached) $ECHO_C" >&6
		13823	else
		13824	ac_func_search_save_LIBS=$LIBS
		13825	ac_cv_search_pthread_join=no
		13826	cat >conftest.$ac_ext <<_ACEOF
		13827	#line $LINENO "configure"
		13828	/* confdefs.h. */
		13829	_ACEOF
		13830	cat confdefs.h >>conftest.$ac_ext
		13831	cat >>conftest.$ac_ext <<_ACEOF
		13832	/* end confdefs.h. */
		13833
		13834	/* Override any gcc2 internal prototype to avoid an error. */
		13835	#ifdef __cplusplus
		13836	extern "C"
		13837	#endif
		13838	/* We use char because int might match the return type of a gcc2
		13839	builtin and then its argument prototype would still apply. */
		13840	char pthread_join ();
		13841	int
		13842	main ()
		13843	{
		13844	pthread_join ();
		13845	;
		13846	return 0;
		13847	}
		13848	_ACEOF
		13849	rm -f conftest.$ac_objext conftest$ac_exeext
		13850	if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
		13851	(eval $ac_link) 2>&5
		13852	ac_status=$?
		13853	echo "$as_me:$LINENO: \$? = $ac_status" >&5
		13854	(exit $ac_status); } &&
		13855	{ ac_try='test -s conftest$ac_exeext'
		13856	{ (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
		13857	(eval $ac_try) 2>&5
		13858	ac_status=$?
		13859	echo "$as_me:$LINENO: \$? = $ac_status" >&5
		13860	(exit $ac_status); }; }; then
		13861	ac_cv_search_pthread_join="none required"
		13862	else
		13863	echo "$as_me: failed program was:" >&5
		13864	sed 's/^/\| /' conftest.$ac_ext >&5
		13865
		13866	fi
		13867	rm -f conftest.$ac_objext conftest$ac_exeext conftest.$ac_ext
		13868	if test "$ac_cv_search_pthread_join" = no; then
		13869	for ac_lib in pthread pthreads; do
		13870	LIBS="-l$ac_lib $ac_func_search_save_LIBS"
		13871	cat >conftest.$ac_ext <<_ACEOF
		13872	#line $LINENO "configure"
		13873	/* confdefs.h. */
		13874	_ACEOF
		13875	cat confdefs.h >>conftest.$ac_ext
		13876	cat >>conftest.$ac_ext <<_ACEOF
		13877	/* end confdefs.h. */
		13878
		13879	/* Override any gcc2 internal prototype to avoid an error. */
		13880	#ifdef __cplusplus
		13881	extern "C"
13882	#endif
13883	/* We use char because int might match the return type of a gcc2
13884	builtin and then its argument prototype would still apply. */
13885	char pthread_join ();
13886	int
13887	main ()
13888	{
13889	pthread_join ();
13890	;
13891	return 0;
13892	}
13893	_ACEOF
13894	rm -f conftest.$ac_objext conftest$ac_exeext
13895	if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
13896	(eval $ac_link) 2>&5
13897	ac_status=$?
13898	echo "$as_me:$LINENO: \$? = $ac_status" >&5
13899	(exit $ac_status); } &&
13900	{ ac_try='test -s conftest$ac_exeext'
13901	{ (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
13902	(eval $ac_try) 2>&5
13903	ac_status=$?
13904	echo "$as_me:$LINENO: \$? = $ac_status" >&5
13905	(exit $ac_status); }; }; then
13906	ac_cv_search_pthread_join="-l$ac_lib"
13907	break
13908	else
13909	echo "$as_me: failed program was:" >&5
13910	sed 's/^/\| /' conftest.$ac_ext >&5
13911
13912	fi
13913	rm -f conftest.$ac_objext conftest$ac_exeext conftest.$ac_ext
13914	done
13915	fi
13916	LIBS=$ac_func_search_save_LIBS
13917	fi
13918	echo "$as_me:$LINENO: result: $ac_cv_search_pthread_join" >&5
13919	echo "${ECHO_T}$ac_cv_search_pthread_join" >&6
13920	if test "$ac_cv_search_pthread_join" != no; then
13921	test "$ac_cv_search_pthread_join" = "none required" \|\| LIBS="$ac_cv_search_pthread_join $LIBS"
13922
13923	else
13924
13925	echo "Can't find a POSIX threads library. If one is actually installed,"
13926	echo "please fix configure.in and submit a patch back to the maintainer."
13927	echo "Thanks!"
13928	exit 1
13929
13930	fi
13931
13932
13933	for ac_func in pthread_join
13934	do
13935	as_ac_var=`echo "ac_cv_func_$ac_func" \| $as_tr_sh`
13936	echo "$as_me:$LINENO: checking for $ac_func" >&5
13937	echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
13938	if eval "test \"\${$as_ac_var+set}\" = set"; then
13939	echo $ECHO_N "(cached) $ECHO_C" >&6
13940	else
13941	cat >conftest.$ac_ext <<_ACEOF
13942	#line $LINENO "configure"
13943	/* confdefs.h. */
13944	_ACEOF
13945	cat confdefs.h >>conftest.$ac_ext
13946	cat >>conftest.$ac_ext <<_ACEOF
13947	/* end confdefs.h. */
13948	/* System header to define __stub macros and hopefully few prototypes,
13949	which can conflict with char $ac_func (); below.
13950	Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
13951	<limits.h> exists even on freestanding compilers. */
13952	#ifdef __STDC__
13953	# include <limits.h>
13954	#else
13955	# include <assert.h>
13956	#endif
13957	/* Override any gcc2 internal prototype to avoid an error. */
13958	#ifdef __cplusplus
13959	extern "C"
13960	{
13961	#endif
13962	/* We use char because int might match the return type of a gcc2
13963	builtin and then its argument prototype would still apply. */
13964	char $ac_func ();
13965	/* The GNU C library defines this for functions which it implements
13966	to always fail with ENOSYS. Some functions are actually named
13967	something starting with __ and the normal name is an alias. */
13968	#if defined (__stub_$ac_func) \|\| defined (__stub___$ac_func)
13969	choke me
13970	#else
13971	char (*f) () = $ac_func;
13972	#endif
13973	#ifdef __cplusplus
13974	}
13975	#endif
13976
13977	int
13978	main ()
13979	{
13980	return f != $ac_func;
13981	;
13982	return 0;
13983	}
13984	_ACEOF
13985	rm -f conftest.$ac_objext conftest$ac_exeext
13986	if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
13987	(eval $ac_link) 2>&5
13988	ac_status=$?
13989	echo "$as_me:$LINENO: \$? = $ac_status" >&5
13990	(exit $ac_status); } &&
13991	{ ac_try='test -s conftest$ac_exeext'
13992	{ (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
13993	(eval $ac_try) 2>&5
13994	ac_status=$?
13995	echo "$as_me:$LINENO: \$? = $ac_status" >&5
13996	(exit $ac_status); }; }; then
13997	eval "$as_ac_var=yes"
13998	else
13999	echo "$as_me: failed program was:" >&5
14000	sed 's/^/\| /' conftest.$ac_ext >&5
14001
14002	eval "$as_ac_var=no"
14003	fi
14004	rm -f conftest.$ac_objext conftest$ac_exeext conftest.$ac_ext
14005	fi
14006	echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
14007	echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
14008	if test `eval echo '${'$as_ac_var'}'` = yes; then
14009	cat >>confdefs.h <<_ACEOF
14010	#define `echo "HAVE_$ac_func" \| $as_tr_cpp` 1
14011	_ACEOF
14012
14013	fi
14014	done
14015
14016	# End PTHREADS mods by James Lemley, 2003
13818		14017
13819	# /***********************************************************************	14018	# /***********************************************************************
13820	# // Write output files	14019	# // Write output files