diff -urp clamav-0.93-orig/clamd/server-th.c clamav-0.93-new/clamd/server-th.c --- clamav-0.93-orig/clamd/server-th.c 2008-03-06 19:41:03.000000000 +0100 +++ clamav-0.93-new/clamd/server-th.c 2008-06-09 11:00:02.000000000 +0200 @@ -83,6 +83,16 @@ typedef struct client_conn_tag { int nsockets; } client_conn_t; +static void scanner_thread_cleanup(void *arg) +{ + client_conn_t *conn = (client_conn_t *) arg; + + shutdown(conn->sd, 2); + closesocket(conn->sd); + cl_free(conn->engine); + free(conn); +} + static void scanner_thread(void *arg) { client_conn_t *conn = (client_conn_t *) arg; @@ -110,6 +120,9 @@ static void scanner_thread(void *arg) if(!timeout) timeout = -1; + /* register cleanup procedure in this thread */ + pthread_cleanup_push(scanner_thread_cleanup, arg); + do { ret = command(conn->sd, conn->engine, conn->limits, conn->options, conn->copt, timeout); if (ret < 0) { @@ -155,10 +168,7 @@ static void scanner_thread(void *arg) } } while (session); - shutdown(conn->sd, 2); - closesocket(conn->sd); - cl_free(conn->engine); - free(conn); + pthread_cleanup_pop(1); return; } @@ -603,6 +613,10 @@ int acceptloop_th(int *socketds, int nso pthread_mutex_lock(&reload_mutex); if(reload) { +#ifdef OPTIMIZE_MEMORY_FOOTPRINT + /* Signal all worker threads to STOP, wait till no more acive threads */ + thrmgr_worker_stop_wait(thr_pool); +#endif pthread_mutex_unlock(&reload_mutex); engine = reload_db(engine, dboptions, copt, FALSE, &ret); if(ret) { @@ -611,10 +625,15 @@ int acceptloop_th(int *socketds, int nso close(new_sd); break; } + pthread_mutex_lock(&reload_mutex); reload = 0; time(&reloaded_time); pthread_mutex_unlock(&reload_mutex); +#ifdef OPTIMIZE_MEMORY_FOOTPRINT + /* Resume thread pool worker threads */ + thrmgr_setstate(thr_pool, POOL_VALID); +#endif #ifdef CLAMUKO if(cfgopt(copt, "ClamukoScanOnAccess")->enabled) { logg("Stopping and restarting Clamuko.\n"); diff -urp clamav-0.93-orig/clamd/thrmgr.c clamav-0.93-new/clamd/thrmgr.c --- clamav-0.93-orig/clamd/thrmgr.c 2008-03-06 19:41:03.000000000 +0100 +++ clamav-0.93-new/clamd/thrmgr.c 2008-06-09 11:01:34.000000000 +0200 @@ -210,6 +210,98 @@ threadpool_t *thrmgr_new(int max_threads return threadpool; } +#ifdef OPTIMIZE_MEMORY_FOOTPRINT +/** + * thrmgr_worker_stop_wait : set state to POOL_STOP, wake all thread worker, wait for them + * to exit before continuing. + */ +void thrmgr_worker_stop_wait(threadpool_t * const threadpool) +{ + struct timespec timeout; + int ret_cond; + int loop = 2; + + if (!threadpool || (threadpool->state != POOL_VALID)) { + return; + } + if (pthread_mutex_lock(&threadpool->pool_mutex) != 0) { + logg("!Mutex lock failed\n"); + exit(-1); + } + threadpool->state = POOL_STOP; + + /* wait for threads to exit */ + if (threadpool->thr_alive > 0) { +#ifdef CL_DEBUG + logg("*%u active threads: waking them and entering wait loop\n", threadpool->thr_alive); +#endif + if (pthread_cond_broadcast(&(threadpool->pool_cond)) != 0) { + pthread_mutex_unlock(&threadpool->pool_mutex); + logg("!Fatal: failed in cond broadcast 'pool_cond'\n"); + return; + } + } + /* now, wait for the threads to exit, make 'loop' number of tries, */ + while (threadpool->thr_alive > 0 && loop--) { +#ifdef CL_DEBUG + logg("*%u active threads. Waiting.\n", threadpool->thr_alive); +#endif + timeout.tv_sec = time(NULL) + (threadpool->idle_timeout/2) + 10L; + timeout.tv_nsec = 0; + ret_cond = pthread_cond_timedwait (&threadpool->pool_cond, &threadpool->pool_mutex, &timeout); + if (ret_cond == ETIMEDOUT) { +#ifdef CL_DEBUG + logg("*%u active threads. Continue to wait.\n", threadpool->thr_alive); +#endif + } else if (ret_cond == 0) { +#ifdef CL_DEBUG + logg("*Received signal. %u active threads.\n", threadpool->thr_alive); +#endif + } + } + if (pthread_mutex_unlock(&threadpool->pool_mutex) != 0) { + logg("!Mutex unlock failed\n"); + exit(-1); + } +} +#endif +#ifdef OPTIMIZE_MEMORY_FOOTPRINT +void thrmgr_setstate(threadpool_t * const threadpool, pool_state_t state ) +{ + if (pthread_mutex_lock(&threadpool->pool_mutex) != 0) { + logg("!Mutex lock failed\n"); + exit(-1); + } + threadpool->state = state; + if (pthread_mutex_unlock(&threadpool->pool_mutex) != 0) { + logg("!Mutex unlock failed\n"); + exit(-1); + } +} +#endif + +static void *thrmgr_worker_cleanup(void *arg) +{ + threadpool_t *threadpool = (threadpool_t *) arg; + + if (pthread_mutex_lock(&(threadpool->pool_mutex)) != 0) { + /* Fatal error */ + logg("!Fatal: mutex lock failed\n"); + exit(-2); + } + (threadpool->thr_alive) && threadpool->thr_alive--; + /* logg("*Thread clean up, %u active threads.", threadpool->thr_alive); */ + if (threadpool->thr_alive == 0) { + /* signal that all threads are finished */ + pthread_cond_broadcast(&threadpool->pool_cond); + } + if (pthread_mutex_unlock(&(threadpool->pool_mutex)) != 0) { + /* Fatal error */ + logg("!Fatal: mutex unlock failed\n"); + exit(-2); + } +} + static void *thrmgr_worker(void *arg) { threadpool_t *threadpool = (threadpool_t *) arg; @@ -217,58 +309,44 @@ static void *thrmgr_worker(void *arg) int retval, must_exit = FALSE; struct timespec timeout; + /* Register cleanup procedure for worker in current thread */ + pthread_cleanup_push(thrmgr_worker_cleanup, arg); + /* loop looking for work */ for (;;) { if (pthread_mutex_lock(&(threadpool->pool_mutex)) != 0) { - /* Fatal error */ logg("!Fatal: mutex lock failed\n"); exit(-2); } timeout.tv_sec = time(NULL) + threadpool->idle_timeout; timeout.tv_nsec = 0; threadpool->thr_idle++; - while (((job_data=work_queue_pop(threadpool->queue)) == NULL) + while ( must_exit == FALSE + && ((job_data = work_queue_pop(threadpool->queue)) == NULL) && (threadpool->state != POOL_EXIT)) { /* Sleep, awaiting wakeup */ pthread_cond_signal(&threadpool->idle_cond); retval = pthread_cond_timedwait(&(threadpool->pool_cond), &(threadpool->pool_mutex), &timeout); - if (retval == ETIMEDOUT) { + if (retval == ETIMEDOUT) must_exit = TRUE; - break; - } } threadpool->thr_idle--; if (threadpool->state == POOL_EXIT) { must_exit = TRUE; } - + if (pthread_mutex_unlock(&(threadpool->pool_mutex)) != 0) { - /* Fatal error */ logg("!Fatal: mutex unlock failed\n"); exit(-2); } - if (job_data) { - threadpool->handler(job_data); - } else if (must_exit) { - break; - } - } - if (pthread_mutex_lock(&(threadpool->pool_mutex)) != 0) { - /* Fatal error */ - logg("!Fatal: mutex lock failed\n"); - exit(-2); - } - threadpool->thr_alive--; - if (threadpool->thr_alive == 0) { - /* signal that all threads are finished */ - pthread_cond_broadcast(&threadpool->pool_cond); - } - if (pthread_mutex_unlock(&(threadpool->pool_mutex)) != 0) { - /* Fatal error */ - logg("!Fatal: mutex unlock failed\n"); - exit(-2); + if (must_exit) break; + if (job_data) threadpool->handler(job_data); + if (threadpool->state == POOL_STOP) break; } + + pthread_yield(); /* do not remove on premptive kernel e.g linux 2.6 */ + pthread_cleanup_pop(1); return NULL; } @@ -289,7 +367,6 @@ int thrmgr_dispatch(threadpool_t *thread if (threadpool->state != POOL_VALID) { if (pthread_mutex_unlock(&(threadpool->pool_mutex)) != 0) { logg("!Mutex unlock failed\n"); - return FALSE; } return FALSE; } diff -urp clamav-0.93-orig/clamd/thrmgr.h clamav-0.93-new/clamd/thrmgr.h --- clamav-0.93-orig/clamd/thrmgr.h 2008-03-06 19:41:03.000000000 +0100 +++ clamav-0.93-new/clamd/thrmgr.h 2008-06-09 11:00:02.000000000 +0200 @@ -26,6 +26,13 @@ #include #endif +/** + * OPTIMIZE_MEMORY_FOOTPRINT : #ifdef, it will force all worker threads to terminate + * before switching to new database, thereby, avoiding fragmenting memory due which + * will cause swelling resident memory during run-time. + */ +#define OPTIMIZE_MEMORY_FOOTPRINT + typedef struct work_item_tag { struct work_item_tag *next; void *data; @@ -41,6 +48,7 @@ typedef struct work_queue_tag { typedef enum { POOL_INVALID, POOL_VALID, + POOL_STOP, /* All worker threads should exit */ POOL_EXIT } pool_state_t; @@ -66,4 +74,13 @@ threadpool_t *thrmgr_new(int max_threads void thrmgr_destroy(threadpool_t *threadpool); int thrmgr_dispatch(threadpool_t *threadpool, void *user_data); +#ifdef OPTIMIZE_MEMORY_FOOTPRINT +/** + * thrmgr_worker_stop_wait : set state to POOL_STOP, wake all thread worker, wait for them + * to exit before continuing. + */ +void thrmgr_worker_stop_wait(threadpool_t * const threadpool); +void thrmgr_setstate(threadpool_t * const threadpool, pool_state_t state); +#endif + #endif diff -urp clamav-0.93-orig/libclamav/readdb.c clamav-0.93-new/libclamav/readdb.c --- clamav-0.93-orig/libclamav/readdb.c 2008-04-02 22:17:28.000000000 +0200 +++ clamav-0.93-new/libclamav/readdb.c 2008-06-09 11:00:02.000000000 +0200 @@ -1684,7 +1684,9 @@ void cl_free(struct cl_engine *engine) pthread_mutex_lock(&cli_ref_mutex); #endif - engine->refcount--; + if(engine->refcount) + engine->refcount--; + if(engine->refcount) { #ifdef CL_THREAD_SAFE pthread_mutex_unlock(&cli_ref_mutex);