syslog-ng-anon

 This patch adds the capability to syslog-ng that allows you to strip

 out any given regexp or all IP addresses from log messages before

 they are written to disk. The goal is to give the system administrator

 the means to implement site logging policies, by allowing them easy

 control over exactly what data they retain in their logfiles,

 regardless of what a particular daemon might think is best.

Background:

 Data retention has become a hot legal topic for ISPs and other Online

 Service Providers (OSPs). There are many instances where it is preferable

 to keep less information on users than is collected by default on many

 systems. In the United States it is not currently required to retain

 data on users of a server, but you may be required to provide all data

 on a user which you have retained. OSPs can protect themselves from legal

 hassles and added work by choosing what data they wish to retain.

 From "Best Practices for Online Service Providers"

 (http://www.eff.org/osp):

  As an intermediary, the OSP [Online Service Provider] finds itself in

  a position to collect and store detailed information about its users

  and their online activities that may be of great interest to third

  parties. The USA PATRIOT Act also provides the government with

  expanded powers to request this information. As a result, OSP owners

  must deal with requests from law enforcement and lawyers to hand over

  private user information and logs. Yet, compliance with these demands

  takes away from an OSP's goal of providing users with reliable,

  secure network services. In this paper, EFF offers some suggestions,

  both legal and technical, for best practices that balance the needs

  of OSPs and their users' privacy and civil liberties.

  Rather than scrubbing the information you don't want in logs, this patch

  ensures that the information is never written to disk. Also, for those 

  daemons which log through syslog facilities, this patch provides a 

  convenient single configuration to limit what you wish to log.

  Here are some related links:

  Best Practices for Online Service Providers

  http://www.eff.org/osp

  http://www.eff.org/osp/20040819_OSPBestPractices.pdf

  EPIC International Data Retention Page

  http://www.epic.org/privacy/intl/data_retention.html

  Working Paper on Usage Log Data Management (from Computer, Freedom, and 

  Privacy conference) http://cryptome.org/usage-logs.htm

Installing syslog-ng-anon 

 Applying the patch

  This patch has been tested against the following versions of syslog-ng:

 	. version 1.6.7

 	. Debian package syslog-ng_1.6.7-2

  To use this patch, obtain the source for syslog-ng 

  (http://www.balabit.com/downloads/syslog-ng/1.6/src/) and the latest

  syslog-ng-anon patch (http://dev.riseup.net/patches/syslog-ng/). 

  Uncompress the syslog-ng source and then apply the patch:

  % tar -zxvf syslog-ng.tar.gz

  % cd syslog-ng

  % patch -p1 < syslog-ng-anon.diff

  Then compile and install syslog-ng as normal.

 Debian package

  Alternately, you can install syslog-ng-anon from this repository:

  deb http://deb.riseup.net/debian unstable main

 How to use it

  This patch adds the filter "strip". For example:

 	filter f_strip {strip(<regexp>);};

  This will strip out all matches of the regular expression on logs to

  which the filter is applied and replaces all matches with the fixed length

  four dashes ("----").

  In place of a regular expression, you can put "ips", which will replace all

  internet addresses with 0.0.0.0. For example:

 	filter f_strip {strip(ips);};

  You can alter what the replacement strings are by using replace:

#

# Configuration file for syslog-ng under Debian.

# Customized for riseup.net using syslog-ng-anon patch

# (http://dev.riseup.net/patches/syslog-ng/)

#

# see http://www.campin.net/syslog-ng/expanded-syslog-ng.conf

# for examples.

#

# levels: emerg alert crit err warning notice info debug

#

############################################################

## global options

options {

    chain_hostnames(0);

    time_reopen(10);

    time_reap(360);

    sync(0);

    log_fifo_size(2048);

    create_dirs(yes);

    group(adm);

    perm(0640);

    dir_perm(0755);

    use_dns(no);

};

############################################################

## universal source

source s_all {

    internal();

    unix-stream("/dev/log");

    file("/proc/kmsg" log_prefix("kernel: "));

};

############################################################

## generic destinations

destination df_facility_dot_info   { file("/var/log/$FACILITY.info");   };

destination df_facility_dot_notice { file("/var/log/$FACILITY.notice"); };

destination df_facility_dot_warn   { file("/var/log/$FACILITY.warn");   };

destination df_facility_dot_err    { file("/var/log/$FACILITY.err");    };

destination df_facility_dot_crit   { file("/var/log/$FACILITY.crit");   };

############################################################

## generic filters

filter f_strip { strip(ips); };

filter f_at_least_info   { level(info..emerg);   };

filter f_at_least_notice { level(notice..emerg); };

filter f_at_least_warn   { level(warn..emerg);   };

filter f_at_least_err    { level(err..emerg);    };

filter f_at_least_crit   { level(crit..emerg);   };

############################################################

## auth.log

filter f_auth { facility(auth, authpriv); };

destination df_auth { file("/var/log/auth.log"); };

log {

    source(s_all);

    filter(f_auth);

    destination(df_auth);

};

############################################################

## daemon.log

filter f_daemon { facility(daemon); };

destination df_daemon { file("/var/log/daemon.log"); };

log {

    source(s_all);

    filter(f_daemon);

    destination(df_daemon);

};

############################################################

## kern.log

filter f_kern { facility(kern); };

destination df_kern { file("/var/log/kern.log"); };

log {

    source(s_all);

    filter(f_kern);

    destination(df_kern);

};

############################################################

## user.log

filter f_user { facility(user); };

destination df_user { file("/var/log/user.log"); };

log {

    source(s_all);

    filter(f_user);

    destination(df_user);

};

############################################################

## sympa.log

filter f_sympa { program("^(sympa|bounced|archived|task_manager)"); };

destination d_sympa { file("/var/log/sympa.log"); };

log {

	source(s_all);

	filter(f_sympa);

	destination(d_sympa);

	flags(final);

};

############################################################

## wwsympa.log

filter f_wwsympa { program("^wwsympa"); };

destination d_wwsympa { file("/var/log/wwsympa.log"); };

log {

	source(s_all);

	filter(f_wwsympa);

	filter(f_strip);

	destination(d_wwsympa);

	flags(final);

};

############################################################

## ldap.log

filter f_ldap { program("slapd"); };

destination d_ldap { file("/var/log/ldap.log"); };

log {

	source(s_all);

	filter(f_ldap);

	destination(d_ldap);

	flags(final);

};

############################################################

## postfix.log

# special source because of chroot jail

#source s_postfix { unix-stream("/var/spool/postfix/dev/log" keep-alive(yes)); }; 

filter f_postfix { program("^postfix/"); };

destination d_postfix { file("/var/log/postfix.log"); };

log {

	source(s_all);

	filter(f_postfix);

	filter(f_strip);

	destination(d_postfix);

	flags(final);

};

############################################################

## courier.log

filter f_courier { program("courier|imap|pop"); };

destination d_courier { file("/var/log/courier.log"); };

log {

	source(s_all);

	filter(f_courier);

	filter(f_strip);

	destination(d_courier);

	flags(final);

};

############################################################

## maildrop.log

filter f_maildrop { program("^maildrop"); };

destination d_maildrop { file("/var/log/maildrop.log"); };

log {

	source(s_all);

	filter(f_maildrop);

	destination(d_courier);

	flags(final);

};

############################################################

## mail.log

filter f_mail { facility(mail); };

destination df_mail { file("/var/log/mail.log"); };

log {

    source(s_all);

    filter(f_mail);

    destination(df_mail);

};

############################################################

## messages.log

filter f_messages {

	level(debug,info,notice)

	and not facility(auth,authpriv,daemon,mail,user,kern);

};

destination df_messages { file("/var/log/messages.log"); };

log {

    source(s_all);

    filter(f_messages);

    destination(df_messages);

};

############################################################

## errors.log

filter f_errors {

	level(warn,err,crit,alert,emerg)

	and not facility(auth,authpriv,daemon,mail,user,kern);

};

destination df_errors { file("/var/log/errors.log"); };

log {

	source(s_all);

	filter(f_errors);

	destination(df_errors);

};

############################################################

## emergencies

filter f_emerg { level(emerg); };

destination du_all { usertty("*"); };

log {

	source(s_all);

	filter(f_emerg);

	destination(du_all);

};

############################################################

## console messages

filter f_xconsole {

    facility(daemon,mail)

    or level(debug,info,notice,warn)

    or (facility(news)

    and level(crit,err,notice));

};

destination dp_xconsole { pipe("/dev/xconsole"); };

log {

    source(s_all);

    filter(f_xconsole);

    destination(dp_xconsole);

};

	| KW_STRIP '(' string ')'		{ $$ = make_filter_strip($3); free($3); }

	| KW_REPLACE '(' string string ')'		{ $$ = make_filter_replace($3,$4); free($3); free($4); }

        { "strip",		KW_STRIP },

        { "replace",	KW_REPLACE },

       (replace string)

struct filter_expr_node *make_filter_strip(const char *re)

{

	if (strcasecmp(re,"ips") == 0)

		return make_filter_replace(re,"0.0.0.0");

	else

		return make_filter_replace(re,"----");

}

#define FMIN(a,b) (a)<(b) ? (a):(b)

static int do_filter_replace(struct filter_expr_node *c, 

			   struct log_filter *rule UNUSED,

			   struct log_info *log)

{

	CAST(filter_expr_re, self, c);

	char * buffer = log->msg->data;

	int snippet_size;

	regmatch_t pmatch;

	char new_msg[2048];

	char * new_msg_max = new_msg+2048;

	char * new_msg_ptr = new_msg;

	int replace_length = strlen(self->replace->data);

	int error = regexec(&self->regex, buffer, 1, &pmatch, 0);

	if (error != 0) return 1;

	while (error==0) {

		/* copy string snippet which preceeds matched text */

		snippet_size = FMIN(pmatch.rm_so, new_msg_max-new_msg_ptr);

		memcpy(new_msg_ptr, buffer, snippet_size);

		new_msg_ptr += snippet_size;

		/* copy replacement string */

		snippet_size = FMIN(replace_length, new_msg_max-new_msg_ptr);

		memcpy(new_msg_ptr, self->replace->data, snippet_size);

		new_msg_ptr += snippet_size;

		/* search for next match */

		buffer += pmatch.rm_eo;

		error = regexec (&self->regex, buffer, 1, &pmatch, REG_NOTBOL);

	}

	/* copy the rest of the old msg */

	snippet_size = FMIN(strlen(buffer),new_msg_max-new_msg_ptr);

	memcpy(new_msg_ptr, buffer, snippet_size); 

	new_msg_ptr += snippet_size;

	ol_string_free(log->msg);

	log->msg = c_format_cstring("%s", new_msg_ptr-new_msg,new_msg);

	return 1;

}

struct filter_expr_node *make_filter_replace(const char *re, const char *replacement)

{

	int regerr;

	NEW(filter_expr_re, self);

	self->super.eval = do_filter_replace;

	self->replace = format_cstring(replacement);

	if (strcasecmp(re,"ips") == 0) {

		re = "(25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])([\\.\\-](25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])){3}";

	}

	regerr = regcomp(&self->regex, re, REG_ICASE | REG_EXTENDED);

	if (regerr) {

		char errorbuf[256];

		regerror(regerr, &self->regex, errorbuf, sizeof(errorbuf));

		werror("Error compiling regular expression: \"%z\" (%z)\n", re, errorbuf);

		KILL(self);

		return NULL;

	}

	return &self->super;

}

  struct ol_string *replace;

  ol_string_free(i->replace);

struct filter_expr_node *make_filter_strip(const char *re);

struct filter_expr_node *make_filter_replace(const char *re, const char *replacement);