[prev in list] [next in list] [prev in thread] [next in thread] 

List:       opensuse-commit
Subject:    commit mcelog for openSUSE:Factory
From:       root () hilbert ! suse ! de (h_root)
Date:       2010-10-28 13:37:57
Message-ID: 20101028133757.7A6B9202AD () hilbert ! suse ! de
[Download RAW message or body]


Hello community,

here is the log from the commit of package mcelog for openSUSE:Factory
checked in at Thu Oct 28 15:37:57 CEST 2010.



--------
--- mcelog/mcelog.changes	2010-02-19 00:48:59.000000000 +0100
+++ mcelog/mcelog.changes	2010-10-25 15:51:12.000000000 +0200
@@ -1,0 +2,7 @@
+Mon Oct 25 15:48:57 CEST 2010 - trenn@suse.de
+
+- Add Sandybridge/Westmere decode support
+- Fix domainname for email notification
+- Update to latest git version
+
+-------------------------------------------------------------------

calling whatdependson for head-i586


Old:
----
  fix_cache_trigger.patch
  mcelog-1.0pre3.tar.bz2

New:
----
  README.email_setup
  email.patch
  mcelog-1.0pre3.6363f5b719e9.tar.bz2
  mcelog.sysconfig

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Other differences:
------------------
++++++ mcelog.spec ++++++
--- /var/tmp/diff_new_pack.IqJrIB/_old	2010-10-28 15:37:35.000000000 +0200
+++ /var/tmp/diff_new_pack.IqJrIB/_new	2010-10-28 15:37:35.000000000 +0200
@@ -1,5 +1,5 @@
 #
-# spec file for package mcelog (Version 1.0pre3)
+# spec file for package mcelog (Version 1.0pre3.6363f5b719e9)
 #
 # Copyright (c) 2010 SUSE LINUX Products GmbH, Nuernberg, Germany.
 #
@@ -21,14 +21,18 @@
 Name:           mcelog
 License:        GPLv2+
 Summary:        Log Machine Check Events
-Version:        1.0pre3
+Version:        1.0pre3.6363f5b719e9
 Release:        1
 AutoReqProv:    on
 ExclusiveArch:  x86_64
-PreReq:         %insserv_prereq
+BuildRequires:  libesmtp-devel
+Requires:       logrotate
+PreReq:         %insserv_prereq %fillup_prereq
 Source:         mcelog-%{version}.tar.bz2
 Source1:        mcelog.service
-Patch1:         fix_cache_trigger.patch
+Source2:        mcelog.sysconfig
+Source6:        README.email_setup
+Patch1:         email.patch
 Group:          System/Monitoring
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 
@@ -53,6 +57,7 @@
 %patch1 -p1
 
 %build
+export SUSE_ASNEEDED=0
 make CFLAGS="$RPM_OPT_FLAGS"
 
 %install 
@@ -65,12 +70,19 @@
 ln -s /etc/init.d/mcelog $RPM_BUILD_ROOT/usr/sbin/rcmcelog
 install -m644 mcelog.logrotate $RPM_BUILD_ROOT/etc/logrotate.d/mcelog
 
+mkdir -p $RPM_BUILD_ROOT/var/adm/fillup-templates
+install -m 644 %SOURCE2 $RPM_BUILD_ROOT/var/adm/fillup-templates/sysconfig.mcelog
+
+mkdir -p $RPM_BUILD_ROOT/%_docdir/%name
+install -m 644 %SOURCE6 $RPM_BUILD_ROOT/%_docdir/%name/README.email_setup
+install -m 644 lk10-mcelog.pdf $RPM_BUILD_ROOT/%_docdir/%name/lk10-mcelog.pdf
+
 %clean
 rm -rf $RPM_BUILD_ROOT
 
 %post
 if [ -c /dev/mcelog ];then
-    %{insserv_force_if_yast mcelog}
+    %{fillup_and_insserv -y mcelog}
 fi
 
 %preun
@@ -88,7 +100,9 @@
 /etc/init.d/mcelog
 %dir /etc/mcelog
 %config /etc/mcelog/mcelog.conf
+/var/adm/fillup-templates/sysconfig.mcelog
 /etc/mcelog/*trigger
 /usr/sbin/rcmcelog
+%_docdir/%name
 
 %changelog

++++++ README.email_setup ++++++
MACHINE CHECK EXCPETION NOTIFICATION VIA EMAIL
==============================================

(C)opyright by Thomas Renninger <trenn@suse.de> Novell Inc. 2010

The setup to send Machine Check Exceptions (MCEs) via email relies on a
working smtp server listening on localhost on port 25.

How this can easily be configured is can be read up here:
http://en.opensuse.org/Mail_server_HOWTO
in the "Outgoing" section.

Test your setup by trying to send test mails via the "mail" shell command,
included in the mailx package.

Specify the email address where the MCEs should get mailed to here:
/etc/sysconfig/mcelog

You can filter MCE mails by matching against these mail headers.
Either one of these headers are set:
  - X-Mcelog-Uncorrectable
  - X-Mcelog-Correctable

and one of these are set:
  - X-Mcelog-Memory
  - X-Mcelog-CPU
  - X-Mcelog-Misc


NOTE: If broken HW results in an MCE storm of dozens and hundreds of MCEs,
mcelog will not sending them all to not overload the machine and network
traffic. If in doubt, check the local mcelog log files.


Autoyast
--------

For people making use of autoyast to spread similar installations on multiple
machines, here are some hints how to set up the email notification through
autoyast. Please read the autoyast documentation first if you are not familiar
with how to create an autoyast.xml file.

This simply sets the email address, notifications should get send to:

  <sysconfig config:type="list">
    <sysconfig_entry>
      <sysconfig_key>MCELOG_ADMIN_EMAIL</sysconfig_key>
      <sysconfig_path>/etc/sysconfig/mcelog</sysconfig_path>
      <sysconfig_value>trenn@suse.de</sysconfig_value>
    </sysconfig_entry>
 </sysconfig>


This is an example of how to set up postfix to listen on localhost and
sending/forwarding all mails coming in there through the smtp server
relay.suse.de.
The alias at the beginning forwards local machine notifications sent to root,
to trenn@suse.de. Like that mails interesting for the administrator can easily
be collected and sent to one email address. But this is just one possible mail
set up example.

<mail>
    <aliases config:type="list">
      <alias>
        <alias>root</alias>
        <destinations>trenn@suse.de</destinations>
      </alias>
    </aliases>
    <connection_type config:type="symbol">permanent</connection_type>
    <listen_remote config:type="boolean">false</listen_remote>
    <masquerade_other_domains config:type="list">
      <domain>suse.de</domain>
    </masquerade_other_domains>
    <mta config:type="symbol">postfix</mta>
    <outgoing_mail_server>relay.suse.de</outgoing_mail_server>
    <postfix_mda config:type="symbol">local</postfix_mda>
    <use_amavis config:type="boolean">false</use_amavis>
</mail>
++++++ email.patch ++++++
---
 Makefile |   10 +++
 email.c  |  184 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 email.h  |   32 ++++++++++
 mcelog.c |   93 +++++++++++++++++++++++++++++++
 mcelog.h |    1 
 msg.c    |    8 ++
 6 files changed, 325 insertions(+), 3 deletions(-)

Index: mcelog-1.0pre3.6363f5b719e9/Makefile
===================================================================
--- mcelog-1.0pre3.6363f5b719e9.orig/Makefile
+++ mcelog-1.0pre3.6363f5b719e9/Makefile
@@ -1,3 +1,4 @@
+CONFIG_EMAIL := 1
 CFLAGS := -g -Os
 prefix := /usr
 etcprefix :=
@@ -34,7 +35,8 @@ OBJ := p4.o k8.o mcelog.o dmi.o tsc.o co
        client.o cache.o sysfs.o yellow.o page.o rbtree.o 	 \
        xeon75xx.o sandy-bridge.o
 DISKDB_OBJ := diskdb.o dimm.o db.o
-CLEAN := mcelog dmi tsc dbquery .depend .depend.X dbquery.o ${DISKDB_OBJ}
+EMAIL_OBJ := email.o
+CLEAN := mcelog dmi tsc dbquery .depend .depend.X dbquery.o ${DISKDB_OBJ} ${EMAIL_OBJ}
 DOC := mce.pdf
 
 ADD_DEFINES :=
@@ -46,6 +48,12 @@ OBJ += ${DISKDB_OBJ}
 all: dbquery
 endif
 
+ifdef CONFIG_EMAIL
+ADD_DEFINES := -DCONFIG_EMAIL=1
+LDFLAGS := -lesmtp
+OBJ += ${EMAIL_OBJ}
+endif
+
 SRC := $(OBJ:.o=.c)
 
 mcelog: ${OBJ}
Index: mcelog-1.0pre3.6363f5b719e9/email.c
===================================================================
--- /dev/null
+++ mcelog-1.0pre3.6363f5b719e9/email.c
@@ -0,0 +1,184 @@
+#include <unistd.h>
+#include <signal.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+#define __USE_GNU
+/* To fetch the dnsname */
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+
+#include <libesmtp.h>
+#include "mcelog.h"
+#include "email.h"
+
+#define MAX_STRING_LEN 512
+char c_recipient[MAX_STRING_LEN] = "";
+static int debug;
+static char dnsname[MAX_STRING_LEN];
+
+static char buf[128];
+#define ERROR() { fprintf (stderr, "SMTP problem [%d] %s\n", __LINE__,	\
+			 smtp_strerror (smtp_errno (), buf, sizeof buf));	\
+		return -1; }
+
+
+void email_usage(void) {
+	fprintf(stderr,
+		"--email address     Requires daemon mode\n");
+}
+
+int email_cmd(int opt, int ac, char **av)
+{
+	char *arg = optarg;
+
+	switch (opt) {
+	case O_EMAIL_ADDRESS:
+		if (arg) {
+			if (strlen(arg) >= MAX_STRING_LEN) {
+				Eprintf("email address too long"
+					" [max:%d]\n", MAX_STRING_LEN);
+				return 0;
+			}
+			strcpy(c_recipient, arg);
+			return 1;
+		}
+	case O_EMAIL_DEBUG:
+		debug = 1;
+		return 0;
+	}
+	return 0;
+}
+
+/* Callback to prnt the recipient status */
+static void
+print_recipient_status (smtp_recipient_t recipient,
+                        const char *mailbox, void *arg)
+{
+	const smtp_status_t *status;
+
+	status = smtp_recipient_status (recipient);
+	if (debug)
+		printf ("%s: %d %s", mailbox, status->code, status->text);
+}
+
+void setup_mail_header(FILE *fp, struct mce *m)
+{
+	char host[MAX_STRING_LEN];
+	struct addrinfo hints;
+	struct addrinfo *res=NULL;
+	int    ret, retry=3;
+
+	/* Taken from net-tools hostname.c showhname() */
+	memset(&hints, 0, sizeof(struct addrinfo));
+	hints.ai_family   = AF_UNSPEC;
+	hints.ai_flags    = AI_CANONNAME | AI_CANONIDN;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_protocol = 0;
+
+	if (gethostname(host, MAX_STRING_LEN)) {
+		fprintf(stderr, "Cannot get host name\n");
+		return;
+	}
+
+	do {
+		ret = getaddrinfo(host, NULL, &hints, &res);
+	} while(ret == EAI_AGAIN && retry-- > 0
+                             && usleep(50000) == 0);
+
+	if (ret != 0 || res == NULL) {
+		fprintf(stderr, "Could not retrieve hostname\n");
+		return;
+	}
+
+	memset(dnsname, '\0', MAX_STRING_LEN);
+	strncpy(dnsname, res->ai_canonname, MAX_STRING_LEN - 1);
+
+	fprintf(fp, "Return-Path: <dummy@will_get_overridden.net>\r\n"
+		"Subject: Machine Check Exception on %s detected\r\n"
+		"MIME-Version: 1.0\r\n"
+		"Content-Type: text/plain;\r\n"
+		"  charset=iso-8859-1\r\n"
+		"Content-Transfer-Encoding: 7bit\r\n\r\n", dnsname);
+	freeaddrinfo(res);
+}
+
+
+int send_mail(FILE *fp)
+{
+	char smtp_host[MAX_STRING_LEN] = "localhost:25";
+	char from[MAX_STRING_LEN];
+
+	const smtp_status_t *status;
+	smtp_session_t session;
+	smtp_message_t message;
+	smtp_recipient_t recipient;
+	struct sigaction sa;
+
+	session = smtp_create_session ();
+	message = smtp_add_message (session);
+
+	snprintf(from, MAX_STRING_LEN, "root@%s", dnsname);
+
+	/* NB.  libESMTP sets timeouts as it progresses through the protocol.
+	   In addition the remote server might close its socket on a timeout.
+	   Consequently libESMTP may sometimes try to write to a socket with
+	   no reader.  Ignore SIGPIPE, then the program doesn't get killed
+	   if/when this happens. */
+	sa.sa_handler = SIG_IGN;
+	sigemptyset (&sa.sa_mask);
+	sa.sa_flags = 0;
+	sigaction (SIGPIPE, &sa, NULL);
+
+	/* Set the host running the SMTP server.  LibESMTP has a default port
+	   number of 587, however this is not widely deployed so the port
+	   is specified as 25 along with the default MTA host. */
+	if (!smtp_set_server (session, smtp_host))
+		ERROR();
+
+	smtp_set_reverse_path (message, from);
+
+	/* RFC 2822 doesn't require recipient headers but a To: header would
+	   be nice to have if not present. */
+	smtp_set_header (message, "To", NULL, NULL);
+
+	/* RFC 2822 doesn't require recipient headers but a To: header would
+	   be nice to have if not present. */
+	if (!smtp_set_header (message, "From", "mcelog", from))
+		ERROR();
+
+	smtp_set_message_fp (message, fp);
+
+	recipient = smtp_add_recipient (message, c_recipient);
+	if (!recipient)
+		ERROR();
+	if (!smtp_dsn_set_notify (recipient, Notify_NEVER))
+		ERROR();
+
+	/* Initiate a connection to the SMTP server and transfer the
+	   message. */
+	if (!smtp_start_session (session))
+		Eprintf("SMTP server problem %s\n",
+			smtp_strerror (smtp_errno (), buf, sizeof buf));
+	else {
+		/* Report on the success or otherwise of the mail transfer.
+		 */
+		if (debug) {
+			status = smtp_message_transfer_status (message);
+			printf ("%d %s", status->code,
+				(status->text != NULL) ? status->text : "\n");
+		}
+		smtp_enumerate_recipients (message, print_recipient_status, NULL);
+	}
+
+	if (debug)
+		fprintf(stderr, "Email sent successfully!\n");
+
+	/* Free resources consumed by the program.
+	 */
+	smtp_destroy_session (session);
+	return 0;
+}
Index: mcelog-1.0pre3.6363f5b719e9/email.h
===================================================================
--- /dev/null
+++ mcelog-1.0pre3.6363f5b719e9/email.h
@@ -0,0 +1,32 @@
+#ifndef _MCELOG_EMAIL_H_
+#define _MCELOG_EMAIL_H_
+
+extern FILE *email_fd;
+extern int email_mode;
+
+#ifdef CONFIG_EMAIL
+extern int send_mail(FILE *email_fd);
+extern void setup_mail_header(FILE *email_fd, struct mce *m);
+extern void email_usage(void);
+extern int email_cmd(int opt, int ac, char **av);
+
+#define EMAIL_OPTIONS					\
+	{ "email", 1, NULL, O_EMAIL_ADDRESS },		\
+	{ "email-debug", 0, NULL, O_EMAIL_DEBUG },
+
+enum email_options {
+	O_EMAIL_ADDRESS = O_EMAIL,
+	O_EMAIL_DEBUG,
+};
+
+#else
+/*
+static int send_mail(FILE *email_fd) { return 0; }
+static void setup_mail_header(FILE *email_fd) { return; };
+*/
+static void email_usage(void) { return; }
+static int email_cmd(int opt, int ac, char **av) { return 0; }
+#define EMAIL_OPTIONS
+#endif
+
+#endif
Index: mcelog-1.0pre3.6363f5b719e9/mcelog.c
===================================================================
--- mcelog-1.0pre3.6363f5b719e9.orig/mcelog.c
+++ mcelog-1.0pre3.6363f5b719e9/mcelog.c
@@ -37,6 +37,7 @@
 #include <assert.h>
 #include <signal.h>
 #include <pwd.h>
+#include <sys/wait.h>
 #include "mcelog.h"
 #include "paths.h"
 #include "k8.h"
@@ -58,6 +59,9 @@
 #include "yellow.h"
 #include "page.h"
 
+#include "email.h"
+int email_mode;
+
 enum cputype cputype = CPU_GENERIC;	
 
 char *logfn = LOG_DEV_FILENAME; 
@@ -69,7 +73,7 @@ static double cpumhz;
 static int cpumhz_forced;
 int ascii_mode;
 int dump_raw_ascii;
-int daemon_mode;
+int daemon_mode = 0;
 static char *inputfile;
 char *processor_flags;
 static int foreground;
@@ -792,6 +796,7 @@ void usage(void)
 "--num-errors N      Only process N errors (for testing)\n"
 "--pidfile file	     Write pid of daemon into file\n"
 		);
+	email_usage();
 	diskdb_usage();
 	print_cputypes();
 	exit(1);
@@ -855,6 +860,7 @@ static struct option options[] = {
 	{ "num-errors", 1, NULL, O_NUMERRORS },
 	{ "pidfile", 1, NULL, O_PIDFILE },
 	DISKDB_OPTIONS
+	EMAIL_OPTIONS
 	{}
 };
 
@@ -1026,11 +1032,86 @@ static void drop_cred(void)
 	}
 }
 
+#ifdef CONFIG_EMAIL
+pid_t c_pid;
+
+/* Not more than 12 mails in 5 mins... */
+#define LAST_LIMIT_COUNT (60 * 5)
+#define LIMIT_COUNT      12
+static time_t last_limit_count;
+static int limit_count;
+static const char *mail_thread = "mcelog_mail_thread";
+
+
+static int setup_email(struct mce *m) {
+	int pdes[2];
+	static int suppressed;
+	int ret;
+
+	if (time(NULL) - last_limit_count < LAST_LIMIT_COUNT) {
+		if (limit_count >= LIMIT_COUNT && !suppressed) {
+			Eprintf("email rate limit [%d mails per %d mins]"
+				" reached, mails supressed\n",
+				LIMIT_COUNT, LAST_LIMIT_COUNT / 60);
+			suppressed = 1;
+		}
+		if (suppressed)
+			return -1;
+	} else {
+		suppressed = 0;
+		limit_count = 0;
+		last_limit_count = time(NULL);
+	}
+
+	limit_count++;
+
+	ret = pipe(pdes);
+	if (ret)
+		return ret;
+
+	c_pid = mcelog_fork(mail_thread);
+	if ( c_pid == 0 ) { /* child */
+		FILE *x = fdopen(pdes[0], "r");
+		close(pdes[1]);
+		send_mail(x);
+		exit(0);
+	} else {
+		close(pdes[0]);
+		/* something went wrong, better close... */
+		if (email_fd)
+			fclose(email_fd);
+		/* Wprintf will now also write into this pipe */
+		email_fd = fdopen(pdes[1], "w");
+		setup_mail_header(email_fd, m);
+	}
+	return 0;
+}
+
+static int finish_email(void) {
+	int status;
+
+	fclose(email_fd);
+	fprintf(stderr, "Email set up for sending\n");
+	/* Anything else we can make sure we do not get orphaned threads? */
+	waitpid (c_pid, &status, WUNTRACED);
+	if (WIFSTOPPED(status)){
+		kill(c_pid, 9);
+		SYSERRprintf("Killed stopped email thread %d\n",
+			     c_pid);
+		return -1;
+	}
+	email_fd = NULL;
+	return 0;
+}
+
+#endif
+
 static void process(int fd, unsigned recordlen, unsigned loglen, char *buf)
 {	
 	int i; 
 	int len;
 	int finish = 0;
+	int mail_setup;
 
 	if (recordlen == 0) {
 		Wprintf("no data in mce record\n");
@@ -1041,19 +1122,23 @@ static void process(int fd, unsigned rec
 	if (len < 0) 
 		err("read"); 
 
-	for (i = 0; (i < len / (int)recordlen) && !finish; i++) { 
+	for (i = 0; (i < len / (int)recordlen) && !finish; i++) {
 		struct mce *mce = (struct mce *)(buf + i*recordlen);
 		mce_prepare(mce);
 		if (numerrors > 0 && --numerrors == 0)
 			finish = 1;
 		if (!mce_filter(mce, recordlen)) 
 			continue;
+		if (email_mode)
+			mail_setup = setup_email(mce);
 		if (!dump_raw_ascii) {
 			disclaimer();
 			Wprintf("MCE %d\n", i);
 			dump_mce(mce, recordlen);
 		} else
 			dump_mce_raw_ascii(mce, recordlen);
+		if (email_mode && !mail_setup)
+			finish_email();
 		flushlog();
 	}
 
@@ -1161,6 +1246,8 @@ int main(int ac, char **av)
 			exit(0);
 		} else if (diskdb_cmd(opt, ac, av)) {
 			exit(0);
+		} else if (email_cmd(opt, ac, av)) {
+			email_mode = 1;
 		} else if (opt == 0)
 			break;		    
 	} 
@@ -1169,6 +1256,8 @@ int main(int ac, char **av)
 		logfn = av[optind++];
 	if (av[optind])
 		usage();
+	/* email sending only in daemon mode */
+	email_mode &= daemon_mode;
 	checkdmi();
 	general_setup();
 		
Index: mcelog-1.0pre3.6363f5b719e9/msg.c
===================================================================
--- mcelog-1.0pre3.6363f5b719e9.orig/msg.c
+++ mcelog-1.0pre3.6363f5b719e9/msg.c
@@ -8,10 +8,13 @@
 #include "mcelog.h"
 #include "msg.h"
 #include "memutil.h"
+#include "email.h"
+
 
 enum syslog_opt syslog_opt = SYSLOG_REMARK;
 int syslog_level = LOG_WARNING;
 static FILE *output_fh;
+       FILE *email_fd;
 static char *output_fn;
 
 int need_stdout(void)
@@ -135,6 +138,11 @@ int Wprintf(char *fmt, ...)
 		n = vfprintf(output_fh ? output_fh : stdout, fmt, ap);
 		va_end(ap);
 	}
+	if (email_fd) {
+ 		va_start(ap,fmt);
+ 		n = vfprintf(email_fd, fmt, ap);
+ 		va_end(ap);
+ 	}
 	return n;
 }
 
Index: mcelog-1.0pre3.6363f5b719e9/mcelog.h
===================================================================
--- mcelog-1.0pre3.6363f5b719e9.orig/mcelog.h
+++ mcelog-1.0pre3.6363f5b719e9/mcelog.h
@@ -122,6 +122,7 @@ enum cputype {
 enum option_ranges {
 	O_COMMON = 500,
 	O_DISKDB = 1000,
+	O_EMAIL  = 1500,
 };
 
 enum syslog_opt { 
++++++ mcelog.service ++++++
--- /var/tmp/diff_new_pack.IqJrIB/_old	2010-10-28 15:37:35.000000000 +0200
+++ /var/tmp/diff_new_pack.IqJrIB/_new	2010-10-28 15:37:35.000000000 +0200
@@ -31,7 +31,7 @@
 ### END INIT INFO
 
 . /etc/rc.status
-#. /etc/sysconfig/mcelog
+[ -r /etc/sysconfig/mcelog ] && . /etc/sysconfig/mcelog
 
 # Shell functions sourced from /etc/rc.status:
 #      rc_check         check and set local and overall rc status
@@ -48,6 +48,9 @@
 mcelog="/usr/sbin/mcelog"
 mcelog_pid="/var/run/mcelog.pid"
 mcelog_params=" --daemon --pidfile ${mcelog_pid}"
+if [ "${MCELOG_ADMIN_EMAIL}x" != "x"  ];then
+    mcelog_params="$mcelog_params --email $MCELOG_ADMIN_EMAIL"
+fi
 mcedev=/dev/mcelog
 
 check="checkproc ${mcelog}"

++++++ mcelog.sysconfig ++++++
## Path:        Hardware/machine_check
## Description: email address machine check exceptions are sent to
## Type:        string
## Default:     ""
#
# Machine check exceptions like memory (correctable or uncorrectable
# ECC errors), processor or other HW are sent with a detailed description
# to this address. Also read README.email_setup for further details.
#
MCELOG_ADMIN_EMAIL="trenn@suse.de"

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++



Remember to have fun...

-- 
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org

[prev in list] [next in list] [prev in thread] [next in thread] 

Configure | About | News | Add a list | Sponsored by KoreLogic