[prev in list] [next in list] [prev in thread] [next in thread]
List: linux-ha-dev
Subject: [Linux-ha-dev] A test tool for heartbeat messaging layer
From: "Zhu, Yi" <yi.zhu () intel ! com>
Date: 2003-08-29 10:47:24
[Download RAW message or body]
Hi,
I've wrote a test tool to simulate packets retransmission. It can be used
for heartbeat messaging layer code testing.
Please patch the testhook patch upon the ordered layer patch and recompile
the project (because it will call the APIs provided by the ordered layer).
Because it will also modify the heartbeat daemon code, please restart
heartbeat service as well. After make install, there will be a test tool
called rexmit_test installed in /usr/lib/heartbeat. The usage of this tool
is as follow.
Usage: rexmit_test [-l|--listen] [-v|--verbose]
[-s|--send <number>] [-o|--order][-d|--drop <index>] [-D|--delay <index>:<number>]
To run the test tool, you need run two instances on two different nodes.
One in listen (or receive) mode and the other in send mode.
This command need root privilege to read and write the PING FIFOs.
For example,
(Listen or receive on Node1)
$ rexmit_test -l -v
(Send packets on Node2)
$ rexmit_test -s3 -v
Send 3 no order guaranteed packets to all nodes in the cluster.
$ rexmit_test -s5 -o Node1
Send 5 ordered packets to only Node1.
$ rexmit_test -s3 -d1 -o
Send 3 ordered packets to the cluster and the second (0 is the first)
packet will be droped by the receiver (its retransmission packet after
this will not be droped).
$ rexmit_test -s19 -D0:40 -v -o Node1
Send 19 ordered packets to Node1 and the first packet will be delayed to
receive by the receiver after 40 packets.
See heartbeat/rexmit_test.c file more information.
Thanks,
--
-----------------------------------------------------------------
Opinions expressed are those of the author and do not represent
Intel Corp.
Zhu Yi (Chuyee)
Intel China Software Lab (ICSL)
22nd Floor, ShanghaiMart Tower No. 2299 Yan'an Road(West)
Shanghai 200336, PRC
Tel: 8621-52574545-1261
Fax: 8621-62366119
GnuPG v1.0.6 (GNU/Linux)
http://cn.geocities.com/chewie_chuyee/gpg.txt or
$ gpg --keyserver wwwkeys.pgp.net --recv-keys 71C34820
1024D/71C34820 C939 2B0B FBCE 1D51 109A 55E5 8650 DB90 71C3 4820
["heartbeat-testhook.patch" (TEXT/PLAIN)]
diff -Nura --exclude CVS linux-ha/heartbeat/api_test.c linux-ha-dev/heartbeat/api_test.c
--- linux-ha/heartbeat/api_test.c 2003-08-13 19:31:19.000000000 +0800
+++ linux-ha-dev/heartbeat/api_test.c 2003-08-28 01:55:25.000000000 +0800
@@ -105,6 +105,7 @@
int j;
(void)_ha_msg_h_Id;
+ (void)_heartbeat_h_Id;
cl_log_set_entity(argv[0]);
cl_log_enable_stderr(TRUE);
diff -Nura --exclude CVS linux-ha/heartbeat/heartbeat.c linux-ha-dev/heartbeat/heartbeat.c
--- linux-ha/heartbeat/heartbeat.c 2003-08-13 19:31:19.000000000 +0800
+++ linux-ha-dev/heartbeat/heartbeat.c 2003-08-27 17:59:12.000000000 +0800
@@ -1627,6 +1627,12 @@
longclock_t messagetime;
const char * cseq;
seqno_t seqno = 0;
+#ifdef TESTHOOK
+ static struct ha_msg * delay_msg = NULL;
+ static seqno_t delay_seq = 0;
+ static seqno_t orig_seq = 0;
+ static const char * delay_from = NULL;
+#endif
@@ -1660,6 +1666,31 @@
}
}
+#ifdef TESTHOOK
+ /* If it's time to process the delayed packet */
+ cseq = ha_msg_value(msg, F_SEQ);
+ from = ha_msg_value(msg, F_ORIG);
+ if (cseq != NULL && sscanf(cseq, "%lx", &seqno) == 1 && delay_msg
+ && seqno >= delay_seq && strcmp(from, delay_from) == 0){
+ cl_log(LOG_WARNING, "Process delayed seq 0x%lx "
+ "and drop current seq 0x%lx from %s"
+ , orig_seq
+ , seqno
+ , from);
+ msg = delay_msg;
+ /*
+ * The memory for delay_msg will be lost after the
+ * function, however it's just a test code, don't worry
+ * about it. :-P
+ */
+ delay_msg = NULL;
+ delay_seq = 0;
+ }else if (delay_msg && orig_seq == seqno) {
+ /* This is a rexmit packet, not time to send, discard it */
+ return;
+ }
+#endif
+
/* Extract message type, originator, timestamp, auth */
type = ha_msg_value(msg, F_TYPE);
from = ha_msg_value(msg, F_ORIG);
@@ -1735,6 +1766,49 @@
#endif
}
+#ifdef TESTHOOK
+ cl_log(LOG_INFO, "Seqno 0x%lx node %s", seqno, thisnode->nodename);
+ /* All the test packet has the T_TEST TYPE */
+ if (strcmp(type, T_TEST) == 0) {
+ const char * ctest;
+
+ if (thisnode == curnode)
+ goto out;
+ if ((ctest = ha_msg_value(msg, F_TESTOPT)) == NULL)
+ goto out;
+ if (strcmp(ctest, "drop") == 0) {
+ /* Throw this packet here to test rexmit */
+ static seqno_t dropseq = 0;
+
+ if (seqno != dropseq) {
+ cl_log(LOG_WARNING, "Discard seq 0x%lx from %s"
+ , seqno
+ , thisnode->nodename);
+ dropseq = seqno;
+ return;
+ }else
+ dropseq = 0;
+ }
+ if (strncmp(ctest, "delay ", 6) == 0 && !delay_msg
+ && seqno != orig_seq) {
+ if (sscanf(ctest+6, "%lx", &delay_seq) != 1)
+ goto out;
+ orig_seq = seqno;
+ delay_seq += seqno;
+ delay_msg = ha_msg_copy(msg);
+ delay_from = ha_msg_value(delay_msg, F_ORIG);
+ cl_log(LOG_WARNING, "Save delay seq 0x%lx from %s until"
+ " seq 0x%lx in address <0x%p>"
+ , seqno
+ , thisnode->nodename
+ , delay_seq
+ , delay_msg);
+ return;
+ }
+ }
+out:
+#endif
+
/* Throw away some incoming packets if testing is enabled */
if (TESTRCV) {
if (thisnode != curnode && TestRand(rcv_loss_prob)) {
diff -Nura --exclude CVS linux-ha/heartbeat/Makefile.am linux-ha-dev/heartbeat/Makefile.am
--- linux-ha/heartbeat/Makefile.am 2003-08-13 19:31:19.000000000 +0800
+++ linux-ha-dev/heartbeat/Makefile.am 2003-08-14 10:42:46.000000000 +0800
@@ -65,7 +65,8 @@
libhbclient_la_LDFLAGS = -version-info 0:0:0
## binary progs
-halib_PROGRAMS = heartbeat send_arp findif api_test get_hw_addr
+halib_PROGRAMS = heartbeat send_arp findif api_test get_hw_addr \
+ rexmit_test
## SOURCES
heartbeat_SOURCES = heartbeat.c auth.c \
@@ -83,7 +84,7 @@
-dlpreopen force \
-dlopen self
-heartbeat_CFLAGS = $(AM_CFLAGS)
+heartbeat_CFLAGS = $(AM_CFLAGS) -DTESTHOOK
send_arp_SOURCES = send_arp.c
send_arp_CFLAGS = @LIBNETDEFINES@ $(CFLAGS) $(INCLUDES)
@@ -109,6 +110,10 @@
api_test_LDADD = $(top_builddir)/lib/clplumbing/libplumb.la \
libhbclient.la $(gliblib)
+rexmit_test_SOURCES = rexmit_test.c
+rexmit_test_LDADD = $(top_builddir)/lib/clplumbing/libplumb.la \
+ libhbclient.la $(gliblib)
+
## SCRIPTS/DATA
ha_DATA = shellfuncs README.config
ha_SCRIPTS = harc
diff -Nura --exclude CVS linux-ha/heartbeat/rexmit_test.c linux-ha-dev/heartbeat/rexmit_test.c
--- linux-ha/heartbeat/rexmit_test.c 1970-01-01 08:00:00.000000000 +0800
+++ linux-ha-dev/heartbeat/rexmit_test.c 2003-08-28 01:55:48.000000000 +0800
@@ -0,0 +1,369 @@
+/*
+ * rexmit_test: Test program for testing the heartbeat protocol packets
+ * retransmission and ordering.
+ *
+ * Copyright (C) 2003 Yi Zhu <yi.zhu@intel.com>
+ * Copyright (C) 2000 Alan Robertson <alanr@unix.sh>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <portability.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <getopt.h>
+#include <errno.h>
+#include <clplumbing/cl_log.h>
+#include <clplumbing/cl_signal.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <sys/time.h>
+#include <sys/stat.h>
+#include <stdarg.h>
+#include <syslog.h>
+#include <hb_api_core.h>
+#include <hb_api.h>
+
+#define RECV 0
+#define SEND 1
+
+/*
+ * A heartbeat API test program...
+ */
+
+void NodeStatus(const char * node, const char * status, void * private);
+void LinkStatus(const char * node, const char *, const char *, void*);
+void gotsig(int nsig);
+void usage(void);
+
+void
+NodeStatus(const char * node, const char * status, void * private)
+{
+ cl_log(LOG_NOTICE, "Status update: Node %s now has status %s\n"
+ , node, status);
+}
+
+void
+LinkStatus(const char * node, const char * lnk, const char * status
+, void * private)
+{
+ cl_log(LOG_NOTICE, "Link Status update: Link %s/%s now has status %s\n"
+ , node, lnk, status);
+}
+
+int quitnow = 0;
+void gotsig(int nsig)
+{
+ (void)nsig;
+ quitnow = 1;
+}
+
+void usage()
+{
+ fprintf(stderr, "Usage: rexmit_test [-l|--listen] [-v|--verbose]\n");
+ fprintf(stderr, "\t\t [-s|--send <number>] [-o|--order]"
+ "[-d|--drop <index>] [-D|--delay <index>:<number>]\n");
+}
+
+int
+main(int argc, char ** argv)
+{
+ struct ha_msg* msg = NULL;
+ ll_cluster_t* hb;
+ const char * node;
+ const char * intf;
+ const char * tonode = NULL;
+ int msgcount=0;
+ int policy = RECV;
+ int c;
+ int opt_listen = 0;
+ int opt_send = 0;
+ int opt_ordered = 0;
+ int opt_drop = -1;
+ int opt_delay_idx = -1;
+ int opt_delay_num = 0;
+ int opt_verbose = 0;
+ int i;
+
+ (void)_ha_msg_h_Id;
+ (void)_heartbeat_h_Id;
+
+ while (1) {
+ int optidx = 0;
+ static struct option long_options[] = {
+ {"listen", 0, 0, 'l'},
+ {"send", 2, 0, 's'},
+ {"ordered", 0, 0, 'o'},
+ {"drop", 1, 0, 'd'},
+ {"delay", 1, 0, 'D'},
+ {"verbose", 0, 0, 'v'},
+ {0, 0, 0, 0}
+ };
+ c = getopt_long(argc, argv, "ls::od:D:v", long_options, &optidx);
+ if (c == -1)
+ break;
+ switch (c) {
+ case 'l':
+ opt_listen = 1;
+ break;
+ case 's':
+ if (optarg) {
+ if (sscanf(optarg, "%d", &opt_send) != 1) {
+ usage();
+ exit(1);
+ }
+ }else
+ opt_send = 1;
+ break;
+ case 'o':
+ opt_ordered = 1;
+ break;
+ case 'd':
+ if (optarg && sscanf(optarg, "%d", &opt_drop) != 1) {
+ usage();
+ exit(1);
+ }
+ break;
+ case 'D':
+ if (optarg && sscanf(optarg, "%d:%d", &opt_delay_idx
+ , &opt_delay_num) != 2) {
+ usage();
+ exit(1);
+ }
+ break;
+ case 'v':
+ opt_verbose = 1;
+ break;
+ default:
+ usage();
+ exit(1);
+ }
+ }
+ if (optind == argc) {
+ tonode = NULL;
+ }else if (optind == argc - 1) {
+ tonode = argv[optind];
+ }else {
+ usage();
+ exit(1);
+ }
+ if (opt_listen == 1 && opt_send > 0) {
+ usage();
+ exit(1);
+ }else if (opt_send > 0) {
+ policy = SEND;
+ if (opt_drop >= opt_send) {
+ fprintf(stderr, "Error: drop packet index > send "
+ "packets number\n");
+ exit(1);
+ }
+ if (opt_delay_num < 0 || opt_delay_idx >= opt_send) {
+ fprintf(stderr, "Error: delay number < 0 or "
+ "delay index > send packets\n");
+ exit(1);
+ }
+ }else if (opt_ordered || opt_drop != -1 || opt_delay_idx != -1
+ || opt_delay_num) {
+ usage();
+ exit(1);
+ }else
+ policy = RECV;
+
+ cl_log_set_entity(argv[0]);
+ if (opt_verbose)
+ cl_log_enable_stderr(TRUE);
+ cl_log_set_facility(LOG_USER);
+ hb = ll_cluster_new("heartbeat");
+ cl_log(LOG_INFO, "PID=%ld\n", (long)getpid());
+ cl_log(LOG_INFO, "Signing in with heartbeat\n");
+ if (hb->llc_ops->signon(hb, "ping")!= HA_OK) {
+ cl_log(LOG_ERR, "Cannot sign on with heartbeat\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(1);
+ }
+
+ if (hb->llc_ops->set_nstatus_callback(hb, NodeStatus, NULL) !=HA_OK){
+ cl_log(LOG_ERR, "Cannot set node status callback\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(2);
+ }
+
+ if (hb->llc_ops->set_ifstatus_callback(hb, LinkStatus, NULL)!=HA_OK){
+ cl_log(LOG_ERR, "Cannot set if status callback\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(3);
+ }
+
+ cl_log(LOG_INFO, "Starting node walk\n");
+ if (hb->llc_ops->init_nodewalk(hb) != HA_OK) {
+ cl_log(LOG_ERR, "Cannot start node walk\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(5);
+ }
+ while((node = hb->llc_ops->nextnode(hb))!= NULL) {
+ cl_log(LOG_INFO, "Cluster node: %s: status: %s\n", node
+ , hb->llc_ops->node_status(hb, node));
+ if (hb->llc_ops->init_ifwalk(hb, node) != HA_OK) {
+ cl_log(LOG_ERR, "Cannot start if walk\n");
+ cl_log(LOG_ERR, "REASON: %s\n"
+ , hb->llc_ops->errmsg(hb));
+ exit(6);
+ }
+ while ((intf = hb->llc_ops->nextif(hb))) {
+ cl_log(LOG_ERR, "\tnode %s: intf: %s ifstatus: %s\n"
+ , node, intf
+ , hb->llc_ops->if_status(hb, node, intf));
+ }
+ if (hb->llc_ops->end_ifwalk(hb) != HA_OK) {
+ cl_log(LOG_ERR, "Cannot end if walk\n");
+ cl_log(LOG_ERR, "REASON: %s\n"
+ , hb->llc_ops->errmsg(hb));
+ exit(7);
+ }
+ }
+ if (hb->llc_ops->end_nodewalk(hb) != HA_OK) {
+ cl_log(LOG_ERR, "Cannot end node walk\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(8);
+ }
+
+ CL_SIGINTERRUPT(SIGINT, 1);
+ CL_SIGNAL(SIGINT, gotsig);
+
+ /* New code begins here */
+ if (policy == RECV)
+ goto receive;
+
+ /* SEND code path */
+ for (i = 0; i < opt_send; i++) {
+ msg = ha_msg_new(0);
+ ha_msg_add(msg, F_TYPE, T_TEST);
+ if (i == opt_drop) {
+ /*
+ * Indicate this packet will be dropped
+ * and rexmitted later.
+ */
+ ha_msg_add(msg, F_TESTOPT, "drop");
+ }
+ if (i == opt_delay_idx) {
+ char str[32];
+ /*
+ * Delay this packet opt_delay_num number later
+ */
+ snprintf(str, sizeof(str), "delay %x", opt_delay_num);
+ ha_msg_add(msg, F_TESTOPT, str);
+ }
+ if (tonode == NULL) {
+ typedef int (*ops_clustersend_t)(ll_cluster_t *
+ , struct ha_msg *);
+ ops_clustersend_t ops_clustersend;
+
+ if (opt_ordered)
+ ops_clustersend = (void *)
+ hb->llc_ops->send_ordered_clustermsg;
+ else
+ ops_clustersend = (void *)
+ hb->llc_ops->sendclustermsg;
+
+ if (ops_clustersend(hb, msg) == HA_OK)
+ cl_log(LOG_INFO, "Sent msg %d to cluster\n", i);
+ else
+ cl_log(LOG_ERR, "FAIL to send msg %d to cluster"
+ "\n", i);
+ }else {
+ typedef int (*ops_nodesend_t)(ll_cluster_t *
+ , struct ha_msg *, const char *);
+ ops_nodesend_t ops_nodesend;
+
+ if (opt_ordered)
+ ops_nodesend = (void *)
+ hb->llc_ops->send_ordered_nodemsg;
+ else
+ ops_nodesend = (void *)
+ hb->llc_ops->sendnodemsg;
+
+ if (ops_nodesend(hb, msg, tonode) == HA_OK)
+ cl_log(LOG_INFO, "Sent msg %d to node %s\n", i
+ , tonode);
+ else
+ cl_log(LOG_ERR, "FAIL to send msg %d to node %s"
+ "\n", i, tonode);
+ }
+ if (opt_verbose)
+ ha_log_message(msg);
+ ha_msg_del(msg); msg=NULL;
+ }
+ goto finish;
+
+receive:
+ /* RECV code path */
+ cl_log(LOG_INFO, "Waiting for messages...\n");
+ errno = 0;
+
+ for(; !quitnow && (msg=hb->llc_ops->readmsg(hb, 1)) != NULL;) {
+ const char * type;
+ const char * orig;
+ const char * cseq;
+ const char * dropit;
+ unsigned long seqno;
+ ++msgcount;
+ if ((type = ha_msg_value(msg, F_TYPE)) == NULL) {
+ type = "?";
+ }
+ if ((orig = ha_msg_value(msg, F_ORIG)) == NULL) {
+ orig = "?";
+ }
+ cl_log(LOG_NOTICE, "Got message %d of type [%s] from [%s]\n"
+ , msgcount, type, orig);
+
+ if (strcmp(type, T_TEST) == 0) {
+ if ((cseq = ha_msg_value(msg, F_SEQ)) != NULL)
+ if (sscanf(cseq, "%lx", &seqno) != 1) {
+ cl_log(LOG_ERR, "bad seqno");
+ continue;
+ }
+ if ((dropit = ha_msg_value(msg, F_TESTOPT)) != NULL)
+ fprintf(stderr, "Got message %d from %s,"
+ " seqno=[%lx], F_TESTOPT=[%s]\n"
+ , msgcount, orig, seqno, dropit);
+ else
+ fprintf(stderr, "Got message %d from %s,"
+ " seqno=[%lx]\n"
+ , msgcount, orig, seqno);
+ if (opt_verbose)
+ ha_log_message(msg);
+ }
+ ha_msg_del(msg); msg=NULL;
+ }
+ if (!quitnow) {
+ cl_log(LOG_ERR, "read_hb_msg returned NULL");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ }
+finish:
+ if (hb->llc_ops->signoff(hb) != HA_OK) {
+ cl_log(LOG_ERR, "Cannot sign off from heartbeat.\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(10);
+ }
+ if (hb->llc_ops->delete(hb) != HA_OK) {
+ cl_log(LOG_ERR, "Cannot delete API object.\n");
+ cl_log(LOG_ERR, "REASON: %s\n", hb->llc_ops->errmsg(hb));
+ exit(11);
+ }
+ return 0;
+}
diff -Nura --exclude CVS linux-ha/include/ha_msg.h linux-ha-dev/include/ha_msg.h
--- linux-ha/include/ha_msg.h 2003-08-13 19:31:19.000000000 +0800
+++ linux-ha-dev/include/ha_msg.h 2003-08-14 10:42:46.000000000 +0800
@@ -77,6 +77,7 @@
#define F_NODETYPE "nodetype" /* Type of node */
#define F_RTYPE "rtype" /* Resource type */
#define F_ORDERSEQ "oseq" /* Order Sequence number */
+#define F_TESTOPT "testopt" /* Test Option flag */
/* Message types */
#define T_STATUS "status" /* Status (heartbeat) */
@@ -90,6 +91,7 @@
#define T_APIREQ "hbapi-req" /* Heartbeat API request */
#define T_APIRESP "hbapi-resp" /* Heartbeat API response */
#define T_APICLISTAT "hbapi-clstat" /* Client status notification" */
+#define T_TEST "test" /* Test Type */
#define NOSEQ_PREFIX "NS_" /* PREFIX: Give no sequence number */
/* Used for messages which can't be retransmitted */
_______________________________________________________
Linux-HA-Dev: Linux-HA-Dev@lists.linux-ha.org
http://lists.linux-ha.org/mailman/listinfo/linux-ha-dev
Home Page: http://linux-ha.org/
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic