[prev in list] [next in list] [prev in thread] [next in thread]
List: openssl-users
Subject: Mysterious hang with openssl and asan on ubuntu 18.04
From: Dan Kegel <dank () kegel ! com>
Date: 2020-02-24 6:07:32
Message-ID: CAPF-yOZY6VHJb2SsqqW1KiPE+G67u1ZxCJ6p5V6__1Ogtjm_Jg () mail ! gmail ! com
[Download RAW message or body]
Hi folks.
The project I'm working on exhibits a hang in one test case when
dealing with openssl connections, usually on 8 core machines,
when built with address sanitizer enabled. This is mature,
theoretically well-debugged production code. With help from c-reduce,
I minimized the rather complex test case to 108 lines of C
linked to openssl. The hang occurs only on Ubuntu 18.04, not
on ubuntu 19.10 or 20.04 beta. Before I go trying to run c-reduce
on openssl, I thought I'd run the so-far minimal reproducer by folks
here and see if anyone can think of a change between openssl 1.1.1 and
1.1.13
(well, to be exact, ubuntu's 1.1.1-1ubuntu2.1~18.04.5 and 1.1.1d-2ubuntu3)
that might account for this.
The following script reproduces the hang reliably for me in under a minute
(sometimes under a second):
#!/bin/sh
set -ex
gcc -g -O2 -fsanitize=address -pthread bug.i -lssl -o bug
export ASAN_OPTIONS=detect_stack_use_after_return=1
export LSAN_OPTIONS=verbosity=1
for iter in $(seq 1 1000)
do
./bug
done
echo "No hang found."
where bug.i contains:
-- snip --
static int readers = 40;
static int once_control = 0;
static int test_secs = 1;
typedef int pid_t;
struct timeval {
long tv_sec;
long tv_usec;
};
struct timespec {
long tv_sec;
long tv_nsec;
};
typedef unsigned long int pthread_t;
enum __itimer_which {
ITIMER_REAL,
};
struct itimerval {
struct timeval it_interval;
struct timeval it_value;
};
typedef int sig_atomic_t;
typedef void(*__sighandler_t);
struct sigaction {
struct {
__sighandler_t sa_handler;
} __sigaction_handler;
};
typedef struct ssl_ctx_st SSL_CTX;
typedef struct ssl_method_st SSL_METHOD;
static SSL_CTX *context;
static struct timespec start_time;
static struct timespec goal_end_time;
static volatile sig_atomic_t keep_on_chugging = 1;
void do_once() {
const struct ssl_method_st *p = 0;
SSL_CTX *context = SSL_CTX_new(p);
}
void thread_main(void *v) { }
void do_work() {
pthread_t net_5;
pthread_once(&once_control, do_once);
pthread_create(&net_5, 0, thread_main, 0);
}
static _Bool is_time_to_quit(void) {
struct timespec now_time;
clock_gettime(0, &now_time);
long long remaining_nsecs =
(goal_end_time.tv_sec - now_time.tv_sec) * 1000000000ULL;
remaining_nsecs += goal_end_time.tv_nsec - now_time.tv_nsec;
if (remaining_nsecs < 0) return 1;
return 0;
}
static void set_flag_for_exit(int signo) {
if (is_time_to_quit()) keep_on_chugging = 0;
}
static void set_timer(void) {
struct itimerval iv = {{0}, {0}};
iv.it_value.tv_usec = 3000;
iv.it_interval.tv_usec = 3000;
if (setitimer(ITIMER_REAL, &iv, ((void *)0)) < 0)
abort();
if (is_time_to_quit()) exit(0);
}
static void create_children(void) {
int i;
struct sigaction act;
memset(&act, 0, sizeof(act));
act.__sigaction_handler.sa_handler = set_flag_for_exit;
sigaction(14, &act, ((void *)0));
act.__sigaction_handler.sa_handler = ((__sighandler_t)1);
pid_t pid;
for (i = 0; i < readers; i++) {
if (i < readers) {
if ((pid = fork()) < 0)
abort();
if (pid == 0) {
set_timer();
do_work();
_exit(0);
}
}
}
}
static int reap_children(void) {
int i;
int status;
for (i = 0; i < readers; i++) wait(&status);
}
int main(int argc, char *argv[]) {
clock_gettime(0, &start_time);
goal_end_time = start_time;
goal_end_time.tv_sec += test_secs;
create_children();
reap_children();
}
-- snip --
[Attachment #3 (text/html)]
<div dir="ltr">Hi folks.<div><br></div><div>The project I'm working on exhibits \
a hang in one test case when</div><div>dealing with openssl connections, usually on 8 \
core machines,</div><div>when built with address sanitizer enabled. This is \
mature,</div><div>theoretically well-debugged production code. With help from \
c-reduce,</div><div>I minimized the rather complex test case to 108 lines of \
C</div><div>linked to openssl. The hang occurs only on Ubuntu 18.04, \
not</div><div>on ubuntu 19.10 or 20.04 beta. Before I go trying to run c-reduce \
</div><div>on openssl, I thought I'd run the so-far minimal reproducer by \
folks</div><div>here and see if anyone can think of a change between openssl 1.1.1 \
and 1.1.13</div><div>(well, to be exact, ubuntu's 1.1.1-1ubuntu2.1~18.04.5 and \
1.1.1d-2ubuntu3)</div><div>that might account for this.</div><div><br></div><div>The \
following script reproduces the hang reliably for me in under a \
minute</div><div>(sometimes under a \
second):</div><div><br></div><div>#!/bin/sh<br>set -ex<br>gcc -g -O2 \
-fsanitize=address -pthread bug.i -lssl -o bug<br>export \
ASAN_OPTIONS=detect_stack_use_after_return=1<br>export \
LSAN_OPTIONS=verbosity=1<br>for iter in $(seq 1 1000)<br>do<br> \
./bug<br>done<br>echo "No hang found."<br></div><div><br></div><div>where \
bug.i contains:</div><div><div><br></div><div>-- snip --</div></div><div>static int \
readers = 40;<br>static int once_control = 0;<br>static int test_secs = \
1;<br><br>typedef int pid_t;<br>struct timeval {<br> long tv_sec;<br> long \
tv_usec;<br>};<br>struct timespec {<br> long tv_sec;<br> long \
tv_nsec;<br>};<br>typedef unsigned long int pthread_t;<br>enum __itimer_which {<br> \
ITIMER_REAL,<br>};<br>struct itimerval {<br> struct timeval it_interval;<br> \
struct timeval it_value;<br>};<br>typedef int sig_atomic_t;<br>typedef \
void(*__sighandler_t);<br>struct sigaction {<br> struct {<br> __sighandler_t \
sa_handler;<br> } __sigaction_handler;<br>};<br><br>typedef struct ssl_ctx_st \
SSL_CTX;<br>typedef struct ssl_method_st SSL_METHOD;<br>static SSL_CTX \
*context;<br><br>static struct timespec start_time;<br>static struct timespec \
goal_end_time;<br>static volatile sig_atomic_t keep_on_chugging = 1;<br><br>void \
do_once() {<br> const struct ssl_method_st *p = 0;<br> SSL_CTX *context = \
SSL_CTX_new(p);<br>}<br><br>void thread_main(void *v) { }<br><br>void do_work() {<br> \
pthread_t net_5;<br> pthread_once(&once_control, do_once);<br> \
pthread_create(&net_5, 0, thread_main, 0);<br>}<br><br>static _Bool \
is_time_to_quit(void) {<br> struct timespec now_time;<br> clock_gettime(0, \
&now_time);<br> long long remaining_nsecs =<br> (goal_end_time.tv_sec - \
now_time.tv_sec) * 1000000000ULL;<br> remaining_nsecs += goal_end_time.tv_nsec - \
now_time.tv_nsec;<br> if (remaining_nsecs < 0) return 1;<br> return \
0;<br>}<br><br>static void set_flag_for_exit(int signo) {<br> if \
(is_time_to_quit()) keep_on_chugging = 0;<br>}<br><br>static void set_timer(void) \
{<br> struct itimerval iv = {{0}, {0}};<br> iv.it_value.tv_usec = 3000;<br> \
iv.it_interval.tv_usec = 3000;<br> if (setitimer(ITIMER_REAL, &iv, ((void *)0)) \
< 0)<br> abort();<br> if (is_time_to_quit()) exit(0);<br>}<br><br>static \
void create_children(void) {<br> int i;<br> struct sigaction act;<br> \
memset(&act, 0, sizeof(act));<br> act.__sigaction_handler.sa_handler = \
set_flag_for_exit;<br> sigaction(14, &act, ((void *)0));<br> \
act.__sigaction_handler.sa_handler = ((__sighandler_t)1);<br> pid_t pid;<br> for \
(i = 0; i < readers; i++) {<br> if (i < readers) {<br> if ((pid = \
fork()) < 0)<br> abort();<br> if (pid == 0) {<br> \
set_timer();<br> do_work();<br> _exit(0);<br> }<br> \
}<br> }<br>}<br><br>static int reap_children(void) {<br> int i;<br> int \
status;<br> for (i = 0; i < readers; i++) wait(&status);<br>}<br><br>int \
main(int argc, char *argv[]) {<br> clock_gettime(0, &start_time);<br> \
goal_end_time = start_time;<br> goal_end_time.tv_sec += test_secs;<br><br> \
create_children();<br> reap_children();<br>}<br></div><div><br></div><div>-- snip \
--</div></div>
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic