tests: add unit tests covered with Clang sanitizers

Currently we run all tests via Valgrind. This patch adds 2nd batch of
tests which are compiled with Clang AddressSanitizer[1],
LeakSanitizer[2] and UndefinedBehaviorSanitizer[3] in order to catch
more issues during QA on CI.

AddressSanitizer is a fast memory error detector.  The tool can detect
the following types of bugs:

 * Out-of-bounds accesses to heap, stack and globals
 * Use-after-free, use-after-return, use-after-scope
 * Double-free, invalid free

LeakSanitizer is a run-time memory leak detector. It can be combined
with AddressSanitizer to get both memory error and leak detection, or
used in a stand-alone mode.

UndefinedBehaviorSanitizer (UBSan) is a fast undefined behavior
detector. UBSan modifies the program at compile-time to catch various
kinds of undefined behavior during program execution, for example:

 * Using misaligned or null pointer
 * Signed integer overflow
 * Conversion to, from, or between floating-point types which would
   overflow the destination

1. http://clang.llvm.org/docs/AddressSanitizer.html
2. http://http://clang.llvm.org/docs/LeakSanitizer.html
3. http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html

Signed-off-by: Petr Štetiar <ynezz@true.cz>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0b0c9e316bfbdfd777dd08f8a4490a5327627660..dcd455c02dfb36df7a52ab035e654af2452ee6d8 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,6 +48,14 @@ INSTALL(TARGETS ubox ubox-static
 ADD_SUBDIRECTORY(lua)
 ADD_SUBDIRECTORY(examples)
 
+MACRO(ADD_UNIT_TEST_SAN name)
+  ADD_EXECUTABLE(${name}-san ${name}.c)
+  TARGET_COMPILE_OPTIONS(${name}-san PRIVATE -g -fno-omit-frame-pointer -fsanitize=undefined,address,leak -fno-sanitize-recover=all)
+  TARGET_LINK_OPTIONS(${name}-san PRIVATE -fsanitize=undefined,address,leak)
+  TARGET_LINK_LIBRARIES(${name}-san ubox blobmsg_json json_script ${json})
+  TARGET_INCLUDE_DIRECTORIES(${name}-san PRIVATE ${PROJECT_SOURCE_DIR})
+ENDMACRO(ADD_UNIT_TEST_SAN)
+
 IF(UNIT_TESTING)
   ENABLE_TESTING()
   ADD_SUBDIRECTORY(tests)
@@ -62,6 +70,10 @@ IF(EXISTS ${json})
        SET_TARGET_PROPERTIES(blobmsg_json-static
                              PROPERTIES OUTPUT_NAME blobmsg_json)
 
+       IF(UNIT_TESTING)
+               ADD_UNIT_TEST_SAN(jshn)
+       ENDIF(UNIT_TESTING)
+
        ADD_EXECUTABLE(jshn jshn.c)
        TARGET_LINK_LIBRARIES(jshn blobmsg_json ${json})
 

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 1c448257a4d6849f8c78e46bcbd23a6fd1d84cdc..bd2205743318e09b47a5e2d352f610ea1edef536 100644 (file)
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -8,6 +8,7 @@ ENDMACRO(ADD_UNIT_TEST)
 
 FILE(GLOB test_cases "test-*.c")
 FOREACH(test_case ${test_cases})
-       GET_FILENAME_COMPONENT(test_case ${test_case} NAME_WE)
-       ADD_UNIT_TEST(${test_case})
+  GET_FILENAME_COMPONENT(test_case ${test_case} NAME_WE)
+  ADD_UNIT_TEST(${test_case})
+  ADD_UNIT_TEST_SAN(${test_case})
 ENDFOREACH(test_case)

diff --git a/tests/cram/test_avl.t b/tests/cram/test_avl.t
index 19a8d21f1f3e51944817aec6d03768dfee4c9e57..d8d1640008c65069612386fa5dd4737d3c63feb5 100644 (file)
--- a/tests/cram/test_avl.t
+++ b/tests/cram/test_avl.t
@@ -9,3 +9,12 @@ check that avl is producing expected results:
   test_basics: delete 'one' element
   test_basics: for each element reverse: zero two twelve three ten six seven nine four five eleven eight 
   test_basics: delete all elements
+
+  $ test-avl-san
+  test_basics: insert: 0=zero 0=one 0=two 0=three 0=four 0=five 0=six 0=seven 0=eight 0=nine 0=ten 0=eleven 0=twelve 
+  test_basics: insert duplicate: -1=zero -1=one -1=two -1=three -1=four -1=five -1=six -1=seven -1=eight -1=nine -1=ten -1=eleven -1=twelve 
+  test_basics: first=eight last=zero
+  test_basics: for each element: eight eleven five four nine one seven six ten three twelve two zero 
+  test_basics: delete 'one' element
+  test_basics: for each element reverse: zero two twelve three ten six seven nine four five eleven eight 
+  test_basics: delete all elements

diff --git a/tests/cram/test_base64.t b/tests/cram/test_base64.t
index 0a7a9d5c026dc64afa509fc4ea6c627b03e03b48..ade41fb1eb2a6b76b1acb3848fef740f586fd325 100644 (file)
--- a/tests/cram/test_base64.t
+++ b/tests/cram/test_base64.t
@@ -20,14 +20,38 @@ check that base64 is producing expected results:
   5 fooba
   6 foobar
 
+  $ test-b64-san
+  0 
+  4 Zg==
+  4 Zm8=
+  4 Zm9v
+  8 Zm9vYg==
+  8 Zm9vYmE=
+  8 Zm9vYmFy
+  0 
+  1 f
+  2 fo
+  3 foo
+  4 foob
+  5 fooba
+  6 foobar
+
 check that b64_encode and b64_decode assert invalid input
 
-  $ alias check="egrep '(dumped|Assertion)' | sed 's;.*\(b64_.*code\).*\(Assertion.*$\);\1: \2;' | LC_ALL=C sort"
+  $ alias check="egrep '(dumped|Assertion)' output.log | sed 's;.*\(b64_.*code\).*\(Assertion.*$\);\1: \2;' | LC_ALL=C sort"
+
+  $ test-b64_decode 2> output.log; check
+  Aborted (core dumped)
+  b64_decode: Assertion `dest && targsize > 0' failed.
+
+  $ test-b64_encode 2> output.log; check
+  Aborted (core dumped)
+  b64_encode: Assertion `dest && targsize > 0' failed.
 
-  $ test-b64_decode 2>&1 | check
+  $ test-b64_decode-san 2> output.log; check
   Aborted (core dumped)
   b64_decode: Assertion `dest && targsize > 0' failed.
 
-  $ test-b64_encode 2>&1 | check
+  $ test-b64_encode-san 2> output.log; check
   Aborted (core dumped)
   b64_encode: Assertion `dest && targsize > 0' failed.

diff --git a/tests/cram/test_blobmsg.t b/tests/cram/test_blobmsg.t
index 504a056f52c47f52b6ed0083db3f1ad934f8222a..3a5801a5a0f4cf731e61377dd2e50a3c669d6176 100644 (file)
--- a/tests/cram/test_blobmsg.t
+++ b/tests/cram/test_blobmsg.t
@@ -15,3 +15,18 @@ check that blobmsg is producing expected results:
   \tworld : 2 (esc)
   }
   json: {"message":"Hello, world!","testdata":{"double":133.700000,"hello":1,"world":"2"},"list":[0,1,2,133.700000]}
+
+  $ test-blobmsg-san
+  Message: Hello, world!
+  List: {
+  0
+  1
+  2
+  133.700000
+  }
+  Testdata: {
+  \tdouble : 133.700000 (esc)
+  \thello : 1 (esc)
+  \tworld : 2 (esc)
+  }
+  json: {"message":"Hello, world!","testdata":{"double":133.700000,"hello":1,"world":"2"},"list":[0,1,2,133.700000]}

diff --git a/tests/cram/test_jshn.t b/tests/cram/test_jshn.t
index 1881a3d639a5fdf65e2e4cc49f53bd8021600008..b2f28534a7372f3d606ee4abd9e292746ab26d1b 100644 (file)
--- a/tests/cram/test_jshn.t
+++ b/tests/cram/test_jshn.t
@@ -9,12 +9,20 @@ check usage:
   Usage: jshn [-n] [-i] -r <message>|-R <file>|-o <file>|-p <prefix>|-w
   [2]
 
+  $ jshn-san
+  Usage: jshn-san [-n] [-i] -r <message>|-R <file>|-o <file>|-p <prefix>|-w
+  [2]
+
 test bad json:
 
   $ jshn -r '[]'
   Failed to parse message data
   [1]
 
+  $ jshn-san -r '[]'
+  Failed to parse message data
+  [1]
+
 test good json:
 
   $ jshn -r '{"foo": "bar", "baz": {"next": "meep"}}'
@@ -24,16 +32,31 @@ test good json:
   json_add_string 'next' 'meep';
   json_close_object;
 
+  $ jshn-san -r '{"foo": "bar", "baz": {"next": "meep"}}'
+  json_init;
+  json_add_string 'foo' 'bar';
+  json_add_object 'baz';
+  json_add_string 'next' 'meep';
+  json_close_object;
+
 test json from file:
 
   $ echo '[]' > test.json; jshn -R test.json
   Failed to parse message data
   [1]
 
+  $ echo '[]' > test.json; jshn-san -R test.json
+  Failed to parse message data
+  [1]
+
   $ jshn -R nada.json
   Error opening nada.json
   [3]
 
+  $ jshn-san -R nada.json
+  Error opening nada.json
+  [3]
+
   $ echo '{"foo": "bar", "baz": {"next": "meep"}}' > test.json; jshn -R test.json
   json_init;
   json_add_string 'foo' 'bar';
@@ -41,38 +64,74 @@ test json from file:
   json_add_string 'next' 'meep';
   json_close_object;
 
+  $ echo '{"foo": "bar", "baz": {"next": "meep"}}' > test.json; jshn-san -R test.json
+  json_init;
+  json_add_string 'foo' 'bar';
+  json_add_object 'baz';
+  json_add_string 'next' 'meep';
+  json_close_object;
+
 test json formatting without prepared environment:
 
   $ jshn -p procd -w
   { }
 
+  $ jshn-san -p procd -w
+  { }
+
   $ jshn -i -p procd -w
   {
   \t (esc)
   }
 
+  $ jshn-san -i -p procd -w
+  {
+  \t (esc)
+  }
+
   $ jshn -i -n -p procd -w
   {
   \t (esc)
   } (no-eol)
 
+  $ jshn-san -i -n -p procd -w
+  {
+  \t (esc)
+  } (no-eol)
+
   $ jshn -p procd -o test.json; cat test.json
   { }
 
+  $ jshn-san -p procd -o test.json; cat test.json
+  { }
+
   $ jshn -i -p procd -o test.json; cat test.json
   {
   \t (esc)
   }
 
+  $ jshn-san -i -p procd -o test.json; cat test.json
+  {
+  \t (esc)
+  }
+
   $ jshn -i -n -p procd -o test.json; cat test.json
   {
   \t (esc)
   } (no-eol)
 
+  $ jshn-san -i -n -p procd -o test.json; cat test.json
+  {
+  \t (esc)
+  } (no-eol)
+
   $ chmod oug= test.json
   $ jshn -i -n -p procd -o test.json
   Error opening test.json
   [3]
+  $ jshn-san -i -n -p procd -o test.json
+  Error opening test.json
+  [3]
   $ rm -f test.json
 
 test json formatting with prepared environment:
@@ -104,6 +163,9 @@ test json formatting with prepared environment:
   $ jshn -p procd -w
   { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }
 
+  $ jshn-san -p procd -w
+  { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }
+
   $ jshn -i -p procd -w
   {
   \t"name": "urngd", (esc)
@@ -123,6 +185,25 @@ test json formatting with prepared environment:
   \t} (esc)
   }
 
+  $ jshn-san -i -p procd -w
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  }
+
   $ jshn -n -i -p procd -w
   {
   \t"name": "urngd", (esc)
@@ -142,9 +223,31 @@ test json formatting with prepared environment:
   \t} (esc)
   } (no-eol)
 
+  $ jshn-san -n -i -p procd -w
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  } (no-eol)
+
   $ jshn -p procd -o test.json; cat test.json
   { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }
 
+  $ jshn-san -p procd -o test.json; cat test.json
+  { "name": "urngd", "script": "\/etc\/init.d\/urngd", "instances": { "instance1": { "command": [ "\/sbin\/urngd" ] } }, "triggers": [ ], "data": { } }
+
   $ jshn -i -p procd -o test.json; cat test.json
   {
   \t"name": "urngd", (esc)
@@ -164,6 +267,25 @@ test json formatting with prepared environment:
   \t} (esc)
   }
 
+  $ jshn-san -i -p procd -o test.json; cat test.json
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  }
+
   $ jshn -n -i -p procd -o test.json; cat test.json
   {
   \t"name": "urngd", (esc)
@@ -183,7 +305,29 @@ test json formatting with prepared environment:
   \t} (esc)
   } (no-eol)
 
+  $ jshn-san -n -i -p procd -o test.json; cat test.json
+  {
+  \t"name": "urngd", (esc)
+  \t"script": "/etc/init.d/urngd", (esc)
+  \t"instances": { (esc)
+  \t\t"instance1": { (esc)
+  \t\t\t"command": [ (esc)
+  \t\t\t\t"/sbin/urngd" (esc)
+  \t\t\t] (esc)
+  \t\t} (esc)
+  \t}, (esc)
+  \t"triggers": [ (esc)
+  \t\t (esc)
+  \t], (esc)
+  \t"data": { (esc)
+  \t\t (esc)
+  \t} (esc)
+  } (no-eol)
+
   $ chmod oug= test.json
   $ jshn -n -i -p procd -o test.json
   Error opening test.json
   [3]
+  $ jshn-san -n -i -p procd -o test.json
+  Error opening test.json
+  [3]

diff --git a/tests/cram/test_json_script.t b/tests/cram/test_json_script.t
index 3e80a5c10b817a2ed912f5c58a959f5f592fb2fc..4af7f5414ab570c7132764276a402b0ecfe70fb6 100644 (file)
--- a/tests/cram/test_json_script.t
+++ b/tests/cram/test_json_script.t
@@ -3,6 +3,7 @@ set test bin path:
   $ [ -n "$TEST_BIN_DIR" ] && export PATH="$TEST_BIN_DIR:$PATH"
   $ export TEST_INPUTS="$TESTDIR/inputs"
   $ alias js="valgrind --quiet --leak-check=full test-json-script"
+  $ alias js-san="test-json-script-san"
 
 check that json-script is producing expected results:
 
@@ -10,25 +11,46 @@ check that json-script is producing expected results:
   Usage: test-json-script [VARNAME=value] <filename_json_script>
   [254]
 
+  $ js-san
+  Usage: test-json-script-san [VARNAME=value] <filename_json_script>
+  [254]
+
   $ echo '}' > test.json; js test.json
   load JSON data from test.json failed.
 
+  $ echo '}' > test.json; js-san test.json
+  load JSON data from test.json failed.
+
   $ js nada.json 2>&1 | grep load.*failed
   load JSON data from nada.json failed.
 
+  $ js-san nada.json 2>&1 | grep load.*failed
+  load JSON data from nada.json failed.
+
   $ echo '[ [ ] [ ] ]' > test.json; js test.json
   load JSON data from test.json failed.
 
+  $ echo '[ [ ] [ ] ]' > test.json; js-san test.json
+  load JSON data from test.json failed.
+
 check example json-script:
 
   $ js $TEST_INPUTS/json-script.json
   exec  /%/
   exec_if_or 
 
+  $ js-san $TEST_INPUTS/json-script.json
+  exec  /%/
+  exec_if_or 
+
   $ js EXECVAR=meh ORVAR=meep $TEST_INPUTS/json-script.json
   exec meh /%/
   exec_if_or meep
 
+  $ js-san EXECVAR=meh ORVAR=meep $TEST_INPUTS/json-script.json
+  exec meh /%/
+  exec_if_or meep
+
 check has expression:
 
   $ echo '
@@ -43,12 +65,21 @@ check has expression:
   $ js VAR=foo test.json
   echo bar
 
+  $ js-san VAR=foo test.json
+  echo bar
+
   $ js VAR=bar test.json
   echo bar
 
+  $ js-san VAR=bar test.json
+  echo bar
+
   $ js test.json
   echo baz
 
+  $ js-san test.json
+  echo baz
+
 check eq expression:
 
   $ echo '
@@ -63,12 +94,21 @@ check eq expression:
   $ js VAR=bar test.json
   echo foo
 
+  $ js-san VAR=bar test.json
+  echo foo
+
   $ js VAR=xxx test.json
   echo baz
 
+  $ js-san VAR=xxx test.json
+  echo baz
+
   $ js test.json
   echo baz
 
+  $ js-san test.json
+  echo baz
+
 check regex single expression:
 
   $ echo '
@@ -83,14 +123,29 @@ check regex single expression:
   $ js VAR=hello test.json
   echo bar
 
+  $ js-san VAR=hello test.json
+  echo bar
+
   $ js VAR=.ell. test.json
   echo bar
 
+  $ js-san VAR=.ell. test.json
+  echo bar
+
   $ js test.json
   echo baz
 
+  $ js-san test.json
+  echo baz
+
   $ js VAR= test.json
   echo baz
 
+  $ js-san VAR= test.json
+  echo baz
+
   $ js VAR=hell test.json
   echo baz
+
+  $ js-san VAR=hell test.json
+  echo baz

diff --git a/tests/cram/test_list.t b/tests/cram/test_list.t
index f7f18bd3746df858f8eaa1158026d0920c88b307..81affad29f8388d3e38e45dabef9b52ff37cdab3 100644 (file)
--- a/tests/cram/test_list.t
+++ b/tests/cram/test_list.t
@@ -20,3 +20,23 @@ check that list is producing expected results:
   test_basics: list_for_each_entry_reverse: one eleven ten nine eight seven six five four three two 
   test_basics: delete all entries
   test_basics: list_empty: yes
+
+  $ test-list-san
+  test_basics: list_empty: yes
+  test_basics: list_add_tail: zero one two three four five six seven eight nine ten eleven twelve 
+  test_basics: list_empty: no
+  test_basics: first=zero last=twelve
+  test_basics: 'zero' is first, yes
+  test_basics: 'twelve' is last, yes
+  test_basics: removing 'twelve' and 'zero'
+  test_basics: first=one last=eleven
+  test_basics: 'one' is first, yes
+  test_basics: 'eleven' is last, yes
+  test_basics: moving 'one' to the tail
+  test_basics: first=two last=one
+  test_basics: 'two' is first, yes
+  test_basics: 'one' is last, yes
+  test_basics: list_for_each_entry: two three four five six seven eight nine ten eleven one 
+  test_basics: list_for_each_entry_reverse: one eleven ten nine eight seven six five four three two 
+  test_basics: delete all entries
+  test_basics: list_empty: yes

diff --git a/tests/cram/test_runqueue.t b/tests/cram/test_runqueue.t
index 4d4911047c184f0a44a75634faaf18e4a808a302..227f4142955762d3747306993362752ddf144f2f 100644 (file)
--- a/tests/cram/test_runqueue.t
+++ b/tests/cram/test_runqueue.t
@@ -12,3 +12,15 @@ check that runqueue is producing expected results:
   [1/1] cancel 'sleep 1'
   [0/1] finish 'sleep 1'
   All done!
+
+  $ test-runqueue-san
+  [1/1] start 'sleep 1'
+  [1/1] cancel 'sleep 1'
+  [0/1] finish 'sleep 1'
+  [1/1] start 'sleep 1'
+  [1/1] cancel 'sleep 1'
+  [0/1] finish 'sleep 1'
+  [1/1] start 'sleep 1'
+  [1/1] cancel 'sleep 1'
+  [0/1] finish 'sleep 1'
+  All done!