Enable Jepsen tests for RR

Run nemesis tests serially

Signed-off-by: Mark Hannum <[email protected]>

Author: markhannum

cdb2jdbc/Makefile

@@ -1,6 +1,7 @@
 # shortcut for building jdbc driver via make
 BASEDIR:=$(realpath ..)
 VERSION?=$(shell grep ^comdb2 ../pkg/changelog  | sed 's/^comdb2 .//; s/-.*//g')
+MVNFLAGS?=$(shell echo "${COMDB2_MVN_FLAGS}")
 
 .PHONY: jdbc
 MAVEN_VER := $(shell mvn -version 2>/dev/null)
@@ -9,21 +10,21 @@ envSkipTests?='true'
 
 jdbc:
 ifdef MAVEN_VER
-	+mvn package
+	+mvn $(MVNFLAGS) package
 else
 	$(error "Could not find maven")
 endif
 
 clean:
 ifdef MAVEN_VER
-	+mvn clean
+	+mvn $(MVNFLAGS) clean
 else
 	$(error "Could not find maven")
 endif
 
 install: jdbc
 ifdef MAVEN_VER
-	+mvn install
+	+mvn $(MVNFLAGS) install
 else
 	$(error "Could not find maven")
 endif

tests/Makefile

@@ -13,7 +13,10 @@ endif
 include $(TESTSROOTDIR)/Makefile.common
 
 # List of disabled tests
-DISABLED_TESTS:=$(shell awk '/^DISABLED/{f=1;next};/<END>/{f=0}f' TODO | cut -d' ' -f1)
+DISABLED_TESTS:=$(shell awk '/^DISABLED TESTS/{f=1;next};/^END DISABLED/{f=0}f' TODO | cut -d' ' -f1)
+
+# List of serial tests
+SERIAL_TESTS:=$(shell awk '/^SERIAL TESTS/{f=1;next};/^END SERIAL/{f=0}f' TODO | cut -d' ' -f1)
 
 # When SKIPSSL is on, skip running tests which rely on SSL
 ifeq ($(SKIPSSL),1)
@@ -41,41 +44,56 @@ ifeq ($(subst all,,$(MAKECMDGOALS)),) #if cmdgoals empty or 'all'
   DOALL:=1
 endif
 
-# List of all tests that need to run.
+.PHONY: all all_tests all_nonserial_tests all_serial_tests
+
+all_tests: all_nonserial_tests all_serial_tests
+all: init basicops all_tests
+
+CANON_SERIAL := $(patsubst %.test,%, $(SERIAL_TESTS))
+
+# Read alltests.txt if present
 ifeq ($(wildcard alltests.txt),alltests.txt)
-  # If file alltests.txt exists, only run the tests in that file (one line per test)
-  export ALL_TESTS:=$(patsubst %.test, %, $(shell sort -u alltests.txt) )
+  SELECTED := $(patsubst %.test,%, \
+    $(shell awk 'NF && $$1 !~ /^#/' alltests.txt | sed 's/[[:space:]]\+.*//' ))
+  # Ignore DISABLED if using alltests.txt
+  ALL_SERIAL_TESTS := $(filter $(CANON_SERIAL),$(SELECTED))     # preserves order of SELECTED
+  ALL_TESTS        := $(filter-out $(ALL_SERIAL_TESTS),$(SELECTED))
 else
-  ifneq ($(DISABLED_TESTS),)
-    DISABLED_TESTS:=" $(shell echo $(DISABLED_TESTS) | tr ' ' '\n' | sort -u) "
-  endif
-
+  DISABLED_TESTS   := $(strip $(sort $(DISABLED_TESTS)))
+  CANON_DISABLED   := $(patsubst %.test,%, $(DISABLED_TESTS))
   ifeq ($(INCLUDE_LONG_RUNNING),1)
     FLG=
   else
     # by default exclude all tests which run longer than 1 hour
     FLG=--skip-long-running
   endif
+  ALL_CANDIDATES   := $(patsubst %.test,%, $(shell $(TESTSROOTDIR)/tools/get_tests_inorder.sh $(FLG)))
 
-  # This is the list of all non-disabled tests.
-  export ALL_TESTS:=$(patsubst %.test, %, $(filter-out $(DISABLED_TESTS), \
-      $(shell $(TESTSROOTDIR)/tools/get_tests_inorder.sh $(FLG))))
+  ALL_SERIAL_TESTS := $(filter-out $(CANON_DISABLED), $(CANON_SERIAL))
+  ALL_TESTS        := $(filter-out $(ALL_SERIAL_TESTS) $(CANON_DISABLED), $(ALL_CANDIDATES))
 endif
 
+# ----- execution -----
+all_nonserial_tests: $(ALL_TESTS)
+	@echo "Non-serial tests complete"
+
+all_serial_tests: all_nonserial_tests
+	@echo "Running serial tests: $(ALL_SERIAL_TESTS)"
+	@set -e; \
+	for t in $(ALL_SERIAL_TESTS); do \
+	  $(MAKE) --no-print-directory $$t; \
+	done
+
 # get number of TOTAL tests running: used for printing progress report of tests
 TOTAL=$(words $(MAKECMDGOALS) )
 ifeq ($(DOALL),1)
   TOTAL=$(words $(ALL_TESTS) )
 endif
 
-all: init basicops all_tests
 
 init:
 	@mkdir -p ${TESTDIR}
 
-# Run all non-disabled tests.
-all_tests: $(ALL_TESTS)
-
 basicops_nokey: init
 	$(shell which git > /dev/null 2>&1 && git rev-parse --abbrev-ref HEAD | xargs echo "Branch" >> ${TESTDIR}/test.log)
 	$(shell which git > /dev/null 2>&1 && git rev-parse --short HEAD | xargs echo "SHA " >> ${TESTDIR}/test.log)

tests/TODO

@@ -53,24 +53,23 @@ sc_swapfields
 sc_versmistmatch
 sc_resume
 
+# The tests listed in this section can't run in parallel- they
+# modify the environment in a way which will break other tests
+SERIAL TESTS:
+register_linearizable.test  -- nemesis.c partitions the network
+cinsert_linearizable.test
+jepsen_a6_nemesis.test      -- nemesis partitions network
+jepsen_bank_nemesis.test
+jepsen_register_nemesis.test
+jepsen_sets_nemesis.test
+END SERIAL
 
 # The tests listed below will be skipped when ran as part of
 # 'make all'. These tests however can be run individually.
 
 DISABLED TESTS:
 async_sc_bench.test       -- benchmark for paper
 halt_processor_tds.test   -- requires a cluster
-jepsen_a6.test            -- jepsen tests require java & root access
-jepsen_a6_nemesis.test
-jepsen_atomic_writes.test
-jepsen_bank_nemesis.test
-jepsen_bank.test
-jepsen_dirty_reads.test
-jepsen_g2.test
-jepsen_register_nemesis.test
-jepsen_register.test
-jepsen_sets_nemesis.test
-jepsen_sets.test
 killcluster.test
 netloss.test              -- does not pass most of the time; requires kernel 4.x & docker 17.x
 overflowblobs.test        -- timesout
@@ -82,5 +81,6 @@ sc_async_constraints.test
 sigstopcluster.test
 weighted_standing_queue.test -- failing in rhel8 + podman
 sc_resume_partition.test
+END DISABLED
 
 # vim: set sw=4 ts=4 et:

tests/docker/Dockerfile.db

@@ -4,6 +4,7 @@ FROM comdb2test:${REVISION}
 RUN mkdir /var/run/sshd && \
     echo 'root:bigsecret' | chpasswd && \
     sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+    sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication yes/' &&
     echo "StrictHostKeyChecking no" > /etc/ssh/ssh_config && \
     sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \
     echo "export VISIBLE=now" >> /etc/profile

tests/docker/Dockerfile.install

@@ -12,6 +12,7 @@ RUN apt-get update && \
     gdb \
     git \
     iproute2 \
+    iptables \
     iputils-ping \
     jq \
     libevent-dev \
@@ -29,10 +30,12 @@ RUN apt-get update && \
     libz-dev \
     lsof \
     make \
+    maven \
     ncurses-dev \
     netcat-openbsd \
     net-tools \
     openssh-server \
+    openjdk-11-jdk \
     protobuf-c-compiler \
     psmisc \
     pstack \
@@ -50,14 +53,18 @@ RUN apt-get update && \
 
 COPY . /comdb2
 
-RUN cd /comdb2 &&  \
+RUN export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) && \
+    export COMDB2_MVN_FLAGS="-Dhttp.proxyHost=devproxy.bloomberg.com -Dhttp.proxyPort=82 -Dhttps.proxyHost=devproxy.bloomberg.com -Dhttps.proxyPort=82" && \
+    cd /comdb2/cdb2jdbc && \
+    make install && \
+    cd /comdb2 &&  \
     rm -rf build && \
     mkdir build && \
     cd build && \
     cmake -DWITH_TCL=1 -DCOMDB2_TEST=1 .. && \
     make -j4 && \
     make -j4 test-tools && \
-    make install &&  \
+    make install && \
     cp /comdb2/tests/docker/client / && \
     cp /comdb2/tests/docker/server / && \
     apt-get clean

tests/docker/runit

@@ -86,6 +86,8 @@ services:
             dockerfile: tests/docker/Dockerfile.db
             args:
                 - REVISION=$prefix
+        ports:
+            - "3333:22"
         volumes:
             - ./common:/common
             - ./common/c1:/dedicated
@@ -100,6 +102,10 @@ cat >> $compose <<EOF
         command: /client $commands
         cap_add:
             - SYS_PTRACE
+            - SYS_RESOURCE
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_ADMIN
 EOF
 
 for node in $containers; do
@@ -136,6 +142,9 @@ cat >> $compose <<EOF
         command: /server
         cap_add:
             - SYS_PTRACE
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_ADMIN
 EOF
 done
 

tests/jepsen_a6.test/runit

@@ -3,8 +3,8 @@
 # Runs the register linearizability test
 #debug=1
 #debug_trace="-D"
-export CDB2JDBC_STATEMENT_QUERYEFFECTS=1
-export CDB2JDBC_VERIFY_RETRY=0
+
+. ${TESTSROOTDIR}/tools/jepsen_setup.sh
 
 [[ "$debug" == 1 ]] && set -x
 needcluster=0
@@ -98,7 +98,7 @@ function runtestwithoptions
     export CDB2_CONFIG_FILE=${CDB2_CONFIG}
     (
         cd ${TESTSROOTDIR}/tools/linearizable/jepsen
-        lein run test $testargs > $outfile 2>&1 < /dev/null
+        $LEIN run test $testargs > $outfile 2>&1 < /dev/null
         r=$? ; [[ $r != 0 ]] && echo "XXX JEPSEN ERROR Lein test returns $r" >> $outfile
     )
     cat $outfile

tests/jepsen_a6_nemesis.test/runit

@@ -3,8 +3,8 @@
 # Runs the register linearizability test
 #debug=1
 #debug_trace="-D"
-export CDB2JDBC_STATEMENT_QUERYEFFECTS=1
-export CDB2JDBC_VERIFY_RETRY=0
+
+. ${TESTSROOTDIR}/tools/jepsen_setup.sh
 
 [[ "$debug" == 1 ]] && set -x
 needcluster=0
@@ -98,7 +98,7 @@ function runtestwithoptions
     export CDB2_CONFIG_FILE=${CDB2_CONFIG}
     (
         cd ${TESTSROOTDIR}/tools/linearizable/jepsen
-        lein run test $testargs > $outfile 2>&1 < /dev/null
+        $LEIN run test $testargs > $outfile 2>&1 < /dev/null
         r=$? ; [[ $r != 0 ]] && echo "XXX JEPSEN ERROR Lein test returns $r" >> $outfile
     )
     cat $outfile

tests/jepsen_atomic_writes.test/runit

@@ -3,8 +3,8 @@
 # Runs the register linearizability test
 #debug=1
 #debug_trace="-D"
-export CDB2JDBC_STATEMENT_QUERYEFFECTS=1
-export CDB2JDBC_VERIFY_RETRY=0
+
+. ${TESTSROOTDIR}/tools/jepsen_setup.sh
 
 [[ "$debug" == 1 ]] && set -x
 needcluster=0
@@ -98,7 +98,7 @@ function runtestwithoptions
     export CDB2_CONFIG_FILE=${CDB2_CONFIG}
     (
         cd ${TESTSROOTDIR}/tools/linearizable/jepsen
-        lein run test $testargs > $outfile 2>&1 < /dev/null
+        $LEIN run test $testargs > $outfile 2>&1 < /dev/null
         r=$? ; [[ $r != 0 ]] && echo "XXX JEPSEN ERROR Lein test returns $r" >> $outfile
     )
     cat $outfile

tests/jepsen_bank.test/runit

@@ -3,8 +3,8 @@
 # Runs the register linearizability test
 #debug=1
 #debug_trace="-D"
-export CDB2JDBC_STATEMENT_QUERYEFFECTS=1
-export CDB2JDBC_VERIFY_RETRY=0
+
+. ${TESTSROOTDIR}/tools/jepsen_setup.sh
 
 [[ "$debug" == 1 ]] && set -x
 needcluster=0
@@ -97,7 +97,7 @@ function runtestwithoptions
     export CDB2_CONFIG_FILE=${CDB2_CONFIG}
     (
         cd ${TESTSROOTDIR}/tools/linearizable/jepsen
-        lein run test $testargs > $outfile 2>&1 < /dev/null
+        $LEIN run test $testargs > $outfile 2>&1 < /dev/null
         r=$? ; [[ $r != 0 ]] && echo "XXX JEPSEN ERROR Lein test returns $r" >> $outfile
     )
     cat $outfile