Initial pipeline support

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "scons_local"]
 	path = scons_local
-	url = https://review.hpdd.intel.com/coral/scons_local
+	url = https://github.com/daos-stack/scons_local.git

multi-node-test.sh

@@ -0,0 +1,238 @@
+#!/bin/bash
+
+set -ex -o pipefail
+
+# shellcheck disable=SC1091
+if [ -f .localenv ]; then
+    # read (i.e. environment, etc.) overrides
+    . .localenv
+fi
+
+HOSTPREFIX=${HOSTPREFIX-${HOSTNAME%%.*}}
+NFS_SERVER=${NFS_SERVER:-$HOSTPREFIX}
+
+trap 'echo "encountered an unchecked return code, exiting with error"' ERR
+
+# shellcheck disable=SC1091
+. .build_vars-Linux.sh
+
+if [ "$1" = "2" ]; then
+    test_runner_vm=$((${EXECUTOR_NUMBER:-0}*3+7))
+    vm1="$((test_runner_vm+1))"
+    vm2="$((test_runner_vm+2))"
+    vmrange="$vm1-$vm2"
+    test_runner_vm="vm$test_runner_vm"
+    vm1="vm$vm1"
+    vm2="vm$vm2"
+elif [ "$1" = "5" ]; then
+    test_runner_vm="vm1"
+    vmrange="2-6"
+fi
+
+log_base_path="testLogs-${1}_node"
+
+rm -f results_1.yml IOF_[25]-node_junit.xml
+
+curl -O 'https://build.hpdd.intel.com/job/mdtest-iof_devel/lastSuccessfulBuild/arch=x86_64,distro=el7/artifact/artifacts/mpich_files.tar.gz'
+curl -O 'https://build.hpdd.intel.com/job/mdtest-iof_devel/lastSuccessfulBuild/arch=x86_64,distro=el7/artifact/artifacts/mdtest_files.tar.gz'
+
+# shellcheck disable=SC2154
+trap 'set +e
+i=5
+# due to flakiness on wolf-53, try this several times
+while [ $i -gt 0 ]; do
+    pdsh -R ssh -S -w "${HOSTPREFIX}$test_runner_vm,${HOSTPREFIX}vm[$vmrange]" "set -x
+    x=0
+    while [ \$x -lt 30 ] &&
+          grep $DAOS_BASE /proc/mounts &&
+          ! sudo umount $DAOS_BASE; do
+        ps axf
+        sleep 1
+        let x+=1
+    done
+    sudo sed -i -e \"/added by multi-node-test-$1.sh/d\" /etc/fstab
+    sudo rmdir $DAOS_BASE || find $DAOS_BASE || true" 2>&1 | dshbak -c
+    if [ ${PIPESTATUS[0]} = 0 ]; then
+        i=0
+    fi
+    let i-=1
+done' EXIT
+
+DAOS_BASE=${SL_OMPI_PREFIX%/install/*}
+if ! pdsh -R ssh -S -w "${HOSTPREFIX}$test_runner_vm,${HOSTPREFIX}vm[$vmrange]" "set -ex
+ulimit -c unlimited
+sudo mkdir -p $DAOS_BASE
+sudo ed <<EOF /etc/fstab
+\\\$a
+$NFS_SERVER:$PWD $DAOS_BASE nfs defaults 0 0 # added by multi-node-test-$1.sh
+.
+wq
+EOF
+if ! sudo mount $DAOS_BASE; then
+    if [ \"\${HOSTNAME%%%%.*}\" = \"${HOSTPREFIX}$test_runner_vm\" ]; then
+        # could be already mounted from another test running in parallel
+        # let's see what that rc is
+        echo \"mount rc: \${PIPESTATUS[0]}\"
+    else
+        exit \${PIPESTATUS[0]}
+    fi
+fi
+
+# TODO: package this in to an RPM
+pip3 install --user tabulate
+
+# install mdtest
+sudo rm -rf /testbin/
+sudo mkdir -p /testbin/mdtest
+cd /testbin/mdtest
+sudo tar xzvf $DAOS_BASE/mpich_files.tar.gz
+sudo tar xzvf $DAOS_BASE/mdtest_files.tar.gz
+df -h" 2>&1 | dshbak -c; then
+    echo "Cluster setup (i.e. provisioning) failed"
+    exit 1
+fi
+
+#echo "hit enter to continue"
+#read -r
+#exit 0
+
+rm -f  install/Linux/bin/fusermount3
+ln -s "$(command -v fusermount)" install/Linux/bin/fusermount3
+
+# shellcheck disable=SC2029
+if ! ssh "${HOSTPREFIX}$test_runner_vm" "set -ex
+ulimit -c unlimited
+cd $DAOS_BASE
+
+# now run it!
+pushd install/Linux/TESTING/
+if [ \"$1\" = \"2\" ]; then
+    cat <<EOF > scripts/iof_fio_main.cfg
+{
+    \"should_be_host_list\": [\"${HOSTPREFIX}${vm1}\", \"${HOSTPREFIX}${vm2}\"],
+    \"host_list\": [\"${HOSTPREFIX}${test_runner_vm}\", \"${HOSTPREFIX}${vm1}\"],
+    \"test_mode\": \"littleChief\",
+    \"log_base_path\": \"$log_base_path\"
+}
+EOF
+    cp scripts/iof_{fio,ior}_main.cfg
+    cp scripts/iof_{fio,iozone}_main.cfg
+    cp scripts/iof_{fio,mdtest}_main.cfg
+
+    rm -rf $log_base_path/
+    python3 test_runner config=scripts/iof_fio_main.cfg \\
+            scripts/iof_multi_two_node.yml || {
+        rc=\${PIPESTATUS[0]}
+        echo \"Test exited with \$rc\"
+    }
+    mv $log_base_path/testRun{,-fio}
+    find $log_base_path/testRun-fio -name subtest_results.yml \\
+         -exec grep -Hi fail {} \\;
+    python3 test_runner config=scripts/iof_ior_main.cfg \\
+            scripts/iof_multi_two_node.yml || {
+        rc=\${PIPESTATUS[0]}
+        echo \"Test exited with \$rc\"
+    }
+    mv $log_base_path/testRun{,-ior}
+    find $log_base_path/testRun-ior -name subtest_results.yml \\
+         -exec grep -Hi fail {} \\;
+    python3 test_runner config=scripts/iof_iozone_main.cfg \\
+            scripts/iof_multi_two_node.yml || {
+        rc=\${PIPESTATUS[0]}
+        echo \"Test exited with \$rc\"
+    }
+    mv $log_base_path/testRun{,-iozone}
+    find $log_base_path/testRun-iozone -name subtest_results.yml \\
+         -exec grep -Hi fail {} \\;
+    python3 test_runner config=scripts/iof_mdtest_main.cfg \\
+            scripts/iof_multi_two_node.yml || {
+        rc=\${PIPESTATUS[0]}
+        echo \"Test exited with \$rc\"
+    }
+    mv $log_base_path/testRun{,-mdtest}
+    find $log_base_path/testRun-mdtest -name subtest_results.yml \\
+         -exec grep -Hi fail {} \\;
+elif [ \"$1\" = \"5\" ]; then
+    cat <<EOF > scripts/iof_multi_five_node.cfg
+{
+    \"should_be_host_list\": [
+        \"${HOSTPREFIX}vm2\",
+        \"${HOSTPREFIX}vm3\",
+        \"${HOSTPREFIX}vm4\",
+        \"${HOSTPREFIX}vm5\",
+        \"${HOSTPREFIX}vm6\"
+    ],
+    \"host_list\": [\"${HOSTPREFIX}${test_runner_vm}\",
+        \"${HOSTPREFIX}vm2\",
+        \"${HOSTPREFIX}vm3\",
+        \"${HOSTPREFIX}vm4\",
+        \"${HOSTPREFIX}vm5\"
+    ],
+    \"test_mode\": \"littleChief\",
+    \"log_base_path\": \"$log_base_path\"
+}
+EOF
+    rm -rf $log_base_path/
+    python3 test_runner config=scripts/iof_multi_five_node.cfg \\
+            scripts/iof_multi_five_node.yml || {
+        rc=\${PIPESTATUS[0]}
+        echo \"Test exited with \$rc\"
+    }
+    find $log_base_path/testRun -name subtest_results.yml \\
+         -exec grep -Hi fail {} \\;
+fi
+popd
+ls -ld install install/Linux install/Linux/TESTING >&2
+ls -l install/Linux/TESTING >&2
+df -h install/Linux/TESTING >&2
+ls -ltar
+exit \$rc"; then
+    rc=${PIPESTATUS[0]}
+else
+    rc=0
+fi
+
+hostname
+pwd
+scp -r "${HOSTPREFIX}$test_runner_vm:$DAOS_BASE/install/Linux/TESTING/$log_base_path" install/Linux/TESTING/
+#ls -l . "$log_base_path"
+#    i=10
+#    while [ "$i" -gt 0 ]; do
+#        ls -ld install install/Linux install/Linux/TESTING >&2
+#        ls -l install/Linux/TESTING >&2
+#        df -h install/Linux/TESTING >&2
+#        ssh "${HOSTPREFIX}$test_runner_vm" "set -x
+#             set +e
+#             if [ \"$i\" -lt 3 ]; then
+#                 sync; sync
+#             fi
+#             ls -l $DAOS_BASE/install/Linux/TESTING
+#             exit 0" >&2
+#        sleep 5
+#        let i-=1
+#    done
+{
+    cat <<EOF
+TestGroup:
+    submission: $(TZ=UTC date)
+    test_group: IOF_${1}-node
+    testhost: $HOSTNAME
+    user_name: jenkins
+Tests:
+EOF
+    find install/Linux/TESTING/"$log_base_path" \
+         -name subtest_results.yml -print0 | xargs -0 cat
+} > results_1.yml
+cat results_1.yml
+
+PYTHONPATH=scony_python-junit/ jenkins/autotest_utils/results_to_junit.py
+ls -ltar
+
+## collect the logs
+#if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1,$vmrange] \
+#    /tmp/Functional_"$TEST_TAG"/\*daos.log "$PWD"/; then
+#    echo "Copying daos.logs from remote nodes failed"
+#    # pass
+#fi
+ls -l
+exit "$rc"

src/cnss/cnss.c

@@ -980,7 +980,9 @@ int main(int argc, char **argv)
 				       "Plugin %s(%p) calling destroy_plugin_data at %p",
 				       entry->fns->name,
 				       entry->fns->handle,
-				       FN_TO_PVOID(entry->fns->destroy_plugin_data));
+				       FN_TO_PVOID(
+					   entry->fns->destroy_plugin_data
+				       ));
 			entry->fns->destroy_plugin_data(entry->fns->handle);
 		}
 

src/ioc/ioc.h

@@ -453,15 +453,16 @@ struct fuse_lowlevel_ops *iof_get_fuse_ops(uint64_t);
 		IOF_TRACE_DOWN(ioc_req);				\
 	} while (0)
 
-#define IOC_REPLY_IOCTL(handle, req, gah_info)				\
-	do {								\
-		int __rc;						\
-		IOF_TRACE_DEBUG(handle, "Returning ioctl");		\
-		__rc = fuse_reply_ioctl(req, 0, &(gah_info), sizeof(gah_info)); \
-		if (__rc != 0)						\
-			IOF_TRACE_ERROR(handle,				\
+#define IOC_REPLY_IOCTL(handle, req, gah_info)				   \
+	do {								   \
+		int __rc;						   \
+		IOF_TRACE_DEBUG(handle, "Returning ioctl");		   \
+		__rc = fuse_reply_ioctl(req, 0, &(gah_info),		   \
+					sizeof(gah_info));		   \
+		if (__rc != 0)						   \
+			IOF_TRACE_ERROR(handle,				   \
 					"fuse_reply_ioctl returned %d:%s", \
-					__rc, strerror(-__rc));		\
+					__rc, strerror(-__rc));		   \
 	} while (0)
 
 struct ioc_request;

src/ioc/ioc_main.c

@@ -318,10 +318,12 @@ static void gah_decref(struct iof_projection_info *fs_handle)
 		if (ref > 1) {
 			int rc;
 
-			rc = fuse_lowlevel_notify_inval_entry(fs_handle->session,
-							      ie->parent,
-							      ie->name,
-							      strlen(ie->name));
+			rc = fuse_lowlevel_notify_inval_entry(
+			    fs_handle->session,
+			    ie->parent,
+			    ie->name,
+			    strlen(ie->name)
+			);
 
 			IOF_TRACE_INFO(ie, "inval returned %d", rc);
 			if (rc == -ENOENT) {
@@ -1328,22 +1330,25 @@ fh_reset(void *arg)
 	}
 
 	rc = crt_req_create(fh->open_req.fsh->proj.crt_ctx, NULL,
-			    FS_TO_OP(fh->open_req.fsh, open), &fh->open_req.rpc);
+			    FS_TO_OP(fh->open_req.fsh, open),
+				     &fh->open_req.rpc);
 	if (rc || !fh->open_req.rpc) {
 		D_FREE(fh->ie);
 		return false;
 	}
 
 	rc = crt_req_create(fh->open_req.fsh->proj.crt_ctx, NULL,
-			    FS_TO_OP(fh->open_req.fsh, create), &fh->creat_req.rpc);
+			    FS_TO_OP(fh->open_req.fsh, create),
+				     &fh->creat_req.rpc);
 	if (rc || !fh->creat_req.rpc) {
 		D_FREE(fh->ie);
 		crt_req_decref(fh->open_req.rpc);
 		return false;
 	}
 
 	rc = crt_req_create(fh->open_req.fsh->proj.crt_ctx, NULL,
-			    FS_TO_OP(fh->open_req.fsh, close), &fh->release_req.rpc);
+			    FS_TO_OP(fh->open_req.fsh, close),
+				     &fh->release_req.rpc);
 	if (rc || !fh->release_req.rpc) {
 		D_FREE(fh->ie);
 		crt_req_decref(fh->open_req.rpc);
@@ -2173,16 +2178,20 @@ initialize_projection(struct iof_state *iof_state,
 
 	IOF_TRACE_DEBUG(fs_handle, "Setting timeout to %d", fs_info->timeout);
 
-	ret = crt_context_set_timeout(fs_handle->proj.crt_ctx, fs_info->timeout);
+	ret = crt_context_set_timeout(fs_handle->proj.crt_ctx,
+				      fs_info->timeout);
 	if (ret != -DER_SUCCESS) {
 		IOF_TRACE_ERROR(iof_state, "Context timeout not set");
 		D_GOTO(err, 0);
 	}
 
 	for (i = 0; i < fs_handle->ctx_num; i++) {
-		fs_handle->ctx_array[i].crt_ctx       = fs_handle->proj.crt_ctx;
-		fs_handle->ctx_array[i].poll_interval = iof_state->iof_ctx.poll_interval;
-		fs_handle->ctx_array[i].callback_fn   = iof_state->iof_ctx.callback_fn;
+		fs_handle->ctx_array[i].crt_ctx =
+		    fs_handle->proj.crt_ctx;
+		fs_handle->ctx_array[i].poll_interval =
+		    iof_state->iof_ctx.poll_interval;
+		fs_handle->ctx_array[i].callback_fn =
+		    iof_state->iof_ctx.callback_fn;
 
 		/* TODO: Much better error checking is required here, not least
 		 * terminating the thread if there are any failures in the rest
@@ -2713,9 +2722,10 @@ static void iof_finish(void *arg)
 		/*send a detach RPC to IONSS*/
 		rc = crt_req_create(iof_state->iof_ctx.crt_ctx,
 				&group->grp.psr_ep,
-				CRT_PROTO_OPC(iof_state->handshake_proto->cpf_base,
-					iof_state->handshake_proto->cpf_ver,
-					1),
+				CRT_PROTO_OPC(
+				    iof_state->handshake_proto->cpf_base,
+				    iof_state->handshake_proto->cpf_ver,
+				    1),
 				&rpc);
 		if (rc != -DER_SUCCESS || !rpc) {
 			IOF_TRACE_ERROR(iof_state,
@@ -2767,7 +2777,8 @@ static void iof_finish(void *arg)
 
 		rc = crt_context_destroy(iof_state->iof_ctx.crt_ctx, false);
 		if (rc != -DER_SUCCESS)
-			IOF_TRACE_ERROR(iof_state, "Could not destroy context %d",
+			IOF_TRACE_ERROR(iof_state,
+					"Could not destroy context %d",
 					rc);
 		IOF_TRACE_DOWN(&iof_state->iof_ctx);
 	}

src/ioc/ops/create.c

@@ -44,7 +44,8 @@
 static bool
 ioc_create_ll_cb(struct ioc_request *request)
 {
-	struct iof_file_handle		*handle = container_of(request, struct iof_file_handle, creat_req);
+	struct iof_file_handle		*handle = container_of(
+	    request, struct iof_file_handle, creat_req);
 	struct iof_projection_info	*fs_handle = request->fsh;
 	struct iof_create_out		*out = crt_reply_get(request->rpc);
 	struct fuse_file_info		fi = {0};
@@ -183,7 +184,8 @@ void ioc_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name,
 
 	IOF_TRACE_UP(handle, fs_handle, fs_handle->fh_pool->reg.name);
 	IOF_TRACE_UP(&handle->creat_req, handle, "creat_req");
-	IOF_TRACE_LINK(handle->creat_req.rpc, &handle->creat_req, "creat_file_rpc");
+	IOF_TRACE_LINK(handle->creat_req.rpc, &handle->creat_req,
+		       "creat_file_rpc");
 
 	handle->common.projection = &fs_handle->proj;
 	handle->creat_req.req = req;