cosbench

README.md

@@ -1,11 +1,11 @@
 COSBench - Cloud Object Storage Benchmark
 =========================================
 
-COSBench is a benchmarking tool to measure the performance of Cloud Object Storage services. Object storage is an 
+COSBench is a benchmarking tool to measure the performance of Cloud Object Storage services. Object storage is an
 emerging technology that is different from traditional file systems (e.g., NFS) or block device systems (e.g., iSCSI).
 Amazon S3 and Openstack* swift are well-known object storage solutions.
 
-COSBench now supports OpenStack* Swift and Amplidata v2.3, 2.5 and 3.1, as well as custom adaptors.
+COSBench now supports OpenStack* Swift, Amazon* S3, Amplidata v2.3, 2.5 and 3.1, Scality*, Ceph, CDMI, Google* Cloud Storage, Aliyun OSS as well as custom adaptors.
 
 
 Important Notice and Contact Information
@@ -14,11 +14,9 @@ Important Notice and Contact Information
 a) COSBench is not a product, and it does not have a full-time support team. Before you use this tool, please understand 
 the need to invest enough effort to learn how to use it effectively and to address possible bugs.
 
-b) To help COSBench develop further, please become an active member of the community and consider giving back by making 
+b) To help COSBench develop further, please become an active member of the community and consider giving back by making
 contributions.
 
-For other questions, contact jiangang.duan@intel.com or yaguang.wang@intel.com
-
 
 Licensing
 ---------
@@ -52,7 +50,6 @@ If a build from source code is needed, please refer to BUILD.md for details.
 
 Resources
 ---------
-Blog: TBD
 
 Wiki: (https://github.com/intel-cloud/cosbench/wiki)
 
@@ -70,5 +67,13 @@ COSBench-Workload-Generator: (https://github.com/giteshnandre/COSBench-Workload-
 
 COSBench-Plot: (https://github.com/icclab/cosbench-plot)
 
+COSBench-Appliance: (https://susestudio.com/a/8Kp374/cosbench)
+
+COSBench Ansible Playbook:
+
+- (http://www.ksingh.co.in/blog/2016/05/29/deploy-cosbench-using-ansible/)
+- (https://github.com/ksingh7/ansible-role-cosbench)
+- (https://galaxy.ansible.com/ksingh7/cosbench/)
+
 
 = END =

VERSION

@@ -1 +1 @@
-0.4.1
+0.4.2

dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java

@@ -42,14 +42,14 @@ public int read(byte[] b, int off, int len) throws IOException {
 
 	private void recordTime() {
 		if (this.isFirstByte) {
-			this.xferStart = System.currentTimeMillis();
+			this.xferStart = System.nanoTime();
 			this.isFirstByte = false;
 		}
-		this.xferEnd = System.currentTimeMillis();
+		this.xferEnd = System.nanoTime();
 	}
 
 	public long getXferTime() {
-		long xferTime = this.xferEnd - this.xferStart;
+		long xferTime = (this.xferEnd - this.xferStart) / 1000000;
 		return xferTime > 0 ? xferTime : 0L;
 	}
 

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java

@@ -69,7 +69,7 @@ public static Sample doDelete(String conName, String objName,
         if (Thread.interrupted())
             throw new AbortedException();
 
-        long start = System.currentTimeMillis();
+        long start = System.nanoTime();
 
         try {
             session.getApi().deleteObject(conName, objName, config);
@@ -87,11 +87,10 @@ public static Sample doDelete(String conName, String objName,
 					op.getSampleType(), op.getName(), false);
         }
 
-        long end = System.currentTimeMillis();
+        long end = System.nanoTime();
 
-        Date now = new Date(end);
-        return new Sample(now, op.getId(), op.getOpType(), op.getSampleType(),
-				op.getName(), true, end - start, 0L, 0L);
+        return new Sample(new Date(), op.getId(), op.getOpType(), op.getSampleType(),
+				op.getName(), true, (end - start) / 1000000, 0L, 0L);
     }
 
 }

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java

@@ -137,7 +137,7 @@ public Sample doWrite(InputStream in, long length, String conName, String objNam
 
         XferCountingInputStream cin = new XferCountingInputStream(in);
 
-        long start = System.currentTimeMillis();
+        long start = System.nanoTime();
 
         try {
               session.getApi().createObject(conName, objName, cin, length, config);
@@ -153,10 +153,9 @@ public Sample doWrite(InputStream in, long length, String conName, String objNam
             IOUtils.closeQuietly(cin);
         }
 
-        long end = System.currentTimeMillis();
+        long end = System.nanoTime();
 
-        Date now = new Date(end);
-        return new Sample(now,  getId(), getOpType(), getSampleType(),
-				getName(), true, end - start, cin.getXferTime(), cin.getByteCount());
+        return new Sample(new Date(),  getId(), getOpType(), getSampleType(),
+				getName(), true, (end - start) / 1000000, cin.getXferTime(), cin.getByteCount());
     }
 }

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java

@@ -1,5 +1,5 @@
-/** 
- 
+/**
+
 Copyright 2013 Intel Corporation, All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +12,7 @@
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License. 
+limitations under the License.
 */
 
 package com.intel.cosbench.driver.operator;
@@ -37,7 +37,7 @@ public class Lister extends AbstractOperator {
     public static final String OP_TYPE = "list";
 
     private ObjectPicker objPicker = new ObjectPicker();
-    
+
     private byte buffer[] = new byte[1024*1024];
 
     public Lister() {
@@ -75,14 +75,14 @@ private Sample doList(OutputStream out, String conName, String objName,
         InputStream in = null;
         CountingOutputStream cout = new CountingOutputStream(out);
         doLogWarn(session.getLogger(), "listerrr: "+ conName + "/" + objName);//###
-        long start = System.currentTimeMillis();
+        long start = System.nanoTime();
         long xferTime = 0L;
         try {
-        	doLogDebug(session.getLogger(), "worker "+ session.getIndex() + " List target " + conName + "/" + objName);      	
+            doLogDebug(session.getLogger(), "worker "+ session.getIndex() + " List target " + conName + "/" + objName);
 	        in = session.getApi().getList(conName, objName, config);
-	        long xferStart = System.currentTimeMillis();
+	        long xferStart = System.nanoTime();
 	        copyLarge(in, cout);
-	        xferTime = System.currentTimeMillis() - xferStart;
+	        xferTime = (System.nanoTime() - xferStart) / 1000000;
         } catch (StorageInterruptedException sie) {
             doLogErr(session.getLogger(), sie.getMessage(), sie);
             throw new AbortedException();
@@ -95,20 +95,16 @@ private Sample doList(OutputStream out, String conName, String objName,
             IOUtils.closeQuietly(in);
             IOUtils.closeQuietly(cout);
         }
-        long end = System.currentTimeMillis();
+        long end = System.nanoTime();
 
-        Date now = new Date(end);
-		return new Sample(now, getId(), getOpType(), getSampleType(),
-				getName(), true, end - start, xferTime, cout.getByteCount());
+		return new Sample(new Date(), getId(), getOpType(), getSampleType(),
+				getName(), true, (end - start) / 1000000, xferTime, cout.getByteCount());
     }
 
     public OutputStream copyLarge(InputStream input, OutputStream output)
             throws IOException
     {
-            for(int n = 0; -1 != (n = input.read(buffer));)
-            {
-                output.write(buffer, 0, n);
-            }
+            IOUtils.copyLarge(input, output);
 
             return output;
     }

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java

@@ -1,5 +1,5 @@
-/** 
- 
+/**
+
 Copyright 2013 Intel Corporation, All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,8 +12,8 @@
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License. 
-*/ 
+limitations under the License.
+*/
 
 package com.intel.cosbench.driver.operator;
 
@@ -32,18 +32,18 @@
 
 /**
  * This class represents primitive READ operation.
- * 
+ *
  * @author ywang19, qzheng7
- * 
+ *
  */
 class Reader extends AbstractOperator {
 
     public static final String OP_TYPE = "read";
-    
+
     private boolean hashCheck = false;
 
     private ObjectPicker objPicker = new ObjectPicker();
-    
+
     private byte buffer[] = new byte[1024*1024];
 
     public Reader() {
@@ -82,51 +82,48 @@ private Sample doRead(OutputStream out, String conName, String objName,
         InputStream in = null;
         CountingOutputStream cout = new CountingOutputStream(out);
 
-        long start = System.currentTimeMillis();
+        long start = System.nanoTime();
         long xferTime = 0L;
-        long xferTimeCheck = 0L;
         try {
             in = session.getApi().getObject(conName, objName, config);
-            long xferStart = System.currentTimeMillis();
-            if (!hashCheck){
+            long xferStart = System.nanoTime();
+            if (!hashCheck) {
                 copyLarge(in, cout);
-            	long xferEnd = System.currentTimeMillis();
-            	xferTime = xferEnd - xferStart;
-            } else if (!validateChecksum(conName, objName, session, in, cout, xferTimeCheck))
+            } else if (!validateChecksum(conName, objName, session, in, cout)) {
 				return new Sample(new Date(), getId(), getOpType(),
 						getSampleType(), getName(), false);
+            }
+            long xferEnd = System.nanoTime();
+            xferTime = (xferEnd - xferStart) / 1000000;
         } catch (StorageInterruptedException sie) {
             doLogErr(session.getLogger(), sie.getMessage(), sie);
             throw new AbortedException();
         } catch (Exception e) {
         	isUnauthorizedException(e, session);
         	errorStatisticsHandle(e, session, conName + "/" + objName);
-        	
+
             return new Sample(new Date(), getId(), getOpType(), getSampleType(), getName(), false);
         } finally {
             IOUtils.closeQuietly(in);
             IOUtils.closeQuietly(cout);
         }
-        long end = System.currentTimeMillis();
+        long end = System.nanoTime();
 
-        Date now = new Date(end);
-		return new Sample(now, getId(), getOpType(), getSampleType(),
-				getName(), true, end - start, hashCheck ? xferTimeCheck : xferTime, cout.getByteCount());
+		return new Sample(new Date(), getId(), getOpType(), getSampleType(),
+				getName(), true, (end - start)/1000000,
+				xferTime, cout.getByteCount());
     }
 
     public OutputStream copyLarge(InputStream input, OutputStream output)
             throws IOException
     {
-            for(int n = 0; -1 != (n = input.read(buffer));)
-            {
-                output.write(buffer, 0, n);
-            }
+            IOUtils.copyLarge(input, output);
 
             return output;
     }
-    
+
     private static boolean validateChecksum(String conName, String objName,
-            Session session, InputStream in, OutputStream out, long xferTimeCheck)
+            Session session, InputStream in, OutputStream out)
             throws IOException {
         HashUtil util;
         try {
@@ -138,8 +135,7 @@ private static boolean validateChecksum(String conName, String objName,
 
             String storedHash = new String();
             String calculatedHash = new String();
-
-            long xferStart = System.currentTimeMillis();
+
             int br1 = in.read(buf1);
 
             if (br1 <= hashLen) {
@@ -176,8 +172,7 @@ private static boolean validateChecksum(String conName, String objName,
                     br1 = br2;
                 }
             }
-            xferTimeCheck = System.currentTimeMillis() - xferStart;
-
+
             if (!calculatedHash.equals(storedHash)) {
                 if (storedHash.startsWith(HashUtil.GUARD)) {
                     String err =

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java

@@ -91,7 +91,7 @@ public static  Sample doWrite(InputStream in, long length, String conName,
             throw new AbortedException();
 
         XferCountingInputStream cin = new XferCountingInputStream(in);	
-        long start = System.currentTimeMillis();
+        long start = System.nanoTime();
 
         try {
             session.getApi()
@@ -110,10 +110,10 @@ public static  Sample doWrite(InputStream in, long length, String conName,
             IOUtils.closeQuietly(cin);
         }
 
-        long end = System.currentTimeMillis();
-        Date now = new Date(end);
-		return new Sample(now, op.getId(), op.getOpType(), op.getSampleType(),
-				op.getName(), true, end - start, cin.getXferTime(), cin.getByteCount());
+        long end = System.nanoTime();
+		return new Sample(new Date(), op.getId(), op.getOpType(), op.getSampleType(),
+				op.getName(), true, (end - start) / 1000000,
+				cin.getXferTime(), cin.getByteCount());
     }
     /*
      * public static Sample doWrite(byte[] data, String conName, String objName,

dev/cosbench-ecs/META-INF/.ecsmeta/.dirmeta

@@ -0,0 +1 @@
+{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:00 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:46Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null}

dev/cosbench-ecs/META-INF/.ecsmeta/MANIFEST.MF

@@ -0,0 +1 @@
+{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":777,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:49 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null}

dev/cosbench-ecs/META-INF/MANIFEST.MF

@@ -0,0 +1,28 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: cosbench-ecs
+Bundle-SymbolicName: cosbench-ecs
+Bundle-Version: 0.4.2.0
+Bundle-Vendor: emc
+Import-Package: com.intel.cosbench.api.auth,
+ com.intel.cosbench.api.context,
+ com.intel.cosbench.api.storage,
+ com.intel.cosbench.client.http,
+ com.intel.cosbench.config,
+ com.intel.cosbench.log
+Bundle-ClassPath: .,
+ commons-codec-1.4.jar,
+ commons-logging-1.1.1.jar,
+ httpclient-4.5.jar,
+ httpcore-4.4.1.jar,
+ javax.mail-api-1.4.6.jar,
+ jdom2-2.0.5.jar,
+ jersey-apache-client4-1.18.3.jar,
+ jersey-client-1.18.3.jar,
+ jersey-core-1.18.3.jar,
+ log4j-1.2.17.jar,
+ lzma-sdk-4j-9.22.0.jar,
+ object-client-2.1.0.jar,
+ slf4j-api-1.7.5.jar,
+ smart-client-2.0.6.jar,
+ vipr-object-transformations-2.0.3.jar

dev/cosbench-ecs/META-INF/spring/.ecsmeta/.dirmeta

@@ -0,0 +1 @@
+{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:49 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:46Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null}

dev/cosbench-ecs/META-INF/spring/.ecsmeta/plugin-context.xml

@@ -0,0 +1 @@
+{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":774,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:06:25 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-16T16:58:52Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null}