apply_delta now has a C implementation, which is only 25 percent faster for small files ( the overhead of all the rest is very high ), but 4 times faster for large files, where the enormous call overhead coming in with python really starts to show

Author: Byron

_delta_apply.c

@@ -886,3 +886,73 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams)
 	return (PyObject*)dcl;
 }
 
+
+// Write using a write function, taking remaining bytes from a base buffer
+// replaces the corresponding method in python
+static
+PyObject* apply_delta(PyObject* self, PyObject* args)
+{
+	PyObject* pybbuf = 0;
+	PyObject* pydbuf = 0;
+	PyObject* pytbuf = 0;
+	if (!PyArg_ParseTuple(args, "OOO", &pybbuf, &pydbuf, &pytbuf)){
+		PyErr_BadArgument();
+		return NULL;
+	}
+	
+	PyObject* objects[] = { pybbuf, pydbuf, pytbuf };
+	assert(sizeof(objects) / sizeof(PyObject*) == 3);
+	
+	uint i;
+	for(i = 0; i < 3; i++){ 
+		if (!PyObject_CheckReadBuffer(objects[i])){
+			PyErr_SetString(PyExc_ValueError, "Argument must be a buffer-compatible object, like a string, or a memory map");
+			return NULL;
+		}
+	}
+	
+	Py_ssize_t lbbuf; Py_ssize_t ldbuf; Py_ssize_t ltbuf;
+	const uchar* bbuf; const uchar* dbuf;
+	uchar* tbuf;
+	PyObject_AsReadBuffer(pybbuf, (const void**)(&bbuf), &lbbuf);
+	PyObject_AsReadBuffer(pydbuf, (const void**)(&dbuf), &ldbuf);
+	
+	if (PyObject_AsWriteBuffer(pytbuf, (void**)(&tbuf), &ltbuf)){
+		PyErr_SetString(PyExc_ValueError, "Argument 3 must be a writable buffer");
+		return NULL;
+	}
+	
+	const uchar* data = dbuf;
+	const uchar* dend = dbuf + ldbuf;
+	
+	while (data < dend)
+	{
+		const char cmd = *data++;
+		
+		if (cmd & 0x80) 
+		{
+			unsigned long cp_off = 0, cp_size = 0;
+			if (cmd & 0x01) cp_off = *data++;
+			if (cmd & 0x02) cp_off |= (*data++ << 8);
+			if (cmd & 0x04) cp_off |= (*data++ << 16);
+			if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24);
+			if (cmd & 0x10) cp_size = *data++;
+			if (cmd & 0x20) cp_size |= (*data++ << 8);
+			if (cmd & 0x40) cp_size |= (*data++ << 16);
+			if (cp_size == 0) cp_size = 0x10000;
+			
+			memcpy(tbuf, bbuf + cp_off, cp_size); 
+			tbuf += cp_size;
+			
+		} else if (cmd) {
+			memcpy(tbuf, data, cmd);
+			tbuf += cmd;
+			data += cmd;
+		} else {                                                                               
+			PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0");
+			return NULL;
+		}
+	}// END handle command opcodes
+	
+	Py_RETURN_NONE;
+}

_fun.c

@@ -83,7 +83,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args)
 
 static PyMethodDef py_fun[] = {
 	{ "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" },
-	{ "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" },
+	{ "connect_deltas", (PyCFunction)connect_deltas, METH_O, "See python implementation" },
+	{ "apply_delta", (PyCFunction)apply_delta, METH_VARARGS, "See python implementation" },
 	{ NULL, NULL, 0, NULL }
 };
 

stream.py

@@ -22,6 +22,11 @@
 		zlib
 	)
 
+try:
+	from _perf import apply_delta as c_apply_delta
+except ImportError:
+	pass
+
 __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader')
 
 
@@ -413,7 +418,10 @@ def _set_cache_(self, attr):
 			stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE)
 
 			#######################################################################
-			apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write)
+			if 'c_apply_delta' in globals():
+				c_apply_delta(bbuf, ddata, tbuf);
+			else:
+				apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write)
 			#######################################################################
 
 			# finally, swap out source and target buffers. The target is now the 
@@ -430,13 +438,6 @@ def _set_cache_(self, attr):
 		self._mm_target = bbuf
 		self._size = final_target_size
 
-		# TODO: Once that works, figure out the ordering of the opcodes. If they
-		# are always in-order/sequential, an alternate implementation could 
-		# use stream access only. Of course this would mean we would read 
-		# all deltas in advance, analyse the opcode ranges to determine a final
-		# concatenated opcode list which indicates what to copy from which delta
-		# to which position. This preprocessing would allow true streaming
-		
 	def read(self, count=0):
 		bl = self._size - self._br		# bytes left
 		if count < 1 or count > bl: