Added frame for actual implementation of the aggregation - for now we just implement a forward-merge of the chunks, then the reverse version once there is some more knowledge on how this works

Author: Byron

ext/async

@@ -41,7 +41,48 @@
 
 __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 
 			'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', 
-			'is_equal_canonical_sha' )
+			'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas',
+			'merge_deltas')
+
+
+#{ Structures
+
+class DeltaChunk(object):
+	"""Represents a piece of a delta, it can either add new data, or copy existing
+	one from a source buffer"""
+	__slots__ = (
+					'to',		# start offset in the target buffer in bytes 
+					'ts',		# size of this chunk in the target buffer in bytes
+					'so',		# start offset in the source buffer in bytes or None
+					'data'		# chunk of bytes to be added to the target buffer or None 
+				)
+	
+	def __init__(self, to, ts, so, data):
+		self.to = to
+		self.ts = ts
+		self.so = so
+		self.data = data
+		
+	#{ Interface
+		
+	def abssize(self):
+		return self.to + self.ts
+		
+	def apply(self, source, target):
+		"""Apply own data to the target buffer
+		:param source: buffer providing source bytes for copy operations
+		:param target: target buffer large enough to contain all the changes to be applied"""
+		if self.data is not None:
+			# APPEND DATA
+			pass
+		else:
+			# COPY DATA FROM SOURCE
+			pass
+		# END handle chunk mode
+		
+	#} END interface
+
+#} END structures
 
 #{ Routines
 
@@ -164,10 +205,46 @@ def stream_copy(read, write, size, chunk_size):
 	return dbw
 
 
+def reverse_merge_deltas(dcl, dstreams):
+	"""Read the condensed delta chunk information from dstream and merge its information
+	into a list of existing delta chunks
+	:param dcl: list of DeltaChunk objects, may be empty initially, and will be changed
+		during the merge process
+	:param dstreams: iterable of delta stream objects. They must be ordered latest first, 
+		hence the delta to be applied last comes first, then its ancestors
+	:return: None"""
+	raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now")
+	
+def merge_deltas(dcl, dstreams):
+	"""Read the condensed delta chunk information from dstream and merge its information
+	into a list of existing delta chunks
+	:param dcl: list of DeltaChunk objects, may be empty initially, and will be changed
+		during the merge process
+	:param dstreams: iterable of delta stream objects. They must be ordered latest last, 
+		hence the delta to be applied last comes last, its oldest ancestor first
+	:return: None"""
+	for ds in dstreams:
+		buf = ds.read()
+		i, src_size = msb_size(buf)
+		i, target_size = msb_size(buf, i)
+		
+		# parse the commands
+		
+	# END for each delta stream
+	
+def apply_delta_chunks(src_buf, src_buf_size, dcl, target):
+	"""
+	Apply data from a delta chunk list and a source buffer to the target stream
+	
+	:param src_buf: random access data from which the delta was created
+	:param src_buf_size: size of the source buffer in bytes
+	:param delta_buf_size: size fo the delta buffer in bytes
+	:param target: ostream with a write method"""
+	
+	
 def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file):
 	"""
-	Apply data from a delta buffer using a source buffer to the target file, 
-	which will be written to
+	Apply data from a delta buffer using a source buffer to the target file
 	
 	:param src_buf: random access data from which the delta was created
 	:param src_buf_size: size of the source buffer in bytes

fun.py

@@ -7,7 +7,9 @@
 from fun import (
 					msb_size,
 					stream_copy, 
-					apply_delta_data, 
+					apply_delta_data,
+					apply_delta_chunks,
+					merge_deltas,
 					delta_types
 				)
 
@@ -320,9 +322,29 @@ def __init__(self, stream_list):
 		self._br = 0
 
 	def _set_cache_(self, attr):
+		# Aggregate all deltas into one delta in reverse order. Hence we take 
+		# the last delta, and reverse-merge its ancestor delta, until we receive
+		# the final delta data stream.
+		dcl = list()
+		reverse_merge_deltas(dcl, self._dstreams)
+		
+		if len(dcl) == 0:
+			self._size = 0
+			self._mm_target = allocate_memory(0)
+			return
+		# END handle empty list
+		
+		self._size = dcl[-1].abssize()
+		self._mm_target = allocate_memory(self._size)
+		
+		bbuf = allocate_memory(self._bstream.size)
+		stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE)
+		
+		apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target)
+		
+	def _set_cache_old(self, attr):
 		"""If we are here, we apply the actual deltas"""
 
-		# prefetch information
 		buffer_info_list = list()
 		max_target_size = 0
 		for dstream in self._dstreams:

stream.py

@@ -117,6 +117,10 @@ def make_sha(source=''):
 
 def allocate_memory(size):
 	""":return: a file-protocol accessible memory block of the given size"""
+	if size == 0:
+		return _RandomAccessStringIO('')
+	# END handle empty chunks gracefully
+	
 	try:
 		return mmap.mmap(-1, size)	# read-write by default
 	except EnvironmentError: