Reverse delta aggregration appears to be working

Author: Byron

fun.py

@@ -235,6 +235,46 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0):
 		# END for each dc
 	# END handle application values
 
+def delta_list_slice(dcl, absofs, size):
+	""":return: Subsection of this  list at the given absolute  offset, with the given 
+		size in bytes.
+	:return: DeltaChunkList (copy) which represents the given chunk"""
+	if len(dcl) == 0:
+		return DeltaChunkList()
+		
+	absofs = max(absofs, dcl.lbound())
+	size = min(dcl.rbound() - dcl.lbound(), size)
+	cdi = _closest_index(dcl, absofs)	# delta start index
+	cd = dcl[cdi]
+	slen = len(dcl)
+	ndcl = dcl.__class__()
+	
+	if cd.to != absofs:
+		tcd = delta_duplicate(cd)
+		_move_delta_lbound(tcd, absofs - cd.to)
+		_set_delta_rbound(tcd, min(tcd.ts, size))
+		ndcl.append(tcd)
+		size -= tcd.ts
+		cdi += 1
+	# END lbound overlap handling
+	
+	while cdi < slen and size:
+		# are we larger than the current block
+		cd = dcl[cdi]
+		if cd.ts <= size:
+			ndcl.append(delta_duplicate(cd))
+			size -= cd.ts
+		else:
+			tcd = delta_duplicate(cd)
+			_set_delta_rbound(tcd, size)
+			ndcl.append(tcd)
+			size -= tcd.ts
+			break
+		# END hadle size
+		cdi += 1
+	# END for each chunk
+	
+	return ndcl
 
 class DeltaChunkList(list):
 	"""List with special functionality to deal with DeltaChunks.
@@ -270,7 +310,7 @@ def connect_with(self, bdcl):
 		:param bdcl: DeltaChunkList to serve as base"""
 		for dc in self:
 			if not dc.has_data():
-				dc.set_copy_chunklist(bdcl[dc.so:dc.ts])
+				dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts))
 			# END handle overlap
 		# END for each dc
 
@@ -355,49 +395,7 @@ def check_integrity(self, target_size=-1):
 			assert lft.to + lft.ts == rgt.to
 		# END for each pair
 
-	def __getslice__(self, absofs, size):
-		""":return: Subsection of this  list at the given absolute  offset, with the given 
-			size in bytes.
-		:return: DeltaChunkList (copy) which represents the given chunk"""
-		if len(self) == 0:
-			return DeltaChunkList()
-			
-		absofs = max(absofs, self.lbound())
-		size = min(self.rbound() - self.lbound(), size)
-		cdi = _closest_index(self, absofs)	# delta start index
-		cd = self[cdi]
-		slen = len(self)
-		ndcl = self.__class__()
-		
-		if cd.to != absofs:
-			tcd = delta_duplicate(cd)
-			_move_delta_lbound(tcd, absofs - cd.to)
-			_set_delta_rbound(tcd, min(tcd.ts, size))
-			ndcl.append(tcd)
-			size -= tcd.ts
-			cdi += 1
-		# END lbound overlap handling
-		
-		while cdi < slen and size:
-			# are we larger than the current block
-			cd = self[cdi]
-			if cd.ts <= size:
-				ndcl.append(delta_duplicate(cd))
-				size -= cd.ts
-			else:
-				tcd = delta_duplicate(cd)
-				_set_delta_rbound(tcd, size)
-				ndcl.append(tcd)
-				size -= tcd.ts
-				break
-			# END hadle size
-			cdi += 1
-		# END for each chunk
-		
-		# ndcl.check_integrity()
-		return ndcl
 
-	
 class TopdownDeltaChunkList(DeltaChunkList):
 	"""Represents a list which is generated by feeding its ancestor streams one by 
 	one"""
@@ -416,15 +414,76 @@ def connect_with_next_base(self, bdcl):
 			consequtively and in order, towards the earliest ancestor delta
 		:return: True if processing was done. Use it to abort processing of
 			remaining streams"""
+		assert self is not bdcl
 		if self.frozen == 1:
 			# Can that ever be hit ?
 			return False
 		# END early abort
-		# mark us so that the is_reversed method returns True, without us thinking
-		# we are frozen
-		self.frozen = -1
 
-		raise NotImplementedError("todo")
+		nfc = 0								# number of frozen chunks
+		dci = 0								# delta chunk index
+		slen = len(self)					# len of self
+		sold = slen
+		while dci < slen:
+			dc = self[dci]
+			dci += 1
+			
+			if dc.flags:
+				nfc += 1
+				continue
+			# END skip frozen chunks
+			
+			# all data chunks must be frozen, we are topmost already
+			# (Also if its a copy operation onto the lowest base, but we cannot 
+			# determine that without the number of deltas to come)
+			if dc.has_data():
+				dc.flags = True
+				nfc += 1
+				continue
+			# END skip add chunks
+			
+			# copy chunks
+			# integrate the portion of the base list into ourselves. Lists
+			# dont support efficient insertion ( just one at a time ), but for now
+			# we live with it. Internally, its all just a 32/64bit pointer, and
+			# the portions of moved memory should be smallish. Maybe we just rebuild
+			# ourselves in order to reduce the amount of insertions ... 
+			ccl = delta_list_slice(bdcl, dc.so, dc.ts)
+			
+			# move the target bounds into place to match with our chunk
+			ofs = dc.to - dc.so
+			for cdc in ccl:
+				cdc.to += ofs
+			# END update target bounds
+			
+			
+			assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound()
+			
+			if len(ccl) == 1:
+				self[dci-1] = ccl[0]
+			else:
+				
+				# maybe try to compute the expenses here, and pick the right algorithm
+				# It would normally be faster than copying everything physically though
+				# TODO: Use a deque here, and decide by the index whether to extend
+				# or extend left !
+				post_dci = self[dci:]
+				del(self[dci-1:])			# include deletion of dc
+				self.extend(ccl)
+				self.extend(post_dci)
+				
+				slen = len(self)
+				dci += len(ccl)-1			# deleted dc, added rest
+				
+			# END handle chunk replacement
+			
+		# END for each chunk
+		
+		if nfc == slen:
+			self.frozen = True
+			return False
+		# END handle completeness
+		
 		return True
 
 
@@ -648,8 +707,6 @@ def connect_deltas(dstreams, reverse):
 				dcl.connect_with(bdcl)
 		# END handle merge
 
-		# dcl.check_integrity()
-		
 		# prepare next base
 		bdcl = dcl
 		dcl = DeltaChunkList()

test/test_pack.py

@@ -130,7 +130,7 @@ def test_pack(self):
 			self._assert_pack_file(pack, version, size)
 		# END for each pack to test
 
-	def _test_pack_entity(self):
+	def test_pack_entity(self):
 		for packinfo, indexinfo in (	(self.packfile_v2_1, self.packindexfile_v1), 
 										(self.packfile_v2_2, self.packindexfile_v2),
 										(self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)):