out = []
total = 0
- performs_read = False
stashes = {}
stashed_blocks = 0
out.append("%s %s\n" % (xf.style, xf.tgt_ranges.to_string_raw()))
total += tgt_size
elif xf.style == "move":
- performs_read = True
assert xf.tgt_ranges
assert xf.src_ranges.size() == tgt_size
if xf.src_ranges != xf.tgt_ranges:
xf.tgt_ranges.to_string_raw(), src_str))
total += tgt_size
elif xf.style in ("bsdiff", "imgdiff"):
- performs_read = True
assert xf.tgt_ranges
assert xf.src_ranges
if self.version == 1:
max_allowed = cache_size * stash_threshold / self.tgt.blocksize
stashed_blocks = 0
+ new_blocks = 0
# Now go through all the commands. Compute the required stash size on the
# fly. If a command requires excess stash than available, it deletes the
# that will use this stash and replace the command with "new".
use_cmd = stashes[idx][2]
replaced_cmds.append(use_cmd)
- print(" %s replaced due to an explicit stash of %d blocks." % (
- use_cmd, sr.size()))
+ print("%10d %9s %s" % (sr.size(), "explicit", use_cmd))
else:
stashed_blocks += sr.size()
if xf.src_ranges.overlaps(xf.tgt_ranges):
if stashed_blocks + xf.src_ranges.size() > max_allowed:
replaced_cmds.append(xf)
- print(" %s replaced due to an implicit stash of %d blocks." % (
- xf, xf.src_ranges.size()))
+ print("%10d %9s %s" % (xf.src_ranges.size(), "implicit", xf))
# Replace the commands in replaced_cmds with "new"s.
for cmd in replaced_cmds:
def_cmd = stashes[idx][1]
assert (idx, sr) in def_cmd.stash_before
def_cmd.stash_before.remove((idx, sr))
+ new_blocks += sr.size()
cmd.ConvertToNew()
+ print(" Total %d blocks are packed as new blocks due to insufficient "
+ "cache size." % (new_blocks,))
+
def ComputePatches(self, prefix):
print("Reticulating splines...")
diff_q = []
a.goes_after[b] = size
def FindTransfers(self):
+ """Parse the file_map to generate all the transfers."""
+
+ def AddTransfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id,
+ split=False):
+ """Wrapper function for adding a Transfer().
+
+ For BBOTA v3, we need to stash source blocks for resumable feature.
+ However, with the growth of file size and the shrink of the cache
+ partition source blocks are too large to be stashed. If a file occupies
+ too many blocks (greater than MAX_BLOCKS_PER_DIFF_TRANSFER), we split it
+ into smaller pieces by getting multiple Transfer()s.
+
+ The downside is that after splitting, we can no longer use imgdiff but
+ only bsdiff."""
+
+ MAX_BLOCKS_PER_DIFF_TRANSFER = 1024
+
+ # We care about diff transfers only.
+ if style != "diff" or not split:
+ Transfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id)
+ return
+
+ # Change nothing for small files.
+ if (tgt_ranges.size() <= MAX_BLOCKS_PER_DIFF_TRANSFER and
+ src_ranges.size() <= MAX_BLOCKS_PER_DIFF_TRANSFER):
+ Transfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id)
+ return
+
+ pieces = 0
+ while (tgt_ranges.size() > MAX_BLOCKS_PER_DIFF_TRANSFER and
+ src_ranges.size() > MAX_BLOCKS_PER_DIFF_TRANSFER):
+ tgt_split_name = "%s-%d" % (tgt_name, pieces)
+ src_split_name = "%s-%d" % (src_name, pieces)
+ tgt_first = tgt_ranges.first(MAX_BLOCKS_PER_DIFF_TRANSFER)
+ src_first = src_ranges.first(MAX_BLOCKS_PER_DIFF_TRANSFER)
+ Transfer(tgt_split_name, src_split_name, tgt_first, src_first, style,
+ by_id)
+
+ tgt_ranges = tgt_ranges.subtract(tgt_first)
+ src_ranges = src_ranges.subtract(src_first)
+ pieces += 1
+
+ # Handle remaining blocks.
+ if tgt_ranges.size() or src_ranges.size():
+ # Must be both non-empty.
+ assert tgt_ranges.size() and src_ranges.size()
+ tgt_split_name = "%s-%d" % (tgt_name, pieces)
+ src_split_name = "%s-%d" % (src_name, pieces)
+ Transfer(tgt_split_name, src_split_name, tgt_ranges, src_ranges, style,
+ by_id)
+
empty = RangeSet()
for tgt_fn, tgt_ranges in self.tgt.file_map.items():
if tgt_fn == "__ZERO":
# in any file and that are filled with zeros. We have a
# special transfer style for zero blocks.
src_ranges = self.src.file_map.get("__ZERO", empty)
- Transfer(tgt_fn, "__ZERO", tgt_ranges, src_ranges,
- "zero", self.transfers)
+ AddTransfer(tgt_fn, "__ZERO", tgt_ranges, src_ranges,
+ "zero", self.transfers)
continue
elif tgt_fn == "__COPY":
# "__COPY" domain includes all the blocks not contained in any
# file and that need to be copied unconditionally to the target.
- Transfer(tgt_fn, None, tgt_ranges, empty, "new", self.transfers)
+ AddTransfer(tgt_fn, None, tgt_ranges, empty, "new", self.transfers)
continue
elif tgt_fn in self.src.file_map:
# Look for an exact pathname match in the source.
- Transfer(tgt_fn, tgt_fn, tgt_ranges, self.src.file_map[tgt_fn],
- "diff", self.transfers)
+ AddTransfer(tgt_fn, tgt_fn, tgt_ranges, self.src.file_map[tgt_fn],
+ "diff", self.transfers, self.version >= 3)
continue
b = os.path.basename(tgt_fn)
if b in self.src_basenames:
# Look for an exact basename match in the source.
src_fn = self.src_basenames[b]
- Transfer(tgt_fn, src_fn, tgt_ranges, self.src.file_map[src_fn],
- "diff", self.transfers)
+ AddTransfer(tgt_fn, src_fn, tgt_ranges, self.src.file_map[src_fn],
+ "diff", self.transfers, self.version >= 3)
continue
b = re.sub("[0-9]+", "#", b)
# for .so files that contain version numbers in the filename
# that get bumped.)
src_fn = self.src_numpatterns[b]
- Transfer(tgt_fn, src_fn, tgt_ranges, self.src.file_map[src_fn],
- "diff", self.transfers)
+ AddTransfer(tgt_fn, src_fn, tgt_ranges, self.src.file_map[src_fn],
+ "diff", self.transfers, self.version >= 3)
continue
- Transfer(tgt_fn, None, tgt_ranges, empty, "new", self.transfers)
+ AddTransfer(tgt_fn, None, tgt_ranges, empty, "new", self.transfers)
def AbbreviateSourceNames(self):
for k in self.src.file_map.keys():
OSDN Git Service
