Skip to content

Commit babf28a

Browse files
AdityaNPLalexirion10
AdityaNPL
and
alexirion10
authored
fix(striped_backups): changed logic to get all striped backups (#62)
* fix(striped_backups): changed logic to get all striped backups * test(striped_backups): adding a unit test for backup chains with striped backups * refactor(striped_backups): renames, comments and list handling based on PR suggestions * fix(striped_backups): return empty list if no diff backups * refactor(striped_backups): whitespace * fix(striped_backups): Exception on empty list of backups * fix(striped_backups): also a null check for the list of backups * test(striped_backups): few more thorough positive tests with stripes and duplicates * test(striped_backups): small rename in function * test(striped_backups): few negative tests * refactor(striped_backups): null/empty check for backup list * test(striped_backups): Adding back test for missing link * test(striped_backups): refactor the verify backup chain function * test(striped_backups): refactor `VerifyListIsAValidBackupChain()` * test(striped_backups): refactor duplicate file test to its own function * test(striped_backups): PR suggestions - refactor duplicate test + null check Co-authored-by: Alex Irion <alex.irion@factset.com>
1 parent 3b3af79 commit babf28a

3 files changed

Lines changed: 202 additions & 53 deletions

File tree

‎src/BackupChain.cs‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ namespace AgDatabaseMove
44
using System.Collections.Generic;
55
using System.IO;
66
using System.Linq;
7+
using Exceptions;
78
using SmoFacade;
89

910

@@ -19,26 +20,28 @@ public class BackupChain : IBackupChain
1920
{
2021
private readonly IList<BackupMetadata> _orderedBackups;
2122

23+
// This also handles any striped backups
2224
private BackupChain(IList<BackupMetadata> recentBackups)
2325
{
26+
if (recentBackups == null || recentBackups.Count == 0) {
27+
throw new BackupChainException("There are no recent backups to form a chain");
28+
}
29+
2430
var backups = recentBackups.Distinct(new BackupMetadataEqualityComparer())
25-
.Where(b => IsValidFilePath(b)) // A third party application caused invalid path strings to be inserted into backupmediafamily
31+
.Where(IsValidFilePath) // A third party application caused invalid path strings to be inserted into backupmediafamily
2632
.ToList();
2733

28-
var mostRecentFullBackup = MostRecentFullBackup(recentBackups);
29-
_orderedBackups = new List<BackupMetadata> { mostRecentFullBackup };
30-
31-
var differentialBackup = MostRecentDifferentialBackup(backups, mostRecentFullBackup);
32-
if(differentialBackup != null)
33-
_orderedBackups.Add(differentialBackup);
34+
var orderedBackups = MostRecentFullBackup(backups).ToList();
35+
orderedBackups.AddRange(MostRecentDiffBackup(backups, orderedBackups.First()));
3436

35-
var mostRecentBackup = _orderedBackups.Last();
36-
while(mostRecentBackup != null) {
37-
mostRecentBackup = NextLogBackup(backups, mostRecentBackup);
38-
39-
if(mostRecentBackup != null)
40-
_orderedBackups.Add(mostRecentBackup);
37+
var prevBackup = orderedBackups.Last();
38+
IEnumerable<BackupMetadata> nextLogBackups;
39+
while((nextLogBackups = NextLogBackup(backups, prevBackup)).Any()) {
40+
orderedBackups.AddRange(nextLogBackups);
41+
prevBackup = orderedBackups.Last();
4142
}
43+
44+
_orderedBackups = orderedBackups;
4245
}
4346

4447
/// <summary>
@@ -56,26 +59,44 @@ public BackupChain(Database database) : this(database.RecentBackups()) { }
5659
/// </summary>
5760
public IEnumerable<BackupMetadata> OrderedBackups => _orderedBackups;
5861

59-
private BackupMetadata MostRecentFullBackup(IList<BackupMetadata> backups)
62+
private static IEnumerable<BackupMetadata> MostRecentFullBackup(IEnumerable<BackupMetadata> backups)
6063
{
61-
return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Full).OrderByDescending(d => d.CheckpointLsn)
62-
.First();
64+
var fullBackupsOrdered = backups
65+
.Where(b => b.BackupType == BackupFileTools.BackupType.Full)
66+
.OrderByDescending(d => d.CheckpointLsn).ToList();
67+
68+
if(!fullBackupsOrdered.Any()) {
69+
throw new BackupChainException("Could not find any full backups");
70+
}
71+
72+
var targetCheckpointLsn = fullBackupsOrdered.First().CheckpointLsn;
73+
// get all the stripes of this backup
74+
return fullBackupsOrdered.Where(fullBackup => fullBackup.CheckpointLsn == targetCheckpointLsn);
6375
}
6476

65-
private BackupMetadata MostRecentDifferentialBackup(IList<BackupMetadata> backups, BackupMetadata lastFullBackup)
77+
private static IEnumerable<BackupMetadata> MostRecentDiffBackup(IEnumerable<BackupMetadata> backups, BackupMetadata lastFullBackup)
6678
{
67-
return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Diff &&
68-
b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn)
69-
.OrderByDescending(b => b.LastLsn).FirstOrDefault();
79+
var diffBackupsOrdered = backups
80+
.Where(b => b.BackupType == BackupFileTools.BackupType.Diff &&
81+
b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn)
82+
.OrderByDescending(b => b.LastLsn).ToList();
83+
84+
if (!diffBackupsOrdered.Any()) {
85+
return new List<BackupMetadata>();
86+
}
87+
var targetLastLsn = diffBackupsOrdered.First().LastLsn;
88+
// get all the stripes of this backup
89+
return diffBackupsOrdered.Where(diffBackup => diffBackup.LastLsn == targetLastLsn);
7090
}
7191

72-
private BackupMetadata NextLogBackup(IList<BackupMetadata> backups, BackupMetadata prevBackup)
92+
private static IEnumerable<BackupMetadata> NextLogBackup(IEnumerable<BackupMetadata> backups, BackupMetadata prevBackup)
7393
{
74-
return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log)
75-
.SingleOrDefault(d => prevBackup.LastLsn >= d.FirstLsn && prevBackup.LastLsn + 1 < d.LastLsn);
94+
// also gets all the stripes of the next backup
95+
return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log &&
96+
prevBackup.LastLsn >= b.FirstLsn && prevBackup.LastLsn + 1 < b.LastLsn);
7697
}
7798

78-
private bool IsValidFilePath(BackupMetadata meta)
99+
private static bool IsValidFilePath(BackupMetadata meta)
79100
{
80101
if(BackupFileTools.IsUrl(meta.PhysicalDeviceName))
81102
return true;

‎src/BackupMetadata.cs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@ public bool Equals(BackupMetadata x, BackupMetadata y)
1717
return x.LastLsn == y.LastLsn &&
1818
x.FirstLsn == y.FirstLsn &&
1919
x.BackupType == y.BackupType &&
20-
x.DatabaseName == y.DatabaseName;
20+
x.DatabaseName == y.DatabaseName &&
21+
x.PhysicalDeviceName == y.PhysicalDeviceName;
2122
}
2223

2324
public int GetHashCode(BackupMetadata obj)
2425
{
2526
var hashCode = -1277603921;
2627
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.DatabaseName);
28+
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.PhysicalDeviceName);
2729
hashCode = hashCode * -1521134295 +
2830
EqualityComparer<BackupFileTools.BackupType>.Default.GetHashCode(obj.BackupType);
2931
hashCode = hashCode * -1521134295 + obj.FirstLsn.GetHashCode();
@@ -35,7 +37,7 @@ public int GetHashCode(BackupMetadata obj)
3537
/// <summary>
3638
/// Metadata about backups from msdb.dbo.backupset and msdb.dbo.backupmediafamily
3739
/// </summary>
38-
public class BackupMetadata
40+
public class BackupMetadata : ICloneable
3941
{
4042
public decimal CheckpointLsn { get; set; }
4143
public decimal DatabaseBackupLsn { get; set; }
@@ -52,5 +54,11 @@ public class BackupMetadata
5254
/// https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-2017
5355
/// </summary>
5456
public BackupFileTools.BackupType BackupType { get; set; }
57+
58+
// used during testing
59+
public object Clone()
60+
{
61+
return MemberwiseClone();
62+
}
5563
}
5664
}

‎tests/AgDatabaseMove.Unit/BackupOrder.cs‎

Lines changed: 147 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,48 @@ namespace AgDatabaseMove.Unit
44
using System.Collections;
55
using System.Collections.Generic;
66
using System.Linq;
7+
using Exceptions;
78
using Moq;
89
using SmoFacade;
910
using Xunit;
1011

1112

1213
public class BackupOrder
1314
{
14-
private readonly List<BackupMetadata> _listBackups;
1515

16-
public BackupOrder()
16+
private static IEnumerable<BackupMetadata> CloneBackupMetaDataList(List<BackupMetadata> list)
1717
{
18-
_listBackups = ListBackups();
18+
var result = new List<BackupMetadata>();
19+
list.ForEach(b => {
20+
result.Add((BackupMetadata)b.Clone());
21+
});
22+
result.Reverse();
23+
return result;
1924
}
2025

21-
private static List<BackupMetadata> ListBackups()
26+
private static List<BackupMetadata> GetBackupList()
2227
{
2328
return new List<BackupMetadata> {
2429
new BackupMetadata {
2530
BackupType = BackupFileTools.BackupType.Log,
2631
DatabaseBackupLsn = 126000000943800037,
2732
CheckpointLsn = 126000000953600034,
28-
FirstLsn = 126000000955500001,
29-
LastLsn = 126000000955800001,
33+
FirstLsn = 126000000955200001,
34+
LastLsn = 126000000955500001,
35+
DatabaseName = "TestDb",
36+
ServerName = "ServerA",
37+
PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_020007_343.trn",
38+
StartTime = DateTime.Parse("2018-10-29 02:00:07.000")
39+
},
40+
new BackupMetadata {
41+
BackupType = BackupFileTools.BackupType.Log,
42+
DatabaseBackupLsn = 126000000943800037,
43+
CheckpointLsn = 126000000953600034,
44+
FirstLsn = 126000000955800001,
45+
LastLsn = 126000000965800001,
3046
DatabaseName = "TestDb",
3147
ServerName = "ServerB",
32-
PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_030006_660.trn",
48+
PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_040005_900.trn",
3349
StartTime = DateTime.Parse("2018-10-29 03:00:06.000")
3450
},
3551
new BackupMetadata {
@@ -44,51 +60,155 @@ private static List<BackupMetadata> ListBackups()
4460
StartTime = DateTime.Parse("2018-10-28 00:02:28.000")
4561
},
4662
new BackupMetadata {
47-
BackupType = BackupFileTools.BackupType.Diff,
63+
BackupType = BackupFileTools.BackupType.Log,
4864
DatabaseBackupLsn = 126000000943800037,
4965
CheckpointLsn = 126000000953600034,
50-
FirstLsn = 126000000943800037,
51-
LastLsn = 126000000955200001,
66+
FirstLsn = 126000000955500001,
67+
LastLsn = 126000000955800001,
5268
DatabaseName = "TestDb",
53-
ServerName = "ServerA",
54-
PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_000339_780.diff",
55-
StartTime = DateTime.Parse("2018-10-29 00:03:39.000")
69+
ServerName = "ServerB",
70+
PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_030006_660.trn",
71+
StartTime = DateTime.Parse("2018-10-29 03:00:06.000")
5672
},
5773
new BackupMetadata {
58-
BackupType = BackupFileTools.BackupType.Log,
59-
DatabaseBackupLsn = 126000000882000037,
74+
BackupType = BackupFileTools.BackupType.Diff,
75+
DatabaseBackupLsn = 126000000943800037,
6076
CheckpointLsn = 126000000953600034,
61-
FirstLsn = 126000000955200001,
62-
LastLsn = 126000000955500001,
77+
FirstLsn = 126000000945600000,
78+
LastLsn = 126000000955200001,
6379
DatabaseName = "TestDb",
6480
ServerName = "ServerA",
65-
PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_020007_343.trn",
66-
StartTime = DateTime.Parse("2018-10-29 02:00:07.000")
81+
PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_000339_780.diff",
82+
StartTime = DateTime.Parse("2018-10-29 00:03:39.000")
6783
}
6884
};
6985
}
7086

71-
[Fact]
72-
public void BackupChainOrdered()
87+
private static List<BackupMetadata> GetBackupListWithoutLogs()
88+
{
89+
var list = GetBackupList();
90+
list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Log);
91+
return list;
92+
}
93+
94+
private static List<BackupMetadata> GetBackupListWithoutDiff()
95+
{
96+
var list = GetBackupList();
97+
list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Diff);
98+
return list;
99+
}
100+
101+
private static List<BackupMetadata> GetBackupListWithStripes()
102+
{
103+
var list = GetBackupList();
104+
var listWithStripes = CloneBackupMetaDataList(list).ToList();
105+
listWithStripes.ForEach(b => {
106+
var path = b.PhysicalDeviceName.Split('.');
107+
b.PhysicalDeviceName = $"{path[0]}_striped.{path[1]}";
108+
});
109+
list.AddRange(listWithStripes);
110+
return list;
111+
}
112+
113+
private static List<BackupMetadata> GetBackupListWithStripesAndDuplicates()
114+
{
115+
var listWithStripes = GetBackupListWithStripes();
116+
var duplicate = CloneBackupMetaDataList(listWithStripes);
117+
listWithStripes.AddRange(duplicate);
118+
return listWithStripes;
119+
}
120+
121+
private static List<BackupMetadata> GetBackupListWithoutFull()
122+
{
123+
var list = GetBackupList();
124+
list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Full);
125+
return list;
126+
}
127+
128+
private static void VerifyListIsAValidBackupChain(List<BackupMetadata> backupChain)
129+
{
130+
bool foundFull, foundDiff, foundLog;
131+
foundFull = foundDiff = foundLog = false;
132+
BackupMetadata full = null;
133+
BackupMetadata lastBackup = null;
134+
135+
BackupMetadata currentBackup;
136+
while((currentBackup = backupChain.FirstOrDefault()) != null) {
137+
138+
if(currentBackup.BackupType == BackupFileTools.BackupType.Full) {
139+
Assert.True(!foundFull && !foundDiff && !foundLog);
140+
foundFull = true;
141+
full = currentBackup;
142+
}
143+
else if(currentBackup.BackupType == BackupFileTools.BackupType.Diff) {
144+
Assert.True(foundFull && !foundDiff && !foundLog);
145+
Assert.Equal(full.CheckpointLsn, currentBackup.DatabaseBackupLsn);
146+
Assert.True(currentBackup.FirstLsn >= lastBackup.LastLsn);
147+
foundDiff = true;
148+
}
149+
else if(currentBackup.BackupType == BackupFileTools.BackupType.Log) {
150+
Assert.True(foundFull);
151+
Assert.True(currentBackup.FirstLsn >= lastBackup.LastLsn);
152+
foundLog = true;
153+
}
154+
155+
lastBackup = currentBackup;
156+
backupChain.RemoveAll(b => b.LastLsn == currentBackup.LastLsn);
157+
}
158+
}
159+
160+
public static IEnumerable<object[]> PositiveTestData => new List<object[]> {
161+
new object[] { GetBackupList() },
162+
new object[] { GetBackupListWithStripes() },
163+
new object[] { GetBackupListWithoutDiff() },
164+
new object[] { GetBackupListWithoutLogs() }
165+
};
166+
167+
[Theory]
168+
[MemberData(nameof(PositiveTestData))]
169+
public void BackupChainIsCorrect(List<BackupMetadata> backupList)
73170
{
74171
var agDatabase = new Mock<IAgDatabase>();
75-
agDatabase.Setup(agd => agd.RecentBackups()).Returns(_listBackups);
172+
agDatabase.Setup(agd => agd.RecentBackups()).Returns(backupList);
76173
var backupChain = new BackupChain(agDatabase.Object);
174+
VerifyListIsAValidBackupChain(backupChain.OrderedBackups.ToList());
175+
}
176+
77177

78-
var expected = _listBackups.OrderBy(bu => bu.FirstLsn);
178+
public static IEnumerable<object[]> NegativeTestData => new List<object[]> {
179+
new object[] { GetBackupListWithoutFull() },
180+
new object[] { new List<BackupMetadata>() }
181+
};
79182

80-
Assert.Equal<IEnumerable>(backupChain.OrderedBackups, expected);
183+
[Theory]
184+
[MemberData(nameof(NegativeTestData))]
185+
public void CanDetectBackupChainIsWrong(List<BackupMetadata> backupList)
186+
{
187+
var agDatabase = new Mock<IAgDatabase>();
188+
agDatabase.Setup(agd => agd.RecentBackups()).Returns(backupList);
189+
Assert.Throws<BackupChainException>(() => new BackupChain(agDatabase.Object));
81190
}
82191

83192
[Fact]
84193
public void MissingLink()
85194
{
86-
var backups = ListBackups().Where(b => b.FirstLsn != 126000000955200001).ToList();
195+
var backups = GetBackupList().Where(b => b.FirstLsn != 126000000955200001).ToList();
87196
var agDatabase = new Mock<IAgDatabase>();
88197
agDatabase.Setup(agd => agd.RecentBackups()).Returns(backups);
89198

90-
var chain = new BackupChain(agDatabase.Object).OrderedBackups;
91-
Assert.NotEqual(chain.Last().LastLsn, ListBackups().Max(b => b.LastLsn));
199+
var chain = new BackupChain(agDatabase.Object).OrderedBackups.ToList();
200+
Assert.NotEqual(chain.Last().LastLsn, GetBackupList().Max(b => b.LastLsn));
201+
VerifyListIsAValidBackupChain(chain);
202+
}
203+
204+
[Fact]
205+
public void DuplicateFiles()
206+
{
207+
var backups = GetBackupListWithStripesAndDuplicates();
208+
var agDatabase = new Mock<IAgDatabase>();
209+
agDatabase.Setup(agd => agd.RecentBackups()).Returns(backups);
210+
var chain = new BackupChain(agDatabase.Object).OrderedBackups.ToList();
211+
Assert.Equal(backups.GroupBy(b => b.PhysicalDeviceName).Count(), chain.Count);
92212
}
93213

94214
// TODO: test skipping of logs if diff last LSN and log last LSN matches

0 commit comments

Comments
 (0)