From 376650521ef4737436521744427a5464dd0b44b7 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Mon, 11 Feb 2019 10:55:51 -0800 Subject: [PATCH 01/11] Add resume functionality to DT tool --- .../ContinuationTokenParser.cs | 57 +++++++++++++++++ .../Microsoft.DataTransfer.AzureTable.csproj | 3 + .../Resumption/AzureTablePrimaryKey.cs | 24 +++++++ .../Resumption/AzureTableResumptionAdaptor.cs | 63 +++++++++++++++++++ .../Sink/TableAPIBulkSinkAdapter.cs | 13 +++- .../Sink/TableAPIBulkSinkAdapterFactory.cs | 7 ++- .../Source/AzureTableSourceAdapter.cs | 21 ++++++- .../Source/AzureTableSourceAdapterFactory.cs | 10 +-- .../Microsoft.DataTransfer.Core.csproj | 4 ++ .../Service/DataTransferContext.cs | 1 + .../Service/DataTransferService.cs | 20 +++++- .../IDataTransferContext.cs | 2 + .../IDataTransferResumptionAdapter.cs | 15 +++++ ...icrosoft.DataTransfer.Extensibility.csproj | 1 + .../Mocks/DataTransferContextMock.cs | 5 ++ 15 files changed, 236 insertions(+), 10 deletions(-) create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs create mode 100644 Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs new file mode 100644 index 0000000..17dfe3e --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs @@ -0,0 +1,57 @@ +using System; +using System.Text; + +namespace Microsoft.DataTransfer.AzureTable +{ + /// + /// The class which is used to encode the continuation token + /// + public class ContinuationTokenParser + { + /// + /// Encode continuation token to fomart: (Version)!(TokenLength)!(CustomBase64EncodedToken) + /// + public static string EncodeContinuationToken(string key) + { + StringBuilder encodedContinuationToken = new StringBuilder(); + // Version of the ContinuationToken + encodedContinuationToken.Append(1); + encodedContinuationToken.Append(exclamationDelimiter); + + UTF8Encoding utf8Encoding = new UTF8Encoding(); + string base64EncodedToken = Convert.ToBase64String(utf8Encoding.GetBytes(key.ToString())); + string customBase64EncodedString = UrlCustomEscapeBase64String(base64EncodedToken); + + //Size is the lenght of base64 encoded key + encodedContinuationToken.Append(customBase64EncodedString.Length); + encodedContinuationToken.Append(exclamationDelimiter); + + encodedContinuationToken.Append(customBase64EncodedString); + return encodedContinuationToken.ToString(); + } + + private static string UrlCustomEscapeBase64String(string token) + { + StringBuilder escapedString = new StringBuilder(); + foreach (char c in token.ToCharArray()) + { + escapedString.Append(TranslateChar(c)); + } + + return escapedString.ToString(); + } + + private static char TranslateChar(char c) + { + switch (c) + { + case '/': return '_'; + case '+': return '*'; + case '=': return '-'; + default: return c; + } + } + + private const char exclamationDelimiter = '!'; + } +} diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Microsoft.DataTransfer.AzureTable.csproj b/AzureTable/Microsoft.DataTransfer.AzureTable/Microsoft.DataTransfer.AzureTable.csproj index da9b82b..f887426 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Microsoft.DataTransfer.AzureTable.csproj +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Microsoft.DataTransfer.AzureTable.csproj @@ -85,6 +85,7 @@ True ConfigurationResources.resx + @@ -95,6 +96,8 @@ True Resources.resx + + diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs new file mode 100644 index 0000000..23f1455 --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.DataTransfer.AzureTable.Resumption +{ + /// + /// + /// + public class AzureTablePrimaryKey + { + /// + /// + /// + public string PartitionKey { get; set; } + + /// + /// + /// + public string RowKey { get; set; } + } +} diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs new file mode 100644 index 0000000..6531548 --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs @@ -0,0 +1,63 @@ +using Microsoft.DataTransfer.Basics; +using Microsoft.DataTransfer.Extensibility; +using Newtonsoft.Json; +using System; +using System.IO; + +namespace Microsoft.DataTransfer.AzureTable.Resumption +{ + /// + /// + /// + public class AzureTableResumptionAdaptor : IDataTransferResumptionAdapter + { + private readonly string _fileName; + + /// + /// + /// + /// + public AzureTableResumptionAdaptor(string fileName) + { + Guard.NotEmpty(nameof(fileName), fileName); + + if (fileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) + { + throw new ArgumentException("File name contains invalid characters."); + } + + _fileName = fileName; + } + + /// + /// + /// + /// + public AzureTablePrimaryKey GetCheckpoint() + { + string fileName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), _fileName); + if (File.Exists(fileName)) + { + return JsonConvert.DeserializeObject(File.ReadAllText(fileName)); + } + + return null; + } + + /// + /// + /// + /// + public void SaveCheckpoint(AzureTablePrimaryKey checkpoint) + { + Guard.NotNull(nameof(checkpoint), checkpoint); + + using (StreamWriter file = File.CreateText( + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), _fileName))) + { + JsonSerializer serializer = new JsonSerializer(); + serializer.Serialize(file, checkpoint); + } + } + } +} diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs index da10b89..57cb98b 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs @@ -3,6 +3,7 @@ using Microsoft.Azure.CosmosDB; using Microsoft.Azure.CosmosDB.Table; using Microsoft.Azure.Storage; + using Microsoft.DataTransfer.AzureTable.Resumption; using Microsoft.DataTransfer.AzureTable.Sink.Bulk; using Microsoft.DataTransfer.AzureTable.Source; using Microsoft.DataTransfer.AzureTable.Utils; @@ -25,6 +26,7 @@ internal sealed class TableAPIBulkSinkAdapter : IDataSinkAdapter private long _maxInputBufferSizeInBytes; private int _throughput; private int _maxLengthInBytesPerBatch; + private IDataTransferResumptionAdapter _resumptionAdapter; private CloudTable cloudtable; private ConcurrentDictionary dict; @@ -37,14 +39,18 @@ public int MaxDegreeOfParallelism } public TableAPIBulkSinkAdapter(string connectionString, string tableName, - bool overwrite, long maxInputBufferSizeInBytes, int throughput, int batchSize) + bool overwrite, long maxInputBufferSizeInBytes, int throughput, int batchSize, + IDataTransferResumptionAdapter resumptionAdapter) { + Guard.NotNull(nameof(resumptionAdapter), resumptionAdapter); + _connectionString = connectionString; _tableName = tableName; _overwrite = overwrite; _maxInputBufferSizeInBytes = maxInputBufferSizeInBytes; _throughput = throughput; _maxLengthInBytesPerBatch = batchSize; + _resumptionAdapter = resumptionAdapter; CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_connectionString); @@ -69,6 +75,11 @@ public async Task InitializeAsync(CancellationToken cancellation) public async Task WriteAsync(IDataItem dataItem, CancellationToken cancellation) { var item = GetITableEntityFromIDataItem(dataItem); + if (dict.Count == 0) + { + _resumptionAdapter.SaveCheckpoint( + new AzureTablePrimaryKey { PartitionKey = item.PartitionKey, RowKey = item.RowKey }); + } inputSizeTracker.Add(item); diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs index 2633bb3..6835943 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs @@ -1,6 +1,7 @@ namespace Microsoft.DataTransfer.TableAPI.Sink.Bulk { using Microsoft.DataTransfer.AzureTable; + using Microsoft.DataTransfer.AzureTable.Resumption; using Microsoft.DataTransfer.Basics; using Microsoft.DataTransfer.Extensibility; using Microsoft.DataTransfer.Extensibility.Basics; @@ -38,7 +39,8 @@ public async Task CreateAsync(ITableAPIBulkSinkAdapterConfigur if (String.IsNullOrEmpty(configuration.TableName)) throw Errors.TableNameMissing(); - long maxInputBufferSizeInBytes = 10 * 1024 * 1024; + //long maxInputBufferSizeInBytes = 10 * 1024 * 1024; + long maxInputBufferSizeInBytes = 100 * 1024; if (configuration.MaxInputBufferSize.HasValue) maxInputBufferSizeInBytes = configuration.MaxInputBufferSize.Value; @@ -58,7 +60,8 @@ public async Task CreateAsync(ITableAPIBulkSinkAdapterConfigur var sink = new TableAPIBulkSinkAdapter(configuration.ConnectionString, configuration.TableName, configuration.Overwrite, - maxInputBufferSizeInBytes, throughput, batchSize); + maxInputBufferSizeInBytes, throughput, batchSize, + new AzureTableResumptionAdaptor(context.RunConfigSignature)); await sink.InitializeAsync(cancellation); diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs index 0984f9e..eca6729 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs @@ -1,6 +1,8 @@ using Microsoft.Azure.CosmosDB.Table; using Microsoft.Azure.Storage; using Microsoft.DataTransfer.AzureTable.Client; +using Microsoft.DataTransfer.AzureTable.Resumption; +using Microsoft.DataTransfer.Basics; using Microsoft.DataTransfer.Extensibility; using System.Collections.Generic; using System.Threading; @@ -18,12 +20,15 @@ sealed class AzureTableSourceAdapter : IDataSourceAdapter private readonly IAzureTableSourceAdapterInstanceConfiguration configuration; private readonly CloudTable table; private readonly TableQuery query; + private readonly IDataTransferResumptionAdapter _resumptionAdapter; private Task> segmentDownloadTask; private int currentEntityIndex; - public AzureTableSourceAdapter(IAzureTableSourceAdapterInstanceConfiguration configuration) + public AzureTableSourceAdapter(IAzureTableSourceAdapterInstanceConfiguration configuration, + IDataTransferResumptionAdapter resumptionAdapter) { + Guard.NotNull(nameof(resumptionAdapter), resumptionAdapter); this.configuration = configuration; string connectionString = System.Text.RegularExpressions.Regex.Replace( @@ -41,13 +46,25 @@ public AzureTableSourceAdapter(IAzureTableSourceAdapterInstanceConfiguration con FilterString = configuration.Filter, SelectColumns = configuration.Projection == null ? null : new List(configuration.Projection) }; + _resumptionAdapter = resumptionAdapter; } public async Task ReadNextAsync(ReadOutputByRef readOutput, CancellationToken cancellation) { if (segmentDownloadTask == null) { - MoveToNextSegment(null, cancellation); + TableContinuationToken continuationToken = null; + var checkpoint = _resumptionAdapter.GetCheckpoint(); + if (checkpoint != null) + { + continuationToken = new TableContinuationToken + { + NextPartitionKey = ContinuationTokenParser.EncodeContinuationToken(checkpoint.PartitionKey), + NextRowKey = ContinuationTokenParser.EncodeContinuationToken(checkpoint.RowKey) + }; + } + + MoveToNextSegment(continuationToken, cancellation); } var currentSegment = await segmentDownloadTask; diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs index 26e7f45..5c86678 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs @@ -1,4 +1,5 @@ -using Microsoft.DataTransfer.Basics; +using Microsoft.DataTransfer.AzureTable.Resumption; +using Microsoft.DataTransfer.Basics; using Microsoft.DataTransfer.Extensibility; using Microsoft.DataTransfer.Extensibility.Basics; using System; @@ -29,17 +30,18 @@ public string Description /// Task that represents asynchronous create operation. public Task CreateAsync(IAzureTableSourceAdapterConfiguration configuration, IDataTransferContext context, CancellationToken cancellation) { - return Task.Factory.StartNew(() => Create(configuration), cancellation); + return Task.Factory.StartNew(() => Create(configuration, context), cancellation); } - private static IDataSourceAdapter Create(IAzureTableSourceAdapterConfiguration configuration) + private static IDataSourceAdapter Create(IAzureTableSourceAdapterConfiguration configuration, IDataTransferContext context) { Guard.NotNull("configuration", configuration); if (String.IsNullOrEmpty(configuration.ConnectionString)) throw Errors.ConnectionStringMissing(); - return new AzureTableSourceAdapter(CreateInstanceConfiguration(configuration)); + return new AzureTableSourceAdapter(CreateInstanceConfiguration(configuration), + new AzureTableResumptionAdaptor(context.RunConfigSignature)); } private static IAzureTableSourceAdapterInstanceConfiguration CreateInstanceConfiguration(IAzureTableSourceAdapterConfiguration configuration) diff --git a/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj b/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj index 520538c..4b82ce8 100644 --- a/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj +++ b/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj @@ -42,6 +42,10 @@ + + ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + True + diff --git a/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs b/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs index 4ed37c2..30187f9 100644 --- a/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs +++ b/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs @@ -6,5 +6,6 @@ sealed class DataTransferContext : IDataTransferContext { public string SourceName { get; set; } public string SinkName { get; set; } + public string RunConfigSignature { get; set; } } } diff --git a/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs b/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs index 137a96a..4046ba8 100644 --- a/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs +++ b/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs @@ -3,6 +3,8 @@ using Microsoft.DataTransfer.ServiceModel; using Microsoft.DataTransfer.ServiceModel.Entities; using Microsoft.DataTransfer.ServiceModel.Statistics; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -52,10 +54,13 @@ public async Task TransferAsync(string sourceName, object sourceConfiguration, if (!sinks.TryGetValue(sinkName, out sinkFactoryAdapter)) throw Errors.UnknownDataSink(sinkName); + var jsonSerilizer = new JsonSerializer(); var context = new DataTransferContext { SourceName = sourceName, - SinkName = sinkName + SinkName = sinkName, + RunConfigSignature = GetStringSha256Hash( + JsonConvert.SerializeObject(sourceConfiguration) + JsonConvert.SerializeObject(sinkConfiguration)) }; try @@ -75,5 +80,18 @@ public async Task TransferAsync(string sourceName, object sourceConfiguration, statistics.Stop(); } } + + private static string GetStringSha256Hash(string text) + { + if (string.IsNullOrEmpty(text)) + return string.Empty; + + using (var sha = new System.Security.Cryptography.SHA256Managed()) + { + byte[] textData = System.Text.Encoding.UTF8.GetBytes(text); + byte[] hash = sha.ComputeHash(textData); + return BitConverter.ToString(hash).Replace("-", string.Empty); + } + } } } diff --git a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs index a371b77..9c79825 100644 --- a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs +++ b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs @@ -15,5 +15,7 @@ public interface IDataTransferContext /// Gets name of the data sink adapter. /// string SinkName { get; } + + string RunConfigSignature { get; } } } diff --git a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs new file mode 100644 index 0000000..eb4d067 --- /dev/null +++ b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.DataTransfer.Extensibility +{ + public interface IDataTransferResumptionAdapter + { + void SaveCheckpoint(TCheckpoint checkpoint); + + TCheckpoint GetCheckpoint(); + } +} diff --git a/Core/Microsoft.DataTransfer.Extensibility/Microsoft.DataTransfer.Extensibility.csproj b/Core/Microsoft.DataTransfer.Extensibility/Microsoft.DataTransfer.Extensibility.csproj index 7279ca9..6f15d99 100644 --- a/Core/Microsoft.DataTransfer.Extensibility/Microsoft.DataTransfer.Extensibility.csproj +++ b/Core/Microsoft.DataTransfer.Extensibility/Microsoft.DataTransfer.Extensibility.csproj @@ -51,6 +51,7 @@ + diff --git a/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs b/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs index e31d2d7..5a82e17 100644 --- a/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs +++ b/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs @@ -15,5 +15,10 @@ public string SinkName { get { return "TestSink"; } } + + public string RunConfigSignature + { + get { return "TestRunConfigSignature"; } + } } } From 5b98e2ebaa1462a511744f50be07175354cd9507 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Mon, 11 Feb 2019 16:01:30 -0800 Subject: [PATCH 02/11] minor changes in the resumption adaptor --- .../Resumption/AzureTableResumptionAdaptor.cs | 16 +++++++++------- .../Sink/TableAPIBulkSinkAdapterFactory.cs | 5 ++--- .../Source/AzureTableSourceAdapter.cs | 4 ---- .../Source/AzureTableSourceAdapterFactory.cs | 2 +- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs index 6531548..205f742 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs @@ -11,7 +11,7 @@ namespace Microsoft.DataTransfer.AzureTable.Resumption /// public class AzureTableResumptionAdaptor : IDataTransferResumptionAdapter { - private readonly string _fileName; + private readonly string _fileFullPath; /// /// @@ -26,7 +26,11 @@ public AzureTableResumptionAdaptor(string fileName) throw new ArgumentException("File name contains invalid characters."); } - _fileName = fileName; + var localAppDataFolder = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + var folderName = Path.Combine(localAppDataFolder, "dt"); + Directory.CreateDirectory(folderName); + + _fileFullPath = Path.Combine(folderName, fileName); } /// @@ -35,10 +39,9 @@ public AzureTableResumptionAdaptor(string fileName) /// public AzureTablePrimaryKey GetCheckpoint() { - string fileName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), _fileName); - if (File.Exists(fileName)) + if (File.Exists(_fileFullPath)) { - return JsonConvert.DeserializeObject(File.ReadAllText(fileName)); + return JsonConvert.DeserializeObject(File.ReadAllText(_fileFullPath)); } return null; @@ -52,8 +55,7 @@ public void SaveCheckpoint(AzureTablePrimaryKey checkpoint) { Guard.NotNull(nameof(checkpoint), checkpoint); - using (StreamWriter file = File.CreateText( - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), _fileName))) + using (StreamWriter file = File.CreateText(_fileFullPath)) { JsonSerializer serializer = new JsonSerializer(); serializer.Serialize(file, checkpoint); diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs index befd000..8434fec 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs @@ -39,8 +39,7 @@ public async Task CreateAsync(ITableAPIBulkSinkAdapterConfigur if (String.IsNullOrEmpty(configuration.TableName)) throw Errors.TableNameMissing(); - //long maxInputBufferSizeInBytes = 10 * 1024 * 1024; - long maxInputBufferSizeInBytes = 100 * 1024; + long maxInputBufferSizeInBytes = 10 * 1024 * 1024; if (configuration.MaxInputBufferSize.HasValue) maxInputBufferSizeInBytes = configuration.MaxInputBufferSize.Value; @@ -64,7 +63,7 @@ public async Task CreateAsync(ITableAPIBulkSinkAdapterConfigur var sink = new TableAPIBulkSinkAdapter(configuration.ConnectionString, configuration.TableName, configuration.Overwrite, maxInputBufferSizeInBytes, throughput, batchSize, - new AzureTableResumptionAdaptor(context.RunConfigSignature)); + new AzureTableResumptionAdaptor(context.RunConfigSignature + ".json")); await sink.InitializeAsync(cancellation); diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs index 2b6fae5..3918f2a 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs @@ -8,11 +8,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.Azure.CosmosDB.Table; -using Microsoft.Azure.Storage; using Microsoft.Azure.Storage.RetryPolicies; -using Microsoft.DataTransfer.AzureTable.Client; -using Microsoft.DataTransfer.Extensibility; namespace Microsoft.DataTransfer.AzureTable.Source { diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs index 5c86678..99e9379 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs @@ -41,7 +41,7 @@ private static IDataSourceAdapter Create(IAzureTableSourceAdapterConfiguration c throw Errors.ConnectionStringMissing(); return new AzureTableSourceAdapter(CreateInstanceConfiguration(configuration), - new AzureTableResumptionAdaptor(context.RunConfigSignature)); + new AzureTableResumptionAdaptor(context.RunConfigSignature + ".json")); } private static IAzureTableSourceAdapterInstanceConfiguration CreateInstanceConfiguration(IAzureTableSourceAdapterConfiguration configuration) From 0396f15c1decc8f201c2e07b12e1f5adf50cfdf8 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Thu, 14 Feb 2019 13:21:13 -0800 Subject: [PATCH 03/11] use cmd line to control the function --- .../Sink/TableAPIBulkSinkAdapter.cs | 4 +--- .../Sink/TableAPIBulkSinkAdapterFactory.cs | 2 +- .../Source/AzureTableSourceAdapter.cs | 3 +-- .../Source/AzureTableSourceAdapterFactory.cs | 2 +- .../Handlers/OneTimeDataTransferHandler.cs | 8 +++++-- .../IInfrastructureConfiguration.cs | 7 +++++-- Core/Microsoft.DataTransfer.Core/Errors.cs | 5 +++++ .../Resources.Designer.cs | 13 ++++++++++-- .../Resources.resx | 3 +++ .../Service/DataTransferContext.cs | 1 + .../Service/DataTransferService.cs | 21 +++++++++++++++++-- .../IDataTransferContext.cs | 8 +++++++ .../ConfigurationResources.Designer.cs | 13 ++++++++++-- .../ConfigurationResources.resx | 3 +++ .../IDataTransferService.cs | 4 +++- .../ITransferCommonConfiguration.cs | 21 +++++++++++++++++++ ...Microsoft.DataTransfer.ServiceModel.csproj | 1 + .../Mocks/DataTransferContextMock.cs | 5 +++++ 18 files changed, 106 insertions(+), 18 deletions(-) create mode 100644 Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs index 835c5bc..cfc3947 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs @@ -44,8 +44,6 @@ public TableAPIBulkSinkAdapter(string connectionString, string tableName, bool overwrite, long maxInputBufferSizeInBytes, int throughput, int batchSize, IDataTransferResumptionAdapter resumptionAdapter) { - Guard.NotNull(nameof(resumptionAdapter), resumptionAdapter); - _connectionString = connectionString; _tableName = tableName; _overwrite = overwrite; @@ -84,7 +82,7 @@ public async Task WriteAsync(IDataItem dataItem, CancellationToken cancellation) var item = GetITableEntityFromIDataItem(dataItem); if (dict.Count == 0) { - _resumptionAdapter.SaveCheckpoint( + _resumptionAdapter?.SaveCheckpoint( new AzureTablePrimaryKey { PartitionKey = item.PartitionKey, RowKey = item.RowKey }); } diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs index 8434fec..1c6eb61 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapterFactory.cs @@ -63,7 +63,7 @@ public async Task CreateAsync(ITableAPIBulkSinkAdapterConfigur var sink = new TableAPIBulkSinkAdapter(configuration.ConnectionString, configuration.TableName, configuration.Overwrite, maxInputBufferSizeInBytes, throughput, batchSize, - new AzureTableResumptionAdaptor(context.RunConfigSignature + ".json")); + context.EnableResumeFunction ? new AzureTableResumptionAdaptor(context.RunConfigSignature + ".json") : null); await sink.InitializeAsync(cancellation); diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs index 3918f2a..8026e10 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs @@ -31,7 +31,6 @@ sealed class AzureTableSourceAdapter : IDataSourceAdapter public AzureTableSourceAdapter(IAzureTableSourceAdapterInstanceConfiguration configuration, IDataTransferResumptionAdapter resumptionAdapter) { - Guard.NotNull(nameof(resumptionAdapter), resumptionAdapter); this.configuration = configuration; string connectionString = System.Text.RegularExpressions.Regex.Replace( @@ -62,7 +61,7 @@ public async Task ReadNextAsync(ReadOutputByRef readOutput, Cancellat if (segmentDownloadTask == null) { TableContinuationToken continuationToken = null; - var checkpoint = _resumptionAdapter.GetCheckpoint(); + var checkpoint = _resumptionAdapter?.GetCheckpoint(); if (checkpoint != null) { continuationToken = new TableContinuationToken diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs index 99e9379..f1d600e 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapterFactory.cs @@ -41,7 +41,7 @@ private static IDataSourceAdapter Create(IAzureTableSourceAdapterConfiguration c throw Errors.ConnectionStringMissing(); return new AzureTableSourceAdapter(CreateInstanceConfiguration(configuration), - new AzureTableResumptionAdaptor(context.RunConfigSignature + ".json")); + context.EnableResumeFunction ? new AzureTableResumptionAdaptor(context.RunConfigSignature + ".json") : null); } private static IAzureTableSourceAdapterInstanceConfiguration CreateInstanceConfiguration(IAzureTableSourceAdapterConfiguration configuration) diff --git a/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs b/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs index f6bf774..b2ae32d 100644 --- a/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs +++ b/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs @@ -16,16 +16,19 @@ sealed class OneTimeDataTransferHandler : ITransferHandler private readonly IDataAdapterConfigurationFactory dataAdapterConfiguration; private readonly ITransferStatisticsHandler statisticsHandler; private readonly ITransferStatisticsConfiguration statisticsConfiguration; + private readonly ITransferCommonConfiguration commonConfiguration; private readonly IOneTimeDataTransferConfiguration configuration; public OneTimeDataTransferHandler(IDataTransferService transferService, IDataAdapterConfigurationFactory dataAdapterConfiguration, - ITransferStatisticsHandler statisticsHandler, ITransferStatisticsConfiguration statisticsConfiguration, IOneTimeDataTransferConfiguration configuration) + ITransferStatisticsHandler statisticsHandler, ITransferStatisticsConfiguration statisticsConfiguration, + IOneTimeDataTransferConfiguration configuration, ITransferCommonConfiguration commonConfiguration) { this.transferService = transferService; this.dataAdapterConfiguration = dataAdapterConfiguration; this.statisticsHandler = statisticsHandler; this.statisticsConfiguration = statisticsConfiguration; + this.commonConfiguration = commonConfiguration; this.configuration = configuration; } @@ -60,7 +63,8 @@ await transferService // With statistics statistics, // Allow cancellation - cancellation.Token); + cancellation.Token, + commonConfiguration.EnableResumeFunction); } } diff --git a/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs b/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs index 03f396b..b0f515f 100644 --- a/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs +++ b/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs @@ -1,4 +1,5 @@ -using Microsoft.DataTransfer.ServiceModel.Errors; +using Microsoft.DataTransfer.ServiceModel; +using Microsoft.DataTransfer.ServiceModel.Errors; using Microsoft.DataTransfer.ServiceModel.Statistics; namespace Microsoft.DataTransfer.ConsoleHost.Configuration @@ -9,5 +10,7 @@ namespace Microsoft.DataTransfer.ConsoleHost.Configuration /// /// This needs to be public to allow automatic proxy class generation. /// - public interface IInfrastructureConfiguration : ITransferStatisticsConfiguration, IErrorDetailsConfiguration { } + public interface IInfrastructureConfiguration : + ITransferStatisticsConfiguration, IErrorDetailsConfiguration, ITransferCommonConfiguration + { } } diff --git a/Core/Microsoft.DataTransfer.Core/Errors.cs b/Core/Microsoft.DataTransfer.Core/Errors.cs index 741b123..35eec05 100644 --- a/Core/Microsoft.DataTransfer.Core/Errors.cs +++ b/Core/Microsoft.DataTransfer.Core/Errors.cs @@ -34,5 +34,10 @@ public static Exception NonGenericDataAdapterFactoryType(Type type) { return new InvalidOperationException(FormatMessage(Resources.NonGenericDataAdapterFactoryTypeFormat, type)); } + + public static Exception UnsupportedDataSourceOrSinkForResumption(string name) + { + return new NotSupportedException(FormatMessage(Resources.UnsupportedDataSourceOrSinkForResumptionFormat, name)); + } } } diff --git a/Core/Microsoft.DataTransfer.Core/Resources.Designer.cs b/Core/Microsoft.DataTransfer.Core/Resources.Designer.cs index 4008b80..f835b1f 100644 --- a/Core/Microsoft.DataTransfer.Core/Resources.Designer.cs +++ b/Core/Microsoft.DataTransfer.Core/Resources.Designer.cs @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version:4.0.30319.34209 +// Runtime Version:4.0.30319.42000 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. @@ -19,7 +19,7 @@ namespace Microsoft.DataTransfer.Core { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { @@ -104,5 +104,14 @@ internal static string UnknownDataSourceFormat { return ResourceManager.GetString("UnknownDataSourceFormat", resourceCulture); } } + + /// + /// Looks up a localized string similar to Data sink or source '{0}' is not supported for data transfer resumption. + /// + internal static string UnsupportedDataSourceOrSinkForResumptionFormat { + get { + return ResourceManager.GetString("UnsupportedDataSourceOrSinkForResumptionFormat", resourceCulture); + } + } } } diff --git a/Core/Microsoft.DataTransfer.Core/Resources.resx b/Core/Microsoft.DataTransfer.Core/Resources.resx index 79da0e4..6a2c461 100644 --- a/Core/Microsoft.DataTransfer.Core/Resources.resx +++ b/Core/Microsoft.DataTransfer.Core/Resources.resx @@ -132,4 +132,7 @@ Data source '{0}' is not known + + Data sink or source '{0}' is not supported for data transfer resumption + \ No newline at end of file diff --git a/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs b/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs index 30187f9..0c98589 100644 --- a/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs +++ b/Core/Microsoft.DataTransfer.Core/Service/DataTransferContext.cs @@ -7,5 +7,6 @@ sealed class DataTransferContext : IDataTransferContext public string SourceName { get; set; } public string SinkName { get; set; } public string RunConfigSignature { get; set; } + public bool EnableResumeFunction { get; set; } } } diff --git a/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs b/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs index 4046ba8..05a57b2 100644 --- a/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs +++ b/Core/Microsoft.DataTransfer.Core/Service/DataTransferService.cs @@ -19,6 +19,12 @@ sealed class DataTransferService : IDataTransferService private IReadOnlyDictionary sources; private IReadOnlyDictionary sinks; + private IReadOnlyList resumeFunctionSupportSourcesAndSinks = new List + { + "AzureTable", + "TableAPIBulk" + }; + public DataTransferService( IReadOnlyDictionary sources, IReadOnlyDictionary sinks, @@ -44,7 +50,8 @@ public IReadOnlyDictionary GetKnownSinks() } public async Task TransferAsync(string sourceName, object sourceConfiguration, - string sinkName, object sinkConfiguration, ITransferStatistics statistics, CancellationToken cancellation) + string sinkName, object sinkConfiguration, ITransferStatistics statistics, + CancellationToken cancellation, bool enableResumeFunction = false) { IDataSourceAdapterFactoryAdapter sourceFactoryAdapter; if (!sources.TryGetValue(sourceName, out sourceFactoryAdapter)) @@ -54,13 +61,23 @@ public async Task TransferAsync(string sourceName, object sourceConfiguration, if (!sinks.TryGetValue(sinkName, out sinkFactoryAdapter)) throw Errors.UnknownDataSink(sinkName); + if (enableResumeFunction) + { + if (!resumeFunctionSupportSourcesAndSinks.Contains(sourceName)) + throw Errors.UnsupportedDataSourceOrSinkForResumption(sourceName); + + if (!resumeFunctionSupportSourcesAndSinks.Contains(sinkName)) + throw Errors.UnsupportedDataSourceOrSinkForResumption(sinkName); + } + var jsonSerilizer = new JsonSerializer(); var context = new DataTransferContext { SourceName = sourceName, SinkName = sinkName, RunConfigSignature = GetStringSha256Hash( - JsonConvert.SerializeObject(sourceConfiguration) + JsonConvert.SerializeObject(sinkConfiguration)) + JsonConvert.SerializeObject(sourceConfiguration) + JsonConvert.SerializeObject(sinkConfiguration)), + EnableResumeFunction = enableResumeFunction }; try diff --git a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs index 9c79825..1dd31fd 100644 --- a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs +++ b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferContext.cs @@ -16,6 +16,14 @@ public interface IDataTransferContext /// string SinkName { get; } + /// + /// Gets the signature of the source and sink configuration + /// string RunConfigSignature { get; } + + /// + /// Whether to enable the resume function + /// + bool EnableResumeFunction { get; } } } diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs index 676bbaa..6337813 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs +++ b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version:4.0.30319.34209 +// Runtime Version:4.0.30319.42000 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. @@ -19,7 +19,7 @@ namespace Microsoft.DataTransfer.ServiceModel { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] public class ConfigurationResources { @@ -69,6 +69,15 @@ public static string Errors_DetailsFormat { } } + /// + /// Looks up a localized string similar to Optional. Whether to save the checkpoint file and resume from there if the data transfer action is stopped for some reason. + /// + public static string Statistics_EnableResumeFunction { + get { + return ResourceManager.GetString("Statistics_EnableResumeFunction", resourceCulture); + } + } + /// /// Looks up a localized string similar to Optional. Name of the CSV file to redirect data transfer failures. /// diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx index c2c2616..3450a89 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx +++ b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx @@ -129,4 +129,7 @@ Optional, default is {0}. Time interval to refresh on-screen data transfer progress + + Optional. Whether to save the checkpoint file and resume from there if the data transfer action is stopped for some reason + \ No newline at end of file diff --git a/Core/Microsoft.DataTransfer.ServiceModel/IDataTransferService.cs b/Core/Microsoft.DataTransfer.ServiceModel/IDataTransferService.cs index 6512dd9..1a27bdd 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/IDataTransferService.cs +++ b/Core/Microsoft.DataTransfer.ServiceModel/IDataTransferService.cs @@ -31,12 +31,14 @@ public interface IDataTransferService /// Name of the target data adapter. /// Target data adapter configuration. /// Instance of to report data transfer progress to. + /// Whether to enable saving and resuming from the last checkpoint. /// Cancellation token. /// Task that represents asynchronous data transfer operation. Task TransferAsync( string sourceName, object sourceConfiguration, string sinkName, object sinkConfiguration, ITransferStatistics statistics, - CancellationToken cancellation); + CancellationToken cancellation, + bool enableResumeFunction = false); } } diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs b/Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs new file mode 100644 index 0000000..f38121d --- /dev/null +++ b/Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.DataTransfer.ServiceModel +{ + /// + /// The common configuration for data transfer + /// + public interface ITransferCommonConfiguration + { + /// + /// Whether to allow saving or resuming from a checkpoint + /// + [Display(ResourceType = typeof(ConfigurationResources), Description = "Statistics_EnableResumeFunction")] + bool EnableResumeFunction { get; } + } +} diff --git a/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj b/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj index 3a5daf5..90df1ed 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj +++ b/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj @@ -56,6 +56,7 @@ + diff --git a/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs b/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs index 5a82e17..6b88e18 100644 --- a/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs +++ b/Shared/Microsoft.DataTransfer.TestsCommon/Mocks/DataTransferContextMock.cs @@ -20,5 +20,10 @@ public string RunConfigSignature { get { return "TestRunConfigSignature"; } } + + public bool EnableResumeFunction + { + get { return false; } + } } } From 4e87eeb7213b983bd63947de9de2ba03b60d88f0 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Thu, 14 Feb 2019 16:27:15 -0800 Subject: [PATCH 04/11] Add uts for ContinuationTokenParser and more comments --- .../ContinuationTokenParserTests.cs | 18 ++++ ...t.DataTransfer.AzureTable.UnitTests.csproj | 93 +++++++++++++++++++ .../Properties/AssemblyInfo.cs | 20 ++++ .../packages.config | 5 + .../Resumption/AzureTablePrimaryKey.cs | 6 +- .../Resumption/AzureTableResumptionAdaptor.cs | 14 +-- .../Source/AzureTableSourceAdapter.cs | 11 +-- DataTransfer.sln | 21 ++--- 8 files changed, 159 insertions(+), 29 deletions(-) create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/ContinuationTokenParserTests.cs create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/ContinuationTokenParserTests.cs b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/ContinuationTokenParserTests.cs new file mode 100644 index 0000000..1793ee5 --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/ContinuationTokenParserTests.cs @@ -0,0 +1,18 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.DataTransfer.AzureTable.UnitTests +{ + [TestClass] + public class ContinuationTokenParserTests + { + [TestMethod] + public void EncodeContinuationToken_StringEncoded() + { + var testString = "test"; + var encodedToken = ContinuationTokenParser.EncodeContinuationToken(testString); + + Assert.AreEqual("1!8!dGVzdA--", encodedToken, "The encoded token should be as expected."); + } + } +} diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj new file mode 100644 index 0000000..2d010d0 --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj @@ -0,0 +1,93 @@ + + + + Debug + AnyCPU + {428B3874-8780-4939-AC5A-79C3980E01C7} + Library + Properties + Microsoft.DataTransfer.AzureTable.UnitTests + Microsoft.DataTransfer.AzureTable.UnitTests + v4.5.2 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\packages\Castle.Core.3.3.3\lib\net45\Castle.Core.dll + True + + + ..\..\packages\Microsoft.Azure.DocumentDB.2.2.1\lib\net45\Microsoft.Azure.Documents.Client.dll + + + ..\..\packages\Moq.4.5.21\lib\net45\Moq.dll + True + + + ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + True + + + + + + + + + + + + + + + + + + + + + + + + + + {366ba489-e851-4899-9ba3-2f9c7599d24b} + Microsoft.DataTransfer.AzureTable + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + \ No newline at end of file diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..001fcbd --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("Microsoft.DataTransfer.AzureTable.UnitTests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Microsoft.DataTransfer.AzureTable.UnitTests")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +[assembly: ComVisible(false)] + +[assembly: Guid("428b3874-8780-4939-ac5a-79c3980e01c7")] + +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config new file mode 100644 index 0000000..e5ca672 --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs index 23f1455..aba42c6 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs @@ -7,17 +7,17 @@ namespace Microsoft.DataTransfer.AzureTable.Resumption { /// - /// + /// Define the checkpoint for AzureTable data transfer /// public class AzureTablePrimaryKey { /// - /// + /// The partition key of the checkpoint /// public string PartitionKey { get; set; } /// - /// + /// The row key of the checkpoint /// public string RowKey { get; set; } } diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs index 205f742..2af375f 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs @@ -7,16 +7,16 @@ namespace Microsoft.DataTransfer.AzureTable.Resumption { /// - /// + /// Adaptor for the resume functionality for data transfer between Azure Table Storage /// public class AzureTableResumptionAdaptor : IDataTransferResumptionAdapter { private readonly string _fileFullPath; /// - /// + /// Constructor /// - /// + /// The name of the checkpoint file public AzureTableResumptionAdaptor(string fileName) { Guard.NotEmpty(nameof(fileName), fileName); @@ -34,9 +34,9 @@ public AzureTableResumptionAdaptor(string fileName) } /// - /// + /// Get the checkpoint from the file /// - /// + /// The checkpoint public AzureTablePrimaryKey GetCheckpoint() { if (File.Exists(_fileFullPath)) @@ -48,9 +48,9 @@ public AzureTablePrimaryKey GetCheckpoint() } /// - /// + /// Save the checkpoint to the file /// - /// + /// The checkpoint to store public void SaveCheckpoint(AzureTablePrimaryKey checkpoint) { Guard.NotNull(nameof(checkpoint), checkpoint); diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs index 8026e10..a843ad1 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Source/AzureTableSourceAdapter.cs @@ -1,14 +1,13 @@ -using Microsoft.Azure.CosmosDB.Table; -using Microsoft.Azure.Storage; -using Microsoft.DataTransfer.AzureTable.Client; -using Microsoft.DataTransfer.AzureTable.Resumption; -using Microsoft.DataTransfer.Basics; -using Microsoft.DataTransfer.Extensibility; using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.CosmosDB.Table; +using Microsoft.Azure.Storage; using Microsoft.Azure.Storage.RetryPolicies; +using Microsoft.DataTransfer.AzureTable.Client; +using Microsoft.DataTransfer.AzureTable.Resumption; +using Microsoft.DataTransfer.Extensibility; namespace Microsoft.DataTransfer.AzureTable.Source { diff --git a/DataTransfer.sln b/DataTransfer.sln index ff274e0..869b9d2 100644 --- a/DataTransfer.sln +++ b/DataTransfer.sln @@ -150,6 +150,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DataTransfer.Basi EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DataTransfer.MongoDb.UnitTests", "MongoDb\Microsoft.DataTransfer.MongoDb.UnitTests\Microsoft.DataTransfer.MongoDb.UnitTests.csproj", "{2AC0E216-9ED4-4D1D-A264-73905A37F6AA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DataTransfer.AzureTable.UnitTests", "AzureTable\Microsoft.DataTransfer.AzureTable.UnitTests\Microsoft.DataTransfer.AzureTable.UnitTests.csproj", "{428B3874-8780-4939-AC5A-79C3980E01C7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -356,18 +358,10 @@ Global {2AC0E216-9ED4-4D1D-A264-73905A37F6AA}.Debug|x64.Build.0 = Debug|x64 {2AC0E216-9ED4-4D1D-A264-73905A37F6AA}.Release|x64.ActiveCfg = Release|x64 {2AC0E216-9ED4-4D1D-A264-73905A37F6AA}.Release|x64.Build.0 = Release|x64 - {3C508C20-FC36-439A-AD68-5FE2170172D0}.Debug|x64.ActiveCfg = Debug|x64 - {3C508C20-FC36-439A-AD68-5FE2170172D0}.Debug|x64.Build.0 = Debug|x64 - {3C508C20-FC36-439A-AD68-5FE2170172D0}.Release|x64.ActiveCfg = Release|x64 - {3C508C20-FC36-439A-AD68-5FE2170172D0}.Release|x64.Build.0 = Release|x64 - {152DEF82-24B7-46B8-A6E4-09F90D5332B1}.Debug|x64.ActiveCfg = Debug|x64 - {152DEF82-24B7-46B8-A6E4-09F90D5332B1}.Debug|x64.Build.0 = Debug|x64 - {152DEF82-24B7-46B8-A6E4-09F90D5332B1}.Release|x64.ActiveCfg = Release|x64 - {152DEF82-24B7-46B8-A6E4-09F90D5332B1}.Release|x64.Build.0 = Release|x64 - {A5013A12-18EA-40B6-83E0-4A59DC1B9E74}.Debug|x64.ActiveCfg = Debug|x64 - {A5013A12-18EA-40B6-83E0-4A59DC1B9E74}.Debug|x64.Build.0 = Debug|x64 - {A5013A12-18EA-40B6-83E0-4A59DC1B9E74}.Release|x64.ActiveCfg = Release|x64 - {A5013A12-18EA-40B6-83E0-4A59DC1B9E74}.Release|x64.Build.0 = Release|x64 + {428B3874-8780-4939-AC5A-79C3980E01C7}.Debug|x64.ActiveCfg = Debug|x64 + {428B3874-8780-4939-AC5A-79C3980E01C7}.Debug|x64.Build.0 = Debug|x64 + {428B3874-8780-4939-AC5A-79C3980E01C7}.Release|x64.ActiveCfg = Release|x64 + {428B3874-8780-4939-AC5A-79C3980E01C7}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -423,9 +417,10 @@ Global {6544F78F-4EE3-489E-87B7-5FCA9C4D50BD} = {1BD0D669-8E45-4E7C-A20F-707A1887E8ED} {DA182D5C-79F4-4AF6-BF15-6E4496353A6A} = {F9CAC1F5-436E-4406-BACC-FC18C8FE36C5} {2AC0E216-9ED4-4D1D-A264-73905A37F6AA} = {7F83D352-1039-4B8F-B63C-56231421056A} + {428B3874-8780-4939-AC5A-79C3980E01C7} = {7FF55CC5-9069-49E5-B16E-6437AE4892A7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {D06EC8A2-02FC-48D4-BC6B-D86A26FED0CC} EnterpriseLibraryConfigurationToolBinariesPathV6 = packages\EnterpriseLibrary.TransientFaultHandling.6.0.1304.0\lib\portable-net45+win+wp8;packages\EnterpriseLibrary.TransientFaultHandling.Data.6.0.1304.1\lib\NET45 + SolutionGuid = {D06EC8A2-02FC-48D4-BC6B-D86A26FED0CC} EndGlobalSection EndGlobal From f963fba273e076116df2b2da532eac45e52cdf55 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Wed, 20 Feb 2019 15:35:12 -0800 Subject: [PATCH 05/11] Address comments --- ...t.DataTransfer.AzureTable.UnitTests.csproj | 5 ++++- .../Properties/AssemblyInfo.cs | 10 ---------- .../packages.config | 2 ++ .../ContinuationTokenParser.cs | 19 ++++++++++--------- .../Resumption/AzureTablePrimaryKey.cs | 2 +- .../Resumption/AzureTableResumptionAdaptor.cs | 10 ++++------ .../IDataTransferResumptionAdapter.cs | 12 ++++++++++++ 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj index 2d010d0..100e2a2 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Microsoft.DataTransfer.AzureTable.UnitTests.csproj @@ -54,7 +54,7 @@ - + @@ -68,6 +68,9 @@ + + Properties\CommonAssemblyInfo.cs + diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs index 001fcbd..d46d5e1 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/Properties/AssemblyInfo.cs @@ -4,17 +4,7 @@ [assembly: AssemblyTitle("Microsoft.DataTransfer.AzureTable.UnitTests")] [assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] [assembly: AssemblyProduct("Microsoft.DataTransfer.AzureTable.UnitTests")] -[assembly: AssemblyCopyright("Copyright © 2019")] -[assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: ComVisible(false)] - [assembly: Guid("428b3874-8780-4939-ac5a-79c3980e01c7")] - -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config index e5ca672..bcf1b2a 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.UnitTests/packages.config @@ -2,4 +2,6 @@ + + \ No newline at end of file diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs index 17dfe3e..9c8fff9 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs @@ -6,25 +6,28 @@ namespace Microsoft.DataTransfer.AzureTable /// /// The class which is used to encode the continuation token /// - public class ContinuationTokenParser + public static class ContinuationTokenParser { + private const char ExclamationDelimiter = '!'; + /// - /// Encode continuation token to fomart: (Version)!(TokenLength)!(CustomBase64EncodedToken) + /// Generates the encoded continuation token with fomart (Version)!(TokenLength)!(CustomBase64EncodedToken) /// + /// The string that you want to encode into continuation token + /// The encoded continuation token public static string EncodeContinuationToken(string key) { StringBuilder encodedContinuationToken = new StringBuilder(); // Version of the ContinuationToken encodedContinuationToken.Append(1); - encodedContinuationToken.Append(exclamationDelimiter); + encodedContinuationToken.Append(ExclamationDelimiter); - UTF8Encoding utf8Encoding = new UTF8Encoding(); - string base64EncodedToken = Convert.ToBase64String(utf8Encoding.GetBytes(key.ToString())); + string base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes(key.ToString())); string customBase64EncodedString = UrlCustomEscapeBase64String(base64EncodedToken); - //Size is the lenght of base64 encoded key + // Size is the lenght of base64 encoded key encodedContinuationToken.Append(customBase64EncodedString.Length); - encodedContinuationToken.Append(exclamationDelimiter); + encodedContinuationToken.Append(ExclamationDelimiter); encodedContinuationToken.Append(customBase64EncodedString); return encodedContinuationToken.ToString(); @@ -51,7 +54,5 @@ private static char TranslateChar(char c) default: return c; } } - - private const char exclamationDelimiter = '!'; } } diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs index aba42c6..2776b51 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTablePrimaryKey.cs @@ -7,7 +7,7 @@ namespace Microsoft.DataTransfer.AzureTable.Resumption { /// - /// Define the checkpoint for AzureTable data transfer + /// Define the checkpoint for Azure Table data transfer /// public class AzureTablePrimaryKey { diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs index 2af375f..6a92bb0 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs @@ -12,9 +12,10 @@ namespace Microsoft.DataTransfer.AzureTable.Resumption public class AzureTableResumptionAdaptor : IDataTransferResumptionAdapter { private readonly string _fileFullPath; + private const string _folderName = "resume_checkpoint"; /// - /// Constructor + /// Create an instance of /// /// The name of the checkpoint file public AzureTableResumptionAdaptor(string fileName) @@ -26,11 +27,8 @@ public AzureTableResumptionAdaptor(string fileName) throw new ArgumentException("File name contains invalid characters."); } - var localAppDataFolder = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); - var folderName = Path.Combine(localAppDataFolder, "dt"); - Directory.CreateDirectory(folderName); - - _fileFullPath = Path.Combine(folderName, fileName); + Directory.CreateDirectory(_folderName); + _fileFullPath = Path.Combine(_folderName, fileName); } /// diff --git a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs index eb4d067..9717359 100644 --- a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs +++ b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs @@ -6,10 +6,22 @@ namespace Microsoft.DataTransfer.Extensibility { + /// + /// The interface for the resumption adapters in data transfer + /// + /// The type of checkpoint that you want to create public interface IDataTransferResumptionAdapter { + /// + /// Save the resumption checkpoint + /// + /// The checkpoint that you want to save void SaveCheckpoint(TCheckpoint checkpoint); + /// + /// Get the resumption checkpoint + /// + /// The checkpoint that you saved in the last run of data transfer action TCheckpoint GetCheckpoint(); } } From 1e5a20be95de12d614d961e9916fb58ea5fb2769 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Wed, 20 Feb 2019 17:59:53 -0800 Subject: [PATCH 06/11] Remove extra stringbuilder object in ContinuationTokenParser --- .../ContinuationTokenParser.cs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs index 9c8fff9..8f0bbea 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/ContinuationTokenParser.cs @@ -23,25 +23,17 @@ public static string EncodeContinuationToken(string key) encodedContinuationToken.Append(ExclamationDelimiter); string base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes(key.ToString())); - string customBase64EncodedString = UrlCustomEscapeBase64String(base64EncodedToken); // Size is the lenght of base64 encoded key - encodedContinuationToken.Append(customBase64EncodedString.Length); + encodedContinuationToken.Append(base64EncodedToken.Length); encodedContinuationToken.Append(ExclamationDelimiter); - encodedContinuationToken.Append(customBase64EncodedString); - return encodedContinuationToken.ToString(); - } - - private static string UrlCustomEscapeBase64String(string token) - { - StringBuilder escapedString = new StringBuilder(); - foreach (char c in token.ToCharArray()) + foreach (char c in base64EncodedToken.ToCharArray()) { - escapedString.Append(TranslateChar(c)); + encodedContinuationToken.Append(TranslateChar(c)); } - return escapedString.ToString(); + return encodedContinuationToken.ToString(); } private static char TranslateChar(char c) From 2bbb56c1d17ac66de6daaa5ac791c2c2fd8ed0a7 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Wed, 27 Feb 2019 16:55:14 -0800 Subject: [PATCH 07/11] Add functional test for resume functionality --- .../AzureTableDataSourceAdapterTests.cs | 43 ++++++++++++++++++- .../AzureTableDataTransferContextMock.cs | 15 +++++++ .../AzureTableHelper.cs | 2 +- ...Transfer.AzureTable.FunctionalTests.csproj | 1 + .../Resumption/AzureTableResumptionAdaptor.cs | 11 +++++ .../Sink/TableAPIBulkSinkAdapter.cs | 2 +- .../IDataTransferResumptionAdapter.cs | 5 +++ 7 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataTransferContextMock.cs diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataSourceAdapterTests.cs b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataSourceAdapterTests.cs index ab1822e..518e2ed 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataSourceAdapterTests.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataSourceAdapterTests.cs @@ -1,4 +1,5 @@ -using Microsoft.DataTransfer.AzureTable.Source; +using Microsoft.DataTransfer.AzureTable.Resumption; +using Microsoft.DataTransfer.AzureTable.Source; using Microsoft.DataTransfer.Extensibility; using Microsoft.DataTransfer.TestsCommon; using Microsoft.DataTransfer.TestsCommon.Mocks; @@ -114,6 +115,46 @@ public async Task ReadEntitiesWithAllInternalFields_AllInternalPropertiesRead() await ReadAndVerifyFields(configuration, new[] { "RowKey", "PartitionKey", "Timestamp" }); } + [TestMethod, Timeout(120000)] + public async Task ResumeFunctionality_ReadFromTheCheckpoint() + { + var configuration = Mocks + .Of(c => + c.ConnectionString == Settings.AzureStorageConnectionString && + c.Table == tableName && + c.InternalFields == AzureTableInternalFields.None) + .First(); + + var checkpointItem = sampleData[100]; + var resumptionAdapter = new AzureTableResumptionAdaptor("checkpoint.json"); + var checkpoint = new AzureTablePrimaryKey() + { + PartitionKey = "", + RowKey = checkpointItem["id"].ToString() + }; + resumptionAdapter.SaveCheckpoint(checkpoint); + + var dataContext = new AzureTableDataTransferContextMock() + { + SinkName = "TestSink", + SourceName = "TestSource", + EnableResumeFunction = true, + RunConfigSignature = "checkpoint" + }; + + using (var adapter = await new AzureTableSourceAdapterFactory() + .CreateAsync(configuration, dataContext, CancellationToken.None)) + { + var readOutput = new ReadOutputByRef(); + var dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None); + + foreach(var field in dataItem.GetFieldNames()) + { + Assert.AreEqual(checkpointItem[field].ToString(), dataItem.GetValue(field).ToString()); + } + } + } + private async Task> ReadData(IAzureTableSourceAdapterConfiguration configuration) { using (var adapter = await new AzureTableSourceAdapterFactory() diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataTransferContextMock.cs b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataTransferContextMock.cs new file mode 100644 index 0000000..fe8ddb8 --- /dev/null +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableDataTransferContextMock.cs @@ -0,0 +1,15 @@ +using Microsoft.DataTransfer.Extensibility; + +namespace Microsoft.DataTransfer.AzureTable.FunctionalTests +{ + sealed class AzureTableDataTransferContextMock : IDataTransferContext + { + public string SourceName { get; set; } + + public string SinkName { get; set; } + + public string RunConfigSignature { get; set; } + + public bool EnableResumeFunction { get; set; } + } +} diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableHelper.cs b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableHelper.cs index 22668b5..9198144 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableHelper.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/AzureTableHelper.cs @@ -15,7 +15,7 @@ public static void CreateTable(string connectionString, string tableName, IReadO TableBatchOperation batch = new TableBatchOperation(); foreach (var entity in data) { - batch.Insert(new DictionaryTableEntity(Guid.NewGuid().ToString(), entity)); + batch.Insert(new DictionaryTableEntity(entity["id"].ToString(), entity)); if (batch.Count >= 100) { diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/Microsoft.DataTransfer.AzureTable.FunctionalTests.csproj b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/Microsoft.DataTransfer.AzureTable.FunctionalTests.csproj index 9137ac8..c59ec66 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/Microsoft.DataTransfer.AzureTable.FunctionalTests.csproj +++ b/AzureTable/Microsoft.DataTransfer.AzureTable.FunctionalTests/Microsoft.DataTransfer.AzureTable.FunctionalTests.csproj @@ -116,6 +116,7 @@ Properties\CommonAssemblyInfo.cs + diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs index 6a92bb0..0386fc5 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs @@ -59,5 +59,16 @@ public void SaveCheckpoint(AzureTablePrimaryKey checkpoint) serializer.Serialize(file, checkpoint); } } + + /// + /// Delete the file which stores the checkpoint + /// + public void DeleteCheckpoint() + { + if (File.Exists(_fileFullPath)) + { + File.Delete(_fileFullPath); + } + } } } diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs index cfc3947..eb6894b 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Sink/TableAPIBulkSinkAdapter.cs @@ -80,7 +80,7 @@ public async Task InitializeAsync(CancellationToken cancellation) public async Task WriteAsync(IDataItem dataItem, CancellationToken cancellation) { var item = GetITableEntityFromIDataItem(dataItem); - if (dict.Count == 0) + if (dict.Count == 0 && !cancellation.IsCancellationRequested) { _resumptionAdapter?.SaveCheckpoint( new AzureTablePrimaryKey { PartitionKey = item.PartitionKey, RowKey = item.RowKey }); diff --git a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs index 9717359..b45ba5d 100644 --- a/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs +++ b/Core/Microsoft.DataTransfer.Extensibility/IDataTransferResumptionAdapter.cs @@ -23,5 +23,10 @@ public interface IDataTransferResumptionAdapter /// /// The checkpoint that you saved in the last run of data transfer action TCheckpoint GetCheckpoint(); + + /// + /// Cleanup the checkpoint file if needed + /// + void DeleteCheckpoint(); } } From 6d07e052fdf0d26c0c3826fa9311ec28697a923f Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Fri, 1 Mar 2019 16:34:58 -0800 Subject: [PATCH 08/11] Move the config into statistics --- .../Handlers/OneTimeDataTransferHandler.cs | 6 ++---- .../IInfrastructureConfiguration.cs | 2 +- .../ITransferCommonConfiguration.cs | 21 ------------------- ...Microsoft.DataTransfer.ServiceModel.csproj | 1 - .../ITransferStatisticsConfiguration.cs | 6 ++++++ TestSettings/Local/TestSettings.xml.template | 10 --------- .../InfrastructureConfiguration.cs | 2 ++ 7 files changed, 11 insertions(+), 37 deletions(-) delete mode 100644 Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs delete mode 100644 TestSettings/Local/TestSettings.xml.template diff --git a/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs b/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs index b2ae32d..efcafe9 100644 --- a/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs +++ b/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs @@ -16,19 +16,17 @@ sealed class OneTimeDataTransferHandler : ITransferHandler private readonly IDataAdapterConfigurationFactory dataAdapterConfiguration; private readonly ITransferStatisticsHandler statisticsHandler; private readonly ITransferStatisticsConfiguration statisticsConfiguration; - private readonly ITransferCommonConfiguration commonConfiguration; private readonly IOneTimeDataTransferConfiguration configuration; public OneTimeDataTransferHandler(IDataTransferService transferService, IDataAdapterConfigurationFactory dataAdapterConfiguration, ITransferStatisticsHandler statisticsHandler, ITransferStatisticsConfiguration statisticsConfiguration, - IOneTimeDataTransferConfiguration configuration, ITransferCommonConfiguration commonConfiguration) + IOneTimeDataTransferConfiguration configuration) { this.transferService = transferService; this.dataAdapterConfiguration = dataAdapterConfiguration; this.statisticsHandler = statisticsHandler; this.statisticsConfiguration = statisticsConfiguration; - this.commonConfiguration = commonConfiguration; this.configuration = configuration; } @@ -64,7 +62,7 @@ await transferService statistics, // Allow cancellation cancellation.Token, - commonConfiguration.EnableResumeFunction); + statisticsConfiguration.EnableResumeFunction); } } diff --git a/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs b/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs index b0f515f..59c6640 100644 --- a/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs +++ b/Console/Microsoft.DataTransfer.ConsoleHost/Configuration/IInfrastructureConfiguration.cs @@ -11,6 +11,6 @@ namespace Microsoft.DataTransfer.ConsoleHost.Configuration /// This needs to be public to allow automatic proxy class generation. /// public interface IInfrastructureConfiguration : - ITransferStatisticsConfiguration, IErrorDetailsConfiguration, ITransferCommonConfiguration + ITransferStatisticsConfiguration, IErrorDetailsConfiguration { } } diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs b/Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs deleted file mode 100644 index f38121d..0000000 --- a/Core/Microsoft.DataTransfer.ServiceModel/ITransferCommonConfiguration.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.DataTransfer.ServiceModel -{ - /// - /// The common configuration for data transfer - /// - public interface ITransferCommonConfiguration - { - /// - /// Whether to allow saving or resuming from a checkpoint - /// - [Display(ResourceType = typeof(ConfigurationResources), Description = "Statistics_EnableResumeFunction")] - bool EnableResumeFunction { get; } - } -} diff --git a/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj b/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj index 90df1ed..3a5daf5 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj +++ b/Core/Microsoft.DataTransfer.ServiceModel/Microsoft.DataTransfer.ServiceModel.csproj @@ -56,7 +56,6 @@ - diff --git a/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs b/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs index 94da7ac..67a2612 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs +++ b/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs @@ -25,5 +25,11 @@ public interface ITransferStatisticsConfiguration /// [Display(ResourceType = typeof(DynamicConfigurationResources), Description = "Statistics_ProgressUpdateInterval")] TimeSpan? ProgressUpdateInterval { get; } + + /// + /// Whether to allow saving or resuming from a checkpoint + /// + [Display(ResourceType = typeof(ConfigurationResources), Description = "Statistics_EnableResumeFunction")] + bool EnableResumeFunction { get; } } } diff --git a/TestSettings/Local/TestSettings.xml.template b/TestSettings/Local/TestSettings.xml.template deleted file mode 100644 index eb1498f..0000000 --- a/TestSettings/Local/TestSettings.xml.template +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs b/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs index c8aee84..7c1126d 100644 --- a/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs +++ b/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs @@ -35,6 +35,8 @@ public TimeSpan? ProgressUpdateInterval set { SetProperty(ref progressUpdateInterval, value); } } + public bool EnableResumeFunction => false; + public InfrastructureConfiguration() { ErrorDetails = InfrastructureDefaults.Current.ErrorDetails; From 668be3d2536cba16e838517825aed8bb1917660b Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Thu, 7 Mar 2019 14:28:52 -0800 Subject: [PATCH 09/11] Revert changes to TestSetting template --- TestSettings/Local/TestSettings.xml.template | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 TestSettings/Local/TestSettings.xml.template diff --git a/TestSettings/Local/TestSettings.xml.template b/TestSettings/Local/TestSettings.xml.template new file mode 100644 index 0000000..eb1498f --- /dev/null +++ b/TestSettings/Local/TestSettings.xml.template @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file From 7683400ff9f8639580688205c3124b5a08aaf5a3 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Tue, 12 Mar 2019 09:58:50 -0700 Subject: [PATCH 10/11] Update the description of the cmd line parameter --- .../Resumption/AzureTableResumptionAdaptor.cs | 2 +- .../ConfigurationResources.Designer.cs | 2 +- .../ConfigurationResources.resx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs index 0386fc5..46cc2a4 100644 --- a/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs +++ b/AzureTable/Microsoft.DataTransfer.AzureTable/Resumption/AzureTableResumptionAdaptor.cs @@ -12,7 +12,7 @@ namespace Microsoft.DataTransfer.AzureTable.Resumption public class AzureTableResumptionAdaptor : IDataTransferResumptionAdapter { private readonly string _fileFullPath; - private const string _folderName = "resume_checkpoint"; + private const string _folderName = "ResumeCheckpoint"; /// /// Create an instance of diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs index e0a3427..da1b758 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs +++ b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs @@ -88,7 +88,7 @@ public static string Statistics_EnableCosmosTableLog { } /// - /// Looks up a localized string similar to Optional. Whether to save the checkpoint file and resume from there if the data transfer action is stopped for some reason. + /// Looks up a localized string similar to Optional. Allows to save checkpoint files in folder ./ResumeCheckpoint and resume from there the next time you run with the same source and target configuration. /// public static string Statistics_EnableResumeFunction { get { diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx index 0e60d9b..39be8b4 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx +++ b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx @@ -136,6 +136,6 @@ Optional, default is {0}. Time interval to refresh on-screen data transfer progress - Optional. Whether to save the checkpoint file and resume from there if the data transfer action is stopped for some reason + Optional. Allows to save checkpoint files in folder ./ResumeCheckpoint and resume from there the next time you run with the same source and target configuration \ No newline at end of file From d7f0ba60825781cf6b24118071730085835d4959 Mon Sep 17 00:00:00 2001 From: Dongyang Cheng Date: Tue, 12 Mar 2019 10:27:44 -0700 Subject: [PATCH 11/11] Update the name of resume functionality cmd config parameter --- .../App/Handlers/OneTimeDataTransferHandler.cs | 2 +- .../ConfigurationResources.Designer.cs | 4 ++-- .../ConfigurationResources.resx | 2 +- .../Statistics/ITransferStatisticsConfiguration.cs | 4 ++-- .../Steps/InfrastructureSetup/InfrastructureConfiguration.cs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs b/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs index 966c833..74cec9b 100644 --- a/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs +++ b/Console/Microsoft.DataTransfer.ConsoleHost/App/Handlers/OneTimeDataTransferHandler.cs @@ -61,7 +61,7 @@ await transferService statistics, // Allow cancellation cancellation.Token, - statisticsConfiguration.EnableResumeFunction); + statisticsConfiguration.EnableResumption); } } diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs index da1b758..cb018b5 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs +++ b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.Designer.cs @@ -90,9 +90,9 @@ public static string Statistics_EnableCosmosTableLog { /// /// Looks up a localized string similar to Optional. Allows to save checkpoint files in folder ./ResumeCheckpoint and resume from there the next time you run with the same source and target configuration. /// - public static string Statistics_EnableResumeFunction { + public static string Statistics_EnableResumption { get { - return ResourceManager.GetString("Statistics_EnableResumeFunction", resourceCulture); + return ResourceManager.GetString("Statistics_EnableResumption", resourceCulture); } } diff --git a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx index 39be8b4..dadb6b2 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx +++ b/Core/Microsoft.DataTransfer.ServiceModel/ConfigurationResources.resx @@ -135,7 +135,7 @@ Optional, default is {0}. Time interval to refresh on-screen data transfer progress - + Optional. Allows to save checkpoint files in folder ./ResumeCheckpoint and resume from there the next time you run with the same source and target configuration \ No newline at end of file diff --git a/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs b/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs index f76beb6..4f22f7c 100644 --- a/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs +++ b/Core/Microsoft.DataTransfer.ServiceModel/Statistics/ITransferStatisticsConfiguration.cs @@ -41,7 +41,7 @@ public interface ITransferStatisticsConfiguration /// /// Whether to allow saving or resuming from a checkpoint /// - [Display(ResourceType = typeof(ConfigurationResources), Description = "Statistics_EnableResumeFunction")] - bool EnableResumeFunction { get; } + [Display(ResourceType = typeof(ConfigurationResources), Description = "Statistics_EnableResumption")] + bool EnableResumption { get; } } } diff --git a/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs b/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs index 6e5e728..8bfcd62 100644 --- a/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs +++ b/Wpf/Microsoft.DataTransfer.WpfHost/Steps/InfrastructureSetup/InfrastructureConfiguration.cs @@ -36,7 +36,7 @@ public TimeSpan? ProgressUpdateInterval set { SetProperty(ref progressUpdateInterval, value); } } - public bool EnableResumeFunction => false; + public bool EnableResumption => false; public bool EnableCosmosTableLog {