diff --git a/.gitignore b/.gitignore
index 16569ab..1e9a901 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
######################
*.suo
*.user
+.vs
# VS File #
###########
diff --git a/Madd0.AzureStorageDriver/ConnectionDialog.xaml b/Madd0.AzureStorageDriver/ConnectionDialog.xaml
index 9de7b89..6286da3 100644
--- a/Madd0.AzureStorageDriver/ConnectionDialog.xaml
+++ b/Madd0.AzureStorageDriver/ConnectionDialog.xaml
@@ -80,6 +80,50 @@
Text="?" />
+
+ Schema loading parallelism:
+
+
+
+
+ The number of concurrent calls that will be made to Azure Table Storage to determine the columns available for each table.
+
+
+
+
+
+
+
+ Table rollover date format:
+
+
+
+
+ The date format that is appended to table names in table rollover. This is used to reduce the number of azure tables that need to be inspected to load schema information.
+
+
+
+
+
+
diff --git a/Madd0.AzureStorageDriver/Model/StorageAccountProperties.cs b/Madd0.AzureStorageDriver/Model/StorageAccountProperties.cs
index 43d93cd..1603037 100644
--- a/Madd0.AzureStorageDriver/Model/StorageAccountProperties.cs
+++ b/Madd0.AzureStorageDriver/Model/StorageAccountProperties.cs
@@ -135,6 +135,34 @@ public int NumberOfRows
set { this._driverData.SetElementValue("NumberOfRows", value); }
}
+ ///
+ /// Returns the maximum number of parallel model loading
+ /// operations can occur when loading schema for the azure table storage tables
+ ///
+ public int ModelLoadMaxParallelism
+ {
+ get { return (int?)this._driverData.Element("ModelLoadMaxParallelism") ??
+ (System.Environment.ProcessorCount * 2); }
+ set
+ {
+ this._driverData.SetElementValue("ModelLoadMaxParallelism", value);
+ }
+ }
+
+ ///
+ /// A date format which is table names can potentially end in
+ /// for table rollover. This prevents loading duplicate schemas
+ /// for rollover tables
+ ///
+ public string TableRolloverDateFormat
+ {
+ get { return (string)this._driverData.Element("TableRolloverDateFormat") ?? "yyyyMMdd"; }
+ set
+ {
+ this._driverData.SetElementValue("TableRolloverDateFormat", value);
+ }
+ }
+
///
/// Gets a instace for the current connection.
///
diff --git a/Madd0.AzureStorageDriver/SchemaBuilder.cs b/Madd0.AzureStorageDriver/SchemaBuilder.cs
index b6ec952..de62d81 100644
--- a/Madd0.AzureStorageDriver/SchemaBuilder.cs
+++ b/Madd0.AzureStorageDriver/SchemaBuilder.cs
@@ -12,12 +12,16 @@ namespace Madd0.AzureStorageDriver
using System.Collections.Generic;
using System.IO;
using System.Linq;
+ using System.Net;
using System.Reflection;
+ using System.Threading.Tasks;
using LINQPad.Extensibility.DataContext;
using Madd0.AzureStorageDriver.Properties;
using Microsoft.CSharp;
using Microsoft.WindowsAzure.Storage.Table;
+
+
///
/// Provides the methods necessary to determining the storage account's schema and to building
/// the typed data context .
@@ -62,7 +66,13 @@ public static List GetSchemaAndBuildAssembly(StorageAccountPropert
/// storage model.
private static IEnumerable GetModel(StorageAccountProperties properties)
{
+ // make sure that we can make at least ModelLoadMaxParallelism concurrent
+ // cals to azure table storage
+ ServicePointManager.DefaultConnectionLimit = properties.ModelLoadMaxParallelism;
+
var tableClient = properties.GetStorageAccount().CreateCloudTableClient();
+
+ string rolloverFormat = properties.TableRolloverDateFormat;
// First get a list of all tables
var model = (from tableName in tableClient.ListTables()
@@ -71,18 +81,42 @@ private static IEnumerable GetModel(StorageAccountProperties propert
Name = tableName.Name
}).ToList();
- // Then go through them
- foreach (var table in model)
+ var schemas = model
+ .GroupBy(table =>
+ {
+ string schemaName = table.Name;
+ DateTime rollover;
+ for (int i = table.Name.Length - 1; i > 0; i--)
+ {
+ string tail = schemaName.Substring(i);
+ if(DateTime.TryParseExact(tail, rolloverFormat, null, System.Globalization.DateTimeStyles.None, out rollover))
+ {
+ schemaName = schemaName.Substring(0, i);
+ break;
+ }
+ }
+ return schemaName;
+ })
+ .ToList();
+
+ var options = new ParallelOptions()
+ {
+ MaxDegreeOfParallelism = properties.ModelLoadMaxParallelism
+ };
+
+ Parallel.ForEach(schemas, options, group =>
{
- var tableColumns = tableClient.GetTableReference(table.Name).ExecuteQuery(new TableQuery().Take(properties.NumberOfRows))
+ var threadTableClient = properties.GetStorageAccount().CreateCloudTableClient();
+
+ var tableColumns = threadTableClient.GetTableReference(group.Last().Name).ExecuteQuery(new TableQuery().Take(properties.NumberOfRows))
.SelectMany(row => row.Properties)
.GroupBy(column => column.Key)
.Select(grp => new TableColumn
- {
- Name = grp.Key,
- TypeName = GetType(grp.First().Value.PropertyType)
- });
-
+ {
+ Name = grp.Key,
+ TypeName = GetType(grp.First().Value.PropertyType)
+ });
+
var baseColumns = new List
{
new TableColumn { Name = "PartitionKey", TypeName = GetType(EdmType.String) },
@@ -91,9 +125,12 @@ private static IEnumerable GetModel(StorageAccountProperties propert
new TableColumn { Name = "ETag", TypeName = GetType(EdmType.String) }
};
- table.Columns = tableColumns.Concat(baseColumns).ToArray();
- }
-
+ foreach(var table in group)
+ {
+ table.Columns = tableColumns.Concat(baseColumns).ToArray();
+ }
+ });
+
return model;
}