Enable date rollover pruning

.gitignore

@@ -2,6 +2,7 @@
 ######################
 *.suo
 *.user
+.vs
 
 # VS File #
 ###########

Madd0.AzureStorageDriver/ConnectionDialog.xaml

@@ -80,6 +80,50 @@
                                    Text="?" />
                     </Grid>
                 </StackPanel>
+                <StackPanel Margin="0 10" Orientation="Horizontal">
+                    <TextBlock>Schema loading parallelism:</TextBlock>
+                    <TextBox Width="50"
+                             Margin="5 0"
+                             Text="{Binding ModelLoadMaxParallelism}"
+                             TextAlignment="Right" />
+                    <Grid>
+                        <Grid.ToolTip>
+                            <ToolTip>
+                                <TextBlock>The number of concurrent calls that will be made to Azure Table Storage to determine the columns available for each table.</TextBlock>
+                            </ToolTip>
+                        </Grid.ToolTip>
+                        <Ellipse Width="15"
+                                 Height="15"
+                                 Fill="#3CA1C8" />
+                        <TextBlock Margin="0 1 0 0"
+                                   HorizontalAlignment="Center"
+                                   FontWeight="Bold"
+                                   Foreground="White"
+                                   Text="?" />
+                    </Grid>
+                </StackPanel>
+                <StackPanel Margin="0 10" Orientation="Horizontal">
+                    <TextBlock>Table rollover date format:</TextBlock>
+                    <TextBox Width="100"
+                             Margin="5 0"
+                             Text="{Binding TableRolloverDateFormat}"
+                             TextAlignment="Left" />
+                    <Grid>
+                        <Grid.ToolTip>
+                            <ToolTip>
+                                <TextBlock>The date format that is appended to table names in table rollover. This is used to reduce the number of azure tables that need to be inspected to load schema information.</TextBlock>
+                            </ToolTip>
+                        </Grid.ToolTip>
+                        <Ellipse Width="15"
+                                 Height="15"
+                                 Fill="#3CA1C8" />
+                        <TextBlock Margin="0 1 0 0"
+                                   HorizontalAlignment="Center"
+                                   FontWeight="Bold"
+                                   Foreground="White"
+                                   Text="?" />
+                    </Grid>
+                </StackPanel>
             </StackPanel>
         </Expander>
 

Madd0.AzureStorageDriver/Model/StorageAccountProperties.cs

@@ -135,6 +135,34 @@ public int NumberOfRows
             set { this._driverData.SetElementValue("NumberOfRows", value); }
         }
 
+        /// <summary>
+        /// Returns the maximum number of parallel model loading
+        /// operations can occur when loading schema for the azure table storage tables
+        /// </summary>
+        public int ModelLoadMaxParallelism
+        {
+            get { return (int?)this._driverData.Element("ModelLoadMaxParallelism") ??
+                    (System.Environment.ProcessorCount * 2); }
+            set
+            {
+                this._driverData.SetElementValue("ModelLoadMaxParallelism", value);
+            }
+        }
+
+        /// <summary>
+        /// A date format which is table names can potentially end in
+        /// for table rollover. This prevents loading duplicate schemas
+        /// for rollover tables
+        /// </summary>
+        public string TableRolloverDateFormat
+        {
+            get { return (string)this._driverData.Element("TableRolloverDateFormat") ?? "yyyyMMdd"; }
+            set
+            {                
+                this._driverData.SetElementValue("TableRolloverDateFormat", value);
+            }
+        }
+
         /// <summary>
         /// Gets a <see cref="CloudStorageAccount"/> instace for the current connection.
         /// </summary>

Madd0.AzureStorageDriver/SchemaBuilder.cs

@@ -12,12 +12,16 @@ namespace Madd0.AzureStorageDriver
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Reflection;
+    using System.Threading.Tasks;
     using LINQPad.Extensibility.DataContext;
     using Madd0.AzureStorageDriver.Properties;
     using Microsoft.CSharp;
     using Microsoft.WindowsAzure.Storage.Table;
 
+
+
     /// <summary>
     /// Provides the methods necessary to determining the storage account's schema and to building 
     /// the typed data context .
@@ -62,7 +66,13 @@ public static List<ExplorerItem> GetSchemaAndBuildAssembly(StorageAccountPropert
         /// storage model.</returns>
         private static IEnumerable<CloudTable> GetModel(StorageAccountProperties properties)
         {
+            // make sure that we can make at least ModelLoadMaxParallelism concurrent
+            // cals to azure table storage
+            ServicePointManager.DefaultConnectionLimit = properties.ModelLoadMaxParallelism;
+
             var tableClient = properties.GetStorageAccount().CreateCloudTableClient();
+
+            string rolloverFormat = properties.TableRolloverDateFormat;
 
             // First get a list of all tables
             var model = (from tableName in tableClient.ListTables()
@@ -71,18 +81,42 @@ private static IEnumerable<CloudTable> GetModel(StorageAccountProperties propert
                              Name = tableName.Name
                          }).ToList();
 
-            // Then go through them
-            foreach (var table in model)
+            var schemas = model
+                .GroupBy(table =>
+                {
+                    string schemaName = table.Name;
+                    DateTime rollover;
+                    for (int i = table.Name.Length - 1; i > 0; i--)
+                    {
+                        string tail = schemaName.Substring(i);
+                        if(DateTime.TryParseExact(tail, rolloverFormat, null, System.Globalization.DateTimeStyles.None, out rollover))
+                        {
+                            schemaName = schemaName.Substring(0, i);
+                            break;
+                        }
+                    }
+                    return schemaName;
+                })
+                .ToList();
+
+            var options = new ParallelOptions()
+            {
+                MaxDegreeOfParallelism = properties.ModelLoadMaxParallelism
+            };
+
+            Parallel.ForEach(schemas, options, group =>
             {
-                var tableColumns = tableClient.GetTableReference(table.Name).ExecuteQuery(new TableQuery().Take(properties.NumberOfRows))
+                var threadTableClient = properties.GetStorageAccount().CreateCloudTableClient();
+
+                var tableColumns = threadTableClient.GetTableReference(group.Last().Name).ExecuteQuery(new TableQuery().Take(properties.NumberOfRows))
                     .SelectMany(row => row.Properties)
                     .GroupBy(column => column.Key)
                     .Select(grp => new TableColumn
-                     {
-                         Name = grp.Key,
-                         TypeName = GetType(grp.First().Value.PropertyType)
-                     });
-
+                    {
+                        Name = grp.Key,
+                        TypeName = GetType(grp.First().Value.PropertyType)
+                    });
+                
                 var baseColumns = new List<TableColumn>
                 {
                     new TableColumn { Name = "PartitionKey", TypeName = GetType(EdmType.String) },
@@ -91,9 +125,12 @@ private static IEnumerable<CloudTable> GetModel(StorageAccountProperties propert
                     new TableColumn { Name = "ETag", TypeName = GetType(EdmType.String) }
                 };
 
-                table.Columns = tableColumns.Concat(baseColumns).ToArray();
-            }
-
+                foreach(var table in group)
+                {
+                    table.Columns = tableColumns.Concat(baseColumns).ToArray();
+                }
+            });
+
             return model;
         }