Skip to content
This repository has been archived by the owner on Nov 7, 2023. It is now read-only.

Commit

Permalink
Merge pull request #44 from Azure/Release-1.8.1
Browse files Browse the repository at this point in the history
Release 1.8.1 addresses two bugs and updates the build system.
  • Loading branch information
David Noble authored Mar 2, 2018
2 parents c0a7449 + 7bed841 commit af17729
Show file tree
Hide file tree
Showing 76 changed files with 513 additions and 402 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{C23E6A57-2C64-4924-B6C6-BAA511C9DDD3}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -19,7 +19,7 @@
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -28,7 +28,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{8A191B07-775F-43C8-ACB6-F742073612EA}</ProjectGuid>
<OutputType>library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -14,7 +14,7 @@
<ProjectTypeGuids>{60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -23,7 +23,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
Expand Down Expand Up @@ -121,7 +121,7 @@
<value>Optional, default is 2MB. Specify the batch size in bytes</value>
</data>
<data name="BulkSink_MaxInputBufferSize" xml:space="preserve">
<value>Optional, default is 1GB. Approximate estimate of input bytes to buffer before flushing data to sink</value>
<value>Optional, default is 10MB. Approximate estimate of input bytes to buffer before flushing data to sink</value>
<comment>This is a hint to the import tool to keep memory usage close to this limit. It is not a strictly enforced limit</comment>
</data>
<data name="BulkSink_Overwrite" xml:space="preserve">
Expand Down Expand Up @@ -151,4 +151,4 @@
<data name="Source_Table" xml:space="preserve">
<value>Name of the Azure Table</value>
</data>
</root>
</root>
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{366BA489-E851-4899-9BA3-2F9C7599D24B}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -14,7 +14,7 @@
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -23,7 +23,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using Microsoft.Azure.CosmosDB.Table;
using Microsoft.Azure.CosmosDB.Table;
using Microsoft.DataTransfer.TableAPI.Sink.Bulk;
using System.Collections.Generic;

Expand All @@ -11,6 +11,7 @@ public class BatchSizeTracker
{
private InputSizeTracker _inputSizeTracker;
private long _maxBatchSizeInBytes;
private const int _maxEntriesPerBatch = 100;

/// <summary>
/// Create an instance of the batch size tracker
Expand All @@ -32,17 +33,21 @@ public IEnumerable<IEnumerable<TableOperation>> Split(IEnumerable<TableOperation
{
var result = new List<List<TableOperation>>();
long sum = 0;
int numEntries = 0;

foreach (var op in list)
{
var entity = op.Entity;
long docLength = _inputSizeTracker.GetDocumentLength(entity.PartitionKey, entity.RowKey);

if (result.Count > 0 && (sum += docLength) <= _maxBatchSizeInBytes)
if (result.Count > 0 &&
(sum += docLength) <= _maxBatchSizeInBytes &&
(numEntries += 1) <= _maxEntriesPerBatch)
result[result.Count - 1].Add(op);
else
{
sum = docLength;
numEntries = 1;
result.Add(new List<TableOperation> { op });
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Microsoft.DataTransfer.TableAPI.Sink.Bulk
namespace Microsoft.DataTransfer.TableAPI.Sink.Bulk
{
using Microsoft.DataTransfer.AzureTable;
using Microsoft.DataTransfer.Basics;
Expand Down Expand Up @@ -38,7 +38,7 @@ public async Task<IDataSinkAdapter> CreateAsync(ITableAPIBulkSinkAdapterConfigur
if (String.IsNullOrEmpty(configuration.TableName))
throw Errors.TableNameMissing();

long maxInputBufferSizeInBytes = 1 * 1024 * 1024 * 1024;
long maxInputBufferSizeInBytes = 10 * 1024 * 1024;
if (configuration.MaxInputBufferSize.HasValue)
maxInputBufferSizeInBytes = configuration.MaxInputBufferSize.Value;

Expand Down
5 changes: 3 additions & 2 deletions AzureTable/Microsoft.DataTransfer.AzureTable/packages.config
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Azure.CosmosDB.Table" version="0.9.0-preview" targetFramework="net451" />
<package id="Microsoft.Azure.DocumentDB" version="1.19.0" targetFramework="net451" />
Expand All @@ -16,4 +16,5 @@
<package id="System.Linq.Queryable" version="4.0.0" targetFramework="net451" />
<package id="System.Net.Requests" version="4.0.11" targetFramework="net451" />
<package id="System.Spatial" version="5.8.3" targetFramework="net451" />
</packages>
<package id="WindowsAzure.Storage-PremiumTable" version="0.1.0-preview" targetFramework="net451" />
</packages>
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{043A0CB1-09E0-4FCA-B75F-6FA25A1C9235}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -12,7 +12,7 @@
<TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -21,7 +21,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{323766CD-3E2E-42E0-BC6E-CB7785057154}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -12,7 +12,7 @@
<TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -21,7 +21,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{018130D4-F8DF-4C38-B29B-558FB6B60966}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -17,7 +17,7 @@
<IsCodedUITest>False</IsCodedUITest>
<TestProjectType>UnitTest</TestProjectType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -26,7 +26,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{B034BCDD-283E-4E35-B617-C21CF4AFF7DE}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -12,8 +12,8 @@
<TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -23,8 +23,8 @@
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<PlatformTarget>x64</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down Expand Up @@ -95,6 +95,7 @@
<EmbeddedResource Include="Resources.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
<SubType>Designer</SubType>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,6 @@
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("e1df6f4c-c514-498d-a4e4-2966901c5d0a")]

#if DEBUG
#if !SignedAssembly
[assembly: InternalsVisibleTo("Microsoft.DataTransfer.ConsoleHost.UnitTests")]
#endif
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProjectGuid>{E94B085B-0C3F-47AE-84B6-5A39B12BE291}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
Expand All @@ -17,7 +17,7 @@
<IsCodedUITest>False</IsCodedUITest>
<TestProjectType>UnitTest</TestProjectType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -26,7 +26,7 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("40b2b118-51d2-4d95-91dc-4f35b2c17f62")]

[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProductVersion>8.0.30703</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{01631F6D-813C-47D7-9CA7-3A6E493356F5}</ProjectGuid>
Expand All @@ -14,7 +14,7 @@
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand All @@ -24,7 +24,7 @@
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("29872a91-b4f6-4c64-96da-f461013b89eb")]

#if DEBUG
#if !SignedAssembly
[assembly: InternalsVisibleTo("Microsoft.DataTransfer.FunctionalTests")]
[assembly: InternalsVisibleTo("Microsoft.DataTransfer.Core.UnitTests")]
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]
Expand Down
Loading

0 comments on commit af17729

Please sign in to comment.