Skip to content

Commit d8f127b

Browse files
vinjiangvinjiang
authored
Merge pull request #178 from EmmaZhu/parallellist
Parallellist
2 parents 576ec65 + 7a88c41 commit d8f127b

File tree

57 files changed

+2602
-936
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+2602
-936
lines changed

‎DataMovement.sln

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
3-
# Visual Studio 14
4-
VisualStudioVersion = 14.0.25420.1
3+
# Visual Studio 15
4+
VisualStudioVersion = 15.0.26228.10
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataMovement", "lib\DataMovement.csproj", "{B821E031-09CC-48F0-BDC6-2793228D4027}"
77
EndProject

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Microsoft Azure Storage Data Movement Library (0.11.0)
1+
# Microsoft Azure Storage Data Movement Library (0.12.0)
22

33
The Microsoft Azure Storage Data Movement Library designed for high-performance uploading, downloading and copying Azure Storage Blob and File. This library is based on the core data movement framework that powers [AzCopy](https://azure.microsoft.com/documentation/articles/storage-use-azcopy/).
44

‎changelog.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
2019.07.05 Version 0.12.0
2+
* All Services on both .Net Framework and .Net Core
3+
- Upgraded Microsoft.Azure.Storage.Blob from 10.0.1 to 10.0.3
4+
- Upgraded Microsoft.Azure.Storage.File from 10.0.1 to 10.0.3
5+
* File Service on both .Net Framework and .Net Core
6+
- Improved listing performance when transfer source is Azure File Directory.
7+
18
2019.04.25 Version 0.11.0
29
* All Services on both .Net Framework and .Net Core
310
- Upgraded azure storage blob client library from 9.4.2 to 10.0.1

‎lib/DataMovement.csproj

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<DebugType>full</DebugType>
2323
<Optimize>false</Optimize>
2424
<OutputPath>bin\Debug\</OutputPath>
25-
<DefineConstants>TRACE;DEBUG;$(DefineConstants)</DefineConstants>
25+
<DefineConstants>TRACE;DEBUG;TEST_HOOK;$(DefineConstants)</DefineConstants>
2626
<ErrorReport>prompt</ErrorReport>
2727
<WarningLevel>4</WarningLevel>
2828
<Prefer32Bit>false</Prefer32Bit>
@@ -54,14 +54,14 @@
5454
<Reference Include="Microsoft.Azure.KeyVault.Core, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
5555
<HintPath>..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll</HintPath>
5656
</Reference>
57-
<Reference Include="Microsoft.Azure.Storage.Blob, Version=10.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
58-
<HintPath>..\packages\Microsoft.Azure.Storage.Blob.10.0.1\lib\net452\Microsoft.Azure.Storage.Blob.dll</HintPath>
57+
<Reference Include="Microsoft.Azure.Storage.Blob, Version=10.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
58+
<HintPath>..\packages\Microsoft.Azure.Storage.Blob.10.0.3\lib\net452\Microsoft.Azure.Storage.Blob.dll</HintPath>
5959
</Reference>
60-
<Reference Include="Microsoft.Azure.Storage.Common, Version=10.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
61-
<HintPath>..\packages\Microsoft.Azure.Storage.Common.10.0.1\lib\net452\Microsoft.Azure.Storage.Common.dll</HintPath>
60+
<Reference Include="Microsoft.Azure.Storage.Common, Version=10.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
61+
<HintPath>..\packages\Microsoft.Azure.Storage.Common.10.0.3\lib\net452\Microsoft.Azure.Storage.Common.dll</HintPath>
6262
</Reference>
63-
<Reference Include="Microsoft.Azure.Storage.File, Version=10.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
64-
<HintPath>..\packages\Microsoft.Azure.Storage.File.10.0.1\lib\net452\Microsoft.Azure.Storage.File.dll</HintPath>
63+
<Reference Include="Microsoft.Azure.Storage.File, Version=10.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
64+
<HintPath>..\packages\Microsoft.Azure.Storage.File.10.0.3\lib\net452\Microsoft.Azure.Storage.File.dll</HintPath>
6565
</Reference>
6666
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=1.8.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
6767
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>
@@ -85,6 +85,7 @@
8585
</Compile>
8686
<Compile Include="AssemblyInfo.cs" />
8787
<Compile Include="ChunkedMemoryStream.cs" />
88+
<Compile Include="DirectoryListingScheduler.cs" />
8889
<Compile Include="Extensions\StorageCopyState.cs" />
8990
<Compile Include="FileNativeMethods.cs" />
9091
<Compile Include="LongPathFile.cs" />
@@ -128,6 +129,7 @@
128129
<Compile Include="TransferControllers\TransferWriters\RangeBasedWriter.cs" />
129130
<Compile Include="TransferControllers\TransferWriters\StreamedWriter.cs" />
130131
<Compile Include="TransferEnumerators\AzureBlobEntry.cs" />
132+
<Compile Include="TransferEnumerators\AzureFileDirectoryEntry.cs" />
131133
<Compile Include="TransferEnumerators\AzureBlobEnumerator.cs" />
132134
<Compile Include="TransferEnumerators\AzureFileEntry.cs" />
133135
<Compile Include="TransferEnumerators\AzureFileEnumerator.cs" />
@@ -160,11 +162,13 @@
160162
<Compile Include="TransferJobs\DirectoryLocation.cs" />
161163
<Compile Include="TransferJobs\DirectoryTransfer.cs" />
162164
<Compile Include="TransferJobs\FileLocation.cs" />
163-
<Compile Include="TransferJobs\MultipleObjectsTransfer.cs" />
165+
<Compile Include="TransferJobs\FlatDirectoryTransfer.cs" />
166+
<Compile Include="TransferJobs\HierarchyDirectoryTransfer.cs" />
164167
<Compile Include="TransferJobs\SerializableTransferLocation.cs" />
165168
<Compile Include="TransferJobs\SingleObjectCheckpoint.cs" />
166169
<Compile Include="TransferJobs\SingleObjectTransfer.cs" />
167170
<Compile Include="TransferJobs\StreamLocation.cs" />
171+
<Compile Include="TransferJobs\SubDirectoryTransfer.cs" />
168172
<Compile Include="TransferJobs\Transfer.cs" />
169173
<Compile Include="TransferJobs\TransferJob.cs" />
170174
<Compile Include="Constants.cs" />

‎lib/DirectoryListingScheduler.cs

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="DirectoryListingScheduler.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
namespace Microsoft.Azure.Storage.DataMovement
9+
{
10+
using System;
11+
using System.Collections.Concurrent;
12+
using System.Collections.Generic;
13+
using System.Text;
14+
using System.Threading;
15+
using System.Threading.Tasks;
16+
17+
class DirectoryListingScheduler : IDisposable
18+
{
19+
// Value to control concurrent of directory listing.
20+
// By testing result with downloading from Azure File Directory on 16 core machines on .Net on Windows and on .Net Core on Linux
21+
// it has best downloading speed with 2 listing threads on .Net on Windows and with 4 listing threads on .Net Core on Linux.
22+
#if DOTNET5_4
23+
private const int MaxParallelListingThreads = 4;
24+
#else
25+
private const int MaxParallelListingThreads = 2;
26+
#endif
27+
28+
SemaphoreSlim semaphore = null;
29+
private static DirectoryListingScheduler SchedulerInstance = null;
30+
private static object SingletonLock = new object();
31+
32+
private DirectoryListingScheduler()
33+
{
34+
semaphore = new SemaphoreSlim(MaxParallelListingThreads, MaxParallelListingThreads);
35+
}
36+
37+
public static DirectoryListingScheduler Instance()
38+
{
39+
if (null == SchedulerInstance)
40+
{
41+
lock (SingletonLock)
42+
{
43+
if (null == SchedulerInstance)
44+
{
45+
SchedulerInstance = new DirectoryListingScheduler();
46+
}
47+
}
48+
}
49+
50+
return SchedulerInstance;
51+
}
52+
53+
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")]
54+
public Task Schedule(
55+
SubDirectoryTransfer subDirectoryTransfer,
56+
CancellationToken cancellationToken,
57+
Action persistDirTransfer,
58+
int timeOut)
59+
{
60+
if (this.semaphore.Wait(timeOut, cancellationToken))
61+
{
62+
try
63+
{
64+
if (null != persistDirTransfer)
65+
{
66+
persistDirTransfer();
67+
}
68+
}
69+
catch
70+
{
71+
this.semaphore.Release();
72+
throw;
73+
}
74+
75+
Task task = subDirectoryTransfer.ExecuteAsync(cancellationToken);
76+
task.ContinueWith((sourceTask) =>
77+
{
78+
this.semaphore.Release();
79+
return sourceTask;
80+
});
81+
82+
return task;
83+
}
84+
else
85+
{
86+
return null;
87+
}
88+
}
89+
90+
/// <summary>
91+
/// Public dispose method to release all resources owned.
92+
/// </summary>
93+
public void Dispose()
94+
{
95+
this.Dispose(true);
96+
GC.SuppressFinalize(this);
97+
}
98+
99+
private void Dispose(bool disposing)
100+
{
101+
if (disposing)
102+
{
103+
if (null != this.semaphore)
104+
{
105+
this.semaphore.Dispose();
106+
this.semaphore = null;
107+
}
108+
}
109+
}
110+
}
111+
}

‎lib/Exceptions/TransferErrorCode.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ public enum TransferErrorCode
116116
/// </summary>
117117
UnsupportedDummyTransfer = 21,
118118

119+
/// <summary>
120+
/// Failed when trying in ShouldTransferCallbackAsync.
121+
/// </summary>
122+
FailedCheckingShouldTransfer = 22,
123+
119124
/// <summary>
120125
/// Uncategorized transfer error.
121126
/// </summary>

‎lib/Extensions/StorageCopyState.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
namespace Microsoft.Azure.Storage.DataMovement.Extensions
88
{
99
using System;
10+
using Microsoft.Azure.Storage.Blob;
1011

1112
internal enum StorageCopyStatus
1213
{

‎lib/Extensions/StorageExtensions.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,12 @@ internal static string ConvertToString(this object instance)
144144
return file.SnapshotQualifiedUri.AbsoluteUri;
145145
}
146146

147+
CloudFileDirectory cloudFileDirectory = instance as CloudFileDirectory;
148+
if (null != cloudFileDirectory)
149+
{
150+
return cloudFileDirectory.SnapshotQualifiedUri.AbsoluteUri;
151+
}
152+
147153
return instance.ToString();
148154
}
149155

0 commit comments

Comments
 (0)