From 2e31c0b0f7cbb061dc4a0f78bcd2a8cf8a1c74fa Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Sun, 24 Feb 2019 02:51:09 -0800 Subject: [PATCH 1/5] Add task to download files from helix results container --- .../HelixPoolProvider/HelixJobCreator.cs | 1 + .../JobSender/ISentJob.cs | 1 + .../JobSender/JobDefinition.cs | 21 ++-- .../JobSender/SentJob.cs | 4 +- .../Sdk/DownloadFromResultsContainer.cs | 104 ++++++++++++++++++ .../Sdk/SendHelixJob.cs | 7 ++ ...crosoft.DotNet.Helix.Sdk.MonoQueue.targets | 3 + .../tools/Microsoft.DotNet.Helix.Sdk.props | 1 + .../DownloadFromResultsContainer.props | 5 + .../DownloadFromResultsContainer.targets | 30 +++++ 10 files changed, 167 insertions(+), 10 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.props create mode 100644 src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets diff --git a/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs b/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs index e4239c628de..7f49fd2e599 100644 --- a/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs +++ b/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs @@ -98,6 +98,7 @@ public async Task CreateJob() .WithContainerName(_configuration.ContainerName) .WithCorrelationPayloadUris(AgentPayloadUri) .WithStorageAccountConnectionString(_configuration.ConnectionString) + .WithResultsStorageAccountConnectionString(_configuration.ConnectionString) .DefineWorkItem(_agentRequestItem.agentId) .WithCommand(ConstructCommand()) .WithFiles(credentialsPath, agentSettingsPath, StartupScriptPath) diff --git a/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs b/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs index 3d3b2d1e886..cab5c25cf2b 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs @@ -5,5 +5,6 @@ namespace Microsoft.DotNet.Helix.Client public interface ISentJob { string CorrelationId { get; } + string ResultsContainerUri { get; } } } diff --git a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs index fb15cc1be02..1913468375d 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs @@ -148,12 +148,15 @@ public async Task SendAsync(Action log = null) IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName); var jobList = new List(); - IBlobHelper resultsStorage = null; - IBlobContainer resultsStorageContainer = null; - if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString)) + IBlobContainer resultsStorageContainer; + if (string.IsNullOrEmpty(ResultsStorageAccountConnectionString)) { - resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString); - resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetContainerName); + resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName); + } + else + { + IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString); + resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName); } List correlationPayloadUris = @@ -188,14 +191,14 @@ public async Task SendAsync(Action log = null) Creator = Creator, MaxRetryCount = MaxRetryCount ?? 0, JobStartIdentifier = jobStartIdentifier, - ResultsUri = resultsStorageContainer?.Uri, - ResultsUriRSAS = resultsStorageContainer?.ReadSas, - ResultsUriWSAS = resultsStorageContainer?.WriteSas, + ResultsUri = resultsStorageContainer.Uri, + ResultsUriRSAS = resultsStorageContainer.ReadSas, + ResultsUriWSAS = resultsStorageContainer.WriteSas, }), ex => log?.Invoke($"Starting job failed with {ex}\nRetrying...")); - return new SentJob(JobApi, newJob); + return new SentJob(JobApi, newJob, resultsStorageContainer.Uri); } public IJobDefinitionWithTargetQueue WithBuild(string buildNumber) diff --git a/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs b/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs index ef70969ed89..cb3cb5c08fe 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs @@ -9,13 +9,15 @@ namespace Microsoft.DotNet.Helix.Client { internal class SentJob : ISentJob { - public SentJob(IJob jobApi, JobCreationResult newJob) + public SentJob(IJob jobApi, JobCreationResult newJob, string resultsContainerUri) { JobApi = jobApi; CorrelationId = newJob.Name; + ResultsContainerUri = resultsContainerUri; } public IJob JobApi { get; } public string CorrelationId { get; } + public string ResultsContainerUri { get; } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs new file mode 100644 index 00000000000..90e970ddd98 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs @@ -0,0 +1,104 @@ +using Microsoft.Build.Framework; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Helix.Sdk +{ + public class DownloadFromResultsContainer : BaseTask + { + [Required] + public ITaskItem[] WorkItems { get; set; } + + [Required] + public string ResultsContainer { get; set; } + + [Required] + public string PathToDownload { get; set; } + + [Required] + public string JobId { get; set; } + + [Required] + public ITaskItem[] MetadataToWrite { get; set; } + + private const string MetadataFile = "metadata.txt"; + + public override bool Execute() + { + if (string.IsNullOrEmpty(ResultsContainer)) + { + LogRequiredParameterError(nameof(ResultsContainer)); + } + + if (string.IsNullOrEmpty(PathToDownload)) + { + LogRequiredParameterError(nameof(PathToDownload)); + } + + if (string.IsNullOrEmpty(JobId)) + { + LogRequiredParameterError(nameof(JobId)); + } + + if (!Log.HasLoggedErrors) + { + Log.LogMessage(MessageImportance.High, $"Downloading result files for job {JobId}"); + ExecuteCore().GetAwaiter().GetResult(); + } + + return !Log.HasLoggedErrors; + } + + private async Task ExecuteCore() + { + DirectoryInfo directory = Directory.CreateDirectory(Path.Combine(PathToDownload, JobId)); + using (FileStream stream = File.Open(Path.Combine(directory.FullName, MetadataFile), FileMode.Create, FileAccess.Write)) + using (var writer = new StreamWriter(stream)) + { + foreach (ITaskItem metadata in MetadataToWrite) + { + await writer.WriteLineAsync(metadata.GetMetadata("Identity")); + } + } + + ResultsContainer = ResultsContainer.EndsWith("/") ? ResultsContainer : ResultsContainer + "/"; + await Task.WhenAll(WorkItems.Select(wi => DownloadFilesForWorkItem(wi, directory.FullName))); + return; + } + + private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath) + { + if (workItem.GetRequiredMetadata(Log, "DownloadFilesFromResults", out string files)) + { + string workItemName = workItem.GetMetadata("Identity"); + string[] filesToDownload = files.Split(';'); + + DirectoryInfo destinationDir = Directory.CreateDirectory(Path.Combine(directoryPath, workItemName)); + foreach (var file in filesToDownload) + { + try + { + string destinationFile = Path.Combine(destinationDir.FullName, file); + Log.LogMessage(MessageImportance.Normal, $"Downloading {file} => {destinationFile}..."); + CloudBlob blob = new CloudBlob(new Uri($"{ResultsContainer}{workItemName}/{file}")); + await blob.DownloadToFileAsync(destinationFile, FileMode.Create); + } + catch (StorageException e) + { + Log.LogWarning($"Failed to download {file} from results container: {e.Message}"); + } + } + }; + return; + } + + private void LogRequiredParameterError(string parameter) + { + Log.LogError($"Required parameter {nameof(parameter)} string was null or empty"); + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs index 4c9dcdbb091..cca9ac014cc 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs @@ -68,6 +68,12 @@ public class SendHelixJob : HelixTask [Output] public string JobCorrelationId { get; set; } + /// + /// When the task finishes, the results container uri should be available in case we want to download files. + /// + [Output] + public string ResultsContainerUri { get; set; } + /// /// A collection of commands that will run for each work item before any work item commands. /// Use ';' to separate commands and escape a ';' with ';;' @@ -209,6 +215,7 @@ protected override async Task ExecuteCore() ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg)); JobCorrelationId = job.CorrelationId; + ResultsContainerUri = job.ResultsContainerUri; } string mcUri = await GetMissionControlResultUri(); diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets index a9be83c0ab7..eb390c11f14 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets @@ -59,10 +59,13 @@ WorkItems="@(HelixWorkItem)" HelixProperties="@(HelixProperties)"> + @(HelixWorkItem->Count()) + $(HelixTargetQueue) + $(HelixResultsContainer) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props index fc13be29ca1..a34d4a226a0 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props @@ -15,4 +15,5 @@ + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.props b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.props new file mode 100644 index 00000000000..70712c6b520 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.props @@ -0,0 +1,5 @@ + + + <_HelixMultiQueueTargets>$(_HelixMultiQueueTargets);$(MSBuildThisFileDirectory)DownloadFromResultsContainer.targets + + \ No newline at end of file diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets new file mode 100644 index 00000000000..379f9572a50 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets @@ -0,0 +1,30 @@ + + + + <_workItemsWithDownloadMetadata Include="@(HelixWorkItem)" Condition="'%(HelixWorkItem.DownloadFilesFromResults)' != ''" /> + + + + + + + + $([MSBuild]::NormalizePath('$(BUILD_SOURCESDIRECTORY)', 'artifacts', helixresults')) + $([MSBuild]::NormalizePath('$(MSBuildStartupDirectory)', 'helixresults')) + + + + + + + \ No newline at end of file From b0fcfd9ede93a770c07283c2104e8ad99eed9e25 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Mon, 25 Feb 2019 23:22:19 -0800 Subject: [PATCH 2/5] PR Feedback --- .../HelixPoolProvider/HelixJobCreator.cs | 1 - .../JobSender/IJobDefinition.cs | 1 + .../JobSender/ISentJob.cs | 1 + .../JobSender/JobDefinition.cs | 28 +++++++++++------ .../JobSender/SentJob.cs | 4 ++- .../Sdk/DownloadFromResultsContainer.cs | 19 +++++++----- .../Sdk/SendHelixJob.cs | 17 ++++++++++ ...crosoft.DotNet.Helix.Sdk.MonoQueue.targets | 9 +++++- .../DownloadFromResultsContainer.targets | 31 ++++++++++--------- 9 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs b/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs index 7f49fd2e599..e4239c628de 100644 --- a/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs +++ b/src/HelixPoolProvider/HelixPoolProvider/HelixJobCreator.cs @@ -98,7 +98,6 @@ public async Task CreateJob() .WithContainerName(_configuration.ContainerName) .WithCorrelationPayloadUris(AgentPayloadUri) .WithStorageAccountConnectionString(_configuration.ConnectionString) - .WithResultsStorageAccountConnectionString(_configuration.ConnectionString) .DefineWorkItem(_agentRequestItem.agentId) .WithCommand(ConstructCommand()) .WithFiles(credentialsPath, agentSettingsPath, StartupScriptPath) diff --git a/src/Microsoft.DotNet.Helix/JobSender/IJobDefinition.cs b/src/Microsoft.DotNet.Helix/JobSender/IJobDefinition.cs index 6461df50099..8e6a94b1049 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/IJobDefinition.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/IJobDefinition.cs @@ -19,6 +19,7 @@ public interface IJobDefinition IJobDefinition WithStorageAccountConnectionString(string accountConnectionString); IJobDefinition WithResultsContainerName(string resultsContainerName); IJobDefinition WithResultsStorageAccountConnectionString(string resultsAccountConnectionString); + IJobDefinition WithDefaultResultsContainer(); IJobDefinition WithMaxRetryCount(int? maxRetryCount); Task SendAsync(Action log = null); } diff --git a/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs b/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs index cab5c25cf2b..ef87437845d 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/ISentJob.cs @@ -6,5 +6,6 @@ public interface ISentJob { string CorrelationId { get; } string ResultsContainerUri { get; } + string ResultsContainerReadSAS { get; } } } diff --git a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs index 1913468375d..d029ee11cab 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs @@ -21,6 +21,7 @@ internal class JobDefinition : IJobDefinitionWithSource, { private readonly Dictionary _properties; private readonly List _workItems; + private bool _withDefaultResultsContainer; public JobDefinition(IJob jobApi) { @@ -133,6 +134,12 @@ public IJobDefinition WithResultsStorageAccountConnectionString(string resultsAc return this; } + public IJobDefinition WithDefaultResultsContainer() + { + _withDefaultResultsContainer = true; + return this; + } + public async Task SendAsync(Action log = null) { IBlobHelper storage; @@ -148,16 +155,17 @@ public async Task SendAsync(Action log = null) IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName); var jobList = new List(); - IBlobContainer resultsStorageContainer; - if (string.IsNullOrEmpty(ResultsStorageAccountConnectionString)) - { - resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName); - } - else + IBlobContainer resultsStorageContainer = null; + if (!string.IsNullOrEmpty(ResultsStorageAccountConnectionString)) { + IBlobHelper resultsStorage = new ConnectionStringBlobHelper(ResultsStorageAccountConnectionString); resultsStorageContainer = await resultsStorage.GetContainerAsync(TargetResultsContainerName); } + else if (_withDefaultResultsContainer) + { + resultsStorageContainer = await storage.GetContainerAsync(TargetResultsContainerName); + } List correlationPayloadUris = (await Task.WhenAll(CorrelationPayloads.Select(p => p.UploadAsync(storageContainer, log)))).ToList(); @@ -191,14 +199,14 @@ public async Task SendAsync(Action log = null) Creator = Creator, MaxRetryCount = MaxRetryCount ?? 0, JobStartIdentifier = jobStartIdentifier, - ResultsUri = resultsStorageContainer.Uri, - ResultsUriRSAS = resultsStorageContainer.ReadSas, - ResultsUriWSAS = resultsStorageContainer.WriteSas, + ResultsUri = resultsStorageContainer?.Uri, + ResultsUriRSAS = resultsStorageContainer?.ReadSas, + ResultsUriWSAS = resultsStorageContainer?.WriteSas, }), ex => log?.Invoke($"Starting job failed with {ex}\nRetrying...")); - return new SentJob(JobApi, newJob, resultsStorageContainer.Uri); + return new SentJob(JobApi, newJob, resultsStorageContainer?.Uri, string.IsNullOrEmpty(Creator) ? resultsStorageContainer?.ReadSas : string.Empty); } public IJobDefinitionWithTargetQueue WithBuild(string buildNumber) diff --git a/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs b/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs index cb3cb5c08fe..bfebf0283cd 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/SentJob.cs @@ -9,15 +9,17 @@ namespace Microsoft.DotNet.Helix.Client { internal class SentJob : ISentJob { - public SentJob(IJob jobApi, JobCreationResult newJob, string resultsContainerUri) + public SentJob(IJob jobApi, JobCreationResult newJob, string resultsContainerUri, string resultsContainerReadSAS) { JobApi = jobApi; CorrelationId = newJob.Name; ResultsContainerUri = resultsContainerUri; + ResultsContainerReadSAS = resultsContainerReadSAS; } public IJob JobApi { get; } public string CorrelationId { get; } public string ResultsContainerUri { get; } + public string ResultsContainerReadSAS { get; } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs index 90e970ddd98..19e3a460122 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs @@ -1,5 +1,6 @@ using Microsoft.Build.Framework; using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; using Microsoft.WindowsAzure.Storage.Blob; using System; using System.IO; @@ -17,7 +18,7 @@ public class DownloadFromResultsContainer : BaseTask public string ResultsContainer { get; set; } [Required] - public string PathToDownload { get; set; } + public string OutputDirectory { get; set; } [Required] public string JobId { get; set; } @@ -25,6 +26,8 @@ public class DownloadFromResultsContainer : BaseTask [Required] public ITaskItem[] MetadataToWrite { get; set; } + public string ResultsContainerReadSAS { get; set; } + private const string MetadataFile = "metadata.txt"; public override bool Execute() @@ -34,9 +37,9 @@ public override bool Execute() LogRequiredParameterError(nameof(ResultsContainer)); } - if (string.IsNullOrEmpty(PathToDownload)) + if (string.IsNullOrEmpty(OutputDirectory)) { - LogRequiredParameterError(nameof(PathToDownload)); + LogRequiredParameterError(nameof(OutputDirectory)); } if (string.IsNullOrEmpty(JobId)) @@ -55,7 +58,7 @@ public override bool Execute() private async Task ExecuteCore() { - DirectoryInfo directory = Directory.CreateDirectory(Path.Combine(PathToDownload, JobId)); + DirectoryInfo directory = Directory.CreateDirectory(Path.Combine(OutputDirectory, JobId)); using (FileStream stream = File.Open(Path.Combine(directory.FullName, MetadataFile), FileMode.Create, FileAccess.Write)) using (var writer = new StreamWriter(stream)) { @@ -72,7 +75,7 @@ private async Task ExecuteCore() private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath) { - if (workItem.GetRequiredMetadata(Log, "DownloadFilesFromResults", out string files)) + if (workItem.TryGetMetadata("DownloadFilesFromResults", out string files)) { string workItemName = workItem.GetMetadata("Identity"); string[] filesToDownload = files.Split(';'); @@ -84,7 +87,9 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory { string destinationFile = Path.Combine(destinationDir.FullName, file); Log.LogMessage(MessageImportance.Normal, $"Downloading {file} => {destinationFile}..."); - CloudBlob blob = new CloudBlob(new Uri($"{ResultsContainer}{workItemName}/{file}")); + + var uri = new Uri($"{ResultsContainer}{workItemName}/{file}"); + CloudBlob blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new CloudBlob(uri) : new CloudBlob(uri, new StorageCredentials(ResultsContainerReadSAS)); await blob.DownloadToFileAsync(destinationFile, FileMode.Create); } catch (StorageException e) @@ -98,7 +103,7 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory private void LogRequiredParameterError(string parameter) { - Log.LogError($"Required parameter {nameof(parameter)} string was null or empty"); + Log.LogError($"Required parameter {parameter} string was null or empty"); } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs index cca9ac014cc..df7ef3939ba 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs @@ -74,6 +74,12 @@ public class SendHelixJob : HelixTask [Output] public string ResultsContainerUri { get; set; } + /// + /// If the job is internal, we need to give the DownloadFromResultsContainer task the Write SAS to download files. + /// + [Output] + public string ResultsContainerReadSAS { get; set; } + /// /// A collection of commands that will run for each work item before any work item commands. /// Use ';' to separate commands and escape a ';' with ';;' @@ -134,6 +140,11 @@ public class SendHelixJob : HelixTask /// public int MaxRetryCount { get; set; } + /// + /// Currently, if we're using the download results feature, we need to return the results container back to know where to download results from. Currently, the Helix API is the one that provides this container and we have no way to get it back. If this property is set to true, we will create the container before sending the job and tell the API to use this container. + /// + public bool UsingDownloadResultsFeature { get; set; } + private CommandPayload _commandPayload; protected override async Task ExecuteCore() @@ -205,6 +216,11 @@ protected override async Task ExecuteCore() } } + if (UsingDownloadResultsFeature) + { + def = def.WithDefaultResultsContainer(); + } + // don't send the job if we have errors if (Log.HasLoggedErrors) { @@ -216,6 +232,7 @@ protected override async Task ExecuteCore() ISentJob job = await def.SendAsync(msg => Log.LogMessage(msg)); JobCorrelationId = job.CorrelationId; ResultsContainerUri = job.ResultsContainerUri; + ResultsContainerReadSAS = job.ResultsContainerReadSAS; } string mcUri = await GetMissionControlResultUri(); diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets index eb390c11f14..939c0a8374a 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets @@ -44,6 +44,10 @@ set -x;$(HelixPreCommands) + + <_usingDownloadResultsFeature>false + <_usingDownloadResultsFeature Condition="'@(HelixWorkItem -> HasMetadata('DownloadFilesFromResults'))' != ''">true + + HelixProperties="@(HelixProperties)" + UsingDownloadResultsFeature="$(_usingDownloadResultsFeature)"> + @(HelixWorkItem->Count()) $(HelixTargetQueue) $(HelixResultsContainer) + $(HelixResultsContainerReadSAS) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets index 379f9572a50..f0bf9c8e7e7 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets @@ -1,30 +1,31 @@ + Condition="$(WaitForWorkItemCompletion)" + AfterTargets="CoreTest" + Inputs="unused" + Outputs="%(SentJob.Identity)"> <_workItemsWithDownloadMetadata Include="@(HelixWorkItem)" Condition="'%(HelixWorkItem.DownloadFilesFromResults)' != ''" /> - - - + + + - $([MSBuild]::NormalizePath('$(BUILD_SOURCESDIRECTORY)', 'artifacts', helixresults')) - $([MSBuild]::NormalizePath('$(MSBuildStartupDirectory)', 'helixresults')) + $([MSBuild]::NormalizePath('$(BUILD_SOURCESDIRECTORY)', 'artifacts', helixresults')) + + Condition="'@(_workItemsWithDownloadMetadata)' != '' AND '%(SentJob.ResultsContainerUri)' != '' AND '$(HelixResultsDestinationDir)' != ''" + WorkItems="@(_workItemsWithDownloadMetadata)" + ResultsContainer="%(SentJob.ResultsContainerUri)" + OutputDirectory="$(HelixResultsDestinationDir)" + MetadataToWrite="@(HelixDownloadResultsMetadata)" + JobId="%(SentJob.Identity)" + ResultsContainerReadSAS="%(SentJob.ResultsContainerReadSAS)" /> \ No newline at end of file From ae6703bb526150d2b45fd8fc3f87a7abe96801ec Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Feb 2019 10:36:11 -0800 Subject: [PATCH 3/5] Fix warning conditions to only be emitted when feature is ON --- .../download-results/DownloadFromResultsContainer.targets | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets index f0bf9c8e7e7..427c6034981 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/download-results/DownloadFromResultsContainer.targets @@ -14,13 +14,15 @@ $([MSBuild]::NormalizePath('$(BUILD_SOURCESDIRECTORY)', 'artifacts', helixresults')) + + <_shouldDownloadResults>false + <_shouldDownloadResults Condition="'@(_workItemsWithDownloadMetadata)' != '' AND '$(HelixResultsDestinationDir)' != ''">true - - + Date: Tue, 26 Feb 2019 16:16:41 -0800 Subject: [PATCH 4/5] Implement ICancelableTask --- .../Sdk/DownloadFromResultsContainer.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs index 19e3a460122..7cc8ff1f518 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs @@ -5,11 +5,12 @@ using System; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace Microsoft.DotNet.Helix.Sdk { - public class DownloadFromResultsContainer : BaseTask + public class DownloadFromResultsContainer : BaseTask, ICancelableTask { [Required] public ITaskItem[] WorkItems { get; set; } @@ -30,6 +31,10 @@ public class DownloadFromResultsContainer : BaseTask private const string MetadataFile = "metadata.txt"; + private readonly CancellationTokenSource _cancellationSource = new CancellationTokenSource(); + + public void Cancel() => _cancellationSource.Cancel(); + public override bool Execute() { if (string.IsNullOrEmpty(ResultsContainer)) @@ -69,12 +74,14 @@ private async Task ExecuteCore() } ResultsContainer = ResultsContainer.EndsWith("/") ? ResultsContainer : ResultsContainer + "/"; - await Task.WhenAll(WorkItems.Select(wi => DownloadFilesForWorkItem(wi, directory.FullName))); + await Task.WhenAll(WorkItems.Select(wi => DownloadFilesForWorkItem(wi, directory.FullName, _cancellationSource.Token))); return; } - private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath) + private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath, CancellationToken ct) { + ct.ThrowIfCancellationRequested(); + if (workItem.TryGetMetadata("DownloadFilesFromResults", out string files)) { string workItemName = workItem.GetMetadata("Identity"); From 3e862cb5f0b45d4a205ac226a421df819731dcaf Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Feb 2019 18:22:06 -0800 Subject: [PATCH 5/5] Add more detailed warning when blob not found --- src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs index 7cc8ff1f518..62d32a70001 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs @@ -101,7 +101,7 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory } catch (StorageException e) { - Log.LogWarning($"Failed to download {file} from results container: {e.Message}"); + Log.LogWarning($"Failed to download {workItemName}/{file} blob from results container: {e.Message}"); } } };