Skip to content

Commit 9492691

Browse files
JoannaaKLAvaStancuTingluoHuang
authored
Avastancu/joannaakl/service container error log (#2110)
* adding support for a service container docker logs * Adding Unit test to ContainerOperationProvider * Adding another test to ContainerOperationProvider * placed the docker logs output in dedicated ##group section * Removed the exception thrown if the service container was not healthy * Removed duplicated logging to the executionContext * Updated the container logs sub-section message * Print service containers only if they were healthy Unhealthy service logs are printed in ContainerHealthCheckLogs called prior to this step. * Removed recently added method to inspect docker logs The method was doing the same thing as the existing DockerLogs method. * Added execution context error This will make a failed health check more visible in the UI without disrupting the execution of the program. * Removing the section 'Waiting for all services to be ready' Since nested subsections are not being displayed properly and we already need one subsection per service error. * Update src/Runner.Worker/Container/DockerCommandManager.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Update src/Test/L0/TestHostContext.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Change the logic for printing Service Containers logs Service container logs will be printed in the 'Start containers' section only if there is an error. Healthy services will have their logs printed in the 'Stop Containers' section. * Removed unused import * Added back section group. * Moved service containers error logs to separate group sections * Removed the test testing the old logic flow. * Remove unnecessary 'IsAnyUnhealthy' flag * Remove printHello() function * Add newline to TestHostContext * Remove unnecessary field 'UnhealthyContainers' * Rename boolean flag indicating service container failure * Refactor healthcheck logic to separate method to enable unit testing. * Remove the default value for bool variable * Update src/Runner.Worker/ContainerOperationProvider.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Update src/Runner.Worker/ContainerOperationProvider.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Rename Healthcheck back to ContainerHealthcheck * Make test sequential * Unextract the container error logs method * remove test asserting thrown exception * Add configure await * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang <tingluohuang@github.com> * Add back test asserting exception * Check service exit code if there is no healtcheck configured * Remove unnecessary healthcheck for healthy service container * Revert "Check service exit code if there is no healtcheck configured" This reverts commit fec24e8. Co-authored-by: Ava S <avastancu@github.com> Co-authored-by: Tingluo Huang <tingluohuang@github.com>
1 parent dca4f67 commit 9492691

3 files changed

Lines changed: 150 additions & 19 deletions

File tree

‎src/Runner.Worker/Container/ContainerInfo.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container,
9292
public bool IsJobContainer { get; set; }
9393
public bool IsAlpine { get; set; }
9494

95+
public bool FailedInitialization { get; set; }
96+
9597
public IDictionary<string, string> ContainerEnvironmentVariables
9698
{
9799
get

‎src/Runner.Worker/ContainerOperationProvider.cs‎

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,41 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec
9898
await StartContainerAsync(executionContext, container);
9999
}
100100

101+
await RunContainersHealthcheck(executionContext, containers);
102+
}
103+
104+
public async Task RunContainersHealthcheck(IExecutionContext executionContext, List<ContainerInfo> containers)
105+
{
101106
executionContext.Output("##[group]Waiting for all services to be ready");
107+
108+
var unhealthyContainers = new List<ContainerInfo>();
102109
foreach (var container in containers.Where(c => !c.IsJobContainer))
103110
{
104-
await ContainerHealthcheck(executionContext, container);
111+
var healthcheck = await ContainerHealthcheck(executionContext, container);
112+
113+
if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase))
114+
{
115+
unhealthyContainers.Add(container);
116+
}
117+
else
118+
{
119+
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
120+
}
105121
}
106122
executionContext.Output("##[endgroup]");
123+
124+
if (unhealthyContainers.Count > 0)
125+
{
126+
foreach (var container in unhealthyContainers)
127+
{
128+
executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed.");
129+
await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId);
130+
executionContext.Error($"Failed to initialize container {container.ContainerImage}");
131+
container.FailedInitialization = true;
132+
executionContext.Output("##[endgroup]");
133+
}
134+
throw new InvalidOperationException("One or more containers failed to start.");
135+
}
107136
}
108137

109138
public async Task StopContainersAsync(IExecutionContext executionContext, object data)
@@ -299,16 +328,15 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai
299328

300329
if (!string.IsNullOrEmpty(container.ContainerId))
301330
{
302-
if (!container.IsJobContainer)
331+
if (!container.IsJobContainer && !container.FailedInitialization)
303332
{
304-
// Print logs for service container jobs (not the "action" job itself b/c that's already logged).
305-
executionContext.Output($"Print service container logs: {container.ContainerDisplayName}");
333+
executionContext.Output($"Print service container logs: {container.ContainerDisplayName}");
306334

307-
int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId);
308-
if (logsExitCode != 0)
309-
{
310-
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
311-
}
335+
int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId);
336+
if (logsExitCode != 0)
337+
{
338+
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
339+
}
312340
}
313341

314342
executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}");
@@ -395,14 +423,14 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex
395423
}
396424
}
397425

398-
private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
426+
private async Task<string> ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
399427
{
400428
string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\"";
401429
string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault();
402430
if (string.IsNullOrEmpty(serviceHealth))
403431
{
404432
// Container has no HEALTHCHECK
405-
return;
433+
return String.Empty;
406434
}
407435
var retryCount = 0;
408436
while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase))
@@ -413,14 +441,7 @@ private async Task ContainerHealthcheck(IExecutionContext executionContext, Cont
413441
serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault();
414442
retryCount++;
415443
}
416-
if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase))
417-
{
418-
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
419-
}
420-
else
421-
{
422-
throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}.");
423-
}
444+
return serviceHealth;
424445
}
425446

426447
private async Task<string> ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container)

‎src/Test/L0/Worker/ContainerOperationProviderL0.cs‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
using GitHub.Runner.Worker;
2+
using GitHub.Runner.Worker.Container;
3+
using Xunit;
4+
using Moq;
5+
using GitHub.Runner.Worker.Container.ContainerHooks;
6+
using System.Threading.Tasks;
7+
using System.Collections.Generic;
8+
using System.Runtime.CompilerServices;
9+
using GitHub.DistributedTask.WebApi;
10+
using System;
11+
12+
namespace GitHub.Runner.Common.Tests.Worker
13+
{
14+
15+
public sealed class ContainerOperationProviderL0
16+
{
17+
18+
private TestHostContext _hc;
19+
private Mock<IExecutionContext> _ec;
20+
private Mock<IDockerCommandManager> _dockerManager;
21+
private Mock<IContainerHookManager> _containerHookManager;
22+
private ContainerOperationProvider containerOperationProvider;
23+
private Mock<IJobServerQueue> serverQueue;
24+
private Mock<IPagingLogger> pagingLogger;
25+
private List<string> healthyDockerStatus = new List<string> { "healthy" };
26+
private List<string> unhealthyDockerStatus = new List<string> { "unhealthy" };
27+
private List<string> dockerLogs = new List<string> { "log1", "log2", "log3" };
28+
29+
List<ContainerInfo> containers = new List<ContainerInfo>();
30+
31+
[Fact]
32+
[Trait("Level", "L0")]
33+
[Trait("Category", "Worker")]
34+
public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertFailedTask()
35+
{
36+
//Arrange
37+
Setup();
38+
_dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(unhealthyDockerStatus));
39+
40+
//Act
41+
try
42+
{
43+
await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers);
44+
}
45+
catch (InvalidOperationException)
46+
{
47+
48+
//Assert
49+
Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed);
50+
}
51+
}
52+
53+
[Fact]
54+
[Trait("Level", "L0")]
55+
[Trait("Category", "Worker")]
56+
public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown()
57+
{
58+
//Arrange
59+
Setup();
60+
_dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(unhealthyDockerStatus));
61+
62+
//Act and Assert
63+
await Assert.ThrowsAsync<InvalidOperationException>(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers));
64+
65+
}
66+
67+
[Fact]
68+
[Trait("Level", "L0")]
69+
[Trait("Category", "Worker")]
70+
public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask()
71+
{
72+
//Arrange
73+
Setup();
74+
_dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(healthyDockerStatus));
75+
76+
//Act
77+
await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers);
78+
79+
//Assert
80+
Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded);
81+
82+
}
83+
84+
private void Setup([CallerMemberName] string testName = "")
85+
{
86+
containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" });
87+
_hc = new TestHostContext(this, testName);
88+
_ec = new Mock<IExecutionContext>();
89+
serverQueue = new Mock<IJobServerQueue>();
90+
pagingLogger = new Mock<IPagingLogger>();
91+
92+
_dockerManager = new Mock<IDockerCommandManager>();
93+
_containerHookManager = new Mock<IContainerHookManager>();
94+
containerOperationProvider = new ContainerOperationProvider();
95+
96+
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
97+
_hc.SetSingleton<IJobServerQueue>(serverQueue.Object);
98+
_hc.SetSingleton<IPagingLogger>(pagingLogger.Object);
99+
100+
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
101+
_hc.SetSingleton<IContainerHookManager>(_containerHookManager.Object);
102+
103+
_ec.Setup(x => x.Global).Returns(new GlobalContext());
104+
105+
containerOperationProvider.Initialize(_hc);
106+
}
107+
}
108+
}

0 commit comments

Comments
 (0)