support defaults. (#369)

Author: TingluoHuang

src/Runner.Worker/ExecutionContext.cs

@@ -48,6 +48,7 @@ public interface IExecutionContext : IRunnerService
         PlanFeatures Features { get; }
         Variables Variables { get; }
         Dictionary<string, string> IntraActionState { get; }
+        IDictionary<String, IDictionary<String, String>> JobDefaults { get; }
         Dictionary<string, VariableValue> JobOutputs { get; }
         IDictionary<String, String> EnvironmentVariables { get; }
         IDictionary<String, ContextScope> Scopes { get; }
@@ -140,6 +141,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext
         public List<ServiceEndpoint> Endpoints { get; private set; }
         public Variables Variables { get; private set; }
         public Dictionary<string, string> IntraActionState { get; private set; }
+        public IDictionary<String, IDictionary<String, String>> JobDefaults { get; private set; }
         public Dictionary<string, VariableValue> JobOutputs { get; private set; }
         public IDictionary<String, String> EnvironmentVariables { get; private set; }
         public IDictionary<String, ContextScope> Scopes { get; private set; }
@@ -270,6 +272,7 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r
                 child.IntraActionState = intraActionState;
             }
             child.EnvironmentVariables = EnvironmentVariables;
+            child.JobDefaults = JobDefaults;
             child.Scopes = Scopes;
             child.FileTable = FileTable;
             child.StepsContext = StepsContext;
@@ -565,6 +568,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation
             // Environment variables shared across all actions
             EnvironmentVariables = new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer);
 
+            // Job defaults shared across all actions
+            JobDefaults = new Dictionary<string, IDictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
+
             // Job Outputs
             JobOutputs = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase);
 

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -58,12 +58,21 @@ public override void PrintActionDetails(ActionRunStage stage)
             string shellCommandPath = null;
             bool validateShellOnHost = !(StepHost is ContainerStepHost);
             string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse<string>());
-            Inputs.TryGetValue("shell", out var shell);
+            string shell = null;
+            if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell))
+            {
+                // TODO: figure out how defaults interact with template later
+                // for now, we won't check job.defaults if we are inside a template.
+                if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults))
+                {
+                    runDefaults.TryGetValue("shell", out shell);
+                }
+            }
             if (string.IsNullOrEmpty(shell))
             {
 #if OS_WINDOWS
                 shellCommand = "pwsh";
-                if(validateShellOnHost)
+                if (validateShellOnHost)
                 {
                     shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                     if (string.IsNullOrEmpty(shellCommandPath))
@@ -139,11 +148,36 @@ public async Task RunAsync(ActionRunStage stage)
             Inputs.TryGetValue("script", out var contents);
             contents = contents ?? string.Empty;
 
-            Inputs.TryGetValue("workingDirectory", out var workingDirectory);
+            string workingDirectory = null;
+            if (!Inputs.TryGetValue("workingDirectory", out workingDirectory))
+            {
+                // TODO: figure out how defaults interact with template later
+                // for now, we won't check job.defaults if we are inside a template.
+                if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults))
+                {
+                    if (runDefaults.TryGetValue("working-directory", out workingDirectory))
+                    {
+                        ExecutionContext.Debug("Overwrite 'working-directory' base on job defaults.");
+                    }
+                }
+            }
             var workspaceDir = githubContext["workspace"] as StringContextData;
             workingDirectory = Path.Combine(workspaceDir, workingDirectory ?? string.Empty);
 
-            Inputs.TryGetValue("shell", out var shell);
+            string shell = null;
+            if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell))
+            {
+                // TODO: figure out how defaults interact with template later
+                // for now, we won't check job.defaults if we are inside a template.
+                if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults))
+                {
+                    if (runDefaults.TryGetValue("shell", out shell))
+                    {
+                        ExecutionContext.Debug("Overwrite 'shell' base on job defaults.");
+                    }
+                }
+            }
+
             var isContainerStepHost = StepHost is ContainerStepHost;
 
             string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse<string>());

src/Runner.Worker/JobExtension.cs

@@ -161,6 +161,26 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
                         }
                     }
 
+                    // Evaluate the job defaults
+                    context.Debug("Evaluating job defaults");
+                    foreach (var token in message.Defaults)
+                    {
+                        var defaults = token.AssertMapping("defaults");
+                        if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
+                        {
+                            context.JobDefaults["run"] = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+                            var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
+                            var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues);
+                            foreach (var pair in jobDefaults)
+                            {
+                                if (!string.IsNullOrEmpty(pair.Value))
+                                {
+                                    context.JobDefaults["run"][pair.Key] = pair.Value;
+                                }
+                            }
+                        }
+                    }
+
                     // Build up 2 lists of steps, pre-job, job
                     // Download actions not already in the cache
                     Trace.Info("Downloading actions");

src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs

@@ -41,7 +41,8 @@ public AgentJobRequestMessage(
             IEnumerable<JobStep> steps,
             IEnumerable<ContextScope> scopes,
             IList<String> fileTable,
-            TemplateToken jobOutputs)
+            TemplateToken jobOutputs,
+            IList<TemplateToken> defaults)
         {
             this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest;
             this.Plan = plan;
@@ -69,6 +70,11 @@ public AgentJobRequestMessage(
                 m_environmentVariables = new List<TemplateToken>(environmentVariables);
             }
 
+            if (defaults?.Count > 0)
+            {
+                m_defaults = new List<TemplateToken>(defaults);
+            }
+
             this.ContextData = new Dictionary<String, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
             if (contextData?.Count > 0)
             {
@@ -213,6 +219,21 @@ public IList<TemplateToken> EnvironmentVariables
             }
         }
 
+        /// <summary>
+        /// Gets the hierarchy of defaults to overlay, last wins.
+        /// </summary>
+        public IList<TemplateToken> Defaults
+        {
+            get
+            {
+                if (m_defaults == null)
+                {
+                    m_defaults = new List<TemplateToken>();
+                }
+                return m_defaults;
+            }
+        }
+
         /// <summary>
         /// Gets the collection of variables associated with the current context.
         /// </summary>
@@ -252,6 +273,9 @@ public IList<ContextScope> Scopes
             }
         }
 
+        /// <summary>
+        /// Gets the table of files used when parsing the pipeline (e.g. yaml files)
+        /// </summary>
         public IList<String> FileTable
         {
             get
@@ -372,6 +396,11 @@ private void OnSerializing(StreamingContext context)
                 m_environmentVariables = null;
             }
 
+            if (m_defaults?.Count == 0)
+            {
+                m_defaults = null;
+            }
+
             if (m_fileTable?.Count == 0)
             {
                 m_fileTable = null;
@@ -406,6 +435,9 @@ private void OnSerializing(StreamingContext context)
         [DataMember(Name = "EnvironmentVariables", EmitDefaultValue = false)]
         private List<TemplateToken> m_environmentVariables;
 
+        [DataMember(Name = "Defaults", EmitDefaultValue = false)]
+        private List<TemplateToken> m_defaults;
+
         [DataMember(Name = "FileTable", EmitDefaultValue = false)]
         private List<String> m_fileTable;
 

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs

@@ -14,6 +14,7 @@ public sealed class PipelineTemplateConstants
         public const String Clean = "clean";
         public const String Container = "container";
         public const String ContinueOnError = "continue-on-error";
+        public const String Defaults = "defaults";
         public const String Env = "env";
         public const String Event = "event";
         public const String EventPattern = "github.event";
@@ -29,6 +30,7 @@ public sealed class PipelineTemplateConstants
         public const String Include = "include";
         public const String Inputs = "inputs";
         public const String Job = "job";
+        public const String JobDefaultsRun = "job-defaults-run";
         public const String JobOutputs = "job-outputs";
         public const String Jobs = "jobs";
         public const String Labels = "labels";

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs

@@ -267,6 +267,42 @@ public Dictionary<String, String> EvaluateJobOutput(
             return result;
         }
 
+        public Dictionary<String, String> EvaluateJobDefaultsRun(
+            TemplateToken token,
+            DictionaryContextData contextData)
+        {
+            var result = default(Dictionary<String, String>);
+
+            if (token != null && token.Type != TokenType.Null)
+            {
+                var context = CreateContext(contextData);
+                try
+                {
+                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobDefaultsRun, token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+                    result = new Dictionary<String, String>(StringComparer.OrdinalIgnoreCase);
+                    var mapping = token.AssertMapping("defaults run");
+                    foreach (var pair in mapping)
+                    {
+                        // Literal key
+                        var key = pair.Key.AssertString("defaults run key");
+
+                        // Literal value
+                        var value = pair.Value.AssertString("defaults run value");
+                        result[key.Value] = value.Value;
+                    }
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result;
+        }
+
         public IList<KeyValuePair<String, JobContainer>> EvaluateJobServiceContainers(
             TemplateToken token,
             DictionaryContextData contextData)

src/Sdk/DTPipelines/workflow-v1.0.json

@@ -9,6 +9,7 @@
         "properties": {
           "on": "any",
           "name": "string",
+          "defaults": "workflow-defaults",
           "env": "workflow-env",
           "jobs": "jobs"
         }
@@ -125,6 +126,23 @@
       "string": {}
     },
 
+    "workflow-defaults": {
+      "mapping": {
+        "properties": {
+          "run": "workflow-defaults-run"
+        }
+      }
+    },
+
+    "workflow-defaults-run": {
+      "mapping": {
+        "properties": {
+          "shell": "non-empty-string",
+          "working-directory": "non-empty-string"
+        }
+      }
+    },
+
     "workflow-env": {
       "context": [
         "github",
@@ -161,6 +179,7 @@
           "services": "services",
           "env": "job-env",
           "outputs": "job-outputs",
+          "defaults": "job-defaults",
           "steps": "steps"
         }
       }
@@ -289,6 +308,30 @@
       }
     },
 
+    "job-defaults": {
+      "mapping": {
+        "properties": {
+          "run": "job-defaults-run"
+        }
+      }
+    },
+
+    "job-defaults-run": {
+      "context": [
+        "github",
+        "strategy",
+        "matrix",
+        "needs",
+        "env"
+      ],
+      "mapping": {
+        "properties": {
+          "shell": "non-empty-string",
+          "working-directory": "non-empty-string"
+        }
+      }
+    },
+
     "job-outputs": {
       "mapping": {
         "loose-key-type": "non-empty-string",

src/Test/L0/Listener/JobDispatcherL0.cs

@@ -33,7 +33,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage()
             TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null);
+            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
             result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
             return result;
         }

src/Test/L0/Listener/RunnerL0.cs

@@ -43,7 +43,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName)
             TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null);
+            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
         }
 
         private JobCancelMessage CreateJobCancelMessage()

src/Test/L0/Worker/ActionCommandManagerL0.cs

@@ -150,7 +150,7 @@ public void EchoProcessCommandDebugOn()
                 TimelineReference timeline = new TimelineReference();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,