Refactor: configmanager Package Enhancement

internal/pkg/configmanager/config.go

@@ -5,12 +5,13 @@ import (
 )
 
 // Config records rendered config values and is used as a general config in DevStream.
+// Also see the rawConfig struct in rawconfig.go, it represents the original config.
 type Config struct {
 	// Command line flag have a higher priority than the config file.
 	// If you used the `--plugin-dir` flag with `dtm`, then the "pluginDir" in the config file will be ignored.
-	PluginDir string
-	Tools     Tools `yaml:"tools"`
-	State     *State
+	PluginDir string `yaml:"pluginDir"`
+	Tools     Tools  `yaml:"tools"`
+	State     *State `yaml:"state"`
 }
 
 func (c *Config) validate() error {

internal/pkg/configmanager/config_test.go

@@ -26,19 +26,21 @@ var _ = Describe("getRawConfigFromFile func", func() {
 	var (
 		fLoc    string
 		baseDir string
+		m       Manager
 	)
 	BeforeEach(func() {
 		baseDir = GinkgoT().TempDir()
 		f, err := os.CreateTemp(baseDir, "test")
 		Expect(err).Error().ShouldNot(HaveOccurred())
 		fLoc = f.Name()
+		m.ConfigFilePath = fLoc
 	})
 	When("file not exist", func() {
 		BeforeEach(func() {
 			fLoc = "not_exist"
 		})
 		It("should return err", func() {
-			_, err := getRawConfigFromFile(fLoc)
+			_, err := m.getRawConfigFromFile()
 			Expect(err).Error().Should(HaveOccurred())
 			Expect(err.Error()).Should(ContainSubstring("no such file or directory"))
 		})
@@ -49,7 +51,7 @@ var _ = Describe("getRawConfigFromFile func", func() {
 			Expect(err).Error().ShouldNot(HaveOccurred())
 		})
 		It("should return err", func() {
-			_, err := getRawConfigFromFile(fLoc)
+			_, err := m.getRawConfigFromFile()
 			Expect(err).Error().Should(HaveOccurred())
 			Expect(err.Error()).Should(ContainSubstring("cannot unmarshal"))
 		})
@@ -58,38 +60,36 @@ var _ = Describe("getRawConfigFromFile func", func() {
 
 var _ = Describe("rawConfig struct", func() {
 	var (
-		r          *rawConfig
-		baseDir    string
-		fLoc       string
-		globalVars map[string]any
+		r       *rawConfig
+		baseDir string
+		fLoc    string
 	)
 	BeforeEach(func() {
 		r = &rawConfig{}
 		baseDir = GinkgoT().TempDir()
 		f, err := os.CreateTemp(baseDir, "test")
 		Expect(err).Error().ShouldNot(HaveOccurred())
 		fLoc = f.Name()
-		globalVars = map[string]any{}
 	})
-	Context("GetGlobalVars method", func() {
+	Context("mergeGlobalVars method", func() {
 		When("varFile get content failed", func() {
 			BeforeEach(func() {
 				r.VarFile = "not_exist"
 			})
 			It("should return err", func() {
-				_, err := r.GetGlobalVars()
+				err := r.mergeGlobalVars()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("not exists"))
 			})
 		})
 		When("varFiles is not valid", func() {
 			BeforeEach(func() {
-				r.VarFile = configFileLoc(fLoc)
+				r.VarFile = fLoc
 				err := os.WriteFile(fLoc, []byte("not_Valid_Yaml{{}}"), 0666)
 				Expect(err).Error().ShouldNot(HaveOccurred())
 			})
 			It("should return err", func() {
-				_, err := r.GetGlobalVars()
+				err := r.mergeGlobalVars()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("cannot unmarshal"))
 			})
@@ -102,7 +102,7 @@ var _ = Describe("rawConfig struct", func() {
 				r.AppFile = "not_exist"
 			})
 			It("should return err", func() {
-				_, err := r.GetToolsFromApps(globalVars)
+				_, err := r.getToolsFromApps()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("not exists"))
 			})
@@ -111,23 +111,23 @@ var _ = Describe("rawConfig struct", func() {
 			BeforeEach(func() {
 				err := os.WriteFile(fLoc, []byte("not_Valid_Yaml{{}}"), 0666)
 				Expect(err).Error().ShouldNot(HaveOccurred())
-				r.AppFile = configFileLoc(fLoc)
+				r.AppFile = fLoc
 			})
 			It("should return err", func() {
-				_, err := r.GetToolsFromApps(globalVars)
+				_, err := r.getToolsFromApps()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("yaml parse path[$.apps[*]] failed"))
 			})
 		})
 	})
 
-	Context("GetTools method", func() {
+	Context("getToolsOutOfApps method", func() {
 		When("toolsFile get content failed", func() {
 			BeforeEach(func() {
 				r.ToolFile = "not_exist"
 			})
 			It("should return err", func() {
-				_, err := r.GetTools(globalVars)
+				_, err := r.getToolsOutOfApps()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("not exists"))
 			})
@@ -136,10 +136,10 @@ var _ = Describe("rawConfig struct", func() {
 			BeforeEach(func() {
 				err := os.WriteFile(fLoc, []byte("not_Valid_Yaml{{}}"), 0666)
 				Expect(err).Error().ShouldNot(HaveOccurred())
-				r.ToolFile = configFileLoc(fLoc)
+				r.ToolFile = fLoc
 			})
 			It("should return err", func() {
-				_, err := r.GetTools(globalVars)
+				_, err := r.getToolsOutOfApps()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("yaml parse path[$.tools[*]] failed"))
 			})
@@ -155,7 +155,7 @@ tools:
       key1: [[ var1 ]]`)
 			})
 			It("should return err", func() {
-				_, err := r.GetTools(globalVars)
+				_, err := r.getToolsOutOfApps()
 				Expect(err).Error().Should(HaveOccurred())
 			})
 		})
@@ -170,7 +170,7 @@ tools:
       key1: {{}}`)
 			})
 			It("should return err", func() {
-				_, err := r.GetTools(globalVars)
+				_, err := r.getToolsOutOfApps()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("unexpected mapping key"))
 			})
@@ -185,17 +185,13 @@ tools:
   dependsOn: [ "not_exist" ]
   options:
     key1: [[ var1 ]]`)
-				globalVars = map[string]any{
-					"var1": "global",
-				}
 			})
 			It("should return err", func() {
-				_, err := r.GetTools(globalVars)
+				_, err := r.getToolsOutOfApps()
 				Expect(err).Error().Should(HaveOccurred())
 				Expect(err.Error()).Should(ContainSubstring("tool default's dependency not_exist doesn't exist"))
 			})
 		})
-
 	})
 
 	Context("getPipelineTemplatesMap method", func() {

internal/pkg/configmanager/configmanager.go

@@ -1,5 +1,17 @@
 package configmanager
 
+import (
+	"bytes"
+	"errors"
+	"io"
+	"os"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/devstream-io/devstream/pkg/util/file"
+	"github.com/devstream-io/devstream/pkg/util/log"
+)
+
 type Manager struct {
 	ConfigFilePath string
 }
@@ -13,41 +25,55 @@ func NewManager(configFilePath string) *Manager {
 // LoadConfig reads an input file as a general config.
 // It will return "non-nil, nil" or "nil, err".
 func (m *Manager) LoadConfig() (*Config, error) {
-	// 1. get rawConfig from config.yaml file
-	rawConfig, err := getRawConfigFromFile(m.ConfigFilePath)
+	rawConf, err := m.getRawConfigFromFile()
 	if err != nil {
 		return nil, err
 	}
 
-	// 2. get all globals vars
-	globalVars, err := rawConfig.GetGlobalVars()
+	tools, err := rawConf.getAllTools()
 	if err != nil {
 		return nil, err
 	}
 
-	// 3. get Tools from Apps
-	appTools, err := rawConfig.GetToolsFromApps(globalVars)
+	config := &Config{
+		PluginDir: rawConf.PluginDir,
+		State:     rawConf.State,
+		Tools:     tools,
+	}
+	if err = config.validate(); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+// getRawConfigFromFile generate new rawConfig options
+func (m *Manager) getRawConfigFromFile() (*rawConfig, error) {
+	// 1. get baseDir from configFile
+	baseDir, err := file.GetFileAbsDirPath(m.ConfigFilePath)
 	if err != nil {
 		return nil, err
 	}
 
-	// 4. get Tools from rawConfig
-	tools, err := rawConfig.GetTools(globalVars)
+	// 2. read the original main config file
+	configBytes, err := os.ReadFile(m.ConfigFilePath)
 	if err != nil {
 		return nil, err
 	}
 
-	tools = append(tools, appTools...)
+	// TODO(daniel-hutao): We should change the documents to delete all "---" with config file. After a while, delete the following line of code, or prompt the user with "This is the wrong way" when "---" be detected
 
-	config := &Config{
-		PluginDir: rawConfig.PluginDir,
-		State:     rawConfig.State,
-		Tools:     tools,
-	}
+	// replace all "---", otherwise yaml.Unmarshal can only read the content before the first "---"
+	configBytes = bytes.Replace(configBytes, []byte("---"), []byte("\n"), -1)
 
-	// 5. validate config
-	if err := config.validate(); err != nil {
+	// 3. decode total yaml files to get rawConfig
+	var rawConf rawConfig
+	if err := yaml.Unmarshal(configBytes, &rawConf); err != nil && !errors.Is(err, io.EOF) {
+		log.Errorf("Please verify the format of your config. Error: %s.", err)
 		return nil, err
 	}
-	return config, nil
+
+	rawConf.configFileBaseDir = baseDir
+	rawConf.totalConfigBytes = configBytes
+	return &rawConf, nil
 }

internal/pkg/configmanager/configmanager_test.go

@@ -437,7 +437,7 @@ tools:
 			})
 		})
 
-		When("GetGlobalVars failed", func() {
+		When("mergeGlobalVars failed", func() {
 			BeforeEach(func() {
 				err := os.WriteFile(fLoc, []byte(`
 varFile: not_exist
@@ -453,7 +453,7 @@ state:
 			})
 		})
 
-		When("GetTools failed", func() {
+		When("getToolsOutOfApps failed", func() {
 			BeforeEach(func() {
 				err := os.WriteFile(fLoc, []byte(`
 toolFile: not_exist