From b7b15f820ceb2316a150fb079c84a72a7e76e772 Mon Sep 17 00:00:00 2001 From: "notzippy@gmail.com" Date: Thu, 21 Apr 2022 15:10:51 -0700 Subject: [PATCH 1/3] Added vendor test --- .travis.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 606fdb8c..0e38b2ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - "1.13.x" - - "1.14.x" + - "1.18.x" + - "1.17.x" - "tip" os: @@ -66,6 +66,14 @@ script: - revel new --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp3 --package revelframework.com - revel test --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp3 + # Check vendored version of revel + - cd $INITIALWD + - revel new --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp4 --package revelframework.com + - cd my/testapp4 + - go mod vendor + - cd ../.. + - revel test --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp4 + matrix: allow_failures: - go: tip From e9e804adff301d46a84fcd5b9c71374a007edc57 Mon Sep 17 00:00:00 2001 From: "notzippy@gmail.com" Date: Thu, 21 Apr 2022 16:13:40 -0700 Subject: [PATCH 2/3] Added code to copy non go files into vendor folder. --- model/revel_container.go | 197 +++++++++++++++++++++++++++++++++++- parser2/source_processor.go | 3 +- 2 files changed, 198 insertions(+), 2 deletions(-) diff --git a/model/revel_container.go b/model/revel_container.go index 80bbbee5..f54e9bb6 100644 --- a/model/revel_container.go +++ b/model/revel_container.go @@ -2,10 +2,16 @@ package model import ( + "bufio" "fmt" + "go/build" + "io" + "io/fs" + "os" "path/filepath" "sort" "strings" + "unicode" "github.com/revel/cmd/utils" "github.com/revel/config" @@ -93,6 +99,15 @@ type ( FireEventFunction func(key Event, value interface{}) (response EventResponse) ImportFunction func(pkgName string) error } + Mod struct { + ImportPath string + SourcePath string + Version string + SourceVersion string + Dir string // full path, $GOPATH/pkg/mod/ + Pkgs []string // sub-pkg import paths + VendorList []string // files to vendor + } ) // Simple Wrapped RevelCallback. @@ -235,6 +250,31 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) { // Reorder module order by key name, a poor mans sort but at least it is consistent sort.Strings(keys) + modtxtPath := filepath.Join(rp.SourcePath, "vendor", "modules.txt") + if utils.Exists(modtxtPath) { + // Parse out require sections of module.txt + modules := rp.vendorInitilizeLocal(modtxtPath, keys) + for _, mod := range modules { + for _, vendorFile := range mod.VendorList { + x := strings.Index(vendorFile, mod.Dir) + if x < 0 { + utils.Logger.Crit("Error! vendor file doesn't belong to mod, strange.", "vendorFile", "mod.Dir", mod.Dir) + } + + localPath := fmt.Sprintf("%s%s", mod.ImportPath, vendorFile[len(mod.Dir):]) + localFile := filepath.Join(rp.SourcePath, "vendor", localPath) + + utils.Logger.Infof("vendoring %s\n", localPath) + + os.MkdirAll(filepath.Dir(localFile), os.ModePerm) + if _, err := copyFile(vendorFile, localFile); err != nil { + fmt.Printf("Error! %s - unable to copy file %s\n", err.Error(), vendorFile) + os.Exit(1) + } + } + } + } + for _, key := range keys { moduleImportPath := rp.Config.StringDefault(key, "") if moduleImportPath == "" { @@ -242,8 +282,8 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) { } modulePath, err := rp.ResolveImportPath(moduleImportPath) + utils.Logger.Info("Resolving import path ", "modulePath", modulePath, "module_import_path", moduleImportPath, "error", err) if err != nil { - utils.Logger.Info("Missing module ", "module_import_path", moduleImportPath, "error", err) if err := callback.PackageResolver(moduleImportPath); err != nil { return fmt.Errorf("failed to resolve package %w", err) @@ -266,6 +306,88 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) { return } +// Adds a module paths to the container object. +func (rp *RevelContainer) vendorInitilizeLocal(modtxtPath string, revel_modules_keys []string) []*Mod { + revel_modules := []string{} + for _, key := range revel_modules_keys { + moduleImportPath := rp.Config.StringDefault(key, "") + if moduleImportPath == "" { + continue + } + revel_modules = append(revel_modules, moduleImportPath) + } + f, _ := os.Open(modtxtPath) + defer f.Close() + scanner := bufio.NewScanner(f) + scanner.Split(bufio.ScanLines) + + var ( + mod *Mod + err error + ) + modules := []*Mod{} + + for scanner.Scan() { + line := scanner.Text() + + // Look for # character + if line[0] == 35 { + s := strings.Split(line, " ") + if (len(s) != 6 && len(s) != 3) || s[1] == "explicit" { + continue + } + + mod = &Mod{ + ImportPath: s[1], + Version: s[2], + } + if s[2] == "=>" { + // issue https://github.com/golang/go/issues/33848 added these, + // see comments. I think we can get away with ignoring them. + continue + } + // Handle "replace" in module file if any + if len(s) > 3 && s[3] == "=>" { + mod.SourcePath = s[4] + + // Handle replaces with a relative target. For example: + // "replace github.com/status-im/status-go/protocol => ./protocol" + if strings.HasPrefix(s[4], ".") || strings.HasPrefix(s[4], "/") { + mod.Dir, err = filepath.Abs(s[4]) + if err != nil { + fmt.Printf("invalid relative path: %v", err) + os.Exit(1) + } + } else { + mod.SourceVersion = s[5] + mod.Dir = pkgModPath(mod.SourcePath, mod.SourceVersion) + } + } else { + mod.Dir = pkgModPath(mod.ImportPath, mod.Version) + } + + if _, err := os.Stat(mod.Dir); os.IsNotExist(err) { + utils.Logger.Critf("Error! %q module path does not exist, check $GOPATH/pkg/mod\n", mod.Dir) + } + + // Determine if we need to examine this mod, based on the list of modules being imported + for _, importPath := range revel_modules { + if strings.HasPrefix(importPath, mod.ImportPath) { + updateModVendorList(mod, importPath) + } + } + + // Build list of files to module path source to project vendor folder + modules = append(modules, mod) + + continue + } + + mod.Pkgs = append(mod.Pkgs, line) + } + return modules +} + // Adds a module paths to the container object. func (rp *RevelContainer) addModulePaths(name, importPath, modulePath string) { utils.Logger.Info("Adding module path", "name", name, "import path", importPath, "system path", modulePath) @@ -313,3 +435,76 @@ func (rp *RevelContainer) ResolveImportPath(importPath string) (string, error) { } return pkgs[0].PkgPath, fmt.Errorf("%w: %s", ErrNoFiles, importPath) } + +func normString(str string) (normStr string) { + for _, char := range str { + if unicode.IsUpper(char) { + normStr += "!" + string(unicode.ToLower(char)) + } else { + normStr += string(char) + } + } + return +} + +func pkgModPath(importPath, version string) string { + goPath := build.Default.GOPATH + if goPath == "" { + if goPath = os.Getenv("GOPATH"); goPath == "" { + // the default GOPATH for go v1.11 + goPath = filepath.Join(os.Getenv("HOME"), "go") + } + } + + normPath := normString(importPath) + normVersion := normString(version) + + return filepath.Join(goPath, "pkg", "mod", fmt.Sprintf("%s@%s", normPath, normVersion)) +} + +func copyFile(src, dst string) (int64, error) { + srcStat, err := os.Stat(src) + if err != nil { + return 0, err + } + + if !srcStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", src) + } + + srcFile, err := os.Open(src) + if err != nil { + return 0, err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return 0, err + } + defer dstFile.Close() + + return io.Copy(dstFile, srcFile) +} + +func updateModVendorList(mod *Mod, importPath string) { + vendorList := []string{} + pathPrefix := filepath.Join(mod.Dir, importPath[len(mod.ImportPath):]) + + filepath.WalkDir(pathPrefix, func(path string, d fs.DirEntry, err error) (e error) { + if d.IsDir() { + return + } + + if err != nil { + utils.Logger.Crit("Failed to walk vendor dir") + } + utils.Logger.Info("Adding to file in vendor list", "path", path) + vendorList = append(vendorList, path) + return + }) + + utils.Logger.Info("For module", "module", mod.ImportPath, "files", len(vendorList)) + + mod.VendorList = append(mod.VendorList, vendorList...) +} diff --git a/parser2/source_processor.go b/parser2/source_processor.go index 8393905d..ad32f387 100644 --- a/parser2/source_processor.go +++ b/parser2/source_processor.go @@ -128,7 +128,8 @@ func (s *SourceProcessor) processPath(path string, info os.FileInfo, err error) } // Ignore files and folders not marked tmp (since those are generated) - if !info.IsDir() || info.Name() == "tmp" { + // Also ignore files in the vendor folder. + if !info.IsDir() || info.Name() == "tmp" || strings.HasPrefix(path, filepath.Join(s.revelContainer.BasePath, "vendor")) { return nil } From ff8817efd9397d33c960c46569e22d9d5bf5d733 Mon Sep 17 00:00:00 2001 From: "notzippy@gmail.com" Date: Mon, 25 Apr 2022 15:41:54 -0700 Subject: [PATCH 3/3] Added `github.com/revel/revel` as an import path to be always copied. --- .vscode/tasks.json | 2 +- model/revel_container.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 5b7e0a28..3db6de11 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,7 @@ { "label": "Clean-Test-Project", "type": "shell", - "command": "rm -rf ${workspaceRoot}/.temp/revel/reveltest" + "command": "rm -rf ${workspaceRoot}/.temp/revel/reveltest && mkdir -p ${workspaceRoot}/.temp/revel/GOPATH" }, { "label": "Update Go Mod", diff --git a/model/revel_container.go b/model/revel_container.go index f54e9bb6..f56b283a 100644 --- a/model/revel_container.go +++ b/model/revel_container.go @@ -308,7 +308,7 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) { // Adds a module paths to the container object. func (rp *RevelContainer) vendorInitilizeLocal(modtxtPath string, revel_modules_keys []string) []*Mod { - revel_modules := []string{} + revel_modules := []string{"github.com/revel/revel"} for _, key := range revel_modules_keys { moduleImportPath := rp.Config.StringDefault(key, "") if moduleImportPath == "" {