package handler import ( "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "sync" ) type PathInfo struct { Dirs []string // 目录路径(包含所有层级) Files []string // 文件路径 } var pathPool = sync.Pool{ New: func() interface{} { return &PathInfo{ Dirs: make([]string, 0, 1024), // 预分配容量 Files: make([]string, 0, 4096), } }, } func GoHeavy(info *PathInfo) (PathInfo, error) { m := make(map[string]bool) for i := range info.Files { path := info.Files[i] lastIndex := strings.LastIndex(path, "/") if lastIndex == -1 { lastIndex = strings.LastIndex(path, "\\") } if lastIndex == -1 { continue } //if _, ok := m[path[:lastIndex]]; !ok { // fmt.Println(path[:lastIndex]) //} m[path[:lastIndex]] = true } var dirs = make([]string, 0, len(m)) fmt.Println(len(info.Dirs)) for i := range info.Dirs { path := info.Dirs[i] if !m[path] { fmt.Println(info.Dirs[i]) dirs = append(dirs, info.Dirs[i]) } } info.Dirs = dirs return *info, nil } func GetPathInfo(root string) (*PathInfo, error) { //获取文件名 index := strings.LastIndex(root, "/") if index == -1 { index = strings.LastIndex(root, "\\") } //RootFileName := "" //if index == -1 { // RootFileName = root //} else { // RootFileName = root[:index+1] //} // 复用对象减少内存分配[9](@ref) info := pathPool.Get().(*PathInfo) // 路径有效性校验[5,6](@ref) root = filepath.Clean(root) stat, err := os.Stat(root) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, &os.PathError{Op: "stat", Path: root, Err: err} // 增强错误信息[8](@ref) } return nil, err } // 文件直接返回[4](@ref) if !stat.IsDir() { info.Files = append(info.Files, root) return info, nil } // 使用WalkDir优化遍历性能[4,9](@ref) err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { // 处理遍历错误但继续执行[7](@ref) if errors.Is(err, fs.ErrPermission) { return nil // 跳过权限错误 } return err } // 排除根目录自身 if path == root { return nil } // 动态扩容检测[9](@ref) if len(info.Dirs)+len(info.Files) > 1e8 { return errors.New("超出内存安全阈值(1,000,000条路径)") } //fmt.Println(path[len(RootFileName):]) if d.IsDir() { info.Dirs = append(info.Dirs, path) } else { info.Files = append(info.Files, path) } return nil }) if err != nil { return nil, &os.PathError{Op: "walk", Path: root, Err: err} // 错误包装[6](@ref) } return info, nil } // 流式处理版本(适用于超大数据集) func StreamPathInfo(root string) (<-chan string, <-chan error) { paths := make(chan string, 1000) errs := make(chan error, 1) go func() { defer close(paths) defer close(errs) err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if path != root { select { case paths <- path: default: return errors.New("消费者处理过慢") } } return nil }) if err != nil { errs <- &os.PathError{Op: "stream", Path: root, Err: err} } }() return paths, errs }