package main import ( "encoding/binary" "errors" "file-manger-server/config" "file-manger-server/db" "file-manger-server/server" "file-manger-server/service" "fmt" "io/fs" "os" "path/filepath" "strings" "sync" ) func main() { //初始化 config.ReadConfig() db.Open() //tcp服务器启动 go server.Run() //gin服务器启动 service.RunGin() os.Exit(0) // 确保程序退出 //TestHandlerReceiveFile() //TestFiles() } func TestHandlerReceiveFile() { bytes := []byte{ 0x00, } for i := 0; i < 32; i++ { bytes = append(bytes, byte(i+3)) } var fileName = "test.txt" //文件名长度 bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName))) //文件名 bytes = append(bytes, []byte(fileName)...) //文件大小 bytes = binary.BigEndian.AppendUint64(bytes, uint64(10)) var path = "./file/" + fileName bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path))) //文件路径 bytes = append(bytes, []byte(path)...) //分片大小 bytes = binary.BigEndian.AppendUint32(bytes, uint32(0)) //创建时间 bytes = binary.BigEndian.AppendUint64(bytes, uint64(999)) //修改时间 bytes = binary.BigEndian.AppendUint64(bytes, uint64(9999)) //最后访问时间 bytes = binary.BigEndian.AppendUint64(bytes, uint64(99999)) bytes = append(bytes, []byte("1234567890中文")...) server.HandlerReceiveFile(nil, bytes) } func TestBigFileSend() { //var str = "\tif err != nil {\n\t\treturn errors.New(\"FileUser transactionId is not found\")\n\t}\n\tif fileUser.UserId == 0 {\n\t\treturn errors.New(\"FileUser userId is not found\")\n\t}" //var data = []byte(str + str + str + str + str + str + "\t//config.ReadConfig()\n\t//db.Open()\n\t////server.Run()\n\t// server") ////切片1 //{ // bytes := []byte{ // 0x00, // } // for i := 0; i < 32; i++ { // bytes = append(bytes, byte(i+3)) // } // var fileName = "test.txt" // //文件名长度 // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName))) // //文件名 // bytes = append(bytes, []byte(fileName)...) // //文件大小 // bytes = binary.BigEndian.AppendUint64(bytes, uint64(1024*2+8)) // var path = "file/" + fileName // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path))) // //文件路径 // bytes = append(bytes, []byte(path)...) // //分片大小 // bytes = binary.BigEndian.AppendUint32(bytes, uint32(2)) // //创建时间 // bytes = binary.BigEndian.AppendUint64(bytes, uint64(999)) // //修改时间 // bytes = binary.BigEndian.AppendUint64(bytes, uint64(9999)) // //最后访问时间 // bytes = binary.BigEndian.AppendUint64(bytes, uint64(99999)) // server.HandlerReceiveFile(nil, bytes) //} //{ // //切片2 // bytes := []byte{ // 0x01, // } // for i := 0; i < 32; i++ { // bytes = append(bytes, byte(i+3)) // } // var fileName = "test.txt" // //切片id // bytes = binary.BigEndian.AppendUint32(bytes, uint32(0)) // var path = "file/" + fileName // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path))) // //文件路径 // bytes = append(bytes, []byte(path)...) // fmt.Println("=================================") // fmt.Println("data len : ", len(data)) // fmt.Println("=================================") // bytes = append(bytes, data...) // server.HandlerReceiveFile(nil, bytes) //} // //{ // //切片4 // bytes := []byte{ // 0x01, // } // for i := 0; i < 32; i++ { // bytes = append(bytes, byte(i+3)) // } // var fileName = "test.txt" // //切片id // bytes = binary.BigEndian.AppendUint32(bytes, uint32(2)) // var path = "file/" + fileName // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path))) // //文件路径 // bytes = append(bytes, []byte(path)...) // var end = "\n| end |" // bytes = append(bytes, []byte(end)...) // // fmt.Println("=================================") // fmt.Println("end len : ", len(end)) // fmt.Println("=================================") // server.HandlerReceiveFile(nil, bytes) //} //{ // //切片2 // bytes := []byte{ // 0x01, // } // for i := 0; i < 32; i++ { // bytes = append(bytes, byte(i+3)) // } // var fileName = "test.txt" // //切片id // bytes = binary.BigEndian.AppendUint32(bytes, uint32(1)) // var path = "file/" + fileName // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path))) // //文件路径 // bytes = append(bytes, []byte(path)...) // fmt.Println("=================================") // fmt.Println("data len : ", len(data)) // fmt.Println("=================================") // bytes = append(bytes, data...) // server.HandlerReceiveFile(nil, bytes) //} { bytes := []byte{ 0x00, } for i := 0; i < 32; i++ { bytes = append(bytes, byte(i+3)) } var fileName = "testtxt" //文件名长度 bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName))) //文件名 bytes = append(bytes, []byte(fileName)...) //文件大小 bytes = binary.BigEndian.AppendUint64(bytes, uint64(0xFFFFFFFFFFFFFFFF)) var path = "file/" + fileName bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path))) //文件路径 bytes = append(bytes, []byte(path)...) //分片大小 bytes = binary.BigEndian.AppendUint32(bytes, uint32(0)) //创建时间 bytes = binary.BigEndian.AppendUint64(bytes, uint64(1999)) //修改时间 bytes = binary.BigEndian.AppendUint64(bytes, uint64(19999)) //最后访问时间 bytes = binary.BigEndian.AppendUint64(bytes, uint64(199999)) server.HandlerReceiveFile(nil, bytes) } } func TestFiles() { info, err := GetPathInfo("D:\\") if err != nil { panic(err) } heavy, err := GoHeavy(info) if err != nil { panic(err) } fmt.Println(heavy.Dirs) } type PathInfo struct { Dirs []string // 目录路径(包含所有层级) Files []string // 文件路径 } var pathPool = sync.Pool{ New: func() interface{} { return &PathInfo{ Dirs: make([]string, 0, 1024), // 预分配容量 Files: make([]string, 0, 4096), } }, } func GoHeavy(info *PathInfo) (PathInfo, error) { m := make(map[string]bool) for i := range info.Files { path := info.Files[i] lastIndex := strings.LastIndex(path, "/") if lastIndex == -1 { lastIndex = strings.LastIndex(path, "\\") } if lastIndex == -1 { continue } //if _, ok := m[path[:lastIndex]]; !ok { // fmt.Println(path[:lastIndex]) //} m[path[:lastIndex]] = true } var dirs = make([]string, 0, len(m)) fmt.Println(len(info.Dirs)) for i := range info.Dirs { path := info.Dirs[i] if !m[path] { fmt.Println(info.Dirs[i]) dirs = append(dirs, info.Dirs[i]) } } info.Dirs = dirs return *info, nil } func GetPathInfo(root string) (*PathInfo, error) { //获取文件名 index := strings.LastIndex(root, "/") if index == -1 { index = strings.LastIndex(root, "\\") } //RootFileName := "" //if index == -1 { // RootFileName = root //} else { // RootFileName = root[:index+1] //} // 复用对象减少内存分配[9](@ref) info := pathPool.Get().(*PathInfo) // 路径有效性校验[5,6](@ref) root = filepath.Clean(root) stat, err := os.Stat(root) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, &os.PathError{Op: "stat", Path: root, Err: err} // 增强错误信息[8](@ref) } return nil, err } // 文件直接返回[4](@ref) if !stat.IsDir() { info.Files = append(info.Files, root) return info, nil } // 使用WalkDir优化遍历性能[4,9](@ref) err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { // 处理遍历错误但继续执行[7](@ref) if errors.Is(err, fs.ErrPermission) { return nil // 跳过权限错误 } return err } // 排除根目录自身 if path == root { return nil } // 动态扩容检测[9](@ref) if len(info.Dirs)+len(info.Files) > 1e8 { return errors.New("超出内存安全阈值(1,000,000条路径)") } //fmt.Println(path[len(RootFileName):]) if d.IsDir() { info.Dirs = append(info.Dirs, path) } else { info.Files = append(info.Files, path) } return nil }) if err != nil { return nil, &os.PathError{Op: "walk", Path: root, Err: err} // 错误包装[6](@ref) } return info, nil } // 流式处理版本(适用于超大数据集) func StreamPathInfo(root string) (<-chan string, <-chan error) { paths := make(chan string, 1000) errs := make(chan error, 1) go func() { defer close(paths) defer close(errs) err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if path != root { select { case paths <- path: default: return errors.New("消费者处理过慢") } } return nil }) if err != nil { errs <- &os.PathError{Op: "stream", Path: root, Err: err} } }() return paths, errs }