|
|
@@ -0,0 +1,334 @@
|
|
|
+package main
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/binary"
|
|
|
+ "errors"
|
|
|
+ "file-manger-server/server"
|
|
|
+ "fmt"
|
|
|
+ "io/fs"
|
|
|
+ "os"
|
|
|
+ "path/filepath"
|
|
|
+ "strings"
|
|
|
+ "sync"
|
|
|
+)
|
|
|
+
|
|
|
+func main() {
|
|
|
+ //config.ReadConfig()
|
|
|
+ //db.Open()
|
|
|
+ ////server.Run()
|
|
|
+ ////service.RunGin()
|
|
|
+ //TestHandlerReceiveFile()
|
|
|
+ TestFiles()
|
|
|
+}
|
|
|
+func TestHandlerReceiveFile() {
|
|
|
+ bytes := []byte{
|
|
|
+ 0x00,
|
|
|
+ }
|
|
|
+ for i := 0; i < 32; i++ {
|
|
|
+ bytes = append(bytes, byte(i+3))
|
|
|
+ }
|
|
|
+ var fileName = "test.txt"
|
|
|
+ //文件名长度
|
|
|
+ bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName)))
|
|
|
+ //文件名
|
|
|
+ bytes = append(bytes, []byte(fileName)...)
|
|
|
+ //文件大小
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(10))
|
|
|
+ var path = "./file/" + fileName
|
|
|
+ bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
|
|
|
+ //文件路径
|
|
|
+ bytes = append(bytes, []byte(path)...)
|
|
|
+ //分片大小
|
|
|
+ bytes = binary.BigEndian.AppendUint32(bytes, uint32(0))
|
|
|
+ //创建时间
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(999))
|
|
|
+ //修改时间
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(9999))
|
|
|
+ //最后访问时间
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(99999))
|
|
|
+ bytes = append(bytes, []byte("1234567890中文")...)
|
|
|
+ server.HandlerReceiveFile(nil, bytes)
|
|
|
+}
|
|
|
+func TestBigFileSend() {
|
|
|
+ //var str = "\tif err != nil {\n\t\treturn errors.New(\"FileUser transactionId is not found\")\n\t}\n\tif fileUser.UserId == 0 {\n\t\treturn errors.New(\"FileUser userId is not found\")\n\t}"
|
|
|
+ //var data = []byte(str + str + str + str + str + str + "\t//config.ReadConfig()\n\t//db.Open()\n\t////server.Run()\n\t// server")
|
|
|
+ ////切片1
|
|
|
+ //{
|
|
|
+ // bytes := []byte{
|
|
|
+ // 0x00,
|
|
|
+ // }
|
|
|
+ // for i := 0; i < 32; i++ {
|
|
|
+ // bytes = append(bytes, byte(i+3))
|
|
|
+ // }
|
|
|
+ // var fileName = "test.txt"
|
|
|
+ // //文件名长度
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName)))
|
|
|
+ // //文件名
|
|
|
+ // bytes = append(bytes, []byte(fileName)...)
|
|
|
+ // //文件大小
|
|
|
+ // bytes = binary.BigEndian.AppendUint64(bytes, uint64(1024*2+8))
|
|
|
+ // var path = "file/" + fileName
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
|
|
|
+ // //文件路径
|
|
|
+ // bytes = append(bytes, []byte(path)...)
|
|
|
+ // //分片大小
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(2))
|
|
|
+ // //创建时间
|
|
|
+ // bytes = binary.BigEndian.AppendUint64(bytes, uint64(999))
|
|
|
+ // //修改时间
|
|
|
+ // bytes = binary.BigEndian.AppendUint64(bytes, uint64(9999))
|
|
|
+ // //最后访问时间
|
|
|
+ // bytes = binary.BigEndian.AppendUint64(bytes, uint64(99999))
|
|
|
+ // server.HandlerReceiveFile(nil, bytes)
|
|
|
+ //}
|
|
|
+ //{
|
|
|
+ // //切片2
|
|
|
+ // bytes := []byte{
|
|
|
+ // 0x01,
|
|
|
+ // }
|
|
|
+ // for i := 0; i < 32; i++ {
|
|
|
+ // bytes = append(bytes, byte(i+3))
|
|
|
+ // }
|
|
|
+ // var fileName = "test.txt"
|
|
|
+ // //切片id
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(0))
|
|
|
+ // var path = "file/" + fileName
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
|
|
|
+ // //文件路径
|
|
|
+ // bytes = append(bytes, []byte(path)...)
|
|
|
+ // fmt.Println("=================================")
|
|
|
+ // fmt.Println("data len : ", len(data))
|
|
|
+ // fmt.Println("=================================")
|
|
|
+ // bytes = append(bytes, data...)
|
|
|
+ // server.HandlerReceiveFile(nil, bytes)
|
|
|
+ //}
|
|
|
+ //
|
|
|
+ //{
|
|
|
+ // //切片4
|
|
|
+ // bytes := []byte{
|
|
|
+ // 0x01,
|
|
|
+ // }
|
|
|
+ // for i := 0; i < 32; i++ {
|
|
|
+ // bytes = append(bytes, byte(i+3))
|
|
|
+ // }
|
|
|
+ // var fileName = "test.txt"
|
|
|
+ // //切片id
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(2))
|
|
|
+ // var path = "file/" + fileName
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
|
|
|
+ // //文件路径
|
|
|
+ // bytes = append(bytes, []byte(path)...)
|
|
|
+ // var end = "\n| end |"
|
|
|
+ // bytes = append(bytes, []byte(end)...)
|
|
|
+ //
|
|
|
+ // fmt.Println("=================================")
|
|
|
+ // fmt.Println("end len : ", len(end))
|
|
|
+ // fmt.Println("=================================")
|
|
|
+ // server.HandlerReceiveFile(nil, bytes)
|
|
|
+ //}
|
|
|
+ //{
|
|
|
+ // //切片2
|
|
|
+ // bytes := []byte{
|
|
|
+ // 0x01,
|
|
|
+ // }
|
|
|
+ // for i := 0; i < 32; i++ {
|
|
|
+ // bytes = append(bytes, byte(i+3))
|
|
|
+ // }
|
|
|
+ // var fileName = "test.txt"
|
|
|
+ // //切片id
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(1))
|
|
|
+ // var path = "file/" + fileName
|
|
|
+ // bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
|
|
|
+ // //文件路径
|
|
|
+ // bytes = append(bytes, []byte(path)...)
|
|
|
+ // fmt.Println("=================================")
|
|
|
+ // fmt.Println("data len : ", len(data))
|
|
|
+ // fmt.Println("=================================")
|
|
|
+ // bytes = append(bytes, data...)
|
|
|
+ // server.HandlerReceiveFile(nil, bytes)
|
|
|
+ //}
|
|
|
+ {
|
|
|
+ bytes := []byte{
|
|
|
+ 0x00,
|
|
|
+ }
|
|
|
+ for i := 0; i < 32; i++ {
|
|
|
+ bytes = append(bytes, byte(i+3))
|
|
|
+ }
|
|
|
+ var fileName = "testtxt"
|
|
|
+ //文件名长度
|
|
|
+ bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName)))
|
|
|
+ //文件名
|
|
|
+ bytes = append(bytes, []byte(fileName)...)
|
|
|
+ //文件大小
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(0xFFFFFFFFFFFFFFFF))
|
|
|
+ var path = "file/" + fileName
|
|
|
+ bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
|
|
|
+ //文件路径
|
|
|
+ bytes = append(bytes, []byte(path)...)
|
|
|
+ //分片大小
|
|
|
+ bytes = binary.BigEndian.AppendUint32(bytes, uint32(0))
|
|
|
+ //创建时间
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(1999))
|
|
|
+ //修改时间
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(19999))
|
|
|
+ //最后访问时间
|
|
|
+ bytes = binary.BigEndian.AppendUint64(bytes, uint64(199999))
|
|
|
+ server.HandlerReceiveFile(nil, bytes)
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+func TestFiles() {
|
|
|
+ info, err := GetPathInfo("D:\\")
|
|
|
+ if err != nil {
|
|
|
+ panic(err)
|
|
|
+ }
|
|
|
+ heavy, err := GoHeavy(info)
|
|
|
+ if err != nil {
|
|
|
+ panic(err)
|
|
|
+ }
|
|
|
+ fmt.Println(heavy.Dirs)
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+type PathInfo struct {
|
|
|
+ Dirs []string // 目录路径(包含所有层级)
|
|
|
+ Files []string // 文件路径
|
|
|
+}
|
|
|
+
|
|
|
+var pathPool = sync.Pool{
|
|
|
+ New: func() interface{} {
|
|
|
+ return &PathInfo{
|
|
|
+ Dirs: make([]string, 0, 1024), // 预分配容量
|
|
|
+ Files: make([]string, 0, 4096),
|
|
|
+ }
|
|
|
+ },
|
|
|
+}
|
|
|
+
|
|
|
+func GoHeavy(info *PathInfo) (PathInfo, error) {
|
|
|
+ m := make(map[string]bool)
|
|
|
+ for i := range info.Files {
|
|
|
+ path := info.Files[i]
|
|
|
+ lastIndex := strings.LastIndex(path, "/")
|
|
|
+
|
|
|
+ if lastIndex == -1 {
|
|
|
+ lastIndex = strings.LastIndex(path, "\\")
|
|
|
+ }
|
|
|
+ if lastIndex == -1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ //if _, ok := m[path[:lastIndex]]; !ok {
|
|
|
+ // fmt.Println(path[:lastIndex])
|
|
|
+ //}
|
|
|
+ m[path[:lastIndex]] = true
|
|
|
+ }
|
|
|
+ var dirs = make([]string, 0, len(m))
|
|
|
+ fmt.Println(len(info.Dirs))
|
|
|
+ for i := range info.Dirs {
|
|
|
+ path := info.Dirs[i]
|
|
|
+ if !m[path] {
|
|
|
+ fmt.Println(info.Dirs[i])
|
|
|
+ dirs = append(dirs, info.Dirs[i])
|
|
|
+ }
|
|
|
+ }
|
|
|
+ info.Dirs = dirs
|
|
|
+ return *info, nil
|
|
|
+}
|
|
|
+
|
|
|
+func GetPathInfo(root string) (*PathInfo, error) {
|
|
|
+ //获取文件名
|
|
|
+ index := strings.LastIndex(root, "/")
|
|
|
+ if index == -1 {
|
|
|
+ index = strings.LastIndex(root, "\\")
|
|
|
+ }
|
|
|
+ //RootFileName := ""
|
|
|
+ //if index == -1 {
|
|
|
+ // RootFileName = root
|
|
|
+ //} else {
|
|
|
+ // RootFileName = root[:index+1]
|
|
|
+ //}
|
|
|
+
|
|
|
+ // 复用对象减少内存分配[9](@ref)
|
|
|
+ info := pathPool.Get().(*PathInfo)
|
|
|
+
|
|
|
+ // 路径有效性校验[5,6](@ref)
|
|
|
+ root = filepath.Clean(root)
|
|
|
+ stat, err := os.Stat(root)
|
|
|
+ if err != nil {
|
|
|
+ if errors.Is(err, fs.ErrNotExist) {
|
|
|
+ return nil, &os.PathError{Op: "stat", Path: root, Err: err} // 增强错误信息[8](@ref)
|
|
|
+ }
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+
|
|
|
+ // 文件直接返回[4](@ref)
|
|
|
+ if !stat.IsDir() {
|
|
|
+ info.Files = append(info.Files, root)
|
|
|
+ return info, nil
|
|
|
+ }
|
|
|
+
|
|
|
+ // 使用WalkDir优化遍历性能[4,9](@ref)
|
|
|
+ err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
|
|
+ if err != nil {
|
|
|
+ // 处理遍历错误但继续执行[7](@ref)
|
|
|
+ if errors.Is(err, fs.ErrPermission) {
|
|
|
+ return nil // 跳过权限错误
|
|
|
+ }
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ // 排除根目录自身
|
|
|
+ if path == root {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
+ // 动态扩容检测[9](@ref)
|
|
|
+ if len(info.Dirs)+len(info.Files) > 1e8 {
|
|
|
+ return errors.New("超出内存安全阈值(1,000,000条路径)")
|
|
|
+ }
|
|
|
+ //fmt.Println(path[len(RootFileName):])
|
|
|
+ if d.IsDir() {
|
|
|
+ info.Dirs = append(info.Dirs, path)
|
|
|
+ } else {
|
|
|
+ info.Files = append(info.Files, path)
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+ })
|
|
|
+
|
|
|
+ if err != nil {
|
|
|
+ return nil, &os.PathError{Op: "walk", Path: root, Err: err} // 错误包装[6](@ref)
|
|
|
+ }
|
|
|
+
|
|
|
+ return info, nil
|
|
|
+}
|
|
|
+
|
|
|
+// 流式处理版本(适用于超大数据集)
|
|
|
+func StreamPathInfo(root string) (<-chan string, <-chan error) {
|
|
|
+ paths := make(chan string, 1000)
|
|
|
+ errs := make(chan error, 1)
|
|
|
+
|
|
|
+ go func() {
|
|
|
+ defer close(paths)
|
|
|
+ defer close(errs)
|
|
|
+
|
|
|
+ err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if path != root {
|
|
|
+ select {
|
|
|
+ case paths <- path:
|
|
|
+ default:
|
|
|
+ return errors.New("消费者处理过慢")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+ })
|
|
|
+
|
|
|
+ if err != nil {
|
|
|
+ errs <- &os.PathError{Op: "stream", Path: root, Err: err}
|
|
|
+ }
|
|
|
+ }()
|
|
|
+
|
|
|
+ return paths, errs
|
|
|
+}
|