file-manger-server/main.go

package main

import (
	"encoding/binary"
	"errors"
	"file-manger-server/server"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

func main() {
	//config.ReadConfig()
	//db.Open()
	////server.Run()
	////service.RunGin()
	//TestHandlerReceiveFile()
	TestFiles()
}
func TestHandlerReceiveFile() {
	bytes := []byte{
		0x00,
	}
	for i := 0; i < 32; i++ {
		bytes = append(bytes, byte(i+3))
	}
	var fileName = "test.txt"
	//文件名长度
	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName)))
	//文件名
	bytes = append(bytes, []byte(fileName)...)
	//文件大小
	bytes = binary.BigEndian.AppendUint64(bytes, uint64(10))
	var path = "./file/" + fileName
	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
	//文件路径
	bytes = append(bytes, []byte(path)...)
	//分片大小
	bytes = binary.BigEndian.AppendUint32(bytes, uint32(0))
	//创建时间
	bytes = binary.BigEndian.AppendUint64(bytes, uint64(999))
	//修改时间
	bytes = binary.BigEndian.AppendUint64(bytes, uint64(9999))
	//最后访问时间
	bytes = binary.BigEndian.AppendUint64(bytes, uint64(99999))
	bytes = append(bytes, []byte("1234567890中文")...)
	server.HandlerReceiveFile(nil, bytes)
}
func TestBigFileSend() {
	//var str = "\tif err != nil {\n\t\treturn errors.New(\"FileUser transactionId is not found\")\n\t}\n\tif fileUser.UserId == 0 {\n\t\treturn errors.New(\"FileUser userId is not found\")\n\t}"
	//var data = []byte(str + str + str + str + str + str + "\t//config.ReadConfig()\n\t//db.Open()\n\t////server.Run()\n\t// server")
	////切片1
	//{
	//	bytes := []byte{
	//		0x00,
	//	}
	//	for i := 0; i < 32; i++ {
	//		bytes = append(bytes, byte(i+3))
	//	}
	//	var fileName = "test.txt"
	//	//文件名长度
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName)))
	//	//文件名
	//	bytes = append(bytes, []byte(fileName)...)
	//	//文件大小
	//	bytes = binary.BigEndian.AppendUint64(bytes, uint64(1024*2+8))
	//	var path = "file/" + fileName
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
	//	//文件路径
	//	bytes = append(bytes, []byte(path)...)
	//	//分片大小
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(2))
	//	//创建时间
	//	bytes = binary.BigEndian.AppendUint64(bytes, uint64(999))
	//	//修改时间
	//	bytes = binary.BigEndian.AppendUint64(bytes, uint64(9999))
	//	//最后访问时间
	//	bytes = binary.BigEndian.AppendUint64(bytes, uint64(99999))
	//	server.HandlerReceiveFile(nil, bytes)
	//}
	//{
	//	//切片2
	//	bytes := []byte{
	//		0x01,
	//	}
	//	for i := 0; i < 32; i++ {
	//		bytes = append(bytes, byte(i+3))
	//	}
	//	var fileName = "test.txt"
	//	//切片id
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(0))
	//	var path = "file/" + fileName
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
	//	//文件路径
	//	bytes = append(bytes, []byte(path)...)
	//	fmt.Println("=================================")
	//	fmt.Println("data len : ", len(data))
	//	fmt.Println("=================================")
	//	bytes = append(bytes, data...)
	//	server.HandlerReceiveFile(nil, bytes)
	//}
	//
	//{
	//	//切片4
	//	bytes := []byte{
	//		0x01,
	//	}
	//	for i := 0; i < 32; i++ {
	//		bytes = append(bytes, byte(i+3))
	//	}
	//	var fileName = "test.txt"
	//	//切片id
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(2))
	//	var path = "file/" + fileName
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
	//	//文件路径
	//	bytes = append(bytes, []byte(path)...)
	//	var end = "\n| end |"
	//	bytes = append(bytes, []byte(end)...)
	//
	//	fmt.Println("=================================")
	//	fmt.Println("end len : ", len(end))
	//	fmt.Println("=================================")
	//	server.HandlerReceiveFile(nil, bytes)
	//}
	//{
	//	//切片2
	//	bytes := []byte{
	//		0x01,
	//	}
	//	for i := 0; i < 32; i++ {
	//		bytes = append(bytes, byte(i+3))
	//	}
	//	var fileName = "test.txt"
	//	//切片id
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(1))
	//	var path = "file/" + fileName
	//	bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
	//	//文件路径
	//	bytes = append(bytes, []byte(path)...)
	//	fmt.Println("=================================")
	//	fmt.Println("data len : ", len(data))
	//	fmt.Println("=================================")
	//	bytes = append(bytes, data...)
	//	server.HandlerReceiveFile(nil, bytes)
	//}
	{
		bytes := []byte{
			0x00,
		}
		for i := 0; i < 32; i++ {
			bytes = append(bytes, byte(i+3))
		}
		var fileName = "testtxt"
		//文件名长度
		bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(fileName)))
		//文件名
		bytes = append(bytes, []byte(fileName)...)
		//文件大小
		bytes = binary.BigEndian.AppendUint64(bytes, uint64(0xFFFFFFFFFFFFFFFF))
		var path = "file/" + fileName
		bytes = binary.BigEndian.AppendUint32(bytes, uint32(len(path)))
		//文件路径
		bytes = append(bytes, []byte(path)...)
		//分片大小
		bytes = binary.BigEndian.AppendUint32(bytes, uint32(0))
		//创建时间
		bytes = binary.BigEndian.AppendUint64(bytes, uint64(1999))
		//修改时间
		bytes = binary.BigEndian.AppendUint64(bytes, uint64(19999))
		//最后访问时间
		bytes = binary.BigEndian.AppendUint64(bytes, uint64(199999))
		server.HandlerReceiveFile(nil, bytes)
	}

}

func TestFiles() {
	info, err := GetPathInfo("D:\\")
	if err != nil {
		panic(err)
	}
	heavy, err := GoHeavy(info)
	if err != nil {
		panic(err)
	}
	fmt.Println(heavy.Dirs)

}

type PathInfo struct {
	Dirs  []string // 目录路径（包含所有层级）
	Files []string // 文件路径
}

var pathPool = sync.Pool{
	New: func() interface{} {
		return &PathInfo{
			Dirs:  make([]string, 0, 1024), // 预分配容量
			Files: make([]string, 0, 4096),
		}
	},
}

func GoHeavy(info *PathInfo) (PathInfo, error) {
	m := make(map[string]bool)
	for i := range info.Files {
		path := info.Files[i]
		lastIndex := strings.LastIndex(path, "/")

		if lastIndex == -1 {
			lastIndex = strings.LastIndex(path, "\\")
		}
		if lastIndex == -1 {
			continue
		}
		//if _, ok := m[path[:lastIndex]]; !ok {
		//	fmt.Println(path[:lastIndex])
		//}
		m[path[:lastIndex]] = true
	}
	var dirs = make([]string, 0, len(m))
	fmt.Println(len(info.Dirs))
	for i := range info.Dirs {
		path := info.Dirs[i]
		if !m[path] {
			fmt.Println(info.Dirs[i])
			dirs = append(dirs, info.Dirs[i])
		}
	}
	info.Dirs = dirs
	return *info, nil
}

func GetPathInfo(root string) (*PathInfo, error) {
	//获取文件名
	index := strings.LastIndex(root, "/")
	if index == -1 {
		index = strings.LastIndex(root, "\\")
	}
	//RootFileName := ""
	//if index == -1 {
	//	RootFileName = root
	//} else {
	//	RootFileName = root[:index+1]
	//}

	// 复用对象减少内存分配[9](@ref)
	info := pathPool.Get().(*PathInfo)

	// 路径有效性校验[5,6](@ref)
	root = filepath.Clean(root)
	stat, err := os.Stat(root)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return nil, &os.PathError{Op: "stat", Path: root, Err: err} // 增强错误信息[8](@ref)
		}
		return nil, err
	}

	// 文件直接返回[4](@ref)
	if !stat.IsDir() {
		info.Files = append(info.Files, root)
		return info, nil
	}

	// 使用WalkDir优化遍历性能[4,9](@ref)
	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			// 处理遍历错误但继续执行[7](@ref)
			if errors.Is(err, fs.ErrPermission) {
				return nil // 跳过权限错误
			}
			return err
		}

		// 排除根目录自身
		if path == root {
			return nil
		}

		// 动态扩容检测[9](@ref)
		if len(info.Dirs)+len(info.Files) > 1e8 {
			return errors.New("超出内存安全阈值(1,000,000条路径)")
		}
		//fmt.Println(path[len(RootFileName):])
		if d.IsDir() {
			info.Dirs = append(info.Dirs, path)
		} else {
			info.Files = append(info.Files, path)
		}
		return nil
	})

	if err != nil {
		return nil, &os.PathError{Op: "walk", Path: root, Err: err} // 错误包装[6](@ref)
	}

	return info, nil
}

// 流式处理版本（适用于超大数据集）
func StreamPathInfo(root string) (<-chan string, <-chan error) {
	paths := make(chan string, 1000)
	errs := make(chan error, 1)

	go func() {
		defer close(paths)
		defer close(errs)

		err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
			if err != nil {
				return err
			}
			if path != root {
				select {
				case paths <- path:
				default:
					return errors.New("消费者处理过慢")
				}
			}
			return nil
		})

		if err != nil {
			errs <- &os.PathError{Op: "stream", Path: root, Err: err}
		}
	}()

	return paths, errs
}