commit 011eb4348b4c63f76b1eac5ee6bcbb976fd68eea Author: 4566704 <4566704@qq.com> Date: Fri Sep 25 15:21:21 2026 +0800 feat: 独立通用大文件分片上传服务端核心（自 zogo/fileupload v0.3.2 抽出） - 六接口路由：init / chunk / merge / check / task / abort - Repository 接口注入存储，宿主实现 12 方法即可接入 - OnMerged / OnInstantHit 回调衔接宿主业务（如写云端文件表） - Envelope 注入响应封装，与宿主 httpx 解耦 - 与 zogo v0.3.2 的 fileupload 子包同源，后续在此仓库独立演进 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5f8baef --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.exe +*.test +*.out +.data/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..5cf2ef8 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# fileupload + +通用大文件分片上传 Go/Gin 服务端核心（独立库，原 zogo/fileupload 子包）。 +六接口（init/chunk/merge/check/task/abort）+ 分片幂等 + 断点续传 + `.part` 原子改名 + 合并整文件 MD5 校验 + 秒传。 +不依赖宿主的响应封装、ORM 或业务表——存储走 `Repository` 接口注入，业务衔接走 `Options` 回调。 + +## 用法 + +```go +import "git.zeroonesoft.cn/golib/fileupload" + +opts := fileupload.Options{ + ChunkRoot: "./data/upload-chunks", // 分片临时目录（建议在静态根之外） + CloudDir: "./data/uploads/cloud", // 合并产物目录 + WebPathPrefix: "cloud", // storagePath 的 Web 前缀（须与 CloudDir 相对静态根的子目录一致） + URLPrefix: "/uploads", // fileUrl = URLPrefix + "/" + storagePath + MaxTotalSizeMB: 2048, // 0=不限 + OnMerged: func(t fileupload.TaskInfo) error { + // 合并成功：宿主写自己的业务表（如云盘记录行）；失败则 merge 整体失败 + return nil + }, + OnInstantHit: func(t fileupload.TaskInfo) error { + // 秒传命中：宿主确保业务侧可见行（可空回调） + return nil + }, + Envelope: func(c *gin.Context, resp any, err error) { + // 宿主自己的响应封装，如 httpx.HandleResult(c, resp, err) + }, +} +_ = fileupload.Mount(api.Group("/upload"), opts, myRepo, + fileupload.WithIdentity(func(c *gin.Context) (string, int64) { + return c.GetString("UserName"), c.GetInt64("TenantId") + })) +``` + +`Repository`（12 个方法）由宿主按自己的表结构实现；软删/硬删自定，但删除后查询必须不可见。 +鉴权由宿主路由组的 JWT 中间件统一负责，本库不感知。 + +## 版本 + +- v0.1.0 首版：自 FileUpload 项目 restful/upload 模块抽出 diff --git a/fileupload.go b/fileupload.go new file mode 100644 index 0000000..33aef29 --- /dev/null +++ b/fileupload.go @@ -0,0 +1,137 @@ +// Package fileupload 通用大文件分片上传（Go/Gin 服务端核心）。 +// +// 职责：init/chunk/merge/check/task/abort 六接口的路由挂载与业务逻辑—— +// 分片幂等、断点续传、.part 原子改名、合并时整文件 MD5 校验、秒传。 +// 宿主差异全部通过 Options 与 Repository 注入，本包不依赖任何宿主的 +// 响应封装、ORM 生成代码或业务表： +// +// fileupload.Mount(api.Group("/upload"), opts, repo) +// +// 与业务层的衔接： +// - merge 成功 → OnMerged(TaskInfo)（宿主写自己的业务表/云盘记录）； +// 回调失败则 merge 整体失败、任务置 error +// - 秒传命中 → OnInstantHit(TaskInfo)（宿主确保业务侧可见行，可空回调） +// - 响应封装 → Options.Envelope（必填，如 httpx.HandleResult / gin.H{code:200,...}） +package fileupload + +import ( + "errors" + "math" + "path" + "strings" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +// TaskInfo 上传任务的库级视图（与宿主的表结构解耦）。 +type TaskInfo struct { + Id int64 + UploadId string + FileName string + FileSize int64 + FileMd5 string + MimeType string + ChunkSize int64 + TotalChunk int32 + Status string // uploading / merging / done / error + StoragePath string // 相对路径（如 cloud/xxx），由宿主的目录约定决定 + Operator string + TenantId int64 +} + +// ChunkInfo 分片记录的库级视图。 +type ChunkInfo struct { + UploadId string + ChunkIndex int32 + ChunkSize int64 + ChunkMd5 string +} + +// Status 常量。 +const ( + StatusUploading = "uploading" + StatusMerging = "merging" + StatusDone = "done" + StatusError = "error" +) + +// DefaultMaxChunkBytes 单分片硬上限（防滥用，与常见 multipart 内存缓冲同量级）。 +const DefaultMaxChunkBytes int64 = 64 << 20 + +// Envelope 宿主响应封装：resp 非 nil 时写成功响应，err 非 nil 时写失败响应。 +type Envelope func(c *gin.Context, resp any, err error) + +// Options 宿主注入配置。 +type Options struct { + // ChunkRoot 分片临时根目录；库在其下建 / 子目录。 + // 建议置于宿主静态根之外，避免分片被静态服务暴露。 + ChunkRoot string + // CloudDir 合并产物目录（宿主的云盘/文件目录约定）。 + CloudDir string + // WebPathPrefix storagePath 的 Web 相对前缀（如 "cloud"），空则无前缀。 + // 最终 storagePath = WebPathPrefix + "/" + 产物名（存库、回调、BS 拼接均用该值）， + // 需与 CloudDir 相对宿主静态根的子目录一致。 + WebPathPrefix string + // URLPrefix 拼接 fileUrl 的静态前缀，默认 "/uploads"； + // 最终 fileUrl = URLPrefix + "/" + storagePath。 + URLPrefix string + // MaxTotalSizeMB 文件总大小上限（MB），0=不限制。 + MaxTotalSizeMB int64 + // MaxChunkBytes 单分片字节上限，0=DefaultMaxChunkBytes(64MB)。 + MaxChunkBytes int64 + // OnMerged 合并成功回调（宿主写业务表等）。返回 err 则 merge 失败、任务置 error。 + OnMerged func(t TaskInfo) error + // OnInstantHit 秒传命中回调（宿主确保业务侧可见行）。可空。 + OnInstantHit func(t TaskInfo) error + // Envelope 响应封装（必填）。 + Envelope Envelope +} + +func (o *Options) validate() error { + if o.ChunkRoot == "" || o.CloudDir == "" { + return errors.New("fileupload: Options.ChunkRoot / CloudDir 必填") + } + if o.Envelope == nil { + return errors.New("fileupload: Options.Envelope 必填") + } + if o.MaxChunkBytes <= 0 { + o.MaxChunkBytes = DefaultMaxChunkBytes + } + if o.URLPrefix == "" { + o.URLPrefix = "/uploads" + } + return nil +} + +func (o *Options) maxChunkBytes() int64 { + if o.MaxChunkBytes > 0 { + return o.MaxChunkBytes + } + return DefaultMaxChunkBytes +} + +// expectChunks 期望分片数：ceil(fileSize/chunkSize)，空文件视为 1 个空分片。 +func expectChunks(fileSize, chunkSize int64) int { + if chunkSize <= 0 { + return 0 + } + n := int(math.Ceil(float64(fileSize) / float64(chunkSize))) + if n == 0 { + n = 1 + } + return n +} + +// sanitizeFileName 取路径基名并剥离路径穿越/非法字符。 +func sanitizeFileName(name string) string { + name = path.Base(strings.TrimSpace(name)) + name = strings.ReplaceAll(name, "\\", "") + if name == "." || name == ".." || name == "/" || name == "" { + return "" + } + return name +} + +// newUploadID 生成上传任务 ID（uuid v4，URL 安全）。 +func newUploadID() string { return uuid.NewString() } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9ebe761 --- /dev/null +++ b/go.mod @@ -0,0 +1,40 @@ +module git.zeroonesoft.cn/golib/fileupload + +go 1.25.0 + +require ( + github.com/gin-gonic/gin v1.12.0 + github.com/google/uuid v1.6.0 +) + +require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0b62b83 --- /dev/null +++ b/go.sum @@ -0,0 +1,91 @@ +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mount.go b/mount.go new file mode 100644 index 0000000..7c9c1f1 --- /dev/null +++ b/mount.go @@ -0,0 +1,119 @@ +package fileupload + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" +) + +// Mount 挂载分片上传路由到 r（宿主自行决定路由组，如 api.Group("/upload")）： +// +// POST /init 初始化（JSON：fileName/fileSize/fileMD5/mimeType/chunkSize/totalChunk） +// POST /chunk 上传分片（multipart：uploadId/chunkIndex/chunk），幂等 +// POST /merge 合并分片（JSON：uploadId），幂等 +// GET /check?fileMD5= 秒传/续传检查 +// GET /task/:uploadId 任务详情与已确认分片 +// DELETE /task/:uploadId 取消任务并清理 +// +// 身份（operator/tenant）由宿主在 handler 外层取好后经 identity 注入； +// 响应统一走 Options.Envelope。 +type Identity func(c *gin.Context) (operator string, tenantID int64) + +type mountConfig struct { + identity Identity +} + +// MountOption 挂载选项。 +type MountOption func(*mountConfig) + +// WithIdentity 注入身份提取（从 JWT 中间件写入的 context 取 UserId/UserName 等）。 +func WithIdentity(fn Identity) MountOption { + return func(m *mountConfig) { m.identity = fn } +} + +// Mount 挂载路由。r 建议为 api.Group("/upload")。 +func Mount(r *gin.RouterGroup, opts Options, repo Repository, mos ...MountOption) error { + if err := opts.validate(); err != nil { + return err + } + if repo == nil { + return errors.New("fileupload: Repository 必填") + } + mc := &mountConfig{} + for _, mo := range mos { + mo(mc) + } + svc := newService(opts, repo) + + r.POST("/init", func(c *gin.Context) { + var req InitReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"code": 1, "msg": "参数错误: " + err.Error()}) + return + } + operator, tenantID := mc.identity(c) + resp, err := svc.init(&req, operator, tenantID) + opts.Envelope(c, resp, err) + }) + + r.POST("/chunk", func(c *gin.Context) { + form, err := c.MultipartForm() + if err != nil { + opts.Envelope(c, nil, errors.New("请使用 multipart/form-data 上传")) + return + } + files := form.File["chunk"] + if len(files) != 1 { + opts.Envelope(c, nil, errors.New("缺少分片文件字段 chunk")) + return + } + chunkIndex := atoiDefault(c.PostForm("chunkIndex"), -1) + resp, err := svc.chunk(c.PostForm("uploadId"), chunkIndex, files[0]) + if err != nil { + opts.Envelope(c, nil, err) + return + } + opts.Envelope(c, resp, nil) + }) + + r.POST("/merge", func(c *gin.Context) { + var req MergeReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"code": 1, "msg": "参数错误: " + err.Error()}) + return + } + resp, err := svc.merge(req.UploadId) + opts.Envelope(c, resp, err) + }) + + r.GET("/check", func(c *gin.Context) { + resp, err := svc.check(c.Query("fileMD5")) + opts.Envelope(c, resp, err) + }) + + r.GET("/task/:uploadId", func(c *gin.Context) { + resp, err := svc.taskInfo(c.Param("uploadId")) + opts.Envelope(c, resp, err) + }) + + r.DELETE("/task/:uploadId", func(c *gin.Context) { + resp, err := svc.abort(c.Param("uploadId")) + opts.Envelope(c, resp, err) + }) + return nil +} + +func atoiDefault(s string, def int) int { + if s == "" { + return def + } + n := 0 + for _, r := range s { + if r < '0' || r > '9' { + return def + } + n = n*10 + int(r-'0') + } + return n +} diff --git a/repository.go b/repository.go new file mode 100644 index 0000000..ae23837 --- /dev/null +++ b/repository.go @@ -0,0 +1,32 @@ +package fileupload + +// Repository 存储层接口：宿主用自己的 ORM/表结构实现（本包不关心表怎么建）。 +// 语义约定： +// - Delete* 为软删或硬删由宿主决定，但之后 GetTaskByUploadID / 各 Find 必须查不到； +// - Find* 只需返回未删除的最新一条。 +type Repository interface { + // CreateTask 新建任务（Id/UploadId 已由本包赋值）。 + CreateTask(t *TaskInfo) error + // GetTaskByUploadID 按 uploadId 取未删除任务；不存在返回 (nil, nil)。 + GetTaskByUploadID(uploadID string) (*TaskInfo, error) + // FindDoneByMD5 取同 MD5 已完成任务中最新一条；无则 (nil, nil)。 + FindDoneByMD5(md5 string) (*TaskInfo, error) + // FindUploadingByMD5 取同 MD5 进行中任务一条（断点续传）；无则 (nil, nil)。 + FindUploadingByMD5(md5 string) (*TaskInfo, error) + // UpdateTaskMerged 合并成功：置 done 并记录 storagePath。 + UpdateTaskMerged(id int64, storagePath string) error + // SetTaskStatus 更新任务状态（error/merging 等流转）。 + SetTaskStatus(id int64, status string) error + // CountChunks 统计任务已确认分片数。 + CountChunks(uploadID string) (int, error) + // ChunkExists 分片是否已确认（幂等依据）。 + ChunkExists(uploadID string, index int32) (bool, error) + // CreateChunk 写入分片确认记录。 + CreateChunk(c *ChunkInfo) error + // ListChunkIndexes 已确认分片序号升序。 + ListChunkIndexes(uploadID string) ([]int32, error) + // DeleteChunks 删除任务的分片记录。 + DeleteChunks(uploadID string) error + // DeleteTaskByUploadID 删除任务。 + DeleteTaskByUploadID(uploadID string) error +} diff --git a/service.go b/service.go new file mode 100644 index 0000000..ef16207 --- /dev/null +++ b/service.go @@ -0,0 +1,382 @@ +package fileupload + +import ( + "crypto/md5" + "errors" + "fmt" + "io" + "mime/multipart" + "os" + "path/filepath" + "strconv" + "time" +) + +// service 核心业务，持 Options + Repository。 +type service struct { + opts Options + repo Repository +} + +func newService(opts Options, repo Repository) *service { + return &service{opts: opts, repo: repo} +} + +func (s *service) chunkDir(uploadID string) string { + return filepath.Join(s.opts.ChunkRoot, uploadID) +} + +func (s *service) fileURL(storagePath string) string { + return s.opts.URLPrefix + "/" + storagePath +} + +// ── init ──────────────────────────────────────────────────────────── + +type InitReq struct { + FileName string `json:"fileName" binding:"required,max=255"` + FileSize int64 `json:"fileSize" binding:"min=0"` + FileMD5 string `json:"fileMD5" binding:"required,len=32"` + MimeType string `json:"mimeType" binding:"max=128"` + ChunkSize int64 `json:"chunkSize" binding:"required,min=1"` + TotalChunk int `json:"totalChunk" binding:"required,min=1"` +} + +func (s *service) init(req *InitReq, operator string, tenantID int64) (any, error) { + if req.ChunkSize <= 0 || req.ChunkSize > s.opts.maxChunkBytes() { + return nil, fmt.Errorf("chunkSize 超出范围 (0, %d]", s.opts.maxChunkBytes()) + } + if req.FileSize < 0 { + return nil, errors.New("fileSize 非法") + } + if len(req.FileMD5) != 32 { + return nil, errors.New("fileMD5 必须为 32 位十六进制") + } + if expect := expectChunks(req.FileSize, req.ChunkSize); req.TotalChunk != expect { + return nil, fmt.Errorf("totalChunk(%d) 与 fileSize(%d)/chunkSize(%d) 不匹配，应为 %d", + req.TotalChunk, req.FileSize, req.ChunkSize, expect) + } + if s.opts.MaxTotalSizeMB > 0 && req.FileSize > s.opts.MaxTotalSizeMB*1024*1024 { + return nil, fmt.Errorf("文件超过总大小上限 %dMB", s.opts.MaxTotalSizeMB) + } + + // 1. 秒传：同 MD5 已完成任务，复用其文件 + done, err := s.repo.FindDoneByMD5(req.FileMD5) + if err != nil { + return nil, err + } + if done != nil && done.StoragePath != "" { + if s.opts.OnInstantHit != nil { + if err := s.opts.OnInstantHit(*done); err != nil { + return nil, err + } + } + return map[string]any{ + "uploadId": "", + "skipUpload": true, + "fileUrl": s.fileURL(done.StoragePath), + }, nil + } + + // 2. 断点续传：同 MD5 进行中任务 + resuming, err := s.repo.FindUploadingByMD5(req.FileMD5) + if err != nil { + return nil, err + } + if resuming != nil { + indexes, err := s.repo.ListChunkIndexes(resuming.UploadId) + if err != nil { + return nil, err + } + return map[string]any{ + "uploadId": resuming.UploadId, + "skipUpload": false, + "uploadedChunks": indexes, + }, nil + } + + // 3. 新建任务 + name := sanitizeFileName(req.FileName) + if name == "" { + return nil, errors.New("文件名非法") + } + task := TaskInfo{ + UploadId: newUploadID(), + FileName: name, + FileSize: req.FileSize, + FileMd5: req.FileMD5, + MimeType: req.MimeType, + ChunkSize: req.ChunkSize, + TotalChunk: int32(req.TotalChunk), + Status: StatusUploading, + Operator: operator, + TenantId: tenantID, + } + if err := s.repo.CreateTask(&task); err != nil { + return nil, err + } + return map[string]any{ + "uploadId": task.UploadId, + "skipUpload": false, + "uploadedChunks": []int32{}, + }, nil +} + +// ── chunk ─────────────────────────────────────────────────────────── + +func (s *service) chunk(uploadID string, chunkIndex int, fh *multipart.FileHeader) (any, error) { + if uploadID == "" { + return nil, errors.New("缺少 uploadId") + } + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("上传任务不存在") + } + if task.Status != StatusUploading { + return nil, fmt.Errorf("任务状态异常: %s", task.Status) + } + if chunkIndex < 0 || int32(chunkIndex) >= task.TotalChunk { + return nil, fmt.Errorf("chunkIndex %d 超出范围 [0, %d)", chunkIndex, task.TotalChunk) + } + if fh.Size > s.opts.maxChunkBytes() { + return nil, fmt.Errorf("分片超过单片上限 %d MB", s.opts.maxChunkBytes()/1024/1024) + } + + // 幂等：已确认的分片直接成功 + exists, err := s.repo.ChunkExists(uploadID, int32(chunkIndex)) + if err != nil { + return nil, err + } + if exists { + return map[string]any{"chunkIndex": chunkIndex}, nil + } + + dir := s.chunkDir(uploadID) + if err := os.MkdirAll(dir, 0o766); err != nil { + return nil, err + } + chunkPath := filepath.Join(dir, strconv.Itoa(chunkIndex)) + dst, err := os.Create(chunkPath) + if err != nil { + return nil, err + } + src, err := fh.Open() + if err != nil { + _ = dst.Close() + return nil, err + } + hash := md5.New() + written, copyErr := io.Copy(io.MultiWriter(dst, hash), src) + closeErr := dst.Close() + _ = src.Close() + if copyErr != nil { + return nil, copyErr + } + if closeErr != nil { + return nil, closeErr + } + + if err := s.repo.CreateChunk(&ChunkInfo{ + UploadId: uploadID, + ChunkIndex: int32(chunkIndex), + ChunkSize: written, + ChunkMd5: fmt.Sprintf("%x", hash.Sum(nil)), + }); err != nil { + // 并发下另一请求刚写入同分片：唯一冲突视为幂等成功 + again, e2 := s.repo.ChunkExists(uploadID, int32(chunkIndex)) + if e2 == nil && again { + return map[string]any{"chunkIndex": chunkIndex}, nil + } + return nil, err + } + return map[string]any{"chunkIndex": chunkIndex}, nil +} + +// ── merge ─────────────────────────────────────────────────────────── + +type MergeReq struct { + UploadId string `json:"uploadId" binding:"required"` +} + +func (s *service) merge(uploadID string) (any, error) { + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("上传任务不存在") + } + + // 幂等：已完成直接返回原地址；合并中拒绝重复触发 + if task.Status == StatusDone && task.StoragePath != "" { + return map[string]any{"fileUrl": s.fileURL(task.StoragePath)}, nil + } + if task.Status == StatusMerging { + return nil, errors.New("任务正在合并中，请勿重复提交") + } + + count, err := s.repo.CountChunks(uploadID) + if err != nil { + return nil, err + } + if count < int(task.TotalChunk) { + return nil, fmt.Errorf("分片不完整，已上传 %d/%d", count, task.TotalChunk) + } + + _ = s.repo.SetTaskStatus(task.Id, StatusMerging) + + // 先写 .part，全部成功后改名，避免半成品文件残留 + partPath := filepath.Join(s.chunkDir(uploadID), "merged.part") + part, err := os.Create(partPath) + if err != nil { + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + hash := md5.New() + for i := int32(0); i < task.TotalChunk; i++ { + src, err := os.Open(filepath.Join(s.chunkDir(uploadID), strconv.Itoa(int(i)))) + if err != nil { + _ = part.Close() + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, fmt.Errorf("读取分片 %d 失败: %w", i, err) + } + _, copyErr := io.Copy(part, io.TeeReader(src, hash)) + _ = src.Close() + if copyErr != nil { + _ = part.Close() + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, fmt.Errorf("合并分片 %d 失败: %w", i, copyErr) + } + } + if err := part.Close(); err != nil { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + + // 整文件 MD5 校验：保证秒传可信 + if actual := fmt.Sprintf("%x", hash.Sum(nil)); actual != task.FileMd5 { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, fmt.Errorf("合并结果校验失败：实际 MD5 %s 与声称的 %s 不一致", actual, task.FileMd5) + } + + // 改名进宿主产物目录 + if err := os.MkdirAll(s.opts.CloudDir, 0o766); err != nil { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + storageName := strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + task.FileName + // storagePath 是 Web 相对路径（含 WebPathPrefix 前缀，如 cloud/xxx）， + // 物理落盘固定在 CloudDir 根下（前缀只是 URL/拼接约定，不是子目录） + finalPath := filepath.Join(s.opts.CloudDir, storageName) + storagePath := storageName + if s.opts.WebPathPrefix != "" { + storagePath = s.opts.WebPathPrefix + "/" + storageName + } + if err := os.Rename(partPath, finalPath); err != nil { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + + task.Status = StatusDone + task.StoragePath = storagePath + if err := s.repo.UpdateTaskMerged(task.Id, storagePath); err != nil { + return nil, err + } + + // 宿主业务衔接（如写云盘记录行）；失败则 merge 整体失败 + if s.opts.OnMerged != nil { + if err := s.opts.OnMerged(*task); err != nil { + return nil, err + } + } + + // 分片目录异步清理 + go func(dir string) { + _ = os.RemoveAll(dir) + }(s.chunkDir(uploadID)) + + return map[string]any{"fileUrl": s.fileURL(storagePath)}, nil +} + +// ── check / task / abort ──────────────────────────────────────────── + +func (s *service) check(fileMD5 string) (any, error) { + if len(fileMD5) != 32 { + return map[string]any{"status": "none"}, nil + } + done, err := s.repo.FindDoneByMD5(fileMD5) + if err != nil { + return nil, err + } + if done != nil && done.StoragePath != "" { + return map[string]any{ + "status": StatusDone, + "fileUrl": s.fileURL(done.StoragePath), + }, nil + } + resuming, err := s.repo.FindUploadingByMD5(fileMD5) + if err != nil { + return nil, err + } + if resuming != nil { + indexes, err := s.repo.ListChunkIndexes(resuming.UploadId) + if err != nil { + return nil, err + } + return map[string]any{ + "status": StatusUploading, + "uploadId": resuming.UploadId, + "uploadedChunks": indexes, + }, nil + } + return map[string]any{"status": "none"}, nil +} + +func (s *service) taskInfo(uploadID string) (any, error) { + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("任务不存在") + } + indexes, err := s.repo.ListChunkIndexes(uploadID) + if err != nil { + return nil, err + } + return map[string]any{"task": task, "uploadedChunks": indexes}, nil +} + +func (s *service) abort(uploadID string) (any, error) { + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("上传任务不存在") + } + if task.Status == StatusDone { + return nil, errors.New("任务已完成，无需取消") + } + if task.Status == StatusMerging { + return nil, errors.New("任务正在合并中，无法取消") + } + if err := s.repo.DeleteChunks(uploadID); err != nil { + return nil, err + } + if err := s.repo.DeleteTaskByUploadID(uploadID); err != nil { + return nil, err + } + go func(dir string) { + _ = os.RemoveAll(dir) + }(s.chunkDir(uploadID)) + return nil, nil +}