From 011eb4348b4c63f76b1eac5ee6bcbb976fd68eea Mon Sep 17 00:00:00 2001 From: 4566704 <4566704@qq.com> Date: Fri, 25 Sep 2026 15:21:21 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E7=8B=AC=E7=AB=8B=E9=80=9A=E7=94=A8?= =?UTF-8?q?=E5=A4=A7=E6=96=87=E4=BB=B6=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AF=E6=A0=B8=E5=BF=83=EF=BC=88=E8=87=AA?= =?UTF-8?q?=20zogo/fileupload=20v0.3.2=20=E6=8A=BD=E5=87=BA=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 六接口路由：init / chunk / merge / check / task / abort - Repository 接口注入存储，宿主实现 12 方法即可接入 - OnMerged / OnInstantHit 回调衔接宿主业务（如写云端文件表） - Envelope 注入响应封装，与宿主 httpx 解耦 - 与 zogo v0.3.2 的 fileupload 子包同源，后续在此仓库独立演进 --- .gitignore | 4 + README.md | 41 ++++++ fileupload.go | 137 ++++++++++++++++++ go.mod | 40 ++++++ go.sum | 91 ++++++++++++ mount.go | 119 ++++++++++++++++ repository.go | 32 +++++ service.go | 382 ++++++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 846 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 fileupload.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 mount.go create mode 100644 repository.go create mode 100644 service.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5f8baef --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.exe +*.test +*.out +.data/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..5cf2ef8 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# fileupload + +通用大文件分片上传 Go/Gin 服务端核心（独立库，原 zogo/fileupload 子包）。 +六接口（init/chunk/merge/check/task/abort）+ 分片幂等 + 断点续传 + `.part` 原子改名 + 合并整文件 MD5 校验 + 秒传。 +不依赖宿主的响应封装、ORM 或业务表——存储走 `Repository` 接口注入，业务衔接走 `Options` 回调。 + +## 用法 + +```go +import "git.zeroonesoft.cn/golib/fileupload" + +opts := fileupload.Options{ + ChunkRoot: "./data/upload-chunks", // 分片临时目录（建议在静态根之外） + CloudDir: "./data/uploads/cloud", // 合并产物目录 + WebPathPrefix: "cloud", // storagePath 的 Web 前缀（须与 CloudDir 相对静态根的子目录一致） + URLPrefix: "/uploads", // fileUrl = URLPrefix + "/" + storagePath + MaxTotalSizeMB: 2048, // 0=不限 + OnMerged: func(t fileupload.TaskInfo) error { + // 合并成功：宿主写自己的业务表（如云盘记录行）；失败则 merge 整体失败 + return nil + }, + OnInstantHit: func(t fileupload.TaskInfo) error { + // 秒传命中：宿主确保业务侧可见行（可空回调） + return nil + }, + Envelope: func(c *gin.Context, resp any, err error) { + // 宿主自己的响应封装，如 httpx.HandleResult(c, resp, err) + }, +} +_ = fileupload.Mount(api.Group("/upload"), opts, myRepo, + fileupload.WithIdentity(func(c *gin.Context) (string, int64) { + return c.GetString("UserName"), c.GetInt64("TenantId") + })) +``` + +`Repository`（12 个方法）由宿主按自己的表结构实现；软删/硬删自定，但删除后查询必须不可见。 +鉴权由宿主路由组的 JWT 中间件统一负责，本库不感知。 + +## 版本 + +- v0.1.0 首版：自 FileUpload 项目 restful/upload 模块抽出 diff --git a/fileupload.go b/fileupload.go new file mode 100644 index 0000000..33aef29 --- /dev/null +++ b/fileupload.go @@ -0,0 +1,137 @@ +// Package fileupload 通用大文件分片上传（Go/Gin 服务端核心）。 +// +// 职责：init/chunk/merge/check/task/abort 六接口的路由挂载与业务逻辑—— +// 分片幂等、断点续传、.part 原子改名、合并时整文件 MD5 校验、秒传。 +// 宿主差异全部通过 Options 与 Repository 注入，本包不依赖任何宿主的 +// 响应封装、ORM 生成代码或业务表： +// +// fileupload.Mount(api.Group("/upload"), opts, repo) +// +// 与业务层的衔接： +// - merge 成功 → OnMerged(TaskInfo)（宿主写自己的业务表/云盘记录）； +// 回调失败则 merge 整体失败、任务置 error +// - 秒传命中 → OnInstantHit(TaskInfo)（宿主确保业务侧可见行，可空回调） +// - 响应封装 → Options.Envelope（必填，如 httpx.HandleResult / gin.H{code:200,...}） +package fileupload + +import ( + "errors" + "math" + "path" + "strings" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +// TaskInfo 上传任务的库级视图（与宿主的表结构解耦）。 +type TaskInfo struct { + Id int64 + UploadId string + FileName string + FileSize int64 + FileMd5 string + MimeType string + ChunkSize int64 + TotalChunk int32 + Status string // uploading / merging / done / error + StoragePath string // 相对路径（如 cloud/xxx），由宿主的目录约定决定 + Operator string + TenantId int64 +} + +// ChunkInfo 分片记录的库级视图。 +type ChunkInfo struct { + UploadId string + ChunkIndex int32 + ChunkSize int64 + ChunkMd5 string +} + +// Status 常量。 +const ( + StatusUploading = "uploading" + StatusMerging = "merging" + StatusDone = "done" + StatusError = "error" +) + +// DefaultMaxChunkBytes 单分片硬上限（防滥用，与常见 multipart 内存缓冲同量级）。 +const DefaultMaxChunkBytes int64 = 64 << 20 + +// Envelope 宿主响应封装：resp 非 nil 时写成功响应，err 非 nil 时写失败响应。 +type Envelope func(c *gin.Context, resp any, err error) + +// Options 宿主注入配置。 +type Options struct { + // ChunkRoot 分片临时根目录；库在其下建 / 子目录。 + // 建议置于宿主静态根之外，避免分片被静态服务暴露。 + ChunkRoot string + // CloudDir 合并产物目录（宿主的云盘/文件目录约定）。 + CloudDir string + // WebPathPrefix storagePath 的 Web 相对前缀（如 "cloud"），空则无前缀。 + // 最终 storagePath = WebPathPrefix + "/" + 产物名（存库、回调、BS 拼接均用该值）， + // 需与 CloudDir 相对宿主静态根的子目录一致。 + WebPathPrefix string + // URLPrefix 拼接 fileUrl 的静态前缀，默认 "/uploads"； + // 最终 fileUrl = URLPrefix + "/" + storagePath。 + URLPrefix string + // MaxTotalSizeMB 文件总大小上限（MB），0=不限制。 + MaxTotalSizeMB int64 + // MaxChunkBytes 单分片字节上限，0=DefaultMaxChunkBytes(64MB)。 + MaxChunkBytes int64 + // OnMerged 合并成功回调（宿主写业务表等）。返回 err 则 merge 失败、任务置 error。 + OnMerged func(t TaskInfo) error + // OnInstantHit 秒传命中回调（宿主确保业务侧可见行）。可空。 + OnInstantHit func(t TaskInfo) error + // Envelope 响应封装（必填）。 + Envelope Envelope +} + +func (o *Options) validate() error { + if o.ChunkRoot == "" || o.CloudDir == "" { + return errors.New("fileupload: Options.ChunkRoot / CloudDir 必填") + } + if o.Envelope == nil { + return errors.New("fileupload: Options.Envelope 必填") + } + if o.MaxChunkBytes <= 0 { + o.MaxChunkBytes = DefaultMaxChunkBytes + } + if o.URLPrefix == "" { + o.URLPrefix = "/uploads" + } + return nil +} + +func (o *Options) maxChunkBytes() int64 { + if o.MaxChunkBytes > 0 { + return o.MaxChunkBytes + } + return DefaultMaxChunkBytes +} + +// expectChunks 期望分片数：ceil(fileSize/chunkSize)，空文件视为 1 个空分片。 +func expectChunks(fileSize, chunkSize int64) int { + if chunkSize <= 0 { + return 0 + } + n := int(math.Ceil(float64(fileSize) / float64(chunkSize))) + if n == 0 { + n = 1 + } + return n +} + +// sanitizeFileName 取路径基名并剥离路径穿越/非法字符。 +func sanitizeFileName(name string) string { + name = path.Base(strings.TrimSpace(name)) + name = strings.ReplaceAll(name, "\\", "") + if name == "." || name == ".." || name == "/" || name == "" { + return "" + } + return name +} + +// newUploadID 生成上传任务 ID（uuid v4，URL 安全）。 +func newUploadID() string { return uuid.NewString() } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9ebe761 --- /dev/null +++ b/go.mod @@ -0,0 +1,40 @@ +module git.zeroonesoft.cn/golib/fileupload + +go 1.25.0 + +require ( + github.com/gin-gonic/gin v1.12.0 + github.com/google/uuid v1.6.0 +) + +require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0b62b83 --- /dev/null +++ b/go.sum @@ -0,0 +1,91 @@ +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mount.go b/mount.go new file mode 100644 index 0000000..7c9c1f1 --- /dev/null +++ b/mount.go @@ -0,0 +1,119 @@ +package fileupload + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" +) + +// Mount 挂载分片上传路由到 r（宿主自行决定路由组，如 api.Group("/upload")）： +// +// POST /init 初始化（JSON：fileName/fileSize/fileMD5/mimeType/chunkSize/totalChunk） +// POST /chunk 上传分片（multipart：uploadId/chunkIndex/chunk），幂等 +// POST /merge 合并分片（JSON：uploadId），幂等 +// GET /check?fileMD5= 秒传/续传检查 +// GET /task/:uploadId 任务详情与已确认分片 +// DELETE /task/:uploadId 取消任务并清理 +// +// 身份（operator/tenant）由宿主在 handler 外层取好后经 identity 注入； +// 响应统一走 Options.Envelope。 +type Identity func(c *gin.Context) (operator string, tenantID int64) + +type mountConfig struct { + identity Identity +} + +// MountOption 挂载选项。 +type MountOption func(*mountConfig) + +// WithIdentity 注入身份提取（从 JWT 中间件写入的 context 取 UserId/UserName 等）。 +func WithIdentity(fn Identity) MountOption { + return func(m *mountConfig) { m.identity = fn } +} + +// Mount 挂载路由。r 建议为 api.Group("/upload")。 +func Mount(r *gin.RouterGroup, opts Options, repo Repository, mos ...MountOption) error { + if err := opts.validate(); err != nil { + return err + } + if repo == nil { + return errors.New("fileupload: Repository 必填") + } + mc := &mountConfig{} + for _, mo := range mos { + mo(mc) + } + svc := newService(opts, repo) + + r.POST("/init", func(c *gin.Context) { + var req InitReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"code": 1, "msg": "参数错误: " + err.Error()}) + return + } + operator, tenantID := mc.identity(c) + resp, err := svc.init(&req, operator, tenantID) + opts.Envelope(c, resp, err) + }) + + r.POST("/chunk", func(c *gin.Context) { + form, err := c.MultipartForm() + if err != nil { + opts.Envelope(c, nil, errors.New("请使用 multipart/form-data 上传")) + return + } + files := form.File["chunk"] + if len(files) != 1 { + opts.Envelope(c, nil, errors.New("缺少分片文件字段 chunk")) + return + } + chunkIndex := atoiDefault(c.PostForm("chunkIndex"), -1) + resp, err := svc.chunk(c.PostForm("uploadId"), chunkIndex, files[0]) + if err != nil { + opts.Envelope(c, nil, err) + return + } + opts.Envelope(c, resp, nil) + }) + + r.POST("/merge", func(c *gin.Context) { + var req MergeReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"code": 1, "msg": "参数错误: " + err.Error()}) + return + } + resp, err := svc.merge(req.UploadId) + opts.Envelope(c, resp, err) + }) + + r.GET("/check", func(c *gin.Context) { + resp, err := svc.check(c.Query("fileMD5")) + opts.Envelope(c, resp, err) + }) + + r.GET("/task/:uploadId", func(c *gin.Context) { + resp, err := svc.taskInfo(c.Param("uploadId")) + opts.Envelope(c, resp, err) + }) + + r.DELETE("/task/:uploadId", func(c *gin.Context) { + resp, err := svc.abort(c.Param("uploadId")) + opts.Envelope(c, resp, err) + }) + return nil +} + +func atoiDefault(s string, def int) int { + if s == "" { + return def + } + n := 0 + for _, r := range s { + if r < '0' || r > '9' { + return def + } + n = n*10 + int(r-'0') + } + return n +} diff --git a/repository.go b/repository.go new file mode 100644 index 0000000..ae23837 --- /dev/null +++ b/repository.go @@ -0,0 +1,32 @@ +package fileupload + +// Repository 存储层接口：宿主用自己的 ORM/表结构实现（本包不关心表怎么建）。 +// 语义约定： +// - Delete* 为软删或硬删由宿主决定，但之后 GetTaskByUploadID / 各 Find 必须查不到； +// - Find* 只需返回未删除的最新一条。 +type Repository interface { + // CreateTask 新建任务（Id/UploadId 已由本包赋值）。 + CreateTask(t *TaskInfo) error + // GetTaskByUploadID 按 uploadId 取未删除任务；不存在返回 (nil, nil)。 + GetTaskByUploadID(uploadID string) (*TaskInfo, error) + // FindDoneByMD5 取同 MD5 已完成任务中最新一条；无则 (nil, nil)。 + FindDoneByMD5(md5 string) (*TaskInfo, error) + // FindUploadingByMD5 取同 MD5 进行中任务一条（断点续传）；无则 (nil, nil)。 + FindUploadingByMD5(md5 string) (*TaskInfo, error) + // UpdateTaskMerged 合并成功：置 done 并记录 storagePath。 + UpdateTaskMerged(id int64, storagePath string) error + // SetTaskStatus 更新任务状态（error/merging 等流转）。 + SetTaskStatus(id int64, status string) error + // CountChunks 统计任务已确认分片数。 + CountChunks(uploadID string) (int, error) + // ChunkExists 分片是否已确认（幂等依据）。 + ChunkExists(uploadID string, index int32) (bool, error) + // CreateChunk 写入分片确认记录。 + CreateChunk(c *ChunkInfo) error + // ListChunkIndexes 已确认分片序号升序。 + ListChunkIndexes(uploadID string) ([]int32, error) + // DeleteChunks 删除任务的分片记录。 + DeleteChunks(uploadID string) error + // DeleteTaskByUploadID 删除任务。 + DeleteTaskByUploadID(uploadID string) error +} diff --git a/service.go b/service.go new file mode 100644 index 0000000..ef16207 --- /dev/null +++ b/service.go @@ -0,0 +1,382 @@ +package fileupload + +import ( + "crypto/md5" + "errors" + "fmt" + "io" + "mime/multipart" + "os" + "path/filepath" + "strconv" + "time" +) + +// service 核心业务，持 Options + Repository。 +type service struct { + opts Options + repo Repository +} + +func newService(opts Options, repo Repository) *service { + return &service{opts: opts, repo: repo} +} + +func (s *service) chunkDir(uploadID string) string { + return filepath.Join(s.opts.ChunkRoot, uploadID) +} + +func (s *service) fileURL(storagePath string) string { + return s.opts.URLPrefix + "/" + storagePath +} + +// ── init ──────────────────────────────────────────────────────────── + +type InitReq struct { + FileName string `json:"fileName" binding:"required,max=255"` + FileSize int64 `json:"fileSize" binding:"min=0"` + FileMD5 string `json:"fileMD5" binding:"required,len=32"` + MimeType string `json:"mimeType" binding:"max=128"` + ChunkSize int64 `json:"chunkSize" binding:"required,min=1"` + TotalChunk int `json:"totalChunk" binding:"required,min=1"` +} + +func (s *service) init(req *InitReq, operator string, tenantID int64) (any, error) { + if req.ChunkSize <= 0 || req.ChunkSize > s.opts.maxChunkBytes() { + return nil, fmt.Errorf("chunkSize 超出范围 (0, %d]", s.opts.maxChunkBytes()) + } + if req.FileSize < 0 { + return nil, errors.New("fileSize 非法") + } + if len(req.FileMD5) != 32 { + return nil, errors.New("fileMD5 必须为 32 位十六进制") + } + if expect := expectChunks(req.FileSize, req.ChunkSize); req.TotalChunk != expect { + return nil, fmt.Errorf("totalChunk(%d) 与 fileSize(%d)/chunkSize(%d) 不匹配，应为 %d", + req.TotalChunk, req.FileSize, req.ChunkSize, expect) + } + if s.opts.MaxTotalSizeMB > 0 && req.FileSize > s.opts.MaxTotalSizeMB*1024*1024 { + return nil, fmt.Errorf("文件超过总大小上限 %dMB", s.opts.MaxTotalSizeMB) + } + + // 1. 秒传：同 MD5 已完成任务，复用其文件 + done, err := s.repo.FindDoneByMD5(req.FileMD5) + if err != nil { + return nil, err + } + if done != nil && done.StoragePath != "" { + if s.opts.OnInstantHit != nil { + if err := s.opts.OnInstantHit(*done); err != nil { + return nil, err + } + } + return map[string]any{ + "uploadId": "", + "skipUpload": true, + "fileUrl": s.fileURL(done.StoragePath), + }, nil + } + + // 2. 断点续传：同 MD5 进行中任务 + resuming, err := s.repo.FindUploadingByMD5(req.FileMD5) + if err != nil { + return nil, err + } + if resuming != nil { + indexes, err := s.repo.ListChunkIndexes(resuming.UploadId) + if err != nil { + return nil, err + } + return map[string]any{ + "uploadId": resuming.UploadId, + "skipUpload": false, + "uploadedChunks": indexes, + }, nil + } + + // 3. 新建任务 + name := sanitizeFileName(req.FileName) + if name == "" { + return nil, errors.New("文件名非法") + } + task := TaskInfo{ + UploadId: newUploadID(), + FileName: name, + FileSize: req.FileSize, + FileMd5: req.FileMD5, + MimeType: req.MimeType, + ChunkSize: req.ChunkSize, + TotalChunk: int32(req.TotalChunk), + Status: StatusUploading, + Operator: operator, + TenantId: tenantID, + } + if err := s.repo.CreateTask(&task); err != nil { + return nil, err + } + return map[string]any{ + "uploadId": task.UploadId, + "skipUpload": false, + "uploadedChunks": []int32{}, + }, nil +} + +// ── chunk ─────────────────────────────────────────────────────────── + +func (s *service) chunk(uploadID string, chunkIndex int, fh *multipart.FileHeader) (any, error) { + if uploadID == "" { + return nil, errors.New("缺少 uploadId") + } + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("上传任务不存在") + } + if task.Status != StatusUploading { + return nil, fmt.Errorf("任务状态异常: %s", task.Status) + } + if chunkIndex < 0 || int32(chunkIndex) >= task.TotalChunk { + return nil, fmt.Errorf("chunkIndex %d 超出范围 [0, %d)", chunkIndex, task.TotalChunk) + } + if fh.Size > s.opts.maxChunkBytes() { + return nil, fmt.Errorf("分片超过单片上限 %d MB", s.opts.maxChunkBytes()/1024/1024) + } + + // 幂等：已确认的分片直接成功 + exists, err := s.repo.ChunkExists(uploadID, int32(chunkIndex)) + if err != nil { + return nil, err + } + if exists { + return map[string]any{"chunkIndex": chunkIndex}, nil + } + + dir := s.chunkDir(uploadID) + if err := os.MkdirAll(dir, 0o766); err != nil { + return nil, err + } + chunkPath := filepath.Join(dir, strconv.Itoa(chunkIndex)) + dst, err := os.Create(chunkPath) + if err != nil { + return nil, err + } + src, err := fh.Open() + if err != nil { + _ = dst.Close() + return nil, err + } + hash := md5.New() + written, copyErr := io.Copy(io.MultiWriter(dst, hash), src) + closeErr := dst.Close() + _ = src.Close() + if copyErr != nil { + return nil, copyErr + } + if closeErr != nil { + return nil, closeErr + } + + if err := s.repo.CreateChunk(&ChunkInfo{ + UploadId: uploadID, + ChunkIndex: int32(chunkIndex), + ChunkSize: written, + ChunkMd5: fmt.Sprintf("%x", hash.Sum(nil)), + }); err != nil { + // 并发下另一请求刚写入同分片：唯一冲突视为幂等成功 + again, e2 := s.repo.ChunkExists(uploadID, int32(chunkIndex)) + if e2 == nil && again { + return map[string]any{"chunkIndex": chunkIndex}, nil + } + return nil, err + } + return map[string]any{"chunkIndex": chunkIndex}, nil +} + +// ── merge ─────────────────────────────────────────────────────────── + +type MergeReq struct { + UploadId string `json:"uploadId" binding:"required"` +} + +func (s *service) merge(uploadID string) (any, error) { + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("上传任务不存在") + } + + // 幂等：已完成直接返回原地址；合并中拒绝重复触发 + if task.Status == StatusDone && task.StoragePath != "" { + return map[string]any{"fileUrl": s.fileURL(task.StoragePath)}, nil + } + if task.Status == StatusMerging { + return nil, errors.New("任务正在合并中，请勿重复提交") + } + + count, err := s.repo.CountChunks(uploadID) + if err != nil { + return nil, err + } + if count < int(task.TotalChunk) { + return nil, fmt.Errorf("分片不完整，已上传 %d/%d", count, task.TotalChunk) + } + + _ = s.repo.SetTaskStatus(task.Id, StatusMerging) + + // 先写 .part，全部成功后改名，避免半成品文件残留 + partPath := filepath.Join(s.chunkDir(uploadID), "merged.part") + part, err := os.Create(partPath) + if err != nil { + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + hash := md5.New() + for i := int32(0); i < task.TotalChunk; i++ { + src, err := os.Open(filepath.Join(s.chunkDir(uploadID), strconv.Itoa(int(i)))) + if err != nil { + _ = part.Close() + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, fmt.Errorf("读取分片 %d 失败: %w", i, err) + } + _, copyErr := io.Copy(part, io.TeeReader(src, hash)) + _ = src.Close() + if copyErr != nil { + _ = part.Close() + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, fmt.Errorf("合并分片 %d 失败: %w", i, copyErr) + } + } + if err := part.Close(); err != nil { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + + // 整文件 MD5 校验：保证秒传可信 + if actual := fmt.Sprintf("%x", hash.Sum(nil)); actual != task.FileMd5 { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, fmt.Errorf("合并结果校验失败：实际 MD5 %s 与声称的 %s 不一致", actual, task.FileMd5) + } + + // 改名进宿主产物目录 + if err := os.MkdirAll(s.opts.CloudDir, 0o766); err != nil { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + storageName := strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + task.FileName + // storagePath 是 Web 相对路径（含 WebPathPrefix 前缀，如 cloud/xxx）， + // 物理落盘固定在 CloudDir 根下（前缀只是 URL/拼接约定，不是子目录） + finalPath := filepath.Join(s.opts.CloudDir, storageName) + storagePath := storageName + if s.opts.WebPathPrefix != "" { + storagePath = s.opts.WebPathPrefix + "/" + storageName + } + if err := os.Rename(partPath, finalPath); err != nil { + _ = os.Remove(partPath) + _ = s.repo.SetTaskStatus(task.Id, StatusError) + return nil, err + } + + task.Status = StatusDone + task.StoragePath = storagePath + if err := s.repo.UpdateTaskMerged(task.Id, storagePath); err != nil { + return nil, err + } + + // 宿主业务衔接（如写云盘记录行）；失败则 merge 整体失败 + if s.opts.OnMerged != nil { + if err := s.opts.OnMerged(*task); err != nil { + return nil, err + } + } + + // 分片目录异步清理 + go func(dir string) { + _ = os.RemoveAll(dir) + }(s.chunkDir(uploadID)) + + return map[string]any{"fileUrl": s.fileURL(storagePath)}, nil +} + +// ── check / task / abort ──────────────────────────────────────────── + +func (s *service) check(fileMD5 string) (any, error) { + if len(fileMD5) != 32 { + return map[string]any{"status": "none"}, nil + } + done, err := s.repo.FindDoneByMD5(fileMD5) + if err != nil { + return nil, err + } + if done != nil && done.StoragePath != "" { + return map[string]any{ + "status": StatusDone, + "fileUrl": s.fileURL(done.StoragePath), + }, nil + } + resuming, err := s.repo.FindUploadingByMD5(fileMD5) + if err != nil { + return nil, err + } + if resuming != nil { + indexes, err := s.repo.ListChunkIndexes(resuming.UploadId) + if err != nil { + return nil, err + } + return map[string]any{ + "status": StatusUploading, + "uploadId": resuming.UploadId, + "uploadedChunks": indexes, + }, nil + } + return map[string]any{"status": "none"}, nil +} + +func (s *service) taskInfo(uploadID string) (any, error) { + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("任务不存在") + } + indexes, err := s.repo.ListChunkIndexes(uploadID) + if err != nil { + return nil, err + } + return map[string]any{"task": task, "uploadedChunks": indexes}, nil +} + +func (s *service) abort(uploadID string) (any, error) { + task, err := s.repo.GetTaskByUploadID(uploadID) + if err != nil { + return nil, err + } + if task == nil { + return nil, errors.New("上传任务不存在") + } + if task.Status == StatusDone { + return nil, errors.New("任务已完成，无需取消") + } + if task.Status == StatusMerging { + return nil, errors.New("任务正在合并中，无法取消") + } + if err := s.repo.DeleteChunks(uploadID); err != nil { + return nil, err + } + if err := s.repo.DeleteTaskByUploadID(uploadID); err != nil { + return nil, err + } + go func(dir string) { + _ = os.RemoveAll(dir) + }(s.chunkDir(uploadID)) + return nil, nil +}