[feature/backend] overall enhancement of image uploading
All checks were successful
Build Backend / Build Docker Image (push) Successful in 5m3s

This commit is contained in:
CDN 2025-02-23 04:42:48 +08:00
parent 6e1be3d513
commit 3e6181e578
Signed by: CDN
GPG key ID: 0C656827F9F80080
13 changed files with 740 additions and 314 deletions

View file

@ -21,10 +21,50 @@ auth:
message: "Registration is currently disabled. Please contact administrator." # 禁用时的提示信息
storage:
driver: local
type: local # local or s3
local:
root: storage
base_url: http://localhost:8080/storage
root_dir: "./storage/media"
s3:
region: "us-east-1"
bucket: "your-bucket-name"
access_key_id: "your-access-key-id"
secret_access_key: "your-secret-access-key"
endpoint: "" # Optional, for MinIO or other S3-compatible services
custom_url: "" # Optional, for CDN or custom domain (e.g., https://cdn.example.com/media)
proxy_s3: false # If true, backend will proxy S3 requests instead of redirecting
upload:
limits:
image:
max_size: 10 # MB
allowed_types:
- image/jpeg
- image/png
- image/gif
- image/webp
- image/svg+xml
video:
max_size: 500 # MB
allowed_types:
- video/mp4
- video/webm
audio:
max_size: 50 # MB
allowed_types:
- audio/mpeg
- audio/ogg
- audio/wav
document:
max_size: 20 # MB
allowed_types:
- application/pdf
- application/msword
- application/vnd.openxmlformats-officedocument.wordprocessingml.document
- application/vnd.ms-excel
- application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- application/zip
- application/x-rar-compressed
- text/plain
- text/csv
logging:
level: debug

View file

@ -16,11 +16,14 @@ type Media struct {
func (Media) Fields() []ent.Field {
return []ent.Field{
field.String("storage_id").
StorageKey("storage_id").
NotEmpty().
Unique(),
field.String("original_name").
StorageKey("original_name").
NotEmpty(),
field.String("mime_type").
StorageKey("mime_type").
NotEmpty(),
field.Int64("size").
Positive(),

View file

@ -9,6 +9,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.17.59
github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1
github.com/chai2010/webp v1.1.1
github.com/disintegration/imaging v1.6.2
github.com/gin-gonic/gin v1.10.0
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/uuid v1.6.0
@ -16,7 +17,6 @@ require (
github.com/rs/zerolog v1.33.0
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.10.0
go.uber.org/mock v0.5.0
golang.org/x/crypto v0.33.0
golang.org/x/time v0.10.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
@ -72,6 +72,7 @@ require (
github.com/zclconf/go-cty v1.16.2 // indirect
github.com/zclconf/go-cty-yaml v1.1.0 // indirect
golang.org/x/arch v0.14.0 // indirect
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
golang.org/x/mod v0.23.0 // indirect
golang.org/x/net v0.35.0 // indirect
golang.org/x/sync v0.11.0 // indirect

View file

@ -61,6 +61,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E=
@ -108,8 +110,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
@ -119,8 +119,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -156,12 +154,13 @@ github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
github.com/zclconf/go-cty-yaml v1.1.0 h1:nP+jp0qPHv2IhUVqmQSzjvqAWcObN0KBkUl2rWBdig0=
github.com/zclconf/go-cty-yaml v1.1.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs=
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4=
golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
@ -173,10 +172,13 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=

View file

@ -49,7 +49,7 @@ type StorageConfig struct {
Type string `yaml:"type"`
Local LocalStorage `yaml:"local"`
S3 S3Storage `yaml:"s3"`
Upload types.UploadConfig `yaml:"upload"`
Upload UploadConfig `yaml:"upload"`
}
type LocalStorage struct {
@ -66,6 +66,27 @@ type S3Storage struct {
ProxyS3 bool `yaml:"proxy_s3"`
}
type UploadConfig struct {
Limits struct {
Image struct {
MaxSize int `yaml:"max_size"`
AllowedTypes []string `yaml:"allowed_types"`
} `yaml:"image"`
Video struct {
MaxSize int `yaml:"max_size"`
AllowedTypes []string `yaml:"allowed_types"`
} `yaml:"video"`
Audio struct {
MaxSize int `yaml:"max_size"`
AllowedTypes []string `yaml:"allowed_types"`
} `yaml:"audio"`
Document struct {
MaxSize int `yaml:"max_size"`
AllowedTypes []string `yaml:"allowed_types"`
} `yaml:"document"`
} `yaml:"limits"`
}
// Load loads configuration from a YAML file
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)

View file

@ -25,8 +25,9 @@ func NewHandler(cfg *config.Config, service service.Service) *Handler {
}
// RegisterRoutes registers all the routes
func (h *Handler) RegisterRoutes(r *gin.Engine) {
api := r.Group("/api/v1")
func (h *Handler) RegisterRoutes(router *gin.Engine) {
// API routes
api := router.Group("/api/v1")
{
// Auth routes
auth := api.Group("/auth")
@ -93,6 +94,9 @@ func (h *Handler) RegisterRoutes(r *gin.Engine) {
media.DELETE("/:id", h.DeleteMedia)
}
}
// Public media files
router.GET("/media/:year/:month/:filename", h.GetMediaFile)
}
// Category handlers
@ -246,10 +250,10 @@ func (h *Handler) GetPost(c *gin.Context) {
contents := make([]gin.H, 0, len(post.Edges.Contents))
for _, content := range post.Edges.Contents {
contents = append(contents, gin.H{
"language_code": content.LanguageCode,
"title": content.Title,
"language_code": content.LanguageCode,
"title": content.Title,
"content_markdown": content.ContentMarkdown,
"summary": content.Summary,
"summary": content.Summary,
})
}
response["edges"].(gin.H)["contents"] = contents
@ -294,7 +298,7 @@ func (h *Handler) CreatePost(c *gin.Context) {
}
type AddPostContentRequest struct {
LanguageCode string `json:"language_code" binding:"required"`
LanguageCode string `json:"language_code" binding:"required"`
Title string `json:"title" binding:"required"`
ContentMarkdown string `json:"content_markdown" binding:"required"`
Summary string `json:"summary" binding:"required"`
@ -326,10 +330,10 @@ func (h *Handler) AddPostContent(c *gin.Context) {
"title": content.Title,
"content_markdown": content.ContentMarkdown,
"language_code": content.LanguageCode,
"summary": content.Summary,
"summary": content.Summary,
"meta_keywords": content.MetaKeywords,
"meta_description": content.MetaDescription,
"edges": gin.H{},
"edges": gin.H{},
})
}
@ -535,11 +539,3 @@ func (h *Handler) AddDailyContent(c *gin.Context) {
c.JSON(http.StatusCreated, content)
}
// Helper functions
func stringPtr(s *string) string {
if s == nil {
return ""
}
return *s
}

View file

@ -5,9 +5,11 @@ import (
"io"
"net/http"
"strconv"
"strings"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
"path/filepath"
)
// Media handlers
@ -33,17 +35,29 @@ func (h *Handler) ListMedia(c *gin.Context) {
return
}
c.JSON(http.StatusOK, media)
c.JSON(http.StatusOK, gin.H{
"data": media,
})
}
func (h *Handler) UploadMedia(c *gin.Context) {
// Get user ID from context (set by auth middleware)
userID, exists := c.Get("user_id")
userIDStr, exists := c.Get("user_id")
if !exists {
c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
return
}
// Convert user ID to int
userID, err := strconv.Atoi(userIDStr.(string))
if err != nil {
log.Error().Err(err).
Str("user_id", fmt.Sprintf("%v", userIDStr)).
Msg("Failed to convert user ID to int")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal server error"})
return
}
// Get file from form
file, err := c.FormFile("file")
if err != nil {
@ -51,38 +65,107 @@ func (h *Handler) UploadMedia(c *gin.Context) {
return
}
// 文件大小限制
if file.Size > 10*1024*1024 { // 10MB
c.JSON(http.StatusBadRequest, gin.H{"error": "File size exceeds the limit (10MB)"})
// 获取文件类型和扩展名
contentType := file.Header.Get("Content-Type")
ext := strings.ToLower(filepath.Ext(file.Filename))
if contentType == "" {
// 如果 Content-Type 为空,尝试从文件扩展名判断
switch ext {
case ".jpg", ".jpeg":
contentType = "image/jpeg"
case ".png":
contentType = "image/png"
case ".gif":
contentType = "image/gif"
case ".webp":
contentType = "image/webp"
case ".mp4":
contentType = "video/mp4"
case ".webm":
contentType = "video/webm"
case ".mp3":
contentType = "audio/mpeg"
case ".ogg":
contentType = "audio/ogg"
case ".wav":
contentType = "audio/wav"
case ".pdf":
contentType = "application/pdf"
case ".doc":
contentType = "application/msword"
case ".docx":
contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
}
}
// 根据 Content-Type 确定文件类型和限制
var maxSize int64
var allowedTypes []string
var fileType string
limits := h.cfg.Storage.Upload.Limits
switch {
case strings.HasPrefix(contentType, "image/"):
maxSize = int64(limits.Image.MaxSize) * 1024 * 1024
allowedTypes = limits.Image.AllowedTypes
fileType = "image"
case strings.HasPrefix(contentType, "video/"):
maxSize = int64(limits.Video.MaxSize) * 1024 * 1024
allowedTypes = limits.Video.AllowedTypes
fileType = "video"
case strings.HasPrefix(contentType, "audio/"):
maxSize = int64(limits.Audio.MaxSize) * 1024 * 1024
allowedTypes = limits.Audio.AllowedTypes
fileType = "audio"
case strings.HasPrefix(contentType, "application/"):
maxSize = int64(limits.Document.MaxSize) * 1024 * 1024
allowedTypes = limits.Document.AllowedTypes
fileType = "document"
default:
c.JSON(http.StatusBadRequest, gin.H{
"error": "Unsupported file type",
})
return
}
// 文件类型限制
allowedTypes := map[string]bool{
"image/jpeg": true,
"image/png": true,
"image/gif": true,
"video/mp4": true,
"video/webm": true,
"audio/mpeg": true,
"audio/ogg": true,
"application/pdf": true,
// 检查文件类型是否允许
typeAllowed := false
for _, allowed := range allowedTypes {
if contentType == allowed {
typeAllowed = true
break
}
}
contentType := file.Header.Get("Content-Type")
if _, ok := allowedTypes[contentType]; !ok {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file type"})
if !typeAllowed {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("Unsupported %s type: %s", fileType, contentType),
})
return
}
// 检查文件大小
if file.Size > maxSize {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("File size exceeds the limit (%d MB) for %s files", limits.Image.MaxSize, fileType),
})
return
}
// Upload file
media, err := h.service.Upload(c.Request.Context(), file, userID.(int))
media, err := h.service.Upload(c.Request.Context(), file, userID)
if err != nil {
log.Error().Err(err).Msg("Failed to upload media")
log.Error().Err(err).
Str("filename", file.Filename).
Str("content_type", contentType).
Int("user_id", userID).
Msg("Failed to upload media")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to upload media"})
return
}
c.JSON(http.StatusCreated, media)
c.JSON(http.StatusCreated, gin.H{
"data": media,
})
}
func (h *Handler) GetMedia(c *gin.Context) {
@ -101,7 +184,7 @@ func (h *Handler) GetMedia(c *gin.Context) {
}
// Get file content
reader, info, err := h.service.GetFile(c.Request.Context(), id)
reader, info, err := h.service.GetFile(c.Request.Context(), media.StorageID)
if err != nil {
log.Error().Err(err).Msg("Failed to get media file")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get media file"})
@ -122,16 +205,18 @@ func (h *Handler) GetMedia(c *gin.Context) {
}
func (h *Handler) GetMediaFile(c *gin.Context) {
id, err := strconv.Atoi(c.Param("id"))
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid media ID"})
return
}
year := c.Param("year")
month := c.Param("month")
filename := c.Param("filename")
// Get file content
reader, info, err := h.service.GetFile(c.Request.Context(), id)
reader, info, err := h.service.GetFile(c.Request.Context(), filename) // 直接使用完整的文件名
if err != nil {
log.Error().Err(err).Msg("Failed to get media file")
log.Error().Err(err).
Str("year", year).
Str("month", month).
Str("filename", filename).
Msg("Failed to get media file")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get media file"})
return
}
@ -151,20 +236,33 @@ func (h *Handler) GetMediaFile(c *gin.Context) {
func (h *Handler) DeleteMedia(c *gin.Context) {
// Get user ID from context (set by auth middleware)
userID, exists := c.Get("user_id")
userIDStr, exists := c.Get("user_id")
if !exists {
c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
return
}
// Convert user ID to int
userID, err := strconv.Atoi(userIDStr.(string))
if err != nil {
log.Error().Err(err).
Str("user_id", fmt.Sprintf("%v", userIDStr)).
Msg("Failed to convert user ID to int")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal server error"})
return
}
id, err := strconv.Atoi(c.Param("id"))
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid media ID"})
return
}
if err := h.service.DeleteMedia(c.Request.Context(), id, userID.(int)); err != nil {
log.Error().Err(err).Msg("Failed to delete media")
if err := h.service.DeleteMedia(c.Request.Context(), id, userID); err != nil {
log.Error().Err(err).
Int("media_id", id).
Int("user_id", userID).
Msg("Failed to delete media")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete media"})
return
}

View file

@ -3,146 +3,120 @@ package middleware
import (
"bytes"
"fmt"
"io"
"net/http"
"path/filepath"
"strings"
"tss-rocks-be/internal/config"
"github.com/gin-gonic/gin"
"tss-rocks-be/internal/types"
)
const (
defaultMaxMemory = 32 << 20 // 32 MB
maxHeaderBytes = 512 // 用于MIME类型检测的最大字节数
)
// ValidateUpload 创建文件上传验证中间件
func ValidateUpload(cfg *types.UploadConfig) gin.HandlerFunc {
// ValidateUpload 验证上传的文件
func ValidateUpload(cfg *config.UploadConfig) gin.HandlerFunc {
return func(c *gin.Context) {
// 检查是否是multipart/form-data请求
if !strings.HasPrefix(c.GetHeader("Content-Type"), "multipart/form-data") {
c.JSON(http.StatusBadRequest, gin.H{
"error": "Content-Type must be multipart/form-data",
})
// Get file from form
file, err := c.FormFile("file")
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "No file uploaded"})
c.Abort()
return
}
// 解析multipart表单
if err := c.Request.ParseMultipartForm(defaultMaxMemory); err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("Failed to parse form: %v", err),
})
c.Abort()
return
}
// 获取文件类型和扩展名
contentType := file.Header.Get("Content-Type")
ext := strings.ToLower(filepath.Ext(file.Filename))
form := c.Request.MultipartForm
if form == nil || form.File == nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": "No file uploaded",
})
c.Abort()
return
}
// 遍历所有上传的文件
for _, files := range form.File {
for _, file := range files {
// 检查文件大小
if file.Size > int64(cfg.MaxSize)<<20 { // 转换为字节
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("File %s exceeds maximum size of %d MB", file.Filename, cfg.MaxSize),
})
c.Abort()
return
}
// 检查文件扩展名
ext := strings.ToLower(filepath.Ext(file.Filename))
if !contains(cfg.AllowedExtensions, ext) {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("File extension %s is not allowed", ext),
})
c.Abort()
return
}
// 打开文件
src, err := file.Open()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": fmt.Sprintf("Failed to open file: %v", err),
})
c.Abort()
return
}
defer src.Close()
// 读取文件头部用于MIME类型检测
header := make([]byte, maxHeaderBytes)
n, err := src.Read(header)
if err != nil && err != io.EOF {
c.JSON(http.StatusInternalServerError, gin.H{
"error": fmt.Sprintf("Failed to read file: %v", err),
})
c.Abort()
return
}
header = header[:n]
// 检测MIME类型
contentType := http.DetectContentType(header)
if !contains(cfg.AllowedTypes, contentType) {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("File type %s is not allowed", contentType),
})
c.Abort()
return
}
// 将文件指针重置到开始位置
_, err = src.Seek(0, 0)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": fmt.Sprintf("Failed to read file: %v", err),
})
c.Abort()
return
}
// 将文件内容读入缓冲区
buf := &bytes.Buffer{}
_, err = io.Copy(buf, src)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": fmt.Sprintf("Failed to read file: %v", err),
})
c.Abort()
return
}
// 将验证过的文件内容和类型保存到上下文中
c.Set("validated_file_"+file.Filename, buf)
c.Set("validated_content_type_"+file.Filename, contentType)
// 如果 Content-Type 为空,尝试从文件扩展名判断
if contentType == "" {
switch ext {
case ".jpg", ".jpeg":
contentType = "image/jpeg"
case ".png":
contentType = "image/png"
case ".gif":
contentType = "image/gif"
case ".webp":
contentType = "image/webp"
case ".mp4":
contentType = "video/mp4"
case ".webm":
contentType = "video/webm"
case ".mp3":
contentType = "audio/mpeg"
case ".ogg":
contentType = "audio/ogg"
case ".wav":
contentType = "audio/wav"
case ".pdf":
contentType = "application/pdf"
case ".doc":
contentType = "application/msword"
case ".docx":
contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
}
}
// 根据 Content-Type 确定文件类型和限制
var maxSize int64
var allowedTypes []string
var fileType string
limits := cfg.Limits
switch {
case strings.HasPrefix(contentType, "image/"):
maxSize = int64(limits.Image.MaxSize) * 1024 * 1024
allowedTypes = limits.Image.AllowedTypes
fileType = "image"
case strings.HasPrefix(contentType, "video/"):
maxSize = int64(limits.Video.MaxSize) * 1024 * 1024
allowedTypes = limits.Video.AllowedTypes
fileType = "video"
case strings.HasPrefix(contentType, "audio/"):
maxSize = int64(limits.Audio.MaxSize) * 1024 * 1024
allowedTypes = limits.Audio.AllowedTypes
fileType = "audio"
case strings.HasPrefix(contentType, "application/"):
maxSize = int64(limits.Document.MaxSize) * 1024 * 1024
allowedTypes = limits.Document.AllowedTypes
fileType = "document"
default:
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("Unsupported file type: %s", contentType),
})
c.Abort()
return
}
// 检查文件类型是否允许
typeAllowed := false
for _, allowed := range allowedTypes {
if contentType == allowed {
typeAllowed = true
break
}
}
if !typeAllowed {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("Unsupported %s type: %s", fileType, contentType),
})
c.Abort()
return
}
// 检查文件大小
if file.Size > maxSize {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("File size exceeds the limit (%d MB) for %s files", limits.Image.MaxSize, fileType),
})
c.Abort()
return
}
c.Next()
}
}
// contains 检查切片中是否包含指定的字符串
func contains(slice []string, str string) bool {
for _, s := range slice {
if s == str {
return true
}
}
return false
}
// GetValidatedFile 从上下文中获取验证过的文件内容
func GetValidatedFile(c *gin.Context, filename string) (*bytes.Buffer, string, bool) {
file, exists := c.Get("validated_file_" + filename)

View file

@ -1,11 +1,14 @@
package service
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
@ -26,6 +29,8 @@ import (
"tss-rocks-be/ent/user"
"tss-rocks-be/internal/storage"
"github.com/chai2010/webp"
"github.com/disintegration/imaging"
"github.com/google/uuid"
"golang.org/x/crypto/bcrypt"
)
@ -419,21 +424,71 @@ func (s *serviceImpl) Upload(ctx context.Context, file *multipart.FileHeader, us
// Open the uploaded file
src, err := openFile(file)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to open file: %v", err)
}
defer src.Close()
// 获取文件类型和扩展名
contentType := file.Header.Get("Content-Type")
ext := strings.ToLower(filepath.Ext(file.Filename))
if contentType == "" {
// 如果 Content-Type 为空,尝试从文件扩展名判断
switch ext {
case ".jpg", ".jpeg":
contentType = "image/jpeg"
case ".png":
contentType = "image/png"
case ".gif":
contentType = "image/gif"
case ".webp":
contentType = "image/webp"
case ".mp4":
contentType = "video/mp4"
case ".webm":
contentType = "video/webm"
case ".mp3":
contentType = "audio/mpeg"
case ".ogg":
contentType = "audio/ogg"
case ".wav":
contentType = "audio/wav"
case ".pdf":
contentType = "application/pdf"
case ".doc":
contentType = "application/msword"
case ".docx":
contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
}
}
// 如果是图片,检查是否需要转换为 WebP
var fileToSave multipart.File = src
var finalContentType = contentType
if strings.HasPrefix(contentType, "image/") && contentType != "image/webp" {
// 转换为 WebP
webpFile, err := convertToWebP(src)
if err != nil {
return nil, fmt.Errorf("failed to convert image to WebP: %v", err)
}
fileToSave = webpFile
finalContentType = "image/webp"
ext = ".webp"
}
// 生成带扩展名的存储文件名
storageFilename := uuid.New().String() + ext
// Save the file to storage
fileInfo, err := s.storage.Save(ctx, file.Filename, file.Header.Get("Content-Type"), src)
fileInfo, err := s.storage.Save(ctx, storageFilename, finalContentType, fileToSave)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to save file: %v", err)
}
// Create media record
return s.client.Media.Create().
SetStorageID(fileInfo.ID).
SetOriginalName(file.Filename).
SetMimeType(fileInfo.ContentType).
SetMimeType(finalContentType).
SetSize(fileInfo.Size).
SetURL(fileInfo.URL).
SetCreatedBy(strconv.Itoa(userID)).
@ -444,13 +499,8 @@ func (s *serviceImpl) GetMedia(ctx context.Context, id int) (*ent.Media, error)
return s.client.Media.Get(ctx, id)
}
func (s *serviceImpl) GetFile(ctx context.Context, id int) (io.ReadCloser, *storage.FileInfo, error) {
media, err := s.GetMedia(ctx, id)
if err != nil {
return nil, nil, err
}
return s.storage.Get(ctx, media.StorageID)
func (s *serviceImpl) GetFile(ctx context.Context, storageID string) (io.ReadCloser, *storage.FileInfo, error) {
return s.storage.Get(ctx, storageID)
}
func (s *serviceImpl) DeleteMedia(ctx context.Context, id int, userID int) error {
@ -1065,3 +1115,47 @@ func (s *serviceImpl) DeleteDaily(ctx context.Context, id string, currentUserID
return s.client.Daily.DeleteOneID(id).Exec(ctx)
}
// convertToWebP 将图片转换为 WebP 格式
func convertToWebP(src multipart.File) (multipart.File, error) {
// 读取原始图片
img, err := imaging.Decode(src)
if err != nil {
return nil, fmt.Errorf("failed to decode image: %v", err)
}
// 创建一个新的缓冲区来存储 WebP 图片
buf := new(bytes.Buffer)
// 将图片编码为 WebP 格式
// 设置较高的质量以保持图片质量
err = webp.Encode(buf, img, &webp.Options{
Lossless: false,
Quality: 90,
})
if err != nil {
return nil, fmt.Errorf("failed to encode image to WebP: %v", err)
}
// 创建一个新的临时文件来存储转换后的图片
tmpFile, err := os.CreateTemp("", "webp-*.webp")
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %v", err)
}
// 写入转换后的数据
if _, err := io.Copy(tmpFile, buf); err != nil {
tmpFile.Close()
os.Remove(tmpFile.Name())
return nil, fmt.Errorf("failed to write WebP data: %v", err)
}
// 将文件指针移回开始位置
if _, err := tmpFile.Seek(0, 0); err != nil {
tmpFile.Close()
os.Remove(tmpFile.Name())
return nil, fmt.Errorf("failed to seek file: %v", err)
}
return tmpFile, nil
}

View file

@ -41,7 +41,7 @@ type Service interface {
ListMedia(ctx context.Context, limit, offset int) ([]*ent.Media, error)
Upload(ctx context.Context, file *multipart.FileHeader, userID int) (*ent.Media, error)
GetMedia(ctx context.Context, id int) (*ent.Media, error)
GetFile(ctx context.Context, id int) (io.ReadCloser, *storage.FileInfo, error)
GetFile(ctx context.Context, storageID string) (io.ReadCloser, *storage.FileInfo, error)
DeleteMedia(ctx context.Context, id int, userID int) error
// Contributor operations

View file

@ -11,6 +11,8 @@ import (
"path/filepath"
"strings"
"time"
"github.com/rs/zerolog/log"
)
type LocalStorage struct {
@ -44,6 +46,24 @@ func (s *LocalStorage) generateID() (string, error) {
return hex.EncodeToString(bytes), nil
}
func (s *LocalStorage) generateFilePath(id string, ext string, createTime time.Time) string {
// Create year/month directory structure
year := createTime.Format("2006")
month := createTime.Format("01")
// If id already has an extension, don't add ext
if filepath.Ext(id) != "" {
return filepath.Join(year, month, id)
}
// Otherwise, add the extension if provided
filename := id
if ext != "" {
filename = id + ext
}
return filepath.Join(year, month, filename)
}
func (s *LocalStorage) saveMetadata(id string, info *FileInfo) error {
metaPath := filepath.Join(s.metaDir, id+".meta")
file, err := os.Create(metaPath)
@ -89,11 +109,64 @@ func (s *LocalStorage) Save(ctx context.Context, name string, contentType string
return nil, fmt.Errorf("failed to generate file ID: %w", err)
}
// Create the file path
filePath := filepath.Join(s.rootDir, id)
// Get file extension from original name or content type
ext := filepath.Ext(name)
if ext == "" {
// If no extension in name, try to get it from content type
switch contentType {
case "image/jpeg":
ext = ".jpg"
case "image/png":
ext = ".png"
case "image/gif":
ext = ".gif"
case "image/webp":
ext = ".webp"
case "image/svg+xml":
ext = ".svg"
case "video/mp4":
ext = ".mp4"
case "video/webm":
ext = ".webm"
case "audio/mpeg":
ext = ".mp3"
case "audio/ogg":
ext = ".ogg"
case "audio/wav":
ext = ".wav"
case "application/pdf":
ext = ".pdf"
case "application/msword":
ext = ".doc"
case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
ext = ".docx"
case "application/vnd.ms-excel":
ext = ".xls"
case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
ext = ".xlsx"
case "application/zip":
ext = ".zip"
case "application/x-rar-compressed":
ext = ".rar"
case "text/plain":
ext = ".txt"
case "text/csv":
ext = ".csv"
}
}
// Create the file path with year/month structure
now := time.Now()
relPath := s.generateFilePath(id, ext, now)
fullPath := filepath.Join(s.rootDir, relPath)
// Create directory if it doesn't exist
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
return nil, fmt.Errorf("failed to create directory: %w", err)
}
// Create the file
file, err := os.Create(filePath)
file, err := os.Create(fullPath)
if err != nil {
return nil, fmt.Errorf("failed to create file: %w", err)
}
@ -102,52 +175,73 @@ func (s *LocalStorage) Save(ctx context.Context, name string, contentType string
// Copy the content
size, err := io.Copy(file, reader)
if err != nil {
// Clean up the file if there's an error
os.Remove(filePath)
os.Remove(fullPath) // Clean up on error
return nil, fmt.Errorf("failed to write file content: %w", err)
}
now := time.Now()
// Save metadata
info := &FileInfo{
ID: id,
Name: name,
Size: size,
ContentType: contentType,
Size: size,
CreatedAt: now,
UpdatedAt: now,
URL: fmt.Sprintf("/api/media/file/%s", id),
URL: fmt.Sprintf("/media/%s/%s/%s", now.Format("2006"), now.Format("01"), filepath.Base(relPath)),
}
// Save metadata
if err := s.saveMetadata(id, info); err != nil {
os.Remove(filePath)
return nil, err
os.Remove(fullPath) // Clean up on error
return nil, fmt.Errorf("failed to save metadata: %w", err)
}
return info, nil
}
func (s *LocalStorage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) {
filePath := filepath.Join(s.rootDir, id)
// 从 id 中提取文件扩展名和基础 ID
ext := filepath.Ext(id)
baseID := strings.TrimSuffix(id, ext)
// Open the file
// 获取文件的创建时间(从元数据或当前时间)
metaPath := filepath.Join(s.metaDir, baseID+".meta")
var createTime time.Time
if stat, err := os.Stat(metaPath); err == nil {
createTime = stat.ModTime()
} else {
createTime = time.Now() // 如果找不到元数据,使用当前时间
}
// 生成完整的文件路径
year := createTime.Format("2006")
month := createTime.Format("01")
filePath := filepath.Join(s.rootDir, year, month, id) // 直接使用完整的 id包含扩展名
// 调试日志
log.Debug().
Str("id", id).
Str("baseID", baseID).
Str("ext", ext).
Str("filePath", filePath).
Time("createTime", createTime).
Msg("Attempting to get file")
// 打开文件
file, err := os.Open(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil, fmt.Errorf("file not found: %s", id)
return nil, nil, fmt.Errorf("file not found: %s (path: %s)", id, filePath)
}
return nil, nil, fmt.Errorf("failed to open file: %w", err)
}
// Get file info
// 获取文件信息
stat, err := file.Stat()
if err != nil {
file.Close()
return nil, nil, fmt.Errorf("failed to get file info: %w", err)
}
// Load metadata
name, contentType, err := s.loadMetadata(id)
// 加载元数据
name, contentType, err := s.loadMetadata(baseID)
if err != nil {
file.Close()
return nil, nil, err
@ -158,27 +252,44 @@ func (s *LocalStorage) Get(ctx context.Context, id string) (io.ReadCloser, *File
Name: name,
Size: stat.Size(),
ContentType: contentType,
CreatedAt: stat.ModTime(),
CreatedAt: createTime,
UpdatedAt: stat.ModTime(),
URL: fmt.Sprintf("/api/media/file/%s", id),
URL: fmt.Sprintf("/media/%s/%s/%s", year, month, id),
}
return file, info, nil
}
func (s *LocalStorage) Delete(ctx context.Context, id string) error {
filePath := filepath.Join(s.rootDir, id)
if err := os.Remove(filePath); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("file not found: %s", id)
}
return fmt.Errorf("failed to delete file: %w", err)
// 从 id 中提取文件扩展名
ext := filepath.Ext(id)
baseID := strings.TrimSuffix(id, ext)
// 获取文件的创建时间(从元数据或当前时间)
metaPath := filepath.Join(s.metaDir, baseID+".meta")
var createTime time.Time
if stat, err := os.Stat(metaPath); err == nil {
createTime = stat.ModTime()
} else {
createTime = time.Now() // 如果找不到元数据,使用当前时间
}
// Remove metadata
metaPath := filepath.Join(s.metaDir, id+".meta")
if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove metadata: %w", err)
// 生成完整的文件路径
relPath := s.generateFilePath(baseID, ext, createTime)
filePath := filepath.Join(s.rootDir, relPath)
// 删除文件
if err := os.Remove(filePath); err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("failed to delete file: %w", err)
}
}
// 删除元数据
if err := os.Remove(metaPath); err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("failed to delete metadata: %w", err)
}
}
return nil

View file

@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"time"
@ -48,14 +49,25 @@ func (s *S3Storage) generateID() (string, error) {
return hex.EncodeToString(bytes), nil
}
func (s *S3Storage) getObjectURL(id string) string {
func (s *S3Storage) generateObjectKey(id string, ext string, createTime time.Time) string {
// Create year/month structure
year := createTime.Format("2006")
month := createTime.Format("01")
filename := id
if ext != "" {
filename = id + ext
}
return fmt.Sprintf("%s/%s/%s", year, month, filename)
}
func (s *S3Storage) getObjectURL(key string) string {
if s.customURL != "" {
return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), id)
return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), key)
}
if s.proxyS3 {
return fmt.Sprintf("/api/media/file/%s", id)
return fmt.Sprintf("/media/%s", key)
}
return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, id)
return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, key)
}
func (s *S3Storage) Save(ctx context.Context, name string, contentType string, reader io.Reader) (*FileInfo, error) {
@ -65,10 +77,60 @@ func (s *S3Storage) Save(ctx context.Context, name string, contentType string, r
return nil, fmt.Errorf("failed to generate file ID: %w", err)
}
// Get file extension from original name or content type
ext := filepath.Ext(name)
if ext == "" {
// If no extension in name, try to get it from content type
switch contentType {
case "image/jpeg":
ext = ".jpg"
case "image/png":
ext = ".png"
case "image/gif":
ext = ".gif"
case "image/webp":
ext = ".webp"
case "image/svg+xml":
ext = ".svg"
case "video/mp4":
ext = ".mp4"
case "video/webm":
ext = ".webm"
case "audio/mpeg":
ext = ".mp3"
case "audio/ogg":
ext = ".ogg"
case "audio/wav":
ext = ".wav"
case "application/pdf":
ext = ".pdf"
case "application/msword":
ext = ".doc"
case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
ext = ".docx"
case "application/vnd.ms-excel":
ext = ".xls"
case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
ext = ".xlsx"
case "application/zip":
ext = ".zip"
case "application/x-rar-compressed":
ext = ".rar"
case "text/plain":
ext = ".txt"
case "text/csv":
ext = ".csv"
}
}
// Create the object key with year/month structure
now := time.Now()
key := s.generateObjectKey(id, ext, now)
// Check if the file exists
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(id),
Key: aws.String(key),
})
if err == nil {
return nil, fmt.Errorf("file already exists with ID: %s", id)
@ -82,37 +144,50 @@ func (s *S3Storage) Save(ctx context.Context, name string, contentType string, r
// Upload the file
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(id),
Key: aws.String(key),
Body: reader,
ContentType: aws.String(contentType),
Metadata: map[string]string{
"x-amz-meta-original-name": name,
"x-amz-meta-created-at": now.Format(time.RFC3339),
},
})
if err != nil {
return nil, fmt.Errorf("failed to upload file: %w", err)
}
now := time.Now()
info := &FileInfo{
ID: id,
Name: name,
Size: 0, // Size is not available until after upload
ContentType: contentType,
CreatedAt: now,
UpdatedAt: now,
URL: s.getObjectURL(id),
URL: s.getObjectURL(key),
}
return info, nil
}
func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) {
// Get the object from S3
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(id),
})
// Try to find the file with different extensions
exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav",
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
var result *s3.GetObjectOutput
var err error
var key string
for _, ext := range exts {
key = s.generateObjectKey(id, ext, time.Now())
result, err = s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
if err == nil {
break
}
}
if err != nil {
return nil, nil, fmt.Errorf("failed to get file from S3: %w", err)
}
@ -122,111 +197,117 @@ func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInf
Name: result.Metadata["x-amz-meta-original-name"],
Size: aws.ToInt64(result.ContentLength),
ContentType: aws.ToString(result.ContentType),
CreatedAt: aws.ToTime(result.LastModified),
UpdatedAt: aws.ToTime(result.LastModified),
URL: s.getObjectURL(id),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
URL: s.getObjectURL(key),
}
return result.Body, info, nil
}
func (s *S3Storage) Delete(ctx context.Context, id string) error {
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(id),
})
if err != nil {
return fmt.Errorf("failed to delete file from S3: %w", err)
// Try to find and delete the file with different extensions
exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav",
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
var lastErr error
for _, ext := range exts {
key := s.generateObjectKey(id, ext, time.Now())
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
if err == nil {
return nil
}
lastErr = err
}
return nil
return fmt.Errorf("failed to delete file from S3: %w", lastErr)
}
func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) {
// Try to find the file with different extensions
exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav",
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
for _, ext := range exts {
key := s.generateObjectKey(id, ext, time.Now())
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
if err == nil {
return true, nil
}
}
return false, nil
}
func (s *S3Storage) List(ctx context.Context, prefix string, limit int, offset int) ([]*FileInfo, error) {
var files []*FileInfo
var continuationToken *string
count := 0
skip := offset
// Skip objects for offset
for i := 0; i < offset/1000; i++ {
output, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
for {
input := &s3.ListObjectsV2Input{
Bucket: aws.String(s.bucket),
Prefix: aws.String(prefix),
ContinuationToken: continuationToken,
MaxKeys: aws.Int32(1000),
})
MaxKeys: aws.Int32(100), // Fetch in batches of 100
}
result, err := s.client.ListObjectsV2(ctx, input)
if err != nil {
return nil, fmt.Errorf("failed to list files from S3: %w", err)
}
if !aws.ToBool(output.IsTruncated) {
return files, nil
}
continuationToken = output.NextContinuationToken
}
// Get the actual objects
output, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(s.bucket),
Prefix: aws.String(prefix),
ContinuationToken: continuationToken,
MaxKeys: aws.Int32(int32(limit)),
})
if err != nil {
return nil, fmt.Errorf("failed to list files from S3: %w", err)
}
for _, obj := range output.Contents {
// Get the object metadata
head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: obj.Key,
})
var contentType string
var originalName string
if err != nil {
var noSuchKey *types.NoSuchKey
if errors.As(err, &noSuchKey) {
// If the object doesn't exist (which shouldn't happen normally),
// we'll still include it in the list but with empty metadata
contentType = ""
originalName = aws.ToString(obj.Key)
} else {
// Process each object
for _, obj := range result.Contents {
if skip > 0 {
skip--
continue
}
} else {
contentType = aws.ToString(head.ContentType)
originalName = head.Metadata["x-amz-meta-original-name"]
if originalName == "" {
originalName = aws.ToString(obj.Key)
// Get object metadata
head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: obj.Key,
})
if err != nil {
continue // Skip files we can't get metadata for
}
// Extract the ID from the key (remove extension if present)
id := aws.ToString(obj.Key)
if ext := filepath.Ext(id); ext != "" {
id = id[:len(id)-len(ext)]
}
info := &FileInfo{
ID: id,
Name: head.Metadata["x-amz-meta-original-name"],
Size: aws.ToInt64(obj.Size),
ContentType: aws.ToString(head.ContentType),
CreatedAt: aws.ToTime(obj.LastModified),
UpdatedAt: aws.ToTime(obj.LastModified),
URL: s.getObjectURL(aws.ToString(obj.Key)),
}
files = append(files, info)
count++
if count >= limit {
return files, nil
}
}
files = append(files, &FileInfo{
ID: aws.ToString(obj.Key),
Name: originalName,
Size: aws.ToInt64(obj.Size),
ContentType: contentType,
CreatedAt: aws.ToTime(obj.LastModified),
UpdatedAt: aws.ToTime(obj.LastModified),
URL: s.getObjectURL(aws.ToString(obj.Key)),
})
if !aws.ToBool(result.IsTruncated) {
break
}
continuationToken = result.NextContinuationToken
}
return files, nil
}
func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) {
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(id),
})
if err != nil {
var nsk *types.NoSuchKey
if ok := errors.As(err, &nsk); ok {
return false, nil
}
return false, fmt.Errorf("failed to check file existence in S3: %w", err)
}
return true, nil
}