From 3e6181e578399941c2d4db5a8b78e72cf29ff720 Mon Sep 17 00:00:00 2001 From: cdn0x12 Date: Sun, 23 Feb 2025 04:42:48 +0800 Subject: [PATCH] [feature/backend] overall enhancement of image uploading --- backend/config/config.yaml.example | 46 ++++- backend/ent/schema/media.go | 3 + backend/go.mod | 3 +- backend/go.sum | 14 +- backend/internal/config/config.go | 23 ++- backend/internal/handler/handler.go | 28 ++- backend/internal/handler/media.go | 162 ++++++++++++---- backend/internal/middleware/upload.go | 214 +++++++++------------ backend/internal/service/impl.go | 116 +++++++++-- backend/internal/service/service.go | 2 +- backend/internal/storage/local.go | 173 ++++++++++++++--- backend/internal/storage/s3.go | 265 +++++++++++++++++--------- frontend/vite.config.ts | 5 + 13 files changed, 740 insertions(+), 314 deletions(-) diff --git a/backend/config/config.yaml.example b/backend/config/config.yaml.example index 220ace8..894b281 100644 --- a/backend/config/config.yaml.example +++ b/backend/config/config.yaml.example @@ -21,10 +21,50 @@ auth: message: "Registration is currently disabled. Please contact administrator." # 禁用时的提示信息 storage: - driver: local + type: local # local or s3 local: - root: storage - base_url: http://localhost:8080/storage + root_dir: "./storage/media" + s3: + region: "us-east-1" + bucket: "your-bucket-name" + access_key_id: "your-access-key-id" + secret_access_key: "your-secret-access-key" + endpoint: "" # Optional, for MinIO or other S3-compatible services + custom_url: "" # Optional, for CDN or custom domain (e.g., https://cdn.example.com/media) + proxy_s3: false # If true, backend will proxy S3 requests instead of redirecting + upload: + limits: + image: + max_size: 10 # MB + allowed_types: + - image/jpeg + - image/png + - image/gif + - image/webp + - image/svg+xml + video: + max_size: 500 # MB + allowed_types: + - video/mp4 + - video/webm + audio: + max_size: 50 # MB + allowed_types: + - audio/mpeg + - audio/ogg + - audio/wav + document: + max_size: 20 # MB + allowed_types: + - application/pdf + - application/msword + - application/vnd.openxmlformats-officedocument.wordprocessingml.document + - application/vnd.ms-excel + - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + - application/zip + - application/x-rar-compressed + - text/plain + - text/csv logging: level: debug diff --git a/backend/ent/schema/media.go b/backend/ent/schema/media.go index 5c41d70..3ad19d5 100644 --- a/backend/ent/schema/media.go +++ b/backend/ent/schema/media.go @@ -16,11 +16,14 @@ type Media struct { func (Media) Fields() []ent.Field { return []ent.Field{ field.String("storage_id"). + StorageKey("storage_id"). NotEmpty(). Unique(), field.String("original_name"). + StorageKey("original_name"). NotEmpty(), field.String("mime_type"). + StorageKey("mime_type"). NotEmpty(), field.Int64("size"). Positive(), diff --git a/backend/go.mod b/backend/go.mod index d42c65c..e2141f5 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.59 github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1 github.com/chai2010/webp v1.1.1 + github.com/disintegration/imaging v1.6.2 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 @@ -16,7 +17,6 @@ require ( github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.10.0 - go.uber.org/mock v0.5.0 golang.org/x/crypto v0.33.0 golang.org/x/time v0.10.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -72,6 +72,7 @@ require ( github.com/zclconf/go-cty v1.16.2 // indirect github.com/zclconf/go-cty-yaml v1.1.0 // indirect golang.org/x/arch v0.14.0 // indirect + golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect golang.org/x/mod v0.23.0 // indirect golang.org/x/net v0.35.0 // indirect golang.org/x/sync v0.11.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 4274445..37a92d6 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -61,6 +61,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= @@ -108,8 +110,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -119,8 +119,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -156,12 +154,13 @@ github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zclconf/go-cty-yaml v1.1.0 h1:nP+jp0qPHv2IhUVqmQSzjvqAWcObN0KBkUl2rWBdig0= github.com/zclconf/go-cty-yaml v1.1.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= @@ -173,10 +172,13 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index faa6bdd..f7eedd8 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -49,7 +49,7 @@ type StorageConfig struct { Type string `yaml:"type"` Local LocalStorage `yaml:"local"` S3 S3Storage `yaml:"s3"` - Upload types.UploadConfig `yaml:"upload"` + Upload UploadConfig `yaml:"upload"` } type LocalStorage struct { @@ -66,6 +66,27 @@ type S3Storage struct { ProxyS3 bool `yaml:"proxy_s3"` } +type UploadConfig struct { + Limits struct { + Image struct { + MaxSize int `yaml:"max_size"` + AllowedTypes []string `yaml:"allowed_types"` + } `yaml:"image"` + Video struct { + MaxSize int `yaml:"max_size"` + AllowedTypes []string `yaml:"allowed_types"` + } `yaml:"video"` + Audio struct { + MaxSize int `yaml:"max_size"` + AllowedTypes []string `yaml:"allowed_types"` + } `yaml:"audio"` + Document struct { + MaxSize int `yaml:"max_size"` + AllowedTypes []string `yaml:"allowed_types"` + } `yaml:"document"` + } `yaml:"limits"` +} + // Load loads configuration from a YAML file func Load(path string) (*Config, error) { data, err := os.ReadFile(path) diff --git a/backend/internal/handler/handler.go b/backend/internal/handler/handler.go index f459f47..11c5b3e 100644 --- a/backend/internal/handler/handler.go +++ b/backend/internal/handler/handler.go @@ -25,8 +25,9 @@ func NewHandler(cfg *config.Config, service service.Service) *Handler { } // RegisterRoutes registers all the routes -func (h *Handler) RegisterRoutes(r *gin.Engine) { - api := r.Group("/api/v1") +func (h *Handler) RegisterRoutes(router *gin.Engine) { + // API routes + api := router.Group("/api/v1") { // Auth routes auth := api.Group("/auth") @@ -93,6 +94,9 @@ func (h *Handler) RegisterRoutes(r *gin.Engine) { media.DELETE("/:id", h.DeleteMedia) } } + + // Public media files + router.GET("/media/:year/:month/:filename", h.GetMediaFile) } // Category handlers @@ -246,10 +250,10 @@ func (h *Handler) GetPost(c *gin.Context) { contents := make([]gin.H, 0, len(post.Edges.Contents)) for _, content := range post.Edges.Contents { contents = append(contents, gin.H{ - "language_code": content.LanguageCode, - "title": content.Title, + "language_code": content.LanguageCode, + "title": content.Title, "content_markdown": content.ContentMarkdown, - "summary": content.Summary, + "summary": content.Summary, }) } response["edges"].(gin.H)["contents"] = contents @@ -294,7 +298,7 @@ func (h *Handler) CreatePost(c *gin.Context) { } type AddPostContentRequest struct { - LanguageCode string `json:"language_code" binding:"required"` + LanguageCode string `json:"language_code" binding:"required"` Title string `json:"title" binding:"required"` ContentMarkdown string `json:"content_markdown" binding:"required"` Summary string `json:"summary" binding:"required"` @@ -326,10 +330,10 @@ func (h *Handler) AddPostContent(c *gin.Context) { "title": content.Title, "content_markdown": content.ContentMarkdown, "language_code": content.LanguageCode, - "summary": content.Summary, + "summary": content.Summary, "meta_keywords": content.MetaKeywords, "meta_description": content.MetaDescription, - "edges": gin.H{}, + "edges": gin.H{}, }) } @@ -535,11 +539,3 @@ func (h *Handler) AddDailyContent(c *gin.Context) { c.JSON(http.StatusCreated, content) } - -// Helper functions -func stringPtr(s *string) string { - if s == nil { - return "" - } - return *s -} diff --git a/backend/internal/handler/media.go b/backend/internal/handler/media.go index 68d4a4b..1ad429b 100644 --- a/backend/internal/handler/media.go +++ b/backend/internal/handler/media.go @@ -5,9 +5,11 @@ import ( "io" "net/http" "strconv" + "strings" "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" + "path/filepath" ) // Media handlers @@ -33,17 +35,29 @@ func (h *Handler) ListMedia(c *gin.Context) { return } - c.JSON(http.StatusOK, media) + c.JSON(http.StatusOK, gin.H{ + "data": media, + }) } func (h *Handler) UploadMedia(c *gin.Context) { // Get user ID from context (set by auth middleware) - userID, exists := c.Get("user_id") + userIDStr, exists := c.Get("user_id") if !exists { c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) return } + // Convert user ID to int + userID, err := strconv.Atoi(userIDStr.(string)) + if err != nil { + log.Error().Err(err). + Str("user_id", fmt.Sprintf("%v", userIDStr)). + Msg("Failed to convert user ID to int") + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal server error"}) + return + } + // Get file from form file, err := c.FormFile("file") if err != nil { @@ -51,38 +65,107 @@ func (h *Handler) UploadMedia(c *gin.Context) { return } - // 文件大小限制 - if file.Size > 10*1024*1024 { // 10MB - c.JSON(http.StatusBadRequest, gin.H{"error": "File size exceeds the limit (10MB)"}) + // 获取文件类型和扩展名 + contentType := file.Header.Get("Content-Type") + ext := strings.ToLower(filepath.Ext(file.Filename)) + if contentType == "" { + // 如果 Content-Type 为空，尝试从文件扩展名判断 + switch ext { + case ".jpg", ".jpeg": + contentType = "image/jpeg" + case ".png": + contentType = "image/png" + case ".gif": + contentType = "image/gif" + case ".webp": + contentType = "image/webp" + case ".mp4": + contentType = "video/mp4" + case ".webm": + contentType = "video/webm" + case ".mp3": + contentType = "audio/mpeg" + case ".ogg": + contentType = "audio/ogg" + case ".wav": + contentType = "audio/wav" + case ".pdf": + contentType = "application/pdf" + case ".doc": + contentType = "application/msword" + case ".docx": + contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + } + } + + // 根据 Content-Type 确定文件类型和限制 + var maxSize int64 + var allowedTypes []string + var fileType string + + limits := h.cfg.Storage.Upload.Limits + switch { + case strings.HasPrefix(contentType, "image/"): + maxSize = int64(limits.Image.MaxSize) * 1024 * 1024 + allowedTypes = limits.Image.AllowedTypes + fileType = "image" + case strings.HasPrefix(contentType, "video/"): + maxSize = int64(limits.Video.MaxSize) * 1024 * 1024 + allowedTypes = limits.Video.AllowedTypes + fileType = "video" + case strings.HasPrefix(contentType, "audio/"): + maxSize = int64(limits.Audio.MaxSize) * 1024 * 1024 + allowedTypes = limits.Audio.AllowedTypes + fileType = "audio" + case strings.HasPrefix(contentType, "application/"): + maxSize = int64(limits.Document.MaxSize) * 1024 * 1024 + allowedTypes = limits.Document.AllowedTypes + fileType = "document" + default: + c.JSON(http.StatusBadRequest, gin.H{ + "error": "Unsupported file type", + }) return } - // 文件类型限制 - allowedTypes := map[string]bool{ - "image/jpeg": true, - "image/png": true, - "image/gif": true, - "video/mp4": true, - "video/webm": true, - "audio/mpeg": true, - "audio/ogg": true, - "application/pdf": true, + // 检查文件类型是否允许 + typeAllowed := false + for _, allowed := range allowedTypes { + if contentType == allowed { + typeAllowed = true + break + } } - contentType := file.Header.Get("Content-Type") - if _, ok := allowedTypes[contentType]; !ok { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file type"}) + if !typeAllowed { + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("Unsupported %s type: %s", fileType, contentType), + }) + return + } + + // 检查文件大小 + if file.Size > maxSize { + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("File size exceeds the limit (%d MB) for %s files", limits.Image.MaxSize, fileType), + }) return } // Upload file - media, err := h.service.Upload(c.Request.Context(), file, userID.(int)) + media, err := h.service.Upload(c.Request.Context(), file, userID) if err != nil { - log.Error().Err(err).Msg("Failed to upload media") + log.Error().Err(err). + Str("filename", file.Filename). + Str("content_type", contentType). + Int("user_id", userID). + Msg("Failed to upload media") c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to upload media"}) return } - c.JSON(http.StatusCreated, media) + c.JSON(http.StatusCreated, gin.H{ + "data": media, + }) } func (h *Handler) GetMedia(c *gin.Context) { @@ -101,7 +184,7 @@ func (h *Handler) GetMedia(c *gin.Context) { } // Get file content - reader, info, err := h.service.GetFile(c.Request.Context(), id) + reader, info, err := h.service.GetFile(c.Request.Context(), media.StorageID) if err != nil { log.Error().Err(err).Msg("Failed to get media file") c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get media file"}) @@ -122,16 +205,18 @@ func (h *Handler) GetMedia(c *gin.Context) { } func (h *Handler) GetMediaFile(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid media ID"}) - return - } + year := c.Param("year") + month := c.Param("month") + filename := c.Param("filename") // Get file content - reader, info, err := h.service.GetFile(c.Request.Context(), id) + reader, info, err := h.service.GetFile(c.Request.Context(), filename) // 直接使用完整的文件名 if err != nil { - log.Error().Err(err).Msg("Failed to get media file") + log.Error().Err(err). + Str("year", year). + Str("month", month). + Str("filename", filename). + Msg("Failed to get media file") c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get media file"}) return } @@ -151,20 +236,33 @@ func (h *Handler) GetMediaFile(c *gin.Context) { func (h *Handler) DeleteMedia(c *gin.Context) { // Get user ID from context (set by auth middleware) - userID, exists := c.Get("user_id") + userIDStr, exists := c.Get("user_id") if !exists { c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) return } + // Convert user ID to int + userID, err := strconv.Atoi(userIDStr.(string)) + if err != nil { + log.Error().Err(err). + Str("user_id", fmt.Sprintf("%v", userIDStr)). + Msg("Failed to convert user ID to int") + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal server error"}) + return + } + id, err := strconv.Atoi(c.Param("id")) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid media ID"}) return } - if err := h.service.DeleteMedia(c.Request.Context(), id, userID.(int)); err != nil { - log.Error().Err(err).Msg("Failed to delete media") + if err := h.service.DeleteMedia(c.Request.Context(), id, userID); err != nil { + log.Error().Err(err). + Int("media_id", id). + Int("user_id", userID). + Msg("Failed to delete media") c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete media"}) return } diff --git a/backend/internal/middleware/upload.go b/backend/internal/middleware/upload.go index 91f9e53..c5a3756 100644 --- a/backend/internal/middleware/upload.go +++ b/backend/internal/middleware/upload.go @@ -3,146 +3,120 @@ package middleware import ( "bytes" "fmt" - "io" "net/http" "path/filepath" "strings" + "tss-rocks-be/internal/config" + "github.com/gin-gonic/gin" - "tss-rocks-be/internal/types" ) -const ( - defaultMaxMemory = 32 << 20 // 32 MB - maxHeaderBytes = 512 // 用于MIME类型检测的最大字节数 -) - -// ValidateUpload 创建文件上传验证中间件 -func ValidateUpload(cfg *types.UploadConfig) gin.HandlerFunc { +// ValidateUpload 验证上传的文件 +func ValidateUpload(cfg *config.UploadConfig) gin.HandlerFunc { return func(c *gin.Context) { - // 检查是否是multipart/form-data请求 - if !strings.HasPrefix(c.GetHeader("Content-Type"), "multipart/form-data") { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "Content-Type must be multipart/form-data", - }) + // Get file from form + file, err := c.FormFile("file") + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "No file uploaded"}) c.Abort() return } - // 解析multipart表单 - if err := c.Request.ParseMultipartForm(defaultMaxMemory); err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": fmt.Sprintf("Failed to parse form: %v", err), - }) - c.Abort() - return - } + // 获取文件类型和扩展名 + contentType := file.Header.Get("Content-Type") + ext := strings.ToLower(filepath.Ext(file.Filename)) - form := c.Request.MultipartForm - if form == nil || form.File == nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "No file uploaded", - }) - c.Abort() - return - } - - // 遍历所有上传的文件 - for _, files := range form.File { - for _, file := range files { - // 检查文件大小 - if file.Size > int64(cfg.MaxSize)<<20 { // 转换为字节 - c.JSON(http.StatusBadRequest, gin.H{ - "error": fmt.Sprintf("File %s exceeds maximum size of %d MB", file.Filename, cfg.MaxSize), - }) - c.Abort() - return - } - - // 检查文件扩展名 - ext := strings.ToLower(filepath.Ext(file.Filename)) - if !contains(cfg.AllowedExtensions, ext) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": fmt.Sprintf("File extension %s is not allowed", ext), - }) - c.Abort() - return - } - - // 打开文件 - src, err := file.Open() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": fmt.Sprintf("Failed to open file: %v", err), - }) - c.Abort() - return - } - defer src.Close() - - // 读取文件头部用于MIME类型检测 - header := make([]byte, maxHeaderBytes) - n, err := src.Read(header) - if err != nil && err != io.EOF { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": fmt.Sprintf("Failed to read file: %v", err), - }) - c.Abort() - return - } - header = header[:n] - - // 检测MIME类型 - contentType := http.DetectContentType(header) - if !contains(cfg.AllowedTypes, contentType) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": fmt.Sprintf("File type %s is not allowed", contentType), - }) - c.Abort() - return - } - - // 将文件指针重置到开始位置 - _, err = src.Seek(0, 0) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": fmt.Sprintf("Failed to read file: %v", err), - }) - c.Abort() - return - } - - // 将文件内容读入缓冲区 - buf := &bytes.Buffer{} - _, err = io.Copy(buf, src) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": fmt.Sprintf("Failed to read file: %v", err), - }) - c.Abort() - return - } - - // 将验证过的文件内容和类型保存到上下文中 - c.Set("validated_file_"+file.Filename, buf) - c.Set("validated_content_type_"+file.Filename, contentType) + // 如果 Content-Type 为空，尝试从文件扩展名判断 + if contentType == "" { + switch ext { + case ".jpg", ".jpeg": + contentType = "image/jpeg" + case ".png": + contentType = "image/png" + case ".gif": + contentType = "image/gif" + case ".webp": + contentType = "image/webp" + case ".mp4": + contentType = "video/mp4" + case ".webm": + contentType = "video/webm" + case ".mp3": + contentType = "audio/mpeg" + case ".ogg": + contentType = "audio/ogg" + case ".wav": + contentType = "audio/wav" + case ".pdf": + contentType = "application/pdf" + case ".doc": + contentType = "application/msword" + case ".docx": + contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } } + // 根据 Content-Type 确定文件类型和限制 + var maxSize int64 + var allowedTypes []string + var fileType string + + limits := cfg.Limits + switch { + case strings.HasPrefix(contentType, "image/"): + maxSize = int64(limits.Image.MaxSize) * 1024 * 1024 + allowedTypes = limits.Image.AllowedTypes + fileType = "image" + case strings.HasPrefix(contentType, "video/"): + maxSize = int64(limits.Video.MaxSize) * 1024 * 1024 + allowedTypes = limits.Video.AllowedTypes + fileType = "video" + case strings.HasPrefix(contentType, "audio/"): + maxSize = int64(limits.Audio.MaxSize) * 1024 * 1024 + allowedTypes = limits.Audio.AllowedTypes + fileType = "audio" + case strings.HasPrefix(contentType, "application/"): + maxSize = int64(limits.Document.MaxSize) * 1024 * 1024 + allowedTypes = limits.Document.AllowedTypes + fileType = "document" + default: + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("Unsupported file type: %s", contentType), + }) + c.Abort() + return + } + + // 检查文件类型是否允许 + typeAllowed := false + for _, allowed := range allowedTypes { + if contentType == allowed { + typeAllowed = true + break + } + } + if !typeAllowed { + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("Unsupported %s type: %s", fileType, contentType), + }) + c.Abort() + return + } + + // 检查文件大小 + if file.Size > maxSize { + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("File size exceeds the limit (%d MB) for %s files", limits.Image.MaxSize, fileType), + }) + c.Abort() + return + } + c.Next() } } -// contains 检查切片中是否包含指定的字符串 -func contains(slice []string, str string) bool { - for _, s := range slice { - if s == str { - return true - } - } - return false -} - // GetValidatedFile 从上下文中获取验证过的文件内容 func GetValidatedFile(c *gin.Context, filename string) (*bytes.Buffer, string, bool) { file, exists := c.Get("validated_file_" + filename) diff --git a/backend/internal/service/impl.go b/backend/internal/service/impl.go index fed9495..98203ee 100644 --- a/backend/internal/service/impl.go +++ b/backend/internal/service/impl.go @@ -1,11 +1,14 @@ package service import ( + "bytes" "context" "errors" "fmt" "io" "mime/multipart" + "os" + "path/filepath" "sort" "strconv" "strings" @@ -26,6 +29,8 @@ import ( "tss-rocks-be/ent/user" "tss-rocks-be/internal/storage" + "github.com/chai2010/webp" + "github.com/disintegration/imaging" "github.com/google/uuid" "golang.org/x/crypto/bcrypt" ) @@ -419,21 +424,71 @@ func (s *serviceImpl) Upload(ctx context.Context, file *multipart.FileHeader, us // Open the uploaded file src, err := openFile(file) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open file: %v", err) } defer src.Close() + // 获取文件类型和扩展名 + contentType := file.Header.Get("Content-Type") + ext := strings.ToLower(filepath.Ext(file.Filename)) + if contentType == "" { + // 如果 Content-Type 为空，尝试从文件扩展名判断 + switch ext { + case ".jpg", ".jpeg": + contentType = "image/jpeg" + case ".png": + contentType = "image/png" + case ".gif": + contentType = "image/gif" + case ".webp": + contentType = "image/webp" + case ".mp4": + contentType = "video/mp4" + case ".webm": + contentType = "video/webm" + case ".mp3": + contentType = "audio/mpeg" + case ".ogg": + contentType = "audio/ogg" + case ".wav": + contentType = "audio/wav" + case ".pdf": + contentType = "application/pdf" + case ".doc": + contentType = "application/msword" + case ".docx": + contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + } + } + + // 如果是图片，检查是否需要转换为 WebP + var fileToSave multipart.File = src + var finalContentType = contentType + if strings.HasPrefix(contentType, "image/") && contentType != "image/webp" { + // 转换为 WebP + webpFile, err := convertToWebP(src) + if err != nil { + return nil, fmt.Errorf("failed to convert image to WebP: %v", err) + } + fileToSave = webpFile + finalContentType = "image/webp" + ext = ".webp" + } + + // 生成带扩展名的存储文件名 + storageFilename := uuid.New().String() + ext + // Save the file to storage - fileInfo, err := s.storage.Save(ctx, file.Filename, file.Header.Get("Content-Type"), src) + fileInfo, err := s.storage.Save(ctx, storageFilename, finalContentType, fileToSave) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to save file: %v", err) } // Create media record return s.client.Media.Create(). SetStorageID(fileInfo.ID). SetOriginalName(file.Filename). - SetMimeType(fileInfo.ContentType). + SetMimeType(finalContentType). SetSize(fileInfo.Size). SetURL(fileInfo.URL). SetCreatedBy(strconv.Itoa(userID)). @@ -444,13 +499,8 @@ func (s *serviceImpl) GetMedia(ctx context.Context, id int) (*ent.Media, error) return s.client.Media.Get(ctx, id) } -func (s *serviceImpl) GetFile(ctx context.Context, id int) (io.ReadCloser, *storage.FileInfo, error) { - media, err := s.GetMedia(ctx, id) - if err != nil { - return nil, nil, err - } - - return s.storage.Get(ctx, media.StorageID) +func (s *serviceImpl) GetFile(ctx context.Context, storageID string) (io.ReadCloser, *storage.FileInfo, error) { + return s.storage.Get(ctx, storageID) } func (s *serviceImpl) DeleteMedia(ctx context.Context, id int, userID int) error { @@ -1065,3 +1115,47 @@ func (s *serviceImpl) DeleteDaily(ctx context.Context, id string, currentUserID return s.client.Daily.DeleteOneID(id).Exec(ctx) } + +// convertToWebP 将图片转换为 WebP 格式 +func convertToWebP(src multipart.File) (multipart.File, error) { + // 读取原始图片 + img, err := imaging.Decode(src) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %v", err) + } + + // 创建一个新的缓冲区来存储 WebP 图片 + buf := new(bytes.Buffer) + + // 将图片编码为 WebP 格式 + // 设置较高的质量以保持图片质量 + err = webp.Encode(buf, img, &webp.Options{ + Lossless: false, + Quality: 90, + }) + if err != nil { + return nil, fmt.Errorf("failed to encode image to WebP: %v", err) + } + + // 创建一个新的临时文件来存储转换后的图片 + tmpFile, err := os.CreateTemp("", "webp-*.webp") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %v", err) + } + + // 写入转换后的数据 + if _, err := io.Copy(tmpFile, buf); err != nil { + tmpFile.Close() + os.Remove(tmpFile.Name()) + return nil, fmt.Errorf("failed to write WebP data: %v", err) + } + + // 将文件指针移回开始位置 + if _, err := tmpFile.Seek(0, 0); err != nil { + tmpFile.Close() + os.Remove(tmpFile.Name()) + return nil, fmt.Errorf("failed to seek file: %v", err) + } + + return tmpFile, nil +} diff --git a/backend/internal/service/service.go b/backend/internal/service/service.go index 38fa1da..ee6aedd 100644 --- a/backend/internal/service/service.go +++ b/backend/internal/service/service.go @@ -41,7 +41,7 @@ type Service interface { ListMedia(ctx context.Context, limit, offset int) ([]*ent.Media, error) Upload(ctx context.Context, file *multipart.FileHeader, userID int) (*ent.Media, error) GetMedia(ctx context.Context, id int) (*ent.Media, error) - GetFile(ctx context.Context, id int) (io.ReadCloser, *storage.FileInfo, error) + GetFile(ctx context.Context, storageID string) (io.ReadCloser, *storage.FileInfo, error) DeleteMedia(ctx context.Context, id int, userID int) error // Contributor operations diff --git a/backend/internal/storage/local.go b/backend/internal/storage/local.go index 2b1d8bf..d8ccff7 100644 --- a/backend/internal/storage/local.go +++ b/backend/internal/storage/local.go @@ -11,6 +11,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/rs/zerolog/log" ) type LocalStorage struct { @@ -44,6 +46,24 @@ func (s *LocalStorage) generateID() (string, error) { return hex.EncodeToString(bytes), nil } +func (s *LocalStorage) generateFilePath(id string, ext string, createTime time.Time) string { + // Create year/month directory structure + year := createTime.Format("2006") + month := createTime.Format("01") + + // If id already has an extension, don't add ext + if filepath.Ext(id) != "" { + return filepath.Join(year, month, id) + } + + // Otherwise, add the extension if provided + filename := id + if ext != "" { + filename = id + ext + } + return filepath.Join(year, month, filename) +} + func (s *LocalStorage) saveMetadata(id string, info *FileInfo) error { metaPath := filepath.Join(s.metaDir, id+".meta") file, err := os.Create(metaPath) @@ -89,11 +109,64 @@ func (s *LocalStorage) Save(ctx context.Context, name string, contentType string return nil, fmt.Errorf("failed to generate file ID: %w", err) } - // Create the file path - filePath := filepath.Join(s.rootDir, id) + // Get file extension from original name or content type + ext := filepath.Ext(name) + if ext == "" { + // If no extension in name, try to get it from content type + switch contentType { + case "image/jpeg": + ext = ".jpg" + case "image/png": + ext = ".png" + case "image/gif": + ext = ".gif" + case "image/webp": + ext = ".webp" + case "image/svg+xml": + ext = ".svg" + case "video/mp4": + ext = ".mp4" + case "video/webm": + ext = ".webm" + case "audio/mpeg": + ext = ".mp3" + case "audio/ogg": + ext = ".ogg" + case "audio/wav": + ext = ".wav" + case "application/pdf": + ext = ".pdf" + case "application/msword": + ext = ".doc" + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + ext = ".docx" + case "application/vnd.ms-excel": + ext = ".xls" + case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + ext = ".xlsx" + case "application/zip": + ext = ".zip" + case "application/x-rar-compressed": + ext = ".rar" + case "text/plain": + ext = ".txt" + case "text/csv": + ext = ".csv" + } + } + + // Create the file path with year/month structure + now := time.Now() + relPath := s.generateFilePath(id, ext, now) + fullPath := filepath.Join(s.rootDir, relPath) + + // Create directory if it doesn't exist + if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } // Create the file - file, err := os.Create(filePath) + file, err := os.Create(fullPath) if err != nil { return nil, fmt.Errorf("failed to create file: %w", err) } @@ -102,52 +175,73 @@ func (s *LocalStorage) Save(ctx context.Context, name string, contentType string // Copy the content size, err := io.Copy(file, reader) if err != nil { - // Clean up the file if there's an error - os.Remove(filePath) + os.Remove(fullPath) // Clean up on error return nil, fmt.Errorf("failed to write file content: %w", err) } - now := time.Now() + // Save metadata info := &FileInfo{ ID: id, Name: name, - Size: size, ContentType: contentType, + Size: size, CreatedAt: now, - UpdatedAt: now, - URL: fmt.Sprintf("/api/media/file/%s", id), + URL: fmt.Sprintf("/media/%s/%s/%s", now.Format("2006"), now.Format("01"), filepath.Base(relPath)), } - - // Save metadata if err := s.saveMetadata(id, info); err != nil { - os.Remove(filePath) - return nil, err + os.Remove(fullPath) // Clean up on error + return nil, fmt.Errorf("failed to save metadata: %w", err) } return info, nil } func (s *LocalStorage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) { - filePath := filepath.Join(s.rootDir, id) + // 从 id 中提取文件扩展名和基础 ID + ext := filepath.Ext(id) + baseID := strings.TrimSuffix(id, ext) - // Open the file + // 获取文件的创建时间（从元数据或当前时间） + metaPath := filepath.Join(s.metaDir, baseID+".meta") + var createTime time.Time + if stat, err := os.Stat(metaPath); err == nil { + createTime = stat.ModTime() + } else { + createTime = time.Now() // 如果找不到元数据，使用当前时间 + } + + // 生成完整的文件路径 + year := createTime.Format("2006") + month := createTime.Format("01") + filePath := filepath.Join(s.rootDir, year, month, id) // 直接使用完整的 id（包含扩展名） + + // 调试日志 + log.Debug(). + Str("id", id). + Str("baseID", baseID). + Str("ext", ext). + Str("filePath", filePath). + Time("createTime", createTime). + Msg("Attempting to get file") + + // 打开文件 file, err := os.Open(filePath) if err != nil { if os.IsNotExist(err) { - return nil, nil, fmt.Errorf("file not found: %s", id) + return nil, nil, fmt.Errorf("file not found: %s (path: %s)", id, filePath) } return nil, nil, fmt.Errorf("failed to open file: %w", err) } - // Get file info + // 获取文件信息 stat, err := file.Stat() if err != nil { file.Close() return nil, nil, fmt.Errorf("failed to get file info: %w", err) } - // Load metadata - name, contentType, err := s.loadMetadata(id) + // 加载元数据 + name, contentType, err := s.loadMetadata(baseID) if err != nil { file.Close() return nil, nil, err @@ -158,27 +252,44 @@ func (s *LocalStorage) Get(ctx context.Context, id string) (io.ReadCloser, *File Name: name, Size: stat.Size(), ContentType: contentType, - CreatedAt: stat.ModTime(), + CreatedAt: createTime, UpdatedAt: stat.ModTime(), - URL: fmt.Sprintf("/api/media/file/%s", id), + URL: fmt.Sprintf("/media/%s/%s/%s", year, month, id), } return file, info, nil } func (s *LocalStorage) Delete(ctx context.Context, id string) error { - filePath := filepath.Join(s.rootDir, id) - if err := os.Remove(filePath); err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("file not found: %s", id) - } - return fmt.Errorf("failed to delete file: %w", err) + // 从 id 中提取文件扩展名 + ext := filepath.Ext(id) + baseID := strings.TrimSuffix(id, ext) + + // 获取文件的创建时间（从元数据或当前时间） + metaPath := filepath.Join(s.metaDir, baseID+".meta") + var createTime time.Time + if stat, err := os.Stat(metaPath); err == nil { + createTime = stat.ModTime() + } else { + createTime = time.Now() // 如果找不到元数据，使用当前时间 } - // Remove metadata - metaPath := filepath.Join(s.metaDir, id+".meta") - if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to remove metadata: %w", err) + // 生成完整的文件路径 + relPath := s.generateFilePath(baseID, ext, createTime) + filePath := filepath.Join(s.rootDir, relPath) + + // 删除文件 + if err := os.Remove(filePath); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to delete file: %w", err) + } + } + + // 删除元数据 + if err := os.Remove(metaPath); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to delete metadata: %w", err) + } } return nil diff --git a/backend/internal/storage/s3.go b/backend/internal/storage/s3.go index bea236d..88f8825 100644 --- a/backend/internal/storage/s3.go +++ b/backend/internal/storage/s3.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "strings" "time" @@ -48,14 +49,25 @@ func (s *S3Storage) generateID() (string, error) { return hex.EncodeToString(bytes), nil } -func (s *S3Storage) getObjectURL(id string) string { +func (s *S3Storage) generateObjectKey(id string, ext string, createTime time.Time) string { + // Create year/month structure + year := createTime.Format("2006") + month := createTime.Format("01") + filename := id + if ext != "" { + filename = id + ext + } + return fmt.Sprintf("%s/%s/%s", year, month, filename) +} + +func (s *S3Storage) getObjectURL(key string) string { if s.customURL != "" { - return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), id) + return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), key) } if s.proxyS3 { - return fmt.Sprintf("/api/media/file/%s", id) + return fmt.Sprintf("/media/%s", key) } - return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, id) + return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, key) } func (s *S3Storage) Save(ctx context.Context, name string, contentType string, reader io.Reader) (*FileInfo, error) { @@ -65,10 +77,60 @@ func (s *S3Storage) Save(ctx context.Context, name string, contentType string, r return nil, fmt.Errorf("failed to generate file ID: %w", err) } + // Get file extension from original name or content type + ext := filepath.Ext(name) + if ext == "" { + // If no extension in name, try to get it from content type + switch contentType { + case "image/jpeg": + ext = ".jpg" + case "image/png": + ext = ".png" + case "image/gif": + ext = ".gif" + case "image/webp": + ext = ".webp" + case "image/svg+xml": + ext = ".svg" + case "video/mp4": + ext = ".mp4" + case "video/webm": + ext = ".webm" + case "audio/mpeg": + ext = ".mp3" + case "audio/ogg": + ext = ".ogg" + case "audio/wav": + ext = ".wav" + case "application/pdf": + ext = ".pdf" + case "application/msword": + ext = ".doc" + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + ext = ".docx" + case "application/vnd.ms-excel": + ext = ".xls" + case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + ext = ".xlsx" + case "application/zip": + ext = ".zip" + case "application/x-rar-compressed": + ext = ".rar" + case "text/plain": + ext = ".txt" + case "text/csv": + ext = ".csv" + } + } + + // Create the object key with year/month structure + now := time.Now() + key := s.generateObjectKey(id, ext, now) + // Check if the file exists _, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(s.bucket), - Key: aws.String(id), + Key: aws.String(key), }) if err == nil { return nil, fmt.Errorf("file already exists with ID: %s", id) @@ -82,37 +144,50 @@ func (s *S3Storage) Save(ctx context.Context, name string, contentType string, r // Upload the file _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.bucket), - Key: aws.String(id), + Key: aws.String(key), Body: reader, ContentType: aws.String(contentType), Metadata: map[string]string{ "x-amz-meta-original-name": name, + "x-amz-meta-created-at": now.Format(time.RFC3339), }, }) if err != nil { return nil, fmt.Errorf("failed to upload file: %w", err) } - now := time.Now() info := &FileInfo{ ID: id, Name: name, Size: 0, // Size is not available until after upload ContentType: contentType, CreatedAt: now, - UpdatedAt: now, - URL: s.getObjectURL(id), + URL: s.getObjectURL(key), } return info, nil } func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) { - // Get the object from S3 - result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(id), - }) + // Try to find the file with different extensions + exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav", + ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"} + + var result *s3.GetObjectOutput + var err error + var key string + + for _, ext := range exts { + key = s.generateObjectKey(id, ext, time.Now()) + result, err = s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err == nil { + break + } + } + if err != nil { return nil, nil, fmt.Errorf("failed to get file from S3: %w", err) } @@ -122,111 +197,117 @@ func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInf Name: result.Metadata["x-amz-meta-original-name"], Size: aws.ToInt64(result.ContentLength), ContentType: aws.ToString(result.ContentType), - CreatedAt: aws.ToTime(result.LastModified), - UpdatedAt: aws.ToTime(result.LastModified), - URL: s.getObjectURL(id), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + URL: s.getObjectURL(key), } return result.Body, info, nil } func (s *S3Storage) Delete(ctx context.Context, id string) error { - _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(id), - }) - if err != nil { - return fmt.Errorf("failed to delete file from S3: %w", err) + // Try to find and delete the file with different extensions + exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav", + ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"} + + var lastErr error + for _, ext := range exts { + key := s.generateObjectKey(id, ext, time.Now()) + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err == nil { + return nil + } + lastErr = err } - return nil + return fmt.Errorf("failed to delete file from S3: %w", lastErr) +} + +func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) { + // Try to find the file with different extensions + exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav", + ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"} + + for _, ext := range exts { + key := s.generateObjectKey(id, ext, time.Now()) + _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err == nil { + return true, nil + } + } + + return false, nil } func (s *S3Storage) List(ctx context.Context, prefix string, limit int, offset int) ([]*FileInfo, error) { var files []*FileInfo var continuationToken *string + count := 0 + skip := offset - // Skip objects for offset - for i := 0; i < offset/1000; i++ { - output, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + for { + input := &s3.ListObjectsV2Input{ Bucket: aws.String(s.bucket), Prefix: aws.String(prefix), ContinuationToken: continuationToken, - MaxKeys: aws.Int32(1000), - }) + MaxKeys: aws.Int32(100), // Fetch in batches of 100 + } + + result, err := s.client.ListObjectsV2(ctx, input) if err != nil { return nil, fmt.Errorf("failed to list files from S3: %w", err) } - if !aws.ToBool(output.IsTruncated) { - return files, nil - } - continuationToken = output.NextContinuationToken - } - // Get the actual objects - output, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(prefix), - ContinuationToken: continuationToken, - MaxKeys: aws.Int32(int32(limit)), - }) - if err != nil { - return nil, fmt.Errorf("failed to list files from S3: %w", err) - } - - for _, obj := range output.Contents { - // Get the object metadata - head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: obj.Key, - }) - - var contentType string - var originalName string - - if err != nil { - var noSuchKey *types.NoSuchKey - if errors.As(err, &noSuchKey) { - // If the object doesn't exist (which shouldn't happen normally), - // we'll still include it in the list but with empty metadata - contentType = "" - originalName = aws.ToString(obj.Key) - } else { + // Process each object + for _, obj := range result.Contents { + if skip > 0 { + skip-- continue } - } else { - contentType = aws.ToString(head.ContentType) - originalName = head.Metadata["x-amz-meta-original-name"] - if originalName == "" { - originalName = aws.ToString(obj.Key) + + // Get object metadata + head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: obj.Key, + }) + if err != nil { + continue // Skip files we can't get metadata for + } + + // Extract the ID from the key (remove extension if present) + id := aws.ToString(obj.Key) + if ext := filepath.Ext(id); ext != "" { + id = id[:len(id)-len(ext)] + } + + info := &FileInfo{ + ID: id, + Name: head.Metadata["x-amz-meta-original-name"], + Size: aws.ToInt64(obj.Size), + ContentType: aws.ToString(head.ContentType), + CreatedAt: aws.ToTime(obj.LastModified), + UpdatedAt: aws.ToTime(obj.LastModified), + URL: s.getObjectURL(aws.ToString(obj.Key)), + } + files = append(files, info) + count++ + + if count >= limit { + return files, nil } } - files = append(files, &FileInfo{ - ID: aws.ToString(obj.Key), - Name: originalName, - Size: aws.ToInt64(obj.Size), - ContentType: contentType, - CreatedAt: aws.ToTime(obj.LastModified), - UpdatedAt: aws.ToTime(obj.LastModified), - URL: s.getObjectURL(aws.ToString(obj.Key)), - }) + if !aws.ToBool(result.IsTruncated) { + break + } + continuationToken = result.NextContinuationToken } return files, nil } - -func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) { - _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(id), - }) - if err != nil { - var nsk *types.NoSuchKey - if ok := errors.As(err, &nsk); ok { - return false, nil - } - return false, fmt.Errorf("failed to check file existence in S3: %w", err) - } - return true, nil -} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index ac62e5e..7a0e3cd 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -24,6 +24,11 @@ export default defineConfig({ changeOrigin: true, secure: false, }, + '/media': { + target: 'http://localhost:8080', + changeOrigin: true, + secure: false, + }, }, }, });