[feature/backend] overall enhancement of image uploading

backend/internal/storage/local.go

@@ -11,6 +11,8 @@ import (
     "path/filepath"
     "strings"
     "time"
+
+    "github.com/rs/zerolog/log"
 )
 
 type LocalStorage struct {
@@ -44,6 +46,24 @@ func (s *LocalStorage) generateID() (string, error) {
     return hex.EncodeToString(bytes), nil
 }
 
+func (s *LocalStorage) generateFilePath(id string, ext string, createTime time.Time) string {
+    // Create year/month directory structure
+    year := createTime.Format("2006")
+    month := createTime.Format("01")
+    
+    // If id already has an extension, don't add ext
+    if filepath.Ext(id) != "" {
+        return filepath.Join(year, month, id)
+    }
+    
+    // Otherwise, add the extension if provided
+    filename := id
+    if ext != "" {
+        filename = id + ext
+    }
+    return filepath.Join(year, month, filename)
+}
+
 func (s *LocalStorage) saveMetadata(id string, info *FileInfo) error {
     metaPath := filepath.Join(s.metaDir, id+".meta")
     file, err := os.Create(metaPath)
@@ -89,11 +109,64 @@ func (s *LocalStorage) Save(ctx context.Context, name string, contentType string
         return nil, fmt.Errorf("failed to generate file ID: %w", err)
     }
 
-    // Create the file path
-    filePath := filepath.Join(s.rootDir, id)
+    // Get file extension from original name or content type
+    ext := filepath.Ext(name)
+    if ext == "" {
+        // If no extension in name, try to get it from content type
+        switch contentType {
+        case "image/jpeg":
+            ext = ".jpg"
+        case "image/png":
+            ext = ".png"
+        case "image/gif":
+            ext = ".gif"
+        case "image/webp":
+            ext = ".webp"
+        case "image/svg+xml":
+            ext = ".svg"
+        case "video/mp4":
+            ext = ".mp4"
+        case "video/webm":
+            ext = ".webm"
+        case "audio/mpeg":
+            ext = ".mp3"
+        case "audio/ogg":
+            ext = ".ogg"
+        case "audio/wav":
+            ext = ".wav"
+        case "application/pdf":
+            ext = ".pdf"
+        case "application/msword":
+            ext = ".doc"
+        case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            ext = ".docx"
+        case "application/vnd.ms-excel":
+            ext = ".xls"
+        case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+            ext = ".xlsx"
+        case "application/zip":
+            ext = ".zip"
+        case "application/x-rar-compressed":
+            ext = ".rar"
+        case "text/plain":
+            ext = ".txt"
+        case "text/csv":
+            ext = ".csv"
+        }
+    }
+
+    // Create the file path with year/month structure
+    now := time.Now()
+    relPath := s.generateFilePath(id, ext, now)
+    fullPath := filepath.Join(s.rootDir, relPath)
+
+    // Create directory if it doesn't exist
+    if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+        return nil, fmt.Errorf("failed to create directory: %w", err)
+    }
 
     // Create the file
-    file, err := os.Create(filePath)
+    file, err := os.Create(fullPath)
     if err != nil {
         return nil, fmt.Errorf("failed to create file: %w", err)
     }
@@ -102,52 +175,73 @@ func (s *LocalStorage) Save(ctx context.Context, name string, contentType string
     // Copy the content
     size, err := io.Copy(file, reader)
     if err != nil {
-        // Clean up the file if there's an error
-        os.Remove(filePath)
+        os.Remove(fullPath) // Clean up on error
         return nil, fmt.Errorf("failed to write file content: %w", err)
     }
 
-    now := time.Now()
+    // Save metadata
     info := &FileInfo{
         ID:          id,
         Name:        name,
-        Size:        size,
         ContentType: contentType,
+        Size:        size,
         CreatedAt:   now,
-        UpdatedAt:   now,
-        URL:         fmt.Sprintf("/api/media/file/%s", id),
+        URL:         fmt.Sprintf("/media/%s/%s/%s", now.Format("2006"), now.Format("01"), filepath.Base(relPath)),
     }
-
-    // Save metadata
     if err := s.saveMetadata(id, info); err != nil {
-        os.Remove(filePath)
-        return nil, err
+        os.Remove(fullPath) // Clean up on error
+        return nil, fmt.Errorf("failed to save metadata: %w", err)
     }
 
     return info, nil
 }
 
 func (s *LocalStorage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) {
-    filePath := filepath.Join(s.rootDir, id)
+    // 从 id 中提取文件扩展名和基础 ID
+    ext := filepath.Ext(id)
+    baseID := strings.TrimSuffix(id, ext)
 
-    // Open the file
+    // 获取文件的创建时间（从元数据或当前时间）
+    metaPath := filepath.Join(s.metaDir, baseID+".meta")
+    var createTime time.Time
+    if stat, err := os.Stat(metaPath); err == nil {
+        createTime = stat.ModTime()
+    } else {
+        createTime = time.Now() // 如果找不到元数据，使用当前时间
+    }
+
+    // 生成完整的文件路径
+    year := createTime.Format("2006")
+    month := createTime.Format("01")
+    filePath := filepath.Join(s.rootDir, year, month, id) // 直接使用完整的 id（包含扩展名）
+
+    // 调试日志
+    log.Debug().
+        Str("id", id).
+        Str("baseID", baseID).
+        Str("ext", ext).
+        Str("filePath", filePath).
+        Time("createTime", createTime).
+        Msg("Attempting to get file")
+
+    // 打开文件
     file, err := os.Open(filePath)
     if err != nil {
         if os.IsNotExist(err) {
-            return nil, nil, fmt.Errorf("file not found: %s", id)
+            return nil, nil, fmt.Errorf("file not found: %s (path: %s)", id, filePath)
         }
         return nil, nil, fmt.Errorf("failed to open file: %w", err)
     }
 
-    // Get file info
+    // 获取文件信息
     stat, err := file.Stat()
     if err != nil {
         file.Close()
         return nil, nil, fmt.Errorf("failed to get file info: %w", err)
     }
 
-    // Load metadata
-    name, contentType, err := s.loadMetadata(id)
+    // 加载元数据
+    name, contentType, err := s.loadMetadata(baseID)
     if err != nil {
         file.Close()
         return nil, nil, err
@@ -158,27 +252,44 @@ func (s *LocalStorage) Get(ctx context.Context, id string) (io.ReadCloser, *File
         Name:        name,
         Size:        stat.Size(),
         ContentType: contentType,
-        CreatedAt:   stat.ModTime(),
+        CreatedAt:   createTime,
         UpdatedAt:   stat.ModTime(),
-        URL:         fmt.Sprintf("/api/media/file/%s", id),
+        URL:         fmt.Sprintf("/media/%s/%s/%s", year, month, id),
     }
 
     return file, info, nil
 }
 
 func (s *LocalStorage) Delete(ctx context.Context, id string) error {
-    filePath := filepath.Join(s.rootDir, id)
-    if err := os.Remove(filePath); err != nil {
-        if os.IsNotExist(err) {
-            return fmt.Errorf("file not found: %s", id)
-        }
-        return fmt.Errorf("failed to delete file: %w", err)
+    // 从 id 中提取文件扩展名
+    ext := filepath.Ext(id)
+    baseID := strings.TrimSuffix(id, ext)
+
+    // 获取文件的创建时间（从元数据或当前时间）
+    metaPath := filepath.Join(s.metaDir, baseID+".meta")
+    var createTime time.Time
+    if stat, err := os.Stat(metaPath); err == nil {
+        createTime = stat.ModTime()
+    } else {
+        createTime = time.Now() // 如果找不到元数据，使用当前时间
     }
 
-    // Remove metadata
-    metaPath := filepath.Join(s.metaDir, id+".meta")
-    if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) {
-        return fmt.Errorf("failed to remove metadata: %w", err)
+    // 生成完整的文件路径
+    relPath := s.generateFilePath(baseID, ext, createTime)
+    filePath := filepath.Join(s.rootDir, relPath)
+
+    // 删除文件
+    if err := os.Remove(filePath); err != nil {
+        if !os.IsNotExist(err) {
+            return fmt.Errorf("failed to delete file: %w", err)
+        }
+    }
+
+    // 删除元数据
+    if err := os.Remove(metaPath); err != nil {
+        if !os.IsNotExist(err) {
+            return fmt.Errorf("failed to delete metadata: %w", err)
+        }
     }
 
     return nil

backend/internal/storage/s3.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -48,14 +49,25 @@ func (s *S3Storage) generateID() (string, error) {
 	return hex.EncodeToString(bytes), nil
 }
 
-func (s *S3Storage) getObjectURL(id string) string {
+func (s *S3Storage) generateObjectKey(id string, ext string, createTime time.Time) string {
+	// Create year/month structure
+	year := createTime.Format("2006")
+	month := createTime.Format("01")
+	filename := id
+	if ext != "" {
+		filename = id + ext
+	}
+	return fmt.Sprintf("%s/%s/%s", year, month, filename)
+}
+
+func (s *S3Storage) getObjectURL(key string) string {
 	if s.customURL != "" {
-		return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), id)
+		return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), key)
 	}
 	if s.proxyS3 {
-		return fmt.Sprintf("/api/media/file/%s", id)
+		return fmt.Sprintf("/media/%s", key)
 	}
-	return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, id)
+	return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, key)
 }
 
 func (s *S3Storage) Save(ctx context.Context, name string, contentType string, reader io.Reader) (*FileInfo, error) {
@@ -65,10 +77,60 @@ func (s *S3Storage) Save(ctx context.Context, name string, contentType string, r
 		return nil, fmt.Errorf("failed to generate file ID: %w", err)
 	}
 
+	// Get file extension from original name or content type
+	ext := filepath.Ext(name)
+	if ext == "" {
+		// If no extension in name, try to get it from content type
+		switch contentType {
+		case "image/jpeg":
+			ext = ".jpg"
+		case "image/png":
+			ext = ".png"
+		case "image/gif":
+			ext = ".gif"
+		case "image/webp":
+			ext = ".webp"
+		case "image/svg+xml":
+			ext = ".svg"
+		case "video/mp4":
+			ext = ".mp4"
+		case "video/webm":
+			ext = ".webm"
+		case "audio/mpeg":
+			ext = ".mp3"
+		case "audio/ogg":
+			ext = ".ogg"
+		case "audio/wav":
+			ext = ".wav"
+		case "application/pdf":
+			ext = ".pdf"
+		case "application/msword":
+			ext = ".doc"
+		case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+			ext = ".docx"
+		case "application/vnd.ms-excel":
+			ext = ".xls"
+		case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+			ext = ".xlsx"
+		case "application/zip":
+			ext = ".zip"
+		case "application/x-rar-compressed":
+			ext = ".rar"
+		case "text/plain":
+			ext = ".txt"
+		case "text/csv":
+			ext = ".csv"
+		}
+	}
+
+	// Create the object key with year/month structure
+	now := time.Now()
+	key := s.generateObjectKey(id, ext, now)
+
 	// Check if the file exists
 	_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
 		Bucket: aws.String(s.bucket),
-		Key:    aws.String(id),
+		Key:    aws.String(key),
 	})
 	if err == nil {
 		return nil, fmt.Errorf("file already exists with ID: %s", id)
@@ -82,37 +144,50 @@ func (s *S3Storage) Save(ctx context.Context, name string, contentType string, r
 	// Upload the file
 	_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
 		Bucket:      aws.String(s.bucket),
-		Key:         aws.String(id),
+		Key:         aws.String(key),
 		Body:        reader,
 		ContentType: aws.String(contentType),
 		Metadata: map[string]string{
 			"x-amz-meta-original-name": name,
+			"x-amz-meta-created-at":    now.Format(time.RFC3339),
 		},
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to upload file: %w", err)
 	}
 
-	now := time.Now()
 	info := &FileInfo{
 		ID:          id,
 		Name:        name,
 		Size:        0, // Size is not available until after upload
 		ContentType: contentType,
 		CreatedAt:   now,
-		UpdatedAt:   now,
-		URL:         s.getObjectURL(id),
+		URL:         s.getObjectURL(key),
 	}
 
 	return info, nil
 }
 
 func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) {
-	// Get the object from S3
-	result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(id),
-	})
+	// Try to find the file with different extensions
+	exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav", 
+		".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
+
+	var result *s3.GetObjectOutput
+	var err error
+	var key string
+
+	for _, ext := range exts {
+		key = s.generateObjectKey(id, ext, time.Now())
+		result, err = s.client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(key),
+		})
+		if err == nil {
+			break
+		}
+	}
+
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to get file from S3: %w", err)
 	}
@@ -122,111 +197,117 @@ func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInf
 		Name:        result.Metadata["x-amz-meta-original-name"],
 		Size:        aws.ToInt64(result.ContentLength),
 		ContentType: aws.ToString(result.ContentType),
-		CreatedAt:   aws.ToTime(result.LastModified),
-		UpdatedAt:   aws.ToTime(result.LastModified),
-		URL:         s.getObjectURL(id),
+		CreatedAt:   time.Now(),
+		UpdatedAt:   time.Now(),
+		URL:         s.getObjectURL(key),
 	}
 
 	return result.Body, info, nil
 }
 
 func (s *S3Storage) Delete(ctx context.Context, id string) error {
-	_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(id),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to delete file from S3: %w", err)
+	// Try to find and delete the file with different extensions
+	exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav", 
+		".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
+
+	var lastErr error
+	for _, ext := range exts {
+		key := s.generateObjectKey(id, ext, time.Now())
+		_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(key),
+		})
+		if err == nil {
+			return nil
+		}
+		lastErr = err
 	}
 
-	return nil
+	return fmt.Errorf("failed to delete file from S3: %w", lastErr)
+}
+
+func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) {
+	// Try to find the file with different extensions
+	exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav", 
+		".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
+
+	for _, ext := range exts {
+		key := s.generateObjectKey(id, ext, time.Now())
+		_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(key),
+		})
+		if err == nil {
+			return true, nil
+		}
+	}
+
+	return false, nil
 }
 
 func (s *S3Storage) List(ctx context.Context, prefix string, limit int, offset int) ([]*FileInfo, error) {
 	var files []*FileInfo
 	var continuationToken *string
+	count := 0
+	skip := offset
 
-	// Skip objects for offset
-	for i := 0; i < offset/1000; i++ {
-		output, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+	for {
+		input := &s3.ListObjectsV2Input{
 			Bucket:            aws.String(s.bucket),
 			Prefix:            aws.String(prefix),
 			ContinuationToken: continuationToken,
-			MaxKeys:           aws.Int32(1000),
-		})
+			MaxKeys:          aws.Int32(100), // Fetch in batches of 100
+		}
+
+		result, err := s.client.ListObjectsV2(ctx, input)
 		if err != nil {
 			return nil, fmt.Errorf("failed to list files from S3: %w", err)
 		}
-		if !aws.ToBool(output.IsTruncated) {
-			return files, nil
-		}
-		continuationToken = output.NextContinuationToken
-	}
 
-	// Get the actual objects
-	output, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
-		Bucket:            aws.String(s.bucket),
-		Prefix:            aws.String(prefix),
-		ContinuationToken: continuationToken,
-		MaxKeys:           aws.Int32(int32(limit)),
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to list files from S3: %w", err)
-	}
-
-	for _, obj := range output.Contents {
-		// Get the object metadata
-		head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
-			Bucket: aws.String(s.bucket),
-			Key:    obj.Key,
-		})
-
-		var contentType string
-		var originalName string
-
-		if err != nil {
-			var noSuchKey *types.NoSuchKey
-			if errors.As(err, &noSuchKey) {
-				// If the object doesn't exist (which shouldn't happen normally),
-				// we'll still include it in the list but with empty metadata
-				contentType = ""
-				originalName = aws.ToString(obj.Key)
-			} else {
+		// Process each object
+		for _, obj := range result.Contents {
+			if skip > 0 {
+				skip--
 				continue
 			}
-		} else {
-			contentType = aws.ToString(head.ContentType)
-			originalName = head.Metadata["x-amz-meta-original-name"]
-			if originalName == "" {
-				originalName = aws.ToString(obj.Key)
+
+			// Get object metadata
+			head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
+				Bucket: aws.String(s.bucket),
+				Key:    obj.Key,
+			})
+			if err != nil {
+				continue // Skip files we can't get metadata for
+			}
+
+			// Extract the ID from the key (remove extension if present)
+			id := aws.ToString(obj.Key)
+			if ext := filepath.Ext(id); ext != "" {
+				id = id[:len(id)-len(ext)]
+			}
+
+			info := &FileInfo{
+				ID:          id,
+				Name:        head.Metadata["x-amz-meta-original-name"],
+				Size:        aws.ToInt64(obj.Size),
+				ContentType: aws.ToString(head.ContentType),
+				CreatedAt:   aws.ToTime(obj.LastModified),
+				UpdatedAt:   aws.ToTime(obj.LastModified),
+				URL:         s.getObjectURL(aws.ToString(obj.Key)),
+			}
+			files = append(files, info)
+			count++
+
+			if count >= limit {
+				return files, nil
 			}
 		}
 
-		files = append(files, &FileInfo{
-			ID:          aws.ToString(obj.Key),
-			Name:        originalName,
-			Size:        aws.ToInt64(obj.Size),
-			ContentType: contentType,
-			CreatedAt:   aws.ToTime(obj.LastModified),
-			UpdatedAt:   aws.ToTime(obj.LastModified),
-			URL:         s.getObjectURL(aws.ToString(obj.Key)),
-		})
+		if !aws.ToBool(result.IsTruncated) {
+			break
+		}
+		continuationToken = result.NextContinuationToken
 	}
 
 	return files, nil
 }
-
-func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) {
-	_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(id),
-	})
-	if err != nil {
-		var nsk *types.NoSuchKey
-		if ok := errors.As(err, &nsk); ok {
-			return false, nil
-		}
-		return false, fmt.Errorf("failed to check file existence in S3: %w", err)
-	}
-	return true, nil
-}