CDN
3e6181e578
All checks were successful
Build Backend / Build Docker Image (push) Successful in 5m3s
313 lines
8.3 KiB
Go
313 lines
8.3 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
)
|
|
|
|
type S3Storage struct {
|
|
client s3Client
|
|
bucket string
|
|
customURL string
|
|
proxyS3 bool
|
|
}
|
|
|
|
// s3Client is the interface that wraps the basic S3 client operations we need
|
|
type s3Client interface {
|
|
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
|
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
|
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
|
|
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
|
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
|
}
|
|
|
|
func NewS3Storage(client s3Client, bucket string, customURL string, proxyS3 bool) *S3Storage {
|
|
return &S3Storage{
|
|
client: client,
|
|
bucket: bucket,
|
|
customURL: customURL,
|
|
proxyS3: proxyS3,
|
|
}
|
|
}
|
|
|
|
func (s *S3Storage) generateID() (string, error) {
|
|
bytes := make([]byte, 16)
|
|
if _, err := rand.Read(bytes); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(bytes), nil
|
|
}
|
|
|
|
func (s *S3Storage) generateObjectKey(id string, ext string, createTime time.Time) string {
|
|
// Create year/month structure
|
|
year := createTime.Format("2006")
|
|
month := createTime.Format("01")
|
|
filename := id
|
|
if ext != "" {
|
|
filename = id + ext
|
|
}
|
|
return fmt.Sprintf("%s/%s/%s", year, month, filename)
|
|
}
|
|
|
|
func (s *S3Storage) getObjectURL(key string) string {
|
|
if s.customURL != "" {
|
|
return fmt.Sprintf("%s/%s", strings.TrimRight(s.customURL, "/"), key)
|
|
}
|
|
if s.proxyS3 {
|
|
return fmt.Sprintf("/media/%s", key)
|
|
}
|
|
return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", s.bucket, key)
|
|
}
|
|
|
|
func (s *S3Storage) Save(ctx context.Context, name string, contentType string, reader io.Reader) (*FileInfo, error) {
|
|
// Generate a unique ID for the file
|
|
id, err := s.generateID()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to generate file ID: %w", err)
|
|
}
|
|
|
|
// Get file extension from original name or content type
|
|
ext := filepath.Ext(name)
|
|
if ext == "" {
|
|
// If no extension in name, try to get it from content type
|
|
switch contentType {
|
|
case "image/jpeg":
|
|
ext = ".jpg"
|
|
case "image/png":
|
|
ext = ".png"
|
|
case "image/gif":
|
|
ext = ".gif"
|
|
case "image/webp":
|
|
ext = ".webp"
|
|
case "image/svg+xml":
|
|
ext = ".svg"
|
|
case "video/mp4":
|
|
ext = ".mp4"
|
|
case "video/webm":
|
|
ext = ".webm"
|
|
case "audio/mpeg":
|
|
ext = ".mp3"
|
|
case "audio/ogg":
|
|
ext = ".ogg"
|
|
case "audio/wav":
|
|
ext = ".wav"
|
|
case "application/pdf":
|
|
ext = ".pdf"
|
|
case "application/msword":
|
|
ext = ".doc"
|
|
case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
|
|
ext = ".docx"
|
|
case "application/vnd.ms-excel":
|
|
ext = ".xls"
|
|
case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
|
|
ext = ".xlsx"
|
|
case "application/zip":
|
|
ext = ".zip"
|
|
case "application/x-rar-compressed":
|
|
ext = ".rar"
|
|
case "text/plain":
|
|
ext = ".txt"
|
|
case "text/csv":
|
|
ext = ".csv"
|
|
}
|
|
}
|
|
|
|
// Create the object key with year/month structure
|
|
now := time.Now()
|
|
key := s.generateObjectKey(id, ext, now)
|
|
|
|
// Check if the file exists
|
|
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err == nil {
|
|
return nil, fmt.Errorf("file already exists with ID: %s", id)
|
|
}
|
|
|
|
var noSuchKey *types.NoSuchKey
|
|
if !errors.As(err, &noSuchKey) {
|
|
return nil, fmt.Errorf("failed to check if file exists: %w", err)
|
|
}
|
|
|
|
// Upload the file
|
|
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
Body: reader,
|
|
ContentType: aws.String(contentType),
|
|
Metadata: map[string]string{
|
|
"x-amz-meta-original-name": name,
|
|
"x-amz-meta-created-at": now.Format(time.RFC3339),
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to upload file: %w", err)
|
|
}
|
|
|
|
info := &FileInfo{
|
|
ID: id,
|
|
Name: name,
|
|
Size: 0, // Size is not available until after upload
|
|
ContentType: contentType,
|
|
CreatedAt: now,
|
|
URL: s.getObjectURL(key),
|
|
}
|
|
|
|
return info, nil
|
|
}
|
|
|
|
func (s *S3Storage) Get(ctx context.Context, id string) (io.ReadCloser, *FileInfo, error) {
|
|
// Try to find the file with different extensions
|
|
exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav",
|
|
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
|
|
|
|
var result *s3.GetObjectOutput
|
|
var err error
|
|
var key string
|
|
|
|
for _, ext := range exts {
|
|
key = s.generateObjectKey(id, ext, time.Now())
|
|
result, err = s.client.GetObject(ctx, &s3.GetObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err == nil {
|
|
break
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to get file from S3: %w", err)
|
|
}
|
|
|
|
info := &FileInfo{
|
|
ID: id,
|
|
Name: result.Metadata["x-amz-meta-original-name"],
|
|
Size: aws.ToInt64(result.ContentLength),
|
|
ContentType: aws.ToString(result.ContentType),
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
URL: s.getObjectURL(key),
|
|
}
|
|
|
|
return result.Body, info, nil
|
|
}
|
|
|
|
func (s *S3Storage) Delete(ctx context.Context, id string) error {
|
|
// Try to find and delete the file with different extensions
|
|
exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav",
|
|
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
|
|
|
|
var lastErr error
|
|
for _, ext := range exts {
|
|
key := s.generateObjectKey(id, ext, time.Now())
|
|
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
lastErr = err
|
|
}
|
|
|
|
return fmt.Errorf("failed to delete file from S3: %w", lastErr)
|
|
}
|
|
|
|
func (s *S3Storage) Exists(ctx context.Context, id string) (bool, error) {
|
|
// Try to find the file with different extensions
|
|
exts := []string{"", ".jpg", ".png", ".gif", ".webp", ".svg", ".mp4", ".webm", ".mp3", ".ogg", ".wav",
|
|
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar", ".txt", ".csv"}
|
|
|
|
for _, ext := range exts {
|
|
key := s.generateObjectKey(id, ext, time.Now())
|
|
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err == nil {
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
func (s *S3Storage) List(ctx context.Context, prefix string, limit int, offset int) ([]*FileInfo, error) {
|
|
var files []*FileInfo
|
|
var continuationToken *string
|
|
count := 0
|
|
skip := offset
|
|
|
|
for {
|
|
input := &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(s.bucket),
|
|
Prefix: aws.String(prefix),
|
|
ContinuationToken: continuationToken,
|
|
MaxKeys: aws.Int32(100), // Fetch in batches of 100
|
|
}
|
|
|
|
result, err := s.client.ListObjectsV2(ctx, input)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list files from S3: %w", err)
|
|
}
|
|
|
|
// Process each object
|
|
for _, obj := range result.Contents {
|
|
if skip > 0 {
|
|
skip--
|
|
continue
|
|
}
|
|
|
|
// Get object metadata
|
|
head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(s.bucket),
|
|
Key: obj.Key,
|
|
})
|
|
if err != nil {
|
|
continue // Skip files we can't get metadata for
|
|
}
|
|
|
|
// Extract the ID from the key (remove extension if present)
|
|
id := aws.ToString(obj.Key)
|
|
if ext := filepath.Ext(id); ext != "" {
|
|
id = id[:len(id)-len(ext)]
|
|
}
|
|
|
|
info := &FileInfo{
|
|
ID: id,
|
|
Name: head.Metadata["x-amz-meta-original-name"],
|
|
Size: aws.ToInt64(obj.Size),
|
|
ContentType: aws.ToString(head.ContentType),
|
|
CreatedAt: aws.ToTime(obj.LastModified),
|
|
UpdatedAt: aws.ToTime(obj.LastModified),
|
|
URL: s.getObjectURL(aws.ToString(obj.Key)),
|
|
}
|
|
files = append(files, info)
|
|
count++
|
|
|
|
if count >= limit {
|
|
return files, nil
|
|
}
|
|
}
|
|
|
|
if !aws.ToBool(result.IsTruncated) {
|
|
break
|
|
}
|
|
continuationToken = result.NextContinuationToken
|
|
}
|
|
|
|
return files, nil
|
|
}
|