diff --git a/api/resource.go b/api/resource.go index d730fbb1e..91b0183c7 100644 --- a/api/resource.go +++ b/api/resource.go @@ -11,6 +11,7 @@ type Resource struct { // Domain specific fields Filename string `json:"filename"` Blob []byte `json:"-"` + InternalPath string `json:"internalPath"` ExternalLink string `json:"externalLink"` Type string `json:"type"` Size int64 `json:"size"` @@ -27,6 +28,7 @@ type ResourceCreate struct { // Domain specific fields Filename string `json:"filename"` Blob []byte `json:"-"` + InternalPath string `json:"internalPath"` ExternalLink string `json:"externalLink"` Type string `json:"type"` Size int64 `json:"-"` diff --git a/api/system.go b/api/system.go index 18c397ce7..4d3669bc6 100644 --- a/api/system.go +++ b/api/system.go @@ -18,5 +18,8 @@ type SystemStatus struct { AdditionalScript string `json:"additionalScript"` // Customized server profile, including server name and external url. CustomizedProfile CustomizedProfile `json:"customizedProfile"` - StorageServiceID int `json:"storageServiceId"` + // Storage service ID. + StorageServiceID int `json:"storageServiceId"` + // Local storage path + LocalStoragePath string `json:"localStoragePath"` } diff --git a/api/system_setting.go b/api/system_setting.go index d2e039e1f..269e851c8 100644 --- a/api/system_setting.go +++ b/api/system_setting.go @@ -27,6 +27,8 @@ const ( SystemSettingCustomizedProfileName SystemSettingName = "customizedProfile" // SystemSettingStorageServiceIDName is the key type of storage service ID. SystemSettingStorageServiceIDName SystemSettingName = "storageServiceId" + // SystemSettingLocalStoragePathName is the key type of local storage path. + SystemSettingLocalStoragePathName SystemSettingName = "localStoragePath" // SystemSettingOpenAIConfigName is the key type of OpenAI config. SystemSettingOpenAIConfigName SystemSettingName = "openAIConfig" ) @@ -70,6 +72,8 @@ func (key SystemSettingName) String() string { return "customizedProfile" case SystemSettingStorageServiceIDName: return "storageServiceId" + case SystemSettingLocalStoragePathName: + return "localStoragePath" case SystemSettingOpenAIConfigName: return "openAIConfig" } @@ -142,6 +146,12 @@ func (upsert SystemSettingUpsert) Validate() error { return fmt.Errorf("failed to unmarshal system setting storage service id value") } return nil + } else if upsert.Name == SystemSettingLocalStoragePathName { + value := "" + err := json.Unmarshal([]byte(upsert.Value), &value) + if err != nil { + return fmt.Errorf("failed to unmarshal system setting local storage path value") + } } else if upsert.Name == SystemSettingOpenAIConfigName { value := OpenAIConfig{} err := json.Unmarshal([]byte(upsert.Value), &value) diff --git a/server/resource.go b/server/resource.go index f638bc940..c49e51f86 100644 --- a/server/resource.go +++ b/server/resource.go @@ -7,7 +7,9 @@ import ( "io" "net/http" "net/url" + "os" "path" + "path/filepath" "regexp" "strconv" "strings" @@ -105,13 +107,13 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { } defer src.Close() - systemSetting, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: api.SystemSettingStorageServiceIDName}) + systemSettingStorageServiceID, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: api.SystemSettingStorageServiceIDName}) if err != nil && common.ErrorCode(err) != common.NotFound { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) } storageServiceID := 0 - if systemSetting != nil { - err = json.Unmarshal([]byte(systemSetting.Value), &storageServiceID) + if systemSettingStorageServiceID != nil { + err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal storage service id").SetInternal(err) } @@ -119,6 +121,7 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { var resourceCreate *api.ResourceCreate if storageServiceID == 0 { + // Database storage. fileBytes, err := io.ReadAll(src) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "Failed to read file").SetInternal(err) @@ -130,6 +133,47 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { Size: size, Blob: fileBytes, } + } else if storageServiceID == -1 { + // Local storage. + systemSettingLocalStoragePath, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: api.SystemSettingLocalStoragePathName}) + if err != nil && common.ErrorCode(err) != common.NotFound { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err) + } + localStoragePath := "" + if systemSettingLocalStoragePath != nil { + err = json.Unmarshal([]byte(systemSettingLocalStoragePath.Value), &localStoragePath) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal storage service id").SetInternal(err) + } + } + filePath := localStoragePath + if !strings.Contains(filePath, "{filename}") { + filePath = path.Join(filePath, "{filename}") + } + filePath = path.Join(s.Profile.Data, replacePathTemplate(filePath, filename)) + dirPath := filepath.Dir(filePath) + err = os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create directory").SetInternal(err) + } + dst, err := os.Create(filePath) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create file").SetInternal(err) + } + defer dst.Close() + + _, err = io.Copy(dst, src) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "Failed to copy file").SetInternal(err) + } + + resourceCreate = &api.ResourceCreate{ + CreatorID: userID, + Filename: filename, + Type: filetype, + Size: size, + InternalPath: filePath, + } } else { storage, err := s.Store.FindStorage(ctx, &api.StorageFind{ID: &storageServiceID}) if err != nil { @@ -138,38 +182,11 @@ func (s *Server) registerResourceRoutes(g *echo.Group) { if storage.Type == api.StorageS3 { s3Config := storage.Config.S3Config - t := time.Now() var s3FileKey string - if s3Config.Path == "" { - s3FileKey = filename - } else { - s3FileKey = fileKeyPattern.ReplaceAllStringFunc(s3Config.Path, func(s string) string { - switch s { - case "{filename}": - return filename - case "{timestamp}": - return fmt.Sprintf("%d", t.Unix()) - case "{year}": - return fmt.Sprintf("%d", t.Year()) - case "{month}": - return fmt.Sprintf("%02d", t.Month()) - case "{day}": - return fmt.Sprintf("%02d", t.Day()) - case "{hour}": - return fmt.Sprintf("%02d", t.Hour()) - case "{minute}": - return fmt.Sprintf("%02d", t.Minute()) - case "{second}": - return fmt.Sprintf("%02d", t.Second()) - } - return s - }) - - if !strings.Contains(s3Config.Path, "{filename}") { - s3FileKey = path.Join(s3FileKey, filename) - } + if !strings.Contains(s3Config.Path, "{filename}") { + s3FileKey = path.Join(s3Config.Path, "{filename}") } - + s3FileKey = replacePathTemplate(s3FileKey, filename) s3client, err := s3.NewClient(ctx, &s3.Config{ AccessKey: s3Config.AccessKey, SecretKey: s3Config.SecretKey, @@ -387,16 +404,29 @@ func (s *Server) registerResourcePublicRoutes(g *echo.Group) { return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to find resource by ID: %v", resourceID)).SetInternal(err) } + blob := resource.Blob + if resource.InternalPath != "" { + src, err := os.Open(resource.InternalPath) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to open the local resource: %s", resource.InternalPath)).SetInternal(err) + } + defer src.Close() + blob, err = io.ReadAll(src) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to read the local resource: %s", resource.InternalPath)).SetInternal(err) + } + } + c.Response().Writer.Header().Set(echo.HeaderCacheControl, "max-age=31536000, immutable") c.Response().Writer.Header().Set(echo.HeaderContentSecurityPolicy, "default-src 'self'") resourceType := strings.ToLower(resource.Type) if strings.HasPrefix(resourceType, "text") { resourceType = echo.MIMETextPlainCharsetUTF8 } else if strings.HasPrefix(resourceType, "video") || strings.HasPrefix(resourceType, "audio") { - http.ServeContent(c.Response(), c.Request(), resource.Filename, time.Unix(resource.UpdatedTs, 0), bytes.NewReader(resource.Blob)) + http.ServeContent(c.Response(), c.Request(), resource.Filename, time.Unix(resource.UpdatedTs, 0), bytes.NewReader(blob)) return nil } - return c.Stream(http.StatusOK, resourceType, bytes.NewReader(resource.Blob)) + return c.Stream(http.StatusOK, resourceType, bytes.NewReader(blob)) }) } @@ -422,3 +452,29 @@ func (s *Server) createResourceCreateActivity(c echo.Context, resource *api.Reso } return err } + +func replacePathTemplate(path string, filename string) string { + t := time.Now() + path = fileKeyPattern.ReplaceAllStringFunc(path, func(s string) string { + switch s { + case "{filename}": + return filename + case "{timestamp}": + return fmt.Sprintf("%d", t.Unix()) + case "{year}": + return fmt.Sprintf("%d", t.Year()) + case "{month}": + return fmt.Sprintf("%02d", t.Month()) + case "{day}": + return fmt.Sprintf("%02d", t.Day()) + case "{hour}": + return fmt.Sprintf("%02d", t.Hour()) + case "{minute}": + return fmt.Sprintf("%02d", t.Minute()) + case "{second}": + return fmt.Sprintf("%02d", t.Second()) + } + return s + }) + return path +} diff --git a/server/system.go b/server/system.go index d1fc53abb..9fbd284ce 100644 --- a/server/system.go +++ b/server/system.go @@ -52,6 +52,7 @@ func (s *Server) registerSystemRoutes(g *echo.Group) { ExternalURL: "", }, StorageServiceID: 0, + LocalStoragePath: "", } systemSettingList, err := s.Store.FindSystemSettingList(ctx, &api.SystemSettingFind{}) @@ -86,6 +87,8 @@ func (s *Server) registerSystemRoutes(g *echo.Group) { systemStatus.CustomizedProfile = customizedProfile } else if systemSetting.Name == api.SystemSettingStorageServiceIDName { systemStatus.StorageServiceID = int(baseValue.(float64)) + } else if systemSetting.Name == api.SystemSettingLocalStoragePathName { + systemStatus.LocalStoragePath = baseValue.(string) } } diff --git a/store/db/migration/dev/LATEST__SCHEMA.sql b/store/db/migration/dev/LATEST__SCHEMA.sql index 64e1bacf5..60d0063b7 100644 --- a/store/db/migration/dev/LATEST__SCHEMA.sql +++ b/store/db/migration/dev/LATEST__SCHEMA.sql @@ -74,6 +74,7 @@ CREATE TABLE resource ( updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')), filename TEXT NOT NULL DEFAULT '', blob BLOB DEFAULT NULL, + internal_path TEXT NOT NULL DEFAULT '', external_link TEXT NOT NULL DEFAULT '', type TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, diff --git a/store/resource.go b/store/resource.go index 57f33a494..0d54a99c6 100644 --- a/store/resource.go +++ b/store/resource.go @@ -24,6 +24,7 @@ type resourceRaw struct { // Domain specific fields Filename string Blob []byte + InternalPath string ExternalLink string Type string Size int64 @@ -43,6 +44,7 @@ func (raw *resourceRaw) toResource() *api.Resource { // Domain specific fields Filename: raw.Filename, Blob: raw.Blob, + InternalPath: raw.InternalPath, ExternalLink: raw.ExternalLink, Type: raw.Type, Size: raw.Size, @@ -195,9 +197,9 @@ func (s *Store) createResourceImpl(ctx context.Context, tx *sql.Tx, create *api. values := []any{create.Filename, create.Blob, create.ExternalLink, create.Type, create.Size, create.CreatorID} placeholders := []string{"?", "?", "?", "?", "?", "?"} if s.profile.IsDev() { - fields = append(fields, "visibility") - values = append(values, create.Visibility) - placeholders = append(placeholders, "?") + fields = append(fields, "visibility", "internal_path") + values = append(values, create.Visibility, create.InternalPath) + placeholders = append(placeholders, "?", "?") } query := ` @@ -218,7 +220,7 @@ func (s *Store) createResourceImpl(ctx context.Context, tx *sql.Tx, create *api. &resourceRaw.CreatorID, } if s.profile.IsDev() { - dests = append(dests, &resourceRaw.Visibility) + dests = append(dests, &resourceRaw.Visibility, &resourceRaw.InternalPath) } dests = append(dests, []any{&resourceRaw.CreatedTs, &resourceRaw.UpdatedTs}...) if err := tx.QueryRowContext(ctx, query, values...).Scan(dests...); err != nil { @@ -247,7 +249,7 @@ func (s *Store) patchResourceImpl(ctx context.Context, tx *sql.Tx, patch *api.Re fields := []string{"id", "filename", "external_link", "type", "size", "creator_id", "created_ts", "updated_ts"} if s.profile.IsDev() { - fields = append(fields, "visibility") + fields = append(fields, "visibility", "internal_path") } query := ` @@ -267,7 +269,7 @@ func (s *Store) patchResourceImpl(ctx context.Context, tx *sql.Tx, patch *api.Re &resourceRaw.UpdatedTs, } if s.profile.IsDev() { - dests = append(dests, &resourceRaw.Visibility) + dests = append(dests, &resourceRaw.Visibility, &resourceRaw.InternalPath) } if err := tx.QueryRowContext(ctx, query, args...).Scan(dests...); err != nil { return nil, FormatError(err) @@ -297,7 +299,7 @@ func (s *Store) findResourceListImpl(ctx context.Context, tx *sql.Tx, find *api. fields = append(fields, "resource.blob") } if s.profile.IsDev() { - fields = append(fields, "resource.visibility") + fields = append(fields, "visibility", "internal_path") } query := fmt.Sprintf(` @@ -334,7 +336,7 @@ func (s *Store) findResourceListImpl(ctx context.Context, tx *sql.Tx, find *api. dests = append(dests, &resourceRaw.Blob) } if s.profile.IsDev() { - dests = append(dests, &resourceRaw.Visibility) + dests = append(dests, &resourceRaw.Visibility, &resourceRaw.InternalPath) } if err := rows.Scan(dests...); err != nil { return nil, FormatError(err) diff --git a/web/src/components/Settings/StorageSection.tsx b/web/src/components/Settings/StorageSection.tsx index 239891caa..17523548d 100644 --- a/web/src/components/Settings/StorageSection.tsx +++ b/web/src/components/Settings/StorageSection.tsx @@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next"; import { useGlobalStore } from "../../store/module"; import * as api from "../../helpers/api"; import showCreateStorageServiceDialog from "../CreateStorageServiceDialog"; +import showUpdateLocalStorageDialog from "../UpdateLocalStorageDialog"; import Dropdown from "../base/Dropdown"; import { showCommonDialog } from "../Dialog/CommonDialog"; @@ -27,10 +28,6 @@ const StorageSection = () => { }; const handleActiveStorageServiceChanged = async (storageId: StorageId) => { - if (storageList.length === 0) { - return; - } - await api.upsertSystemSetting({ name: "storageServiceId", value: JSON.stringify(storageId), @@ -70,6 +67,7 @@ const StorageSection = () => { }} > + {storageList.map((storage) => (