mirror of https://github.com/usememos/memos
refactor: update storage setting
Steven
1 year ago
parent
f25c7d9b24
commit
320963098f
@ -1,110 +0,0 @@
|
|||||||
package resourcepresign
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/usememos/memos/plugin/storage/s3"
|
|
||||||
storepb "github.com/usememos/memos/proto/gen/store"
|
|
||||||
"github.com/usememos/memos/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RunPreSignLinks is a background runner that pre-signs external links stored in the database.
|
|
||||||
// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.
|
|
||||||
func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {
|
|
||||||
for {
|
|
||||||
if err := signExternalLinks(ctx, dataStore); err != nil {
|
|
||||||
slog.Error("failed to pre-sign links", err)
|
|
||||||
} else {
|
|
||||||
slog.Debug("pre-signed links")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-time.After(s3.LinkLifetime / 2):
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func signExternalLinks(ctx context.Context, dataStore *store.Store) error {
|
|
||||||
objectStore, err := findObjectStorage(ctx, dataStore)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "find object storage")
|
|
||||||
}
|
|
||||||
if objectStore == nil || !objectStore.Config.PreSign {
|
|
||||||
// object storage not set or not supported
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
resources, err := dataStore.ListResources(ctx, &store.FindResource{
|
|
||||||
GetBlob: false,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "list resources")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, resource := range resources {
|
|
||||||
if resource.ExternalLink == "" {
|
|
||||||
// not for object store
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if strings.Contains(resource.ExternalLink, "?") && time.Since(time.Unix(resource.UpdatedTs, 0)) < s3.LinkLifetime/2 {
|
|
||||||
// resource not signed (hack for migration)
|
|
||||||
// resource was recently updated - skipping
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
newLink, err := objectStore.PreSignLink(ctx, resource.ExternalLink)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("failed to pre-sign link", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().Unix()
|
|
||||||
if _, err := dataStore.UpdateResource(ctx, &store.UpdateResource{
|
|
||||||
ID: resource.ID,
|
|
||||||
UpdatedTs: &now,
|
|
||||||
ExternalLink: &newLink,
|
|
||||||
}); err != nil {
|
|
||||||
return errors.Wrapf(err, "update resource %d link to %q", resource.ID, newLink)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// findObjectStorage returns current default storage if it's S3-compatible or nil otherwise.
|
|
||||||
// Returns error only in case of internal problems (ie: database or configuration issues).
|
|
||||||
// May return nil client and nil error.
|
|
||||||
func findObjectStorage(ctx context.Context, dataStore *store.Store) (*s3.Client, error) {
|
|
||||||
workspaceStorageSetting, err := dataStore.GetWorkspaceStorageSetting(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Failed to find workspaceStorageSetting")
|
|
||||||
}
|
|
||||||
if workspaceStorageSetting.StorageType != storepb.WorkspaceStorageSetting_STORAGE_TYPE_EXTERNAL || workspaceStorageSetting.ActivedExternalStorageId == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
storage, err := dataStore.GetStorage(ctx, &store.FindStorage{ID: workspaceStorageSetting.ActivedExternalStorageId})
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Failed to find storage")
|
|
||||||
}
|
|
||||||
if storage == nil || storage.Type != storepb.Storage_S3 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
s3Config := storage.Config.GetS3Config()
|
|
||||||
return s3.NewClient(ctx, &s3.Config{
|
|
||||||
AccessKey: s3Config.AccessKey,
|
|
||||||
SecretKey: s3Config.SecretKey,
|
|
||||||
EndPoint: s3Config.EndPoint,
|
|
||||||
Region: s3Config.Region,
|
|
||||||
Bucket: s3Config.Bucket,
|
|
||||||
URLPrefix: s3Config.UrlPrefix,
|
|
||||||
URLSuffix: s3Config.UrlSuffix,
|
|
||||||
PreSign: s3Config.PreSign,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@ -1,257 +0,0 @@
|
|||||||
import { Button, IconButton, Input, Checkbox, Typography } from "@mui/joy";
|
|
||||||
import React, { useEffect, useState } from "react";
|
|
||||||
import { toast } from "react-hot-toast";
|
|
||||||
import { storageServiceClient } from "@/grpcweb";
|
|
||||||
import { S3Config, Storage, Storage_Type } from "@/types/proto/api/v1/storage_service";
|
|
||||||
import { useTranslate } from "@/utils/i18n";
|
|
||||||
import { generateDialog } from "./Dialog";
|
|
||||||
import Icon from "./Icon";
|
|
||||||
import LearnMore from "./LearnMore";
|
|
||||||
import RequiredBadge from "./RequiredBadge";
|
|
||||||
|
|
||||||
interface Props extends DialogProps {
|
|
||||||
storage?: Storage;
|
|
||||||
confirmCallback?: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
const CreateStorageServiceDialog: React.FC<Props> = (props: Props) => {
|
|
||||||
const t = useTranslate();
|
|
||||||
const { destroy, storage, confirmCallback } = props;
|
|
||||||
const [basicInfo, setBasicInfo] = useState({
|
|
||||||
title: "",
|
|
||||||
});
|
|
||||||
const [type] = useState<Storage_Type>(Storage_Type.S3);
|
|
||||||
const [s3Config, setS3Config] = useState<S3Config>({
|
|
||||||
endPoint: "",
|
|
||||||
region: "",
|
|
||||||
accessKey: "",
|
|
||||||
secretKey: "",
|
|
||||||
path: "",
|
|
||||||
bucket: "",
|
|
||||||
urlPrefix: "",
|
|
||||||
urlSuffix: "",
|
|
||||||
preSign: false,
|
|
||||||
});
|
|
||||||
const isCreating = storage === undefined;
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
if (storage) {
|
|
||||||
setBasicInfo({
|
|
||||||
title: storage.title,
|
|
||||||
});
|
|
||||||
if (storage.type === "S3") {
|
|
||||||
setS3Config(S3Config.fromPartial(storage.config?.s3Config || {}));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
const handleCloseBtnClick = () => {
|
|
||||||
destroy();
|
|
||||||
};
|
|
||||||
|
|
||||||
const allowConfirmAction = () => {
|
|
||||||
if (basicInfo.title === "") {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (type === "S3") {
|
|
||||||
if (
|
|
||||||
s3Config.endPoint === "" ||
|
|
||||||
s3Config.region === "" ||
|
|
||||||
s3Config.accessKey === "" ||
|
|
||||||
s3Config.secretKey === "" ||
|
|
||||||
s3Config.bucket === ""
|
|
||||||
) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleConfirmBtnClick = async () => {
|
|
||||||
try {
|
|
||||||
if (isCreating) {
|
|
||||||
await storageServiceClient.createStorage({
|
|
||||||
storage: Storage.fromPartial({
|
|
||||||
title: basicInfo.title,
|
|
||||||
type: type,
|
|
||||||
config: {
|
|
||||||
s3Config: s3Config,
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
await storageServiceClient.updateStorage({
|
|
||||||
storage: Storage.fromPartial({
|
|
||||||
id: storage?.id,
|
|
||||||
title: basicInfo.title,
|
|
||||||
type: type,
|
|
||||||
config: {
|
|
||||||
s3Config: s3Config,
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
updateMask: ["title", "config"],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
console.error(error);
|
|
||||||
toast.error(error.response.data.message);
|
|
||||||
}
|
|
||||||
if (confirmCallback) {
|
|
||||||
confirmCallback();
|
|
||||||
}
|
|
||||||
destroy();
|
|
||||||
};
|
|
||||||
|
|
||||||
const setPartialS3Config = (state: Partial<S3Config>) => {
|
|
||||||
setS3Config({
|
|
||||||
...s3Config,
|
|
||||||
...state,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<>
|
|
||||||
<div className="dialog-header-container">
|
|
||||||
<span>{t(isCreating ? "setting.storage-section.create-storage" : "setting.storage-section.update-storage")}</span>
|
|
||||||
<IconButton size="sm" onClick={handleCloseBtnClick}>
|
|
||||||
<Icon.X className="w-5 h-auto" />
|
|
||||||
</IconButton>
|
|
||||||
</div>
|
|
||||||
<div className="dialog-content-container min-w-[19rem]">
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("common.name")}
|
|
||||||
<RequiredBadge />
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("common.name")}
|
|
||||||
value={basicInfo.title}
|
|
||||||
onChange={(e) =>
|
|
||||||
setBasicInfo({
|
|
||||||
...basicInfo,
|
|
||||||
title: e.target.value,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.endpoint")}
|
|
||||||
<RequiredBadge />
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.s3-compatible-url")}
|
|
||||||
value={s3Config.endPoint}
|
|
||||||
onChange={(e) => setPartialS3Config({ endPoint: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.region")}
|
|
||||||
<RequiredBadge />
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.region-placeholder")}
|
|
||||||
value={s3Config.region}
|
|
||||||
onChange={(e) => setPartialS3Config({ region: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.accesskey")}
|
|
||||||
<RequiredBadge />
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.accesskey-placeholder")}
|
|
||||||
value={s3Config.accessKey}
|
|
||||||
onChange={(e) => setPartialS3Config({ accessKey: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.secretkey")}
|
|
||||||
<RequiredBadge />
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.secretkey-placeholder")}
|
|
||||||
value={s3Config.secretKey}
|
|
||||||
onChange={(e) => setPartialS3Config({ secretKey: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.bucket")}
|
|
||||||
<RequiredBadge />
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.bucket-placeholder")}
|
|
||||||
value={s3Config.bucket}
|
|
||||||
onChange={(e) => setPartialS3Config({ bucket: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<div className="flex flex-row items-center mb-1">
|
|
||||||
<Typography level="body-md">{t("setting.storage-section.path")}</Typography>
|
|
||||||
<LearnMore
|
|
||||||
className="ml-1"
|
|
||||||
title={t("setting.storage-section.path-description")}
|
|
||||||
url="https://usememos.com/docs/advanced-settings/local-storage"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.path-placeholder") + "/{year}/{month}/{filename}"}
|
|
||||||
value={s3Config.path}
|
|
||||||
onChange={(e) => setPartialS3Config({ path: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.url-prefix")}
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.url-prefix-placeholder")}
|
|
||||||
value={s3Config.urlPrefix}
|
|
||||||
onChange={(e) => setPartialS3Config({ urlPrefix: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Typography className="!mb-1" level="body-md">
|
|
||||||
{t("setting.storage-section.url-suffix")}
|
|
||||||
</Typography>
|
|
||||||
<Input
|
|
||||||
className="mb-2"
|
|
||||||
placeholder={t("setting.storage-section.url-suffix-placeholder")}
|
|
||||||
value={s3Config.urlSuffix}
|
|
||||||
onChange={(e) => setPartialS3Config({ urlSuffix: e.target.value })}
|
|
||||||
fullWidth
|
|
||||||
/>
|
|
||||||
<Checkbox
|
|
||||||
className="mb-2"
|
|
||||||
label={t("setting.storage-section.presign-placeholder")}
|
|
||||||
checked={s3Config.preSign}
|
|
||||||
onChange={(e) => setPartialS3Config({ preSign: e.target.checked })}
|
|
||||||
/>
|
|
||||||
<div className="mt-2 w-full flex flex-row justify-end items-center space-x-1">
|
|
||||||
<Button variant="plain" color="neutral" onClick={handleCloseBtnClick}>
|
|
||||||
{t("common.cancel")}
|
|
||||||
</Button>
|
|
||||||
<Button onClick={handleConfirmBtnClick} disabled={!allowConfirmAction()}>
|
|
||||||
{t(isCreating ? "common.create" : "common.update")}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</>
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
function showCreateStorageServiceDialog(storage?: Storage, confirmCallback?: () => void) {
|
|
||||||
generateDialog(
|
|
||||||
{
|
|
||||||
className: "create-storage-service-dialog",
|
|
||||||
dialogName: "create-storage-service-dialog",
|
|
||||||
},
|
|
||||||
CreateStorageServiceDialog,
|
|
||||||
{ storage, confirmCallback },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export default showCreateStorageServiceDialog;
|
|
||||||
Loading…
Reference in New Issue