From fee7fcd6608b9d07da1f146c33c9bb8f898f708f Mon Sep 17 00:00:00 2001 From: boojack Date: Fri, 10 Apr 2026 22:23:00 +0800 Subject: [PATCH] fix(frontend): restore sitemap and robots routes --- server/router/frontend/frontend.go | 77 +++++++++++++++++++- server/router/frontend/frontend_test.go | 93 +++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 server/router/frontend/frontend_test.go diff --git a/server/router/frontend/frontend.go b/server/router/frontend/frontend.go index dedee6390..6034484cd 100644 --- a/server/router/frontend/frontend.go +++ b/server/router/frontend/frontend.go @@ -3,10 +3,14 @@ package frontend import ( "context" "embed" + "encoding/xml" "io/fs" + "net/http" + "strings" "github.com/labstack/echo/v5" "github.com/labstack/echo/v5/middleware" + "github.com/pkg/errors" "github.com/usememos/memos/internal/profile" "github.com/usememos/memos/internal/util" @@ -28,10 +32,10 @@ func NewFrontendService(profile *profile.Profile, store *store.Store) *FrontendS } } -func (*FrontendService) Serve(_ context.Context, e *echo.Echo) { +func (s *FrontendService) Serve(_ context.Context, e *echo.Echo) { skipper := func(c *echo.Context) bool { // Skip API routes. - if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1") { + if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1", "/robots.txt", "/sitemap.xml") { return true } // For index.html and root path, set no-cache headers to prevent browser caching @@ -57,6 +61,8 @@ func (*FrontendService) Serve(_ context.Context, e *echo.Echo) { HTML5: true, // Enable fallback to index.html Skipper: skipper, })) + + s.registerRoutes(e) } func getFileSystem(path string) fs.FS { @@ -66,3 +72,70 @@ func getFileSystem(path string) fs.FS { } return sub } + +func (s *FrontendService) registerRoutes(e *echo.Echo) { + e.GET("/robots.txt", s.getRobotsTXT) + e.GET("/sitemap.xml", s.getSitemapXML) +} + +func (s *FrontendService) getRobotsTXT(c *echo.Context) error { + instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL) + if err != nil { + return err + } + + robotsTXT := strings.Join([]string{ + "User-agent: *", + "Allow: /", + "Host: " + instanceURL, + "Sitemap: " + instanceURL + "/sitemap.xml", + }, "\n") + return c.String(http.StatusOK, robotsTXT) +} + +func (s *FrontendService) getSitemapXML(c *echo.Context) error { + instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL) + if err != nil { + return err + } + + memos, err := s.Store.ListMemos(c.Request().Context(), &store.FindMemo{ + VisibilityList: []store.Visibility{store.Public}, + }) + if err != nil { + return errors.Wrap(err, "failed to list public memos for sitemap") + } + + urls := make([]sitemapURL, 0, len(memos)) + for _, memo := range memos { + urls = append(urls, sitemapURL{ + Loc: instanceURL + "/m/" + memo.UID, + }) + } + + return c.XML(http.StatusOK, sitemapURLSet{ + XMLNS: sitemapXMLNamespace, + URLs: urls, + }) +} + +func normalizeInstanceURL(instanceURL string) (string, error) { + instanceURL = strings.TrimRight(instanceURL, "/") + if instanceURL == "" { + return "", echo.NewHTTPError(http.StatusNotFound, "instance URL is not configured") + } + return instanceURL, nil +} + +type sitemapURLSet struct { + XMLName xml.Name `xml:"urlset"` + XMLNS string `xml:"xmlns,attr"` + URLs []sitemapURL `xml:"url"` +} + +type sitemapURL struct { + Loc string `xml:"loc"` +} + +//nolint:revive +const sitemapXMLNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9" diff --git a/server/router/frontend/frontend_test.go b/server/router/frontend/frontend_test.go new file mode 100644 index 000000000..912c48136 --- /dev/null +++ b/server/router/frontend/frontend_test.go @@ -0,0 +1,93 @@ +package frontend + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/internal/profile" + "github.com/usememos/memos/store" + teststore "github.com/usememos/memos/store/test" +) + +func TestFrontendService_RobotsTXT(t *testing.T) { + ctx := context.Background() + testStore := teststore.NewTestingStore(ctx, t) + profile := &profile.Profile{ + InstanceURL: "https://demo.usememos.com/", + } + + e := echo.New() + NewFrontendService(profile, testStore).Serve(ctx, e) + + req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "text/plain; charset=UTF-8", rec.Header().Get("Content-Type")) + require.Equal(t, "User-agent: *\nAllow: /\nHost: https://demo.usememos.com\nSitemap: https://demo.usememos.com/sitemap.xml", rec.Body.String()) +} + +func TestFrontendService_SitemapXML(t *testing.T) { + ctx := context.Background() + testStore := teststore.NewTestingStore(ctx, t) + profile := &profile.Profile{ + InstanceURL: "https://demo.usememos.com", + } + + user, err := testStore.CreateUser(ctx, &store.User{ + Username: "sitemap-owner", + Role: store.RoleUser, + Email: "sitemap-owner@example.com", + }) + require.NoError(t, err) + + _, err = testStore.CreateMemo(ctx, &store.Memo{ + UID: "publicmemo", + CreatorID: user.ID, + Content: "public memo", + Visibility: store.Public, + }) + require.NoError(t, err) + + _, err = testStore.CreateMemo(ctx, &store.Memo{ + UID: "privatememo", + CreatorID: user.ID, + Content: "private memo", + Visibility: store.Private, + }) + require.NoError(t, err) + + e := echo.New() + NewFrontendService(profile, testStore).Serve(ctx, e) + + req := httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + require.Equal(t, http.StatusOK, rec.Code) + require.Contains(t, rec.Header().Get("Content-Type"), "application/xml") + require.Contains(t, rec.Body.String(), `https://demo.usememos.com/m/publicmemo`) + require.NotContains(t, rec.Body.String(), "privatememo") +} + +func TestFrontendService_SitemapRoutesRequireInstanceURL(t *testing.T) { + ctx := context.Background() + testStore := teststore.NewTestingStore(ctx, t) + + e := echo.New() + NewFrontendService(&profile.Profile{}, testStore).Serve(ctx, e) + + for _, path := range []string{"/robots.txt", "/sitemap.xml"} { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + require.Equal(t, http.StatusNotFound, rec.Code) + } +}