From 45fec80099be2d65a04bdaec4af6137eec0b1015 Mon Sep 17 00:00:00 2001 From: Jason Ernst Date: Fri, 2 Oct 2026 23:48:07 -0700 Subject: [PATCH] Sitemap: real lastmod dates, no duplicates, no tag pages The sitemap library stamped the time of the request as lastmod on every entry that had no date, which was everything but posts. The handler now builds the XML with encoding/xml, as the RSS feed does: pages carry their own update time, the home page and post listings carry their newest post's, and an entry with no known date has no lastmod. Each URL is listed once (a listing and its page shared a path), tag pages are left out since they only re-list posts already present, and changefreq/priority are dropped as search engines ignore them. Removes go-sitemap-generator and its two dependencies. Co-Authored-By: Claude Fable 5.1 --- blog/blog.go | 88 ++++++++++++++++++++++++++++++++--------------- blog/blog_test.go | 17 +++++---- go.mod | 3 -- go.sum | 8 ----- 4 files changed, 72 insertions(+), 44 deletions(-) diff --git a/blog/blog.go b/blog/blog.go index 09b11ac..e6cd21f 100644 --- a/blog/blog.go +++ b/blog/blog.go @@ -28,7 +28,6 @@ import ( "github.com/joho/godotenv" "gorm.io/gorm" - "github.com/ikeikeikeike/go-sitemap-generator/v2/stm" ) // PageFilter is a function that decides whether a page should be shown. @@ -1229,17 +1228,60 @@ func (b *Blog) SiteURL(c *gin.Context) string { return scheme + "://" + c.Request.Host } -// Sitemap serves /sitemap.xml: the home page, every enabled page (nav or -// not), post type listings, published posts (with lastmod), tags in use, -// and whatever URLs enabled plugins list for the pages under their slugs. -// Every loc is under SiteURL. +type sitemapURLSet struct { + XMLName xml.Name `xml:"urlset"` + Xmlns string `xml:"xmlns,attr"` + URLs []sitemapEntry `xml:"url"` +} + +type sitemapEntry struct { + Loc string `xml:"loc"` + LastMod string `xml:"lastmod,omitempty"` +} + +// Sitemap serves /sitemap.xml: the home page, post type listings, every +// enabled page (nav or not), published posts, and whatever URLs enabled +// plugins list for the pages under their slugs. Tag pages are left out: +// they only re-list posts that are already here. Every loc is under +// SiteURL and appears once. lastmod is the date the content last changed +// — for the home page or a listing, that of its newest post — and is left +// out when that is not known, never filled in with the time of the request. func (b *Blog) Sitemap(c *gin.Context) { host := b.SiteURL(c) - sm := stm.NewSitemap(1) - sm.SetDefaultHost(host) - sm.Create() + set := sitemapURLSet{Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9"} + seen := map[string]bool{} + add := func(path string, lastMod time.Time) { + if seen[path] { + return + } + seen[path] = true + entry := sitemapEntry{Loc: host + path} + if !lastMod.IsZero() { + entry.LastMod = lastMod.UTC().Format(time.RFC3339) + } + set.URLs = append(set.URLs, entry) + } + newer := func(a, b time.Time) time.Time { + if b.After(a) { + return b + } + return a + } + + posts := b.GetPosts(false) + var newest time.Time + byType := map[uint]time.Time{} + for _, post := range posts { + newest = newer(newest, post.UpdatedAt) + byType[post.PostTypeID] = newer(byType[post.PostTypeID], post.UpdatedAt) + } - sm.Add(stm.URL{{"loc", "/"}, {"changefreq", "weekly"}, {"priority", 1.0}}) + add("/", newest) + // Before pages: a listing usually has a page at the same path, and the + // listing's date (its newest post) is the one that says when it changed. + for _, pt := range b.GetPostTypes() { + add(pt.Permalink(), byType[pt.ID]) + } var pages []Page (*b.db).Where("enabled = ?", true).Order("nav_order asc").Find(&pages) @@ -1247,32 +1289,24 @@ func (b *Blog) Sitemap(c *gin.Context) { if b.PageFilter != nil && !b.PageFilter(page) { continue } - sm.Add(stm.URL{{"loc", page.PagePermalink()}, {"changefreq", "weekly"}, {"priority", 0.7}}) + add(page.PagePermalink(), page.UpdatedAt) } - for _, pt := range b.GetPostTypes() { - sm.Add(stm.URL{{"loc", pt.Permalink()}, {"changefreq", "weekly"}, {"priority", 0.7}}) - } - - for _, post := range b.GetPosts(false) { - sm.Add(stm.URL{{"loc", post.Permalink()}, {"lastmod", post.UpdatedAt}, {"changefreq", "yearly"}, {"priority", 0.55}}) - } - for _, tag := range b.getTags() { - if len(tag.Posts) > 0 { - sm.Add(stm.URL{{"loc", tag.Permalink()}, {"changefreq", "weekly"}, {"priority", 0.55}}) - } + for _, post := range posts { + add(post.Permalink(), post.UpdatedAt) } if r := pluginRegistryFrom(c); r != nil { for _, u := range r.SitemapURLs(c) { - entry := stm.URL{{"loc", u.Loc}, {"changefreq", "weekly"}, {"priority", 0.6}} - if !u.LastMod.IsZero() { - entry = append(entry, []interface{}{"lastmod", u.LastMod}) - } - sm.Add(entry) + add(u.Loc, u.LastMod) } } - c.Data(http.StatusOK, "application/xml; charset=utf-8", sm.XMLContent()) + out, err := xml.Marshal(set) + if err != nil { + c.String(http.StatusInternalServerError, "sitemap: %v", err) + return + } + c.Data(http.StatusOK, "application/xml; charset=utf-8", append([]byte(xml.Header), out...)) } // feedItems is how many posts the RSS feed carries. diff --git a/blog/blog_test.go b/blog/blog_test.go index 3f3f638..615b5b4 100644 --- a/blog/blog_test.go +++ b/blog/blog_test.go @@ -1550,9 +1550,10 @@ func (p *sitemapPlugin) Sitemap(_ *plugin.HookContext) []plugin.SitemapURL { } // TestSitemap: every URL is under the site_url setting (not a host baked -// into goblog); posts carry lastmod; every enabled page is listed, nav or -// not; pages that do not exist are not invented; enabled plugins that -// implement plugin.Sitemapper add their URLs. +// into goblog) and listed once; posts carry lastmod, and the home page and +// a listing carry their newest post's; tag pages are left out; every +// enabled page is listed, nav or not; pages that do not exist are not +// invented; enabled plugins that implement plugin.Sitemapper add their URLs. func TestSitemap(t *testing.T) { db, _ := gorm.Open(sqlite.Open(":memory:")) db.AutoMigrate(&auth.BlogUser{}, &blog.PostType{}, &blog.Post{}, &blog.Tag{}, &blog.Comment{}, &blog.Page{}, &blog.Setting{}, &plugin.PluginSetting{}) @@ -1562,6 +1563,7 @@ func TestSitemap(t *testing.T) { updated := time.Date(2026, 8, 15, 12, 0, 0, 0, time.UTC) db.Create(&blog.Post{Title: "Hello", Slug: "hello", Content: "hi", PostTypeID: pt.ID, CreatedAt: updated, UpdatedAt: updated, Tags: []blog.Tag{{Name: "go"}}}) db.Create(&blog.Post{Title: "Draft", Slug: "draft", Content: "hi", PostTypeID: pt.ID, Draft: true}) + db.Create(&blog.Page{Title: "Writing", Slug: "posts", PageType: blog.PageTypeAbout, ShowInNav: true, Enabled: true}) db.Create(&blog.Page{Title: "About", Slug: "about", PageType: blog.PageTypeAbout, ShowInNav: true, Enabled: true}) db.Create(&blog.Page{Title: "Hidden", Slug: "hidden", PageType: blog.PageTypeAbout, ShowInNav: false, Enabled: true}) db.Create(&blog.Page{Title: "Off", Slug: "off", PageType: blog.PageTypeAbout, ShowInNav: true, Enabled: false}) @@ -1584,13 +1586,13 @@ func TestSitemap(t *testing.T) { t.Fatalf("code=%d type=%q", w.Code, w.Header().Get("Content-Type")) } for _, want := range []string{ - "https://www.example.test/", + "https://www.example.test/2026-08-15T12:00:00Z", + "https://www.example.test/posts2026-08-15T12:00:00Z", "https://www.example.test/about", "https://www.example.test/hidden", "https://www.example.test/posts", "https://www.example.test/posts/2026/08/15/hello", "2026-08-15", - "https://www.example.test/tag/go", "https://www.example.test/dir", "https://www.example.test/dir/hello", "2026-09-01", @@ -1599,11 +1601,14 @@ func TestSitemap(t *testing.T) { t.Errorf("sitemap missing %q in:\n%s", want, body) } } - for _, gone := range []string{"jasonernst.com", "/off<", "/draft<", "/archives<", "/tags<"} { + for _, gone := range []string{"jasonernst.com", "/off<", "/draft<", "/archives<", "/tags<", "/tag/"} { if strings.Contains(body, gone) { t.Errorf("sitemap must not contain %q:\n%s", gone, body) } } + if n := strings.Count(body, "https://www.example.test/posts"); n != 1 { + t.Errorf("/posts listed %d times, want 1:\n%s", n, body) + } // Without site_url the request's own host is used. db.Where("key = ?", "site_url").Delete(&blog.Setting{}) diff --git a/go.mod b/go.mod index ae5a849..41f590d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/gin-contrib/static v1.1.8 github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 - github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 github.com/joho/godotenv v1.5.1 github.com/stretchr/testify v1.12.1 gorm.io/driver/mysql v1.6.0 @@ -30,13 +29,11 @@ require ( require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/beevik/etree v1.1.0 // indirect github.com/bytedance/gopkg v0.1.4 // indirect github.com/bytedance/sonic v1.15.2 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/cloudwego/base64x v0.1.7 // indirect github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect - github.com/fatih/structs v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.1 // indirect github.com/go-playground/locales v0.14.1 // indirect diff --git a/go.sum b/go.sum index 3680375..6a04b47 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,12 @@ filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= -github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM= github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4= github.com/bytedance/sonic v1.15.2 h1:90H+rcF/FwLXwfB1cudOLq/je83n683Utf4Cbp0xHCo= github.com/bytedance/sonic v1.15.2/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= -github.com/clbanning/mxj v1.8.3 h1:2r/KCJi52w2MRz+K+UMa/1d7DdCjnLqYJfnbr7dYNWI= -github.com/clbanning/mxj v1.8.3/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI= github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -21,8 +17,6 @@ github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a h1:UwSIFv5g github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q= github.com/extism/go-sdk v1.7.1 h1:lWJos6uY+tRFdlIHR+SJjwFDApY7OypS/2nMhiVQ9Sw= github.com/extism/go-sdk v1.7.1/go.mod h1:IT+Xdg5AZM9hVtpFUA+uZCJMge/hbvshl8bwzLtFyKA= -github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gin-contrib/sessions v1.1.2 h1:utWRqOHT/kbqxxqF/mtutV9wfGLi1+uu5YuvWUhj/Ow= @@ -66,8 +60,6 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca h1:T54Ema1DU8ngI+aef9ZhAhNGQhcRTrWxVeG07F+c/Rw= github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM= -github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2/go.mod h1:WtaVKD9TeruTED9ydiaOJU08qGoEPP/LyzTKiD3jEsw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=