Sitemap: real lastmod dates, no duplicates, no tag pages

blog/blog.go

@@ -28,7 +28,6 @@ import (
 	"github.com/joho/godotenv"
 	"gorm.io/gorm"
 
-	"github.com/ikeikeikeike/go-sitemap-generator/v2/stm"
 )
 
 // PageFilter is a function that decides whether a page should be shown.
@@ -1229,50 +1228,85 @@ func (b *Blog) SiteURL(c *gin.Context) string {
 	return scheme + "://" + c.Request.Host
 }
 
-// Sitemap serves /sitemap.xml: the home page, every enabled page (nav or
-// not), post type listings, published posts (with lastmod), tags in use,
-// and whatever URLs enabled plugins list for the pages under their slugs.
-// Every loc is under SiteURL.
+type sitemapURLSet struct {
+	XMLName xml.Name       `xml:"urlset"`
+	Xmlns   string         `xml:"xmlns,attr"`
+	URLs    []sitemapEntry `xml:"url"`
+}
+
+type sitemapEntry struct {
+	Loc     string `xml:"loc"`
+	LastMod string `xml:"lastmod,omitempty"`
+}
+
+// Sitemap serves /sitemap.xml: the home page, post type listings, every
+// enabled page (nav or not), published posts, and whatever URLs enabled
+// plugins list for the pages under their slugs. Tag pages are left out:
+// they only re-list posts that are already here. Every loc is under
+// SiteURL and appears once. lastmod is the date the content last changed
+// — for the home page or a listing, that of its newest post — and is left
+// out when that is not known, never filled in with the time of the request.
 func (b *Blog) Sitemap(c *gin.Context) {
 	host := b.SiteURL(c)
-	sm := stm.NewSitemap(1)
-	sm.SetDefaultHost(host)
-	sm.Create()
+	set := sitemapURLSet{Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9"}
+	seen := map[string]bool{}
+	add := func(path string, lastMod time.Time) {
+		if seen[path] {
+			return
+		}
+		seen[path] = true
+		entry := sitemapEntry{Loc: host + path}
+		if !lastMod.IsZero() {
+			entry.LastMod = lastMod.UTC().Format(time.RFC3339)
+		}
+		set.URLs = append(set.URLs, entry)
+	}
+	newer := func(a, b time.Time) time.Time {
+		if b.After(a) {
+			return b
+		}
+		return a
+	}
+
+	posts := b.GetPosts(false)
+	var newest time.Time
+	byType := map[uint]time.Time{}
+	for _, post := range posts {
+		newest = newer(newest, post.UpdatedAt)
+		byType[post.PostTypeID] = newer(byType[post.PostTypeID], post.UpdatedAt)
+	}
 
-	sm.Add(stm.URL{{"loc", "/"}, {"changefreq", "weekly"}, {"priority", 1.0}})
+	add("/", newest)
+	// Before pages: a listing usually has a page at the same path, and the
+	// listing's date (its newest post) is the one that says when it changed.
+	for _, pt := range b.GetPostTypes() {
+		add(pt.Permalink(), byType[pt.ID])
+	}
 
 	var pages []Page
 	(*b.db).Where("enabled = ?", true).Order("nav_order asc").Find(&pages)
 	for _, page := range pages {
 		if b.PageFilter != nil && !b.PageFilter(page) {
 			continue
 		}
-		sm.Add(stm.URL{{"loc", page.PagePermalink()}, {"changefreq", "weekly"}, {"priority", 0.7}})
+		add(page.PagePermalink(), page.UpdatedAt)
 	}
 
-	for _, pt := range b.GetPostTypes() {
-		sm.Add(stm.URL{{"loc", pt.Permalink()}, {"changefreq", "weekly"}, {"priority", 0.7}})
-	}
-
-	for _, post := range b.GetPosts(false) {
-		sm.Add(stm.URL{{"loc", post.Permalink()}, {"lastmod", post.UpdatedAt}, {"changefreq", "yearly"}, {"priority", 0.55}})
-	}
-	for _, tag := range b.getTags() {
-		if len(tag.Posts) > 0 {
-			sm.Add(stm.URL{{"loc", tag.Permalink()}, {"changefreq", "weekly"}, {"priority", 0.55}})
-		}
+	for _, post := range posts {
+		add(post.Permalink(), post.UpdatedAt)
 	}
 	if r := pluginRegistryFrom(c); r != nil {
 		for _, u := range r.SitemapURLs(c) {
-			entry := stm.URL{{"loc", u.Loc}, {"changefreq", "weekly"}, {"priority", 0.6}}
-			if !u.LastMod.IsZero() {
-				entry = append(entry, []interface{}{"lastmod", u.LastMod})
-			}
-			sm.Add(entry)
+			add(u.Loc, u.LastMod)
 		}
 	}
 
-	c.Data(http.StatusOK, "application/xml; charset=utf-8", sm.XMLContent())
+	out, err := xml.Marshal(set)
+	if err != nil {
+		c.String(http.StatusInternalServerError, "sitemap: %v", err)
+		return
+	}
+	c.Data(http.StatusOK, "application/xml; charset=utf-8", append([]byte(xml.Header), out...))
 }
 
 // feedItems is how many posts the RSS feed carries.

blog/blog_test.go

@@ -1550,9 +1550,10 @@ func (p *sitemapPlugin) Sitemap(_ *plugin.HookContext) []plugin.SitemapURL {
 }
 
 // TestSitemap: every URL is under the site_url setting (not a host baked
-// into goblog); posts carry lastmod; every enabled page is listed, nav or
-// not; pages that do not exist are not invented; enabled plugins that
-// implement plugin.Sitemapper add their URLs.
+// into goblog) and listed once; posts carry lastmod, and the home page and
+// a listing carry their newest post's; tag pages are left out; every
+// enabled page is listed, nav or not; pages that do not exist are not
+// invented; enabled plugins that implement plugin.Sitemapper add their URLs.
 func TestSitemap(t *testing.T) {
 	db, _ := gorm.Open(sqlite.Open(":memory:"))
 	db.AutoMigrate(&auth.BlogUser{}, &blog.PostType{}, &blog.Post{}, &blog.Tag{}, &blog.Comment{}, &blog.Page{}, &blog.Setting{}, &plugin.PluginSetting{})
@@ -1562,6 +1563,7 @@ func TestSitemap(t *testing.T) {
 	updated := time.Date(2026, 8, 15, 12, 0, 0, 0, time.UTC)
 	db.Create(&blog.Post{Title: "Hello", Slug: "hello", Content: "hi", PostTypeID: pt.ID, CreatedAt: updated, UpdatedAt: updated, Tags: []blog.Tag{{Name: "go"}}})
 	db.Create(&blog.Post{Title: "Draft", Slug: "draft", Content: "hi", PostTypeID: pt.ID, Draft: true})
+	db.Create(&blog.Page{Title: "Writing", Slug: "posts", PageType: blog.PageTypeAbout, ShowInNav: true, Enabled: true})
 	db.Create(&blog.Page{Title: "About", Slug: "about", PageType: blog.PageTypeAbout, ShowInNav: true, Enabled: true})
 	db.Create(&blog.Page{Title: "Hidden", Slug: "hidden", PageType: blog.PageTypeAbout, ShowInNav: false, Enabled: true})
 	db.Create(&blog.Page{Title: "Off", Slug: "off", PageType: blog.PageTypeAbout, ShowInNav: true, Enabled: false})
@@ -1584,13 +1586,13 @@ func TestSitemap(t *testing.T) {
 		t.Fatalf("code=%d type=%q", w.Code, w.Header().Get("Content-Type"))
 	}
 	for _, want := range []string{
-		"<loc>https://www.example.test/</loc>",
+		"<loc>https://www.example.test/</loc><lastmod>2026-08-15T12:00:00Z</lastmod>",
+		"<loc>https://www.example.test/posts</loc><lastmod>2026-08-15T12:00:00Z</lastmod>",
 		"<loc>https://www.example.test/about</loc>",
 		"<loc>https://www.example.test/hidden</loc>",
 		"<loc>https://www.example.test/posts</loc>",
 		"<loc>https://www.example.test/posts/2026/08/15/hello</loc>",
 		"<lastmod>2026-08-15",
-		"<loc>https://www.example.test/tag/go</loc>",
 		"<loc>https://www.example.test/dir</loc>",
 		"<loc>https://www.example.test/dir/hello</loc>",
 		"<lastmod>2026-09-01",
@@ -1599,11 +1601,14 @@ func TestSitemap(t *testing.T) {
 			t.Errorf("sitemap missing %q in:\n%s", want, body)
 		}
 	}
-	for _, gone := range []string{"jasonernst.com", "/off<", "/draft<", "/archives<", "/tags<"} {
+	for _, gone := range []string{"jasonernst.com", "/off<", "/draft<", "/archives<", "/tags<", "/tag/"} {
 		if strings.Contains(body, gone) {
 			t.Errorf("sitemap must not contain %q:\n%s", gone, body)
 		}
 	}
+	if n := strings.Count(body, "<loc>https://www.example.test/posts</loc>"); n != 1 {
+		t.Errorf("/posts listed %d times, want 1:\n%s", n, body)
+	}
 
 	// Without site_url the request's own host is used.
 	db.Where("key = ?", "site_url").Delete(&blog.Setting{})

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/gin-contrib/static v1.1.8
 	github.com/gin-gonic/gin v1.12.0
 	github.com/google/uuid v1.6.0
-	github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2
 	github.com/joho/godotenv v1.5.1
 	github.com/stretchr/testify v1.12.1
 	gorm.io/driver/mysql v1.6.0
@@ -30,13 +29,11 @@ require (
 require (
 	filippo.io/edwards25519 v1.1.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/beevik/etree v1.1.0 // indirect
 	github.com/bytedance/gopkg v0.1.4 // indirect
 	github.com/bytedance/sonic v1.15.2 // indirect
 	github.com/bytedance/sonic/loader v0.5.1 // indirect
 	github.com/cloudwego/base64x v0.1.7 // indirect
 	github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect
-	github.com/fatih/structs v1.1.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
 	github.com/gin-contrib/sse v1.1.1 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect

go.sum

@@ -2,16 +2,12 @@ filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw=
 filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
-github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
-github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM=
 github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4=
 github.com/bytedance/sonic v1.15.2 h1:90H+rcF/FwLXwfB1cudOLq/je83n683Utf4Cbp0xHCo=
 github.com/bytedance/sonic v1.15.2/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA=
 github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI=
 github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
-github.com/clbanning/mxj v1.8.3 h1:2r/KCJi52w2MRz+K+UMa/1d7DdCjnLqYJfnbr7dYNWI=
-github.com/clbanning/mxj v1.8.3/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
 github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI=
 github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -21,8 +17,6 @@ github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a h1:UwSIFv5g
 github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q=
 github.com/extism/go-sdk v1.7.1 h1:lWJos6uY+tRFdlIHR+SJjwFDApY7OypS/2nMhiVQ9Sw=
 github.com/extism/go-sdk v1.7.1/go.mod h1:IT+Xdg5AZM9hVtpFUA+uZCJMge/hbvshl8bwzLtFyKA=
-github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
-github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
 github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/gin-contrib/sessions v1.1.2 h1:utWRqOHT/kbqxxqF/mtutV9wfGLi1+uu5YuvWUhj/Ow=
@@ -66,8 +60,6 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq
 github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik=
 github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca h1:T54Ema1DU8ngI+aef9ZhAhNGQhcRTrWxVeG07F+c/Rw=
 github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
-github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2 h1:wIdDEle9HEy7vBPjC6oKz6ejs3Ut+jmsYvuOoAW2pSM=
-github.com/ikeikeikeike/go-sitemap-generator/v2 v2.0.2/go.mod h1:WtaVKD9TeruTED9ydiaOJU08qGoEPP/LyzTKiD3jEsw=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=