package main import ( "encoding/xml" "net/http" "net/url" "strconv" "strings" "time" ) type sitemapEntry struct { Loc string LastMod string } type sitemapXML struct { XMLName xml.Name `xml:"urlset"` XMLNS string `xml:"xmlns,attr"` URLs []sitemapURL `xml:"url"` } type sitemapURL struct { Loc string `xml:"loc"` LastMod string `xml:"lastmod,omitempty"` } func (a *app) sitemapHandler(w http.ResponseWriter, r *http.Request) { if a.db == nil { writeError(w, http.StatusServiceUnavailable, "database not configured") return } pages, err := a.list("pages", Doc{}, Sort{{Key: "updatedAt", Value: -1}}) if err != nil { writeError(w, http.StatusInternalServerError, "failed to build sitemap") return } entries := collectSitemapEntries(pages, requestOrigin(r)) body, err := renderSitemapXML(entries) if err != nil { writeError(w, http.StatusInternalServerError, "failed to render sitemap") return } w.Header().Set("Content-Type", "application/xml; charset=utf-8") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(body)) } func (a *app) robotsHandler(w http.ResponseWriter, r *http.Request) { body := renderRobotsTXT(requestOrigin(r)) w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(body)) } func collectSitemapEntries(pages []Doc, baseURL string) []sitemapEntry { seen := make(map[string]struct{}, len(pages)) entries := make([]sitemapEntry, 0, len(pages)) for _, page := range pages { if !isPublishedPublicPage(page) { continue } loc, ok := pagePublicURL(page, baseURL) if !ok { continue } if _, exists := seen[loc]; exists { continue } seen[loc] = struct{}{} entries = append(entries, sitemapEntry{ Loc: loc, LastMod: pageLastMod(page), }) } return entries } func isPublishedPublicPage(page Doc) bool { published := false now := time.Now().UTC() if status, ok := asString(page["status"]); ok && strings.EqualFold(status, "published") { published = true } if value, ok := asBool(page["published"]); ok && value { published = true } if value, ok := asBool(page["isPublished"]); ok && value { published = true } if publishedAt, ok := asTime(page["publishedAt"]); ok { if publishedAt.UTC().After(now) { return false } published = true } if !published { return false } if value, ok := asBool(page["private"]); ok && value { return false } if value, ok := asBool(page["isPrivate"]); ok && value { return false } if visibility, ok := asString(page["visibility"]); ok && strings.EqualFold(visibility, "private") { return false } if access, ok := asString(page["access"]); ok && strings.EqualFold(access, "private") { return false } if !isIndexablePage(page) { return false } slug, ok := asString(page["slug"]) return ok && strings.TrimSpace(slug) != "" } func isIndexablePage(page Doc) bool { seo, ok := asMap(page["seo"]) if !ok { return true } robotsRaw, exists := seo["robots"] if !exists || robotsRaw == nil { return true } robots, ok := asMap(robotsRaw) if !ok { return true } if index, exists := asBool(robots["index"]); exists && !index { return false } return true } func pagePublicURL(page Doc, baseURL string) (string, bool) { seo, _ := asMap(page["seo"]) if canonical, ok := asString(seo["canonicalUrl"]); ok && canonical != "" && isAbsoluteHTTPURL(canonical) { return canonical, true } slug, ok := asString(page["slug"]) if !ok { return "", false } slug = strings.Trim(strings.TrimSpace(slug), "/") if slug == "" { return "", false } return strings.TrimRight(baseURL, "/") + "/pages/" + slug, true } func pageLastMod(page Doc) string { raw, ok := page["updatedAt"] if !ok { return "" } if t, ok := asTime(raw); ok { return t.UTC().Format(time.RFC3339) } return "" } func renderSitemapXML(entries []sitemapEntry) (string, error) { xmlDoc := sitemapXML{XMLNS: "http://www.sitemaps.org/schemas/sitemap/0.9"} xmlDoc.URLs = make([]sitemapURL, 0, len(entries)) for _, entry := range entries { xmlDoc.URLs = append(xmlDoc.URLs, sitemapURL{Loc: entry.Loc, LastMod: entry.LastMod}) } body, err := xml.MarshalIndent(xmlDoc, "", " ") if err != nil { return "", err } return xml.Header + string(body), nil } func renderRobotsTXT(baseURL string) string { base := strings.TrimRight(baseURL, "/") return "User-agent: *\nAllow: /\nSitemap: " + base + "/sitemap.xml\n" } func requestOrigin(r *http.Request) string { scheme := r.Header.Get("X-Forwarded-Proto") if scheme == "" { if r.TLS != nil { scheme = "https" } else { scheme = "http" } } host := r.Header.Get("X-Forwarded-Host") if host == "" { host = r.Host } if host == "" { host = "localhost:8080" } return scheme + "://" + host } func isAbsoluteHTTPURL(value string) bool { u, err := url.Parse(strings.TrimSpace(value)) if err != nil { return false } if u.Host == "" { return false } return u.Scheme == "http" || u.Scheme == "https" } func asMap(raw any) (map[string]any, bool) { switch value := raw.(type) { case map[string]any: return value, true case Doc: return map[string]any(value), true default: return nil, false } } func asString(raw any) (string, bool) { value, ok := raw.(string) if !ok { return "", false } value = strings.TrimSpace(value) if value == "" { return "", false } return value, true } func asBool(raw any) (bool, bool) { value, ok := raw.(bool) return value, ok } func asTime(raw any) (time.Time, bool) { switch value := raw.(type) { case time.Time: if value.IsZero() { return time.Time{}, false } return value, true case int: if value <= 0 { return time.Time{}, false } return time.UnixMilli(int64(value)), true case int32: if value <= 0 { return time.Time{}, false } return time.UnixMilli(int64(value)), true case int64: if value <= 0 { return time.Time{}, false } return time.UnixMilli(value), true case float64: if value <= 0 { return time.Time{}, false } return time.UnixMilli(int64(value)), true case string: trimmed := strings.TrimSpace(value) if trimmed == "" { return time.Time{}, false } if parsed, err := strconv.ParseInt(trimmed, 10, 64); err == nil && parsed > 0 { return time.UnixMilli(parsed), true } if parsed, err := time.Parse(time.RFC3339, trimmed); err == nil && !parsed.IsZero() { return parsed, true } } return time.Time{}, false }