From d2373d3eabcf35b8b181c833f6da5a5431b38ff4 Mon Sep 17 00:00:00 2001 From: Mimic890 <144267059+Mimic890@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:00:00 +0300 Subject: [PATCH 1/4] feat: add SQLite metrics store and per-client rate limiter --- go.mod | 10 +- go.sum | 48 +++- internal/ratelimit/ratelimit.go | 79 +++++++ internal/ratelimit/ratelimit_test.go | 77 +++++++ internal/store/store.go | 318 +++++++++++++++++++++++++++ internal/store/store_test.go | 105 +++++++++ 6 files changed, 634 insertions(+), 3 deletions(-) create mode 100644 internal/ratelimit/ratelimit.go create mode 100644 internal/ratelimit/ratelimit_test.go create mode 100644 internal/store/store.go create mode 100644 internal/store/store_test.go diff --git a/go.mod b/go.mod index f579542..fb3a129 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.27.3 require ( github.com/alecthomas/chroma/v2 v2.27.0 + github.com/altcha-org/altcha-lib-go/v2 v2.4.1 github.com/blevesearch/bleve/v2 v2.6.1 github.com/microcosm-cc/bluemonday v1.0.27 github.com/minio/minio-go/v7 v7.4.1 @@ -14,6 +15,7 @@ require ( go.yaml.in/yaml/v3 v3.0.6 golang.org/x/image v0.47.0 golang.org/x/net v0.62.0 + modernc.org/sqlite v1.61.0 ) require ( @@ -40,7 +42,7 @@ require ( github.com/blevesearch/zapx/v17 v17.2.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dlclark/regexp2/v2 v2.2.1 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dustin/go-humanize v1.1.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect @@ -48,10 +50,13 @@ require ( github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/ncruces/go-strftime v1.1.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.6.0 // indirect github.com/tinylib/msgp v1.6.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect @@ -61,4 +66,7 @@ require ( golang.org/x/text v0.43.0 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/ini.v1 v1.67.3 // indirect + modernc.org/libc v1.79.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 7983681..caa7404 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslx github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/altcha-org/altcha-lib-go/v2 v2.4.1 h1:Y1lIhe+Y4QI/w1KV0wsCEfCzSmaBWJgu4I3gAKtHYaU= +github.com/altcha-org/altcha-lib-go/v2 v2.4.1/go.mod h1:n3KVRk3ziFGC5GIPkiSFMSROrAZ0wezy88///KeYdRo= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= @@ -58,18 +60,22 @@ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55k github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0= github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede h1:YrgBGwxMRK0Vq0WSCWFaZUnTsrA/PZE/xs1QZh+/edg= @@ -81,6 +87,8 @@ github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV91 github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= @@ -91,9 +99,13 @@ github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy7 github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= +github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -126,6 +138,8 @@ golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= @@ -134,6 +148,8 @@ golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -141,3 +157,31 @@ gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= +modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= +modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= +modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go new file mode 100644 index 0000000..e6478bf --- /dev/null +++ b/internal/ratelimit/ratelimit.go @@ -0,0 +1,79 @@ +// Package ratelimit is an in-memory token bucket per client address. +package ratelimit + +import ( + "container/list" + "math" + "net/netip" + "sync" + "time" +) + +// MaxClients bounds memory: past it, the least recently seen client is evicted. +const MaxClients = 10000 + +// Limiter allows N requests per Per for each client, with bursts up to N. +type Limiter struct { + n float64 + per time.Duration + max int + now func() time.Time + mu sync.Mutex + byIP map[netip.Prefix]*list.Element + lru *list.List // front: most recently seen +} + +type bucket struct { + key netip.Prefix + tokens float64 + last time.Time +} + +// New returns a limiter for n requests per per. +func New(n int, per time.Duration) *Limiter { + return &Limiter{n: float64(n), per: per, max: MaxClients, now: time.Now, byIP: map[netip.Prefix]*list.Element{}, lru: list.New()} +} + +// Key groups IPv6 clients by /64 (one subscriber); IPv4 is one address. +// The zero Addr (unknown client) shares one bucket. +func Key(ip netip.Addr) netip.Prefix { + ip = ip.Unmap() + bits := 32 + if ip.Is6() { + bits = 64 + } + p, _ := ip.Prefix(bits) // zero Addr gives the zero Prefix + return p +} + +// Allow takes a token for ip. If none is left it returns false and when to retry. +func (l *Limiter) Allow(ip netip.Addr) (bool, time.Duration) { + key := Key(ip) + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + var b *bucket + if e, ok := l.byIP[key]; ok { + b = e.Value.(*bucket) + l.lru.MoveToFront(e) + b.tokens = min(l.n, b.tokens+now.Sub(b.last).Seconds()*l.n/l.per.Seconds()) + b.last = now + } else { + if l.lru.Len() >= l.max { + old := l.lru.Back() + l.lru.Remove(old) + delete(l.byIP, old.Value.(*bucket).key) + } + b = &bucket{key: key, tokens: l.n, last: now} + l.byIP[key] = l.lru.PushFront(b) + } + if b.tokens >= 1 { + b.tokens-- + return true, 0 + } + wait := time.Duration((1 - b.tokens) * float64(l.per) / l.n) + return false, wait +} + +// RetryAfter is the Retry-After header value: whole seconds, at least 1. +func RetryAfter(d time.Duration) int { return max(1, int(math.Ceil(d.Seconds()))) } diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go new file mode 100644 index 0000000..4347b40 --- /dev/null +++ b/internal/ratelimit/ratelimit_test.go @@ -0,0 +1,77 @@ +package ratelimit + +import ( + "net/netip" + "testing" + "time" +) + +func TestLimiter(t *testing.T) { + now := time.Date(2026, 11, 4, 0, 0, 0, 0, time.UTC) + l := New(3, time.Minute) + l.now = func() time.Time { return now } + a := netip.MustParseAddr("192.0.2.1") + for i := range 3 { + if ok, _ := l.Allow(a); !ok { + t.Fatalf("request %d denied", i) + } + } + ok, wait := l.Allow(a) + if ok || wait != 20*time.Second || RetryAfter(wait) != 20 { + t.Fatalf("4th = %v %v", ok, wait) + } + if ok, _ := l.Allow(netip.MustParseAddr("192.0.2.2")); !ok { + t.Error("other client limited") + } + now = now.Add(20 * time.Second) // one token back + if ok, _ := l.Allow(a); !ok { + t.Error("no refill") + } + if ok, _ := l.Allow(a); ok { + t.Error("refilled too much") + } + now = now.Add(time.Hour) // refill caps at the burst + for range 3 { + l.Allow(a) + } + if ok, _ := l.Allow(a); ok { + t.Error("burst above N") + } +} + +func TestKeyIPv6Slash64(t *testing.T) { + l := New(1, time.Minute) + l.Allow(netip.MustParseAddr("2001:db8:1:2::1")) + if ok, _ := l.Allow(netip.MustParseAddr("2001:db8:1:2:ffff::9")); ok { + t.Error("same /64 not grouped") + } + if ok, _ := l.Allow(netip.MustParseAddr("2001:db8:1:3::1")); !ok { + t.Error("other /64 limited") + } + if Key(netip.MustParseAddr("::ffff:192.0.2.1")) != netip.MustParsePrefix("192.0.2.1/32") { + t.Error("mapped IPv4 not unmapped") + } + l.Allow(netip.Addr{}) + if ok, _ := l.Allow(netip.Addr{}); ok { + t.Error("unknown clients do not share a bucket") + } +} + +func TestMaxClients(t *testing.T) { + l := New(1, time.Hour) + l.max = 3 + ips := []string{"192.0.2.1", "192.0.2.2", "192.0.2.3", "192.0.2.4"} + for _, s := range ips { + l.Allow(netip.MustParseAddr(s)) + } + if len(l.byIP) != 3 || l.lru.Len() != 3 { + t.Fatalf("tracked %d", len(l.byIP)) + } + // The oldest (.1) was evicted, so it gets a fresh bucket; .4 is still limited. + if ok, _ := l.Allow(netip.MustParseAddr("192.0.2.1")); !ok { + t.Error("evicted client still limited") + } + if ok, _ := l.Allow(netip.MustParseAddr("192.0.2.4")); ok { + t.Error("recent client forgotten") + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..dea77cc --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,318 @@ +// Package store keeps per-post metrics, votes and search stats in SQLite. +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + _ "modernc.org/sqlite" // pure-Go driver "sqlite" +) + +// FileName is the database file inside DATA_DIR. +const FileName = "hyperhack.db" + +// Metric names in the counters table. Labels come from fixed lists or post content. +const ( + View = "view" // label "" + ReadSecs = "read_s" // label "": seconds, summed per view and clamped first + ReadView = "read_n" // label "": views that reported any reading time + Depth = "depth" // label "25" / "50" / "75" / "100" + Source = "source" // label: fixed source list + Device = "device" // label: mobile / tablet / desktop + Lang = "lang" // label: UI language + Theme = "theme" // label: light / dark + Outbound = "outbound" // label: a host linked from the post + Copy = "copy" // label "" + VoteCast = "vote" // label "like" / "dislike": votes cast, for vote trends +) + +// Day is the UTC calendar day used for daily rows. +func Day(t time.Time) string { return t.UTC().Format(time.DateOnly) } + +const schema = ` +CREATE TABLE IF NOT EXISTS counters ( + slug TEXT NOT NULL, metric TEXT NOT NULL, label TEXT NOT NULL, n INTEGER NOT NULL, + PRIMARY KEY (slug, metric, label)) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS daily_views ( + day TEXT NOT NULL, slug TEXT NOT NULL, n INTEGER NOT NULL, + PRIMARY KEY (day, slug)) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS votes ( + vid BLOB NOT NULL, slug TEXT NOT NULL, vote INTEGER NOT NULL CHECK (vote IN (-1, 1)), + PRIMARY KEY (vid, slug)) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS votes_slug ON votes (slug, vote); +CREATE TABLE IF NOT EXISTS searches ( + query TEXT PRIMARY KEY, n INTEGER NOT NULL, zero INTEGER NOT NULL, day TEXT NOT NULL) WITHOUT ROWID; +PRAGMA user_version = 1; +` + +// Store is safe for concurrent use. +type Store struct { + db *sql.DB + log *slog.Logger // component store +} + +// Open creates or opens dir/hyperhack.db with mode 0600, WAL and a busy timeout. +func Open(dir string, log *slog.Logger) (*Store, error) { + path := filepath.Join(dir, FileName) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return nil, err + } + _ = f.Close() + if err := os.Chmod(path, 0o600); err != nil { // also fixes a file created by hand + return nil, err + } + db, err := sql.Open("sqlite", "file:"+path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)") + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) // one writer; reads are small + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := db.ExecContext(ctx, schema); err != nil { + _ = db.Close() + return nil, fmt.Errorf("schema: %w", err) + } + return &Store{db: db, log: log.With("component", "store")}, nil +} + +// Close closes the database. +func (s *Store) Close() error { return s.db.Close() } + +// Inc adds N to one counter. +type Inc struct { + Slug, Metric, Label string + N int64 +} + +// Add applies counter increments in one transaction. View increments also go to the daily table. +func (s *Store) Add(ctx context.Context, day string, incs ...Inc) error { + if len(incs) == 0 { + return nil + } + err := s.tx(ctx, func(tx *sql.Tx) error { + for _, in := range incs { + if _, err := tx.ExecContext(ctx, `INSERT INTO counters (slug, metric, label, n) VALUES (?, ?, ?, ?) + ON CONFLICT (slug, metric, label) DO UPDATE SET n = n + excluded.n`, in.Slug, in.Metric, in.Label, in.N); err != nil { + return err + } + if in.Metric == View { + if _, err := tx.ExecContext(ctx, `INSERT INTO daily_views (day, slug, n) VALUES (?, ?, ?) + ON CONFLICT (day, slug) DO UPDATE SET n = n + excluded.n`, day, in.Slug, in.N); err != nil { + return err + } + } + } + return nil + }) + s.log.Debug("counters added", "rows", len(incs), "err", err) + return s.fail("add counters", err) +} + +// Total is one counter value. +type Total struct { + Slug, Metric, Label string + N int64 +} + +// Totals returns every counter, for /metrics. +func (s *Store) Totals(ctx context.Context) ([]Total, error) { + rows, err := s.db.QueryContext(ctx, `SELECT slug, metric, label, n FROM counters ORDER BY slug, metric, label`) + if err != nil { + return nil, s.fail("read counters", err) + } + defer func() { _ = rows.Close() }() + var out []Total + for rows.Next() { + var t Total + if err := rows.Scan(&t.Slug, &t.Metric, &t.Label, &t.N); err != nil { + return nil, s.fail("read counters", err) + } + out = append(out, t) + } + return out, s.fail("read counters", rows.Err()) +} + +// Views is the public all-time view count of a post. +func (s *Store) Views(ctx context.Context, slug string) (int64, error) { + var n int64 + err := s.db.QueryRowContext(ctx, `SELECT n FROM counters WHERE slug = ? AND metric = ? AND label = ''`, slug, View).Scan(&n) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + return n, s.fail("read views", err) +} + +// Popular returns slugs by views since day (inclusive), most viewed first. +// Ties go to the slug order, so the list is stable. +func (s *Store) Popular(ctx context.Context, since string) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT slug FROM daily_views WHERE day >= ? GROUP BY slug ORDER BY SUM(n) DESC, slug`, since) + if err != nil { + return nil, s.fail("read popular", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var slug string + if err := rows.Scan(&slug); err != nil { + return nil, s.fail("read popular", err) + } + out = append(out, slug) + } + return out, s.fail("read popular", rows.Err()) +} + +// Vote values. None removes the vote. +const ( + Dislike = -1 + None = 0 + Like = 1 +) + +// SetVote stores the vote of vid (a hash of the vote cookie) and returns the previous one. +func (s *Store) SetVote(ctx context.Context, vid []byte, slug string, vote int) (prev int, err error) { + err = s.tx(ctx, func(tx *sql.Tx) error { + err := tx.QueryRowContext(ctx, `SELECT vote FROM votes WHERE vid = ? AND slug = ?`, vid, slug).Scan(&prev) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if vote == None { + _, err = tx.ExecContext(ctx, `DELETE FROM votes WHERE vid = ? AND slug = ?`, vid, slug) + } else { + _, err = tx.ExecContext(ctx, `INSERT INTO votes (vid, slug, vote) VALUES (?, ?, ?) + ON CONFLICT (vid, slug) DO UPDATE SET vote = excluded.vote`, vid, slug, vote) + } + return err + }) + return prev, s.fail("set vote", err) +} + +// VoteOf returns the vote of vid on slug, or None. +func (s *Store) VoteOf(ctx context.Context, vid []byte, slug string) (int, error) { + var v int + err := s.db.QueryRowContext(ctx, `SELECT vote FROM votes WHERE vid = ? AND slug = ?`, vid, slug).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return None, nil + } + return v, s.fail("read vote", err) +} + +// Votes is the like and dislike count of one post. +type Votes struct{ Likes, Dislikes int64 } + +// VoteCounts returns the current counts for every voted post. +func (s *Store) VoteCounts(ctx context.Context) (map[string]Votes, error) { + rows, err := s.db.QueryContext(ctx, `SELECT slug, vote, COUNT(*) FROM votes GROUP BY slug, vote`) + if err != nil { + return nil, s.fail("read votes", err) + } + defer func() { _ = rows.Close() }() + out := map[string]Votes{} + for rows.Next() { + var slug string + var vote int + var n int64 + if err := rows.Scan(&slug, &vote, &n); err != nil { + return nil, s.fail("read votes", err) + } + v := out[slug] + if vote == Like { + v.Likes = n + } else { + v.Dislikes = n + } + out[slug] = v + } + return out, s.fail("read votes", rows.Err()) +} + +// PostVotes returns the counts for one post. +func (s *Store) PostVotes(ctx context.Context, slug string) (Votes, error) { + var v Votes + err := s.db.QueryRowContext(ctx, `SELECT COALESCE(SUM(vote = 1), 0), COALESCE(SUM(vote = -1), 0) FROM votes WHERE slug = ?`, slug).Scan(&v.Likes, &v.Dislikes) + return v, s.fail("read votes", err) +} + +// LogSearch counts a normalized query; zero marks a query with no results. +func (s *Store) LogSearch(ctx context.Context, day, query string, zero bool) error { + z := 0 + if zero { + z = 1 + } + _, err := s.db.ExecContext(ctx, `INSERT INTO searches (query, n, zero, day) VALUES (?, 1, ?, ?) + ON CONFLICT (query) DO UPDATE SET n = n + 1, zero = zero + excluded.zero, day = excluded.day`, query, z, day) + return s.fail("log search", err) +} + +// QueryCount is one stored search query. +type QueryCount struct { + Query string + N int64 +} + +// TopSearches returns the n most frequent queries; zero selects zero-result counts. +func (s *Store) TopSearches(ctx context.Context, n int, zero bool) ([]QueryCount, error) { + col := "n" + if zero { + col = "zero" + } + // col is one of two constants, never input. + rows, err := s.db.QueryContext(ctx, `SELECT query, `+col+` FROM searches WHERE `+col+` > 0 ORDER BY `+col+` DESC, query LIMIT ?`, n) + if err != nil { + return nil, s.fail("read searches", err) + } + defer func() { _ = rows.Close() }() + var out []QueryCount + for rows.Next() { + var q QueryCount + if err := rows.Scan(&q.Query, &q.N); err != nil { + return nil, s.fail("read searches", err) + } + out = append(out, q) + } + return out, s.fail("read searches", rows.Err()) +} + +// Prune drops search queries last seen before searchDay, keeps at most maxSearches +// (most frequent), and drops daily views before viewDay. +func (s *Store) Prune(ctx context.Context, searchDay, viewDay string, maxSearches int) error { + err := s.tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, `DELETE FROM searches WHERE day < ?`, searchDay); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM searches WHERE query NOT IN + (SELECT query FROM searches ORDER BY n DESC, day DESC, query LIMIT ?)`, maxSearches); err != nil { + return err + } + _, err := tx.ExecContext(ctx, `DELETE FROM daily_views WHERE day < ?`, viewDay) + return err + }) + return s.fail("prune", err) +} + +func (s *Store) tx(ctx context.Context, fn func(*sql.Tx) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := fn(tx); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() +} + +// fail logs a write or read error once, as ERROR, and wraps it. +func (s *Store) fail(op string, err error) error { + if err == nil { + return nil + } + s.log.Error("sqlite", "op", op, "err", err) + return fmt.Errorf("%s: %w", op, err) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..b67aad8 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,105 @@ +package store + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "slices" + "testing" +) + +func open(t *testing.T) (*Store, string) { + t.Helper() + dir := t.TempDir() + s, err := Open(dir, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + return s, dir +} + +func TestStore(t *testing.T) { + s, dir := open(t) + ctx := context.Background() + if fi, err := os.Stat(filepath.Join(dir, FileName)); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("db file mode: %v %v", fi.Mode(), err) + } + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.Add(ctx, "2026-11-01", Inc{"a", View, "", 5}, Inc{"b", View, "", 1}, Inc{"a", Source, "telegram", 2})) + must(s.Add(ctx, "2026-11-04", Inc{"b", View, "", 3}, Inc{"a", View, "", 1})) + if n, _ := s.Views(ctx, "a"); n != 6 { + t.Errorf("views a = %d", n) + } + if n, _ := s.Views(ctx, "none"); n != 0 { + t.Errorf("views none = %d", n) + } + if p, _ := s.Popular(ctx, "2026-11-02"); !slices.Equal(p, []string{"b", "a"}) { + t.Errorf("popular recent = %v", p) + } + if p, _ := s.Popular(ctx, "2026-10-01"); !slices.Equal(p, []string{"a", "b"}) { + t.Errorf("popular all = %v", p) + } + tot, _ := s.Totals(ctx) + if len(tot) != 3 || tot[0] != (Total{"a", Source, "telegram", 2}) { + t.Errorf("totals = %v", tot) + } + + vid1, vid2 := []byte("v1"), []byte("v2") + if prev, _ := s.SetVote(ctx, vid1, "a", Like); prev != None { + t.Errorf("prev = %d", prev) + } + _, _ = s.SetVote(ctx, vid2, "a", Dislike) + if prev, _ := s.SetVote(ctx, vid2, "a", Like); prev != Dislike { + t.Errorf("prev = %d", prev) + } + if v, _ := s.PostVotes(ctx, "a"); v != (Votes{2, 0}) { + t.Errorf("votes = %+v", v) + } + _, _ = s.SetVote(ctx, vid1, "a", None) + if v, _ := s.VoteOf(ctx, vid1, "a"); v != None { + t.Errorf("vote after none = %d", v) + } + if m, _ := s.VoteCounts(ctx); m["a"] != (Votes{1, 0}) { + t.Errorf("vote counts = %v", m) + } + if _, err := s.SetVote(ctx, vid1, "a", 7); err == nil { + t.Error("vote 7 accepted") + } + + for range 3 { + must(s.LogSearch(ctx, "2026-11-04", "go", false)) + } + must(s.LogSearch(ctx, "2026-11-04", "kubernetes", true)) + must(s.LogSearch(ctx, "2026-08-01", "old", true)) + if top, _ := s.TopSearches(ctx, 10, false); len(top) != 3 || top[0] != (QueryCount{"go", 3}) { + t.Errorf("top = %v", top) + } + if top, _ := s.TopSearches(ctx, 10, true); len(top) != 2 || top[0].Query != "kubernetes" { + t.Errorf("top zero = %v", top) + } + must(s.Prune(ctx, "2026-10-01", "2026-11-02", 1)) + if top, _ := s.TopSearches(ctx, 10, false); len(top) != 1 || top[0].Query != "go" { + t.Errorf("after prune = %v", top) + } + if p, _ := s.Popular(ctx, "2000-01-01"); !slices.Equal(p, []string{"b", "a"}) { + t.Errorf("popular after prune = %v", p) + } + + // Reopen: data survives. + _ = s.Close() + s2, err := Open(dir, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = s2.Close() }() + if n, _ := s2.Views(ctx, "a"); n != 6 { + t.Errorf("views after reopen = %d", n) + } +} From 5d0c4777a6d924ddbb8e66f5647e4d5c6755dd5c Mon Sep 17 00:00:00 2001 From: Mimic890 <144267059+Mimic890@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:00:00 +0300 Subject: [PATCH 2/4] feat: record outbound link hosts and expose sync stats --- internal/content/post.go | 36 +++++++++++++++++++++++++++++++++ internal/content/search_test.go | 7 +++++++ internal/syncer/syncer.go | 32 +++++++++++++++++++++++++++++ internal/syncer/syncer_test.go | 17 ++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/internal/content/post.go b/internal/content/post.go index 74b359c..f594a2f 100644 --- a/internal/content/post.go +++ b/internal/content/post.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "html/template" + "net/url" "path" "regexp" "slices" @@ -36,6 +37,7 @@ type Post struct { HTML template.HTML // sanitized Text string // plain text of HTML, for search Source []byte // the raw .md file, served at /posts/.md + Outbound []string // hosts of external links: the only allowed outbound-click labels } const maxFrontmatter = 64 << 10 @@ -142,6 +144,7 @@ func (r *Renderer) Parse(key string, src []byte) (*Post, error) { return nil, fmt.Errorf("render: %w", err) } p.Text = plainText(string(p.HTML)) + p.Outbound = outboundHosts(string(p.HTML)) p.Source = src return p, nil } @@ -167,6 +170,39 @@ func cleanTags(in []string) ([]string, error) { return out, nil } +// maxOutbound bounds the outbound-click labels per post. +const maxOutbound = 100 + +// outboundHosts lists the lower-case hosts of http(s) links in s, unique, in order. +func outboundHosts(s string) []string { + var hosts []string + z := html.NewTokenizer(strings.NewReader(s)) + for len(hosts) < maxOutbound { + switch z.Next() { + case html.ErrorToken: + return hosts + case html.StartTagToken: + if name, attr := z.TagName(); string(name) != "a" || !attr { + continue + } + for { + k, v, more := z.TagAttr() + if string(k) == "href" { + if u, err := url.Parse(string(v)); err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Hostname() != "" { + if h := strings.ToLower(u.Hostname()); !slices.Contains(hosts, h) { + hosts = append(hosts, h) + } + } + } + if !more { + break + } + } + } + } + return hosts +} + // plainText returns the text nodes of s, separated by spaces. func plainText(s string) string { var b strings.Builder diff --git a/internal/content/search_test.go b/internal/content/search_test.go index 8aa4d96..c503db7 100644 --- a/internal/content/search_test.go +++ b/internal/content/search_test.go @@ -138,3 +138,10 @@ func TestPlainText(t *testing.T) { t.Errorf("plainText = %q, want %q", got, want) } } + +func TestOutboundHosts(t *testing.T) { + got := outboundHosts(`a b c d e f`) + if strings.Join(got, ",") != "github.com,t.me" { + t.Errorf("outboundHosts = %v", got) + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index b01b438..5a5ae5f 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -46,6 +46,26 @@ type Syncer struct { warned map[string]string // key → ETag already warned about, so polls do not repeat WARNs maxWarned bool index atomic.Pointer[content.Index] + + lastAttempt, lastSuccess atomic.Int64 // unix seconds + errors, fetchFailures atomic.Int64 + objects atomic.Int64 +} + +// Stats is the sync status for /metrics. +type Stats struct { + LastAttempt, LastSuccess int64 // unix seconds, 0 if never + Errors int64 // failed syncs (listing or saving state) + FetchFailures int64 // objects that failed to download + Objects int64 // keys in the last listing +} + +// Stats returns the sync status counters. +func (s *Syncer) Stats() Stats { + return Stats{ + LastAttempt: s.lastAttempt.Load(), LastSuccess: s.lastSuccess.Load(), + Errors: s.errors.Load(), FetchFailures: s.fetchFailures.Load(), Objects: s.objects.Load(), + } } type cached struct { @@ -123,11 +143,22 @@ func (s *Syncer) syncRetry(ctx context.Context) error { func (s *Syncer) Sync(ctx context.Context) error { s.mu.Lock() defer s.mu.Unlock() + s.lastAttempt.Store(time.Now().Unix()) + err := s.sync(ctx) + if err != nil && ctx.Err() == nil { + s.errors.Add(1) + } else if err == nil { + s.lastSuccess.Store(time.Now().Unix()) + } + return err +} +func (s *Syncer) sync(ctx context.Context) error { objs, truncated, err := s.bucket.List(ctx, s.opts.MaxObjects) if err != nil { return fmt.Errorf("list: %w", err) } + s.objects.Store(int64(len(objs))) if truncated && !s.maxWarned { s.log.Warn("bucket has more objects than MAX_OBJECTS, the rest are skipped", "max", s.opts.MaxObjects) } @@ -170,6 +201,7 @@ func (s *Syncer) Sync(ctx context.Context) error { _, existed := s.mirror.State()[o.Key] if err := s.fetch(ctx, o, limit); err != nil { warn(o, "fetch failed, keeping the old copy", "err", err) + s.fetchFailures.Add(1) failed++ continue } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index c3cb2fc..2337ecf 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -343,3 +343,20 @@ func TestAssetChangeRerenders(t *testing.T) { func pngEncode(w io.Writer, width, height int) error { return png.Encode(w, image.NewGray(image.Rect(0, 0, width, height))) } + +func TestStats(t *testing.T) { + b := &fakeBucket{objs: map[string]fakeObj{"a.md": {"e", post("A", "a")}, "b.md": {"e", post("B", "b")}}, getErr: map[string]error{"b.md": errors.New("boom")}} + s := newSyncer(t, t.TempDir(), b) + if err := s.Sync(context.Background()); err != nil { + t.Fatal(err) + } + st := s.Stats() + if st.Objects != 2 || st.FetchFailures != 1 || st.Errors != 0 || st.LastSuccess == 0 || st.LastAttempt < st.LastSuccess { + t.Errorf("stats = %+v", st) + } + b.listErr = errors.New("down") + _ = s.Sync(context.Background()) + if st := s.Stats(); st.Errors != 1 { + t.Errorf("errors = %d", st.Errors) + } +} From 809c7bf8bf577d3b477c43de807a1a286de2b00d Mon Sep 17 00:00:00 2001 From: Mimic890 <144267059+Mimic890@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:00:00 +0300 Subject: [PATCH 3/4] feat: add votes with ALTCHA, metrics beacon, rate limits and /metrics --- internal/server/altcha.go | 109 +++++++++ internal/server/altcha_test.go | 114 +++++++++ internal/server/api.go | 196 ++++++++++++++++ internal/server/api_test.go | 264 +++++++++++++++++++++ internal/server/beacon.go | 233 +++++++++++++++++++ internal/server/beacon_test.go | 170 ++++++++++++++ internal/server/metrics.go | 269 ++++++++++++++++++++++ internal/server/pages.go | 147 ++++++++++-- internal/server/pages_test.go | 30 ++- internal/server/reindex.go | 8 +- internal/server/reindex_test.go | 4 +- internal/server/server.go | 36 ++- internal/server/server_test.go | 11 +- internal/server/web/i18n/en.json | 9 +- internal/server/web/i18n/ru.json | 9 +- internal/server/web/static/altcha.js | 43 ++++ internal/server/web/static/post.js | 135 +++++++++++ internal/server/web/static/search.js | 12 +- internal/server/web/static/style.css | 23 +- internal/server/web/templates/home.html | 10 + internal/server/web/templates/layout.html | 2 +- internal/server/web/templates/post.html | 11 +- main.go | 31 ++- 23 files changed, 1819 insertions(+), 57 deletions(-) create mode 100644 internal/server/altcha.go create mode 100644 internal/server/altcha_test.go create mode 100644 internal/server/api.go create mode 100644 internal/server/api_test.go create mode 100644 internal/server/beacon.go create mode 100644 internal/server/beacon_test.go create mode 100644 internal/server/metrics.go create mode 100644 internal/server/web/static/altcha.js create mode 100644 internal/server/web/static/post.js diff --git a/internal/server/altcha.go b/internal/server/altcha.go new file mode 100644 index 0000000..cacfccf --- /dev/null +++ b/internal/server/altcha.go @@ -0,0 +1,109 @@ +package server + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "math/big" + "sync" + "time" + + "github.com/altcha-org/altcha-lib-go/v2" +) + +// ALTCHA v2 settings. The client in static/altcha.js implements exactly this: +// key = SHA-256(salt || nonce || uint32 big-endian counter), cost 1, 32-byte key, +// solved when the hex key starts with keyPrefix. ALTCHA_MAX_NUMBER bounds the counter. +const ( + altchaAlgorithm = "SHA-256" + altchaCost = 1 + maxUsedAltcha = 100000 // replay map cap; past it new solutions are refused +) + +var errAltchaBusy = errors.New("too many open challenges") + +// altchaGuard creates challenges and verifies solutions once (replay protection). +type altchaGuard struct { + sigKey, keySigKey string // two keys derived from ALTCHA_HMAC_KEY + maxNumber int64 + expires time.Duration + now func() time.Time + + mu sync.Mutex + used map[string]int64 // challenge signature → expiresAt (unix s) + lastPurge time.Time +} + +func newAltchaGuard(hmacKey string, maxNumber int64, expires time.Duration) *altchaGuard { + derive := func(label string) string { + m := hmac.New(sha256.New, []byte(hmacKey)) + m.Write([]byte(label)) + return hex.EncodeToString(m.Sum(nil)) + } + return &altchaGuard{ + sigKey: derive("altcha signature"), keySigKey: derive("altcha key signature"), + maxNumber: maxNumber, expires: expires, now: time.Now, used: map[string]int64{}, + } +} + +func (g *altchaGuard) challenge() (altcha.Challenge, error) { + n, err := rand.Int(rand.Reader, big.NewInt(g.maxNumber+1)) + if err != nil { + return altcha.Challenge{}, err + } + counter := int(n.Int64()) + exp := g.now().Add(g.expires) + return altcha.CreateChallenge(altcha.CreateChallengeOptions{ + Algorithm: altchaAlgorithm, + Cost: altchaCost, + Counter: &counter, + DeriveKey: altcha.DeriveKeySHA(), + ExpiresAt: &exp, + HMACSignatureSecret: g.sigKey, + HMACKeySignatureSecret: g.keySigKey, + }) +} + +// verify checks a solution and marks its challenge used. The error says why it failed. +func (g *altchaGuard) verify(p altcha.Payload) error { + params := p.Challenge.Parameters + if params.Algorithm != altchaAlgorithm || params.Cost != altchaCost || params.ExpiresAt <= 0 { + return errors.New("unexpected challenge parameters") + } + res, err := altcha.VerifySolution(altcha.VerifySolutionOptions{ + Challenge: p.Challenge, + Solution: p.Solution, + DeriveKey: altcha.DeriveKeySHA(), + HMACSignatureSecret: g.sigKey, + HMACKeySignatureSecret: g.keySigKey, + }) + switch { + case err != nil: + return err + case res.Expired: + return errors.New("challenge expired") + case !res.Verified: + return errors.New("invalid solution") + } + now := g.now() + g.mu.Lock() + defer g.mu.Unlock() + if now.Sub(g.lastPurge) > time.Minute || len(g.used) >= maxUsedAltcha { + for sig, exp := range g.used { + if exp < now.Unix() { + delete(g.used, sig) + } + } + g.lastPurge = now + } + if _, ok := g.used[p.Challenge.Signature]; ok { + return errors.New("challenge already used") + } + if len(g.used) >= maxUsedAltcha { + return errAltchaBusy + } + g.used[p.Challenge.Signature] = params.ExpiresAt + return nil +} diff --git a/internal/server/altcha_test.go b/internal/server/altcha_test.go new file mode 100644 index 0000000..6b31743 --- /dev/null +++ b/internal/server/altcha_test.go @@ -0,0 +1,114 @@ +package server + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/altcha-org/altcha-lib-go/v2" +) + +// solveWithJS runs static/altcha.js in node (WebCrypto) and returns its payload. +func solveWithJS(t *testing.T, ch altcha.Challenge) altcha.Payload { + t.Helper() + node, err := exec.LookPath("node") + if err != nil { + if os.Getenv("CI") != "" { + t.Fatal("node is required in CI for the ALTCHA solver test") + } + t.Skip("node not installed") + } + src, err := webFS.ReadFile("web/static/altcha.js") + if err != nil { + t.Fatal(err) + } + solver := filepath.Join(t.TempDir(), "altcha.js") + if err := os.WriteFile(solver, src, 0o600); err != nil { + t.Fatal(err) + } + in, _ := json.Marshal(ch) + // The browser file sets globalThis.hyperhackAltcha; load it as-is and call solve. + script := `const fs = require("fs"); +(0, eval)(fs.readFileSync(process.argv[1], "utf8")); +const ch = JSON.parse(fs.readFileSync(0, "utf8")); +globalThis.hyperhackAltcha.solve(ch).then((p) => process.stdout.write(JSON.stringify(p)), (e) => { console.error(e); process.exit(1); });` + cmd := exec.Command(node, "-e", script, solver) + cmd.Stdin = strings.NewReader(string(in)) + out, err := cmd.Output() + if err != nil { + t.Fatalf("node: %v", err) + } + var p altcha.Payload + if err := json.Unmarshal(out, &p); err != nil { + t.Fatalf("payload %s: %v", out, err) + } + return p +} + +func TestAltchaJSSolverPassesLibrary(t *testing.T) { + g := newAltchaGuard("0123456789abcdef0123456789abcdef", 3000, 5*time.Minute) + ch, err := g.challenge() + if err != nil { + t.Fatal(err) + } + p := solveWithJS(t, ch) + + // The library's own v2 verification, with the exact options the server uses. + res, err := altcha.VerifySolution(altcha.VerifySolutionOptions{ + Challenge: p.Challenge, Solution: p.Solution, DeriveKey: altcha.DeriveKeySHA(), + HMACSignatureSecret: g.sigKey, HMACKeySignatureSecret: g.keySigKey, + }) + if err != nil || !res.Verified { + t.Fatalf("VerifySolution = %+v, %v", res, err) + } + // Slow path too: re-derive without the key signature, so the counter itself is checked. + res, err = altcha.VerifySolution(altcha.VerifySolutionOptions{ + Challenge: p.Challenge, Solution: p.Solution, DeriveKey: altcha.DeriveKeySHA(), HMACSignatureSecret: g.sigKey, + }) + if err != nil || !res.Verified { + t.Fatalf("VerifySolution (re-derive) = %+v, %v", res, err) + } + // Same answer as the library solver. + lib, err := altcha.SolveChallenge(altcha.SolveChallengeOptions{Challenge: ch, DeriveKey: altcha.DeriveKeySHA()}) + if err != nil || lib == nil || lib.Counter != p.Solution.Counter || lib.DerivedKey != p.Solution.DerivedKey { + t.Fatalf("library solution %+v, JS %+v", lib, p.Solution) + } + + if err := g.verify(p); err != nil { + t.Fatalf("guard.verify = %v", err) + } + if err := g.verify(p); err == nil || !strings.Contains(err.Error(), "already used") { + t.Fatalf("replay = %v", err) + } +} + +func TestAltchaRejects(t *testing.T) { + g := newAltchaGuard("0123456789abcdef0123456789abcdef", 1000, time.Minute) + ch, _ := g.challenge() + sol, _ := altcha.SolveChallenge(altcha.SolveChallengeOptions{Challenge: ch, DeriveKey: altcha.DeriveKeySHA()}) + + bad := *sol + bad.DerivedKey = strings.Repeat("0", 64) + if err := g.verify(altcha.Payload{Challenge: ch, Solution: bad}); err == nil { + t.Error("wrong key accepted") + } + other := newAltchaGuard("another key, also 32 bytes long!!", 1000, time.Minute) + if err := other.verify(altcha.Payload{Challenge: ch, Solution: *sol}); err == nil { + t.Error("challenge from another key accepted") + } + tampered := ch + tampered.Parameters.ExpiresAt += 3600 + if err := g.verify(altcha.Payload{Challenge: tampered, Solution: *sol}); err == nil { + t.Error("extended expiry accepted") + } + ch2, _ := (&altchaGuard{sigKey: g.sigKey, keySigKey: g.keySigKey, maxNumber: 10, expires: time.Second, now: time.Now}).challenge() + sol2, _ := altcha.SolveChallenge(altcha.SolveChallengeOptions{Challenge: ch2, DeriveKey: altcha.DeriveKeySHA()}) + time.Sleep(1100 * time.Millisecond) + if err := g.verify(altcha.Payload{Challenge: ch2, Solution: *sol2}); err == nil || !strings.Contains(err.Error(), "expired") { + t.Errorf("expired = %v", err) + } +} diff --git a/internal/server/api.go b/internal/server/api.go new file mode 100644 index 0000000..fc5a941 --- /dev/null +++ b/internal/server/api.go @@ -0,0 +1,196 @@ +package server + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "regexp" + + "github.com/altcha-org/altcha-lib-go/v2" + + "github.com/Mimic890/hyperhack/internal/store" +) + +var vidRe = regexp.MustCompile(`^[0-9a-f]{32}$`) + +var voteValues = map[string]int{"like": store.Like, "dislike": store.Dislike, "none": store.None} + +func voteName(v int) string { + for name, n := range voteValues { + if n == v { + return name + } + } + return "none" +} + +// sameOrigin is the CSRF check for /api/*: a cross-site Sec-Fetch-Site is refused, +// and a POST must carry the site's own Origin. +func (p *pages) sameOrigin(r *http.Request) bool { + if s := r.Header.Get("Sec-Fetch-Site"); s != "" && s != "same-origin" { + return false + } + return r.Method == http.MethodGet || r.Header.Get("Origin") == p.origin +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(w).Encode(v) +} + +func apiError(w http.ResponseWriter, status int) { + http.Error(w, http.StatusText(status), status) +} + +// readVID returns the vote cookie if it is well formed. +func readVID(r *http.Request) string { + if c, err := r.Cookie("vid"); err == nil && vidRe.MatchString(c.Value) { + return c.Value + } + return "" +} + +// vidHash is what the store keeps, so a database copy does not hold usable cookies. +func vidHash(vid string) []byte { + h := sha256.Sum256([]byte(vid)) + return h[:] +} + +// ensureVID returns the vote cookie, setting a new random one if missing. +func (p *pages) ensureVID(w http.ResponseWriter, r *http.Request) string { + if vid := readVID(r); vid != "" { + return vid + } + b := make([]byte, 16) + _, _ = rand.Read(b) + vid := hex.EncodeToString(b) + http.SetCookie(w, &http.Cookie{ + Name: "vid", Value: vid, Path: "/", MaxAge: cookieMaxAge, + HttpOnly: true, Secure: p.cfg.SiteURL.Scheme == "https", SameSite: http.SameSiteLaxMode, + }) + return vid +} + +// apiAltcha handles GET /api/altcha: a fresh signed challenge. +func (p *pages) apiAltcha(w http.ResponseWriter, r *http.Request) { + if p.limit(p.lim.vote, "altcha")(w, r) { + return + } + if !p.sameOrigin(r) { + p.vlog.Warn("origin check failed", "endpoint", "altcha") + apiError(w, http.StatusForbidden) + return + } + ch, err := p.altcha.challenge() + if err != nil { + p.vlog.Error("altcha challenge", "err", err) + apiError(w, http.StatusInternalServerError) + return + } + writeJSON(w, ch) +} + +type voteRequest struct { + Slug string `json:"slug"` + Vote string `json:"vote"` + Altcha altcha.Payload `json:"altcha"` +} + +type voteResponse struct { + Likes int64 `json:"likes"` + Dislikes int64 `json:"dislikes"` + Vote string `json:"vote"` +} + +// apiVote handles POST /api/vote: one vote per vid cookie per post, each behind ALTCHA. +func (p *pages) apiVote(w http.ResponseWriter, r *http.Request) { + if p.limit(p.lim.vote, "vote")(w, r) { + return + } + if !p.sameOrigin(r) { + p.vlog.Warn("origin check failed", "endpoint", "vote") + apiError(w, http.StatusForbidden) + return + } + var req voteRequest + if status, err := decodeJSON(w, r, &req); err != nil { + apiError(w, status) + return + } + vote, ok := voteValues[req.Vote] + if !ok { + apiError(w, http.StatusBadRequest) + return + } + ix := p.index() + if ix == nil { + apiError(w, http.StatusServiceUnavailable) + return + } + post := ix.Get(req.Slug) + if post == nil { + apiError(w, http.StatusNotFound) + return + } + if err := p.altcha.verify(req.Altcha); err != nil { + p.vlog.Warn("altcha failed", "reason", err) + apiError(w, http.StatusForbidden) + return + } + ctx := r.Context() + prev, err := p.store.SetVote(ctx, vidHash(p.ensureVID(w, r)), post.Slug, vote) + if err != nil { + apiError(w, http.StatusServiceUnavailable) + return + } + if vote != store.None && vote != prev { + _ = p.store.Add(ctx, store.Day(p.visitors.now()), store.Inc{Slug: post.Slug, Metric: store.VoteCast, Label: req.Vote, N: 1}) + } + counts, err := p.store.PostVotes(ctx, post.Slug) + if err != nil { + apiError(w, http.StatusServiceUnavailable) + return + } + p.vlog.Debug("vote", "slug", post.Slug, "vote", req.Vote, "was", voteName(prev)) + writeJSON(w, voteResponse{Likes: counts.Likes, Dislikes: counts.Dislikes, Vote: req.Vote}) +} + +// apiBeacon handles POST /api/beacon: one delta of a page view's metrics. +func (p *pages) apiBeacon(w http.ResponseWriter, r *http.Request) { + if p.limit(p.lim.beacon, "beacon")(w, r) { + return + } + if !p.sameOrigin(r) { + p.mlog.Warn("origin check failed", "endpoint", "beacon") + apiError(w, http.StatusForbidden) + return + } + var b beacon + if status, err := decodeJSON(w, r, &b); err != nil { + apiError(w, status) + return + } + if b.validate() != nil { + apiError(w, http.StatusBadRequest) + return + } + ix := p.index() + if ix == nil { + apiError(w, http.StatusServiceUnavailable) + return + } + post := ix.Get(b.Slug) + if post == nil { + apiError(w, http.StatusNotFound) // nothing stored for unknown slugs + return + } + day, incs := p.visitors.apply(&b, post, clientIP(r, p.cfg.TrustedProxies), r.UserAgent()) + if err := p.store.Add(r.Context(), day, incs...); err != nil { + apiError(w, http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/server/api_test.go b/internal/server/api_test.go new file mode 100644 index 0000000..f839463 --- /dev/null +++ b/internal/server/api_test.go @@ -0,0 +1,264 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/altcha-org/altcha-lib-go/v2" + + "github.com/Mimic890/hyperhack/internal/config" + "github.com/Mimic890/hyperhack/internal/store" +) + +func (s *testSite) do(method, path, body string, hdr map[string]string, cookies ...*http.Cookie) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, strings.NewReader(body)) + for k, v := range hdr { + r.Header.Set(k, v) + } + for _, c := range cookies { + r.AddCookie(c) + } + rec := httptest.NewRecorder() + s.h.ServeHTTP(rec, r) + return rec +} + +var jsonSameOrigin = map[string]string{"Content-Type": "application/json", "Origin": "https://example.com", "Sec-Fetch-Site": "same-origin"} + +func beaconBody(view string, first bool, time int) string { + b := map[string]any{"slug": "zametka", "view": view, "first": first, "time": time} + if first { + b["source"], b["device"], b["lang"], b["theme"] = "telegram", "mobile", "ru", "dark" + } + out, _ := json.Marshal(b) + return string(out) +} + +func TestOriginCheck(t *testing.T) { + s := seoSite(t) + body := beaconBody(view1, true, 1) + for name, hdr := range map[string]map[string]string{ + "no origin": {"Content-Type": "application/json"}, + "other origin": {"Content-Type": "application/json", "Origin": "https://evil.example"}, + "http origin": {"Content-Type": "application/json", "Origin": "http://example.com"}, + "cross-site": {"Content-Type": "application/json", "Origin": "https://example.com", "Sec-Fetch-Site": "cross-site"}, + "same-site": {"Content-Type": "application/json", "Origin": "https://example.com", "Sec-Fetch-Site": "same-site"}, + } { + if rec := s.do("POST", "/api/beacon", body, hdr); rec.Code != http.StatusForbidden { + t.Errorf("%s: %d, want 403", name, rec.Code) + } + if rec := s.do("POST", "/api/vote", `{}`, hdr); rec.Code != http.StatusForbidden { + t.Errorf("vote %s: %d, want 403", name, rec.Code) + } + } + if rec := s.do("GET", "/api/altcha", "", map[string]string{"Sec-Fetch-Site": "cross-site"}); rec.Code != http.StatusForbidden { + t.Errorf("cross-site altcha = %d", rec.Code) + } + if rec := s.do("POST", "/api/beacon", body, jsonSameOrigin); rec.Code != http.StatusNoContent { + t.Errorf("same origin = %d", rec.Code) + } + if rec := s.do("POST", "/api/beacon", body, map[string]string{"Content-Type": "text/plain", "Origin": "https://example.com"}); rec.Code != http.StatusUnsupportedMediaType { + t.Errorf("text/plain = %d", rec.Code) + } + if rec := s.do("GET", "/api/beacon", "", nil); rec.Code != http.StatusMethodNotAllowed { + t.Errorf("GET beacon = %d", rec.Code) + } +} + +func TestBeaconToMetrics(t *testing.T) { + s := seoSite(t) + send := func(body string, want int) { + t.Helper() + if rec := s.do("POST", "/api/beacon", body, jsonSameOrigin); rec.Code != want { + t.Fatalf("beacon %s = %d, want %d", body, rec.Code, want) + } + } + send(beaconBody(view1, true, 3*3600), 204) + send(beaconBody(view1, false, 2*3600), 204) // view sum clamped at 4 h + send(`{"slug":"zametka","view":"`+view1+`","depth":[25,100],"copies":2}`, 204) + send(beaconBody(view2, true, 10), 204) // same visitor: no new view + send(strings.Replace(beaconBody(view2, true, 1), "zametka", "nope", 1), 404) + send(strings.Replace(beaconBody(view2, true, 1), "telegram", "facebook", 1), 400) + + mustContain(t, s.get("/posts/zametka").Body.String(), "Views: 1") + m := s.get("/metrics").Body.String() + mustContain(t, m, + `hyperhack_post_views_total{slug="zametka"} 1`, + `hyperhack_post_read_seconds_total{slug="zametka"} 14410`, + `hyperhack_post_read_views_total{slug="zametka"} 2`, + `hyperhack_post_read_depth_total{slug="zametka",depth="100"} 1`, + `hyperhack_post_source_total{slug="zametka",source="telegram"} 1`, + `hyperhack_post_code_copies_total{slug="zametka"} 2`, + `hyperhack_sync_objects 3`, + `hyperhack_http_requests_total{route="POST /api/beacon",code="204"} 4`, + `hyperhack_http_request_duration_seconds_bucket{route="POST /api/beacon",le="+Inf"} `, + ) +} + +func solve(t *testing.T, s *testSite) altcha.Payload { + t.Helper() + rec := s.do("GET", "/api/altcha", "", nil) + if rec.Code != 200 || rec.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("altcha = %d", rec.Code) + } + var ch altcha.Challenge + if err := json.Unmarshal(rec.Body.Bytes(), &ch); err != nil { + t.Fatal(err) + } + sol, err := altcha.SolveChallenge(altcha.SolveChallengeOptions{Challenge: ch, DeriveKey: altcha.DeriveKeySHA()}) + if err != nil || sol == nil { + t.Fatalf("solve: %v", err) + } + return altcha.Payload{Challenge: ch, Solution: *sol} +} + +func voteBody(t *testing.T, vote string, p altcha.Payload) string { + b, err := json.Marshal(voteRequest{Slug: "zametka", Vote: vote, Altcha: p}) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestVote(t *testing.T) { + s := seoSite(t) + p := solve(t, s) + rec := s.do("POST", "/api/vote", voteBody(t, "like", p), jsonSameOrigin) + if rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != `{"likes":1,"dislikes":0,"vote":"like"}` { + t.Fatalf("vote = %d %s", rec.Code, rec.Body) + } + var vid *http.Cookie + for _, c := range rec.Result().Cookies() { + if c.Name == "vid" { + vid = c + } + } + if vid == nil || !vidRe.MatchString(vid.Value) || !vid.HttpOnly || !vid.Secure || vid.SameSite != http.SameSiteLaxMode { + t.Fatalf("vid cookie = %+v", vid) + } + // Replay of the same solution. + if rec := s.do("POST", "/api/vote", voteBody(t, "like", p), jsonSameOrigin, vid); rec.Code != http.StatusForbidden { + t.Errorf("replay = %d", rec.Code) + } + page := s.get("/posts/zametka", vid).Body.String() + mustContain(t, page, `data-vote="like" aria-pressed="true"`, `1`, `src="/static/altcha.js?v=`, `src="/static/post.js?v=`) + // Change to dislike with the same cookie, then remove. + if rec := s.do("POST", "/api/vote", voteBody(t, "dislike", solve(t, s)), jsonSameOrigin, vid); !strings.Contains(rec.Body.String(), `{"likes":0,"dislikes":1,"vote":"dislike"}`) { + t.Errorf("change = %d %s", rec.Code, rec.Body) + } + if rec := s.do("POST", "/api/vote", voteBody(t, "none", solve(t, s)), jsonSameOrigin, vid); !strings.Contains(rec.Body.String(), `{"likes":0,"dislikes":0,"vote":"none"}`) { + t.Errorf("remove = %d %s", rec.Code, rec.Body) + } + // Bad inputs. + if rec := s.do("POST", "/api/vote", voteBody(t, "love", solve(t, s)), jsonSameOrigin); rec.Code != 400 { + t.Errorf("bad vote = %d", rec.Code) + } + bad := solve(t, s) + bad.Solution.DerivedKey = strings.Repeat("ab", 32) + if rec := s.do("POST", "/api/vote", voteBody(t, "like", bad), jsonSameOrigin); rec.Code != http.StatusForbidden { + t.Errorf("wrong solution = %d", rec.Code) + } + if rec := s.do("POST", "/api/vote", `{"slug":"zametka","vote":"like"}`, jsonSameOrigin); rec.Code != http.StatusForbidden { + t.Errorf("no altcha = %d", rec.Code) + } + mustContain(t, s.get("/metrics").Body.String(), `hyperhack_post_votes_total{slug="zametka",vote="like"} 1`, `hyperhack_post_votes_total{slug="zametka",vote="dislike"} 1`, `hyperhack_post_likes{slug="zametka"} 0`) +} + +func TestRateLimits(t *testing.T) { + cfg := testConfig() + one := config.Rate{N: 1, Per: time.Hour} + cfg.RateBeacon, cfg.RateSearch, cfg.RateVote = one, one, one + s := newSite(t, cfg, seoSite(t).idx.Load()) + if rec := s.do("POST", "/api/beacon", beaconBody(view1, true, 1), jsonSameOrigin); rec.Code != 204 { + t.Fatalf("first beacon = %d", rec.Code) + } + rec := s.do("POST", "/api/beacon", beaconBody(view1, false, 1), jsonSameOrigin) + if rec.Code != http.StatusTooManyRequests || rec.Header().Get("Retry-After") != "3600" { + t.Errorf("second beacon = %d, Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) + } + s.get("/search?q=go") + rec = s.get("/search?q=go") + if rec.Code != http.StatusTooManyRequests || !strings.Contains(rec.Body.String(), "Too many requests") || rec.Header().Get("Retry-After") == "" { + t.Errorf("search = %d", rec.Code) + } + // /api/altcha and /api/vote share RATE_LIMIT_VOTE. + s.do("GET", "/api/altcha", "", nil) + if rec := s.do("POST", "/api/vote", `{}`, jsonSameOrigin); rec.Code != http.StatusTooManyRequests { + t.Errorf("vote after altcha = %d", rec.Code) + } + // Another client is not affected. + r := httptest.NewRequest("GET", "/search?q=go", nil) + r.RemoteAddr = "198.51.100.7:1000" + rr := httptest.NewRecorder() + s.h.ServeHTTP(rr, r) + if rr.Code != 200 { + t.Errorf("other client = %d", rr.Code) + } +} + +func TestMetricsAccess(t *testing.T) { + s := seoSite(t) + for remote, want := range map[string]int{"203.0.113.9:1": 403, "10.1.2.3:1": 200, "[2001:db8::1]:1": 403, "127.0.0.1:1": 200} { + r := httptest.NewRequest("GET", "/metrics", nil) + r.RemoteAddr = remote + rec := httptest.NewRecorder() + s.h.ServeHTTP(rec, r) + if rec.Code != want { + t.Errorf("%s = %d, want %d", remote, rec.Code, want) + } + if want == 200 && rec.Header().Get("Content-Type") != "text/plain; version=0.0.4; charset=utf-8" { + t.Errorf("content type %q", rec.Header().Get("Content-Type")) + } + } +} + +func TestSearchStats(t *testing.T) { + s := seoSite(t) + s.do("GET", "/search?q=bl", "", map[string]string{"X-Search": "live"}) + s.do("GET", "/search?q=Bleve!", "", map[string]string{"X-Search": "final"}) + s.get("/search?q=nothinghere") + s.get("/search?q=go&page=2") // later pages are not counted + top, _ := s.d.Store.TopSearches(context.Background(), 10, false) + if len(top) != 2 || top[0].Query != "bleve" && top[1].Query != "bleve" { + t.Errorf("stored = %v", top) + } + zero, _ := s.d.Store.TopSearches(context.Background(), 10, true) + if len(zero) != 1 || zero[0].Query != "nothinghere" { + t.Errorf("zero = %v", zero) + } + mustContain(t, s.get("/metrics").Body.String(), "hyperhack_search_queries_total 2", "hyperhack_search_zero_results_total 1", + `hyperhack_search_top_zero_result_queries{query="nothinghere"} 1`, "hyperhack_search_duration_seconds_count 4") +} + +func TestPopular(t *testing.T) { + s := seoSite(t) + ctx := context.Background() + _ = s.d.Store.Add(ctx, store.Day(time.Now()), store.Inc{Slug: "zametka", Metric: store.View, N: 5}, store.Inc{Slug: "go-script-tips", Metric: store.View, N: 9}) + _ = s.d.Store.Add(ctx, store.Day(time.Now().AddDate(0, 0, -40)), store.Inc{Slug: "zametka", Metric: store.View, N: 100}) // outside POPULAR_DAYS + body := s.get("/").Body.String() + i, j := strings.Index(body, `
`) || j < i || + strings.Index(body[i:], "go-script-tips") > strings.Index(body[i:], "zametka") { + t.Errorf("popular section wrong:\n%s", body[max(i, 0):]) + } + if strings.Contains(s.get("/?page=2").Body.String(), `class="popular"`) { + t.Error("popular on page 2") + } +} + +func TestStaticScripts(t *testing.T) { + s := seoSite(t) + for _, name := range []string{"search.js", "altcha.js", "post.js"} { + if rec := s.get("/static/" + name); rec.Code != 200 || rec.Header().Get("Content-Type") != "text/javascript; charset=utf-8" { + t.Errorf("%s = %d", name, rec.Code) + } + } + if rec := s.get("/static/other.js"); rec.Code != 404 { + t.Errorf("other.js = %d", rec.Code) + } +} diff --git a/internal/server/beacon.go b/internal/server/beacon.go new file mode 100644 index 0000000..6709828 --- /dev/null +++ b/internal/server/beacon.go @@ -0,0 +1,233 @@ +package server + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "net/netip" + "regexp" + "slices" + "strconv" + "sync" + "time" + + "github.com/Mimic890/hyperhack/internal/content" + "github.com/Mimic890/hyperhack/internal/ratelimit" + "github.com/Mimic890/hyperhack/internal/store" +) + +const ( + maxAPIBody = 4 << 10 + maxViewSeconds = 4 * 60 * 60 // time-on-page clamp, applied to the sum of a view's deltas + maxOutboundSend = 20 + maxCopiesSend = 100 + maxViews = 100000 // open page views tracked in memory + maxSeen = 200000 // visitor × post entries per day + viewTTL = 24 * time.Hour +) + +var ( + viewIDRe = regexp.MustCompile(`^[0-9a-f]{32}$`) + sources = []string{"telegram", "google", "yandex", "bing", "github", "direct", "internal", "other"} + devices = []string{"mobile", "tablet", "desktop"} + depths = []int{25, 50, 75, 100} +) + +// beacon is one delta from the post script: what happened since its last send. +type beacon struct { + Slug string `json:"slug"` + View string `json:"view"` // random id of this page view, the same in every delta + First bool `json:"first"` // first delta of the view: counts the view + Time int `json:"time"` // visible seconds since the last delta + Depth []int `json:"depth"` // read-depth markers reached since the last delta + Source string `json:"source"` + Device string `json:"device"` + Lang string `json:"lang"` + Theme string `json:"theme"` + Outbound []string `json:"outbound"` // hosts of external links clicked + Copies int `json:"copies"` // copy events inside code blocks +} + +var errBadInput = errors.New("bad input") + +// decodeJSON reads exactly one JSON object with known fields from a small body. +func decodeJSON(w http.ResponseWriter, r *http.Request, v any) (int, error) { + if mt, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")); err != nil || mt != "application/json" { + return http.StatusUnsupportedMediaType, errBadInput + } + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxAPIBody)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + return http.StatusRequestEntityTooLarge, err + } + return http.StatusBadRequest, err + } + if err := dec.Decode(&struct{}{}); err != io.EOF { + return http.StatusBadRequest, errors.New("more than one JSON value") + } + return 0, nil +} + +// validate checks types and ranges. Whether the slug exists is checked by the caller. +func (b *beacon) validate() error { + ok := viewIDRe.MatchString(b.View) && + b.Time >= 0 && b.Time <= maxViewSeconds && + len(b.Depth) <= len(depths) && len(b.Outbound) <= maxOutboundSend && + b.Copies >= 0 && b.Copies <= maxCopiesSend && + enumOK(b.Source, sources) && enumOK(b.Device, devices) && enumOK(b.Lang, langs) && enumOK(b.Theme, themes) + if b.First { + ok = ok && b.Source != "" && b.Device != "" && b.Lang != "" && b.Theme != "" + } + for i, d := range b.Depth { + ok = ok && slices.Contains(depths, d) && !slices.Contains(b.Depth[:i], d) + } + for _, h := range b.Outbound { + ok = ok && h != "" && len(h) <= 253 + } + if !ok { + return errBadInput + } + return nil +} + +func enumOK(v string, allowed []string) bool { return v == "" || slices.Contains(allowed, v) } + +// viewState is the running sum of one page view. +type viewState struct { + slug string + seconds int + created time.Time +} + +// visitors turns beacons into counter increments: views and depth once per visitor +// per post per day (daily HMAC hash), time summed and clamped per page view. +// Nothing here is written to disk; the daily key is dropped at midnight UTC. +type visitors struct { + secret []byte + now func() time.Time + + mu sync.Mutex + day string + key []byte // HMAC(VISITOR_SECRET, day) + seen map[[16]byte]uint8 // hash(visitor, slug) → bit 0 viewed, bits 1–4 depth markers + views map[string]*viewState // page view id → running sum + lastPurge time.Time + fullWarn func(what string) // called once per day when a map is full + warned map[string]bool +} + +func newVisitors(secret string, warn func(string)) *visitors { + return &visitors{secret: []byte(secret), now: time.Now, views: map[string]*viewState{}, fullWarn: warn} +} + +// visitorKey is HMAC(daily_key, ip_prefix + user_agent) bound to the post. Caller holds mu. +func (v *visitors) visitorKey(ip netip.Addr, ua, slug string) [16]byte { + m := hmac.New(sha256.New, v.key) + m.Write([]byte(ratelimit.Key(ip).String())) + m.Write([]byte{0}) + m.Write([]byte(ua)) + visitor := m.Sum(nil) + h := sha256.New() + h.Write(visitor) + h.Write([]byte(slug)) + var k [16]byte + copy(k[:], h.Sum(nil)) + return k +} + +func (v *visitors) rollDay(now time.Time) { + day := store.Day(now) + if day == v.day { + return + } + m := hmac.New(sha256.New, v.secret) + m.Write([]byte(day)) + v.day, v.key, v.seen, v.warned = day, m.Sum(nil), map[[16]byte]uint8{}, map[string]bool{} +} + +func (v *visitors) warnFull(what string) { + if !v.warned[what] { + v.warned[what] = true + if v.fullWarn != nil { + v.fullWarn(what) + } + } +} + +// apply returns the counter increments for one validated beacon on post p. +func (v *visitors) apply(b *beacon, p *content.Post, ip netip.Addr, ua string) (day string, incs []store.Inc) { + now := v.now() + v.mu.Lock() + defer v.mu.Unlock() + v.rollDay(now) + if now.Sub(v.lastPurge) > time.Minute { + for id, vs := range v.views { + if now.Sub(vs.created) > viewTTL { + delete(v.views, id) + } + } + v.lastPurge = now + } + + vs := v.views[b.View] + switch { + case vs == nil && !b.First: + return v.day, nil // unknown view (expired, restart): a view starts only with its first delta + case vs == nil && len(v.views) >= maxViews: + v.warnFull("page views") + return v.day, nil + case vs == nil: + vs = &viewState{slug: p.Slug, created: now} + v.views[b.View] = vs + case vs.slug != p.Slug || b.First: + return v.day, nil // id reused for another post, or a repeated first delta + } + inc := func(metric, label string, n int) { + incs = append(incs, store.Inc{Slug: p.Slug, Metric: metric, Label: label, N: int64(n)}) + } + + k := v.visitorKey(ip, ua, p.Slug) + mask, known := v.seen[k] + if !known && len(v.seen) >= maxSeen { + v.warnFull("visitors") + } else { + if b.First && mask&1 == 0 { + mask |= 1 + inc(store.View, "", 1) + inc(store.Source, b.Source, 1) + inc(store.Device, b.Device, 1) + inc(store.Lang, b.Lang, 1) + inc(store.Theme, b.Theme, 1) + } + for _, d := range b.Depth { + if bit := uint8(2) << slices.Index(depths, d); mask&bit == 0 { + mask |= bit + inc(store.Depth, strconv.Itoa(d), 1) + } + } + v.seen[k] = mask + } + + if add := min(b.Time, maxViewSeconds-vs.seconds); add > 0 { + if vs.seconds == 0 { + inc(store.ReadView, "", 1) // views with reading time: the divisor of the average + } + vs.seconds += add + inc(store.ReadSecs, "", add) + } + for _, h := range b.Outbound { + if slices.Contains(p.Outbound, h) { // labels stay bounded by the post's own links + inc(store.Outbound, h, 1) + } + } + if b.Copies > 0 { + inc(store.Copy, "", b.Copies) + } + return v.day, incs +} diff --git a/internal/server/beacon_test.go b/internal/server/beacon_test.go new file mode 100644 index 0000000..8ea5874 --- /dev/null +++ b/internal/server/beacon_test.go @@ -0,0 +1,170 @@ +package server + +import ( + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" + + "github.com/Mimic890/hyperhack/internal/content" + "github.com/Mimic890/hyperhack/internal/store" +) + +const view1, view2 = "0123456789abcdef0123456789abcdef", "fedcba9876543210fedcba9876543210" + +func sum(incs []store.Inc, metric, label string) int64 { + var n int64 + for _, in := range incs { + if in.Metric == metric && in.Label == label { + n += in.N + } + } + return n +} + +func firstBeacon(view string) *beacon { + return &beacon{Slug: "p", View: view, First: true, Source: "direct", Device: "desktop", Lang: "en", Theme: "dark"} +} + +func TestVisitorsPerViewSum(t *testing.T) { + now := time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC) + v := newVisitors("visitor-secret-visitor-secret-32b", nil) + v.now = func() time.Time { return now } + p := &content.Post{Slug: "p", Outbound: []string{"github.com"}} + ip, ua := netip.MustParseAddr("192.0.2.7"), "UA" + total := func(b *beacon, metric string) int64 { + _, incs := v.apply(b, p, ip, ua) + return sum(incs, metric, "") + } + + b := firstBeacon(view1) + b.Time = 3 * 3600 + if _, incs := v.apply(b, p, ip, ua); sum(incs, store.View, "") != 1 || sum(incs, store.ReadSecs, "") != 3*3600 { + t.Fatalf("first delta = %+v", incs) + } + // Second delta of the same view: 2 h more, but the view's sum is clamped at 4 h. + if got := total(&beacon{Slug: "p", View: view1, Time: 7200}, store.ReadSecs); got != 3600 { + t.Errorf("clamped delta = %d, want 3600", got) + } + if got := total(&beacon{Slug: "p", View: view1, Time: 100}, store.ReadSecs); got != 0 { + t.Errorf("past the clamp = %d, want 0", got) + } + // A new view of the same visitor: time counts, the view does not (daily dedup). + _, incs := v.apply(func() *beacon { b := firstBeacon(view2); b.Time = 60; return b }(), p, ip, ua) + if sum(incs, store.View, "") != 0 || sum(incs, store.ReadSecs, "") != 60 || sum(incs, store.ReadView, "") != 1 { + t.Errorf("second view incs = %+v", incs) + } + // Unknown view id without first, repeated first, other post: ignored. + for _, b := range []*beacon{{Slug: "p", View: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Time: 5}, firstBeacon(view1)} { + if _, incs := v.apply(b, p, ip, ua); incs != nil { + t.Errorf("ignored beacon gave %+v", incs) + } + } + if _, incs := v.apply(&beacon{Slug: "q", View: view1, Time: 5}, &content.Post{Slug: "q"}, ip, ua); incs != nil { + t.Errorf("view id reused for another post: %+v", incs) + } + // Depth once per visitor and post; outbound only to the post's own links. + _, incs = v.apply(&beacon{Slug: "p", View: view2, Depth: []int{25, 50}, Outbound: []string{"github.com", "evil.example"}, Copies: 2}, p, ip, ua) + if sum(incs, store.Depth, "25") != 1 || sum(incs, store.Outbound, "github.com") != 1 || sum(incs, store.Outbound, "evil.example") != 0 || sum(incs, store.Copy, "") != 2 { + t.Errorf("incs = %+v", incs) + } + _, incs = v.apply(&beacon{Slug: "p", View: view1, Depth: []int{25, 100}}, p, ip, ua) + if sum(incs, store.Depth, "25") != 0 || sum(incs, store.Depth, "100") != 1 { + t.Errorf("depth dedup = %+v", incs) + } + // Next day: new daily key, the same visitor counts again. + now = now.Add(24 * time.Hour) + _, incs = v.apply(firstBeacon("11111111111111111111111111111111"), p, ip, ua) + if sum(incs, store.View, "") != 1 { + t.Errorf("next day view = %+v", incs) + } + // Another visitor (other /64) counts the same day. + _, incs = v.apply(firstBeacon("22222222222222222222222222222222"), p, netip.MustParseAddr("2001:db8::1"), ua) + if sum(incs, store.View, "") != 1 { + t.Errorf("other visitor = %+v", incs) + } +} + +func TestVisitorsFirstCountsView(t *testing.T) { + v := newVisitors("visitor-secret-visitor-secret-32b", nil) + p := &content.Post{Slug: "p"} + _, incs := v.apply(firstBeacon(view1), p, netip.MustParseAddr("192.0.2.1"), "UA") + if sum(incs, store.View, "") != 1 || sum(incs, store.Source, "direct") != 1 || sum(incs, store.Theme, "dark") != 1 { + t.Errorf("first = %+v", incs) + } +} + +func TestBeaconValidate(t *testing.T) { + ok := []string{ + `{"slug":"p","view":"` + view1 + `","first":true,"time":5,"depth":[25],"source":"telegram","device":"mobile","lang":"ru","theme":"light"}`, + `{"slug":"p","view":"` + view1 + `","time":14400,"outbound":["github.com"],"copies":100}`, + } + bad := []string{ + `{"slug":"p","view":"short"}`, + `{"slug":"p","view":"` + view1 + `","time":-1}`, + `{"slug":"p","view":"` + view1 + `","time":14401}`, + `{"slug":"p","view":"` + view1 + `","depth":[30]}`, + `{"slug":"p","view":"` + view1 + `","depth":[25,25]}`, + `{"slug":"p","view":"` + view1 + `","source":"facebook"}`, + `{"slug":"p","view":"` + view1 + `","first":true}`, + `{"slug":"p","view":"` + view1 + `","copies":101}`, + `{"slug":"p","view":"` + view1 + `","extra":1}`, + `{"slug":"p","view":"` + view1 + `"}{}`, + `[]`, + `{"slug":"p","view":"` + view1 + `","outbound":["` + strings.Repeat("a", 254) + `"]}`, + } + for _, s := range ok { + if status, err := decodeBeacon(s, "application/json"); status != 0 || err != nil { + t.Errorf("rejected %s: %d %v", s, status, err) + } + } + for _, s := range bad { + if status, err := decodeBeacon(s, "application/json"); err == nil { + t.Errorf("accepted %s (%d)", s, status) + } + } + if status, _ := decodeBeacon(ok[0], "text/plain"); status != 415 { + t.Errorf("text/plain = %d", status) + } + if status, _ := decodeBeacon(`{"slug":"`+strings.Repeat("a", 5000)+`"}`, "application/json; charset=utf-8"); status != 413 { + t.Errorf("big body = %d", status) + } +} + +func decodeBeacon(body, ctype string) (int, error) { + r := httptest.NewRequest("POST", "/api/beacon", strings.NewReader(body)) + r.Header.Set("Content-Type", ctype) + var b beacon + if status, err := decodeJSON(httptest.NewRecorder(), r, &b); err != nil { + return status, err + } + return 0, b.validate() +} + +func FuzzBeacon(f *testing.F) { + f.Add(`{"slug":"p","view":"` + view1 + `","first":true,"time":5,"depth":[25,100],"source":"direct","device":"tablet","lang":"en","theme":"dark","outbound":["github.com"],"copies":1}`) + f.Add(`{"slug":"p","view":"` + view1 + `","time":99999999999999999999}`) + f.Add(`{"depth":[25,50,75,100,25]}`) + v := newVisitors("visitor-secret-visitor-secret-32b", nil) + p := &content.Post{Slug: "p", Outbound: []string{"github.com"}} + f.Fuzz(func(t *testing.T, body string) { + r := httptest.NewRequest("POST", "/api/beacon", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + var b beacon + if _, err := decodeJSON(httptest.NewRecorder(), r, &b); err != nil || b.validate() != nil { + return + } + _, incs := v.apply(&b, p, netip.MustParseAddr("192.0.2.1"), "UA") + for _, in := range incs { + if in.N < 0 || (in.Metric == store.ReadSecs && in.N > maxViewSeconds) { + t.Fatalf("bad increment %+v from %s", in, body) + } + } + for _, vs := range v.views { + if vs.seconds > maxViewSeconds { + t.Fatalf("view sum %d over the clamp", vs.seconds) + } + } + }) +} diff --git a/internal/server/metrics.go b/internal/server/metrics.go new file mode 100644 index 0000000..289b282 --- /dev/null +++ b/internal/server/metrics.go @@ -0,0 +1,269 @@ +package server + +import ( + "bytes" + "fmt" + "net/http" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/Mimic890/hyperhack/internal/store" +) + +const ( + topQueries = 20 // top-N search queries exposed as gauges + topsRefresh = 5 * time.Minute // how often the top-N lists are re-read +) + +var latencyBuckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10} + +type histogram struct { + counts []uint64 // per bucket, not cumulative + sum float64 + n uint64 +} + +func (h *histogram) observe(v float64) { + if h.counts == nil { + h.counts = make([]uint64, len(latencyBuckets)) + } + if i := sort.SearchFloat64s(latencyBuckets, v); i < len(latencyBuckets) { + h.counts[i]++ + } + h.sum += v + h.n++ +} + +type reqKey struct { + route string + code int +} + +// registry holds the in-memory metrics. Labels are bounded: routes are mux +// patterns, codes are HTTP statuses. +type registry struct { + mu sync.Mutex + reqs map[reqKey]uint64 + dur map[string]*histogram + search histogram + queries uint64 + zero uint64 + tops [2][]store.QueryCount // all, zero-result + topsLoaded time.Time +} + +func newRegistry() *registry { + return ®istry{reqs: map[reqKey]uint64{}, dur: map[string]*histogram{}} +} + +func (m *registry) request(pattern string, code int, d time.Duration) { + if pattern == "" { + pattern = "unmatched" + } + m.mu.Lock() + defer m.mu.Unlock() + m.reqs[reqKey{pattern, code}]++ + h := m.dur[pattern] + if h == nil { + h = &histogram{} + m.dur[pattern] = h + } + h.observe(d.Seconds()) +} + +func (m *registry) searchDone(d time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.search.observe(d.Seconds()) +} + +func (m *registry) searchQuery(zero bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.queries++ + if zero { + m.zero++ + } +} + +// expo writes Prometheus text format. +type expo struct{ bytes.Buffer } + +func (e *expo) family(name, typ, help string) { + fmt.Fprintf(e, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ) +} + +// sample writes one line; labels are name, value pairs. +func (e *expo) sample(name string, v float64, labels ...string) { + e.WriteString(name) + if len(labels) > 0 { + e.WriteByte('{') + for i := 0; i+1 < len(labels); i += 2 { + if i > 0 { + e.WriteByte(',') + } + e.WriteString(labels[i] + `="` + escapeLabel(labels[i+1]) + `"`) + } + e.WriteByte('}') + } + e.WriteString(" " + strconv.FormatFloat(v, 'g', -1, 64) + "\n") +} + +func (e *expo) histogram(name string, h *histogram, labels ...string) { + var cum uint64 + for i, b := range latencyBuckets { + if h.counts != nil { + cum += h.counts[i] + } + e.sample(name+"_bucket", float64(cum), append(slices.Clone(labels), "le", strconv.FormatFloat(b, 'g', -1, 64))...) + } + e.sample(name+"_bucket", float64(h.n), append(slices.Clone(labels), "le", "+Inf")...) + e.sample(name+"_sum", h.sum, labels...) + e.sample(name+"_count", float64(h.n), labels...) +} + +var labelEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`) + +func escapeLabel(s string) string { return labelEscaper.Replace(s) } + +// postFamilies maps store metrics to exposed counters and their label name. +var postFamilies = []struct{ metric, name, label, help string }{ + {store.View, "hyperhack_post_views_total", "", "Post views, one per visitor per post per day."}, + {store.ReadSecs, "hyperhack_post_read_seconds_total", "", "Visible reading time; each view's sum is clamped to 4 h."}, + {store.ReadView, "hyperhack_post_read_views_total", "", "Views that reported reading time (divisor for the average)."}, + {store.Depth, "hyperhack_post_read_depth_total", "depth", "Readers who reached a share of the text."}, + {store.Source, "hyperhack_post_source_total", "source", "Views by traffic source."}, + {store.Device, "hyperhack_post_device_total", "device", "Views by device class."}, + {store.Lang, "hyperhack_post_ui_lang_total", "lang", "Views by UI language."}, + {store.Theme, "hyperhack_post_ui_theme_total", "theme", "Views by theme."}, + {store.Outbound, "hyperhack_post_outbound_clicks_total", "host", "Clicks on external links, by host."}, + {store.Copy, "hyperhack_post_code_copies_total", "", "Copies from code blocks."}, + {store.VoteCast, "hyperhack_post_votes_total", "vote", "Likes and dislikes cast."}, +} + +// metricsPage handles GET /metrics: private network only (METRICS_ALLOW). +func (p *pages) metricsPage(w http.ResponseWriter, r *http.Request) { + if !contains(p.cfg.MetricsAllow, clientIP(r, p.cfg.TrustedProxies)) { + p.mlog.Warn("metrics denied", "reason", "address not in METRICS_ALLOW") + apiError(w, http.StatusForbidden) + return + } + ctx := r.Context() + var e expo + + // Post metrics: only posts in the current index, so labels stay bounded. + if ix := p.index(); ix != nil { + totals, err := p.store.Totals(ctx) + if err != nil { + apiError(w, http.StatusServiceUnavailable) + return + } + for _, f := range postFamilies { + e.family(f.name, "counter", f.help) + for _, t := range totals { + post := ix.Get(t.Slug) + if t.Metric != f.metric || post == nil || (f.metric == store.Outbound && !slices.Contains(post.Outbound, t.Label)) { + continue + } + if f.label == "" { + e.sample(f.name, float64(t.N), "slug", t.Slug) + } else { + e.sample(f.name, float64(t.N), "slug", t.Slug, f.label, t.Label) + } + } + } + votes, err := p.store.VoteCounts(ctx) + if err != nil { + apiError(w, http.StatusServiceUnavailable) + return + } + e.family("hyperhack_post_likes", "gauge", "Current likes.") + for _, post := range ix.Posts() { + e.sample("hyperhack_post_likes", float64(votes[post.Slug].Likes), "slug", post.Slug) + } + e.family("hyperhack_post_dislikes", "gauge", "Current dislikes.") + for _, post := range ix.Posts() { + e.sample("hyperhack_post_dislikes", float64(votes[post.Slug].Dislikes), "slug", post.Slug) + } + e.family("hyperhack_posts", "gauge", "Published posts in the index.") + e.sample("hyperhack_posts", float64(len(ix.Posts()))) + } + + if p.syncStats != nil { + st := p.syncStats() + e.family("hyperhack_sync_last_attempt_timestamp_seconds", "gauge", "Start of the last sync.") + e.sample("hyperhack_sync_last_attempt_timestamp_seconds", float64(st.LastAttempt)) + e.family("hyperhack_sync_last_success_timestamp_seconds", "gauge", "End of the last successful sync.") + e.sample("hyperhack_sync_last_success_timestamp_seconds", float64(st.LastSuccess)) + e.family("hyperhack_sync_errors_total", "counter", "Failed syncs.") + e.sample("hyperhack_sync_errors_total", float64(st.Errors)) + e.family("hyperhack_sync_fetch_failures_total", "counter", "Objects that failed to download.") + e.sample("hyperhack_sync_fetch_failures_total", float64(st.FetchFailures)) + e.family("hyperhack_sync_objects", "gauge", "Keys in the last bucket listing.") + e.sample("hyperhack_sync_objects", float64(st.Objects)) + } + + p.refreshTops(r) + m := p.metrics + m.mu.Lock() + e.family("hyperhack_search_queries_total", "counter", "Settled search queries.") + e.sample("hyperhack_search_queries_total", float64(m.queries)) + e.family("hyperhack_search_zero_results_total", "counter", "Settled search queries with no results.") + e.sample("hyperhack_search_zero_results_total", float64(m.zero)) + e.family("hyperhack_search_duration_seconds", "histogram", "Search latency.") + e.histogram("hyperhack_search_duration_seconds", &m.search) + for i, name := range []string{"hyperhack_search_top_queries", "hyperhack_search_top_zero_result_queries"} { + e.family(name, "gauge", "Top stored search queries (normalized) and their count.") + for _, q := range m.tops[i] { + e.sample(name, float64(q.N), "query", q.Query) + } + } + e.family("hyperhack_http_requests_total", "counter", "HTTP requests by route pattern and status.") + keys := make([]reqKey, 0, len(m.reqs)) + for k := range m.reqs { + keys = append(keys, k) + } + slices.SortFunc(keys, func(a, b reqKey) int { + return strings.Compare(a.route+strconv.Itoa(a.code), b.route+strconv.Itoa(b.code)) + }) + for _, k := range keys { + e.sample("hyperhack_http_requests_total", float64(m.reqs[k]), "route", k.route, "code", strconv.Itoa(k.code)) + } + e.family("hyperhack_http_request_duration_seconds", "histogram", "HTTP latency by route pattern.") + routes := make([]string, 0, len(m.dur)) + for k := range m.dur { + routes = append(routes, k) + } + slices.Sort(routes) + for _, route := range routes { + e.histogram("hyperhack_http_request_duration_seconds", m.dur[route], "route", route) + } + m.mu.Unlock() + + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(e.Bytes()) +} + +// refreshTops re-reads the top search queries at most every topsRefresh. +func (p *pages) refreshTops(r *http.Request) { + m := p.metrics + m.mu.Lock() + fresh := time.Since(m.topsLoaded) < topsRefresh + m.mu.Unlock() + if fresh { + return + } + all, err1 := p.store.TopSearches(r.Context(), topQueries, false) + zero, err2 := p.store.TopSearches(r.Context(), topQueries, true) + if err1 != nil || err2 != nil { + return // logged by store; keep the old lists + } + m.mu.Lock() + m.tops, m.topsLoaded = [2][]store.QueryCount{all, zero}, time.Now() + m.mu.Unlock() +} diff --git a/internal/server/pages.go b/internal/server/pages.go index f1bf685..aa63a2a 100644 --- a/internal/server/pages.go +++ b/internal/server/pages.go @@ -2,6 +2,7 @@ package server import ( "bytes" + "context" "crypto/sha256" "embed" "encoding/hex" @@ -23,6 +24,9 @@ import ( "github.com/Mimic890/hyperhack/internal/config" "github.com/Mimic890/hyperhack/internal/content" + "github.com/Mimic890/hyperhack/internal/ratelimit" + "github.com/Mimic890/hyperhack/internal/store" + "github.com/Mimic890/hyperhack/internal/syncer" ) //go:embed web @@ -50,7 +54,7 @@ type pageData struct { CSS string Site site - JS string // search.js URL; empty on pages without the script + JS []string // versioned script URLs for this page // metadata. Desc, Canonical and Image are absolute or empty. Desc, Canonical, Image string @@ -59,10 +63,14 @@ type pageData struct { JSONLD template.JS // from json.Marshal Posts []*content.Post + Popular []*content.Post Page, Pages int PageBase string // pager URL for page 1; other pages add ?page=N - Post *content.Post + Post *content.Post + Views int64 + Votes store.Votes + MyVote int // store.Like, store.Dislike or store.None Tag string // tag page, or the search filter Tags []string // all tags, for the search filter @@ -74,18 +82,35 @@ type pageData struct { } type pages struct { - log *slog.Logger - cfg *config.Config - index func() *content.Index - tmpl map[string]map[string]*template.Template // lang → page → template - css []byte - cssV string - js []byte - jsV string - fonts fs.FS + log *slog.Logger // component http + vlog *slog.Logger // component votes + mlog *slog.Logger // component metrics + cfg *config.Config + index func() *content.Index + store *store.Store + syncStats func() syncer.Stats + tmpl map[string]map[string]*template.Template // lang → page → template + css []byte + cssV string + scripts map[string]script // file name → script + fonts fs.FS + + origin string // SITE_URL origin, for the Origin check + lim limits + altcha *altchaGuard + visitors *visitors + metrics *registry } -func newPages(log *slog.Logger, cfg *config.Config, index func() *content.Index) *pages { +type script struct { + body []byte + url string // versioned +} + +type limits struct{ vote, beacon, search, reindex *ratelimit.Limiter } + +// newPages builds the handlers. log is the base logger; components are added here. +func newPages(log *slog.Logger, cfg *config.Config, d Deps) *pages { msgs := map[string]map[string]string{} for _, l := range langs { b, err := webFS.ReadFile("web/i18n/" + l + ".json") @@ -99,7 +124,21 @@ func newPages(log *slog.Logger, cfg *config.Config, index func() *content.Index) msgs[l] = m } - p := &pages{log: log, cfg: cfg, index: index, tmpl: map[string]map[string]*template.Template{}} + p := &pages{ + log: log.With("component", "http"), vlog: log.With("component", "votes"), mlog: log.With("component", "metrics"), + cfg: cfg, index: d.Index, store: d.Store, syncStats: d.SyncStats, + tmpl: map[string]map[string]*template.Template{}, + origin: cfg.SiteURL.Scheme + "://" + cfg.SiteURL.Host, + lim: limits{ + vote: ratelimit.New(cfg.RateVote.N, cfg.RateVote.Per), beacon: ratelimit.New(cfg.RateBeacon.N, cfg.RateBeacon.Per), + search: ratelimit.New(cfg.RateSearch.N, cfg.RateSearch.Per), reindex: ratelimit.New(cfg.RateReindex.N, cfg.RateReindex.Per), + }, + altcha: newAltchaGuard(cfg.AltchaHMACKey, cfg.AltchaMaxNumber, cfg.AltchaExpires), + metrics: newRegistry(), + } + p.visitors = newVisitors(cfg.VisitorSecret, func(what string) { + p.mlog.Warn("view tracking full for today, new entries are not counted", "map", what) + }) for _, l := range langs { p.tmpl[l] = map[string]*template.Template{} for _, name := range []string{"home", "post", "error", "search", "tag"} { @@ -109,16 +148,20 @@ func newPages(log *slog.Logger, cfg *config.Config, index func() *content.Index) } } - base, err := webFS.ReadFile("web/static/style.css") + css, err := webFS.ReadFile("web/static/style.css") if err != nil { panic(err) } - p.css = append(append(base, tagCSS()...), themedChromaCSS()...) + p.css = append(append(css, tagCSS()...), themedChromaCSS()...) p.cssV = versioned("/static/style.css", p.css) - if p.js, err = webFS.ReadFile("web/static/search.js"); err != nil { - panic(err) + p.scripts = map[string]script{} + for _, name := range []string{"search.js", "altcha.js", "post.js"} { + b, err := webFS.ReadFile("web/static/" + name) + if err != nil { + panic(err) + } + p.scripts[name] = script{body: b, url: versioned("/static/"+name, b)} } - p.jsV = versioned("/static/search.js", p.js) p.fonts, err = fs.Sub(webFS, "web/static/fonts") if err != nil { panic(err) @@ -272,9 +315,12 @@ func (p *pages) render(w http.ResponseWriter, status int, name string, d pageDat func (p *pages) errorPage(w http.ResponseWriter, r *http.Request, status int) { d := p.data(r) d.Title, d.Text = "not_found", "not_found_text" - if status == http.StatusServiceUnavailable { + switch status { + case http.StatusServiceUnavailable: d.Title, d.Text = "not_ready", "not_ready_text" w.Header().Set("Retry-After", "30") + case http.StatusTooManyRequests: // Retry-After is set by limit + d.Title, d.Text = "too_many", "too_many_text" } p.render(w, status, "error", d) } @@ -291,9 +337,24 @@ func (p *pages) home(w http.ResponseWriter, r *http.Request) { return } d.Tags = ix.Tags() + if d.Page == 1 { + d.Popular = p.popular(r.Context(), ix) + } p.render(w, http.StatusOK, "home", d) } +// popular is the top POPULAR_LIMIT posts by views in the last POPULAR_DAYS days (today included). +func (p *pages) popular(ctx context.Context, ix *content.Index) []*content.Post { + slugs, _ := p.store.Popular(ctx, store.Day(time.Now().AddDate(0, 0, 1-p.cfg.PopularDays))) // error logged by store + var out []*content.Post + for _, s := range slugs { + if post := ix.Get(s); post != nil && len(out) < p.cfg.PopularLimit { + out = append(out, post) + } + } + return out +} + // tag handles /tags/{tag...}: posts with this tag, paginated. func (p *pages) tag(w http.ResponseWriter, r *http.Request) { ix := p.index() @@ -313,15 +374,19 @@ func (p *pages) tag(w http.ResponseWriter, r *http.Request) { } // search handles GET /search?q=&tag=&page=. It works without JS; search.js swaps #results. -// shortcut: no RATE_LIMIT_SEARCH yet; it comes with the rate limiter in stage 5. +// Live requests from search.js (X-Search: live) are not stored as queries, so typing +// "s", "se", "sel" does not fill the stats; the settled query is sent once as "final". func (p *pages) search(w http.ResponseWriter, r *http.Request) { + if p.limit(p.lim.search, "search")(w, r) { + return + } ix := p.index() if ix == nil { p.errorPage(w, r, http.StatusServiceUnavailable) return } d := p.data(r) - d.NoIndex, d.JS, d.Tags = true, p.jsV, ix.Tags() + d.NoIndex, d.JS, d.Tags = true, []string{p.scripts["search.js"].url}, ix.Tags() q := r.URL.Query() d.Query = q.Get("q") if rs := []rune(d.Query); len(rs) > content.MaxQueryRunes { @@ -336,7 +401,13 @@ func (p *pages) search(w http.ResponseWriter, r *http.Request) { return } size := p.cfg.PageSize + start := time.Now() hits, total, err := ix.Search(d.Query, d.Tag, d.Page, size) + p.metrics.searchDone(time.Since(start)) + if terms := content.Terms(d.Query); err == nil && len(terms) > 0 && d.Page == 1 && r.Header.Get("X-Search") != "live" { + p.metrics.searchQuery(total == 0) + _ = p.store.LogSearch(r.Context(), store.Day(time.Now()), strings.Join(terms, " "), total == 0) // error logged by store + } if err != nil { p.log.Error("search", "err", err) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) @@ -382,6 +453,13 @@ func (p *pages) post(w http.ResponseWriter, r *http.Request) { d.Image = p.abs(assetPath(post.Cover)) } d.JSONLD = p.jsonLD(post, d.Canonical, d.Image) + d.JS = []string{p.scripts["altcha.js"].url, p.scripts["post.js"].url} + ctx := r.Context() + d.Views, _ = p.store.Views(ctx, post.Slug) // errors are logged by store; the page still renders + d.Votes, _ = p.store.PostVotes(ctx, post.Slug) + if vid := readVID(r); vid != "" { + d.MyVote, _ = p.store.VoteOf(ctx, vidHash(vid), post.Slug) + } p.render(w, http.StatusOK, "post", d) } @@ -474,10 +552,33 @@ func localPath(back string) string { // assetPath is the escaped URL path of a vault asset key. func assetPath(key string) string { return (&url.URL{Path: "/" + key}).EscapedPath() } -func (p *pages) script(w http.ResponseWriter, _ *http.Request) { +func (p *pages) script(w http.ResponseWriter, r *http.Request) { + s, ok := p.scripts[r.PathValue("script")] + if !ok { + p.errorPage(w, r, http.StatusNotFound) + return + } w.Header().Set("Content-Type", "text/javascript; charset=utf-8") w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") // URL carries a content hash - _, _ = w.Write(p.js) + _, _ = w.Write(s.body) +} + +// limit returns a check that answers 429 with Retry-After when the client is over l. +func (p *pages) limit(l *ratelimit.Limiter, endpoint string) func(http.ResponseWriter, *http.Request) bool { + return func(w http.ResponseWriter, r *http.Request) bool { + ok, wait := l.Allow(clientIP(r, p.cfg.TrustedProxies)) + if ok { + return false + } + p.log.Warn("rate limit hit", "endpoint", endpoint) // never the client address + w.Header().Set("Retry-After", strconv.Itoa(ratelimit.RetryAfter(wait))) + if endpoint == "search" { + p.errorPage(w, r, http.StatusTooManyRequests) + } else { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + } + return true + } } func (p *pages) style(w http.ResponseWriter, _ *http.Request) { diff --git a/internal/server/pages_test.go b/internal/server/pages_test.go index 2bd298c..c91658d 100644 --- a/internal/server/pages_test.go +++ b/internal/server/pages_test.go @@ -16,11 +16,33 @@ import ( "github.com/Mimic890/hyperhack/internal/config" "github.com/Mimic890/hyperhack/internal/content" + "github.com/Mimic890/hyperhack/internal/store" + "github.com/Mimic890/hyperhack/internal/syncer" ) func testConfig() *config.Config { u, _ := url.Parse("https://example.com") - return &config.Config{SiteName: "Test ", SiteURL: u, DefaultLang: "en", ReadingWPM: 200, PageSize: 2} + private, _ := config.ParsePrefixes("127.0.0.0/8,10.0.0.0/8,192.0.2.0/24") + lots := config.Rate{N: 1000, Per: time.Minute} + return &config.Config{ + SiteName: "Test ", SiteURL: u, DefaultLang: "en", ReadingWPM: 200, PageSize: 2, + AltchaHMACKey: "0123456789abcdef0123456789abcdef", VisitorSecret: "visitor-secret-visitor-secret-32b", + AltchaExpires: 5 * time.Minute, AltchaMaxNumber: 1000, PopularDays: 30, PopularLimit: 5, + RateVote: lots, RateBeacon: lots, RateSearch: lots, RateReindex: lots, MetricsAllow: private, + } +} + +// testDeps returns working dependencies with a fresh SQLite store. +func testDeps(t *testing.T, index func() *content.Index) Deps { + t.Helper() + st, err := store.Open(t.TempDir(), slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + ready := new(atomic.Bool) + ready.Store(true) + return Deps{Ready: ready, Index: index, Store: st, SyncStats: func() syncer.Stats { return syncer.Stats{Objects: 3} }} } func testIndex(n int) *content.Index { @@ -42,15 +64,15 @@ func testIndex(n int) *content.Index { type testSite struct { h http.Handler idx atomic.Pointer[content.Index] + d Deps } func newSite(t *testing.T, cfg *config.Config, ix *content.Index) *testSite { t.Helper() s := &testSite{} s.idx.Store(ix) - var ready atomic.Bool - ready.Store(true) - s.h = New(slog.New(slog.DiscardHandler), cfg, &ready, nil, s.idx.Load) + s.d = testDeps(t, s.idx.Load) + s.h = New(slog.New(slog.DiscardHandler), cfg, s.d) return s } diff --git a/internal/server/reindex.go b/internal/server/reindex.go index 453c348..724ccad 100644 --- a/internal/server/reindex.go +++ b/internal/server/reindex.go @@ -14,9 +14,8 @@ import ( // reindexWriteTimeout replaces the server WriteTimeout: a sync can take longer than 30 s. const reindexWriteTimeout = 10 * time.Minute -// reindex handles POST /reindex: private network only, Bearer token, waits for the sync. -// shortcut: no RATE_LIMIT_REINDEX yet; it comes with the rate limiter in stage 5. -func reindex(log *slog.Logger, token string, allow, trusted []netip.Prefix, sync func(context.Context) error) http.HandlerFunc { +// reindex handles POST /reindex: private network only, RATE_LIMIT_REINDEX, Bearer token, waits for the sync. +func reindex(log *slog.Logger, token string, allow, trusted []netip.Prefix, limited func(http.ResponseWriter, *http.Request) bool, sync func(context.Context) error) http.HandlerFunc { want := sha256.Sum256([]byte(token)) return func(w http.ResponseWriter, r *http.Request) { if !contains(allow, clientIP(r, trusted)) { @@ -24,6 +23,9 @@ func reindex(log *slog.Logger, token string, allow, trusted []netip.Prefix, sync http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } + if limited(w, r) { // before the token check: limits guessing + return + } got, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") // Hash both sides so the compare takes the same time for any token length. gotSum := sha256.Sum256([]byte(got)) diff --git a/internal/server/reindex_test.go b/internal/server/reindex_test.go index 048add0..5e4d4a6 100644 --- a/internal/server/reindex_test.go +++ b/internal/server/reindex_test.go @@ -30,7 +30,9 @@ func TestReindex(t *testing.T) { } cfg := testConfig() cfg.ReindexToken, cfg.ReindexAllow, cfg.TrustedProxies = token, allow, trusted - h := New(slog.New(slog.DiscardHandler), cfg, new(atomic.Bool), sync, func() *content.Index { return nil }) + d := testDeps(t, func() *content.Index { return nil }) + d.Sync = sync + h := New(slog.New(slog.DiscardHandler), cfg, d) req := func(remote, xff, auth string) int { r := httptest.NewRequest("POST", "/reindex", nil) diff --git a/internal/server/server.go b/internal/server/server.go index 8b83d17..bfc934d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -13,12 +13,23 @@ import ( "github.com/Mimic890/hyperhack/internal/config" "github.com/Mimic890/hyperhack/internal/content" + "github.com/Mimic890/hyperhack/internal/store" + "github.com/Mimic890/hyperhack/internal/syncer" ) -// New returns the root handler. ready gates /healthz; sync runs a bucket sync for /reindex; -// index returns the current posts (nil before the first load). -func New(log *slog.Logger, cfg *config.Config, ready *atomic.Bool, sync func(context.Context) error, index func() *content.Index) http.Handler { - p := newPages(log, cfg, index) +// Deps is what the handlers need from the rest of the service. +type Deps struct { + Ready *atomic.Bool // gates /healthz + Sync func(context.Context) error // bucket sync for /reindex + Index func() *content.Index // current posts, nil before the first load + Store *store.Store + SyncStats func() syncer.Stats +} + +// New returns the root handler. log is the base logger; components are added here. +func New(log *slog.Logger, cfg *config.Config, d Deps) http.Handler { + p := newPages(log, cfg, d) + ready := d.Ready mux := http.NewServeMux() mux.HandleFunc("GET /{$}", p.home) mux.HandleFunc("GET /posts/{slug}", p.post) // also /posts/.md @@ -35,12 +46,16 @@ func New(log *slog.Logger, cfg *config.Config, ready *atomic.Bool, sync func(con } mux.HandleFunc("POST /prefs", p.prefs) mux.HandleFunc("GET /static/style.css", p.style) - mux.HandleFunc("GET /static/search.js", p.script) + mux.HandleFunc("GET /static/{script}", p.script) + mux.HandleFunc("GET /api/altcha", p.apiAltcha) + mux.HandleFunc("POST /api/vote", p.apiVote) + mux.HandleFunc("POST /api/beacon", p.apiBeacon) + mux.HandleFunc("GET /metrics", p.metricsPage) mux.HandleFunc("GET /static/fonts/{file}", p.font) mux.HandleFunc("GET /", p.notFound) - mux.Handle("POST /reindex", reindex(log, cfg.ReindexToken, cfg.ReindexAllow, cfg.TrustedProxies, sync)) + mux.Handle("POST /reindex", reindex(p.log, cfg.ReindexToken, cfg.ReindexAllow, cfg.TrustedProxies, p.limit(p.lim.reindex, "reindex"), d.Sync)) // The "GET /" catch-all would answer GET on POST-only paths with 404; SPEC wants 405. - for _, path := range []string{"/reindex", "/prefs"} { + for _, path := range []string{"/reindex", "/prefs", "/api/vote", "/api/beacon"} { mux.HandleFunc("GET "+path, func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Allow", http.MethodPost) http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) @@ -55,11 +70,11 @@ func New(log *slog.Logger, cfg *config.Config, ready *atomic.Bool, sync func(con } _, _ = w.Write([]byte("ok\n")) }) - return middleware(log, mux) + return middleware(p.log, p.metrics, mux) } -// middleware sets security headers, recovers panics and logs each request on DEBUG. -func middleware(log *slog.Logger, next http.Handler) http.Handler { +// middleware sets security headers, recovers panics, counts and logs each request on DEBUG. +func middleware(log *slog.Logger, m *registry, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() id := requestID() @@ -74,6 +89,7 @@ func middleware(log *slog.Logger, next http.Handler) http.Handler { http.Error(sw, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } } + m.request(r.Pattern, sw.code(), time.Since(start)) // the mux sets Pattern on r log.Debug("request", "request_id", id, "method", r.Method, "path", r.URL.Path, "status", sw.code(), "duration", time.Since(start)) }() securityHeaders(w.Header()) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index d760527..85e2719 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "os" "regexp" - "sync/atomic" "testing" "github.com/Mimic890/hyperhack/internal/content" @@ -20,8 +19,10 @@ func do(h http.Handler, method, path string) *httptest.ResponseRecorder { } func TestRoutes(t *testing.T) { - var ready atomic.Bool - h := New(slog.New(slog.DiscardHandler), testConfig(), &ready, nil, func() *content.Index { return nil }) + d := testDeps(t, func() *content.Index { return nil }) + ready := d.Ready + ready.Store(false) + h := New(slog.New(slog.DiscardHandler), testConfig(), d) if got := do(h, "GET", "/healthz").Code; got != http.StatusServiceUnavailable { t.Errorf("healthz before ready = %d, want 503", got) @@ -48,7 +49,7 @@ func TestRoutes(t *testing.T) { } func TestPanicRecovery(t *testing.T) { - h := middleware(slog.New(slog.DiscardHandler), http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + h := middleware(slog.New(slog.DiscardHandler), newRegistry(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) { panic("secret internal detail") })) rec := do(h, "GET", "/") @@ -61,7 +62,7 @@ func TestPanicRecovery(t *testing.T) { // TestCSPMatchesSpec keeps the CSP header identical to SPEC → Security headers. func TestCSPMatchesSpec(t *testing.T) { re := regexp.MustCompile("`Content-Security-Policy: (default-src 'self'[^`]*)`") - h := New(slog.New(slog.DiscardHandler), testConfig(), new(atomic.Bool), nil, func() *content.Index { return nil }) + h := New(slog.New(slog.DiscardHandler), testConfig(), testDeps(t, func() *content.Index { return nil })) got := do(h, "GET", "/healthz").Header().Get("Content-Security-Policy") for _, f := range []string{"../../SPEC.md", "../../SPEC.ru.md"} { b, err := os.ReadFile(f) diff --git a/internal/server/web/i18n/en.json b/internal/server/web/i18n/en.json index b33e3ec..5c2afca 100644 --- a/internal/server/web/i18n/en.json +++ b/internal/server/web/i18n/en.json @@ -41,5 +41,12 @@ "no_results": "Nothing found. Try other words.", "results": "Search results", "prev": "Previous", - "next": "Next" + "next": "Next", + "popular": "Popular", + "views": "Views: {n}", + "like": "Like", + "dislike": "Dislike", + "vote_error": "Could not save the vote. Try again.", + "too_many": "Too many requests", + "too_many_text": "Please wait a moment and try again." } diff --git a/internal/server/web/i18n/ru.json b/internal/server/web/i18n/ru.json index dd5cf35..8248a68 100644 --- a/internal/server/web/i18n/ru.json +++ b/internal/server/web/i18n/ru.json @@ -41,5 +41,12 @@ "no_results": "Ничего не найдено. Попробуйте другие слова.", "results": "Результаты поиска", "prev": "Назад", - "next": "Дальше" + "next": "Дальше", + "popular": "Популярное", + "views": "Просмотры: {n}", + "like": "Нравится", + "dislike": "Не нравится", + "vote_error": "Не удалось сохранить голос. Попробуйте ещё раз.", + "too_many": "Слишком много запросов", + "too_many_text": "Подождите немного и попробуйте снова." } diff --git a/internal/server/web/static/altcha.js b/internal/server/web/static/altcha.js new file mode 100644 index 0000000..8d94098 --- /dev/null +++ b/internal/server/web/static/altcha.js @@ -0,0 +1,43 @@ +// ALTCHA v2 solver, the same algorithm as altcha-lib DeriveKeySHA with a uint32 counter: +// key = SHA-256(salt || nonce || counter as big-endian uint32), repeated `cost` times, +// first `keyLength` bytes; solved when the hex key starts with `keyPrefix`. +(() => { + const hex = (b) => Array.from(b, (x) => x.toString(16).padStart(2, "0")).join(""); + const unhex = (s) => new Uint8Array((s.match(/../g) || []).map((h) => parseInt(h, 16))); + + async function derive(data, cost, length) { + let d = await crypto.subtle.digest("SHA-256", data); + for (let i = 1; i < cost; i++) d = await crypto.subtle.digest("SHA-256", d); + return new Uint8Array(d).slice(0, length); + } + + // solve returns the payload for the server: the challenge as received plus the solution. + async function solve(challenge, maxCounter = 10000000) { + const p = challenge.parameters; + if (p.algorithm !== "SHA-256") throw new Error("unsupported algorithm"); + const salt = unhex(p.salt); + const nonce = unhex(p.nonce); + const cost = Math.max(1, p.cost || 1); + const length = p.keyLength > 0 ? p.keyLength : 32; + const prefix = String(p.keyPrefix).toLowerCase(); + const batch = 256; + for (let start = 0; start <= maxCounter; start += batch) { + const jobs = []; + for (let n = start; n < start + batch && n <= maxCounter; n++) { + const data = new Uint8Array(salt.length + nonce.length + 4); + data.set(salt); + data.set(nonce, salt.length); + new DataView(data.buffer).setUint32(data.length - 4, n); + jobs.push(derive(data, cost, length)); + } + const keys = await Promise.all(jobs); + for (let i = 0; i < keys.length; i++) { + const key = hex(keys[i]); + if (key.startsWith(prefix)) return { challenge, solution: { counter: start + i, derivedKey: key } }; + } + } + throw new Error("challenge not solved"); + } + + globalThis.hyperhackAltcha = { solve }; +})(); diff --git a/internal/server/web/static/post.js b/internal/server/web/static/post.js new file mode 100644 index 0000000..f2bfd41 --- /dev/null +++ b/internal/server/web/static/post.js @@ -0,0 +1,135 @@ +// Post page: votes (with ALTCHA) and the metrics beacon. The page works without this script. +(() => { + const article = document.querySelector("article.post[data-slug]"); + if (!article) return; + const slug = article.dataset.slug; + const hex = (b) => Array.from(b, (x) => x.toString(16).padStart(2, "0")).join(""); + const post = (url, body, keepalive) => + fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), keepalive }); + + // Votes: click → challenge → solve → vote. Clicking the pressed button removes the vote. + const votes = article.querySelector(".votes"); + // The solver needs crypto.subtle (secure contexts only); without it the buttons stay disabled. + if (votes && globalThis.hyperhackAltcha && globalThis.crypto?.subtle) { + const buttons = votes.querySelectorAll("button[data-vote]"); + const status = votes.querySelector("[role=status]"); + const busy = (on) => { + votes.setAttribute("aria-busy", String(on)); + buttons.forEach((b) => (b.disabled = on)); + }; + buttons.forEach((btn) => { + btn.disabled = false; + btn.addEventListener("click", async () => { + const vote = btn.getAttribute("aria-pressed") === "true" ? "none" : btn.dataset.vote; + busy(true); + status.textContent = ""; + try { + const r1 = await fetch("/api/altcha"); + if (!r1.ok) throw new Error(String(r1.status)); + const altcha = await globalThis.hyperhackAltcha.solve(await r1.json()); + const r2 = await post("/api/vote", { slug, vote, altcha }, false); + if (!r2.ok) throw new Error(String(r2.status)); + const res = await r2.json(); + buttons.forEach((b) => { + b.setAttribute("aria-pressed", String(b.dataset.vote === res.vote)); + b.querySelector(".n").textContent = b.dataset.vote === "like" ? res.likes : res.dislikes; + }); + } catch { + status.textContent = votes.dataset.error; + } finally { + busy(false); + } + }); + }); + } + + // Beacon: on every hide, send what is new since the last send. Every delta carries + // the same random view id; the server sums time per view and counts the view once. + const view = hex(crypto.getRandomValues(new Uint8Array(16))); + let first = true; + let visibleMs = 0; + let since = document.visibilityState === "visible" ? performance.now() : null; + let copies = 0; + const depth = new Set(); + const outbound = []; + + const source = () => { + if (!document.referrer) return "direct"; + let host; + try { + host = new URL(document.referrer).hostname.toLowerCase(); + } catch { + return "other"; + } + if (host === location.hostname) return "internal"; + const known = [ + ["telegram", /(^|\.)(t\.me|telegram\.(org|me))$/], + ["google", /(^|\.)google\.[a-z.]+$/], + ["yandex", /(^|\.)(yandex\.[a-z.]+|ya\.ru)$/], + ["bing", /(^|\.)bing\.com$/], + ["github", /(^|\.)github\.com$/], + ]; + for (const [name, re] of known) if (re.test(host)) return name; + return "other"; + }; + const device = () => (innerWidth < 768 ? "mobile" : innerWidth < 1024 ? "tablet" : "desktop"); + const theme = () => + document.documentElement.dataset.theme || (matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"); + const lang = () => { + const l = document.querySelector(".topbar")?.lang; + return l === "ru" ? "ru" : "en"; + }; + + const context = { source: source(), device: device(), lang: lang(), theme: theme() }; // as at page load + + if ("IntersectionObserver" in globalThis) { + const io = new IntersectionObserver((entries) => { + for (const e of entries) { + if (!e.isIntersecting) continue; + depth.add(Number(e.target.dataset.depth)); + io.unobserve(e.target); + } + }); + article.querySelectorAll(".depth").forEach((m) => io.observe(m)); + } + + const onClick = (e) => { + const a = e.target.closest && e.target.closest(".prose a[href]"); + if (!a || outbound.length >= 20) return; + try { + const u = new URL(a.href); + if ((u.protocol === "https:" || u.protocol === "http:") && u.hostname !== location.hostname) outbound.push(u.hostname.toLowerCase()); + } catch { + // not a URL + } + }; + article.addEventListener("click", onClick); + article.addEventListener("auxclick", onClick); + document.addEventListener("copy", () => { + const n = document.getSelection()?.anchorNode; + const el = n && (n.nodeType === 1 ? n : n.parentElement); + if (el && el.closest("pre, code") && article.contains(el)) copies++; + }); + + const flush = () => { + if (since !== null) { + const now = performance.now(); + visibleMs += now - since; + since = document.visibilityState === "visible" ? now : null; + } + const time = Math.floor(visibleMs / 1000); + visibleMs -= time * 1000; + if (!first && time === 0 && depth.size === 0 && outbound.length === 0 && copies === 0) return; + const b = { slug, view, first, time: Math.min(time, 14400), depth: [...depth], outbound: outbound.splice(0), copies: Math.min(copies, 100) }; + if (first) Object.assign(b, context); + first = false; + depth.clear(); + copies = 0; + post("/api/beacon", b, true).catch(() => {}); + }; + document.addEventListener("visibilitychange", () => { + if (document.visibilityState === "hidden") flush(); + else since = performance.now(); + }); + addEventListener("pagehide", flush); +})(); diff --git a/internal/server/web/static/search.js b/internal/server/web/static/search.js index cda922c..b791d2c 100644 --- a/internal/server/web/static/search.js +++ b/internal/server/web/static/search.js @@ -4,12 +4,14 @@ const results = document.getElementById("results"); const status = document.getElementById("status"); if (!form || !results || !status) return; - let timer, ctrl; - const run = () => { + let timer, settle, ctrl; + // "live" requests update the results while typing and are not stored as queries. + // After a pause the query is sent once more as "final", so only settled queries count. + const run = (mode) => { const url = "/search?" + new URLSearchParams(new FormData(form)); if (ctrl) ctrl.abort(); ctrl = new AbortController(); - fetch(url, { signal: ctrl.signal }) + fetch(url, { signal: ctrl.signal, headers: { "X-Search": mode } }) .then((r) => (r.ok ? r.text() : Promise.reject(r.status))) .then((html) => { const doc = new DOMParser().parseFromString(html, "text/html"); @@ -24,6 +26,8 @@ }; form.addEventListener("input", () => { clearTimeout(timer); - timer = setTimeout(run, 250); + clearTimeout(settle); + timer = setTimeout(() => run("live"), 250); + settle = setTimeout(() => run("final"), 1500); }); })(); diff --git a/internal/server/web/static/style.css b/internal/server/web/static/style.css index e5d361c..d40263f 100644 --- a/internal/server/web/static/style.css +++ b/internal/server/web/static/style.css @@ -158,7 +158,10 @@ main { max-width: 44rem; margin: 0 auto; padding: 2.5rem 1.25rem 4rem; } .post > header h1 { margin: 0; font-size: clamp(1.875rem, 4vw + 1rem, 2.625rem); line-height: 1.15; font-weight: 700; letter-spacing: -0.025em; } .post > header .meta { margin-top: 0.75rem; } .cover { display: block; width: 100%; margin: 2rem 0 0; border-radius: 12px; } -.prose { margin-top: 2rem; overflow-wrap: break-word; } +.prose { position: relative; margin-top: 2rem; overflow-wrap: break-word; } +/* Read-depth markers for the post script (IntersectionObserver). */ +.depth { position: absolute; left: 0; width: 1px; height: 1px; pointer-events: none; } +.d25 { top: 25%; } .d50 { top: 50%; } .d75 { top: 75%; } .d100 { bottom: 0; } .prose > :first-child { margin-top: 0; } .prose h2, .prose h3, .prose h4 { line-height: 1.3; letter-spacing: -0.015em; margin: 2.25rem 0 0.75rem; scroll-margin-top: 1rem; } .prose h2 { font-size: 1.5rem; font-weight: 650; } @@ -178,6 +181,24 @@ code, pre { font-family: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, .prose pre { overflow-x: auto; padding: 1rem 1.125rem; border: 1px solid var(--border); border-radius: 10px; font-size: 0.875rem; line-height: 1.6; } .prose pre, .prose pre.chroma { background: var(--surface); } +/* Votes: enabled by post.js; without JS the counts are still shown. */ +.votes { display: flex; align-items: center; gap: 0.5rem; margin-top: 2.5rem; padding-top: 1.25rem; border-top: 1px solid var(--border); } +.vote { + display: inline-flex; align-items: center; gap: 0.4rem; height: 2.25rem; padding: 0 0.85rem; + border: 1px solid var(--border); border-radius: 999px; background: var(--bg); color: var(--muted); + font: inherit; font-size: 0.875rem; font-weight: 600; font-variant-numeric: tabular-nums; cursor: pointer; +} +.vote:hover:not(:disabled) { color: var(--text); background: var(--surface); } +.vote[aria-pressed="true"] { border-color: var(--accent); background: var(--accent-soft); color: var(--accent); } +.vote:disabled { cursor: default; } +.votes[aria-busy="true"] .vote { opacity: 0.6; } +.vote-status { margin: 0; } + +/* Popular */ +.popular { margin-top: 2.5rem; } +.popular ol { margin: 0; padding-left: 1.25rem; } +.popular li + li { margin-top: 0.35rem; } + .error h1 { margin: 0 0 0.75rem; font-size: 2rem; letter-spacing: -0.02em; } @media (max-width: 30rem) { diff --git a/internal/server/web/templates/home.html b/internal/server/web/templates/home.html index f4dd834..053b62a 100644 --- a/internal/server/web/templates/home.html +++ b/internal/server/web/templates/home.html @@ -10,4 +10,14 @@

{{t "newest"}}

{{- end}}
{{- template "pager" .}} +{{- with .Popular}} +
+ +
    + {{- range .}} +
  1. {{.Title}}
    2. + {{- end}} +
+
+{{- end}} {{end}} diff --git a/internal/server/web/templates/layout.html b/internal/server/web/templates/layout.html index cdd1121..cb603d7 100644 --- a/internal/server/web/templates/layout.html +++ b/internal/server/web/templates/layout.html @@ -34,7 +34,7 @@ {{- end}} -{{- with .JS}} +{{- range .JS}} {{- end}} diff --git a/internal/server/web/templates/post.html b/internal/server/web/templates/post.html index 2e81de1..0151337 100644 --- a/internal/server/web/templates/post.html +++ b/internal/server/web/templates/post.html @@ -1,12 +1,13 @@ {{define "title"}}{{.Post.Title}} — {{.Site.Name}}{{end}} {{define "main"}} +{{- $d := .}} {{- with .Post}} -
+

{{.Title}}

{{template "date" .Date}} {{- if not .Updated.IsZero}} · {{t "updated" "date" (date .Updated)}}{{end}} - · {{t "min_read" "n" (minutes .Words)}}

+ · {{t "min_read" "n" (minutes .Words)}} · {{t "views" "n" $d.Views}}

{{- template "tags" .Tags}}
{{- if .Cover}} @@ -14,7 +15,13 @@

{{.Title}}

{{- end}}
{{.HTML}} +
+
+ + + +
{{- end}} {{end}} diff --git a/main.go b/main.go index b353114..7481136 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "github.com/Mimic890/hyperhack/internal/content" "github.com/Mimic890/hyperhack/internal/mirror" "github.com/Mimic890/hyperhack/internal/server" + "github.com/Mimic890/hyperhack/internal/store" "github.com/Mimic890/hyperhack/internal/syncer" ) @@ -44,6 +45,12 @@ func run() int { mainLog.Error("open vault dir", "err", err) return 1 } + st, err := store.Open(cfg.DataDir, log) + if err != nil { + mainLog.Error("open SQLite", "err", err) + return 1 + } + defer func() { _ = st.Close() }() bucket, err := syncer.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey) if err != nil { mainLog.Error("S3 client", "err", err) @@ -64,7 +71,7 @@ func run() int { httpLog := log.With("component", "http") srv := &http.Server{ Addr: addr, - Handler: server.New(httpLog, cfg, &ready, syn.Sync, syn.Index), + Handler: server.New(log, cfg, server.Deps{Ready: &ready, Sync: syn.Sync, Index: syn.Index, Store: st, SyncStats: syn.Stats}), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 30 * time.Second, @@ -82,6 +89,7 @@ func run() int { defer stopSync() syncErr := make(chan error, 1) go func() { syncErr <- syn.Run(syncCtx, &ready) }() + go prune(syncCtx, st, cfg.PopularDays) code := 0 select { @@ -110,6 +118,27 @@ func run() int { return code } +// Search query TTL and cap (SPEC → Data: bounded growth). +const ( + searchTTLDays = 90 + maxSearches = 10000 +) + +// prune keeps the SQLite tables bounded: hourly, and once at start. +func prune(ctx context.Context, st *store.Store, popularDays int) { + t := time.NewTicker(time.Hour) + defer t.Stop() + for { + now := time.Now() + _ = st.Prune(ctx, store.Day(now.AddDate(0, 0, -searchTTLDays)), store.Day(now.AddDate(0, 0, -popularDays)), maxSearches) // logged by store + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + func newLogger(cfg *config.Config) *slog.Logger { opts := &slog.HandlerOptions{Level: cfg.LogLevel} if cfg.LogFormat == "text" { From 2fa672407c11fce958d1a32494365fb342b38006 Mon Sep 17 00:00:00 2001 From: Mimic890 <144267059+Mimic890@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:00:00 +0300 Subject: [PATCH 4/4] docs: record metrics, votes and limits decisions in both SPEC files --- SPEC.md | 21 +++++++++++---------- SPEC.ru.md | 21 +++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/SPEC.md b/SPEC.md index a45e5da..c2e36ac 100644 --- a/SPEC.md +++ b/SPEC.md @@ -91,7 +91,7 @@ Proposed variables (final names set during implementation): | `ALTCHA_HMAC_KEY` | ALTCHA secret. | | `READING_WPM` | Words per minute for estimated reading time. Default: `200`. | | `LOG_LEVEL`, `LOG_FORMAT` | Log level and format (`json` / `text`). Defaults: `info`, `json`. | -| `DATA_DIR` | Directory for SQLite. Default: `/data`. | +| `DATA_DIR` | Directory for SQLite (`hyperhack.db`). Default: `/data`. | | `VAULT_DIR` | Directory for the local bucket mirror. Default: `/vault`. | | `RATE_LIMIT_VOTE`, `RATE_LIMIT_BEACON`, `RATE_LIMIT_REINDEX` | Per-client limits on write endpoints. Defaults: `10/1m`, `30/1m`, `5/1m`. | | `POPULAR_DAYS`, `POPULAR_LIMIT` | "Popular" window and size. Defaults: `30`, `5`. | @@ -248,14 +248,14 @@ The service stores per-post metrics only. Storage: SQLite. - Clients without JS (most bots) are not counted. Readers with JS off are not counted either. Accepted. - Public: shown on the post. - Feeds the "Popular" list. -- Repeat views from one visitor are not counted (proposed: hash of IP + User-Agent per day; raw IP is not stored). +- Repeat views from one visitor are not counted: daily visitor hash (see Security → Votes and visitor identity); raw IP is not stored. ### Time on page - The server cannot measure it alone. It needs the client script. -- The script sends active reading time when the page is hidden or closed (`visibilitychange` + `navigator.sendBeacon`). +- The script sends active reading time when the page is hidden or closed (`visibilitychange` / `pagehide`, `fetch` with `keepalive`). - Only time with the tab visible counts. -- Stored per post as an aggregate (e.g. average or median). +- Every delta carries a random per-page-view id. The server sums the deltas of a view and clamps the sum to 4 h, then adds it to the post aggregate: average time = total seconds / views with reading time. - Private. Not shown on the site. Metric only (see Monitoring). ### Estimated reading time @@ -337,7 +337,7 @@ All private: metrics only, not shown on the site. Collected by the same client s |---|---|---| | Views per day per post | script | Trends; base for "Popular". | | Read depth: share of readers who reach 25 / 50 / 75 / 100 % of the text | script (`IntersectionObserver` on markers) | Do readers finish the post? | -| Traffic source per post: `telegram`, `google`, `yandex`, `bing`, `github`, `direct`, `other` | script (`document.referrer` domain, mapped to a fixed list) | Where readers come from. | +| Traffic source per post: `telegram`, `google`, `yandex`, `bing`, `github`, `direct`, `internal` (from this site), `other` | script (`document.referrer` domain, mapped to a fixed list) | Where readers come from. | | Outbound link clicks per post (target domain) | script | Which links readers use. | | Code copies per post | script (`copy` event inside code blocks) | Which code is useful. | | Device class: `mobile` / `tablet` / `desktop` | script (viewport width bucket) | Layout priorities. | @@ -349,7 +349,7 @@ Rules: - Prometheus labels must have bounded values: post slug, fixed source list, fixed device list. Never raw URLs, referrers or queries as labels. - Search queries are stored in SQLite (normalized, lower case). `/metrics` exposes only the top `N` as a gauge, refreshed periodically. -- One beacon request per page view carries all script metrics, sent on `visibilitychange` / `pagehide`. It is rate-limited by `RATE_LIMIT_BEACON`. +- The script sends one beacon request per hide (`visibilitychange` / `pagehide`) with what changed since its last send, tagged with the page-view id. The view is counted once. Rate-limited by `RATE_LIMIT_BEACON`. ## Monitoring @@ -452,7 +452,7 @@ Other methods → `405`. Unknown paths → `404`. - `Content-Security-Policy: default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; worker-src 'self'; object-src 'none'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'; upgrade-insecure-requests`. Add `blob:` to `worker-src` only if the ALTCHA build needs it. - No inline scripts or styles. Theme and language come from cookies and are rendered on the server, so no inline script is needed against a theme flash. - `X-Content-Type-Options: nosniff`, `Referrer-Policy: strict-origin-when-cross-origin`, `Cross-Origin-Opener-Policy: same-origin`, `X-Frame-Options: DENY`, `Permissions-Policy: camera=(), microphone=(), geolocation=(), browsing-topics=()`. -- All JS, CSS, fonts and the ALTCHA widget are self-hosted. No CDN, no third-party requests. +- All JS, CSS, fonts and the ALTCHA solver are self-hosted. No CDN, no third-party requests. ### CSRF and CORS @@ -464,11 +464,11 @@ Other methods → `405`. Unknown paths → `404`. - Vote identity: anonymous random cookie `vid` (128 bit, `HttpOnly; Secure; SameSite=Lax; Max-Age=1 year`). No personal data in it. - One vote per `vid` per post. Clearing the cookie allows a new vote, but each vote costs an ALTCHA solution and hits `RATE_LIMIT_VOTE`. - Why not an IP hash: many readers share one IP (mobile CGNAT), and a daily hash would allow one vote per day. -- View dedup: `HMAC-SHA256(daily_key, ip_prefix + user_agent)`, `daily_key = HMAC(VISITOR_SECRET, date)`. Old daily keys are not stored, so days cannot be linked. Raw IPs are never stored or logged. +- View dedup: `HMAC-SHA256(daily_key, ip_prefix + user_agent)`, `daily_key = HMAC(VISITOR_SECRET, date)`. `ip_prefix`: the IPv4 address or the IPv6 `/64`. Today's hashes live in memory only; old daily keys are not stored, so days cannot be linked. Raw IPs are never stored or logged. ### ALTCHA -- Library: `github.com/altcha-org/altcha-lib-go`. Invisible mode: the browser solves the challenge on click. +- Library: `github.com/altcha-org/altcha-lib-go/v2` (v2 challenges). Invisible: the post script solves the challenge on click with its own small solver (`static/altcha.js`), no widget. - `ALTCHA_HMAC_KEY` ≥ 32 random bytes. - Challenge expiry `ALTCHA_EXPIRES` (default 5 min). Expired → reject. - Replay protection in the service: used challenges stay in memory until expiry; reuse → reject. The library does not track this. @@ -514,7 +514,7 @@ Other methods → `405`. Unknown paths → `404`. ### Container -- Static binary (`CGO_ENABLED=0`, `-trimpath`), pure-Go SQLite driver (e.g. `modernc.org/sqlite`). +- Static binary (`CGO_ENABLED=0`, `-trimpath`), pure-Go SQLite driver `modernc.org/sqlite`. - Runtime image: distroless static non-root, or `scratch`. No shell, no package manager. - Compose example: ```yaml @@ -615,3 +615,4 @@ Other methods → `405`. Unknown paths → `404`. - 2026-11-03 — Sync: S3 client `minio-go/v7`, frontmatter via `go.yaml.in/yaml/v3`, GFM. Mirror state in `VAULT_DIR/.hyperhack.json`. S3 down at startup: serve from the mirror; exit 1 only with no mirror. No Remotely Save prefix. Keys with a `.` or `_` segment are ignored (was proposed). `MAX_OBJECTS` counts all keys. An invalid frontmatter field skips the post. 3 S3 attempts per poll. `/reindex` waits for the sync and returns `204`. - 2026-11-04 — Pages: theme and language switch via a no-JS form `POST /prefs`; cookies `theme` / `lang` (`HttpOnly; SameSite=Lax; Max-Age=1 year`, `Secure` on https). Logo from `assets/logo.svg`, built-in mark otherwise. Fonts Inter + JetBrains Mono, self-hosted. Pagination `/?page=N`: page 1 is `/`, other out-of-range values → `404`. `` from the post; the top bar keeps the UI language. CSS and fonts at `/static/*`. - 2026-11-04 — Search, tags, RSS, SEO: Bleve `v2.6.1`, in-memory index, one document mapping per language (`en`, `ru`). Fuzziness by word length: ≤ 3 letters 0, ≤ 6 letters 1, longer 2; prefix on the last word. Snippets keep only ``. Search works without JS; `search.js` swaps the results while typing (debounced). Results page `noindex`. Tags lower case, Obsidian characters, `/tags/` paginated like the main page. 24 tag hue buckets (FNV-1a hash), contrast checked in tests. RSS: `dc:creator` from `AUTHOR_NAME`, `atom:updated` from `updated`. Image `width`/`height` read from the mirror (`golang.org/x/image` for WebP); an asset change re-renders all posts. IndexNow: one ping per sync with new, changed and removed post URLs, not on the first index load. New `AUTHOR_NAME`; `DEFAULT_OG_IMAGE` is a path inside `assets/`. +- 2026-11-04 — Metrics, votes, limits: SQLite `DATA_DIR/hyperhack.db` (`modernc.org/sqlite` `v1.61.0`, `0600`, WAL). All-time counters per post and label (Grafana gets daily trends with `increase()`), daily views only for "Popular" (pruned after `POPULAR_DAYS`). Search queries: live requests from `search.js` (`X-Search: live`) are not stored, the settled query is; TTL 90 days, at most 10 000 rows, top 20 in `/metrics`, refreshed every 5 min. ALTCHA: `altcha-lib-go/v2` `v2.4.1`, v2 challenge with SHA-256, cost 1, `uint32` counter up to `ALTCHA_MAX_NUMBER`, key signature; own small solver `static/altcha.js` instead of the widget; signing keys derived from `ALTCHA_HMAC_KEY`; replay map capped at 100 000. Beacon: a delta on every hide and `pagehide`, sent with `fetch(…, {keepalive: true})` (JSON body), each with a random per-page-view id; the server sums time per view and clamps the sum to 4 h; time on page = average over views with reading time. Views and read depth: once per visitor per post per day; daily hashes in memory only (cap 200 000), `ip_prefix` = IPv4 address or IPv6 `/64`. Source `internal` for referrers from the site itself. Outbound hosts limited to links in the post. `/api/altcha` and `/api/vote` share `RATE_LIMIT_VOTE`; limiter tracks at most 10 000 clients. Vote cookie stored as SHA-256. "Popular" after "Newest" on page 1. `/metrics` in Prometheus text format written with the standard library. diff --git a/SPEC.ru.md b/SPEC.ru.md index bf571b0..fc4f312 100644 --- a/SPEC.ru.md +++ b/SPEC.ru.md @@ -91,7 +91,7 @@ | `ALTCHA_HMAC_KEY` | Секрет ALTCHA. | | `READING_WPM` | Слов в минуту для расчётного времени чтения. По умолчанию: `200`. | | `LOG_LEVEL`, `LOG_FORMAT` | Уровень и формат логов (`json` / `text`). По умолчанию: `info`, `json`. | -| `DATA_DIR` | Папка для SQLite. По умолчанию: `/data`. | +| `DATA_DIR` | Папка для SQLite (`hyperhack.db`). По умолчанию: `/data`. | | `VAULT_DIR` | Папка для локального зеркала бакета. По умолчанию: `/vault`. | | `RATE_LIMIT_VOTE`, `RATE_LIMIT_BEACON`, `RATE_LIMIT_REINDEX` | Лимиты на клиента для эндпоинтов записи. По умолчанию: `10/1m`, `30/1m`, `5/1m`. | | `POPULAR_DAYS`, `POPULAR_LIMIT` | Окно и размер «Популярных». По умолчанию: `30`, `5`. | @@ -248,14 +248,14 @@ Garage не шлёт уведомления об изменениях в бак - Клиенты без JS (почти все боты) не считаются. Читатели с выключенным JS тоже. Принято. - Публично: показывается на посте. - Источник для списка «Популярные». -- Повторные просмотры одного посетителя не считаются (предложено: хеш IP + User-Agent за сутки; сырой IP не хранится). +- Повторные просмотры одного посетителя не считаются: суточный хеш посетителя (см. «Безопасность → Голоса и идентификация посетителя»); сырой IP не хранится. ### Время на странице - Сервер сам его не измерит. Нужен клиентский скрипт. -- Скрипт шлёт активное время чтения, когда страницу скрывают или закрывают (`visibilitychange` + `navigator.sendBeacon`). +- Скрипт шлёт активное время чтения, когда страницу скрывают или закрывают (`visibilitychange` / `pagehide`, `fetch` с `keepalive`). - Считается только время, когда вкладка видима. -- Хранится по посту как агрегат (например, среднее или медиана). +- В каждой дельте — случайный id просмотра страницы. Сервер суммирует дельты просмотра и ограничивает сумму 4 ч, затем добавляет её в агрегат поста: среднее время = сумма секунд / просмотры со временем чтения. - Не публично. На сайте не показывается. Только метрика (см. «Мониторинг»). ### Расчётное время чтения @@ -337,7 +337,7 @@ Garage не шлёт уведомления об изменениях в бак |---|---|---| | Просмотры по дням по посту | скрипт | Тренды; основа «Популярных». | | Глубина чтения: доля читателей, дошедших до 25 / 50 / 75 / 100 % текста | скрипт (`IntersectionObserver` на метках) | Дочитывают ли пост. | -| Источник трафика по посту: `telegram`, `google`, `yandex`, `bing`, `github`, `direct`, `other` | скрипт (домен `document.referrer`, сведённый к фиксированному списку) | Откуда приходят читатели. | +| Источник трафика по посту: `telegram`, `google`, `yandex`, `bing`, `github`, `direct`, `internal` (с этого сайта), `other` | скрипт (домен `document.referrer`, сведённый к фиксированному списку) | Откуда приходят читатели. | | Клики по внешним ссылкам по посту (домен цели) | скрипт | Какими ссылками пользуются. | | Копирования кода по посту | скрипт (событие `copy` внутри блоков кода) | Какой код полезен. | | Класс устройства: `mobile` / `tablet` / `desktop` | скрипт (диапазон ширины viewport) | Приоритеты вёрстки. | @@ -349,7 +349,7 @@ Garage не шлёт уведомления об изменениях в бак - Значения меток Prometheus ограничены: slug поста, фиксированный список источников, фиксированный список устройств. Никогда не делать метками сырые URL, рефереры и запросы. - Поисковые запросы хранятся в SQLite (нормализованные, в нижнем регистре). `/metrics` отдаёт только топ `N` как gauge, обновляется периодически. -- Все метрики скрипта уходят одним beacon-запросом на просмотр, по `visibilitychange` / `pagehide`. Лимит — `RATE_LIMIT_BEACON`. +- Скрипт шлёт один beacon-запрос при каждом скрытии (`visibilitychange` / `pagehide`) с тем, что изменилось с прошлой отправки, и с id просмотра страницы. Просмотр считается один раз. Лимит — `RATE_LIMIT_BEACON`. ## Мониторинг @@ -452,7 +452,7 @@ Garage не шлёт уведомления об изменениях в бак - `Content-Security-Policy: default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; worker-src 'self'; object-src 'none'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'; upgrade-insecure-requests`. `blob:` в `worker-src` добавлять, только если этого требует сборка ALTCHA. - Нет inline-скриптов и inline-стилей. Тема и язык берутся из cookie и рендерятся на сервере, поэтому inline-скрипт против мигания темы не нужен. - `X-Content-Type-Options: nosniff`, `Referrer-Policy: strict-origin-when-cross-origin`, `Cross-Origin-Opener-Policy: same-origin`, `X-Frame-Options: DENY`, `Permissions-Policy: camera=(), microphone=(), geolocation=(), browsing-topics=()`. -- Все JS, CSS, шрифты и виджет ALTCHA лежат на своём сервере. Без CDN, без сторонних запросов. +- Все JS, CSS, шрифты и решатель ALTCHA лежат на своём сервере. Без CDN, без сторонних запросов. ### CSRF и CORS @@ -464,11 +464,11 @@ Garage не шлёт уведомления об изменениях в бак - Личность для голоса: анонимная случайная cookie `vid` (128 бит, `HttpOnly; Secure; SameSite=Lax; Max-Age=1 год`). Личных данных в ней нет. - Один голос на `vid` на пост. Если удалить cookie, можно проголосовать снова, но каждый голос стоит решения ALTCHA и упирается в `RATE_LIMIT_VOTE`. - Почему не хеш IP: многие читатели делятдин IP (мобильный CGNAT), а суточный хеш разрешил бы голос раз в день. -- Дедупликация просмотров: `HMAC-SHA256(daily_key, ip_prefix + user_agent)`, `daily_key = HMAC(VISITOR_SECRET, date)`. Старые суточные ключи не хранятся, поэтому дни нельзя связать. Сырые IP никогда не хранятся и не логируются. +- Дедупликация просмотров: `HMAC-SHA256(daily_key, ip_prefix + user_agent)`, `daily_key = HMAC(VISITOR_SECRET, date)`. `ip_prefix`: адрес IPv4 или `/64` IPv6. Хеши текущего дня живут только в памяти; старые суточные ключи не хранятся, поэтому дни нельзя связать. Сырые IP никогда не хранятся и не логируются. ### ALTCHA -- Библиотека: `github.com/altcha-org/altcha-lib-go`. Невидимый режим: браузер решает задачу при клике. +- Библиотека: `github.com/altcha-org/altcha-lib-go/v2` (задачи v2). Невидимо: скрипт поста решает задачу при клике своим маленьким решателем (`static/altcha.js`), без виджета. - `ALTCHA_HMAC_KEY` ≥ 32 случайных байт. - Срок жизни задачи `ALTCHA_EXPIRES` (по умолчанию 5 мин). Истекла → отказ. - Защита от повтора в сервисе: использованные задачи хранятся в памяти до истечения; повтор → отказ. Библиотека этого не делает. @@ -514,7 +514,7 @@ Garage не шлёт уведомления об изменениях в бак ### Контейнер -- Статический бинарник (`CGO_ENABLED=0`, `-trimpath`), драйвер SQLite на чистом Go (например, `modernc.org/sqlite`). +- Статический бинарник (`CGO_ENABLED=0`, `-trimpath`), драйвер SQLite на чистом Go `modernc.org/sqlite`. - Образ: distroless static non-root или `scratch`. Без shell и пакетного менеджера. - Пример Compose: ```yaml @@ -615,3 +615,4 @@ Garage не шлёт уведомления об изменениях в бак - 2026-11-03 — Синхронизация: клиент S3 `minio-go/v7`, frontmatter через `go.yaml.in/yaml/v3`, GFM. Состояние зеркала в `VAULT_DIR/.hyperhack.json`. S3 недоступен при старте: работаем из зеркала; выход с кодом 1 только без зеркала. Без префикса Remotely Save. Ключи с сегментом на `.` или `_` игнорируются (было предложено). `MAX_OBJECTS` считает все ключи. Неверное поле frontmatter — пост пропускается. 3 попытки S3 за опрос. `/reindex` ждёт синхронизацию и возвращает `204`. - 2026-11-04 — Страницы: тема и язык переключаются формой без JS `POST /prefs`; cookie `theme` / `lang` (`HttpOnly; SameSite=Lax; Max-Age=1 год`, `Secure` на https). Логотип из `assets/logo.svg`, иначе встроенный знак. Шрифты Inter + JetBrains Mono, свои. Пагинация `/?page=N`: страница 1 — это `/`, другие значения вне диапазона → `404`. `` из поста; верхний бар остаётся на языке интерфейса. CSS и шрифты на `/static/*`. - 2026-11-04 — Поиск, теги, RSS, SEO: Bleve `v2.6.1`, индекс в памяти, своя схема документа на каждый язык (`en`, `ru`). Опечатки по длине слова: ≤ 3 букв 0, ≤ 6 букв 1, длиннее 2; префикс по последнему слову. В сниппетах остаётся только ``. Поиск работает без JS; `search.js` подменяет результаты при наборе (с задержкой). У страницы результатов `noindex`. Теги в нижнем регистре, символы как в Obsidian, `/tags/` с пагинацией как на главной. 24 оттенка для тегов (хеш FNV-1a), контраст проверяется в тестах. RSS: `dc:creator` из `AUTHOR_NAME`, `atom:updated` из `updated`. `width`/`height` картинок читаются из зеркала (`golang.org/x/image` для WebP); изменение ассета перерисовывает все посты. IndexNow: один пинг за синхронизацию с URL новых, изменённых и удалённых постов, не при первой загрузке индекса. Новая `AUTHOR_NAME`; `DEFAULT_OG_IMAGE` — путь внутри `assets/`. +- 2026-11-04 — Метрики, голоса, лимиты: SQLite `DATA_DIR/hyperhack.db` (`modernc.org/sqlite` `v1.61.0`, `0600`, WAL). Счётчики за всё время по посту и метке (дневные тренды Grafana считает через `increase()`), дневные просмотры — только для «Популярных» (удаляются старше `POPULAR_DAYS`). Поисковые запросы: живые запросы `search.js` (`X-Search: live`) не сохраняются, сохраняется устоявшийся запрос; TTL 90 дней, не больше 10 000 строк, топ-20 в `/metrics`, обновляется раз в 5 минут. ALTCHA: `altcha-lib-go/v2` `v2.4.1`, задача v2 с SHA-256, cost 1, счётчик `uint32` до `ALTCHA_MAX_NUMBER`, подпись ключа; свой маленький решатель `static/altcha.js` вместо виджета; ключи подписи выводятся из `ALTCHA_HMAC_KEY`; карта повторов ограничена 100 000. Beacon: дельта при каждом скрытии и `pagehide`, отправка через `fetch(…, {keepalive: true})` (тело JSON), в каждой — случайный id просмотра страницы; сервер суммирует время по просмотру и ограничивает сумму 4 ч; время на странице — среднее по просмотрам со временем чтения. Просмотры и глубина чтения: один раз на посетителя, пост и день; суточные хеши только в памяти (не больше 200 000), `ip_prefix` — адрес IPv4 или `/64` IPv6. Источник `internal` для переходов с самого сайта. Внешние домены — только из ссылок поста. `/api/altcha` и `/api/vote` делят `RATE_LIMIT_VOTE`; лимитер помнит не больше 10 000 клиентов. Cookie голоса хранится как SHA-256. «Популярные» после «Новых» на первой странице. `/metrics` в текстовом формате Prometheus, написан на стандартной библиотеке.