From 028064529f09153eb6cb975a885acba5ce209c37 Mon Sep 17 00:00:00 2001 From: edwh Date: Wed, 23 Sep 2026 16:05:36 +0100 Subject: [PATCH 1/3] Network page: load stats async, faster stats query, exclude archived from group count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The network page no longer calculates stats before rendering; the Vue component fetches them from the API (it already did when initialStats was empty), showing "…" until they arrive. - Network::stats() loads only group ids (no member-count scopes or eager loads), and bulkGroupStats pages events by primary key instead of a growing OFFSET. The "add groups" picker selects only id and name. - stats() returns 'groups', counting only non-archived groups; impact stats still include archived groups' past events. Co-Authored-By: Claude Opus 5.5 (1M context) --- app/Group.php | 4 +++- app/Http/Controllers/NetworkController.php | 8 ++++---- app/Network.php | 19 ++++++++++++++----- resources/js/components/NetworkPage.vue | 11 +++++++---- tests/Feature/Networks/NetworkTest.php | 1 + tests/Feature/Stats/NetworkStatsTest.php | 22 ++++++++++++++++++++++ 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/app/Group.php b/app/Group.php index 1a2dbebb32..8461767e1d 100644 --- a/app/Group.php +++ b/app/Group.php @@ -325,7 +325,9 @@ public static function bulkGroupStats(iterable $groups, ?float $eEmissionRatio = ->with('allDevices') ->withCount('allInvited') ->whereIn('events.group', $groupIds) - ->lazy(200) + // lazyById pages on the primary key (WHERE idevents > last) rather than lazy()'s growing + // OFFSET, which made MySQL rescan every earlier row for each chunk on large networks. + ->lazyById(200, 'events.idevents', 'idevents') ->each(function ($event) use (&$statsByGroup, $eEmissionRatio, $uEmissionratio) { $gid = $event->group; if (! isset($statsByGroup[$gid])) { diff --git a/app/Http/Controllers/NetworkController.php b/app/Http/Controllers/NetworkController.php index ef269b8022..4fa561a355 100644 --- a/app/Http/Controllers/NetworkController.php +++ b/app/Http/Controllers/NetworkController.php @@ -55,12 +55,12 @@ public function show(Network $network): View $groupsForAssociating = []; if ($user->can('associateGroups', $network)) { - $groupsForAssociating = $network->groupsNotIn()->sortBy('name'); + $groupsForAssociating = $network->groupsNotIn(); } - // Get network stats - $stats = $network->stats(); - $stats['groups'] = $network->groups->count(); + // Stats are fetched by the page from the API after it renders. Calculating them walks every device + // of every past event in the network, which is too slow to block the page load on for large networks. + $stats = []; // Determine if user can manage tags (NC for this network or Admin) $canManageTags = Fixometer::hasRole($user, 'Administrator') || diff --git a/app/Network.php b/app/Network.php index eb4069c18a..c0ea3840a1 100644 --- a/app/Network.php +++ b/app/Network.php @@ -88,13 +88,17 @@ public function sizedLogo($size) public function groupsNotIn() { - $networkGroupsIds = $this->groups()->pluck('idgroups')->toArray(); + $networkGroupsIds = $this->groups()->withoutGlobalScopes()->without(['networks', 'group_tags'])->pluck('groups.idgroups')->toArray(); - if (empty($networkGroupsIds)) { - return Group::orderBy('name')->get(); + // Only the id and name are needed (for the "add groups" picker). Skip the member-count global scopes + // and the networks/tags eager loads, which are expensive across every group on the platform. + $query = Group::withoutGlobalScopes()->without(['networks', 'group_tags'])->select(['idgroups', 'name'])->orderBy('name'); + + if (! empty($networkGroupsIds)) { + $query->whereNotIn('idgroups', $networkGroupsIds); } - return Group::whereNotIn('idgroups', $networkGroupsIds)->orderBy('name')->get(); + return $query->get(); } public function stats() @@ -102,7 +106,11 @@ public function stats() $eEmissionRatio = \App\Helpers\LcaStats::getEmissionRatioPowered(); $uEmissionratio = \App\Helpers\LcaStats::getEmissionRatioUnpowered(); - $allStats = \App\Group::bulkGroupStats($this->groups, $eEmissionRatio, $uEmissionratio); + // Impact stats include archived groups (their past events still count); the group tally does not. + $groups = $this->groups()->withoutGlobalScopes()->without(['networks', 'group_tags']) + ->get(['groups.idgroups', 'groups.archived_at']); + + $allStats = \App\Group::bulkGroupStats($groups, $eEmissionRatio, $uEmissionratio); $stats = \App\Group::getGroupStatsArrayKeys(); @@ -113,6 +121,7 @@ public function stats() } $stats['parties'] = $stats['parties'] ?? 0; + $stats['groups'] = $groups->whereNull('archived_at')->count(); return $stats; } diff --git a/resources/js/components/NetworkPage.vue b/resources/js/components/NetworkPage.vue index 59cb4264eb..2a4717d43d 100644 --- a/resources/js/components/NetworkPage.vue +++ b/resources/js/components/NetworkPage.vue @@ -23,19 +23,19 @@

{{ __('networks.general.impact') }}

-
{{ stats.groups || 0 }}
+
{{ statsLoaded ? (stats.groups || 0) : '…' }}
{{ __('networks.stats.groups', { count: stats.groups || 0 }) }}
-
{{ stats.parties || 0 }}
+
{{ statsLoaded ? (stats.parties || 0) : '…' }}
{{ __('networks.stats.events', { count: stats.parties || 0 }) }}
-
{{ formatWeight(stats.waste_total) }}
+
{{ statsLoaded ? formatWeight(stats.waste_total) : '…' }}
{{ __('networks.stats.waste_diverted') }}
-
{{ formatWeight(stats.co2_total) }}
+
{{ statsLoaded ? formatWeight(stats.co2_total) : '…' }}
{{ __('networks.stats.co2_prevented') }}
@@ -234,6 +234,9 @@ export default { } }, computed: { + statsLoaded() { + return !!this.stats && Object.keys(this.stats).length > 0 + }, worldBounds() { // The inverted whole-world box: GroupMap treats it as "no location", so // it frames all the (network-filtered) groups instead. diff --git a/tests/Feature/Networks/NetworkTest.php b/tests/Feature/Networks/NetworkTest.php index b256b88299..bc1a36929e 100644 --- a/tests/Feature/Networks/NetworkTest.php +++ b/tests/Feature/Networks/NetworkTest.php @@ -231,6 +231,7 @@ public function network_stats_can_be_queried(): void $response = $this->get("/api/networks/{$network->id}/stats?api_token=1234"); $stats = json_decode($response->getContent(), true); $expectedStats = \App\Group::getGroupStatsArrayKeys(); + $expectedStats['groups'] = 1; $this->assertEquals($expectedStats, $stats); } diff --git a/tests/Feature/Stats/NetworkStatsTest.php b/tests/Feature/Stats/NetworkStatsTest.php index 6b99a94b9a..83ad7f46ee 100644 --- a/tests/Feature/Stats/NetworkStatsTest.php +++ b/tests/Feature/Stats/NetworkStatsTest.php @@ -33,10 +33,32 @@ public function a_network_with_no_groups_has_empty_stats(): void $expect = \App\Group::getGroupStatsArrayKeys(); $expect['parties'] = $expect['parties'] ?? 0; + $expect['groups'] = 0; $this->assertEquals($expect, $network->stats()); } + /** @test */ + public function network_group_count_excludes_archived_groups_but_impact_includes_them(): void + { + $network = Network::factory()->create(); + + $active = Group::factory()->create(); + $archived = Group::factory()->create(['archived_at' => '2024-01-01 00:00:00']); + $network->addGroup($active); + $network->addGroup($archived); + + Party::factory()->moderated()->create([ + 'event_start_utc' => '2000-01-01T10:00:00+00:00', + 'event_end_utc' => '2000-01-01T13:00:00+00:00', + 'group' => $archived->idgroups, + ]); + + $stats = $network->stats(); + $this->assertEquals(1, $stats['groups']); + $this->assertEquals(1, $stats['parties']); + } + /** @test */ public function network_stats_aggregate_misc_estimates_and_footprints_across_groups(): void { From ca9fa07b5eda4de5638ff7dd7aa225f3dfd5defd Mon Sep 17 00:00:00 2001 From: edwh Date: Fri, 25 Sep 2026 16:26:38 +0100 Subject: [PATCH 2/3] Network stats: fix keyset paging dropping events; tag stats shape; failed-load state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - bulkGroupStats(): reorder() before lazyById. past() orders by date, and lazyById only strips an existing order on the id column, so pages were ordered by date and paged by id: events beyond the first 200 were skipped or double-counted (399 of 450 in the new test). - statsForTag() returns 'groups' like Network::stats(). - NetworkPage shows '–' if the stats request fails, not '…' forever. - StatsTestCase declares $_ratioPowered (dynamic property deprecation hid test failure messages under process isolation). Co-Authored-By: Claude Opus 5.5 (1M context) --- app/Group.php | 5 +++- .../Controllers/API/NetworkController.php | 3 ++ resources/js/components/NetworkPage.vue | 14 ++++++--- tests/Feature/Networks/APIv2NetworkTest.php | 1 + tests/Feature/Stats/NetworkStatsTest.php | 30 +++++++++++++++++++ 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/app/Group.php b/app/Group.php index 8461767e1d..65421a9f9f 100644 --- a/app/Group.php +++ b/app/Group.php @@ -326,7 +326,10 @@ public static function bulkGroupStats(iterable $groups, ?float $eEmissionRatio = ->withCount('allInvited') ->whereIn('events.group', $groupIds) // lazyById pages on the primary key (WHERE idevents > last) rather than lazy()'s growing - // OFFSET, which made MySQL rescan every earlier row for each chunk on large networks. + // OFFSET, which made MySQL rescan every earlier row for each chunk on large networks. past() + // orders by date, and keyset paging needs the id to be the only sort key or events are skipped + // and double-counted - hence reorder(). + ->reorder() ->lazyById(200, 'events.idevents', 'idevents') ->each(function ($event) use (&$statsByGroup, $eEmissionRatio, $uEmissionratio) { $gid = $event->group; diff --git a/app/Http/Controllers/API/NetworkController.php b/app/Http/Controllers/API/NetworkController.php index 00d7a6076a..e65cf769a6 100644 --- a/app/Http/Controllers/API/NetworkController.php +++ b/app/Http/Controllers/API/NetworkController.php @@ -57,6 +57,9 @@ private function statsForTag(Network $network, int $tagId): array } } + // Same shape as Network::stats(): the group tally excludes archived groups. + $stats['groups'] = $groups->whereNull('archived_at')->count(); + return $stats; } diff --git a/resources/js/components/NetworkPage.vue b/resources/js/components/NetworkPage.vue index 2a4717d43d..c87cfc943d 100644 --- a/resources/js/components/NetworkPage.vue +++ b/resources/js/components/NetworkPage.vue @@ -23,19 +23,19 @@

{{ __('networks.general.impact') }}

-
{{ statsLoaded ? (stats.groups || 0) : '…' }}
+
{{ statsLoaded ? (stats.groups || 0) : statsPlaceholder }}
{{ __('networks.stats.groups', { count: stats.groups || 0 }) }}
-
{{ statsLoaded ? (stats.parties || 0) : '…' }}
+
{{ statsLoaded ? (stats.parties || 0) : statsPlaceholder }}
{{ __('networks.stats.events', { count: stats.parties || 0 }) }}
-
{{ statsLoaded ? formatWeight(stats.waste_total) : '…' }}
+
{{ statsLoaded ? formatWeight(stats.waste_total) : statsPlaceholder }}
{{ __('networks.stats.waste_diverted') }}
-
{{ statsLoaded ? formatWeight(stats.co2_total) : '…' }}
+
{{ statsLoaded ? formatWeight(stats.co2_total) : statsPlaceholder }}
{{ __('networks.stats.co2_prevented') }}
@@ -219,6 +219,7 @@ export default { data() { return { stats: this.initialStats, + statsFailed: false, tags: this.initialTags, newTagName: '', newTagDescription: '', @@ -237,6 +238,10 @@ export default { statsLoaded() { return !!this.stats && Object.keys(this.stats).length > 0 }, + statsPlaceholder() { + // Loading, or a dash if the stats request failed rather than an ellipsis forever. + return this.statsFailed ? '–' : '…' + }, worldBounds() { // The inverted whole-world box: GroupMap treats it as "no location", so // it frames all the (network-filtered) groups instead. @@ -366,6 +371,7 @@ export default { this.stats = response.data } catch (error) { console.error('Failed to fetch network stats:', error) + this.statsFailed = true } } diff --git a/tests/Feature/Networks/APIv2NetworkTest.php b/tests/Feature/Networks/APIv2NetworkTest.php index 326d949171..34ea412bfe 100644 --- a/tests/Feature/Networks/APIv2NetworkTest.php +++ b/tests/Feature/Networks/APIv2NetworkTest.php @@ -873,6 +873,7 @@ public function testGetNetworkStatsFilteredByTag(): void { $this->assertArrayHasKey('parties', $json); $this->assertArrayHasKey('co2_total', $json); $this->assertArrayHasKey('waste_total', $json); + $this->assertEquals(1, $json['groups']); } /** diff --git a/tests/Feature/Stats/NetworkStatsTest.php b/tests/Feature/Stats/NetworkStatsTest.php index 83ad7f46ee..64c79fad5b 100644 --- a/tests/Feature/Stats/NetworkStatsTest.php +++ b/tests/Feature/Stats/NetworkStatsTest.php @@ -8,6 +8,7 @@ use App\Network; use App\Party; use App\User; +use Carbon\Carbon; use Tests\Feature\Stats\StatsTestCase; /** @@ -38,6 +39,35 @@ public function a_network_with_no_groups_has_empty_stats(): void $this->assertEquals($expect, $network->stats()); } + /** @test */ + public function network_stats_count_every_event_once_across_chunks(): void + { + // More events than one lazyById chunk (200), with dates running opposite to ids, so any ordering other + // than by id would make chunks overlap or skip events. + $network = Network::factory()->create(); + $group1 = Group::factory()->create(); + $group2 = Group::factory()->create(); + $network->addGroup($group1); + $network->addGroup($group2); + + $count = 450; + $expectedVolunteers = 0; + for ($i = 1; $i <= $count; $i++) { + $start = Carbon::parse('2020-01-01 10:00:00')->addDays($i); + Party::factory()->moderated()->create([ + 'group' => $i % 2 ? $group1->idgroups : $group2->idgroups, + 'event_start_utc' => $start->toIso8601String(), + 'event_end_utc' => $start->copy()->addHours(3)->toIso8601String(), + 'volunteers' => $i, + ]); + $expectedVolunteers += $i; + } + + $stats = $network->stats(); + $this->assertEquals($count, $stats['parties']); + $this->assertEquals($expectedVolunteers, $stats['volunteers']); + } + /** @test */ public function network_group_count_excludes_archived_groups_but_impact_includes_them(): void { From 825c71f9c62d7ac7996e65f4d71662e2d68243fb Mon Sep 17 00:00:00 2001 From: edwh Date: Fri, 25 Sep 2026 16:26:49 +0100 Subject: [PATCH 3/3] StatsTestCase: declare $_ratioPowered Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/Feature/Stats/StatsTestCase.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Feature/Stats/StatsTestCase.php b/tests/Feature/Stats/StatsTestCase.php index 5b2b939a68..1a25c835ae 100644 --- a/tests/Feature/Stats/StatsTestCase.php +++ b/tests/Feature/Stats/StatsTestCase.php @@ -13,6 +13,7 @@ class StatsTestCase extends TestCase { protected $_displacementFactor; protected $_ratioUnpowered; + protected $_ratioPowered; protected $_idUnpoweredMisc; protected $_idPoweredMisc; protected $_idUnpoweredNonMisc;