Network page: async stats, faster stats query, exclude archived groups from count

app/Group.php

@@ -325,7 +325,12 @@ public static function bulkGroupStats(iterable $groups, ?float $eEmissionRatio =
             ->with('allDevices')
             ->withCount('allInvited')
             ->whereIn('events.group', $groupIds)
-            ->lazy(200)
+            // lazyById pages on the primary key (WHERE idevents > last) rather than lazy()'s growing
+            // OFFSET, which made MySQL rescan every earlier row for each chunk on large networks.  past()
+            // orders by date, and keyset paging needs the id to be the only sort key or events are skipped
+            // and double-counted - hence reorder().
+            ->reorder()
+            ->lazyById(200, 'events.idevents', 'idevents')
             ->each(function ($event) use (&$statsByGroup, $eEmissionRatio, $uEmissionratio) {
                 $gid = $event->group;
                 if (! isset($statsByGroup[$gid])) {

app/Http/Controllers/API/NetworkController.php

@@ -57,6 +57,9 @@ private function statsForTag(Network $network, int $tagId): array
             }
         }
 
+        // Same shape as Network::stats(): the group tally excludes archived groups.
+        $stats['groups'] = $groups->whereNull('archived_at')->count();
+
         return $stats;
     }
 

app/Http/Controllers/NetworkController.php

@@ -55,12 +55,12 @@ public function show(Network $network): View
         $groupsForAssociating = [];
 
         if ($user->can('associateGroups', $network)) {
-            $groupsForAssociating = $network->groupsNotIn()->sortBy('name');
+            $groupsForAssociating = $network->groupsNotIn();
         }
 
-        // Get network stats
-        $stats = $network->stats();
-        $stats['groups'] = $network->groups->count();
+        // Stats are fetched by the page from the API after it renders.  Calculating them walks every device
+        // of every past event in the network, which is too slow to block the page load on for large networks.
+        $stats = [];
 
         // Determine if user can manage tags (NC for this network or Admin)
         $canManageTags = Fixometer::hasRole($user, 'Administrator') ||

app/Network.php

@@ -88,21 +88,29 @@ public function sizedLogo($size)
 
     public function groupsNotIn()
     {
-        $networkGroupsIds = $this->groups()->pluck('idgroups')->toArray();
+        $networkGroupsIds = $this->groups()->withoutGlobalScopes()->without(['networks', 'group_tags'])->pluck('groups.idgroups')->toArray();
 
-        if (empty($networkGroupsIds)) {
-            return Group::orderBy('name')->get();
+        // Only the id and name are needed (for the "add groups" picker).  Skip the member-count global scopes
+        // and the networks/tags eager loads, which are expensive across every group on the platform.
+        $query = Group::withoutGlobalScopes()->without(['networks', 'group_tags'])->select(['idgroups', 'name'])->orderBy('name');
+
+        if (! empty($networkGroupsIds)) {
+            $query->whereNotIn('idgroups', $networkGroupsIds);
         }
 
-        return Group::whereNotIn('idgroups', $networkGroupsIds)->orderBy('name')->get();
+        return $query->get();
     }
 
     public function stats()
     {
         $eEmissionRatio = \App\Helpers\LcaStats::getEmissionRatioPowered();
         $uEmissionratio = \App\Helpers\LcaStats::getEmissionRatioUnpowered();
 
-        $allStats = \App\Group::bulkGroupStats($this->groups, $eEmissionRatio, $uEmissionratio);
+        // Impact stats include archived groups (their past events still count); the group tally does not.
+        $groups = $this->groups()->withoutGlobalScopes()->without(['networks', 'group_tags'])
+            ->get(['groups.idgroups', 'groups.archived_at']);
+
+        $allStats = \App\Group::bulkGroupStats($groups, $eEmissionRatio, $uEmissionratio);
 
         $stats = \App\Group::getGroupStatsArrayKeys();
 
@@ -113,6 +121,7 @@ public function stats()
         }
 
         $stats['parties'] = $stats['parties'] ?? 0;
+        $stats['groups'] = $groups->whereNull('archived_at')->count();
 
         return $stats;
     }

resources/js/components/NetworkPage.vue

@@ -23,19 +23,19 @@
       <h2>{{ __('networks.general.impact') }}</h2>
       <div class="stats-grid">
         <div class="stat-box">
-          <div class="stat-value">{{ stats.groups || 0 }}</div>
+          <div class="stat-value">{{ statsLoaded ? (stats.groups || 0) : statsPlaceholder }}</div>
           <div class="stat-label">{{ __('networks.stats.groups', { count: stats.groups || 0 }) }}</div>
         </div>
         <div class="stat-box">
-          <div class="stat-value">{{ stats.parties || 0 }}</div>
+          <div class="stat-value">{{ statsLoaded ? (stats.parties || 0) : statsPlaceholder }}</div>
           <div class="stat-label">{{ __('networks.stats.events', { count: stats.parties || 0 }) }}</div>
         </div>
         <div class="stat-box">
-          <div class="stat-value">{{ formatWeight(stats.waste_total) }}</div>
+          <div class="stat-value">{{ statsLoaded ? formatWeight(stats.waste_total) : statsPlaceholder }}</div>
           <div class="stat-label">{{ __('networks.stats.waste_diverted') }}</div>
         </div>
         <div class="stat-box">
-          <div class="stat-value">{{ formatWeight(stats.co2_total) }}</div>
+          <div class="stat-value">{{ statsLoaded ? formatWeight(stats.co2_total) : statsPlaceholder }}</div>
           <div class="stat-label">{{ __('networks.stats.co2_prevented') }}</div>
         </div>
       </div>
@@ -219,6 +219,7 @@ export default {
   data() {
     return {
       stats: this.initialStats,
+      statsFailed: false,
       tags: this.initialTags,
       newTagName: '',
       newTagDescription: '',
@@ -234,6 +235,13 @@ export default {
     }
   },
   computed: {
+    statsLoaded() {
+      return !!this.stats && Object.keys(this.stats).length > 0
+    },
+    statsPlaceholder() {
+      // Loading, or a dash if the stats request failed rather than an ellipsis forever.
+      return this.statsFailed ? '–' : '…'
+    },
     worldBounds() {
       // The inverted whole-world box: GroupMap treats it as "no location", so
       // it frames all the (network-filtered) groups instead.
@@ -363,6 +371,7 @@ export default {
         this.stats = response.data
       } catch (error) {
         console.error('Failed to fetch network stats:', error)
+        this.statsFailed = true
       }
     }
 

tests/Feature/Networks/APIv2NetworkTest.php

@@ -873,6 +873,7 @@ public function testGetNetworkStatsFilteredByTag(): void {
         $this->assertArrayHasKey('parties', $json);
         $this->assertArrayHasKey('co2_total', $json);
         $this->assertArrayHasKey('waste_total', $json);
+        $this->assertEquals(1, $json['groups']);
     }
 
     /**

tests/Feature/Networks/NetworkTest.php

@@ -231,6 +231,7 @@ public function network_stats_can_be_queried(): void
         $response = $this->get("/api/networks/{$network->id}/stats?api_token=1234");
         $stats = json_decode($response->getContent(), true);
         $expectedStats = \App\Group::getGroupStatsArrayKeys();
+        $expectedStats['groups'] = 1;
         $this->assertEquals($expectedStats, $stats);
     }
 

tests/Feature/Stats/NetworkStatsTest.php

@@ -8,6 +8,7 @@
 use App\Network;
 use App\Party;
 use App\User;
+use Carbon\Carbon;
 use Tests\Feature\Stats\StatsTestCase;
 
 /**
@@ -33,10 +34,61 @@ public function a_network_with_no_groups_has_empty_stats(): void
 
         $expect = \App\Group::getGroupStatsArrayKeys();
         $expect['parties'] = $expect['parties'] ?? 0;
+        $expect['groups'] = 0;
 
         $this->assertEquals($expect, $network->stats());
     }
 
+    /** @test */
+    public function network_stats_count_every_event_once_across_chunks(): void
+    {
+        // More events than one lazyById chunk (200), with dates running opposite to ids, so any ordering other
+        // than by id would make chunks overlap or skip events.
+        $network = Network::factory()->create();
+        $group1 = Group::factory()->create();
+        $group2 = Group::factory()->create();
+        $network->addGroup($group1);
+        $network->addGroup($group2);
+
+        $count = 450;
+        $expectedVolunteers = 0;
+        for ($i = 1; $i <= $count; $i++) {
+            $start = Carbon::parse('2020-01-01 10:00:00')->addDays($i);
+            Party::factory()->moderated()->create([
+                'group' => $i % 2 ? $group1->idgroups : $group2->idgroups,
+                'event_start_utc' => $start->toIso8601String(),
+                'event_end_utc' => $start->copy()->addHours(3)->toIso8601String(),
+                'volunteers' => $i,
+            ]);
+            $expectedVolunteers += $i;
+        }
+
+        $stats = $network->stats();
+        $this->assertEquals($count, $stats['parties']);
+        $this->assertEquals($expectedVolunteers, $stats['volunteers']);
+    }
+
+    /** @test */
+    public function network_group_count_excludes_archived_groups_but_impact_includes_them(): void
+    {
+        $network = Network::factory()->create();
+
+        $active = Group::factory()->create();
+        $archived = Group::factory()->create(['archived_at' => '2024-01-01 00:00:00']);
+        $network->addGroup($active);
+        $network->addGroup($archived);
+
+        Party::factory()->moderated()->create([
+            'event_start_utc' => '2000-01-01T10:00:00+00:00',
+            'event_end_utc' => '2000-01-01T13:00:00+00:00',
+            'group' => $archived->idgroups,
+        ]);
+
+        $stats = $network->stats();
+        $this->assertEquals(1, $stats['groups']);
+        $this->assertEquals(1, $stats['parties']);
+    }
+
     /** @test */
     public function network_stats_aggregate_misc_estimates_and_footprints_across_groups(): void
     {

tests/Feature/Stats/StatsTestCase.php

@@ -13,6 +13,7 @@
 {
     protected $_displacementFactor;
     protected $_ratioUnpowered;
+    protected $_ratioPowered;
     protected $_idUnpoweredMisc;
     protected $_idPoweredMisc;
     protected $_idUnpoweredNonMisc;