Data engagement: reporting links, group stats, Fixometer fixes (#919)

.gitignore

@@ -3,6 +3,7 @@
 /public/storage
 /public/uploads
 /public/devices.csv
+/public/exports
 /public/report.html
 npm-debug.log
 yarn-error.log

app/Console/Commands/ExportRepairData.php

@@ -0,0 +1,88 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\RepairDataExport;
+use DB;
+use Illuminate\Console\Command;
+
+/**
+ * Builds the full repair data download, so that the web server can hand it straight out rather than PHP taking a
+ * minute or more to build it on every request.
+ *
+ * Runs hourly, but rebuilds at most once a day, and only if the data has changed.  The hourly run means a fresh
+ * machine, which starts without the file, has one again within the hour.
+ */
+class ExportRepairData extends Command
+{
+    protected $signature = 'export:repair-data {--force : Rebuild even if built recently or nothing has changed}';
+
+    protected $description = 'Build the repair data download served at /exports/repair-data.csv';
+
+    // Rebuild a little under a day after the last build, so the hourly run doesn't drift later each day.
+    private const MAX_AGE_SECONDS = 23 * 3600;
+
+    public static function path(): string
+    {
+        return public_path('exports/repair-data.csv');
+    }
+
+    private function fingerprintPath(): string
+    {
+        return storage_path('app/exports/repair-data.fingerprint');
+    }
+
+    /**
+     * Changes when repairs are added, edited or removed, or events or groups change in a way that could change
+     * which repairs anyone can see.
+     */
+    private function fingerprint(): string
+    {
+        return json_encode([
+            DB::table('devices')->count(),
+            DB::table('devices')->max('updated_at'),
+            DB::table('events')->max('updated_at'),
+            DB::table('groups')->max('updated_at'),
+        ]);
+    }
+
+    public function handle(RepairDataExport $export): int
+    {
+        $path = self::path();
+        $fingerprint = $this->fingerprint();
+
+        if (! $this->option('force') && file_exists($path)) {
+            if (time() - filemtime($path) < self::MAX_AGE_SECONDS) {
+                $this->info('Built recently; not rebuilding.');
+
+                return self::SUCCESS;
+            }
+
+            if (@file_get_contents($this->fingerprintPath()) === $fingerprint) {
+                // Mark it as checked, so we don't check again until tomorrow.
+                touch($path);
+                $this->info('Data unchanged; not rebuilding.');
+
+                return self::SUCCESS;
+            }
+        }
+
+        foreach ([dirname($path), dirname($this->fingerprintPath())] as $dir) {
+            if (! is_dir($dir)) {
+                mkdir($dir, 0755, true);
+            }
+        }
+
+        // Build alongside and then swap in, so nobody downloads a half-written file.
+        $tmp = $path.'.tmp';
+        $file = fopen($tmp, 'w');
+        $export->write($export->query(), $file);
+        fclose($file);
+        rename($tmp, $path);
+
+        file_put_contents($this->fingerprintPath(), $fingerprint);
+        $this->info('Built '.$path);
+
+        return self::SUCCESS;
+    }
+}

app/Console/Kernel.php

@@ -44,6 +44,9 @@ protected function schedule(Schedule $schedule): void
         $schedule->command('event:timezones')->hourly();
 
         $schedule->command('wordpress:event:create_failed')->daily();
+
+        // Rebuilds at most daily; hourly so that a new machine gets the file soon after it starts.
+        $schedule->command('export:repair-data')->hourly()->withoutOverlapping();
     }
 
     /**

app/Device.php

@@ -207,14 +207,17 @@ public function countByClustersYearStatus($group)
         return DB::select($sql, ['group' => $group]);
     }
 
+    /**
+     * The most common categories, powered and unpowered, leaving out the "None of the above" ones.
+     */
     public function findMostSeen($status = null, $cluster = null, $group = null)
     {
         $sql = 'SELECT COUNT(`d`.`category`) AS `counter`, `c`.`name` FROM `'.$this->table.'` AS `d`
                 INNER JOIN `events` AS `e`
                     ON `d`.`event` = `e`.`idevents`
                 INNER JOIN `categories` AS `c`
                     ON `d`.`category` = `c`.`idcategories`
-                WHERE 1=1 and `c`.`powered` = 1 AND `c`.`idcategories` <> '.env('MISC_CATEGORY_ID_POWERED');
+                WHERE `c`.`idcategories` NOT IN ('.(int) env('MISC_CATEGORY_ID_POWERED').', '.(int) env('MISC_CATEGORY_ID_UNPOWERED').')';
 
         if (! is_null($status) && is_numeric($status)) {
             $sql .= ' AND `d`.`repair_status` = :status ';

app/Group.php

@@ -497,6 +497,35 @@
         return $this->networks->contains($network);
     }
 
+    /**
+     * The group reporting dashboard, filtered to this group, or null if reporting isn't set up.
+     *
+     * The dashboard is in the language of the group's network, falling back to English when there's no
+     * dashboard in that language or its base language.
+     */
+    public function reportingUrl(): ?string
+    {
+        $templates = config('restarters.reporting.group_urls', []);
+        $network = $this->networks->first();
+        $language = $network ? $network->default_language : null;
+
+        // A regional language (e.g. Repair Together's fr-BE) uses the dashboard for its base language.
+        $base = $language ? explode('-', $language)[0] : null;
+        $template = ($language ? ($templates[$language] ?? null) : null)
+            ?: ($base ? ($templates[$base] ?? null) : null)
+            ?: ($templates['en'] ?? null);
+
+        if (! $template) {
+            return null;
+        }
+
+        return str_replace(
+            ['{group}', '{group_name}'],
+            [$this->idgroups, rawurlencode($this->name)],
+            $template
+        );
+    }
+
     // If just one of the networks that the group is a member of should push to Wordpress, then we should push.
     // If an group is not approved, then we should not push the events to Wordpress.
     public function eventsShouldPushToWordpress()

app/Http/Controllers/ApiController.php

@@ -7,6 +7,7 @@
 use App\Group;
 use App\Party;
 use App\User;
+use Carbon\Carbon;
 use Auth;
 use DB;
 use Illuminate\Http\Request;
@@ -51,31 +52,35 @@ public static function homepage_data(): JsonResponse
     {
         $result = [];
 
-        $lock = \Cache::lock('homepage_data_lock', 60);
+        $lock = \Cache::lock('homepage_data_v3_lock', 60);
 
-        if (\Cache::has('homepage_data')) {
-            $result = \Cache::get('homepage_data');
+        if (\Cache::has('homepage_data_v3')) {
+            $result = \Cache::get('homepage_data_v3');
         } elseif ($lock->get()) {
             try {
                 $Device = new Device;
 
                 // Aggregate participants and hours in SQL — avoids loading 18k+ event rows into PHP.
                 // hoursVolunteered() formula: cancelled→3, volunteers>0→9+volunteers*ceil(minutes/60), else→21
+                // Events held leaves out cancelled events and those of groups that aren't approved.
                 $eventStats = DB::table('events')
-                    ->whereNull('deleted_at')
-                    ->where('event_end_utc', '<', now())
+                    ->leftJoin('groups', 'groups.idgroups', '=', 'events.group')
+                    ->whereNull('events.deleted_at')
+                    ->where('events.event_end_utc', '<', now())
                     ->selectRaw("
-                        SUM(pax) as participants,
+                        SUM(CASE WHEN events.cancelled = 0 AND groups.approved = 1 THEN 1 ELSE 0 END) as events,
+                        SUM(events.pax) as participants,
                         SUM(CASE
-                            WHEN cancelled = 1 THEN 3
-                            WHEN volunteers > 0 THEN 9 + volunteers * CEIL(TIMESTAMPDIFF(MINUTE, event_start_utc, event_end_utc) / 60)
+                            WHEN events.cancelled = 1 THEN 3
+                            WHEN events.volunteers > 0 THEN 9 + events.volunteers * CEIL(TIMESTAMPDIFF(MINUTE, events.event_start_utc, events.event_end_utc) / 60)
                             ELSE 21
                         END) as hours_volunteered
                     ")
                     ->first();
 
                 $result['participants'] = (int) ($eventStats->participants ?? 0);
                 $result['hours_volunteered'] = (int) ($eventStats->hours_volunteered ?? 0);
+                $result['events'] = (int) ($eventStats->events ?? 0);
 
                 $fixed = $Device->statusCount();
                 $result['items_fixed'] = count($fixed) ? $fixed[0]->counter : 0;
@@ -93,20 +98,21 @@ public static function homepage_data(): JsonResponse
                 $result['fixed_unpowered'] = $devices->fixedUnpoweredCount();
                 $result['total_powered'] = $devices->poweredCount();
                 $result['total_unpowered'] = $devices->unpoweredCount();
+                $result['total_items'] = $result['total_powered'] + $result['total_unpowered'];
 
                 // for backward compatibility (don't break therestartproject.org)
                 $result['weights'] = round($result['waste_total']);
                 $result['ewaste'] = round($result['waste_powered']);
                 $result['unpowered_waste'] = round($result['waste_unpowered']);
                 $result['emissions'] = round($result['co2_total']);
 
-                \Cache::put('homepage_data', $result, 43200);
+                \Cache::put('homepage_data_v3', $result, 43200);
             } finally {
                 $lock->release();
             }
         } else {
             // Another worker is rebuilding — return stale or empty rather than pile on
-            $result = \Cache::get('homepage_data', []);
+            $result = \Cache::get('homepage_data_v3', []);
         }
 
         return response()
@@ -200,75 +206,107 @@ public static function getUserList()
     }
 
     /**
-     * List/search devices.
+     * The where clauses for the device list's search filters.
      */
-    public static function getDevices(Request $request, $page, $size): JsonResponse
+    private static function deviceFilters(Request $request): array
     {
-        $powered = $request->input('powered');
-        $sortBy = $request->input('sortBy');
-        $sortDesc = $request->input('sortDesc');
-        $category = $request->input('category');
-        $brand = $request->input('brand');
-        $model = $request->input('model');
-        $item_type = $request->input('item_type');
-        $status = $request->input('status');
-        $comments = $request->input('comments');
-        $wiki = filter_var($request->input('wiki', false), FILTER_VALIDATE_BOOLEAN);
-        $group = $request->input('group');
-        $from_date = $request->input('from_date');
-        $to_date = $request->input('to_date');
-
-        $wheres = [
-            ['categories.powered', '=', $powered == 'true' ? 1 : 0],
-        ];
+        $wheres = [];
 
-        if ($category) {
-            $wheres[] = ['idcategories', '=', $category];
-        }
+        // No powered filter means both powered and unpowered items.
+        $powered = $request->input('powered');
 
-        if ($brand) {
-            $wheres[] = ['devices.brand', 'LIKE', '%'.$brand.'%'];
+        if ($powered === 'true' || $powered === 'false') {
+            $wheres[] = ['categories.powered', '=', $powered === 'true' ? 1 : 0];
         }
 
-        if ($model) {
-            $wheres[] = ['devices.model', 'LIKE', '%'.$model.'%'];
+        if ($request->input('category')) {
+            $wheres[] = ['idcategories', '=', $request->input('category')];
         }
 
-        if ($item_type) {
-            $wheres[] = ['devices.item_type', 'LIKE', '%'.$item_type.'%'];
-        }
+        // Free text searches.
+        $likes = [
+            'brand' => 'devices.brand',
+            'model' => 'devices.model',
+            'item_type' => 'devices.item_type',
+            'comments' => 'devices.problem',
+            'group' => 'groups.name',
+        ];
 
-        if ($comments) {
-            $wheres[] = ['devices.problem', 'LIKE', '%'.$comments.'%'];
+        foreach ($likes as $param => $column) {
+            if ($request->input($param)) {
+                $wheres[] = [$column, 'LIKE', '%'.$request->input($param).'%'];
+            }
         }
 
-        if ($wiki) {
+        if (filter_var($request->input('wiki', false), FILTER_VALIDATE_BOOLEAN)) {
             $wheres[] = ['devices.wiki', '=', 1];
         }
 
+        $status = $request->input('status');
+
         if ($status) {
-            $wheres[] = ['repair_status', '=', $status];
-        }
+            // The client uses the status strings from the rest of the API; accept the underlying numbers too.
+            $statuses = [
+                Device::REPAIR_STATUS_FIXED_STR => Device::REPAIR_STATUS_FIXED,
+                Device::REPAIR_STATUS_REPAIRABLE_STR => Device::REPAIR_STATUS_REPAIRABLE,
+                Device::REPAIR_STATUS_ENDOFLIFE_STR => Device::REPAIR_STATUS_ENDOFLIFE,
+            ];
 
-        if ($group) {
-            $wheres[] = ['groups.name', 'LIKE', '%'.$group.'%'];
+            $wheres[] = ['repair_status', '=', $statuses[$status] ?? intval($status)];
         }
 
-        if ($from_date) {
-            $wheres[] = ['events.event_start_utc', '>=', $from_date];
+        if ($request->input('from_date')) {
+            $wheres[] = ['events.event_start_utc', '>=', Carbon::parse($request->input('from_date'))->startOfDay()];
         }
 
-        if ($to_date) {
-            $wheres[] = ['events.event_end_utc', '<=', $to_date];
+        if ($request->input('to_date')) {
+            // The date is inclusive - events on that day count.
+            $wheres[] = ['events.event_start_utc', '<', Carbon::parse($request->input('to_date'))->startOfDay()->addDay()];
         }
 
-        // Get the items we want for this page.
+        return $wheres;
+    }
+
+    /**
+     * List/search devices.
+     */
+    public static function getDevices(Request $request, $page, $size): JsonResponse
+    {
+        $request->validate([
+            'from_date' => 'nullable|date',
+            'to_date' => 'nullable|date',
+        ]);
+
+        // The client asks to sort by one of its table columns; map those onto database columns.  Anything else
+        // gets the default of most recent event first.
+        $sortColumns = [
+            'item_type' => 'devices.item_type',
+            'category' => 'categories.name',
+            'device_category.name' => 'categories.name',
+            'brand' => 'devices.brand',
+            'groupname' => 'groups.name',
+            'repair_status' => 'devices.repair_status',
+            'event_date' => 'events.event_start_utc',
+            'created_at' => 'devices.created_at',
+            'iddevices' => 'devices.iddevices',
+        ];
+
+        $sortBy = $sortColumns[$request->input('sortBy')] ?? 'events.event_start_utc';
+        $sortDesc = strtolower($request->input('sortDesc', 'desc')) === 'asc' ? 'asc' : 'desc';
+
+        $wheres = self::deviceFilters($request);
+
+        // Get the items we want for this page.  Select only device columns - the joined tables share column names
+        // such as created_at, which would otherwise overwrite the device's own.
         $query = Device::with(['deviceEvent.theGroup', 'deviceCategory', 'barriers'])
+            ->select('devices.*')
             ->join('events', 'events.idevents', '=', 'devices.event')
             ->join('groups', 'events.group', '=', 'groups.idgroups')
             ->join('categories', 'devices.category', '=', 'categories.idcategories')
+            ->whereNull('events.deleted_at')
             ->where($wheres)
-            ->orderBy($sortBy, $sortDesc);
+            ->orderBy($sortBy, $sortDesc)
+            ->orderBy('devices.iddevices', $sortDesc);
 
         // Get total info across all pages.
         $count = $query->count();

app/Http/Controllers/DeviceController.php

@@ -62,6 +62,8 @@ public function index($search = null): \Illuminate\View\View
             'clusters' => $clusters,
             'barriers' => \App\Helpers\Fixometer::allBarriers(),
             'brands' => $brands,
+            // The nightly file if it's there; a new machine won't have it until the hourly build has run.
+            'download_url' => file_exists(\App\Console\Commands\ExportRepairData::path()) ? '/exports/repair-data.csv' : '/export/devices',
         ]);
     }