Skip to content

Commit 9eef65d

Browse files
nolanpronolanpro
committed
Add metrics for scheduled jobs
1 parent ca4c0bb commit 9eef65d

8 files changed

Lines changed: 246 additions & 14 deletions

File tree

‎ProcessMaker/Console/Commands/ScheduleMultitenantRun.php‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,6 @@ public function handle(): int
6363

6464
throw $e;
6565
} finally {
66-
// Metrics::histogramObserve(
67-
// self::DURATION_METRIC,
68-
// 'Duration of schedule:multitenant-run in seconds',
69-
// [],
70-
// self::DURATION_BUCKETS,
71-
// microtime(true) - $startedAt,
72-
// );
7366
$lock->release();
7467
}
7568
}

‎ProcessMaker/Console/Scheduling/FastCommandEvent.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
use Illuminate\Console\Scheduling\Event;
66
use Illuminate\Console\Scheduling\EventMutex;
77
use Illuminate\Support\Facades\Artisan;
8+
use Symfony\Component\Console\Input\StringInput;
89

910
class FastCommandEvent extends Event
1011
{
@@ -37,6 +38,11 @@ public function __construct(
3738
$this->artisanParameters = $artisanParameters;
3839
}
3940

41+
public function artisanCommandName(): string
42+
{
43+
return (new StringInput($this->artisanCommand))->getFirstArgument() ?? $this->artisanCommand;
44+
}
45+
4046
/**
4147
* Run the command in-process, or shell out when runInBackground is set.
4248
*

‎ProcessMaker/Listeners/ScheduledTaskMetricsSubscriber.php‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
<?php
2+
3+
namespace ProcessMaker\Listeners;
4+
5+
use Illuminate\Console\Events\ScheduledTaskFailed;
6+
use Illuminate\Console\Events\ScheduledTaskFinished;
7+
use Illuminate\Console\Events\ScheduledTaskStarting;
8+
use Illuminate\Console\Scheduling\Event;
9+
use ProcessMaker\Console\Scheduling\FastCommandEvent;
10+
use ProcessMaker\Facades\Metrics;
11+
use ProcessMaker\Multitenancy\Tenant;
12+
use ProcessMaker\Services\MetricsService;
13+
use WeakMap;
14+
15+
class ScheduledTaskMetricsSubscriber
16+
{
17+
public const LAST_SUCCESS_TIMESTAMP = 'schedule_job_last_success_timestamp';
18+
19+
public const LAST_FAILURE_TIMESTAMP = 'schedule_job_last_failure_timestamp';
20+
21+
public const DURATION_SECONDS = 'schedule_job_duration_seconds';
22+
23+
public const RUNS_TOTAL = 'schedule_job_runs_total';
24+
25+
/**
26+
* @var WeakMap<Event, float>
27+
*/
28+
private WeakMap $startedAt;
29+
30+
public function __construct()
31+
{
32+
$this->startedAt = new WeakMap();
33+
}
34+
35+
public function handleStarting(ScheduledTaskStarting $event): void
36+
{
37+
$this->startedAt[$event->task] = microtime(true);
38+
39+
if (Tenant::current() !== null) {
40+
Metrics::clearResolvedInstance(MetricsService::class);
41+
app()->forgetInstance(MetricsService::class);
42+
}
43+
}
44+
45+
public function handleFinished(ScheduledTaskFinished $event): void
46+
{
47+
$job = $this->jobName($event->task);
48+
49+
Metrics::gauge(
50+
self::DURATION_SECONDS,
51+
'Duration of the last scheduled job run in seconds',
52+
['job']
53+
)->set($event->runtime, [$job]);
54+
55+
unset($this->startedAt[$event->task]);
56+
57+
if ($event->task->runInBackground || $event->task->exitCode !== 0) {
58+
return;
59+
}
60+
61+
$this->recordResult($job, 'success');
62+
}
63+
64+
public function handleFailed(ScheduledTaskFailed $event): void
65+
{
66+
$job = $this->jobName($event->task);
67+
68+
if (isset($this->startedAt[$event->task])) {
69+
Metrics::gauge(
70+
self::DURATION_SECONDS,
71+
'Duration of the last scheduled job run in seconds',
72+
['job']
73+
)->set(microtime(true) - $this->startedAt[$event->task], [$job]);
74+
75+
unset($this->startedAt[$event->task]);
76+
}
77+
78+
$this->recordResult($job, 'failure');
79+
}
80+
81+
private function recordResult(string $job, string $status): void
82+
{
83+
$timestampMetric = $status === 'success'
84+
? self::LAST_SUCCESS_TIMESTAMP
85+
: self::LAST_FAILURE_TIMESTAMP;
86+
$resultDescription = $status === 'success' ? 'successful' : 'failed';
87+
88+
Metrics::gauge(
89+
$timestampMetric,
90+
"Unix timestamp of the last {$resultDescription} scheduled job run",
91+
['job']
92+
)->set(now()->timestamp, [$job]);
93+
94+
Metrics::counter(
95+
self::RUNS_TOTAL,
96+
'Total number of scheduled job runs',
97+
['job', 'status']
98+
)->inc([$job, $status]);
99+
}
100+
101+
private function jobName(Event $task): string
102+
{
103+
if ($task instanceof FastCommandEvent) {
104+
return $task->artisanCommandName();
105+
}
106+
107+
if (preg_match('/(?:^|\s)[\'"]?artisan[\'"]?\s+([^\s\'"]+)/', $task->command, $matches)) {
108+
return $matches[1];
109+
}
110+
111+
return $task->getSummaryForDisplay();
112+
}
113+
}

‎ProcessMaker/Multitenancy/PrefixCacheTask.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@
22

33
namespace ProcessMaker\Multitenancy;
44

5+
use Prometheus\Storage\Redis as PrometheusRedis;
56
use Spatie\Multitenancy\Contracts\IsTenant;
67
use Spatie\Multitenancy\Tasks\PrefixCacheTask as SpatiePrefixCacheTask;
78

89
class PrefixCacheTask extends SpatiePrefixCacheTask
910
{
11+
private const LANDLORD_PROMETHEUS_PREFIX = 'PROMETHEUS_';
12+
1013
private $originalSettingsPrefix;
1114

1215
public function makeCurrent(IsTenant $tenant): void
@@ -19,6 +22,8 @@ public function makeCurrent(IsTenant $tenant): void
1922
config()->set('cache.stores.cache_settings.prefix', $tenantSettingsPrefix);
2023
$this->storeName = 'cache_settings';
2124
$this->setCachePrefix($cachePrefix);
25+
26+
PrometheusRedis::setPrefix($cachePrefix . self::LANDLORD_PROMETHEUS_PREFIX);
2227
}
2328

2429
public function forgetCurrent(): void
@@ -28,5 +33,7 @@ public function forgetCurrent(): void
2833
config()->set('cache.stores.cache_settings.prefix', $this->originalSettingsPrefix);
2934
$this->storeName = 'cache_settings';
3035
$this->setCachePrefix($this->originalPrefix);
36+
37+
PrometheusRedis::setPrefix(self::LANDLORD_PROMETHEUS_PREFIX);
3138
}
3239
}

‎ProcessMaker/Multitenancy/SwitchTenant.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ private function overrideConfigs(Application $app, IsTenant $tenant)
140140
// url() helper
141141
app(UrlGenerator::class)->useOrigin($tenant->config['app.url']);
142142

143-
// NOTE: Cache prefix and cache settings prefix are handled in PrefixCacheTask
143+
// NOTE: Cache, cache settings, and Prometheus prefixes are handled in PrefixCacheTask
144144

145145
if (!isset($tenant->config['app.docker_host_url'])) {
146146
// There is no specific override in the tenant's config so set it to the app url

‎ProcessMaker/Providers/ProcessMakerServiceProvider.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
use ProcessMaker\ImportExport\SignalHelper;
4444
use ProcessMaker\Jobs\SmartInbox;
4545
use ProcessMaker\LicensedPackageManifest;
46+
use ProcessMaker\Listeners\ScheduledTaskMetricsSubscriber;
4647
use ProcessMaker\Managers;
4748
use ProcessMaker\Managers\MenuManager;
4849
use ProcessMaker\Managers\ScreenCompiledManager;
@@ -133,6 +134,8 @@ public function register(): void
133134
return new Managers\LoginManager();
134135
});
135136

137+
$this->app->singleton(ScheduledTaskMetricsSubscriber::class);
138+
136139
/*
137140
* Maps our Index Manager as a singleton. The Index Manager is used
138141
* to manage customizations to the search indexer.

‎ProcessMaker/Services/MetricsService.php‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,12 @@
77
use Laravel\Horizon\Contracts\MetricsRepository;
88
use Laravel\Horizon\Contracts\WorkloadRepository;
99
use ProcessMaker\Facades\Metrics;
10-
use ProcessMaker\Multitenancy\Tenant;
1110
use Prometheus\CollectorRegistry;
1211
use Prometheus\Counter;
1312
use Prometheus\Gauge;
1413
use Prometheus\Histogram;
1514
use Prometheus\RenderTextFormat;
1615
use Prometheus\Storage\Redis as PrometheusRedis;
17-
use Redis;
1816
use RuntimeException;
1917

2018
class MetricsService
@@ -46,10 +44,6 @@ public function __construct(private $adapter = null)
4644
if ($adapter === null) {
4745
$redis = app('redis')->client();
4846
$adapter = PrometheusRedis::fromExistingConnection($redis);
49-
if (app()->has(Tenant::BOOTSTRAPPED_TENANT)) {
50-
$tenantInfo = app(Tenant::BOOTSTRAPPED_TENANT);
51-
$adapter->setPrefix('tenant_' . $tenantInfo['id'] . ':PROMETHEUS_');
52-
}
5347
}
5448
$this->collectionRegistry = new CollectorRegistry($adapter);
5549
} catch (Exception $e) {

‎tests/Feature/Console/ScheduledTaskMetricsSubscriberTest.php‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
<?php
2+
3+
namespace Tests\Feature\Console;
4+
5+
use Illuminate\Console\Events\ScheduledTaskFailed;
6+
use Illuminate\Console\Events\ScheduledTaskFinished;
7+
use Illuminate\Console\Events\ScheduledTaskStarting;
8+
use Illuminate\Container\Container;
9+
use Illuminate\Support\Facades\App;
10+
use Illuminate\Support\Facades\Event;
11+
use ProcessMaker\Console\Scheduling\FastSchedule;
12+
use ProcessMaker\Facades\Metrics;
13+
use ProcessMaker\Listeners\ScheduledTaskMetricsSubscriber;
14+
use ProcessMaker\Multitenancy\Tenant;
15+
use ProcessMaker\Services\MetricsService;
16+
use Prometheus\Storage\InMemory;
17+
use ReflectionProperty;
18+
use RuntimeException;
19+
use Tests\TestCase;
20+
21+
class ScheduledTaskMetricsSubscriberTest extends TestCase
22+
{
23+
public function setUpMetrics(): void
24+
{
25+
config(['app.multitenancy' => false]);
26+
27+
App::instance(MetricsService::class, new MetricsService(new InMemory()));
28+
}
29+
30+
public function testRecordsSuccessfulScheduledTaskMetrics(): void
31+
{
32+
$task = (new FastSchedule())->command('emails:send --queue');
33+
34+
Event::dispatch(new ScheduledTaskStarting($task));
35+
$task->exitCode = 0;
36+
Event::dispatch(new ScheduledTaskFinished($task, 2.41));
37+
38+
$metrics = Metrics::renderMetrics();
39+
40+
$this->assertStringContainsString(
41+
ScheduledTaskMetricsSubscriber::LAST_SUCCESS_TIMESTAMP . '{job="emails:send"}',
42+
$metrics
43+
);
44+
$this->assertStringContainsString(
45+
ScheduledTaskMetricsSubscriber::DURATION_SECONDS . '{job="emails:send"} 2.41',
46+
$metrics
47+
);
48+
$this->assertStringContainsString(
49+
ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="success"} 1',
50+
$metrics
51+
);
52+
}
53+
54+
public function testRecordsFailedScheduledTaskMetricsAndDuration(): void
55+
{
56+
$task = (new FastSchedule())->command('emails:send');
57+
58+
Event::dispatch(new ScheduledTaskStarting($task));
59+
Event::dispatch(new ScheduledTaskFailed($task, new RuntimeException('Sending failed')));
60+
61+
$metrics = Metrics::renderMetrics();
62+
63+
$this->assertStringContainsString(
64+
ScheduledTaskMetricsSubscriber::LAST_FAILURE_TIMESTAMP . '{job="emails:send"}',
65+
$metrics
66+
);
67+
$this->assertStringContainsString(
68+
ScheduledTaskMetricsSubscriber::DURATION_SECONDS . '{job="emails:send"}',
69+
$metrics
70+
);
71+
$this->assertStringContainsString(
72+
ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="failure"} 1',
73+
$metrics
74+
);
75+
$this->assertStringNotContainsString(
76+
ScheduledTaskMetricsSubscriber::LAST_SUCCESS_TIMESTAMP . '{job="emails:send"}',
77+
$metrics
78+
);
79+
}
80+
81+
public function testNonZeroExitIsOnlyRecordedAsFailure(): void
82+
{
83+
$task = (new FastSchedule())->command('emails:send');
84+
85+
Event::dispatch(new ScheduledTaskStarting($task));
86+
$task->exitCode = 1;
87+
Event::dispatch(new ScheduledTaskFinished($task, 1.25));
88+
Event::dispatch(new ScheduledTaskFailed($task, new RuntimeException('Exit code 1')));
89+
90+
$metrics = Metrics::renderMetrics();
91+
92+
$this->assertStringContainsString(
93+
ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="failure"} 1',
94+
$metrics
95+
);
96+
$this->assertStringNotContainsString(
97+
ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="success"}',
98+
$metrics
99+
);
100+
}
101+
102+
public function testTenantMetricsServiceIsResolvedAfterTenantBecomesCurrent(): void
103+
{
104+
config(['app.multitenancy' => true]);
105+
app()->instance(config('multitenancy.current_tenant_container_key'), new Tenant(['id' => 123]));
106+
107+
app(ScheduledTaskMetricsSubscriber::class)->handleStarting(
108+
new ScheduledTaskStarting((new FastSchedule())->command('emails:send'))
109+
);
110+
111+
$instances = (new ReflectionProperty(Container::class, 'instances'))->getValue(app());
112+
$this->assertArrayNotHasKey(MetricsService::class, $instances);
113+
114+
app()->forgetInstance(config('multitenancy.current_tenant_container_key'));
115+
}
116+
}

0 commit comments

Comments
 (0)