Implemented /metrics endpoint and reimplemented /stats

closes #418 (/metrics endpoint)

Usage:
```yaml
scrape_configs:
  - job_name: 'engelsystem'
    static_configs:
    - targets: ['engelsystem.example.com:80']
```
This commit is contained in:
Igor Scheller 2018-12-18 02:23:44 +01:00 committed by msquare
parent 3c8d0eeb44
commit c5621b82cf
9 changed files with 725 additions and 52 deletions

View File

@ -4,4 +4,9 @@ use FastRoute\RouteCollector;
/** @var RouteCollector $route */
// Pages
$route->get('/credits', 'CreditsController@index');
// Stats
$route->get('/metrics', 'Metrics\\Controller@metrics');
$route->get('/stats', 'Metrics\\Controller@stats');

View File

@ -1,47 +0,0 @@
<?php
use Engelsystem\Database\DB;
use Engelsystem\Models\User\State;
use Engelsystem\Models\User\User;
function guest_stats()
{
$apiKey = config('api_key');
$request = request();
if ($request->has('api_key')) {
if (!empty($apiKey) && $request->input('api_key') == $apiKey) {
$stats = [];
$stats['user_count'] = User::all()->count();
$stats['arrived_user_count'] = State::whereArrived(true)->count();
$done_shifts_seconds = DB::selectOne('
SELECT SUM(`Shifts`.`end` - `Shifts`.`start`)
FROM `ShiftEntry`
JOIN `Shifts` USING (`SID`)
WHERE `Shifts`.`end` < UNIX_TIMESTAMP()
');
$done_shifts_seconds = (int)array_shift($done_shifts_seconds);
$stats['done_work_hours'] = round($done_shifts_seconds / (60 * 60), 0);
$users_in_action = DB::select('
SELECT `Shifts`.`start`, `Shifts`.`end`
FROM `ShiftEntry`
JOIN `Shifts` ON `Shifts`.`SID`=`ShiftEntry`.`SID`
WHERE UNIX_TIMESTAMP() BETWEEN `Shifts`.`start` AND `Shifts`.`end`
');
$stats['users_in_action'] = count($users_in_action);
header('Content-Type: application/json');
raw_output(json_encode($stats));
return;
}
raw_output(json_encode([
'error' => 'Wrong api_key.'
]));
}
raw_output(json_encode([
'error' => 'Missing parameter api_key.'
]));
}

View File

@ -0,0 +1,131 @@
<?php
namespace Engelsystem\Controllers\Metrics;
use Engelsystem\Config\Config;
use Engelsystem\Controllers\BaseController;
use Engelsystem\Http\Exceptions\HttpForbidden;
use Engelsystem\Http\Request;
use Engelsystem\Http\Response;
class Controller extends BaseController
{
/** @var Config */
protected $config;
/** @var MetricsEngine */
protected $engine;
/** @var Request */
protected $request;
/** @var Response */
protected $response;
/** @var Stats */
protected $stats;
/**
* @param Response $response
* @param MetricsEngine $engine
* @param Config $config
* @param Request $request
* @param Stats $stats
*/
public function __construct(
Response $response,
MetricsEngine $engine,
Config $config,
Request $request,
Stats $stats
) {
$this->config = $config;
$this->engine = $engine;
$this->request = $request;
$this->response = $response;
$this->stats = $stats;
}
/**
* @return Response
*/
public function metrics()
{
$now = microtime(true);
$this->checkAuth();
$data = [
$this->config->get('app_name') . ' stats',
'users' => [
'type' => 'gauge',
['labels' => ['state' => 'incoming'], 'value' => $this->stats->newUsers()],
['labels' => ['state' => 'arrived', 'working' => 'no'], 'value' => $this->stats->arrivedUsers(false)],
['labels' => ['state' => 'arrived', 'working' => 'yes'], 'value' => $this->stats->arrivedUsers(true)],
],
'users_working' => [
'type' => 'gauge',
['labels' => ['freeloader' => false], $this->stats->currentlyWorkingUsers(false)],
['labels' => ['freeloader' => true], $this->stats->currentlyWorkingUsers(true)],
],
'work_seconds' => [
'type' => 'gauge',
['labels' => ['state' => 'done'], 'value' => $this->stats->workSeconds(true, false)],
['labels' => ['state' => 'planned'], 'value' => $this->stats->workSeconds(false, false)],
['labels' => ['state' => 'freeloaded'], 'value' => $this->stats->workSeconds(null, true)],
],
'registration_enabled' => ['type' => 'gauge', $this->config->get('registration_enabled')],
];
$data['scrape_duration_seconds'] = [
'type' => 'gauge',
'help' => 'Duration of the current request',
microtime(true) - $this->request->server->get('REQUEST_TIME_FLOAT', $now)
];
return $this->response
->withHeader('Content-Type', 'text/plain; version=0.0.4')
->withContent($this->engine->get('/metrics', $data));
}
/**
* @return Response
*/
public function stats()
{
$this->checkAuth(true);
$data = [
'user_count' => $this->stats->newUsers() + $this->stats->arrivedUsers(),
'arrived_user_count' => $this->stats->arrivedUsers(),
'done_work_hours' => round($this->stats->workSeconds(true) / 60 / 60, 0),
'users_in_action' => $this->stats->currentlyWorkingUsers(),
];
return $this->response
->withHeader('Content-Type', 'application/json')
->withContent(json_encode($data));
}
/**
* Ensure that the if the request is authorized
*
* @param bool $isJson
*/
protected function checkAuth($isJson = false)
{
$apiKey = $this->config->get('api_key');
if (empty($apiKey) || $this->request->get('api_key') == $apiKey) {
return;
}
$message = 'The api_key is invalid';
$headers = [];
if ($isJson) {
$message = json_encode(['error' => $message]);
$headers['Content-Type'] = 'application/json';
}
throw new HttpForbidden($message, $headers);
}
}

View File

@ -0,0 +1,137 @@
<?php
namespace Engelsystem\Controllers\Metrics;
use Engelsystem\Renderer\EngineInterface;
class MetricsEngine implements EngineInterface
{
/**
* Render metrics
*
* @example $data = ['foo' => [['labels' => ['foo'=>'bar'], 'value'=>42]], 'bar'=>123]
*
* @param string $path
* @param mixed[] $data
* @return string
*/
public function get($path, $data = []): string
{
$return = [];
foreach ($data as $name => $list) {
if (is_int($name)) {
$return[] = '# ' . $this->escape($list);
continue;
}
$list = is_array($list) ? $list : [$list];
$name = 'engelsystem_' . $name;
if (isset($list['help'])) {
$return[] = sprintf('# HELP %s %s', $name, $this->escape($list['help']));
unset($list['help']);
}
if (isset($list['type'])) {
$return[] = sprintf('# TYPE %s %s', $name, $list['type']);
unset($list['type']);
}
$list = (!isset($list['value']) || !isset($list['labels'])) ? $list : [$list];
foreach ($list as $row) {
$row = is_array($row) ? $row : [$row];
$return[] = $this->formatData($name, $row);
}
}
return implode("\n", $return);
}
/**
* @param string $path
* @return bool
*/
public function canRender($path): bool
{
return $path == '/metrics';
}
/**
* @param string $name
* @param array|mixed $row
* @see https://prometheus.io/docs/instrumenting/exposition_formats/
* @return string
*/
protected function formatData($name, $row): string
{
return sprintf(
'%s%s %s',
$name,
$this->renderLabels($row),
$this->renderValue($row));
}
/**
* @param array|mixed $row
* @return mixed
*/
protected function renderLabels($row): string
{
$labels = [];
if (!is_array($row) || empty($row['labels'])) {
return '';
}
foreach ($row['labels'] as $type => $value) {
$labels[$type] = $type . '="' . $this->formatValue($value) . '"';
}
return '{' . implode(',', $labels) . '}';
}
/**
* @param array|mixed $row
* @return mixed
*/
protected function renderValue($row)
{
if (isset($row['value'])) {
return $this->formatValue($row['value']);
}
return $this->formatValue(array_pop($row));
}
/**
* @param mixed $value
* @return mixed
*/
protected function formatValue($value)
{
if (is_bool($value)) {
return (int)$value;
}
return $this->escape($value);
}
/**
* @param mixed $value
* @return mixed
*/
protected function escape($value)
{
$replace = [
'\\' => '\\\\',
'"' => '\\"',
"\n" => '\\n',
];
return str_replace(
array_keys($replace),
array_values($replace),
$value
);
}
}

View File

@ -0,0 +1,144 @@
<?php
namespace Engelsystem\Controllers\Metrics;
use Engelsystem\Database\Database;
use Illuminate\Database\Query\Builder as QueryBuilder;
use Illuminate\Database\Query\Expression as QueryExpression;
class Stats
{
/** @var Database */
protected $db;
/**
* @param Database $db
*/
public function __construct(Database $db)
{
$this->db = $db;
}
/**
* The number of not arrived users
*
* @param null $working
* @return int
*/
public function arrivedUsers($working = null): int
{
$query = $this
->getQuery('users')
->join('users_state', 'user_id', '=', 'id')
->where('arrived', '=', 1);
if (!is_null($working)) {
// @codeCoverageIgnoreStart
$query
->leftJoin('UserWorkLog', 'UserWorkLog.user_id', '=', 'users.id')
->leftJoin('ShiftEntry', 'ShiftEntry.UID', '=', 'users.id')
->groupBy('users.id');
$query->where(function ($query) use ($working) {
/** @var QueryBuilder $query */
if ($working) {
$query
->whereNotNull('ShiftEntry.SID')
->orWhereNotNull('UserWorkLog.work_hours');
return;
}
$query
->whereNull('ShiftEntry.SID')
->whereNull('UserWorkLog.work_hours');
});
// @codeCoverageIgnoreEnd
}
return $query
->count();
}
/**
* The number of not arrived users
*
* @return int
*/
public function newUsers(): int
{
return $this
->getQuery('users')
->join('users_state', 'user_id', '=', 'id')
->where('arrived', '=', 0)
->count();
}
/**
* The number of currently working users
*
* @param null $freeloaded
* @return int
* @codeCoverageIgnore
*/
public function currentlyWorkingUsers($freeloaded = null): int
{
$query = $this
->getQuery('users')
->join('ShiftEntry', 'ShiftEntry.UID', '=', 'users.id')
->join('Shifts', 'Shifts.SID', '=', 'ShiftEntry.SID')
->where('Shifts.start', '<=', time())
->where('Shifts.end', '>', time());
if (!is_null($freeloaded)) {
$query->where('ShiftEntry.freeloaded', '=', $freeloaded);
}
return $query->count();
}
/**
* The number of worked shifts
*
* @param bool|null $done
* @param bool|null $freeloaded
* @return int
* @codeCoverageIgnore
*/
public function workSeconds($done = null, $freeloaded = null): int
{
$query = $this
->getQuery('ShiftEntry')
->join('Shifts', 'Shifts.SID', '=', 'ShiftEntry.SID');
if (!is_null($freeloaded)) {
$query->where('freeloaded', '=', $freeloaded);
}
if (!is_null($done)) {
$query->where('end', ($done == true ? '<' : '>='), time());
}
return $query->sum($this->raw('end - start'));
}
/**
* @param string $table
* @return QueryBuilder
*/
protected function getQuery(string $table): QueryBuilder
{
return $this->db
->getConnection()
->table($table);
}
/**
* @param mixed $value
* @return QueryExpression
* @codeCoverageIgnore
*/
protected function raw($value)
{
return $this->db->getConnection()->raw($value);
}
}

View File

@ -26,7 +26,6 @@ class LegacyMiddleware implements MiddlewareInterface
'shift_entries',
'shifts',
'shifts_json_export',
'stats',
'users',
'user_driver_licenses',
'user_password_recovery',
@ -122,10 +121,6 @@ class LegacyMiddleware implements MiddlewareInterface
case 'shifts_json_export':
require_once realpath(__DIR__ . '/../../includes/controller/shifts_controller.php');
shifts_json_export_controller();
/** @noinspection PhpMissingBreakStatementInspection */
case 'stats':
require_once realpath(__DIR__ . '/../../includes/pages/guest_stats.php');
guest_stats();
case 'user_password_recovery':
require_once realpath(__DIR__ . '/../../includes/controller/users_controller.php');
$title = user_password_recovery_title();

View File

@ -0,0 +1,165 @@
<?php
namespace Engelsystem\Test\Unit\Controllers\Metrics;
use Engelsystem\Config\Config;
use Engelsystem\Controllers\Metrics\Controller;
use Engelsystem\Controllers\Metrics\MetricsEngine;
use Engelsystem\Controllers\Metrics\Stats;
use Engelsystem\Http\Exceptions\HttpForbidden;
use Engelsystem\Http\Request;
use Engelsystem\Http\Response;
use Engelsystem\Test\Unit\TestCase;
use PHPUnit\Framework\MockObject\MockObject;
use Symfony\Component\HttpFoundation\ServerBag;
class ControllerTest extends TestCase
{
/**
* @covers \Engelsystem\Controllers\Metrics\Controller::__construct
* @covers \Engelsystem\Controllers\Metrics\Controller::metrics
*/
public function testMetrics()
{
/** @var Response|MockObject $response */
/** @var Request|MockObject $request */
/** @var MetricsEngine|MockObject $engine */
/** @var Stats|MockObject $stats */
/** @var Config $config */
list($response, $request, $engine, $stats, $config) = $this->getMocks();
$request->server = new ServerBag();
$request->server->set('REQUEST_TIME_FLOAT', 0.0123456789);
$engine->expects($this->once())
->method('get')
->willReturnCallback(function ($path, $data) use ($response) {
$this->assertEquals('/metrics', $path);
$this->assertArrayHasKey('users', $data);
$this->assertArrayHasKey('users_working', $data);
$this->assertArrayHasKey('work_seconds', $data);
$this->assertArrayHasKey('registration_enabled', $data);
$this->assertArrayHasKey('scrape_duration_seconds', $data);
return 'metrics return';
});
$response->expects($this->once())
->method('withHeader')
->with('Content-Type', 'text/plain; version=0.0.4')
->willReturn($response);
$response->expects($this->once())
->method('withContent')
->with('metrics return')
->willReturn($response);
$stats->expects($this->exactly(2))
->method('arrivedUsers')
->withConsecutive([false], [true])
->willReturnOnConsecutiveCalls(7, 43);
$stats->expects($this->exactly(2))
->method('currentlyWorkingUsers')
->withConsecutive([false], [true])
->willReturnOnConsecutiveCalls(10, 1);
$stats->expects($this->exactly(3))
->method('workSeconds')
->withConsecutive([true, false], [false, false], [null, true])
->willReturnOnConsecutiveCalls(60 * 37, 60 * 251, 60 * 3);
$this->setExpects($stats, 'newUsers', null, 9);
$config->set('registration_enabled', 1);
$controller = new Controller($response, $engine, $config, $request, $stats);
$controller->metrics();
}
/**
* @covers \Engelsystem\Controllers\Metrics\Controller::stats
* @covers \Engelsystem\Controllers\Metrics\Controller::checkAuth
*/
public function testStats()
{
/** @var Response|MockObject $response */
/** @var Request|MockObject $request */
/** @var MetricsEngine|MockObject $engine */
/** @var Stats|MockObject $stats */
/** @var Config $config */
list($response, $request, $engine, $stats, $config) = $this->getMocks();
$response->expects($this->once())
->method('withHeader')
->with('Content-Type', 'application/json')
->willReturn($response);
$response->expects($this->once())
->method('withContent')
->with(json_encode([
'user_count' => 13,
'arrived_user_count' => 10,
'done_work_hours' => 99,
'users_in_action' => 5
]))
->willReturn($response);
$request->expects($this->once())
->method('get')
->with('api_key')
->willReturn('ApiKey987');
$config->set('api_key', 'ApiKey987');
$stats->expects($this->once())
->method('workSeconds')
->with(true)
->willReturn(60 * 60 * 99.47);
$this->setExpects($stats, 'newUsers', null, 3);
$this->setExpects($stats, 'arrivedUsers', null, 10, $this->exactly(2));
$this->setExpects($stats, 'currentlyWorkingUsers', null, 5);
$controller = new Controller($response, $engine, $config, $request, $stats);
$controller->stats();
}
/**
* @covers \Engelsystem\Controllers\Metrics\Controller::checkAuth
*/
public function testCheckAuth()
{
/** @var Response|MockObject $response */
/** @var Request|MockObject $request */
/** @var MetricsEngine|MockObject $engine */
/** @var Stats|MockObject $stats */
/** @var Config $config */
list($response, $request, $engine, $stats, $config) = $this->getMocks();
$request->expects($this->once())
->method('get')
->with('api_key')
->willReturn('LoremIpsum!');
$config->set('api_key', 'fooBar!');
$controller = new Controller($response, $engine, $config, $request, $stats);
$this->expectException(HttpForbidden::class);
$this->expectExceptionMessage(json_encode(['error' => 'The api_key is invalid']));
$controller->stats();
}
/**
* @return array
*/
protected function getMocks(): array
{
/** @var Response|MockObject $response */
$response = $this->createMock(Response::class);
/** @var Request|MockObject $request */
$request = $this->createMock(Request::class);
/** @var MetricsEngine|MockObject $engine */
$engine = $this->createMock(MetricsEngine::class);
/** @var Stats|MockObject $stats */
$stats = $this->createMock(Stats::class);
$config = new Config();
return array($response, $request, $engine, $stats, $config);
}
}

View File

@ -0,0 +1,69 @@
<?php
namespace Engelsystem\Test\Unit\Controllers\Metrics;
use Engelsystem\Controllers\Metrics\MetricsEngine;
use Engelsystem\Test\Unit\TestCase;
class MetricsEngineTest extends TestCase
{
/**
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::get
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::formatData
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::renderLabels
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::renderValue
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::formatValue
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::escape
*/
public function testGet()
{
$engine = new MetricsEngine();
$this->assertEquals('', $engine->get('/metrics'));
$this->assertEquals('engelsystem_users 13', $engine->get('/metrics', ['users' => 13]));
$this->assertEquals('engelsystem_bool_val 0', $engine->get('/metrics', ['bool_val' => false]));
$this->assertEquals('# Lorem \n Ipsum', $engine->get('/metrics', ["Lorem \n Ipsum"]));
$this->assertEquals(
'engelsystem_foo{lorem="ip\\\\sum"} \\"lorem\\n\\\\ipsum\\"',
$engine->get('/metrics', [
'foo' => ['labels' => ['lorem' => 'ip\\sum'], 'value' => "\"lorem\n\\ipsum\""]
])
);
$this->assertEquals(
'engelsystem_foo_count{bar="14"} 42',
$engine->get('/metrics', ['foo_count' => ['labels' => ['bar' => 14], 'value' => 42],])
);
$this->assertEquals(
'engelsystem_lorem{test="123"} NaN' . "\n" . 'engelsystem_lorem{test="456"} 999.99',
$engine->get('/metrics', [
'lorem' => [
['labels' => ['test' => 123], 'value' => 'NaN'],
['labels' => ['test' => 456], 'value' => 999.99],
],
])
);
$this->assertEquals(
"# HELP engelsystem_test Some help\\n text\n# TYPE engelsystem_test counter\nengelsystem_test 99",
$engine->get('/metrics', ['test' => ['help' => "Some help\n text", 'type' => 'counter', 'value' => 99]])
);
}
/**
* @covers \Engelsystem\Controllers\Metrics\MetricsEngine::canRender
*/
public function testCanRender()
{
$engine = new MetricsEngine();
$this->assertFalse($engine->canRender('/'));
$this->assertFalse($engine->canRender('/metrics.foo'));
$this->assertTrue($engine->canRender('/metrics'));
}
}

View File

@ -0,0 +1,74 @@
<?php
namespace Engelsystem\Test\Unit\Controllers\Metrics;
use Engelsystem\Controllers\Metrics\Stats;
use Engelsystem\Models\User\State;
use Engelsystem\Models\User\User;
use Engelsystem\Test\Unit\HasDatabase;
use Engelsystem\Test\Unit\TestCase;
use Illuminate\Support\Str;
class StatsTest extends TestCase
{
use HasDatabase;
/**
* @covers \Engelsystem\Controllers\Metrics\Stats::newUsers
* @covers \Engelsystem\Controllers\Metrics\Stats::getQuery
* @covers \Engelsystem\Controllers\Metrics\Stats::__construct
*/
public function testNewUsers()
{
$this->initDatabase();
$this->addUsers();
$stats = new Stats($this->database);
$this->assertEquals(2, $stats->newUsers());
}
/**
* @covers \Engelsystem\Controllers\Metrics\Stats::arrivedUsers
*/
public function testArrivedUsers()
{
$this->initDatabase();
$this->addUsers();
$stats = new Stats($this->database);
$this->assertEquals(3, $stats->arrivedUsers());
}
/**
* Add some example users
*/
protected function addUsers()
{
$this->addUser();
$this->addUser();
$this->addUser(['arrived' => 1]);
$this->addUser(['arrived' => 1, 'active' => 1]);
$this->addUser(['arrived' => 1, 'active' => 1]);
}
/**
* @param array $state
*/
protected function addUser(array $state = [])
{
$name = 'user_' . Str::random(5);
$user = new User([
'name' => $name,
'password' => '',
'email' => $name . '@engel.example.com',
'api_key' => '',
]);
$user->save();
$state = new State($state);
$state->user()
->associate($user)
->save();
}
}