fix: retry fill ES queries on transient connection failure

This commit is contained in:
Bart van Bragt 2026-04-15 14:07:43 +02:00
parent 08353d2473
commit 557604e426
1 changed files with 44 additions and 11 deletions

View File

@ -16,6 +16,7 @@ use Blomstra\Search\Jobs\Job;
use Blomstra\Search\Jobs\UpdateSearchJob; use Blomstra\Search\Jobs\UpdateSearchJob;
use Blomstra\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Elasticsearch\Client; use Elasticsearch\Client;
use Elasticsearch\Common\Exceptions\ElasticsearchException;
use Flarum\Settings\SettingsRepositoryInterface; use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Container\Container;
@ -361,17 +362,7 @@ HELP;
$rangeTo = $continueAt; $rangeTo = $continueAt;
if ($seedMissing) { if ($seedMissing) {
$response = (new Builder($client)) $seeded = $this->queryIndexedIds($client, $targetIndex, $seeder->joinRelation(), $rangeFrom, $rangeTo);
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $seeder->joinRelation()))
)
->search();
$seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} }
/** @var Collection $collection */ /** @var Collection $collection */
@ -571,6 +562,48 @@ HELP;
]; ];
} }
/**
* Query ES for rawIds already indexed in $targetIndex for the given $joinRelation and ID range.
* Retries on transient ES failures (NoNodesAvailableException / other ElasticsearchException)
* by sleeping past the StaticNoPingConnectionPool dead-node timeout (default 60 s) before
* each retry, giving the pool a chance to resurface the node.
*/
protected function queryIndexedIds(
Client $client,
string $targetIndex,
string $joinRelation,
int $rangeFrom,
int $rangeTo,
int $maxRetries = 10
): array {
$attempt = 0;
while (true) {
try {
$response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $joinRelation))
)
->search();
return Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} catch (ElasticsearchException $e) {
$attempt++;
if ($attempt >= $maxRetries) {
throw $e;
}
$this->warn("ES error on range {$rangeFrom}{$rangeTo} (attempt {$attempt}/{$maxRetries}): {$e->getMessage()}. Waiting 65 s before retry…");
sleep(65); // outlast the StaticNoPingConnectionPool dead-node window (default 60 s)
}
}
}
protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int
{ {
$raw = $settings->get("blomstra-search.continued-at.$type"); $raw = $settings->get("blomstra-search.continued-at.$type");