perf: eager-load relationships in UpdateSearchJob and cache permissions in DiscussionSeeder
- Add relationships() to Seeder base class; DiscussionSeeder implements it to return tags/recipientUsers/recipientGroups for eager loading - UpdateSearchJob calls loadMissing(relationships()) before the map loop, eliminating N×R lazy-load queries per batch - Remove refresh:true from bulk call — synchronous ES refresh was the primary throughput bottleneck during indexing - Cache viewForum permissions in DiscussionSeeder per job instance, replacing N×2 Permission queries per document with a single cached query - Increase seeder batch size from 1000 to 2500
This commit is contained in:
parent
439c74f0a3
commit
5d97187e30
|
|
@ -355,13 +355,13 @@ HELP;
|
||||||
$seeded = null;
|
$seeded = null;
|
||||||
|
|
||||||
while ($continueAt !== null) {
|
while ($continueAt !== null) {
|
||||||
$rangeFrom = max(1, $continueAt - 1000);
|
$rangeFrom = max(1, $continueAt - 2500);
|
||||||
$rangeTo = $continueAt;
|
$rangeTo = $continueAt;
|
||||||
|
|
||||||
if ($seedMissing) {
|
if ($seedMissing) {
|
||||||
$response = (new Builder($client))
|
$response = (new Builder($client))
|
||||||
->index($targetIndex)
|
->index($targetIndex)
|
||||||
->size(1000)
|
->size(2500)
|
||||||
->addQuery(
|
->addQuery(
|
||||||
(new BoolQuery())
|
(new BoolQuery())
|
||||||
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
|
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,8 @@ class UpdateSearchJob extends Job
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->models->loadMissing($this->seeder->relationships());
|
||||||
|
|
||||||
// Preparing body for storing.
|
// Preparing body for storing.
|
||||||
$body = $this->models->map(function (Model $model) {
|
$body = $this->models->map(function (Model $model) {
|
||||||
$document = $this->seeder->toDocument($model);
|
$document = $this->seeder->toDocument($model);
|
||||||
|
|
@ -38,9 +40,8 @@ class UpdateSearchJob extends Job
|
||||||
->flatten(1);
|
->flatten(1);
|
||||||
|
|
||||||
$response = $client->bulk([
|
$response = $client->bulk([
|
||||||
'index' => $this->index,
|
'index' => $this->index,
|
||||||
'body' => $body->toArray(),
|
'body' => $body->toArray(),
|
||||||
'refresh' => true,
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
if (Arr::get($response, 'errors') !== true) {
|
if (Arr::get($response, 'errors') !== true) {
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,12 @@ class DiscussionSeeder extends Seeder
|
||||||
}
|
}
|
||||||
|
|
||||||
public function query(): Builder
|
public function query(): Builder
|
||||||
|
{
|
||||||
|
return Discussion::query()
|
||||||
|
->whereNull('hidden_at');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function relationships(): array
|
||||||
{
|
{
|
||||||
$includes = [];
|
$includes = [];
|
||||||
|
|
||||||
|
|
@ -57,9 +63,7 @@ class DiscussionSeeder extends Seeder
|
||||||
$includes[] = 'recipientGroups';
|
$includes[] = 'recipientGroups';
|
||||||
}
|
}
|
||||||
|
|
||||||
return Discussion::query()
|
return $includes;
|
||||||
->whereNull('hidden_at')
|
|
||||||
->with($includes);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function savingOn(Dispatcher $events, callable $callable)
|
public static function savingOn(Dispatcher $events, callable $callable)
|
||||||
|
|
@ -151,37 +155,53 @@ class DiscussionSeeder extends Seeder
|
||||||
return $document;
|
return $document;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* All viewForum permissions keyed by permission string, loaded once per seeder instance.
|
||||||
|
* Avoids N×2 Permission queries inside the per-document map loop.
|
||||||
|
*/
|
||||||
|
private ?Collection $cachedPermissions = null;
|
||||||
|
private ?Collection $cachedGlobalPermission = null;
|
||||||
|
|
||||||
|
private function allPermissions(): Collection
|
||||||
|
{
|
||||||
|
if ($this->cachedPermissions === null) {
|
||||||
|
$this->cachedPermissions = Permission::query()
|
||||||
|
->where(function ($q) {
|
||||||
|
$q->where('permission', 'viewForum')
|
||||||
|
->orWhere('permission', 'like', 'tag%.viewForum');
|
||||||
|
})
|
||||||
|
->get();
|
||||||
|
|
||||||
|
$this->cachedGlobalPermission = $this->cachedPermissions
|
||||||
|
->where('permission', 'viewForum')
|
||||||
|
->pluck('group_id');
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->cachedPermissions;
|
||||||
|
}
|
||||||
|
|
||||||
protected function groupsForDiscussion(Discussion $discussion): array
|
protected function groupsForDiscussion(Discussion $discussion): array
|
||||||
{
|
{
|
||||||
|
$allPerms = $this->allPermissions();
|
||||||
$permissions = collect();
|
$permissions = collect();
|
||||||
|
|
||||||
$globalPermission = Permission::query()
|
|
||||||
->where('permission', 'viewForum')
|
|
||||||
->pluck('group_id');
|
|
||||||
|
|
||||||
if ($this->extensionEnabled('flarum-tags')) {
|
if ($this->extensionEnabled('flarum-tags')) {
|
||||||
/** @var Collection $tags */
|
/** @var Collection $tags */
|
||||||
$tags = $discussion->tags;
|
$tags = $discussion->tags;
|
||||||
|
|
||||||
$tagPermissions = Permission::query()
|
$permissions = $tags->map(function (Tag $tag) use ($allPerms) {
|
||||||
->whereIn(
|
$tagPerms = $allPerms->where('permission', "tag$tag->id.viewForum");
|
||||||
'permission',
|
|
||||||
$tags->pluck('id')->map(fn (int $id) => "tag$id.viewForum")
|
|
||||||
)->get();
|
|
||||||
|
|
||||||
$permissions = $tags->map(function (Tag $tag) use ($tagPermissions) {
|
|
||||||
$permissions = $tagPermissions->where('permission', "tag$tag->id.viewForum");
|
|
||||||
|
|
||||||
if ($tag->is_restricted) {
|
if ($tag->is_restricted) {
|
||||||
$permissions = $permissions->add(['group_id' => Group::ADMINISTRATOR_ID]);
|
$tagPerms = $tagPerms->add(['group_id' => Group::ADMINISTRATOR_ID]);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $permissions->pluck('group_id');
|
return $tagPerms->pluck('group_id');
|
||||||
})->flatten();
|
})->flatten();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!$discussion->is_private && $permissions->isEmpty()) {
|
if (!$discussion->is_private && $permissions->isEmpty()) {
|
||||||
$permissions = $globalPermission;
|
$permissions = $this->cachedGlobalPermission;
|
||||||
}
|
}
|
||||||
|
|
||||||
return $permissions->toArray();
|
return $permissions->toArray();
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,12 @@ abstract class Seeder
|
||||||
|
|
||||||
abstract public function toDocument(Model $model): Document;
|
abstract public function toDocument(Model $model): Document;
|
||||||
|
|
||||||
|
/** Relationships to eager-load on the collection before calling toDocument(). */
|
||||||
|
public function relationships(): array
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
protected function extensionEnabled(string $extension): bool
|
protected function extensionEnabled(string $extension): bool
|
||||||
{
|
{
|
||||||
/** @var ExtensionManager $manager */
|
/** @var ExtensionManager $manager */
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue