From e2e2d1428fa4184ace2529cfb16dae3ebf112148 Mon Sep 17 00:00:00 2001 From: Bart van Bragt Date: Thu, 9 Apr 2026 23:30:57 +0200 Subject: [PATCH] feat: restructure index as parent-child documents with blue-green rebuilds - Switch to ES parent/child join_field: discussion docs hold all metadata, post docs hold content only. Routing ensures parent and child land on the same shard. Removes the old flat-document approach where post data was duplicated onto every comment document. - Add HasChildQuery with inner_hits so the best-matching post ID is surfaced as mostRelevantPost without a second DB query. - Blue-green index rebuilds: --recreate writes into a timestamped pending index, --swap atomically promotes it via alias. Interrupted builds are resumable with --recreate --continue. --swap requires confirmation before proceeding. - Index hidden posts; non-moderators are filtered at query time via is_hidden on the has_child clause. Core\Hidden and Core\Restored trigger re-indexing so moderators with post.hide can search hidden posts. --- src/Api/Controllers/SearchController.php | 249 ++++++------- src/Commands/BuildCommand.php | 436 +++++++++++++++-------- src/Elasticsearch/HasChildQuery.php | 57 +++ src/Jobs/DeletingJob.php | 3 +- src/Jobs/Job.php | 8 +- src/Jobs/UpdateSearchJob.php | 3 +- src/Jobs/ViewsSearchJob.php | 28 +- src/Provider.php | 8 +- src/Save/Document.php | 12 +- src/Seeders/CommentSeeder.php | 81 ++--- src/Seeders/DiscussionSeeder.php | 52 ++- src/Seeders/Seeder.php | 58 +-- 12 files changed, 593 insertions(+), 402 deletions(-) create mode 100644 src/Elasticsearch/HasChildQuery.php diff --git a/src/Api/Controllers/SearchController.php b/src/Api/Controllers/SearchController.php index 938a3f7..32bddb2 100644 --- a/src/Api/Controllers/SearchController.php +++ b/src/Api/Controllers/SearchController.php @@ -12,10 +12,12 @@ namespace Blomstra\Search\Api\Controllers; +use Blomstra\Search\Elasticsearch\HasChildQuery; use Blomstra\Search\Elasticsearch\MatchPhraseQuery; use Blomstra\Search\Elasticsearch\MatchQuery; use Blomstra\Search\Elasticsearch\TermsQuery; -use Blomstra\Search\Save\Document as ElasticDocument; +use Blomstra\Search\Searchers\CommentPostSearcher; +use Blomstra\Search\Searchers\DiscussionSearcher; use Blomstra\Search\Searchers\Searcher; use Elasticsearch\Client; use Flarum\Api\Controller\ListDiscussionsController; @@ -33,9 +35,9 @@ use Illuminate\Support\Arr; use Illuminate\Support\Collection; use Illuminate\Support\Str; use Psr\Http\Message\ServerRequestInterface; +use Psr\Log\LoggerInterface; use Spatie\ElasticsearchQueryBuilder\Builder; use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery; -use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery; use Spatie\ElasticsearchQueryBuilder\Sorts\Sort; use Tobscure\JsonApi\Document; @@ -51,117 +53,100 @@ class SearchController extends ListDiscussionsController 'view_count' => 'view_count', ]; - protected Collection $searchers; protected bool $matchSentences; protected bool $matchWords; + protected ?Searcher $discussionSearcher; + protected ?Searcher $postSearcher; public function __construct(protected Client $elastic, protected UrlGenerator $uri, Container $container, SettingsRepositoryInterface $settings) { - $this->searchers = $this->gatherSearchers($container->tagged('blomstra.search.searchers')); - $this->matchSentences = true; - $this->matchWords = true; - } + $this->matchWords = true; - protected function gatherSearchers(iterable $searchers) - { - return collect($searchers) - ->map(fn ($searcher) => new $searcher()) - ->filter(fn (Searcher $searcher) => $searcher->enabled()); + $searchers = collect($container->tagged('blomstra.search.searchers')); + + $this->discussionSearcher = $searchers->first(fn ($s) => $s instanceof DiscussionSearcher); + $this->postSearcher = $searchers->first(fn ($s) => $s instanceof CommentPostSearcher); } protected function data(ServerRequestInterface $request, Document $document) { - // Not used for now. - $type = Arr::get($request->getQueryParams(), 'type'); - - $actor = RequestUtil::getActor($request); - + $actor = RequestUtil::getActor($request); $filters = $this->extractFilter($request); - - $search = $this->getSearch($filters); - - $limit = $this->extractLimit($request); - $offset = $this->extractOffset($request); - + $search = $this->getSearch($filters); + $limit = $this->extractLimit($request); + $offset = $this->extractOffset($request); $include = array_merge($this->extractInclude($request), ['state']); - $filterQuery = BoolQuery::create(); + $query = BoolQuery::create() + // Always restrict to discussion documents; posts are only searched via has_child. + ->add(TermQuery::create('join_field', 'discussion'), 'filter'); if (!empty($search)) { - if ($this->matchSentences) { - $filterQuery->add($this->sentenceMatch($search)); - } - if ($this->matchWords) { - $filterQuery->add($this->wordMatch($search, 'and')); - } - if ($this->matchWords) { - $filterQuery->add($this->wordMatch($search, 'or')); - } + $query->add($this->buildTextQuery($search, $actor)); } + $this->addFilters($query, $actor, $filters); + $builder = (new Builder($this->elastic)) ->index(resolve('blomstra.search.elastic_index')) ->size($limit + 1) - ->from($this->extractOffset($request)) - ->addQuery( - $this->addFilters($filterQuery, $actor, $filters) - ); + ->from($offset) + ->addQuery($query); $knownSortFields = array_merge(array_values($this->translateSort), ['rawId']); + $logger = resolve(LoggerInterface::class); + + $phpSortField = null; + $phpSortDir = 'desc'; foreach ($this->extractSort($request) as $field => $direction) { $translated = $this->translateSort[$field] ?? $field; if (!in_array($translated, $knownSortFields)) { - resolve(\Psr\Log\LoggerInterface::class)->warning( - "blomstra/search: unknown sort field \"{$field}\", ignoring." - ); + $logger->warning("blomstra/search: unknown sort field \"{$field}\", ignoring."); continue; } $builder->addSort(new Sort($translated, $direction)); + + if ($phpSortField === null && $translated !== 'rawId') { + $phpSortField = $translated; + $phpSortDir = $direction; + } } $response = $builder->search(); Discussion::setStateUser($actor); - // Eager load groups for use in the policies (isAdmin check) if (in_array('mostRelevantPost.user', $include)) { $include[] = 'mostRelevantPost.user.groups'; - // If the first level of the relationship wasn't explicitly included, - // add it so the code below can look for it if (!in_array('mostRelevantPost', $include)) { $include[] = 'mostRelevantPost'; } } - // we need to retrieve all discussion ids and when the results are posts, - // their ids as most relevant post id + // All hits are discussion documents. Extract the best-matching post ID from + // inner_hits when a has_child clause matched (i.e. the match came from a post). $results = Collection::make(Arr::get($response, 'hits.hits')) ->map(function ($hit) { - $type = $hit['_source']['type']; - $id = Str::after($hit['_source']['id'], "$type:"); + // _id is "discussions:123" — parse the numeric part directly. + $discussionId = Str::after($hit['_id'], 'discussions:'); - if ($type === 'posts') { - return [ - 'most_relevant_post_id' => $id, - 'weight' => Arr::get($hit, 'sort.0'), - ]; - } else { - return [ - 'discussion_id' => $id, - 'weight' => Arr::get($hit, 'sort.0'), - ]; - } + // rawId on the inner hit gives us the integer post ID. + $bestPostId = Arr::get($hit, 'inner_hits.best_post.hits.hits.0._source.rawId'); + + return [ + 'discussion_id' => $discussionId, + 'most_relevant_post_id' => $bestPostId, + 'weight' => Arr::get($hit, 'sort.0', Arr::get($hit, '_score', 0)), + ]; }); $document->addPaginationLinks( - $this->uri->to('api')->route('blomstra.search', [ - 'type' => 'discussions', - ]), + $this->uri->to('api')->route('blomstra.search', ['type' => 'discussions']), $request->getQueryParams(), $offset, $limit, @@ -171,28 +156,25 @@ class SearchController extends ListDiscussionsController $results = $results->take($limit); $discussions = Discussion::query() - ->select('discussions.*') - ->join('posts', 'posts.discussion_id', 'discussions.id') - // Extra safety to prevent leaking hidden discussion (titles) towards search results. - ->when($actor->isGuest() || !$actor->hasPermission('discussion.hide'), fn ($query) => $query->whereNull('discussions.hidden_at')) - ->where(function ($query) use ($results) { - $query - ->whereIn('discussions.id', $results->pluck('discussion_id')->filter()) - ->orWhereIn('posts.id', $results->pluck('most_relevant_post_id')->filter()); - }) + ->when( + $actor->isGuest() || !$actor->hasPermission('discussion.hide'), + fn ($q) => $q->whereNull('hidden_at') + ) + ->whereIn('id', $results->pluck('discussion_id')->filter()) ->get() ->each(function (Discussion $discussion) use ($results) { - if (in_array($discussion->id, $results->pluck('discussion_id')->toArray())) { - $discussion->most_relevant_post_id = $discussion->first_post_id; - $discussion->weight = $results->firstWhere('discussion_id', $discussion->id)['weight'] ?? 0; - } else { - $post = $discussion->posts()->whereIn('id', $results->pluck('most_relevant_post_id'))->first(); - $discussion->most_relevant_post_id = $post?->id ?? $discussion->first_post_id; - $discussion->weight = $results->firstWhere('most_relevant_post_id', $post?->id)['weight'] ?? 0; - } + $result = $results->firstWhere('discussion_id', $discussion->id); + + $discussion->most_relevant_post_id = $result['most_relevant_post_id'] + ?? $discussion->first_post_id; + $discussion->weight = $result['weight'] ?? 0; }) ->keyBy('id') - ->sortByDesc('weight') + ->when( + $phpSortField, + fn ($c) => $phpSortDir === 'desc' ? $c->sortByDesc($phpSortField) : $c->sortBy($phpSortField), + fn ($c) => $c->sortByDesc('weight') + ) ->unique(); $this->loadRelations($discussions, $include); @@ -210,27 +192,63 @@ class SearchController extends ListDiscussionsController return $discussions; } - protected function getDocument(string $type): ?ElasticDocument + /** + * Build the text-matching portion of the query. + * + * Discussion titles are matched directly (filtered to join_field=discussion). + * Post bodies are matched via has_child with score_mode=sum so discussions + * with many matching posts score higher than those with a single strong match. + * inner_hits returns the best-scoring post for use as mostRelevantPost. + * Hidden posts are only included in matching for users with post.hide permission. + */ + protected function buildTextQuery(string $search, User $actor): BoolQuery { - $documents = resolve(Container::class)->tagged('blomstra.search.documents'); + $textQuery = BoolQuery::create(); - return collect($documents)->first(function (ElasticDocument $document) use ($type) { - return $document->type() === $type; - }); + if ($this->discussionSearcher?->enabled()) { + $textQuery->add($this->buildShouldClauses($search, $this->discussionSearcher->boost()), 'should'); + } + + if ($this->postSearcher?->enabled()) { + $postQuery = $this->buildShouldClauses($search, $this->postSearcher->boost()); + + // Guests and non-moderators may not see hidden posts; exclude them from child matching. + if ($actor->isGuest() || !$actor->hasPermission('post.hide')) { + $postQuery->add(TermQuery::create('is_hidden', false), 'filter'); + } + + $textQuery->add( + HasChildQuery::create('post', $postQuery)->withInnerHits(), + 'should' + ); + } + + return $textQuery; + } + + protected function buildShouldClauses(string $search, float $boost): BoolQuery + { + $should = BoolQuery::create(); + + if ($this->matchSentences) { + $should->add((new MatchPhraseQuery('content', $search))->boost(2 * $boost), 'should'); + } + if ($this->matchWords) { + $should->add((new MatchQuery('content', $search))->operator('and')->boost(1.8 * $boost), 'should'); + $should->add((new MatchQuery('content', $search))->operator('or')->boost(0.8 * $boost), 'should'); + } + + return $should; } protected function extensionEnabled(string $extension): bool { - /** @var ExtensionManager $manager */ - $manager = resolve(ExtensionManager::class); - - return $manager->isEnabled($extension); + return resolve(ExtensionManager::class)->isEnabled($extension); } - protected function addFilters(BoolQuery $query, User $actor, array $filters = []): BoolQuery + protected function addFilters(BoolQuery $query, User $actor, array $filters = []): void { - $groups = $this->getGroups($actor); - + $groups = $this->getGroups($actor); $onlyPrivate = Str::contains($filters['q'] ?? '', 'is:private'); $subQuery = BoolQuery::create() @@ -263,55 +281,12 @@ class SearchController extends ListDiscussionsController } } - $query->add( - $subQuery, - 'filter' - ); - - return $query; - } - - protected function boolQuery(Query $parent, float $boost = 1): Query - { - $bool = new BoolQuery(); - - /** @var Searcher $searcher */ - foreach ($this->searchers as $searcher) { - $searcher = new $searcher(); - - $bool->add( - BoolQuery::create() - ->add(TermQuery::create('type', $searcher->type()), 'filter') - ->add(clone $parent->boost($boost * $searcher->boost())), - 'should' - ); - } - - return $bool; - } - - protected function sentenceMatch(string $q): Query - { - $query = (new MatchPhraseQuery('content', $q)); - - return $this->boolQuery($query, 2); - } - - protected function wordMatch(string $q, string $operator = 'or'): Query - { - $query = (new MatchQuery('content', $q)) - ->operator($operator); - - $boost = $operator === 'and' ? 1.8 : .8; - - return $this->boolQuery($query, $boost); + $query->add($subQuery, 'filter'); } protected function getGroups(User $actor): Collection { - /** @var Collection $groups */ $groups = $actor->groups->pluck('id'); - $groups->add(Group::GUEST_ID); if ($actor->is_email_confirmed) { @@ -327,9 +302,7 @@ class SearchController extends ListDiscussionsController if ($search) { $q = collect(explode(' ', $search)) - ->filter(function (string $part) { - return $part !== 'is:private'; - }) + ->filter(fn (string $part) => $part !== 'is:private') ->filter() ->join(' '); diff --git a/src/Commands/BuildCommand.php b/src/Commands/BuildCommand.php index 8941432..7428825 100644 --- a/src/Commands/BuildCommand.php +++ b/src/Commands/BuildCommand.php @@ -33,33 +33,290 @@ class BuildCommand extends Command {--max-id= : Limits for each object the number of items to seed} {--throttle= : Number of seconds to wait between pushing to the queue} {--only= : type to run seeder for, eg discussions or posts} - {--recreate : create or recreate the index} - {--mapping : recreate the mapping} - {--continue : continue each object type where you left off} - {--seed-missing : attempt to seed only objects that are missing in the index}'; + {--recreate : Build into a new timestamped pending index (never touches the live alias)} + {--discard : With --recreate, discard any in-progress pending build and start completely fresh} + {--build-only : Deprecated no-op; --recreate no longer auto-swaps (kept for scripting compatibility)} + {--mapping : Push updated mapping to the active index without rebuilding} + {--continue : Resume each seeder from where it left off (use with --recreate when resuming an interrupted build)} + {--seed-missing : Seed only documents missing from the active index} + {--swap : Atomically swap the alias from the active index to the completed pending index} + {--i-am-sure : Required with --swap; confirms the queue is fully drained before promoting}'; - protected $description = 'Rebuilds the complete search server with its documents.'; + protected $description = 'Rebuilds the complete search index.'; - public function handle(Container $container) + public function handle(Container $container): void { - /** @var string $index */ - $index = $container->make('blomstra.search.elastic_index'); - - /** @var array|string[] $seeders */ - $seeders = $container->tagged('blomstra.search.seeders'); - - /** @var Queue $queue */ - $queue = $container->make(Queue::class); - /** @var Client $client */ $client = $container->make(Client::class); /** @var SettingsRepositoryInterface $settings */ $settings = $container->make(SettingsRepositoryInterface::class); - // Elastic scheme definition. - $properties = [ + /** @var string $alias */ + $alias = $container->make('blomstra.search.elastic_index'); + + // --swap: promote the pending index to live, nothing else. + if ($this->option('swap')) { + if (!$this->option('i-am-sure')) { + $this->warn('--swap promotes the pending index to live. This cannot be undone.'); + $this->line(''); + $this->line('Before swapping, ensure the queue is fully drained:'); + $this->line(' php flarum queue:work --stop-when-empty'); + $this->line(''); + + if (!$this->confirm('Has the queue been drained? Proceed with the swap?')) { + return; + } + } + + $this->performSwap($client, $alias, $settings); + return; + } + + /** @var Queue $queue */ + $queue = $container->make(Queue::class); + + /** @var Seeder[] $seeders */ + $seeders = collect($container->tagged('blomstra.search.seeders')); + + // Determine which concrete index to write into. + if ($this->option('recreate')) { + // If a pending build exists, require an explicit choice before touching anything. + $pending = $settings->get('blomstra-search.pending-index'); + if ($pending && $client->indices()->exists(['index' => $pending]) + && !$this->option('continue') && !$this->option('discard') + ) { + $this->warn("An in-progress build already exists: $pending"); + $this->line(''); + $this->line('Choose one of:'); + $this->line(' --recreate --continue Resume each seeder from where it left off'); + $this->line(' --recreate --discard Discard this build and start completely fresh'); + return; + } + + $targetIndex = $this->preparePendingIndex($client, $alias, $settings, $seeders); + } else { + // --mapping / --seed-missing / normal: write directly through the alias. + $targetIndex = $alias; + } + + // Apply or update the mapping. + if ($this->option('recreate') || $this->option('mapping')) { + $client->indices()->putMapping([ + 'index' => $targetIndex, + 'body' => $this->mappingProperties(), + ]); + } + + // Run seeders into $targetIndex. + $only = $this->option('only'); + + /** @var Seeder $seeder */ + foreach ($seeders as $seeder) { + if ($only && $seeder->type() !== $only) { + continue; + } + + $total = 0; + + $continueAt = $this->option('continue') + ? ($this->getContinueAt($settings, $seeder->type()) ?? $seeder->query()->max('id')) + : $seeder->query()->max('id'); + + $seeded = null; + + while ($continueAt !== null) { + $rangeFrom = max(1, $continueAt - 1000); + $rangeTo = $continueAt; + + if ($this->option('seed-missing')) { + $response = (new Builder($client)) + ->index($targetIndex) + ->size(1000) + ->addQuery( + (new BoolQuery()) + ->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo)) + ->add(TermQuery::create('join_field', $seeder->joinRelation())) + ) + ->search(); + + $seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId'); + } + + /** @var Collection $collection */ + $collection = $seeder->query() + ->latest('id') + ->whereBetween('id', [$rangeFrom, $rangeTo]) + ->when($this->option('max-id'), fn ($q, $id) => $q->where('id', '<=', $id)) + ->when($seeded, fn ($q, $seeded) => $q->whereNotIn('id', $seeded)) + ->get(); + + $min = $collection->min('id'); + + if ($this->option('seed-missing') && $collection->isEmpty()) { + $continueAt = $rangeFrom > 2 ? $rangeFrom - 1 : null; + } else { + $continueAt = $min && $min > 2 ? $min - 1 : null; + } + + if ($collection->isNotEmpty()) { + $queue->pushOn(Job::$onQueue, new UpdateSearchJob($collection, $seeder, $targetIndex)); + } + + $this->info("IDs {$rangeFrom}–{$rangeTo} | type: {$seeder->type()} | queued: {$collection->count()}."); + + $total += $collection->count(); + $this->setContinueAt($settings, $seeder->type(), $continueAt); + + if ($throttle = $this->option('throttle')) { + $this->info("Throttling for $throttle seconds"); + sleep($throttle); + } + } + + $this->info("Queued a total of $total {$seeder->type()} for indexing."); + } + + if ($this->option('recreate')) { + $this->info("Build complete. Drain the queue, then promote '$targetIndex' to live:"); + $this->line(" php flarum queue:work --stop-when-empty"); + $this->line(" php flarum blomstra:search:index --swap"); + } + } + + /** + * Prepare the concrete index that the current build will write into. + * + * - If a pending build exists and --discard is not set, resume it. + * - Otherwise create a fresh timestamped index and save it as pending. + */ + protected function preparePendingIndex( + Client $client, + string $alias, + SettingsRepositoryInterface $settings, + iterable $seeders + ): string { + $pending = $settings->get('blomstra-search.pending-index'); + + // --discard: throw away any in-progress build. + if ($this->option('discard') && $pending) { + if ($client->indices()->exists(['index' => $pending])) { + $client->indices()->delete(['index' => $pending]); + $this->info("Discarded pending index: $pending"); + } + $pending = null; + $settings->set('blomstra-search.pending-index', null); + } + + // Resume an existing pending build (only reached when --continue or --discard was specified). + if ($pending && $client->indices()->exists(['index' => $pending])) { + $this->info("Resuming pending index build: $pending"); + return $pending; + } + + // Fresh build: create a new timestamped concrete index. + $pending = $alias . '_' . date('YmdHis'); + + $client->indices()->create([ + 'index' => $pending, + 'body' => ['settings' => $this->indexSettings($settings)], + ]); + + $settings->set('blomstra-search.pending-index', $pending); + + // Clear per-seeder progress so the fresh build starts from the top. + foreach ($seeders as $seeder) { + $this->setContinueAt($settings, $seeder->type(), null); + } + + $this->info("Created pending index: $pending"); + + return $pending; + } + + /** + * Atomically promote the pending index to live by swapping the alias. + * + * Handles both the normal alias-swap case and the one-time migration from + * an older installation where the configured name was a concrete index. + */ + protected function performSwap(Client $client, string $alias, SettingsRepositoryInterface $settings): void + { + $pending = $settings->get('blomstra-search.pending-index'); + + if (!$pending || !$client->indices()->exists(['index' => $pending])) { + $this->error('No pending index found. Run --recreate [--build-only] first.'); + return; + } + + $aliasExists = (bool) $client->indices()->existsAlias(['name' => $alias]); + $indexExists = !$aliasExists && (bool) $client->indices()->exists(['index' => $alias]); + + if ($aliasExists) { + // Normal case: atomic remove-old / add-new in a single API call. + $result = $client->indices()->getAlias(['name' => $alias]); + $oldIndex = array_key_first($result); + + $client->indices()->updateAliases([ + 'body' => ['actions' => [ + ['remove' => ['index' => $oldIndex, 'alias' => $alias]], + ['add' => ['index' => $pending, 'alias' => $alias]], + ]], + ]); + + $this->info("Alias '$alias' → '$pending'. Swap complete."); + $client->indices()->delete(['index' => $oldIndex, 'ignore_unavailable' => true]); + $this->info("Deleted old index: $oldIndex"); + } else { + // One-time migration: the configured name was a concrete index (old install). + // Brief downtime window here is acceptable — it only happens once. + if ($indexExists) { + $client->indices()->delete(['index' => $alias]); + $this->info("Deleted legacy concrete index: $alias"); + } + + $client->indices()->putAlias(['index' => $pending, 'name' => $alias]); + } + + $settings->set('blomstra-search.active-index', $pending); + $settings->set('blomstra-search.pending-index', null); + + $this->info("Alias '$alias' → '$pending'. Swap complete."); + } + + protected function indexSettings(SettingsRepositoryInterface $settings): array + { + return [ + 'index.max_ngram_diff' => 7, + 'analysis' => [ + 'analyzer' => [ + 'flarum_analyzer' => [ + 'type' => $settings->get('blomstra-search.analyzer-language') ?: 'english', + ], + 'flarum_analyzer_partial' => [ + 'type' => 'custom', + 'tokenizer' => 'standard', + 'filter' => ['lowercase', 'partial_search_filter'], + ], + ], + 'filter' => [ + 'partial_search_filter' => [ + 'type' => 'ngram', + 'min_gram' => 3, + 'max_gram' => 10, + 'token_chars' => ['letter', 'digit', 'symbol'], + ], + ], + ], + ]; + } + + protected function mappingProperties(): array + { + return [ 'properties' => [ + 'join_field' => ['type' => 'join', 'relations' => ['discussion' => 'post']], + 'discussion_id' => ['type' => 'integer'], 'content' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'], 'rawId' => ['type' => 'integer'], 'created_at' => ['type' => 'date'], @@ -72,149 +329,20 @@ class BuildCommand extends Command 'recipient_users' => ['type' => 'integer'], 'comment_count' => ['type' => 'integer'], 'view_count' => ['type' => 'integer'], + 'is_hidden' => ['type' => 'boolean'], ], ]; - - // Remove/delete the whole index. - if ($this->option('recreate')) { - // Flush the index. - $client->indices()->delete([ - 'index' => $index, - 'ignore_unavailable' => true, - ]); - - // Create a new index. - $client->indices()->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'index.max_ngram_diff' => 7, - 'analysis' => [ - 'analyzer' => [ - 'flarum_analyzer' => [ - 'type' => $settings->get('blomstra-search.analyzer-language') ?: 'english', - ], - 'flarum_analyzer_partial' => [ - 'type' => 'custom', - 'tokenizer' => 'standard', - 'filter' => [ - 'lowercase', - 'partial_search_filter', - ], - ], - ], - 'filter' => [ - 'partial_search_filter' => [ - 'type' => 'ngram', - 'min_gram' => 3, - 'max_gram' => 10, - 'token_chars' => ['letter', 'digit', 'symbol'], - ], - ], - ], - ], - ], - ]); - } - - // Create the index. - if ($this->option('recreate') || $this->option('mapping')) { - $client->indices()->putMapping([ - 'index' => $index, - 'body' => $properties, - ]); - } - - // Seed only a specific resource. - $only = $this->option('only'); - - /** @var Seeder $seeder */ - foreach ($seeders as $seeder) { - if ($only && $seeder->type() !== $only) { - continue; - } - - $total = 0; - - $continueAt = $this->option('continue') - ? ($this->continueAt($seeder->type()) ?? $seeder->query()->max('id')) - : $seeder->query()->max('id'); - - $seeded = null; - - while ($continueAt !== null) { - $rangeFrom = max(1, $continueAt - 1000); - $rangeTo = $continueAt; - - if ($this->option('seed-missing')) { - $response = (new Builder($client)) - ->index($index) - ->size(1000) - ->addQuery( - (new BoolQuery()) - ->add((new RangeQuery('rawId')) - ->gte($rangeFrom) - ->lte($rangeTo)) - ->add(TermQuery::create('type', $seeder->type())) - ) - ->search(); - - $seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId'); - } - - /** @var Collection $collection */ - $collection = $seeder->query() - ->latest('id') - ->whereBetween('id', [$rangeFrom, $rangeTo]) - ->when($this->option('max-id'), function ($query, $id) { - $query->where('id', '<=', $id); - }) - ->when($seeded, fn ($query, $seeded) => $query->whereNotIn('id', $seeded)) - ->get(); - - $min = $collection->min('id'); - - if ($this->option('seed-missing') && $collection->isEmpty()) { - // All docs in this range are already indexed; advance past the range bottom. - $continueAt = $rangeFrom > 2 ? $rangeFrom - 1 : null; - } else { - $continueAt = $min && $min > 2 ? $min - 1 : null; - } - - if ($collection->isNotEmpty()) { - $queue->pushOn(Job::$onQueue, new UpdateSearchJob($collection, $seeder)); - } - - $this->info("IDs {$rangeFrom}–{$rangeTo} | type: {$seeder->type()} | queued: {$collection->count()}."); - - $total += $collection->count(); - - $this->continueAt( - $seeder->type(), - $continueAt - ); - - if ($throttle = $this->option('throttle')) { - $this->info("Throttling for $throttle seconds"); - sleep($throttle); - } - } - - $this->info("Queued a total of $total {$seeder->type()} for indexing."); - } } - protected function continueAt(string $type, int $at = null) + protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int { - /** @var SettingsRepositoryInterface $settings */ - $settings = resolve(SettingsRepositoryInterface::class); + $raw = $settings->get("blomstra-search.continued-at.$type"); - $key = "blomstra-search.continued-at.$type"; + return $raw !== null ? (int) $raw : null; + } - if ($at) { - $settings->set($key, $at); - } else { - return $settings->get($key); - } + protected function setContinueAt(SettingsRepositoryInterface $settings, string $type, ?int $at): void + { + $settings->set("blomstra-search.continued-at.$type", $at); } } diff --git a/src/Elasticsearch/HasChildQuery.php b/src/Elasticsearch/HasChildQuery.php new file mode 100644 index 0000000..a95fe0a --- /dev/null +++ b/src/Elasticsearch/HasChildQuery.php @@ -0,0 +1,57 @@ +innerHits = true; + + return $this; + } + + public function toArray(): array + { + $hasChild = [ + 'type' => $this->type, + 'score_mode' => $this->scoreMode, + 'query' => $this->query->toArray(), + ]; + + if ($this->innerHits) { + $hasChild['inner_hits'] = ['name' => 'best_post', 'size' => 1]; + } + + return ['has_child' => $hasChild]; + } +} diff --git a/src/Jobs/DeletingJob.php b/src/Jobs/DeletingJob.php index 1f6975c..415e6be 100644 --- a/src/Jobs/DeletingJob.php +++ b/src/Jobs/DeletingJob.php @@ -26,9 +26,10 @@ class DeletingJob extends Job // Preparing body for storing. $body = $this->models->map(function (Model $model) { $document = $this->seeder->toDocument($model); + $routing = $this->seeder->routing($model); return [ - ['delete' => ['_index' => $this->index, '_id' => $document->id]], + ['delete' => ['_index' => $this->index, '_id' => $document->id, 'routing' => $routing]], ]; })->flatten(1); diff --git a/src/Jobs/Job.php b/src/Jobs/Job.php index 782e1a7..4e29cdb 100644 --- a/src/Jobs/Job.php +++ b/src/Jobs/Job.php @@ -22,9 +22,13 @@ abstract class Job extends AbstractJob public static ?string $onQueue = null; - public function __construct(protected Collection $models, protected Seeder $seeder) + /** + * @param string|null $targetIndex Explicit index name for blue-green builds. + * Defaults to the configured alias when null. + */ + public function __construct(protected Collection $models, protected Seeder $seeder, ?string $targetIndex = null) { - $this->index = resolve('blomstra.search.elastic_index'); + $this->index = $targetIndex ?? resolve('blomstra.search.elastic_index'); if (static::$onQueue) { $this->onQueue(static::$onQueue); diff --git a/src/Jobs/UpdateSearchJob.php b/src/Jobs/UpdateSearchJob.php index 3116faa..bf7783f 100644 --- a/src/Jobs/UpdateSearchJob.php +++ b/src/Jobs/UpdateSearchJob.php @@ -28,9 +28,10 @@ class UpdateSearchJob extends Job // Preparing body for storing. $body = $this->models->map(function (Model $model) { $document = $this->seeder->toDocument($model); + $routing = $this->seeder->routing($model); return [ - ['index' => ['_index' => $this->index, '_id' => $document->id]], + ['index' => ['_index' => $this->index, '_id' => $document->id, 'routing' => $routing]], $document->toArray(), ]; }) diff --git a/src/Jobs/ViewsSearchJob.php b/src/Jobs/ViewsSearchJob.php index aa174c7..c08336e 100644 --- a/src/Jobs/ViewsSearchJob.php +++ b/src/Jobs/ViewsSearchJob.php @@ -20,10 +20,12 @@ use Flarum\Queue\AbstractJob; class ViewsSearchJob extends AbstractJob { protected string $index; + protected string $documentType; public function __construct(protected int $discussionId) { - $this->index = resolve('blomstra.search.elastic_index'); + $this->index = resolve('blomstra.search.elastic_index'); + $this->documentType = resolve(DiscussionSerializer::class)->getType(new Discussion()); if (Job::$onQueue) { $this->onQueue(Job::$onQueue); @@ -38,16 +40,20 @@ class ViewsSearchJob extends AbstractJob return; } - $type = resolve(DiscussionSerializer::class)->getType(new Discussion()); + $type = $this->documentType; - $client->update([ - 'index' => $this->index, - 'id' => "$type:{$this->discussionId}", - 'retry_on_conflict' => 3, - 'ignore' => [404], - 'body' => [ - 'doc' => ['view_count' => (int) $discussion->view_count], - ], - ]); + try { + $client->update([ + 'index' => $this->index, + 'id' => "$type:{$this->discussionId}", + 'routing' => (string) $this->discussionId, + 'retry_on_conflict' => 3, + 'body' => [ + 'doc' => ['view_count' => (int) $discussion->view_count], + ], + ]); + } catch (\Elasticsearch\Common\Exceptions\Missing404Exception $e) { + // Document not yet indexed; will be picked up on next --seed-missing or --recreate. + } } } diff --git a/src/Provider.php b/src/Provider.php index 29b38f9..8b13ee2 100644 --- a/src/Provider.php +++ b/src/Provider.php @@ -116,11 +116,9 @@ class Provider extends AbstractServiceProvider $queue->pushOn(Job::$onQueue, new DeletingJob(Collection::make([$model]), $seeder)); }); - if (method_exists($seeder, 'viewingOn')) { - $seeder::viewingOn($events, function (int $discussionId) use ($queue) { - $queue->pushOn(Job::$onQueue, new ViewsSearchJob($discussionId)); - }); - } + $seeder::viewingOn($events, function (int $discussionId) use ($queue) { + $queue->pushOn(Job::$onQueue, new ViewsSearchJob($discussionId)); + }); } } } diff --git a/src/Save/Document.php b/src/Save/Document.php index 372de98..07a7b3f 100644 --- a/src/Save/Document.php +++ b/src/Save/Document.php @@ -13,11 +13,12 @@ namespace Blomstra\Search\Save; use Carbon\Carbon; +use Illuminate\Support\Arr; use Illuminate\Support\Fluent; /** - * @property string $type * @property string $id + * @property int $rawId * @property string $content * @property Carbon $created_at * @property Carbon $updated_at @@ -30,4 +31,13 @@ use Illuminate\Support\Fluent; */ class Document extends Fluent { + /** + * Exclude `id` from the body sent to Elasticsearch. The document ID is + * already stored as `_id` by the bulk API action; duplicating it in + * `_source` wastes space and is redundant. + */ + public function toArray(): array + { + return Arr::except($this->attributes, ['id']); + } } diff --git a/src/Seeders/CommentSeeder.php b/src/Seeders/CommentSeeder.php index 1c94218..77d234d 100644 --- a/src/Seeders/CommentSeeder.php +++ b/src/Seeders/CommentSeeder.php @@ -13,7 +13,9 @@ namespace Blomstra\Search\Seeders; use Blomstra\Search\Save\Document; +use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\PostSerializer; +use Flarum\Discussion\Discussion; use Flarum\Post\CommentPost; use Flarum\Post\Event as Core; use Illuminate\Contracts\Events\Dispatcher; @@ -27,30 +29,29 @@ class CommentSeeder extends Seeder return resolve(PostSerializer::class)->getType(new CommentPost()); } + public function joinRelation(): string + { + return 'post'; + } + + public function routing(Model $model): string + { + return (string) $model->discussion_id; + } + public function query(): Builder { - $includes = ['discussion']; - - if ($this->extensionEnabled('flarum-tags')) { - $includes[] = 'discussion.tags'; - } - - if ($this->extensionEnabled('fof-byobu')) { - $includes[] = 'discussion.recipientUsers'; - $includes[] = 'discussion.recipientGroups'; - } - return CommentPost::query() - ->whereNull('hidden_at') - ->where('type', CommentPost::$type) - ->with($includes); + ->where('type', CommentPost::$type); } public static function savingOn(Dispatcher $events, callable $callable) { $events->listen([ Core\Posted::class, - Core\Revised::class + Core\Revised::class, + Core\Hidden::class, + Core\Restored::class, ], function ($event) use ($callable) { $callable($event->post); }); @@ -65,41 +66,31 @@ class CommentSeeder extends Seeder }); } + /** Cached discussion type string (e.g. "discussions") — resolved once per job. */ + private ?string $discussionType = null; + + private function discussionType(): string + { + return $this->discussionType ??= resolve(DiscussionSerializer::class)->getType(new Discussion()); + } + /** - * @param CommentPost $model + * Post documents only need content for has_child matching and the join + * field to establish the parent-child relationship. All discussion-level + * fields (groups, tags, comment_count, etc.) live on the discussion + * document and are irrelevant here. * - * @return Document + * @param CommentPost $model */ public function toDocument(Model $model): Document { - $document = new Document([ - 'type' => $this->type(), - 'id' => $this->type().':'.$model->id, - 'rawId' => $model->id, - 'content' => $model->content, - 'created_at' => $model->created_at?->toAtomString(), - 'updated_at' => $model->edited_at?->toAtomString(), - 'is_private' => $model->is_private, - 'user_id' => $model->user_id, - 'groups' => $this->groupsForDiscussion($model->discussion), - 'comment_count' => $model->discussion->comment_count, + return new Document([ + 'join_field' => ['name' => $this->joinRelation(), 'parent' => "{$this->discussionType()}:{$model->discussion_id}"], + 'discussion_id' => $model->discussion_id, + 'id' => $this->type().':'.$model->id, + 'rawId' => $model->id, + 'content' => $model->content, + 'is_hidden' => $model->hidden_at !== null, ]); - - if ($this->extensionEnabled('flarum-tags')) { - $document['tags'] = $model->discussion->tags->pluck('id')->toArray(); - } - - if ($this->extensionEnabled('fof-byobu')) { - $document['recipient_users'] = $model->discussion->recipientUsers - ->whereNull('removed_at') - ->pluck('id') - ->toArray(); - $document['recipient_groups'] = $model->discussion->recipientGroups - ->whereNull('removed_at') - ->pluck('id') - ->toArray(); - } - - return $document; } } diff --git a/src/Seeders/DiscussionSeeder.php b/src/Seeders/DiscussionSeeder.php index 3efdd55..406fdf4 100644 --- a/src/Seeders/DiscussionSeeder.php +++ b/src/Seeders/DiscussionSeeder.php @@ -17,10 +17,14 @@ use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Discussion\Discussion; use Flarum\Discussion\Event as Core; use Flarum\Extension\ExtensionManager; +use Flarum\Group\Group; +use Flarum\Group\Permission; +use Flarum\Tags\Tag; use FoF\Byobu\Events as Byobu; use FoF\DiscussionViews\Events\DiscussionWasViewed; use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Model; class DiscussionSeeder extends Seeder @@ -30,6 +34,16 @@ class DiscussionSeeder extends Seeder return resolve(DiscussionSerializer::class)->getType(new Discussion()); } + public function joinRelation(): string + { + return 'discussion'; + } + + public function routing(Model $model): string + { + return (string) $model->id; + } + public function query(): Builder { $includes = []; @@ -99,7 +113,7 @@ class DiscussionSeeder extends Seeder public function toDocument(Model $model): Document { $document = new Document([ - 'type' => $this->type(), + 'join_field' => $this->joinRelation(), 'id' => $this->type().':'.$model->id, 'rawId' => $model->id, 'content' => $model->title, @@ -136,4 +150,40 @@ class DiscussionSeeder extends Seeder return $document; } + + protected function groupsForDiscussion(Discussion $discussion): array + { + $permissions = collect(); + + $globalPermission = Permission::query() + ->where('permission', 'viewForum') + ->pluck('group_id'); + + if ($this->extensionEnabled('flarum-tags')) { + /** @var Collection $tags */ + $tags = $discussion->tags; + + $tagPermissions = Permission::query() + ->whereIn( + 'permission', + $tags->pluck('id')->map(fn (int $id) => "tag$id.viewForum") + )->get(); + + $permissions = $tags->map(function (Tag $tag) use ($tagPermissions) { + $permissions = $tagPermissions->where('permission', "tag$tag->id.viewForum"); + + if ($tag->is_restricted) { + $permissions = $permissions->add(['group_id' => Group::ADMINISTRATOR_ID]); + } + + return $permissions->pluck('group_id'); + })->flatten(); + } + + if (!$discussion->is_private && $permissions->isEmpty()) { + $permissions = $globalPermission; + } + + return $permissions->toArray(); + } } diff --git a/src/Seeders/Seeder.php b/src/Seeders/Seeder.php index a2698a9..54e86f3 100644 --- a/src/Seeders/Seeder.php +++ b/src/Seeders/Seeder.php @@ -13,66 +13,38 @@ namespace Blomstra\Search\Seeders; use Blomstra\Search\Save\Document; -use Flarum\Discussion\Discussion; use Flarum\Extension\ExtensionManager; -use Flarum\Group\Group; -use Flarum\Group\Permission; -use Flarum\Tags\Tag; use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Database\Eloquent\Builder; -use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Model; abstract class Seeder { abstract public function type(): string; + /** + * The join relation name for this document type ('discussion' or 'post'). + * Used in the join_field mapping and for filtering in seed-missing checks. + */ + abstract public function joinRelation(): string; + + /** + * The ES routing key for this model. Parent and child documents for the same + * discussion must share a routing key so ES places them on the same shard. + */ + abstract public function routing(Model $model): string; + abstract public function query(): Builder; abstract public static function savingOn(Dispatcher $events, callable $callable); abstract public static function deletingOn(Dispatcher $events, callable $callable); + /** No-op default; override in seeders that need to react to view events. */ + public static function viewingOn(Dispatcher $events, callable $callable): void {} + abstract public function toDocument(Model $model): Document; - protected function groupsForDiscussion(Discussion $discussion): array - { - $permissions = collect(); - - $globalPermission = Permission::query() - ->where('permission', 'viewForum') - ->pluck('group_id'); - - if ($this->extensionEnabled('flarum-tags')) { - /** @var Collection $tags */ - $tags = $discussion->tags; - - $tagPermissions = Permission::query() - ->whereIn( - 'permission', - $tags->pluck('id')->map(function (int $id) { - return "tag$id.viewForum"; - }) - )->get(); - - $permissions = $tags->map(function (Tag $tag) use ($tagPermissions) { - $permissions = $tagPermissions->where('permission', "tag$tag->id.viewForum"); - - if ($tag->is_restricted) { - $permissions = $permissions->add(['group_id' => Group::ADMINISTRATOR_ID]); - } - - return $permissions->pluck('group_id'); - })->flatten(); - } - - if (!$discussion->is_private && $permissions->isEmpty()) { - $permissions = $globalPermission; - } - - return $permissions->toArray(); - } - protected function extensionEnabled(string $extension): bool { /** @var ExtensionManager $manager */