diff --git a/composer.json b/composer.json index f2013a7..37a262f 100644 --- a/composer.json +++ b/composer.json @@ -24,10 +24,13 @@ } ], "require": { - "php": ">= 8.0", - "flarum/core": "^1.2.0", - "elasticsearch/elasticsearch": "7.*", - "spatie/elasticsearch-query-builder": "^1.3.0" + "php": ">= 8.1", + "flarum/core": "^2.0", + "elasticsearch/elasticsearch": "7.17.2", + "spatie/elasticsearch-query-builder": "^1.4.0" + }, + "require-dev": { + "flarum/core": "^2.x-dev" }, "extra": { "flarum-extension": { @@ -45,5 +48,10 @@ "psr-4": { "Blomstra\\Search\\": "src/" } + }, + "config": { + "allow-plugins": { + "php-http/discovery": true + } } } diff --git a/extend.php b/extend.php index ace56c8..c8479a8 100644 --- a/extend.php +++ b/extend.php @@ -12,11 +12,11 @@ namespace Blomstra\Search; +use Flarum\Discussion\Discussion as FlarumDiscussion; use Flarum\Extend as Flarum; +use Flarum\Post\Post as FlarumPost; return [ - (new Flarum\ServiceProvider())->register(Provider::class), - (new Flarum\Frontend('forum')) ->js(__DIR__.'/js/dist/forum.js'), (new Flarum\Frontend('admin')) @@ -24,13 +24,26 @@ return [ (new Flarum\Locales(__DIR__.'/resources/locale')), + (new Flarum\ServiceProvider()) + ->register(Provider::class), + (new Flarum\Routes('api')) - ->get('/blomstra/search/{type}', 'blomstra.search', Api\Controllers\SearchController::class) ->put('/blomstra/search/index', 'blomstra.search.index', Api\Controllers\IndexController::class), + (new Flarum\Console()) ->command(Commands\BuildCommand::class), (new Flarum\Settings()) ->default('blomstra-search.search-discussion-subjects', true) ->default('blomstra-search.search-post-bodies', true), + + (new Flarum\SearchDriver(Search\ElasticSearchDriver::class)) + ->addSearcher(FlarumDiscussion::class, Discussion\DiscussionSearcher::class) + ->addFilter(Discussion\DiscussionSearcher::class, Discussion\PrivateFilterMutator::class) +// ->addMutator(Discussion\DiscussionSearcher::class, Discussion\PrivateFilterMutator::mutate(...)) + ->setFulltext(Discussion\DiscussionSearcher::class, Discussion\FulltextFilter::class), + + (new Flarum\SearchIndex()) + ->indexer(FlarumDiscussion::class, Discussion\DiscussionIndexer::class) + ->indexer(FlarumPost::class, Post\CommentPostIndexer::class), ]; diff --git a/js/src/forum/PaginatedListStates/extendDiscussionState.tsx b/js/src/forum/PaginatedListStates/extendDiscussionState.tsx deleted file mode 100644 index f5b3a8d..0000000 --- a/js/src/forum/PaginatedListStates/extendDiscussionState.tsx +++ /dev/null @@ -1,33 +0,0 @@ -import app from 'flarum/forum/app'; - -import { override } from 'flarum/common/extend'; - -import DiscussionListState from 'flarum/forum/states/DiscussionListState'; - -export default function extendDiscussionState() { - override(DiscussionListState.prototype, 'loadPage', async function (this: DiscussionListState, original, page: number = 1) { - const preloaded = app.data.apiDocument || null; - - // If existing payload is given or no search is made, fallback on native page. - if (preloaded || !this.requestParams()?.filter?.q) return original.call(this, page); - - const params = this.requestParams(); - params.page = { - offset: this.pageSize * (page - 1), - ...params.page, - }; - - if (Array.isArray(params.include)) { - params.include = params.include.join(','); - } - - // Construct API search URI - const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`; - - // Make API GET request - const results = await app.request({ params, url, method: 'GET' }); - - // Parse API response into models and push to store - return app.store.pushPayload(results); - }); -} diff --git a/js/src/forum/SearchSources/DiscussionsSearchSource.tsx b/js/src/forum/SearchSources/DiscussionsSearchSource.tsx deleted file mode 100644 index c3aff73..0000000 --- a/js/src/forum/SearchSources/DiscussionsSearchSource.tsx +++ /dev/null @@ -1,76 +0,0 @@ -import app from 'flarum/forum/app'; -import highlight from 'flarum/common/helpers/highlight'; -import LinkButton from 'flarum/common/components/LinkButton'; -import Link from 'flarum/common/components/Link'; -import { SearchSource } from 'flarum/forum/components/Search'; -import type Mithril from 'mithril'; - -/** - * The `DiscussionsSearchSource` finds and displays discussion search results in - * the search dropdown. - */ -export default class DiscussionsSearchSource implements SearchSource { - /** - * Map of lowercase search queries to their respective model results - */ - protected results = new Map(); - - /** - * Model name used for assembling API endpoint URL, and for frontend DOM. - */ - private type = 'discussions'; - - async search(query: string): Promise { - query = query.toLowerCase(); - - this.results.set(query, []); - - const params = { - filter: { q: query }, - page: { limit: 3 }, - include: 'mostRelevantPost', - }; - - // Construct API search URI - const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`; - - // Make API GET request - const results = await app.request({ params, url, method: 'GET' }); - - // Parse API response into models and push to store - const models = app.store.pushPayload(results); - - // Add models to results map - this.results.set(query, models); - } - - view(query: string): Array { - query = query.toLowerCase(); - - // Get results from map - const queryResults = this.results.get(query) || []; - - const results = queryResults.map((discussion: any) => { - const mostRelevantPost = discussion.mostRelevantPost(); - - return ( -
  • - -
    {highlight(discussion.title(), query)}
    - {!!mostRelevantPost &&
    {highlight(mostRelevantPost.contentPlain(), query, 100)}
    } - -
    • - ); - }); - - return [ -
  • {app.translator.trans('core.forum.search.discussions_heading')}
    • , -
  • - - {app.translator.trans('core.forum.search.all_discussions_button', { query })} - -
    • , - ...results, - ]; - } -} diff --git a/js/src/forum/index.ts b/js/src/forum/index.ts index a1bd2e8..6a68871 100644 --- a/js/src/forum/index.ts +++ b/js/src/forum/index.ts @@ -1,23 +1,5 @@ import app from 'flarum/forum/app'; -import Search, { SearchAttrs, SearchSource } from 'flarum/forum/components/Search'; - -import { extend } from 'flarum/common/extend'; -import ItemList from 'flarum/common/utils/ItemList'; - -import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource'; -import extendDiscussionState from './PaginatedListStates/extendDiscussionState'; - app.initializers.add('blomstra-search', () => { - extend(Search.prototype, 'sourceItems', function (this: Search, items: ItemList) { - items.replace('discussions', new DiscussionsSearchSource()); - }); + // }); - -app.initializers.add( - 'blomstra-search-early', - () => { - extendDiscussionState(); - }, - 999999 -); diff --git a/src/Api/Client.php b/src/Api/Client.php deleted file mode 100644 index 32a914b..0000000 --- a/src/Api/Client.php +++ /dev/null @@ -1,28 +0,0 @@ -queryParams, 'filter.q')) { - return parent::get('/blomstra/search/discussions'); - } - - return parent::get($path); - } -} diff --git a/src/Api/Controllers/SearchController.php b/src/Api/Controllers/SearchController.php index a0c91e3..a443777 100644 --- a/src/Api/Controllers/SearchController.php +++ b/src/Api/Controllers/SearchController.php @@ -32,7 +32,7 @@ use Illuminate\Support\Arr; use Illuminate\Support\Collection; use Illuminate\Support\Str; use Psr\Http\Message\ServerRequestInterface; -use Spatie\ElasticsearchQueryBuilder\Builder; +use Blomstra\Search\Elasticsearch\Builder; use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery; use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery; diff --git a/src/Commands/BuildCommand.php b/src/Commands/BuildCommand.php index 554e0f9..cc2b110 100644 --- a/src/Commands/BuildCommand.php +++ b/src/Commands/BuildCommand.php @@ -12,27 +12,28 @@ namespace Blomstra\Search\Commands; -use Blomstra\Search\Jobs\Job; -use Blomstra\Search\Jobs\SavingJob; -use Blomstra\Search\Seeders\Seeder; +use Blomstra\Search\Discussion\DiscussionIndexer; +use Blomstra\Search\Post\CommentPostIndexer; use Elasticsearch\Client; +use Flarum\Discussion\Discussion; +use Flarum\Post\Post; +use Flarum\Search\Job\IndexJob; use Flarum\Settings\SettingsRepositoryInterface; use Illuminate\Console\Command; use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Queue\Queue; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Arr; -use Spatie\ElasticsearchQueryBuilder\Builder; +use Blomstra\Search\Elasticsearch\Builder; use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery; use Spatie\ElasticsearchQueryBuilder\Queries\RangeQuery; -use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery; class BuildCommand extends Command { protected $signature = 'blomstra:search:index {--max-id= : Limits for each object the number of items to seed} {--throttle= : Number of seconds to wait between pushing to the queue} - {--only= : type to run seeder for, eg discussions or posts} + {--only= : type of model to index, eg discussions or posts} {--recreate : create or recreate the index} {--mapping : recreate the mapping} {--continue : continue each object type where you left off} @@ -40,118 +41,64 @@ class BuildCommand extends Command protected $description = 'Rebuilds the complete search server with its documents.'; - public function handle(Container $container) + public function handle(Container $container, Client $client, Queue $queue, SettingsRepositoryInterface $settings) { - /** @var string $index */ - $index = $container->make('blomstra.search.elastic_index'); - - /** @var array|string[] $seeders */ - $seeders = $container->tagged('blomstra.search.seeders'); - - /** @var Queue $queue */ - $queue = $container->make(Queue::class); - - /** @var Client $client */ - $client = $container->make(Client::class); - - /** @var SettingsRepositoryInterface $settings */ - $settings = $container->make(SettingsRepositoryInterface::class); - - // Elastic scheme definition. - $properties = [ - 'properties' => [ - 'content' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'], - 'rawId' => ['type' => 'integer'], - 'created_at' => ['type' => 'date'], - 'updated_at' => ['type' => 'date'], - 'is_private' => ['type' => 'boolean'], - 'is_sticky' => ['type' => 'boolean'], - 'groups' => ['type' => 'integer'], - 'tags' => ['type' => 'integer'], - 'recipient_groups' => ['type' => 'integer'], - 'recipient_users' => ['type' => 'integer'], - 'comment_count' => ['type' => 'integer'], - ], + $indexers = [ + 'discussions' => DiscussionIndexer::class, + 'posts' => CommentPostIndexer::class, + ]; + $models = [ + 'discussions' => Discussion::class, + 'posts' => Post::class, ]; - // Remove/delete the whole index. - if ($this->option('recreate')) { - // Flush the index. - $client->indices()->delete([ - 'index' => $index, - 'ignore_unavailable' => true, - ]); - - // Create a new index. - $client->indices()->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'index.max_ngram_diff' => 10, - 'analysis' => [ - 'analyzer' => [ - 'flarum_analyzer' => [ - 'type' => $settings->get('blomstra-search.analyzer-language') ?: 'english', - ], - 'flarum_analyzer_partial' => [ - 'type' => 'custom', - 'tokenizer' => 'standard', - 'filter' => [ - 'lowercase', - 'partial_search_filter', - ], - ], - ], - 'filter' => [ - 'partial_search_filter' => [ - 'type' => 'ngram', - 'min_gram' => 1, - 'max_gram' => 10, - 'token_chars' => ['letter', 'digit', 'symbol'], - ], - ], - ], - ], - ], - ]); - } - - // Create the index. - if ($this->option('recreate') || $this->option('mapping')) { - $client->indices()->putMapping([ - 'index' => $index, - 'body' => $properties, - ]); - } - - // Seed only a specific resource. $only = $this->option('only'); + $onlyIndexers = $only + ? Arr::only($indexers, is_array($only) ? $only : explode(',', $only)) + : $indexers; - /** @var Seeder $seeder */ - foreach ($seeders as $seeder) { - if ($only && $seeder->type() !== $only) { - continue; + foreach ($onlyIndexers as $type => $indexerClass) { + /** @var DiscussionIndexer|CommentPostIndexer $indexer */ + $indexer = $container->make($indexerClass); + + $properties = [ + 'properties' => $indexer->properties(), + ]; + + // Remove/delete the whole index. + if ($this->option('recreate')) { + $indexer->flush(); + $indexer->build(); + } + + // Create the index. + if (! $this->option('recreate') && $this->option('mapping')) { + $client->indices()->putMapping([ + 'index' => $indexer::index(), + 'body' => $properties, + ]); } $total = 0; + $query = $models[$type]::query(); + $continueAt = $this->option('continue') - ? ($this->continueAt($seeder->type()) ?? $seeder->query()->max('id')) - : $seeder->query()->max('id'); + ? ($this->continueAt($type) ?? $query->max('id')) + : $query->max('id'); $seeded = null; while ($continueAt !== null) { if ($this->option('seed-missing')) { $response = (new Builder($client)) - ->index($index) + ->index($indexer::index()) ->size(1000) ->addQuery( (new BoolQuery()) - ->add((new RangeQuery('rawId')) + ->add((new RangeQuery('rawId')) ->gte($continueAt - 1000) ->lte($continueAt)) - ->add(TermQuery::create('type', $seeder->type())) ) ->search(); @@ -159,7 +106,7 @@ class BuildCommand extends Command } /** @var Collection $collection */ - $collection = $seeder->query() + $collection = $query ->latest('id') ->whereBetween('id', [$continueAt - 1000, $continueAt]) ->when($this->option('max-id'), function ($query, $id) { @@ -171,16 +118,14 @@ class BuildCommand extends Command $min = $collection->min('id'); $continueAt = $min && $min > 2 ? $min - 1 : null; - $queue->pushOn(Job::$onQueue, new SavingJob($collection, $seeder)); + $onQueue = property_exists($indexerClass, 'queue') ? $indexerClass::$queue : null; + $queue->pushOn($onQueue, new IndexJob($indexerClass, $collection->all(), IndexJob::SAVE)); - $this->info("Pushed into the index, type: {$seeder->type()}, amount: {$collection->count()}."); + $this->info("Pushed into the index, type: $type, amount: {$collection->count()}."); $total += $collection->count(); - $this->continueAt( - $seeder->type(), - $continueAt - ); + $this->continueAt($type, $continueAt); if ($throttle = $this->option('throttle')) { $this->info("Throttling for $throttle seconds"); diff --git a/src/Discussion/DiscussionIndexer.php b/src/Discussion/DiscussionIndexer.php new file mode 100644 index 0000000..a7db51a --- /dev/null +++ b/src/Discussion/DiscussionIndexer.php @@ -0,0 +1,101 @@ +elastic->save(self::index(), $models, $this->toDocument(...)); + } + + public function delete(array $models): void + { + $this->elastic->delete(self::index(), $models); + } + + function build(): void + { + $this->elastic->build(self::index(), $this->properties()); + } + + function flush(): void + { + $this->elastic->flush(self::index()); + } + + public function properties(): array + { + return [ + 'rawId' => ['type' => 'integer'], + 'discussion_id' => ['type' => 'integer'], + 'title' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'], + 'created_at' => ['type' => 'date'], + 'updated_at' => ['type' => 'date'], + 'last_posted_at' => ['type' => 'date'], + 'is_private' => ['type' => 'boolean'], + 'is_sticky' => ['type' => 'boolean'], + 'is_locked' => ['type' => 'boolean'], + 'groups' => ['type' => 'integer'], + 'tags' => ['type' => 'integer'], + 'recipient_groups' => ['type' => 'integer'], + 'recipient_users' => ['type' => 'integer'], + 'comment_count' => ['type' => 'integer'], + ]; + } + + public function toDocument(Discussion $model): Document + { + $document = new Document([ + 'id' => $model->id, + 'discussion_id' => $model->id, // duplicated for result aggregation in searching with posts. + 'rawId' => $model->id, + 'title' => $model->title, + 'created_at' => $model->created_at?->toAtomString(), + 'updated_at' => $model->last_posted_at?->toAtomString(), + 'is_private' => $model->is_private, + 'user_id' => $model->user_id, + 'groups' => $this->groupsForDiscussion($model), + 'comment_count' => $model->comment_count, + ]); + + if ($this->extensions->isEnabled('flarum-tags')) { + $document['tags'] = $model->tags->pluck('id')->toArray(); + } + + if ($this->extensions->isEnabled('fof-byobu')) { + $document['recipient_users'] = $model->recipientUsers->pluck('id')->toArray(); + $document['recipient_groups'] = $model->recipientGroups->pluck('id')->toArray(); + } + + if ($this->extensions->isEnabled('flarum-sticky')) { + $document['is_sticky'] = $model->is_sticky; + } + + if ($this->extensions->isEnabled('flarum-lock')) { + $document['is_locked'] = $model->is_locked; + } + + return $document; + } +} diff --git a/src/Discussion/DiscussionSearcher.php b/src/Discussion/DiscussionSearcher.php new file mode 100644 index 0000000..5318ab3 --- /dev/null +++ b/src/Discussion/DiscussionSearcher.php @@ -0,0 +1,22 @@ +select('discussions.*'); + } +} diff --git a/src/Discussion/FulltextFilter.php b/src/Discussion/FulltextFilter.php new file mode 100644 index 0000000..68e63bd --- /dev/null +++ b/src/Discussion/FulltextFilter.php @@ -0,0 +1,156 @@ + + */ +class FulltextFilter extends AbstractFulltextFilter +{ + public function __construct( + protected SearchManager $search, + protected Client $elastic + ) { + } + + public function search(SearchState $state, string $value): void + { + $builder = $state->getBuilder(); + + $query = BoolQuery::create() + ->add( + BoolQuery::create() + ->add($this->exactMatch('title', $value, 1.2), 'should') + ->add($this->wordMatch('title', $value, 'and', 1.2), 'should') + ->add($this->wordMatch('title', $value, 'or', 1.2), 'should'), + 'should' + ) + ->add( + BoolQuery::create() + ->add($this->exactMatch('content', $value), 'should', 1) + ->add($this->wordMatch('content', $value, 'and', 1), 'should') + ->add($this->wordMatch('content', $value, 'or', 1), 'should'), + 'should' + ); + + $builder->addQuery($query); + + $builder->collapse('discussion_id'); + + $aggs = FilterAggregation::create('posts', TermsQuery::create('_index', [CommentPostIndexer::index()])) + ->aggregation( + TermsAggregation::create('per_discussion', 'discussion_id') + ->aggregation( + TopHitsAggregation::create('most_relevant_post_id', 1, Sort::create('_score')) + ) + ); + + $builder->addAggregation($aggs); + + $state->setDefaultSort(function (Builder $builder) { + $builder->addSort( + Sort::create('_score', 'desc') + ); + }); + + $state->retrieveDatabaseRecordsUsing(function (array $response, SearchCriteria $criteria): Collection { + $buckets = Collection::make(Arr::get($response, 'aggregations.posts.per_discussion.buckets')) + ->map(fn (array $hit) => [ + 'discussion_id' => $hit['key'], + 'most_relevant_post_id' => Arr::get($hit, 'most_relevant_post_id.most_relevant_post_id.hits.hits.0._id'), + 'most_relevant_post_score' => Arr::get($hit, 'most_relevant_post_id.most_relevant_post_id.hits.hits.0._score'), + ])->keyBy('discussion_id'); + + $results = Collection::make(Arr::get($response, 'hits.hits')) + ->map(fn (array $hit) => [ + 'discussion_id' => $hit['_source']['discussion_id'], + 'most_relevant_post_id' => Arr::get($buckets->get($hit['_source']['discussion_id']), 'most_relevant_post_id'), + 'most_relevant_post_score' => Arr::get($buckets->get($hit['_source']['discussion_id']), 'most_relevant_post_score'), + 'title_score' => $hit['_score'], + ])->keyBy('discussion_id'); + + // We have $hits and $buckets, both are sorted by score. + // $discussionResult represents discussion scores based on the title. + // $buckets represents discussion scores based on the sum of all posts, + // and also the most relevant post id and its score. + // We need to merge these two results into one collection, and sort them by score. + // We also need to make sure that the most relevant post id is set on the discussion. + $results = $results + // Sort by title_score, most_relevant_post_score then posts_score. + ->sortByDesc(function ($result) { + return $result['title_score'] ?? 0 + + $result['most_relevant_post_score'] ?? 0 + + $result['posts_score'] ?? 0; + }) + ->take($criteria->limit); + + $actor = $criteria->actor; + $connection = Post::query()->getConnection(); + + if ($results->isEmpty()) { + return Discussion::whereVisibleTo($actor) + ->select('discussions.*') + ->get(); + } + + return Discussion::whereVisibleTo($actor) + ->select('discussions.*') + ->selectRaw( + $connection->raw('COALESCE(('.$connection->getQueryGrammar()->compileSelect( + $postsQuery = Post::query() + ->select('posts.id') + ->whereIn('posts.id', $results->pluck('most_relevant_post_id')->filter()) + ->whereColumn('discussions.id', '=', 'posts.discussion_id') + ->limit(1) + ->toBase() + ).'), first_post_id) as most_relevant_post_id')->getValue($connection->getQueryGrammar()) + ) + ->mergeBindings($postsQuery) + ->whereIn('discussions.id', $results->pluck('discussion_id')) + ->orderByRaw('FIELD(`discussions`.`id`, ' . implode(',', $results->pluck('discussion_id')->all()) . ')') + ->get(); + }); + } + + protected function exactMatch(string $field, string $q, int $fieldBoost = 1): Query + { + $query = (new MatchPhraseQuery($field, $q)); + + return $query->boost(2 * $fieldBoost); + } + + protected function wordMatch(string $field, string $q, string $operator, int $fieldBoost = 1): Query + { + $query = (new MatchQuery($field, $q)) + ->operator($operator); + + $boost = $operator === 'and' ? 1.8 : .8; + + return $query->boost($boost * $fieldBoost); + } +} diff --git a/src/Discussion/PrivateFilterMutator.php b/src/Discussion/PrivateFilterMutator.php new file mode 100644 index 0000000..b97b016 --- /dev/null +++ b/src/Discussion/PrivateFilterMutator.php @@ -0,0 +1,85 @@ + + */ +class PrivateFilterMutator implements FilterInterface +{ + use AppliesAccessControl; + + public function __construct( + protected ExtensionManager $extensions + ) { + } + + public function getFilterKey(): string + { + return 'private'; + } + + public function filter(SearchState $state, array|string $value, bool $negate): void + { + $actor = $state->getActor(); + + if (! $this->extensions->isEnabled('fof-byobu') || $actor->isGuest()) { + return; + } + + $builder = $state->getBuilder(); + $query = BoolQuery::create(); + + $query->add(self::byobuAccessQuery($state), 'filter'); + + $builder->addQuery($query); + } + + public static function mutate(ElasticSearchState $state): void + { + $builder = $state->getBuilder(); + + // If this filter isn't active, we apply both the private and public queries as should clauses. + if (Arr::first($state->getActiveFilters(), fn ($filter) => $filter->getFilterKey() === 'private')) { + return; + } + + $query = BoolQuery::create(); + + $query + ->add(self::restrictQuery($state), 'should') + ->add(self::byobuAccessQuery($state), 'should'); + + $builder->addQuery($query); + } + + private static function restrictQuery(SearchState $state): BoolQuery + { + return BoolQuery::create() + ->add(TermQuery::create('is_private', 'false')) + ->add(TermsQuery::create('groups', self::groupsForUser($state->getActor()))); + } + + private static function byobuAccessQuery(SearchState $state): BoolQuery + { + $actor = $state->getActor(); + + return BoolQuery::create() + ->add(TermQuery::create('is_private', 'true')) + ->add( + BoolQuery::create() + ->add(TermsQuery::create('recipient_groups', self::groupsForUser($actor)), 'should') + ->add(TermsQuery::create('recipient_users', [$actor->id]), 'should'), + ); + } +} diff --git a/src/Elasticsearch/BucketSortAggregation.php b/src/Elasticsearch/BucketSortAggregation.php new file mode 100644 index 0000000..8725286 --- /dev/null +++ b/src/Elasticsearch/BucketSortAggregation.php @@ -0,0 +1,50 @@ +fields[] = [$field => ['order' => $order]]; + + return $this; + } + + public function size(int $size): self + { + $this->size = $size; + + return $this; + } + + public function payload(): array + { + $parameters = [ + 'sort' => $this->fields, + ]; + + if ($this->size) { + $parameters['size'] = $this->size; + } + + return [ + 'bucket_sort' => $parameters, + ]; + } +} diff --git a/src/Elasticsearch/Builder.php b/src/Elasticsearch/Builder.php new file mode 100644 index 0000000..4988faf --- /dev/null +++ b/src/Elasticsearch/Builder.php @@ -0,0 +1,58 @@ +collapse = [ + 'field' => $field, + ]; + + return $this; + } + + public function getPayload(): array + { + $payload = parent::getPayload(); + + if ($this->collapse) { + $payload['collapse'] = $this->collapse; + } + + return $payload; + } + + public function search(): array + { + $params = $this->getParams(); + + return $this->client->search($params); + } + + public function getParams(): array + { + $payload = $this->getPayload(); + + $params = [ + 'body' => $payload, + ]; + + if ($this->searchIndex) { + $params['index'] = $this->searchIndex; + } + + if ($this->size !== null) { + $params['size'] = $this->size; + } + + if ($this->from !== null) { + $params['from'] = $this->from; + } + + return $params; + } +} diff --git a/src/Elasticsearch/SumAggregation.php b/src/Elasticsearch/SumAggregation.php new file mode 100644 index 0000000..efe7299 --- /dev/null +++ b/src/Elasticsearch/SumAggregation.php @@ -0,0 +1,43 @@ +name = $name; + $this->field = $field; + } + + public function payload(): array + { + $parameters = [ + 'field' => $this->field, + ]; + + if ($this->missing) { + $parameters['missing'] = $this->missing; + } + + return [ + 'sum' => $parameters, + ]; + } +} diff --git a/src/Exceptions/SeedingException.php b/src/Exceptions/IndexingException.php similarity index 70% rename from src/Exceptions/SeedingException.php rename to src/Exceptions/IndexingException.php index d186bf3..40ce609 100644 --- a/src/Exceptions/SeedingException.php +++ b/src/Exceptions/IndexingException.php @@ -14,9 +14,9 @@ namespace Blomstra\Search\Exceptions; use Throwable; -class SeedingException extends \Exception +class IndexingException extends \Exception { - public function __construct($message = '', public array $items, $code = 0, Throwable $previous = null) + public function __construct(string $message, public array $items, $code = 0, Throwable $previous = null) { parent::__construct($message, $code, $previous); } diff --git a/src/Jobs/DeletingJob.php b/src/Jobs/DeletingJob.php deleted file mode 100644 index 1f6975c..0000000 --- a/src/Jobs/DeletingJob.php +++ /dev/null @@ -1,41 +0,0 @@ -models->isEmpty()) { - return; - } - - // Preparing body for storing. - $body = $this->models->map(function (Model $model) { - $document = $this->seeder->toDocument($model); - - return [ - ['delete' => ['_index' => $this->index, '_id' => $document->id]], - ]; - })->flatten(1); - - $response = $client->bulk([ - 'index' => $this->index, - 'body' => $body->toArray(), - 'refresh' => true, - ]); - } -} diff --git a/src/Jobs/Job.php b/src/Jobs/Job.php deleted file mode 100644 index 782e1a7..0000000 --- a/src/Jobs/Job.php +++ /dev/null @@ -1,33 +0,0 @@ -index = resolve('blomstra.search.elastic_index'); - - if (static::$onQueue) { - $this->onQueue(static::$onQueue); - } - } -} diff --git a/src/Jobs/SavingJob.php b/src/Jobs/SavingJob.php deleted file mode 100644 index 9aec901..0000000 --- a/src/Jobs/SavingJob.php +++ /dev/null @@ -1,58 +0,0 @@ -models->isEmpty()) { - return; - } - - // Preparing body for storing. - $body = $this->models->map(function (Model $model) { - $document = $this->seeder->toDocument($model); - - return [ - ['index' => ['_index' => $this->index, '_id' => $document->id]], - $document->toArray(), - ]; - }) - ->flatten(1); - - $response = $client->bulk([ - 'index' => $this->index, - 'body' => $body->toArray(), - 'refresh' => true, - ]); - - if (Arr::get($response, 'errors') !== true) { - return true; - } - - $items = Arr::get($response, 'items'); - - $error = Arr::get(Arr::first($items), 'index.error.reason'); - - throw new SeedingException( - "Failed to seed: $error", - $items - ); - } -} diff --git a/src/Post/CommentPostIndexer.php b/src/Post/CommentPostIndexer.php new file mode 100644 index 0000000..1ea3c12 --- /dev/null +++ b/src/Post/CommentPostIndexer.php @@ -0,0 +1,99 @@ +elastic->save( + self::index(), + array_filter($models, fn (Post $model) => $model->type === CommentPost::$type), + $this->toDocument(...) + ); + } + + function delete(array $models): void + { + $this->elastic->delete( + self::index(), + array_filter($models, fn (Post $model) => $model->type === CommentPost::$type) + ); + } + + public function build(): void + { + $this->elastic->build(self::index(), $this->properties()); + } + + public function flush(): void + { + $this->elastic->flush(self::index()); + } + + public function properties(): array + { + return [ + 'rawId' => ['type' => 'integer'], + 'content' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'], + 'discussion_id' => ['type' => 'integer'], + 'created_at' => ['type' => 'date'], + 'updated_at' => ['type' => 'date'], + 'is_private' => ['type' => 'boolean'], + 'groups' => ['type' => 'integer'], + 'tags' => ['type' => 'integer'], + 'recipient_groups' => ['type' => 'integer'], + 'recipient_users' => ['type' => 'integer'], + 'comment_count' => ['type' => 'integer'], + ]; + } + + public function toDocument(Post $model): Document + { + $document = new Document([ + 'id' => $model->id, + 'type' => $model->type, + 'rawId' => $model->id, + 'discussion_id' => $model->discussion_id, + 'content' => $model->content, + 'created_at' => $model->created_at?->toAtomString(), + 'updated_at' => $model->edited_at?->toAtomString(), + 'is_private' => $model->is_private, + 'user_id' => $model->user_id, + 'groups' => $this->groupsForDiscussion($model->discussion), + 'comment_count' => $model->discussion->comment_count, + ]); + + if ($this->extensions->isEnabled('flarum-tags')) { + $document['tags'] = $model->discussion->tags->pluck('id')->toArray(); + } + + if ($this->extensions->isEnabled('fof-byobu')) { + $document['recipient_users'] = $model->discussion->recipientUsers->pluck('id')->toArray(); + $document['recipient_groups'] = $model->discussion->recipientGroups->pluck('id')->toArray(); + } + + return $document; + } +} diff --git a/src/Provider.php b/src/Provider.php index f8f45b2..bb197ea 100644 --- a/src/Provider.php +++ b/src/Provider.php @@ -12,39 +12,25 @@ namespace Blomstra\Search; -use Blomstra\Search\Jobs\DeletingJob; -use Blomstra\Search\Jobs\Job; -use Blomstra\Search\Jobs\SavingJob; use Elasticsearch\Client as Elastic; use Elasticsearch\ClientBuilder; -use Flarum\Api\Client; use Flarum\Foundation\AbstractServiceProvider; use Flarum\Foundation\Config; -use Flarum\Http\Middleware\ExecuteRoute; use Flarum\Settings\SettingsRepositoryInterface; use Illuminate\Contracts\Container\Container; -use Illuminate\Contracts\Events\Dispatcher; -use Illuminate\Contracts\Queue\Queue; -use Illuminate\Support\Collection; -use Laminas\Stratigility\MiddlewarePipe; use Psr\Log\LoggerInterface; class Provider extends AbstractServiceProvider { - public function register() + public function register(): void { - $this->container->tag([ - Seeders\DiscussionSeeder::class, - Seeders\CommentSeeder::class, - ], 'blomstra.search.seeders'); + $this->container->singleton(Elastic::class, function (Container $container) { + /** @var Config $config */ + $config = $this->container->make(Config::class); - /** @var SettingsRepositoryInterface $settings */ - $settings = $this->container->make(SettingsRepositoryInterface::class); + /** @var SettingsRepositoryInterface $settings */ + $settings = $this->container->make(SettingsRepositoryInterface::class); - /** @var Config $config */ - $config = $this->container->make(Config::class); - - $this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) { $builder = ClientBuilder::create() ->setHosts([$settings->get('blomstra-search.elastic-endpoint')]); @@ -61,59 +47,10 @@ class Provider extends AbstractServiceProvider return $builder->build(); }); - - $this->container->instance( - 'blomstra.search.elastic_index', - $settings->get('blomstra-search.elastic-index', 'flarum') - ); - - $this->container->extend( - Client::class, - function () { - $pipe = new MiddlewarePipe(); - - $exclude = resolve('flarum.api_client.exclude_middleware'); - - $middlewareStack = array_filter(resolve('flarum.api.middleware'), function ($middlewareClass) use ($exclude) { - return !in_array($middlewareClass, $exclude); - }); - - foreach ($middlewareStack as $middleware) { - $pipe->pipe(resolve($middleware)); - } - - $pipe->pipe(new ExecuteRoute()); - - return new Api\Client($pipe); - } - ); - - $this->container->tag([ - Searchers\DiscussionSearcher::class, - Searchers\CommentPostSearcher::class, - ], 'blomstra.search.searchers'); } - public function boot() + public function boot(): void { - /** @var array|string[] $seeders */ - $seeders = $this->container->tagged('blomstra.search.seeders'); - - /** @var Dispatcher $events */ - $events = resolve(Dispatcher::class); - - /** @var Queue $queue */ - $queue = resolve(Queue::class); - - /** @var string|Seeders\Seeder $seeder */ - foreach ($seeders as $seeder) { - $seeder::savingOn($events, function ($model) use ($queue, $seeder) { - $queue->pushOn(Job::$onQueue, new SavingJob(Collection::make([$model]), $seeder)); - }); - - $seeder::deletingOn($events, function ($model) use ($queue, $seeder) { - $queue->pushOn(Job::$onQueue, new DeletingJob(Collection::make([$model]), $seeder)); - }); - } + // } } diff --git a/src/Save/Document.php b/src/Save/Document.php index 372de98..1cecc03 100644 --- a/src/Save/Document.php +++ b/src/Save/Document.php @@ -16,7 +16,6 @@ use Carbon\Carbon; use Illuminate\Support\Fluent; /** - * @property string $type * @property string $id * @property string $content * @property Carbon $created_at diff --git a/src/Seeders/Seeder.php b/src/Search/Concerns/AppliesAccessControl.php similarity index 54% rename from src/Seeders/Seeder.php rename to src/Search/Concerns/AppliesAccessControl.php index a2698a9..c97104e 100644 --- a/src/Seeders/Seeder.php +++ b/src/Search/Concerns/AppliesAccessControl.php @@ -1,39 +1,28 @@ groups->pluck('id'); - abstract public function query(): Builder; + $groups->add(Group::GUEST_ID); - abstract public static function savingOn(Dispatcher $events, callable $callable); + if ($actor->is_email_confirmed) { + $groups->add(Group::MEMBER_ID); + } - abstract public static function deletingOn(Dispatcher $events, callable $callable); - - abstract public function toDocument(Model $model): Document; + return $groups->toArray(); + } protected function groupsForDiscussion(Discussion $discussion): array { @@ -43,8 +32,8 @@ abstract class Seeder ->where('permission', 'viewForum') ->pluck('group_id'); - if ($this->extensionEnabled('flarum-tags')) { - /** @var Collection $tags */ + if (resolve(ExtensionManager::class)->isEnabled('flarum-tags')) { + /** @var \Illuminate\Database\Eloquent\Collection $tags */ $tags = $discussion->tags; $tagPermissions = Permission::query() @@ -72,12 +61,4 @@ abstract class Seeder return $permissions->toArray(); } - - protected function extensionEnabled(string $extension): bool - { - /** @var ExtensionManager $manager */ - $manager = resolve(ExtensionManager::class); - - return $manager->isEnabled($extension); - } } diff --git a/src/Search/ElasticIndex.php b/src/Search/ElasticIndex.php new file mode 100644 index 0000000..d637970 --- /dev/null +++ b/src/Search/ElasticIndex.php @@ -0,0 +1,134 @@ +map(function (Model $model) use ($index, $toDocument) { + $document = $toDocument($model); + + return [ + [ + 'index' => [ + '_index' => $index, + '_id' => $document->id + ] + ], + $document->toArray(), + ]; + }) + ->flatten(1); + + $this->handleResponse( + $this->client->bulk([ + 'index' => $index, + 'body' => $body->toArray(), + 'refresh' => true, + ]) + ); + } + + public function delete(string $index, array $models): void + { + $models = collect($models); + + $body = $models->map(function (Model $model) use ($index) { + return [ + [ + 'delete' => [ + '_index' => $index(), + '_id' => $model->id + ] + ], + ]; + })->flatten(1); + + $this->handleResponse( + $this->client->bulk([ + 'index' => $index(), + 'body' => $body->toArray(), + 'refresh' => true, + ]) + ); + } + + public function build(string $index, array $properties): void + { + $this->client->indices()->create([ + 'index' => $index, + 'body' => [ + 'settings' => [ + 'index.max_ngram_diff' => 10, + 'analysis' => [ + 'analyzer' => [ + 'flarum_analyzer' => [ + 'type' => $this->settings->get('blomstra-search.analyzer-language') ?: 'english', + ], + 'flarum_analyzer_partial' => [ + 'type' => 'custom', + 'tokenizer' => 'standard', + 'filter' => [ + 'lowercase', + 'partial_search_filter', + ], + ], + ], + 'filter' => [ + 'partial_search_filter' => [ + 'type' => 'ngram', + 'min_gram' => 1, + 'max_gram' => 10, + 'token_chars' => ['letter', 'digit', 'symbol'], + ], + ], + ], + ], + 'mappings' => [ + 'properties' => $properties, + ], + ], + ]); + } + + public function flush(string $index): void + { + $this->client->indices()->delete([ + 'index' => $index, + 'ignore_unavailable' => true, + ]); + } + + public function handleResponse(array $response): void + { + if (Arr::get($response, 'errors') !== true) { + return; + } + + $items = Arr::get($response, 'items'); + + $error = Arr::get(Arr::first($items), 'index.error.reason'); + + throw new IndexingException( + "Failed to seed: $error", + $items + ); + } +} diff --git a/src/Search/ElasticSearchDriver.php b/src/Search/ElasticSearchDriver.php new file mode 100644 index 0000000..08bf188 --- /dev/null +++ b/src/Search/ElasticSearchDriver.php @@ -0,0 +1,13 @@ +builder = $builder; + } + + public function getBuilder(): Builder + { + return $this->builder; + } + + public function retrieveDatabaseRecordsUsing(Closure $retrieveDatabaseRecordsUsing): void + { + $this->retrieveDatabaseRecordsUsing = $retrieveDatabaseRecordsUsing; + } + + public function getRetrieveDatabaseRecordsUsing(): ?Closure + { + return $this->retrieveDatabaseRecordsUsing; + } +} diff --git a/src/Search/Searcher.php b/src/Search/Searcher.php new file mode 100644 index 0000000..7dea472 --- /dev/null +++ b/src/Search/Searcher.php @@ -0,0 +1,109 @@ + */ + protected array $mutators, + protected Client $elastic + ) { + } + + abstract public function index(): string; + + public function search(SearchCriteria $criteria): SearchResults + { + $builder = (new Builder($this->elastic)) + ->index($this->index()) + ->size($criteria->limit + 1) + ->from($criteria->offset); + + $state = new ElasticSearchState($criteria->actor, $criteria->isFulltext()); + $state->setBuilder($builder); + + // Default logic for retrieving database records. + // This is normally overriden by the fulltext filter. + $state->retrieveDatabaseRecordsUsing(function (array $response, SearchCriteria $criteria): Collection { + $results = (new Collection(Arr::get($response, 'hits.hits')))->map(function ($hit) { + $type = $hit['_source']['type']; + $id = Str::after($hit['_source']['id'], "$type:"); + + return [ + 'id' => $id, + 'score' => Arr::get($hit, '_score'), + 'weight' => Arr::get($hit, 'sort.0'), + ]; + })->sortByDesc('weight'); + + $ids = $results + ->take($criteria->limit) + ->pluck('id') + ->all(); + + return $this->getQuery($criteria->actor) + ->whereIn('id', $ids) + ->orderByRaw('FIELD(id, ' . implode(',', $ids) . ')') + ->get(); + }); + + $this->filters->apply($state, $criteria->filters); + + $this->applySort($state, $criteria); + + foreach ($this->mutators as $mutator) { + $mutator($state, $criteria); + } + +// echo json_encode($builder->getParams(), JSON_PRETTY_PRINT); +// exit; + + $response = $builder->search(); + +// header('Content-Type: application/json'); +// echo json_encode($response, JSON_PRETTY_PRINT); +// exit; + + $areMoreResults = count($response['hits']['hits']) > $criteria->limit; + + $callback = $state->getRetrieveDatabaseRecordsUsing(); + + if (! $callback) { + throw new \RuntimeException('No callback set to retrieve database records'); + } + + $records = $callback($response, $criteria); + + return new SearchResults($records, $areMoreResults); + } + + protected function applySort(ElasticSearchState $state, SearchCriteria $criteria): void + { + $sort = $criteria->sort; + + if ($criteria->sortIsDefault && ! empty($state->getDefaultSort())) { + $sort = $state->getDefaultSort(); + } + + if (is_callable($sort)) { + $sort($state->getBuilder()); + } else { + foreach (($criteria->sort ?? []) as $field => $direction) { + $state->getBuilder()->addSort(new Sort(Str::snake($field), $direction)); + } + } + } +} diff --git a/src/Searchers/CommentPostSearcher.php b/src/Searchers/CommentPostSearcher.php deleted file mode 100644 index 73bbe78..0000000 --- a/src/Searchers/CommentPostSearcher.php +++ /dev/null @@ -1,27 +0,0 @@ -setting('blomstra-search.search-post-bodies', true); - - return boolval($enabled); - } -} diff --git a/src/Searchers/DiscussionSearcher.php b/src/Searchers/DiscussionSearcher.php deleted file mode 100644 index af32452..0000000 --- a/src/Searchers/DiscussionSearcher.php +++ /dev/null @@ -1,32 +0,0 @@ -setting('blomstra-search.search-discussion-subjects', true); - - return boolval($enabled); - } - - public function boost(): float - { - return 1.5; - } -} diff --git a/src/Searchers/Searcher.php b/src/Searchers/Searcher.php deleted file mode 100644 index e6b6fe7..0000000 --- a/src/Searchers/Searcher.php +++ /dev/null @@ -1,48 +0,0 @@ -seeder; - - if (empty($seeder)) { - throw new \InvalidArgumentException('Implement type or add $seeder'); - } - - return (new $seeder())->type(); - } - - public function enabled(): bool - { - return true; - } - - public function boost(): float - { - return 1; - } - - protected function setting(string $key, $default = null) - { - return resolve(SettingsRepositoryInterface::class)->get($key, $default); - } -} diff --git a/src/Seeders/CommentSeeder.php b/src/Seeders/CommentSeeder.php deleted file mode 100644 index 55e50b4..0000000 --- a/src/Seeders/CommentSeeder.php +++ /dev/null @@ -1,96 +0,0 @@ -getType(new CommentPost()); - } - - public function query(): Builder - { - $includes = ['discussion']; - - if ($this->extensionEnabled('flarum-tags')) { - $includes[] = 'discussion.tags'; - } - - if ($this->extensionEnabled('fof-byobu')) { - $includes[] = 'discussion.recipientUsers'; - $includes[] = 'discussion.recipientGroups'; - } - - return CommentPost::query() - ->whereNull('hidden_at') - ->where('type', CommentPost::$type) - ->with($includes); - } - - public static function savingOn(Dispatcher $events, callable $callable) - { - $events->listen(Posted::class, function (Posted $event) use ($callable) { - $callable($event->post); - }); - } - - public static function deletingOn(Dispatcher $events, callable $callable) - { - $events->listen(Deleted::class, function (Deleted $event) use ($callable) { - $callable($event->post); - }); - } - - /** - * @param CommentPost $model - * - * @return Document - */ - public function toDocument(Model $model): Document - { - $document = new Document([ - 'type' => $this->type(), - 'id' => $this->type().':'.$model->id, - 'rawId' => $model->id, - 'content' => $model->content, - 'content_partial' => $model->content, - 'created_at' => $model->created_at?->toAtomString(), - 'updated_at' => $model->edited_at?->toAtomString(), - 'is_private' => $model->is_private, - 'user_id' => $model->user_id, - 'groups' => $this->groupsForDiscussion($model->discussion), - 'comment_count' => $model->discussion->comment_count, - ]); - - if ($this->extensionEnabled('flarum-tags')) { - $document['tags'] = $model->discussion->tags->pluck('id')->toArray(); - } - - if ($this->extensionEnabled('fof-byobu')) { - $document['recipient_users'] = $model->discussion->recipientUsers->pluck('id')->toArray(); - $document['recipient_groups'] = $model->discussion->recipientGroups->pluck('id')->toArray(); - } - - return $document; - } -} diff --git a/src/Seeders/DiscussionSeeder.php b/src/Seeders/DiscussionSeeder.php deleted file mode 100644 index 8558c50..0000000 --- a/src/Seeders/DiscussionSeeder.php +++ /dev/null @@ -1,101 +0,0 @@ -getType(new Discussion()); - } - - public function query(): Builder - { - $includes = []; - - if ($this->extensionEnabled('flarum-tags')) { - $includes[] = 'tags'; - } - - if ($this->extensionEnabled('fof-byobu')) { - $includes[] = 'recipientUsers'; - $includes[] = 'recipientGroups'; - } - - return Discussion::query() - ->whereNull('hidden_at') - ->with($includes); - } - - public static function savingOn(Dispatcher $events, callable $callable) - { - $events->listen([Started::class, Restored::class], function ($event) use ($callable) { - return $callable($event->discussion); - }); - } - - public static function deletingOn(Dispatcher $events, callable $callable) - { - $events->listen([Deleted::class, Hidden::class], function ($event) use ($callable) { - return $callable($event->discussion); - }); - } - - /** - * @param Discussion $model - * - * @return Document - */ - public function toDocument(Model $model): Document - { - $document = new Document([ - 'type' => $this->type(), - 'id' => $this->type().':'.$model->id, - 'rawId' => $model->id, - 'content' => $model->title, - 'content_partial' => $model->title, - 'created_at' => $model->created_at?->toAtomString(), - 'updated_at' => $model->last_posted_at?->toAtomString(), - 'is_private' => $model->is_private, - 'user_id' => $model->user_id, - 'groups' => $this->groupsForDiscussion($model), - 'comment_count' => $model->comment_count, - ]); - - if ($this->extensionEnabled('flarum-tags')) { - $document['tags'] = $model->tags->pluck('id')->toArray(); - } - - if ($this->extensionEnabled('fof-byobu')) { - $document['recipient_users'] = $model->recipientUsers->pluck('id')->toArray(); - $document['recipient_groups'] = $model->recipientGroups->pluck('id')->toArray(); - } - - if ($this->extensionEnabled('flarum-sticky')) { - $document['is_sticky'] = (bool) $model->is_sticky; - } - - return $document; - } -}