fix

composer.json

@@ -24,10 +24,13 @@
         }
     ],
     "require": {
-        "php": ">= 8.0",
+        "php": ">= 8.1",
-        "flarum/core": "^1.2.0",
+        "flarum/core": "^2.0",
-        "elasticsearch/elasticsearch": "7.*",
+        "elasticsearch/elasticsearch": "7.17.2",
-        "spatie/elasticsearch-query-builder": "^1.3.0"
+        "spatie/elasticsearch-query-builder": "^1.4.0"
+    },
+    "require-dev": {
+        "flarum/core": "^2.x-dev"
     },
     "extra": {
         "flarum-extension": {
@@ -45,5 +48,10 @@
         "psr-4": {
             "Blomstra\\Search\\": "src/"
         }
+    },
+    "config": {
+        "allow-plugins": {
+            "php-http/discovery": true
+        }
     }
 }

extend.php

@@ -12,25 +12,38 @@
 
 namespace Blomstra\Search;
 
+use Flarum\Discussion\Discussion as FlarumDiscussion;
 use Flarum\Extend as Flarum;
+use Flarum\Post\Post as FlarumPost;
 
 return [
-    (new Flarum\ServiceProvider())->register(Provider::class),
-
     (new Flarum\Frontend('forum'))
         ->js(__DIR__.'/js/dist/forum.js'),
     (new Flarum\Frontend('admin'))
         ->js(__DIR__.'/js/dist/admin.js'),
 
-    (new Flarum\Locales(__DIR__.'/resources/locale')),
+    new Flarum\Locales(__DIR__.'/resources/locale'),
+
+    (new Flarum\ServiceProvider())
+        ->register(Provider::class),
 
     (new Flarum\Routes('api'))
-        ->get('/blomstra/search/{type}', 'blomstra.search', Api\Controllers\SearchController::class)
         ->put('/blomstra/search/index', 'blomstra.search.index', Api\Controllers\IndexController::class),
+
     (new Flarum\Console())
         ->command(Commands\BuildCommand::class),
 
     (new Flarum\Settings())
         ->default('blomstra-search.search-discussion-subjects', true)
         ->default('blomstra-search.search-post-bodies', true),
+
+    (new Flarum\SearchDriver(Search\ElasticSearchDriver::class))
+        ->addSearcher(FlarumDiscussion::class, Discussion\DiscussionSearcher::class)
+        ->addFilter(Discussion\DiscussionSearcher::class, Discussion\PrivateFilterMutator::class)
+        ->addMutator(Discussion\DiscussionSearcher::class, Discussion\PrivateFilterMutator::mutate(...))
+        ->setFulltext(Discussion\DiscussionSearcher::class, Discussion\FulltextFilter::class),
+
+    (new Flarum\SearchIndex())
+        ->indexer(FlarumDiscussion::class, Discussion\DiscussionIndexer::class)
+        ->indexer(FlarumPost::class, Post\CommentPostIndexer::class),
 ];

js/src/forum/PaginatedListStates/extendDiscussionState.tsx

@@ -1,33 +0,0 @@
-import app from 'flarum/forum/app';
-
-import { override } from 'flarum/common/extend';
-
-import DiscussionListState from 'flarum/forum/states/DiscussionListState';
-
-export default function extendDiscussionState() {
-  override(DiscussionListState.prototype, 'loadPage', async function (this: DiscussionListState, original, page: number = 1) {
-    const preloaded = app.data.apiDocument || null;
-
-    // If existing payload is given or no search is made,  fallback on native page.
-    if (preloaded || !this.requestParams()?.filter?.q) return original.call(this, page);
-
-    const params = this.requestParams();
-    params.page = {
-      offset: this.pageSize * (page - 1),
-      ...params.page,
-    };
-
-    if (Array.isArray(params.include)) {
-      params.include = params.include.join(',');
-    }
-
-    // Construct API search URI
-    const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`;
-
-    // Make API GET request
-    const results = await app.request({ params, url, method: 'GET' });
-
-    // Parse API response into models and push to store
-    return app.store.pushPayload(results);
-  });
-}

js/src/forum/SearchSources/DiscussionsSearchSource.tsx

@@ -1,76 +0,0 @@
-import app from 'flarum/forum/app';
-import highlight from 'flarum/common/helpers/highlight';
-import LinkButton from 'flarum/common/components/LinkButton';
-import Link from 'flarum/common/components/Link';
-import { SearchSource } from 'flarum/forum/components/Search';
-import type Mithril from 'mithril';
-
-/**
- * The `DiscussionsSearchSource` finds and displays discussion search results in
- * the search dropdown.
- */
-export default class DiscussionsSearchSource implements SearchSource {
-  /**
-   * Map of lowercase search queries to their respective model results
-   */
-  protected results = new Map<string, any[]>();
-
-  /**
-   * Model name used for assembling API endpoint URL, and for frontend DOM.
-   */
-  private type = 'discussions';
-
-  async search(query: string): Promise<void> {
-    query = query.toLowerCase();
-
-    this.results.set(query, []);
-
-    const params = {
-      filter: { q: query },
-      page: { limit: 3 },
-      include: 'mostRelevantPost',
-    };
-
-    // Construct API search URI
-    const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`;
-
-    // Make API GET request
-    const results = await app.request({ params, url, method: 'GET' });
-
-    // Parse API response into models and push to store
-    const models = app.store.pushPayload(results);
-
-    // Add models to results map
-    this.results.set(query, models);
-  }
-
-  view(query: string): Array<Mithril.Vnode> {
-    query = query.toLowerCase();
-
-    // Get results from map
-    const queryResults = this.results.get(query) || [];
-
-    const results = queryResults.map((discussion: any) => {
-      const mostRelevantPost = discussion.mostRelevantPost();
-
-      return (
-        <li className="DiscussionSearchResult" data-index={`${this.type}${discussion.id()}`}>
-          <Link href={app.route.discussion(discussion, mostRelevantPost && mostRelevantPost.number())}>
-            <div className="DiscussionSearchResult-title">{highlight(discussion.title(), query)}</div>
-            {!!mostRelevantPost && <div className="DiscussionSearchResult-excerpt">{highlight(mostRelevantPost.contentPlain(), query, 100)}</div>}
-          </Link>
-        </li>
-      );
-    });
-
-    return [
-      <li className="Dropdown-header">{app.translator.trans('core.forum.search.discussions_heading')}</li>,
-      <li>
-        <LinkButton icon="fas fa-search" href={app.route('index', { q: query })}>
-          {app.translator.trans('core.forum.search.all_discussions_button', { query })}
-        </LinkButton>
-      </li>,
-      ...results,
-    ];
-  }
-}

js/src/forum/index.ts

@@ -1,23 +1,5 @@
 import app from 'flarum/forum/app';
 
-import Search, { SearchAttrs, SearchSource } from 'flarum/forum/components/Search';
-
-import { extend } from 'flarum/common/extend';
-import ItemList from 'flarum/common/utils/ItemList';
-
-import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource';
-import extendDiscussionState from './PaginatedListStates/extendDiscussionState';
-
 app.initializers.add('blomstra-search', () => {
-  extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) {
+  //
-    items.replace('discussions', new DiscussionsSearchSource());
-  });
 });
-
-app.initializers.add(
-  'blomstra-search-early',
-  () => {
-    extendDiscussionState();
-  },
-  999999
-);

src/Api/Client.php

@@ -1,28 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Api;
-
-use Illuminate\Support\Arr;
-use Psr\Http\Message\ResponseInterface;
-
-class Client extends \Flarum\Api\Client
-{
-    public function get(string $path): ResponseInterface
-    {
-        if ($path === '/discussions' && Arr::has($this->queryParams, 'filter.q')) {
-            return parent::get('/blomstra/search/discussions');
-        }
-
-        return parent::get($path);
-    }
-}

src/Api/Controllers/SearchController.php

@@ -12,6 +12,7 @@
 
 namespace Blomstra\Search\Api\Controllers;
 
+use Blomstra\Search\Elasticsearch\Builder;
 use Blomstra\Search\Elasticsearch\MatchPhraseQuery;
 use Blomstra\Search\Elasticsearch\MatchQuery;
 use Blomstra\Search\Elasticsearch\TermsQuery;
@@ -32,7 +33,6 @@ use Illuminate\Support\Arr;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Str;
 use Psr\Http\Message\ServerRequestInterface;
-use Spatie\ElasticsearchQueryBuilder\Builder;
 use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
 use Spatie\ElasticsearchQueryBuilder\Queries\Query;
 use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;

src/Commands/BuildCommand.php

@@ -12,27 +12,28 @@
 
 namespace Blomstra\Search\Commands;
 
-use Blomstra\Search\Jobs\Job;
+use Blomstra\Search\Discussion\DiscussionIndexer;
-use Blomstra\Search\Jobs\SavingJob;
+use Blomstra\Search\Elasticsearch\Builder;
-use Blomstra\Search\Seeders\Seeder;
+use Blomstra\Search\Post\CommentPostIndexer;
 use Elasticsearch\Client;
+use Flarum\Discussion\Discussion;
+use Flarum\Post\Post;
+use Flarum\Search\Job\IndexJob;
 use Flarum\Settings\SettingsRepositoryInterface;
 use Illuminate\Console\Command;
 use Illuminate\Contracts\Container\Container;
 use Illuminate\Contracts\Queue\Queue;
 use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Support\Arr;
-use Spatie\ElasticsearchQueryBuilder\Builder;
 use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
 use Spatie\ElasticsearchQueryBuilder\Queries\RangeQuery;
-use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;
 
 class BuildCommand extends Command
 {
     protected $signature = 'blomstra:search:index
         {--max-id= : Limits for each object the number of items to seed}
         {--throttle= : Number of seconds to wait between pushing to the queue}
-        {--only= : type to run seeder for, eg discussions or posts}
+        {--only= : type of model to index, eg discussions or posts}
         {--recreate : create or recreate the index}
         {--mapping : recreate the mapping}
         {--continue : continue each object type where you left off}
@@ -40,118 +41,64 @@ class BuildCommand extends Command
 
     protected $description = 'Rebuilds the complete search server with its documents.';
 
-    public function handle(Container $container)
+    public function handle(Container $container, Client $client, Queue $queue, SettingsRepositoryInterface $settings)
     {
-        /** @var string $index */
+        $indexers = [
-        $index = $container->make('blomstra.search.elastic_index');
+            'discussions' => DiscussionIndexer::class,
+            'posts'       => CommentPostIndexer::class,
+        ];
+        $models = [
+            'discussions' => Discussion::class,
+            'posts'       => Post::class,
+        ];
 
-        /** @var array|string[] $seeders */
+        $only = $this->option('only');
-        $seeders = $container->tagged('blomstra.search.seeders');
+        $onlyIndexers = $only
+            ? Arr::only($indexers, is_array($only) ? $only : explode(',', $only))
+            : $indexers;
 
-        /** @var Queue $queue */
+        foreach ($onlyIndexers as $type => $indexerClass) {
-        $queue = $container->make(Queue::class);
+            /** @var DiscussionIndexer|CommentPostIndexer $indexer */
+            $indexer = $container->make($indexerClass);
 
-        /** @var Client $client */
-        $client = $container->make(Client::class);
-
-        /** @var SettingsRepositoryInterface $settings */
-        $settings = $container->make(SettingsRepositoryInterface::class);
-
-        // Elastic scheme definition.
             $properties = [
-            'properties' => [
+                'properties' => $indexer->properties(),
-                'content'          => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'],
-                'rawId'            => ['type' => 'integer'],
-                'created_at'       => ['type' => 'date'],
-                'updated_at'       => ['type' => 'date'],
-                'is_private'       => ['type' => 'boolean'],
-                'is_sticky'        => ['type' => 'boolean'],
-                'groups'           => ['type' => 'integer'],
-                'tags'             => ['type' => 'integer'],
-                'recipient_groups' => ['type' => 'integer'],
-                'recipient_users'  => ['type' => 'integer'],
-                'comment_count'    => ['type' => 'integer'],
-            ],
             ];
 
             // Remove/delete the whole index.
             if ($this->option('recreate')) {
-            // Flush the index.
+                $indexer->flush();
-            $client->indices()->delete([
+                $indexer->build();
-                'index'              => $index,
-                'ignore_unavailable' => true,
-            ]);
-
-            // Create a new index.
-            $client->indices()->create([
-                'index' => $index,
-                'body'  => [
-                    'settings' => [
-                        'index.max_ngram_diff' => 10,
-                        'analysis'             => [
-                            'analyzer' => [
-                                'flarum_analyzer' => [
-                                    'type' => $settings->get('blomstra-search.analyzer-language') ?: 'english',
-                                ],
-                                'flarum_analyzer_partial' => [
-                                    'type'      => 'custom',
-                                    'tokenizer' => 'standard',
-                                    'filter'    => [
-                                        'lowercase',
-                                        'partial_search_filter',
-                                    ],
-                                ],
-                            ],
-                            'filter' => [
-                                'partial_search_filter' => [
-                                    'type'        => 'ngram',
-                                    'min_gram'    => 1,
-                                    'max_gram'    => 10,
-                                    'token_chars' => ['letter', 'digit', 'symbol'],
-                                ],
-                            ],
-                        ],
-                    ],
-                ],
-            ]);
             }
 
             // Create the index.
-        if ($this->option('recreate') || $this->option('mapping')) {
+            if (!$this->option('recreate') && $this->option('mapping')) {
                 $client->indices()->putMapping([
-                'index' => $index,
+                    'index' => $indexer::index(),
                     'body'  => $properties,
                 ]);
             }
 
-        // Seed only a specific resource.
-        $only = $this->option('only');
-
-        /** @var Seeder $seeder */
-        foreach ($seeders as $seeder) {
-            if ($only && $seeder->type() !== $only) {
-                continue;
-            }
-
             $total = 0;
 
+            $query = $models[$type]::query();
+
             $continueAt = $this->option('continue')
-                ? ($this->continueAt($seeder->type()) ?? $seeder->query()->max('id'))
+                ? ($this->continueAt($type) ?? $query->max('id'))
-                : $seeder->query()->max('id');
+                : $query->max('id');
 
             $seeded = null;
 
             while ($continueAt !== null) {
                 if ($this->option('seed-missing')) {
                     $response = (new Builder($client))
-                        ->index($index)
+                        ->index($indexer::index())
                         ->size(1000)
                         ->addQuery(
                             (new BoolQuery())
                                 ->add((new RangeQuery('rawId'))
                                 ->gte($continueAt - 1000)
                                 ->lte($continueAt))
-                            ->add(TermQuery::create('type', $seeder->type()))
                         )
                         ->search();
 
@@ -159,7 +106,7 @@ class BuildCommand extends Command
                 }
 
                 /** @var Collection $collection */
-                $collection = $seeder->query()
+                $collection = $query
                     ->latest('id')
                     ->whereBetween('id', [$continueAt - 1000, $continueAt])
                     ->when($this->option('max-id'), function ($query, $id) {
@@ -171,16 +118,14 @@ class BuildCommand extends Command
                 $min = $collection->min('id');
                 $continueAt = $min && $min > 2 ? $min - 1 : null;
 
-                $queue->pushOn(Job::$onQueue, new SavingJob($collection, $seeder));
+                $onQueue = property_exists($indexerClass, 'queue') ? $indexerClass::$queue : null;
+                $queue->pushOn($onQueue, new IndexJob($indexerClass, $collection->all(), IndexJob::SAVE));
 
-                $this->info("Pushed into the index, type: {$seeder->type()}, amount: {$collection->count()}.");
+                $this->info("Pushed into the index, type: $type, amount: {$collection->count()}.");
 
                 $total += $collection->count();
 
-                $this->continueAt(
+                $this->continueAt($type, $continueAt);
-                    $seeder->type(),
-                    $continueAt
-                );
 
                 if ($throttle = $this->option('throttle')) {
                     $this->info("Throttling for $throttle seconds");

src/Discussion/DiscussionIndexer.php

@@ -0,0 +1,111 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Discussion;
+
+use Blomstra\Search\Save\Document;
+use Blomstra\Search\Search\Concerns\AppliesAccessControl;
+use Blomstra\Search\Search\ElasticIndex;
+use Flarum\Discussion\Discussion;
+use Flarum\Extension\ExtensionManager;
+use Flarum\Search\IndexerInterface;
+
+class DiscussionIndexer implements IndexerInterface
+{
+    use AppliesAccessControl;
+
+    public function __construct(
+        protected ElasticIndex $elastic,
+        protected ExtensionManager $extensions
+    ) {
+    }
+
+    public static function index(): string
+    {
+        return 'discussions';
+    }
+
+    public function save(array $models): void
+    {
+        $this->elastic->save(self::index(), $models, $this->toDocument(...));
+    }
+
+    public function delete(array $models): void
+    {
+        $this->elastic->delete(self::index(), $models);
+    }
+
+    public function build(): void
+    {
+        $this->elastic->build(self::index(), $this->properties());
+    }
+
+    public function flush(): void
+    {
+        $this->elastic->flush(self::index());
+    }
+
+    public function properties(): array
+    {
+        return [
+            'rawId'            => ['type' => 'integer'],
+            'discussion_id'    => ['type' => 'integer'],
+            'title'            => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'],
+            'created_at'       => ['type' => 'date'],
+            'updated_at'       => ['type' => 'date'],
+            'last_posted_at'   => ['type' => 'date'],
+            'is_private'       => ['type' => 'boolean'],
+            'is_sticky'        => ['type' => 'boolean'],
+            'is_locked'        => ['type' => 'boolean'],
+            'groups'           => ['type' => 'integer'],
+            'tags'             => ['type' => 'integer'],
+            'recipient_groups' => ['type' => 'integer'],
+            'recipient_users'  => ['type' => 'integer'],
+            'comment_count'    => ['type' => 'integer'],
+        ];
+    }
+
+    public function toDocument(Discussion $model): Document
+    {
+        $document = new Document([
+            'id'              => $model->id,
+            'discussion_id'   => $model->id, // duplicated for result aggregation in searching with posts.
+            'rawId'           => $model->id,
+            'title'           => $model->title,
+            'created_at'      => $model->created_at?->toAtomString(),
+            'updated_at'      => $model->last_posted_at?->toAtomString(),
+            'is_private'      => $model->is_private,
+            'user_id'         => $model->user_id,
+            'groups'          => $this->groupsForDiscussion($model),
+            'comment_count'   => $model->comment_count,
+        ]);
+
+        if ($this->extensions->isEnabled('flarum-tags')) {
+            $document['tags'] = $model->tags->pluck('id')->toArray();
+        }
+
+        if ($this->extensions->isEnabled('fof-byobu')) {
+            $document['recipient_users'] = $model->recipientUsers->pluck('id')->toArray();
+            $document['recipient_groups'] = $model->recipientGroups->pluck('id')->toArray();
+        }
+
+        if ($this->extensions->isEnabled('flarum-sticky')) {
+            $document['is_sticky'] = $model->is_sticky;
+        }
+
+        if ($this->extensions->isEnabled('flarum-lock')) {
+            $document['is_locked'] = $model->is_locked;
+        }
+
+        return $document;
+    }
+}

src/Discussion/DiscussionSearcher.php

@@ -0,0 +1,32 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Discussion;
+
+use Blomstra\Search\Post\CommentPostIndexer;
+use Blomstra\Search\Search\Searcher;
+use Flarum\Discussion\Discussion;
+use Flarum\User\User;
+use Illuminate\Database\Eloquent\Builder;
+
+class DiscussionSearcher extends Searcher
+{
+    public function index(): string
+    {
+        return DiscussionIndexer::index().','.CommentPostIndexer::index();
+    }
+
+    public function getQuery(User $actor): Builder
+    {
+        return Discussion::whereVisibleTo($actor)->select('discussions.*');
+    }
+}

src/Discussion/FulltextFilter.php

@@ -0,0 +1,166 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Discussion;
+
+use Blomstra\Search\Elasticsearch\Builder;
+use Blomstra\Search\Elasticsearch\MatchPhraseQuery;
+use Blomstra\Search\Elasticsearch\MatchQuery;
+use Blomstra\Search\Elasticsearch\TermsQuery;
+use Blomstra\Search\Post\CommentPostIndexer;
+use Blomstra\Search\Search\ElasticSearchState;
+use Elasticsearch\Client;
+use Flarum\Discussion\Discussion;
+use Flarum\Post\Post;
+use Flarum\Search\AbstractFulltextFilter;
+use Flarum\Search\SearchCriteria;
+use Flarum\Search\SearchManager;
+use Flarum\Search\SearchState;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
+use Spatie\ElasticsearchQueryBuilder\Aggregations\FilterAggregation;
+use Spatie\ElasticsearchQueryBuilder\Aggregations\TermsAggregation;
+use Spatie\ElasticsearchQueryBuilder\Aggregations\TopHitsAggregation;
+use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
+use Spatie\ElasticsearchQueryBuilder\Queries\Query;
+use Spatie\ElasticsearchQueryBuilder\Sorts\Sort;
+
+/**
+ * Unlike the default flarum database search,
+ * this search will not get score based on the sum of post scores of a discussion.
+ *
+ * @extends AbstractFulltextFilter<ElasticSearchState>
+ */
+class FulltextFilter extends AbstractFulltextFilter
+{
+    public function __construct(
+        protected SearchManager $search,
+        protected Client $elastic
+    ) {
+    }
+
+    public function search(SearchState $state, string $value): void
+    {
+        $builder = $state->getBuilder();
+
+        $query = BoolQuery::create()
+            ->add(
+                BoolQuery::create()
+                    ->add($this->exactMatch('title', $value, 1.2), 'should')
+                    ->add($this->wordMatch('title', $value, 'and', 1.2), 'should')
+                    ->add($this->wordMatch('title', $value, 'or', 1.2), 'should'),
+                'should'
+            )
+            ->add(
+                BoolQuery::create()
+                    ->add($this->exactMatch('content', $value), 'should', 1)
+                    ->add($this->wordMatch('content', $value, 'and', 1), 'should')
+                    ->add($this->wordMatch('content', $value, 'or', 1), 'should'),
+                'should'
+            );
+
+        $builder->addQuery($query);
+
+        $builder->collapse('discussion_id');
+
+        $aggs = FilterAggregation::create('posts', TermsQuery::create('_index', [CommentPostIndexer::index()]))
+            ->aggregation(
+                TermsAggregation::create('per_discussion', 'discussion_id')
+                    ->aggregation(
+                        TopHitsAggregation::create('most_relevant_post_id', 1, Sort::create('_score'))
+                    )
+            );
+
+        $builder->addAggregation($aggs);
+
+        $state->setDefaultSort(function (Builder $builder) {
+            $builder->addSort(
+                Sort::create('_score', 'desc')
+            );
+        });
+
+        $state->retrieveDatabaseRecordsUsing(function (array $response, SearchCriteria $criteria): Collection {
+            $buckets = Collection::make(Arr::get($response, 'aggregations.posts.per_discussion.buckets'))
+                ->map(fn (array $hit) => [
+                    'discussion_id'            => $hit['key'],
+                    'most_relevant_post_id'    => Arr::get($hit, 'most_relevant_post_id.most_relevant_post_id.hits.hits.0._id'),
+                    'most_relevant_post_score' => Arr::get($hit, 'most_relevant_post_id.most_relevant_post_id.hits.hits.0._score'),
+                ])->keyBy('discussion_id');
+
+            $results = Collection::make(Arr::get($response, 'hits.hits'))
+                ->map(fn (array $hit) => [
+                    'discussion_id'            => $hit['_source']['discussion_id'],
+                    'most_relevant_post_id'    => Arr::get($buckets->get($hit['_source']['discussion_id']), 'most_relevant_post_id'),
+                    'most_relevant_post_score' => Arr::get($buckets->get($hit['_source']['discussion_id']), 'most_relevant_post_score'),
+                    'title_score'              => $hit['_score'],
+                ])->keyBy('discussion_id');
+
+            // We have $hits and $buckets, both are sorted by score.
+            // $discussionResult represents discussion scores based on the title.
+            // $buckets represents discussion scores based on the sum of all posts,
+            // and also the most relevant post id and its score.
+            // We need to merge these two results into one collection, and sort them by score.
+            // We also need to make sure that the most relevant post id is set on the discussion.
+            $results = $results
+                // Sort by title_score, most_relevant_post_score then posts_score.
+                ->sortByDesc(function ($result) {
+                    return $result['title_score'] ?? 0
+                        + $result['most_relevant_post_score'] ?? 0
+                        + $result['posts_score'] ?? 0;
+                })
+                ->take($criteria->limit);
+
+            $actor = $criteria->actor;
+            $connection = Post::query()->getConnection();
+
+            if ($results->isEmpty()) {
+                return Discussion::whereVisibleTo($actor)
+                    ->select('discussions.*')
+                    ->get();
+            }
+
+            return Discussion::whereVisibleTo($actor)
+                ->select('discussions.*')
+                ->selectRaw(
+                    $connection->raw('COALESCE(('.$connection->getQueryGrammar()->compileSelect(
+                        $postsQuery = Post::query()
+                            ->select('posts.id')
+                            ->whereIn('posts.id', $results->pluck('most_relevant_post_id')->filter())
+                            ->whereColumn('discussions.id', '=', 'posts.discussion_id')
+                            ->limit(1)
+                            ->toBase()
+                    ).'), first_post_id) as most_relevant_post_id')->getValue($connection->getQueryGrammar())
+                )
+                ->mergeBindings($postsQuery)
+                ->whereIn('discussions.id', $results->pluck('discussion_id'))
+                ->orderByRaw('FIELD(`discussions`.`id`, '.implode(',', $results->pluck('discussion_id')->all()).')')
+                ->get();
+        });
+    }
+
+    protected function exactMatch(string $field, string $q, int $fieldBoost = 1): Query
+    {
+        $query = (new MatchPhraseQuery($field, $q));
+
+        return $query->boost(2 * $fieldBoost);
+    }
+
+    protected function wordMatch(string $field, string $q, string $operator, int $fieldBoost = 1): Query
+    {
+        $query = (new MatchQuery($field, $q))
+            ->operator($operator);
+
+        $boost = $operator === 'and' ? 1.8 : .8;
+
+        return $query->boost($boost * $fieldBoost);
+    }
+}

src/Discussion/PrivateFilterMutator.php

@@ -0,0 +1,95 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Discussion;
+
+use Blomstra\Search\Elasticsearch\TermsQuery;
+use Blomstra\Search\Search\Concerns\AppliesAccessControl;
+use Blomstra\Search\Search\ElasticSearchState;
+use Flarum\Extension\ExtensionManager;
+use Flarum\Search\Filter\FilterInterface;
+use Flarum\Search\SearchState;
+use Illuminate\Support\Arr;
+use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
+use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;
+
+/**
+ * @implements FilterInterface<ElasticSearchState>
+ */
+class PrivateFilterMutator implements FilterInterface
+{
+    use AppliesAccessControl;
+
+    public function __construct(
+        protected ExtensionManager $extensions
+    ) {
+    }
+
+    public function getFilterKey(): string
+    {
+        return 'private';
+    }
+
+    public function filter(SearchState $state, array|string $value, bool $negate): void
+    {
+        $actor = $state->getActor();
+
+        if (!$this->extensions->isEnabled('fof-byobu') || $actor->isGuest()) {
+            return;
+        }
+
+        $builder = $state->getBuilder();
+        $query = BoolQuery::create();
+
+        $query->add(self::byobuAccessQuery($state), 'filter');
+
+        $builder->addQuery($query);
+    }
+
+    public static function mutate(ElasticSearchState $state): void
+    {
+        $builder = $state->getBuilder();
+
+        // If this filter isn't active, we apply both the private and public queries as should clauses.
+        if (Arr::first($state->getActiveFilters(), fn ($filter) => $filter->getFilterKey() === 'private')) {
+            return;
+        }
+
+        $query = BoolQuery::create();
+
+        $query
+            ->add(self::restrictQuery($state), 'should')
+            ->add(self::byobuAccessQuery($state), 'should');
+
+        $builder->addQuery($query);
+    }
+
+    private static function restrictQuery(SearchState $state): BoolQuery
+    {
+        return BoolQuery::create()
+            ->add(TermQuery::create('is_private', 'false'))
+            ->add(TermsQuery::create('groups', self::groupsForUser($state->getActor())));
+    }
+
+    private static function byobuAccessQuery(SearchState $state): BoolQuery
+    {
+        $actor = $state->getActor();
+
+        return BoolQuery::create()
+            ->add(TermQuery::create('is_private', 'true'))
+            ->add(
+                BoolQuery::create()
+                    ->add(TermsQuery::create('recipient_groups', self::groupsForUser($actor)), 'should')
+                    ->add(TermsQuery::create('recipient_users', [$actor->id]), 'should'),
+            );
+    }
+}

src/Elasticsearch/Builder.php

@@ -0,0 +1,68 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Elasticsearch;
+
+class Builder extends \Spatie\ElasticsearchQueryBuilder\Builder
+{
+    protected array $collapse = [];
+
+    public function collapse(string $field): static
+    {
+        $this->collapse = [
+            'field' => $field,
+        ];
+
+        return $this;
+    }
+
+    public function getPayload(): array
+    {
+        $payload = parent::getPayload();
+
+        if ($this->collapse) {
+            $payload['collapse'] = $this->collapse;
+        }
+
+        return $payload;
+    }
+
+    public function search(): array
+    {
+        $params = $this->getParams();
+
+        return $this->client->search($params);
+    }
+
+    public function getParams(): array
+    {
+        $payload = $this->getPayload();
+
+        $params = [
+            'body' => $payload,
+        ];
+
+        if ($this->searchIndex) {
+            $params['index'] = $this->searchIndex;
+        }
+
+        if ($this->size !== null) {
+            $params['size'] = $this->size;
+        }
+
+        if ($this->from !== null) {
+            $params['from'] = $this->from;
+        }
+
+        return $params;
+    }
+}

src/Elasticsearch/SimpleSearchQuery.php

@@ -1,59 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Elasticsearch;
-
-use Spatie\ElasticsearchQueryBuilder\Queries\Query;
-
-class SimpleSearchQuery implements Query
-{
-    protected float $boost = 1;
-    protected ?string $analyzer = null;
-
-    public static function create(array $field, string $value)
-    {
-        return new self($field, $value);
-    }
-
-    public function boost(float $boost = 1)
-    {
-        $this->boost = $boost;
-
-        return $this;
-    }
-
-    public function analyzer(string $analyzer)
-    {
-        $this->analyzer = $analyzer;
-
-        return $this;
-    }
-
-    public function __construct(
-        protected array $fields,
-        protected string $value
-    ) {
-    }
-
-    public function toArray(): array
-    {
-        return [
-            'simple_query_string' => [
-                'query'            => $this->value,
-                'fields'           => $this->fields,
-                'analyzer'         => $this->analyzer,
-                'default_operator' => 'AND',
-                'boost'            => $this->boost,
-            ],
-        ];
-    }
-}

src/Exceptions/IndexingException.php

@@ -14,9 +14,9 @@ namespace Blomstra\Search\Exceptions;
 
 use Throwable;
 
-class SeedingException extends \Exception
+class IndexingException extends \Exception
 {
-    public function __construct($message = '', public array $items, $code = 0, Throwable $previous = null)
+    public function __construct(string $message, public array $items, $code = 0, Throwable $previous = null)
     {
         parent::__construct($message, $code, $previous);
     }

src/Jobs/DeletingJob.php

@@ -1,41 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Jobs;
-
-use Elasticsearch\Client;
-use Illuminate\Database\Eloquent\Model;
-
-class DeletingJob extends Job
-{
-    public function handle(Client $client)
-    {
-        if ($this->models->isEmpty()) {
-            return;
-        }
-
-        // Preparing body for storing.
-        $body = $this->models->map(function (Model $model) {
-            $document = $this->seeder->toDocument($model);
-
-            return [
-                ['delete' => ['_index' => $this->index, '_id' => $document->id]],
-            ];
-        })->flatten(1);
-
-        $response = $client->bulk([
-            'index'   => $this->index,
-            'body'    => $body->toArray(),
-            'refresh' => true,
-        ]);
-    }
-}

src/Jobs/Job.php

@@ -1,33 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Jobs;
-
-use Blomstra\Search\Seeders\Seeder;
-use Flarum\Queue\AbstractJob;
-use Illuminate\Support\Collection;
-
-abstract class Job extends AbstractJob
-{
-    protected string $index;
-
-    public static ?string $onQueue = null;
-
-    public function __construct(protected Collection $models, protected Seeder $seeder)
-    {
-        $this->index = resolve('blomstra.search.elastic_index');
-
-        if (static::$onQueue) {
-            $this->onQueue(static::$onQueue);
-        }
-    }
-}

src/Jobs/SavingJob.php

@@ -1,58 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Jobs;
-
-use Blomstra\Search\Exceptions\SeedingException;
-use Elasticsearch\Client;
-use Illuminate\Database\Eloquent\Model;
-use Illuminate\Support\Arr;
-
-class SavingJob extends Job
-{
-    public function handle(Client $client)
-    {
-        if ($this->models->isEmpty()) {
-            return;
-        }
-
-        // Preparing body for storing.
-        $body = $this->models->map(function (Model $model) {
-            $document = $this->seeder->toDocument($model);
-
-            return [
-                ['index' => ['_index' => $this->index, '_id' => $document->id]],
-                $document->toArray(),
-            ];
-        })
-        ->flatten(1);
-
-        $response = $client->bulk([
-            'index'   => $this->index,
-            'body'    => $body->toArray(),
-            'refresh' => true,
-        ]);
-
-        if (Arr::get($response, 'errors') !== true) {
-            return true;
-        }
-
-        $items = Arr::get($response, 'items');
-
-        $error = Arr::get(Arr::first($items), 'index.error.reason');
-
-        throw new SeedingException(
-            "Failed to seed: $error",
-            $items
-        );
-    }
-}

src/Post/CommentPostIndexer.php

@@ -0,0 +1,109 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Post;
+
+use Blomstra\Search\Save\Document;
+use Blomstra\Search\Search\Concerns\AppliesAccessControl;
+use Blomstra\Search\Search\ElasticIndex;
+use Flarum\Extension\ExtensionManager;
+use Flarum\Post\CommentPost;
+use Flarum\Post\Post;
+use Flarum\Search\IndexerInterface;
+
+class CommentPostIndexer implements IndexerInterface
+{
+    use AppliesAccessControl;
+
+    public function __construct(
+        protected ElasticIndex $elastic,
+        protected ExtensionManager $extensions
+    ) {
+    }
+
+    public static function index(): string
+    {
+        return 'posts';
+    }
+
+    public function save(array $models): void
+    {
+        $this->elastic->save(
+            self::index(),
+            array_filter($models, fn (Post $model) => $model->type === CommentPost::$type),
+            $this->toDocument(...)
+        );
+    }
+
+    public function delete(array $models): void
+    {
+        $this->elastic->delete(
+            self::index(),
+            array_filter($models, fn (Post $model) => $model->type === CommentPost::$type)
+        );
+    }
+
+    public function build(): void
+    {
+        $this->elastic->build(self::index(), $this->properties());
+    }
+
+    public function flush(): void
+    {
+        $this->elastic->flush(self::index());
+    }
+
+    public function properties(): array
+    {
+        return [
+            'rawId'            => ['type' => 'integer'],
+            'content'          => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'],
+            'discussion_id'    => ['type' => 'integer'],
+            'created_at'       => ['type' => 'date'],
+            'updated_at'       => ['type' => 'date'],
+            'is_private'       => ['type' => 'boolean'],
+            'groups'           => ['type' => 'integer'],
+            'tags'             => ['type' => 'integer'],
+            'recipient_groups' => ['type' => 'integer'],
+            'recipient_users'  => ['type' => 'integer'],
+            'comment_count'    => ['type' => 'integer'],
+        ];
+    }
+
+    public function toDocument(Post $model): Document
+    {
+        $document = new Document([
+            'id'              => $model->id,
+            'type'            => $model->type,
+            'rawId'           => $model->id,
+            'discussion_id'   => $model->discussion_id,
+            'content'         => $model->content,
+            'created_at'      => $model->created_at?->toAtomString(),
+            'updated_at'      => $model->edited_at?->toAtomString(),
+            'is_private'      => $model->is_private,
+            'user_id'         => $model->user_id,
+            'groups'          => $this->groupsForDiscussion($model->discussion),
+            'comment_count'   => $model->discussion->comment_count,
+        ]);
+
+        if ($this->extensions->isEnabled('flarum-tags')) {
+            $document['tags'] = $model->discussion->tags->pluck('id')->toArray();
+        }
+
+        if ($this->extensions->isEnabled('fof-byobu')) {
+            $document['recipient_users'] = $model->discussion->recipientUsers->pluck('id')->toArray();
+            $document['recipient_groups'] = $model->discussion->recipientGroups->pluck('id')->toArray();
+        }
+
+        return $document;
+    }
+}

src/Provider.php

@@ -12,39 +12,25 @@
 
 namespace Blomstra\Search;
 
-use Blomstra\Search\Jobs\DeletingJob;
-use Blomstra\Search\Jobs\Job;
-use Blomstra\Search\Jobs\SavingJob;
 use Elasticsearch\Client as Elastic;
 use Elasticsearch\ClientBuilder;
-use Flarum\Api\Client;
 use Flarum\Foundation\AbstractServiceProvider;
 use Flarum\Foundation\Config;
-use Flarum\Http\Middleware\ExecuteRoute;
 use Flarum\Settings\SettingsRepositoryInterface;
 use Illuminate\Contracts\Container\Container;
-use Illuminate\Contracts\Events\Dispatcher;
-use Illuminate\Contracts\Queue\Queue;
-use Illuminate\Support\Collection;
-use Laminas\Stratigility\MiddlewarePipe;
 use Psr\Log\LoggerInterface;
 
 class Provider extends AbstractServiceProvider
 {
-    public function register()
+    public function register(): void
     {
-        $this->container->tag([
+        $this->container->singleton(Elastic::class, function (Container $container) {
-            Seeders\DiscussionSeeder::class,
+            /** @var Config $config */
-            Seeders\CommentSeeder::class,
+            $config = $this->container->make(Config::class);
-        ], 'blomstra.search.seeders');
 
             /** @var SettingsRepositoryInterface $settings */
             $settings = $this->container->make(SettingsRepositoryInterface::class);
 
-        /** @var Config $config */
-        $config = $this->container->make(Config::class);
-
-        $this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) {
             $builder = ClientBuilder::create()
                 ->setHosts([$settings->get('blomstra-search.elastic-endpoint')]);
 
@@ -61,59 +47,10 @@ class Provider extends AbstractServiceProvider
 
             return $builder->build();
         });
-
-        $this->container->instance(
-            'blomstra.search.elastic_index',
-            $settings->get('blomstra-search.elastic-index', 'flarum')
-        );
-
-        $this->container->extend(
-            Client::class,
-            function () {
-                $pipe = new MiddlewarePipe();
-
-                $exclude = resolve('flarum.api_client.exclude_middleware');
-
-                $middlewareStack = array_filter(resolve('flarum.api.middleware'), function ($middlewareClass) use ($exclude) {
-                    return !in_array($middlewareClass, $exclude);
-                });
-
-                foreach ($middlewareStack as $middleware) {
-                    $pipe->pipe(resolve($middleware));
     }
 
-                $pipe->pipe(new ExecuteRoute());
+    public function boot(): void
-
-                return new Api\Client($pipe);
-            }
-        );
-
-        $this->container->tag([
-            Searchers\DiscussionSearcher::class,
-            Searchers\CommentPostSearcher::class,
-        ], 'blomstra.search.searchers');
-    }
-
-    public function boot()
     {
-        /** @var array|string[] $seeders */
+        //
-        $seeders = $this->container->tagged('blomstra.search.seeders');
-
-        /** @var Dispatcher $events */
-        $events = resolve(Dispatcher::class);
-
-        /** @var Queue $queue */
-        $queue = resolve(Queue::class);
-
-        /** @var string|Seeders\Seeder $seeder */
-        foreach ($seeders as $seeder) {
-            $seeder::savingOn($events, function ($model) use ($queue, $seeder) {
-                $queue->pushOn(Job::$onQueue, new SavingJob(Collection::make([$model]), $seeder));
-            });
-
-            $seeder::deletingOn($events, function ($model) use ($queue, $seeder) {
-                $queue->pushOn(Job::$onQueue, new DeletingJob(Collection::make([$model]), $seeder));
-            });
-        }
     }
 }

src/Save/Document.php

@@ -16,7 +16,6 @@ use Carbon\Carbon;
 use Illuminate\Support\Fluent;
 
 /**
- * @property string      $type
  * @property string      $id
  * @property string      $content
  * @property Carbon      $created_at

src/Search/Concerns/AppliesAccessControl.php

@@ -10,30 +10,29 @@
  *
  */
 
-namespace Blomstra\Search\Seeders;
+namespace Blomstra\Search\Search\Concerns;
 
-use Blomstra\Search\Save\Document;
 use Flarum\Discussion\Discussion;
 use Flarum\Extension\ExtensionManager;
 use Flarum\Group\Group;
 use Flarum\Group\Permission;
 use Flarum\Tags\Tag;
-use Illuminate\Contracts\Events\Dispatcher;
+use Flarum\User\User;
-use Illuminate\Database\Eloquent\Builder;
-use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Database\Eloquent\Model;
 
-abstract class Seeder
+trait AppliesAccessControl
 {
-    abstract public function type(): string;
+    protected static function groupsForUser(User $actor): array
+    {
+        $groups = $actor->groups->pluck('id');
 
-    abstract public function query(): Builder;
+        $groups->add(Group::GUEST_ID);
 
-    abstract public static function savingOn(Dispatcher $events, callable $callable);
+        if ($actor->is_email_confirmed) {
+            $groups->add(Group::MEMBER_ID);
+        }
 
-    abstract public static function deletingOn(Dispatcher $events, callable $callable);
+        return $groups->toArray();
-
+    }
-    abstract public function toDocument(Model $model): Document;
 
     protected function groupsForDiscussion(Discussion $discussion): array
     {
@@ -43,8 +42,8 @@ abstract class Seeder
             ->where('permission', 'viewForum')
             ->pluck('group_id');
 
-        if ($this->extensionEnabled('flarum-tags')) {
+        if (resolve(ExtensionManager::class)->isEnabled('flarum-tags')) {
-            /** @var Collection $tags */
+            /** @var \Illuminate\Database\Eloquent\Collection $tags */
             $tags = $discussion->tags;
 
             $tagPermissions = Permission::query()
@@ -72,12 +71,4 @@ abstract class Seeder
 
         return $permissions->toArray();
     }
-
-    protected function extensionEnabled(string $extension): bool
-    {
-        /** @var ExtensionManager $manager */
-        $manager = resolve(ExtensionManager::class);
-
-        return $manager->isEnabled($extension);
-    }
 }

src/Search/ElasticIndex.php

@@ -0,0 +1,144 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Search;
+
+use Blomstra\Search\Exceptions\IndexingException;
+use Closure;
+use Elasticsearch\Client;
+use Flarum\Settings\SettingsRepositoryInterface;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Arr;
+
+class ElasticIndex
+{
+    public function __construct(
+        protected Client $client,
+        protected SettingsRepositoryInterface $settings
+    ) {
+    }
+
+    public function save(string $index, array $models, Closure $toDocument): void
+    {
+        $models = collect($models);
+
+        // Preparing body for storing.
+        $body = $models->map(function (Model $model) use ($index, $toDocument) {
+            $document = $toDocument($model);
+
+            return [
+                [
+                    'index' => [
+                        '_index' => $index,
+                        '_id'    => $document->id,
+                    ],
+                ],
+                $document->toArray(),
+            ];
+        })
+            ->flatten(1);
+
+        $this->handleResponse(
+            $this->client->bulk([
+                'index'   => $index,
+                'body'    => $body->toArray(),
+                'refresh' => true,
+            ])
+        );
+    }
+
+    public function delete(string $index, array $models): void
+    {
+        $models = collect($models);
+
+        $body = $models->map(function (Model $model) use ($index) {
+            return [
+                [
+                    'delete' => [
+                        '_index' => $index,
+                        '_id'    => $model->id,
+                    ],
+                ],
+            ];
+        })->flatten(1);
+
+        $this->handleResponse(
+            $this->client->bulk([
+                'index'   => $index,
+                'body'    => $body->toArray(),
+                'refresh' => true,
+            ])
+        );
+    }
+
+    public function build(string $index, array $properties): void
+    {
+        $this->client->indices()->create([
+            'index' => $index,
+            'body'  => [
+                'settings' => [
+                    'index.max_ngram_diff' => 10,
+                    'analysis'             => [
+                        'analyzer' => [
+                            'flarum_analyzer' => [
+                                'type' => $this->settings->get('blomstra-search.analyzer-language') ?: 'english',
+                            ],
+                            'flarum_analyzer_partial' => [
+                                'type'      => 'custom',
+                                'tokenizer' => 'standard',
+                                'filter'    => [
+                                    'lowercase',
+                                    'partial_search_filter',
+                                ],
+                            ],
+                        ],
+                        'filter' => [
+                            'partial_search_filter' => [
+                                'type'        => 'ngram',
+                                'min_gram'    => 1,
+                                'max_gram'    => 10,
+                                'token_chars' => ['letter', 'digit', 'symbol'],
+                            ],
+                        ],
+                    ],
+                ],
+                'mappings' => [
+                    'properties' => $properties,
+                ],
+            ],
+        ]);
+    }
+
+    public function flush(string $index): void
+    {
+        $this->client->indices()->delete([
+            'index'              => $index,
+            'ignore_unavailable' => true,
+        ]);
+    }
+
+    public function handleResponse(array $response): void
+    {
+        if (Arr::get($response, 'errors') !== true) {
+            return;
+        }
+
+        $items = Arr::get($response, 'items');
+
+        $error = Arr::get(Arr::first($items), 'index.error.reason');
+
+        throw new IndexingException(
+            "Failed to seed: $error",
+            $items
+        );
+    }
+}

src/Search/ElasticSearchDriver.php

@@ -0,0 +1,23 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Search;
+
+use Flarum\Search\AbstractDriver;
+
+class ElasticSearchDriver extends AbstractDriver
+{
+    public static function name(): string
+    {
+        return 'blomstra-elasticsearch';
+    }
+}

src/Search/ElasticSearchState.php

@@ -0,0 +1,43 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Search;
+
+use Blomstra\Search\Elasticsearch\Builder;
+use Closure;
+use Flarum\Search\SearchState;
+
+class ElasticSearchState extends SearchState
+{
+    protected Builder $builder;
+    protected ?Closure $retrieveDatabaseRecordsUsing = null;
+
+    public function setBuilder(Builder $builder): void
+    {
+        $this->builder = $builder;
+    }
+
+    public function getBuilder(): Builder
+    {
+        return $this->builder;
+    }
+
+    public function retrieveDatabaseRecordsUsing(Closure $retrieveDatabaseRecordsUsing): void
+    {
+        $this->retrieveDatabaseRecordsUsing = $retrieveDatabaseRecordsUsing;
+    }
+
+    public function getRetrieveDatabaseRecordsUsing(): ?Closure
+    {
+        return $this->retrieveDatabaseRecordsUsing;
+    }
+}

src/Search/Searcher.php

@@ -0,0 +1,119 @@
+<?php
+
+/*
+ * This file is part of blomstra/search.
+ *
+ * Copyright (c) 2022 Blomstra Ltd.
+ *
+ * For the full copyright and license information, please view the LICENSE.md
+ * file that was distributed with this source code.
+ *
+ */
+
+namespace Blomstra\Search\Search;
+
+use Blomstra\Search\Elasticsearch\Builder;
+use Elasticsearch\Client;
+use Flarum\Search\Filter\FilterManager;
+use Flarum\Search\SearchCriteria;
+use Flarum\Search\SearcherInterface;
+use Flarum\Search\SearchResults;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Str;
+use Spatie\ElasticsearchQueryBuilder\Sorts\Sort;
+
+abstract class Searcher implements SearcherInterface
+{
+    public function __construct(
+        protected FilterManager $filters,
+        /** @var array<callable> */
+        protected array $mutators,
+        protected Client $elastic
+    ) {
+    }
+
+    abstract public function index(): string;
+
+    public function search(SearchCriteria $criteria): SearchResults
+    {
+        $builder = (new Builder($this->elastic))
+            ->index($this->index())
+            ->size($criteria->limit + 1)
+            ->from($criteria->offset);
+
+        $state = new ElasticSearchState($criteria->actor, $criteria->isFulltext());
+        $state->setBuilder($builder);
+
+        // Default logic for retrieving database records.
+        // This is normally overriden by the fulltext filter.
+        $state->retrieveDatabaseRecordsUsing(function (array $response, SearchCriteria $criteria): Collection {
+            $results = (new Collection(Arr::get($response, 'hits.hits')))->map(function ($hit) {
+                $type = $hit['_source']['type'];
+                $id = Str::after($hit['_source']['id'], "$type:");
+
+                return [
+                    'id'     => $id,
+                    'score'  => Arr::get($hit, '_score'),
+                    'weight' => Arr::get($hit, 'sort.0'),
+                ];
+            })->sortByDesc('weight');
+
+            $ids = $results
+                ->take($criteria->limit)
+                ->pluck('id')
+                ->all();
+
+            return $this->getQuery($criteria->actor)
+                ->whereIn('id', $ids)
+                ->orderByRaw('FIELD(id, '.implode(',', $ids).')')
+                ->get();
+        });
+
+        $this->filters->apply($state, $criteria->filters);
+
+        $this->applySort($state, $criteria);
+
+        foreach ($this->mutators as $mutator) {
+            $mutator($state, $criteria);
+        }
+
+//        echo json_encode($builder->getParams(), JSON_PRETTY_PRINT);
+//        exit;
+
+        $response = $builder->search();
+
+//        header('Content-Type: application/json');
+//        echo json_encode($response, JSON_PRETTY_PRINT);
+//        exit;
+
+        $areMoreResults = count($response['hits']['hits']) > $criteria->limit;
+
+        $callback = $state->getRetrieveDatabaseRecordsUsing();
+
+        if (!$callback) {
+            throw new \RuntimeException('No callback set to retrieve database records');
+        }
+
+        $records = $callback($response, $criteria);
+
+        return new SearchResults($records, $areMoreResults);
+    }
+
+    protected function applySort(ElasticSearchState $state, SearchCriteria $criteria): void
+    {
+        $sort = $criteria->sort;
+
+        if ($criteria->sortIsDefault && !empty($state->getDefaultSort())) {
+            $sort = $state->getDefaultSort();
+        }
+
+        if (is_callable($sort)) {
+            $sort($state->getBuilder());
+        } else {
+            foreach (($criteria->sort ?? []) as $field => $direction) {
+                $state->getBuilder()->addSort(new Sort(Str::snake($field), $direction));
+            }
+        }
+    }
+}

src/Searchers/CommentPostSearcher.php

@@ -1,27 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Searchers;
-
-use Blomstra\Search\Seeders\CommentSeeder;
-
-class CommentPostSearcher extends Searcher
-{
-    protected string|null $seeder = CommentSeeder::class;
-
-    public function enabled(): bool
-    {
-        $enabled = $this->setting('blomstra-search.search-post-bodies', true);
-
-        return boolval($enabled);
-    }
-}

src/Searchers/DiscussionSearcher.php

@@ -1,32 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Searchers;
-
-use Blomstra\Search\Seeders\DiscussionSeeder;
-
-class DiscussionSearcher extends Searcher
-{
-    protected string|null $seeder = DiscussionSeeder::class;
-
-    public function enabled(): bool
-    {
-        $enabled = $this->setting('blomstra-search.search-discussion-subjects', true);
-
-        return boolval($enabled);
-    }
-
-    public function boost(): float
-    {
-        return 1.5;
-    }
-}

src/Searchers/Searcher.php

@@ -1,48 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Searchers;
-
-use Blomstra\Search\Seeders\Seeder;
-use Flarum\Settings\SettingsRepositoryInterface;
-
-abstract class Searcher
-{
-    protected string|null $seeder = null;
-
-    public function type(): string
-    {
-        /** @var Seeder $seeder */
-        $seeder = $this->seeder;
-
-        if (empty($seeder)) {
-            throw new \InvalidArgumentException('Implement type or add $seeder');
-        }
-
-        return (new $seeder())->type();
-    }
-
-    public function enabled(): bool
-    {
-        return true;
-    }
-
-    public function boost(): float
-    {
-        return 1;
-    }
-
-    protected function setting(string $key, $default = null)
-    {
-        return resolve(SettingsRepositoryInterface::class)->get($key, $default);
-    }
-}

src/Seeders/CommentSeeder.php

@@ -1,96 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Seeders;
-
-use Blomstra\Search\Save\Document;
-use Flarum\Api\Serializer\PostSerializer;
-use Flarum\Post\CommentPost;
-use Flarum\Post\Event\Deleted;
-use Flarum\Post\Event\Posted;
-use Illuminate\Contracts\Events\Dispatcher;
-use Illuminate\Database\Eloquent\Builder;
-use Illuminate\Database\Eloquent\Model;
-
-class CommentSeeder extends Seeder
-{
-    public function type(): string
-    {
-        return resolve(PostSerializer::class)->getType(new CommentPost());
-    }
-
-    public function query(): Builder
-    {
-        $includes = ['discussion'];
-
-        if ($this->extensionEnabled('flarum-tags')) {
-            $includes[] = 'discussion.tags';
-        }
-
-        if ($this->extensionEnabled('fof-byobu')) {
-            $includes[] = 'discussion.recipientUsers';
-            $includes[] = 'discussion.recipientGroups';
-        }
-
-        return CommentPost::query()
-            ->whereNull('hidden_at')
-            ->where('type', CommentPost::$type)
-            ->with($includes);
-    }
-
-    public static function savingOn(Dispatcher $events, callable $callable)
-    {
-        $events->listen(Posted::class, function (Posted $event) use ($callable) {
-            $callable($event->post);
-        });
-    }
-
-    public static function deletingOn(Dispatcher $events, callable $callable)
-    {
-        $events->listen(Deleted::class, function (Deleted $event) use ($callable) {
-            $callable($event->post);
-        });
-    }
-
-    /**
-     * @param CommentPost $model
-     *
-     * @return Document
-     */
-    public function toDocument(Model $model): Document
-    {
-        $document = new Document([
-            'type'            => $this->type(),
-            'id'              => $this->type().':'.$model->id,
-            'rawId'           => $model->id,
-            'content'         => $model->content,
-            'content_partial' => $model->content,
-            'created_at'      => $model->created_at?->toAtomString(),
-            'updated_at'      => $model->edited_at?->toAtomString(),
-            'is_private'      => $model->is_private,
-            'user_id'         => $model->user_id,
-            'groups'          => $this->groupsForDiscussion($model->discussion),
-            'comment_count'   => $model->discussion->comment_count,
-        ]);
-
-        if ($this->extensionEnabled('flarum-tags')) {
-            $document['tags'] = $model->discussion->tags->pluck('id')->toArray();
-        }
-
-        if ($this->extensionEnabled('fof-byobu')) {
-            $document['recipient_users'] = $model->discussion->recipientUsers->pluck('id')->toArray();
-            $document['recipient_groups'] = $model->discussion->recipientGroups->pluck('id')->toArray();
-        }
-
-        return $document;
-    }
-}

src/Seeders/DiscussionSeeder.php

@@ -1,101 +0,0 @@
-<?php
-
-/*
- * This file is part of blomstra/search.
- *
- * Copyright (c) 2022 Blomstra Ltd.
- *
- * For the full copyright and license information, please view the LICENSE.md
- * file that was distributed with this source code.
- *
- */
-
-namespace Blomstra\Search\Seeders;
-
-use Blomstra\Search\Save\Document;
-use Flarum\Api\Serializer\DiscussionSerializer;
-use Flarum\Discussion\Discussion;
-use Flarum\Discussion\Event\Deleted;
-use Flarum\Discussion\Event\Hidden;
-use Flarum\Discussion\Event\Restored;
-use Flarum\Discussion\Event\Started;
-use Illuminate\Contracts\Events\Dispatcher;
-use Illuminate\Database\Eloquent\Builder;
-use Illuminate\Database\Eloquent\Model;
-
-class DiscussionSeeder extends Seeder
-{
-    public function type(): string
-    {
-        return resolve(DiscussionSerializer::class)->getType(new Discussion());
-    }
-
-    public function query(): Builder
-    {
-        $includes = [];
-
-        if ($this->extensionEnabled('flarum-tags')) {
-            $includes[] = 'tags';
-        }
-
-        if ($this->extensionEnabled('fof-byobu')) {
-            $includes[] = 'recipientUsers';
-            $includes[] = 'recipientGroups';
-        }
-
-        return Discussion::query()
-            ->whereNull('hidden_at')
-            ->with($includes);
-    }
-
-    public static function savingOn(Dispatcher $events, callable $callable)
-    {
-        $events->listen([Started::class, Restored::class], function ($event) use ($callable) {
-            return $callable($event->discussion);
-        });
-    }
-
-    public static function deletingOn(Dispatcher $events, callable $callable)
-    {
-        $events->listen([Deleted::class, Hidden::class], function ($event) use ($callable) {
-            return $callable($event->discussion);
-        });
-    }
-
-    /**
-     * @param Discussion $model
-     *
-     * @return Document
-     */
-    public function toDocument(Model $model): Document
-    {
-        $document = new Document([
-            'type'            => $this->type(),
-            'id'              => $this->type().':'.$model->id,
-            'rawId'           => $model->id,
-            'content'         => $model->title,
-            'content_partial' => $model->title,
-            'created_at'      => $model->created_at?->toAtomString(),
-            'updated_at'      => $model->last_posted_at?->toAtomString(),
-            'is_private'      => $model->is_private,
-            'user_id'         => $model->user_id,
-            'groups'          => $this->groupsForDiscussion($model),
-            'comment_count'   => $model->comment_count,
-        ]);
-
-        if ($this->extensionEnabled('flarum-tags')) {
-            $document['tags'] = $model->tags->pluck('id')->toArray();
-        }
-
-        if ($this->extensionEnabled('fof-byobu')) {
-            $document['recipient_users'] = $model->recipientUsers->pluck('id')->toArray();
-            $document['recipient_groups'] = $model->recipientGroups->pluck('id')->toArray();
-        }
-
-        if ($this->extensionEnabled('flarum-sticky')) {
-            $document['is_sticky'] = (bool) $model->is_sticky;
-        }
-
-        return $document;
-    }
-}