sbp-jm
/
flarum-ext-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip

This commit is contained in:
Daniel Klabbers 2021-10-15 13:51:36 +02:00
parent cfa3bae1f7
commit 4130e49bee
10 changed files with 135 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extend.php
View File

@ -11,7 +11,7 @@ return [
->js(__DIR__ . '/js/dist/forum.js'),
(new Flarum\Routes('api'))
->get('/blomstra/search/{index}', 'blomstra.search', Api\Controllers\SearchController::class),
->get('/blomstra/search/{type}', 'blomstra.search', Api\Controllers\SearchController::class),
(new Flarum\Console)
->command(Commands\RebuildDocumentsCommand::class)

2
js/src/forum/SearchSources/DiscussionsSearchSource.tsx
View File

@ -18,7 +18,7 @@ export default class DiscussionsSearchSource implements SearchSource {
/**
* Model name used for assembling API endpoint URL, and for frontend DOM.
*/
private type = 'posts';
private type = 'discussion';
async search(query: string): Promise<void> {
query = query.toLowerCase();

39
src/Api/Controllers/SearchController.php
View File

@ -6,9 +6,13 @@ use Blomstra\Search\Elasticsearch\TermsQuery;
use Blomstra\Search\Schemas\Schema;
use Elasticsearch\Client;
use Flarum\Api\Controller\AbstractListController;
use Flarum\Api\Controller\ListDiscussionsController;
use Flarum\Discussion\Filter\DiscussionFilterer;
use Flarum\Discussion\Search\DiscussionSearcher;
use Flarum\Extension\ExtensionManager;
use Flarum\Group\Group;
use Flarum\Http\RequestUtil;
use Flarum\Http\UrlGenerator;
use Flarum\User\User;
use Illuminate\Contracts\Container\Container;
use Illuminate\Support\Arr;
@ -20,33 +24,39 @@ use Spatie\ElasticsearchQueryBuilder\Builder;
use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
use Spatie\ElasticsearchQueryBuilder\Queries\MultiMatchQuery;
use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;
use Spatie\ElasticsearchQueryBuilder\Sorts\Sort;
use Tobscure\JsonApi\Document;
class SearchController extends AbstractListController
class SearchController extends ListDiscussionsController
{
public function __construct()
{
}
protected function data(ServerRequestInterface $request, Document $document)
{
/** @var Client $client */
$client = resolve('blomstra.search.elastic');
$index = Arr::get($request->getQueryParams(), 'index');
$type = Arr::get($request->getQueryParams(), 'type');
$actor = RequestUtil::getActor($request);
$filters = $this->extractFilter($request);
$schema = $this->getSchema($index);
$schema = $this->getSchema($type);
$this->serializer = $schema::serializer();
$filterQuery = (BoolQuery::create())
->add(
MultiMatchQuery::create($filters['q'], array_keys($schema->fulltext(new ($schema::model())))),
'must'
BoolQuery::create()
->add(MultiMatchQuery::create($filters['q'], array_keys($schema->fulltext(new ($schema::model())))))
->add(TermQuery::create('type', $type))
);
$result = (new Builder($client))
->index($index)
$builder = (new Builder($client))
->index(resolve('blomstra.search.elastic_index'))
->size($this->extractLimit($request))
->from($this->extractOffset($request))
->addQuery(
@ -55,18 +65,23 @@ class SearchController extends AbstractListController
->addAggregation(
TermsAggregation::create('discussions', 'discussion_id')
->aggregation(TopHitsAggregation::create('hits', 1))
)
->search();
);
foreach ($this->extractSort($request) as $field => $direction) {
$builder->addSort(new Sort($field, $direction));
}
$result = $builder->search();
return $schema::results(Arr::get($result, 'hits.hits'));
}
protected function getSchema(string $index): ?Schema
protected function getSchema(string $type): ?Schema
{
$mapping = resolve(Container::class)->tagged('blomstra.search.schemas');
return collect($mapping)->first(function (Schema $schema) use ($index) {
return $schema::index() === $index;
return collect($mapping)->first(function (Schema $schema) use ($type) {
return $schema::type() === $type;
});
}

12
src/Commands/RebuildDocumentsCommand.php
View File

@ -13,7 +13,7 @@ use Illuminate\Database\Eloquent\Model;
class RebuildDocumentsCommand extends Command
{
protected $signature = 'blomstra:search:documents:rebuild {--flush : Flush the indices}';
protected $signature = 'blomstra:search:documents:rebuild {--flush : Flushes ALL the documents inside the index}';
protected $description = 'Rebuilds the complete search server with its documents.';
public function handle(Container $container)
@ -27,16 +27,16 @@ class RebuildDocumentsCommand extends Command
/** @var Client $client */
$client = $container->make('blomstra.search.elastic');
// Flush the index.
if ($this->option('flush')) $client->indices()->delete([
'index' => resolve('blomstra.search.elastic_index')
]);
/** @var Schema $schema */
foreach ($schemas as $schema) {
/** @var Model $model */
$model = $schema::model();
// Flush the index.
if ($this->option('flush')) $client->indices()->delete([
'index' => $schema::index()
]);
$total = 0;
$schema::query()->chunk(50, function (Collection $collection) use ($model, $queue, &$total) {

12
src/Elasticsearch/TermsQuery.php
View File

@ -8,24 +8,24 @@ class TermsQuery implements Query
{
protected string $field;
protected array $value;
protected array $values;
public static function create(string $field, array $value): static
public static function create(string $field, array $values): static
{
return new self($field, $value);
return new self($field, $values);
}
public function __construct(string $field, array $value)
public function __construct(string $field, array $values)
{
$this->field = $field;
$this->value = $value;
$this->values = $values;
}
public function toArray(): array
{
return [
'terms' => [
$this->field => $this->value,
$this->field => $this->values,
],
];
}

45
src/Observe/SavingJob.php
View File

@ -29,33 +29,36 @@ class SavingJob extends Job
}
// Set up the index
// if (! $client->indices()->exists([
// 'index' => $schema::index(),
// 'expand_wildcards' => 'none'
// ])) {
// $client->indices()->create([
// 'index' => $schema::index(),
// 'body' => [
// 'mappings' => [
// 'properties' => $properties
// ],
// 'settings' => [
// 'index' => [
// 'query' => [
// 'default_field' => array_keys($schema->fulltext($first))
// ]
// ]
// ]
// ]
// ]);
// }
if (! $client->indices()->exists([
'index' => resolve('blomstra.search.elastic_index'),
'expand_wildcards' => 'none'
])) {
$client->indices()->create([
'index' => $schema::index(),
'body' => [
'mappings' => [
'properties' => $properties
],
'settings' => [
'index' => [
'query' => [
'default_field' => array_keys($schema->fulltext($first))
]
]
]
]
]);
}
}
// Preparing body for storing.
$body = $this->models->map(function (Model $model) use ($schema) {
return [
['index' => ['_id' => $model->getKey(), '_index' => $schema::index()]],
['index' => [
'_id' => $model->getKey(), '_index' => resolve('blomstra.search.elastic_index')]
],
array_merge(
['type' => $schema::type()],
$schema->fulltext($model),
$schema->filters($model))

10
src/Provider.php
View File

@ -20,14 +20,12 @@ class Provider extends AbstractServiceProvider
{
public function register()
{
// $this->container->tag([DiscussionSchema::class], 'blomstra.search.schemas');
$this->container->tag([CommentPostSchema::class], 'blomstra.search.schemas');
$this->container->singleton('blomstra.search.elastic', function (Container $container) {
$config = $container->make('flarum.config') ?? [];
$elastic = Arr::get($config, 'elastic');
$config = $this->container->make('flarum.config') ?? [];
$elastic = Arr::get($config, 'elastic', []);
$this->container->singleton('blomstra.search.elastic', function (Container $container) use ($elastic) {
$builder = ClientBuilder::create()
->setHosts([$elastic['endpoint']])
->setLogger($container->make(LoggerInterface::class));
@ -41,6 +39,8 @@ class Provider extends AbstractServiceProvider
return $builder->build();
});
$this->container->instance('blomstra.search.elastic_index', Arr::get($elastic, 'index', 'flarum'));
}
public function boot()

29
src/Schemas/CommentPostSchema.php
View File

@ -3,6 +3,7 @@
namespace Blomstra\Search\Schemas;
use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Api\Serializer\PostSerializer;
use Flarum\Discussion\Discussion;
use Flarum\Post\CommentPost;
use Flarum\Post\Event\Deleted;
@ -16,22 +17,19 @@ class CommentPostSchema extends Schema
public function filters(CommentPost $post): array
{
$filters = [
'type' => CommentPost::$type,
'author' => $post->user_id,
'created_at' => $post->created_at->toAtomString(),
'createdAt' => $post->created_at->toAtomString(),
'private' => $post->is_private,
'groups' => $this->groupsForDiscussion($post->discussion),
'discussion_id' => $post->discussion?->id
];
if ($this->extensionEnabled('fof-byobu')) {
$filters['recipient-users'] = $post->discussion->recipientUsers->pluck('id')->toArray();
$filters['recipient-groups'] = $post->discussion->recipientGroups->pluck('id')->toArray();
if ($this->extensionEnabled('flarum-flags')) {
$filters['flags_count'] = $post->flags->count();
}
if ($this->extensionEnabled('fof-best-answer')) {
$filters['best-answer-set'] = $post->discussion->best_answer_post_id !== null;
$filters['best-answer-set-at'] = $post->discussion->best_answer_set_at?->toAtomString();
if ($this->extensionEnabled('flarum-approval')) {
$filters['approved'] = $post->is_approved;
}
return $filters;
@ -40,16 +38,10 @@ class CommentPostSchema extends Schema
public function fulltext(CommentPost $post): array
{
return [
'title' => $post->discussion?->title,
'content' => $post->exists ? $post->content : null,
'content' => $post->content,
];
}
public static function index(): string
{
return 'posts';
}
public static function model(): string
{
return CommentPost::class;
@ -92,4 +84,9 @@ class CommentPostSchema extends Schema
$callable($event->post);
});
}
public static function type(): string
{
return CommentPost::$type;
}
}

101
src/Schemas/DiscussionSchema.php
View File

@ -6,77 +6,56 @@ use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Discussion\Discussion;
use Flarum\Discussion\Event\Deleted;
use Flarum\Discussion\Event\Hidden;
use Flarum\Discussion\Event\Restored;
use Flarum\Discussion\Event\Started;
use Flarum\Group\Group;
use Flarum\Group\Permission;
use Flarum\Tags\Tag;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Collection;
class DiscussionSchema extends Schema
{
public function filters(Discussion $discussion): array
{
$filters = [];
$permissions = collect();
$globalPermission = Permission::query()
->where('permission', 'viewForum')
->pluck('group_id');
if ($this->extensionEnabled('flarum-tags')) {
/** @var \Illuminate\Database\Eloquent\Collection $tags */
$tags = $discussion->tags;
$filters['tags'] = $tags->pluck('id')->toArray();
$tagPermissions = Permission::query()
->whereIn(
'permission',
$tags->pluck('id')->map(function (int $id) {
return "tag$id.viewForum";
})
)->get();
$permissions = $tags->map(function (Tag $tag) use ($tagPermissions) {
$permissions = $tagPermissions->where('permission', "tag$tag->id.viewForum");
if ($tag->is_restricted) {
$permissions = $permissions->add(['group_id' => Group::ADMINISTRATOR_ID]);
}
return $permissions->pluck('group_id');
})->flatten();
}
if (! $discussion->is_private && $permissions->isEmpty()) {
$permissions = $globalPermission;
}
$filters['groups'] = $permissions->toArray();
$filters['private'] = $discussion->is_private;
$filters = [
'type' => 'discussions',
'author' => $discussion->user_id,
'createdAt' => $discussion->created_at->toAtomString(),
'lastPostedAt' => $discussion->last_posted_at->toAtomString(),
'private' => $discussion->is_private,
'first_post_id' => $discussion->first_post_id,
'last_post_id' => $discussion->last_post_id,
'commentCount' => $discussion?->comment_count,
'groups' => $this->groupsForDiscussion($discussion),
];
if ($this->extensionEnabled('fof-byobu')) {
$filters['recipient-users'] = $discussion->recipientUsers->pluck('id')->toArray();
$filters['recipient-groups'] = $discussion->recipientGroups->pluck('id')->toArray();
}
if ($this->extensionEnabled('flarum-sticky')) {
$filters['is_sticky'] = $discussion->is_sticky;
}
return $filters;
}
public static function relations()
{
return [
'discussion_id' => [
'type' => 'discussion'
]
];
}
public function fulltext(Discussion $discussion): array
{
return [
'title' => $discussion->title,
'content' => $discussion->firstPost?->content
'title' => $discussion->title
];
}
public static function index(): string
{
return 'discussions';
}
public static function model(): string
{
return Discussion::class;
@ -89,7 +68,7 @@ class DiscussionSchema extends Schema
public static function savingOn(Dispatcher $events, callable $callable)
{
$events->listen(Started::class, function (Started $event) use ($callable) {
$events->listen([Started::class, Restored::class], function ($event) use ($callable) {
return $callable($event->discussion);
});
}
@ -100,4 +79,26 @@ class DiscussionSchema extends Schema
return $callable($event->discussion);
});
}
public static function query(): Builder
{
return Discussion::query();
}
public static function results(array $hits): Collection
{
$postIds = \Illuminate\Support\Collection::make($hits)->keyBy('_source.discussion_id')->pluck('_id');
$discussionIds = Collection::make($hits)->pluck('_source.discussion_id');
return Discussion::query()->findMany($discussionIds)->map(function (Discussion $discussion) use ($postIds) {
$discussion->most_relevant_post_id = $postIds->get($discussion->id);
return $discussion;
})->load('mostRelevantPost');
}
public static function type(): string
{
return 'discussion';
}
}

2
src/Schemas/Schema.php
View File

@ -21,7 +21,7 @@ abstract class Schema
return $manager->isEnabled($extension);
}
abstract public static function index(): string;
abstract public static function type(): string;
abstract public static function model(): string;
abstract public static function query(): Builder;