sbp-jm
/
flarum-ext-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip

This commit is contained in:
Daniel Klabbers 2021-10-15 21:21:30 +02:00
parent 4130e49bee
commit 4fa4ccce99
13 changed files with 406 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/Commands/RebuildDocumentsCommand.php
View File

@ -3,13 +3,12 @@
namespace Blomstra\Search\Commands;
use Blomstra\Search\Observe\SavingJob;
use Blomstra\Search\Schemas\Schema;
use Blomstra\Search\Seeders\Seeder;
use Elasticsearch\Client;
use Illuminate\Console\Command;
use Illuminate\Contracts\Container\Container;
use Illuminate\Contracts\Queue\Queue;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
class RebuildDocumentsCommand extends Command
{
@ -18,8 +17,8 @@ class RebuildDocumentsCommand extends Command
public function handle(Container $container)
{
/** @var array $schemas */
$schemas = $container->tagged('blomstra.search.schemas');
/** @var array $seeders */
$seeders = $container->tagged('blomstra.search.seeders');
/** @var Queue $queue */
$queue = $container->make(Queue::class);
@ -29,19 +28,14 @@ class RebuildDocumentsCommand extends Command
// Flush the index.
if ($this->option('flush')) $client->indices()->delete([
'index' => resolve('blomstra.search.elastic_index')
'index' => $container->make('blomstra.search.elastic_index')
]);
/** @var Schema $schema */
foreach ($schemas as $schema) {
/** @var Model $model */
$model = $schema::model();
/** @var Seeder $seeder */
foreach ($seeders as $seeder) {
$seeder->query()->chunk(50, function (Collection $collection) use ($queue, &$total) {
$total = 0;
$schema::query()->chunk(50, function (Collection $collection) use ($model, $queue, &$total) {
$queue->push(new SavingJob($model, $collection));
$queue->push(new SavingJob($collection));
$this->info("Pushed {$collection->count()} into the index.");

49
src/Documents/CommentDocument.php Normal file
View File

@ -0,0 +1,49 @@
<?php
namespace Blomstra\Search\Documents;
use Flarum\Api\Serializer\PostSerializer;
use Flarum\Post\CommentPost;
class CommentDocument extends Document
{
public function __construct(protected CommentPost $model)
{}
public function fulltext(): array
{
return [
'content' => $this->model->content,
];
}
public function attributes(): array
{
$attributes = [
'author' => $this->model->user_id,
'createdAt' => $this->model->created_at->toAtomString(),
'is_private' => $this->model->is_private,
'discussion_id' => $this->model->discussion?->id
];
if ($this->extensionEnabled('flarum-flags')) {
$attributes['flags_count'] = $this->model->flags->count();
}
if ($this->extensionEnabled('flarum-approval')) {
$attributes['approved'] = $this->model->is_approved;
}
return $attributes;
}
public function serializer(): string
{
return PostSerializer::class;
}
public function model(): string
{
return CommentPost::class;
}
}

54
src/Documents/DiscussionDocument.php Normal file
View File

@ -0,0 +1,54 @@
<?php
namespace Blomstra\Search\Documents;
use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Discussion\Discussion;
class DiscussionDocument extends Document
{
public function __construct(protected Discussion $model)
{}
public function fulltext(): array
{
return [
'content' => $this->model->title
];
}
public function attributes(): array
{
$attributes = [
'author' => $this->model->user_id,
'createdAt' => $this->model->created_at->toAtomString(),
'lastPostedAt' => $this->model->last_posted_at?->toAtomString(),
'is_private' => $this->model->is_private,
'first_post_id' => $this->model->first_post_id,
'last_post_id' => $this->model->last_post_id,
'commentCount' => $this->model?->comment_count,
'groups' => $this->groupsForDiscussion($this->model),
];
if ($this->extensionEnabled('fof-byobu')) {
$attributes['recipient-users'] = $this->model->recipientUsers->pluck('id')->toArray();
$attributes['recipient-groups'] = $this->model->recipientGroups->pluck('id')->toArray();
}
if ($this->extensionEnabled('flarum-sticky')) {
$attributes['is_sticky'] = $this->model->is_sticky;
}
return $attributes;
}
public function serializer(): string
{
return DiscussionSerializer::class;
}
public function model(): string
{
return Discussion::class;
}
}

82
src/Documents/Document.php Normal file
View File

@ -0,0 +1,82 @@
<?php
namespace Blomstra\Search\Documents;
use Flarum\Discussion\Discussion;
use Flarum\Extension\ExtensionManager;
use Flarum\Group\Group;
use Flarum\Group\Permission;
use Flarum\Tags\Tag;
use Illuminate\Database\Eloquent\Collection;
abstract class Document
{
abstract public function fulltext(): array;
abstract public function attributes(): array;
public function id(): string
{
return $this->type() . ':' . $this->model->getKey();
}
public function index(): string
{
return resolve('blomstra.search.elastic_index');
}
public function type(): string
{
return resolve($this->serializer())->getType($this->model);
}
abstract public function serializer(): string;
abstract public function model(): string;
protected function groupsForDiscussion(Discussion $discussion): array
{
$permissions = collect();
$globalPermission = Permission::query()
->where('permission', 'viewForum')
->pluck('group_id');
if ($this->extensionEnabled('flarum-tags')) {
/** @var Collection $tags */
$tags = $discussion->tags;
$filters['tags'] = $tags->pluck('id')->toArray();
$tagPermissions = Permission::query()
->whereIn(
'permission',
$tags->pluck('id')->map(function (int $id) {
return "tag$id.viewForum";
})
)->get();
$permissions = $tags->map(function (Tag $tag) use ($tagPermissions) {
$permissions = $tagPermissions->where('permission', "tag$tag->id.viewForum");
if ($tag->is_restricted) {
$permissions = $permissions->add(['group_id' => Group::ADMINISTRATOR_ID]);
}
return $permissions->pluck('group_id');
})->flatten();
}
if (! $discussion->is_private && $permissions->isEmpty()) {
$permissions = $globalPermission;
}
return $permissions->toArray();
}
protected function extensionEnabled(string $extension): bool
{
/** @var ExtensionManager $manager */
$manager = resolve(ExtensionManager::class);
return $manager->isEnabled($extension);
}
}

30
src/Manager.php Normal file
View File

@ -0,0 +1,30 @@
<?php
namespace Blomstra\Search;
use Blomstra\Search\Documents\Document;
use Illuminate\Contracts\Container\Container;
class Manager
{
public function __construct(protected Container $container)
{}
public function document(string $type): ?string
{
return $this->fromTagged('blomstra.search.documents', $type);
}
protected function fromTagged(string $binding, $search): ?string
{
$entities = $this->container->tagged($binding);
foreach ($entities as $entity) {
$instance = $this->container->make($entity);
if ($instance->type() === $search) return $entity;
}
return null;
}
}

24
src/Observe/DeletingJob.php
View File

@ -3,26 +3,26 @@
namespace Blomstra\Search\Observe;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
use MeiliSearch\Client;
class DeletingJob extends Job
{
public function __construct(protected string $class, protected Collection $models)
{}
public function handle(Client $meili)
{
$schema = $this->getSchema();
if ($this->models->isEmpty()) return;
if (! $schema) return;
$document = $this->getDocument();
$keys = $this->models->map(function (Model $model) {
return $model->getKey();
});
if (! $document) return;
$meili->index($schema::index())->deleteDocuments(
$keys
);
$keys = $this->models->map(function (Model $model) use ($document) {
return (new $document)($model)->id();
})->toArray();
$meili
->index(resolve('blomstra.search.elastic_index'))
->deleteDocuments(
$keys
);
}
}

20
src/Observe/Job.php
View File

@ -2,18 +2,26 @@
namespace Blomstra\Search\Observe;
use Blomstra\Search\Schemas\Schema;
use Blomstra\Search\Documents\Document;
use Flarum\Queue\AbstractJob;
use Illuminate\Contracts\Container\Container;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
abstract class Job extends AbstractJob
{
protected function getSchema(): ?Schema
{
$mapping = resolve(Container::class)->tagged('blomstra.search.schemas');
public function __construct(protected Collection $models)
{}
return collect($mapping)->first(function (Schema $schema) {
return $schema::model() === $this->class;
protected function getDocument(): ?Document
{
$documents = resolve(Container::class)->tagged('blomstra.search.documents');
/** @var Model $model */
$model = $this->models->first();
return collect($documents)->first(function (Document $document) use ($model) {
return $document->model() === get_class($model);
});
}
}

83
src/Observe/SavingJob.php
View File

@ -2,71 +2,76 @@
namespace Blomstra\Search\Observe;
use Blomstra\Search\Documents\Document;
use Elasticsearch\Client;
use Illuminate\Contracts\Container\Container;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
class SavingJob extends Job
{
public function __construct(protected string $class, protected Collection $models)
{}
public function handle(Container $container)
{
if ($this->models->isEmpty()) return;
/** @var Client $client */
$client = $container->make('blomstra.search.elastic');
$schema = $this->getSchema();
$document = $this->getDocument();
if (! $schema) return;
if (! $document) return;
if($first = $this->models->first()) {
$properties = [];
foreach (array_keys($schema->fulltext($first)) as $key) {
$properties[$key] = ['type' => 'text'];
}
// $properties = [];
//
// foreach (array_keys($schema->fulltext($first)) as $key) {
// $properties[$key] = ['type' => 'text'];
// }
// Set up the index
if (! $client->indices()->exists([
'index' => resolve('blomstra.search.elastic_index'),
'expand_wildcards' => 'none'
])) {
$client->indices()->create([
'index' => $schema::index(),
'body' => [
'mappings' => [
'properties' => $properties
],
'settings' => [
'index' => [
'query' => [
'default_field' => array_keys($schema->fulltext($first))
]
]
]
]
]);
}
// if (! $client->indices()->exists([
// 'index' => $schema::index(),
// 'expand_wildcards' => 'none'
// ])) {
// $client->indices()->create([
// 'index' => $schema::index(),
// 'body' => [
// 'mappings' => [
// 'properties' => $properties
// ],
// 'settings' => [
// 'index' => [
// 'query' => [
// 'default_field' => array_keys($schema->fulltext($first))
// ]
// ]
// ]
// ]
// ]);
// }
}
// Preparing body for storing.
$body = $this->models->map(function (Model $model) use ($schema) {
$body = $this->models->map(function (Model $model) use ($document) {
/** @var Document $markup */
$markup = new $document($model);
return [
['index' => [
'_id' => $model->getKey(), '_index' => resolve('blomstra.search.elastic_index')]
[
'index' => [
'_id' => $markup->id(),
'_index' => resolve('blomstra.search.elastic_index')
]
],
array_merge(
['type' => $schema::type()],
$schema->fulltext($model),
$schema->filters($model))
['type' => $markup->type()],
$markup->fulltext(),
$markup->attributes()
)
];
})->flatten(1);
$response = $client->bulk([
'index' => $schema::index(),
'index' => resolve('blomstra.search.elastic_index'),
'body' => $body->toArray(),
'refresh' => true
]);

30
src/Provider.php
View File

@ -2,11 +2,11 @@
namespace Blomstra\Search;
use Blomstra\Search\Documents;
use Blomstra\Search\Observe\DeletingJob;
use Blomstra\Search\Observe\SavingJob;
use Blomstra\Search\Schemas\CommentPostSchema;
use Blomstra\Search\Schemas\DiscussionSchema;
use Blomstra\Search\Schemas\Schema;
use Blomstra\Search\Seeders;
use Elasticsearch\ClientBuilder;
use Flarum\Foundation\AbstractServiceProvider;
use Illuminate\Contracts\Container\Container;
@ -20,7 +20,15 @@ class Provider extends AbstractServiceProvider
{
public function register()
{
$this->container->tag([CommentPostSchema::class], 'blomstra.search.schemas');
$this->container->tag([
Documents\CommentDocument::class,
Documents\DiscussionDocument::class
], 'blomstra.search.documents');
$this->container->tag([
Seeders\CommentSeeder::class,
Seeders\DiscussionSeeder::class
], 'blomstra.search.seeders');
$config = $this->container->make('flarum.config') ?? [];
$elastic = Arr::get($config, 'elastic', []);
@ -45,8 +53,8 @@ class Provider extends AbstractServiceProvider
public function boot()
{
/** @var array $schemas */
$schemas = $this->container->tagged('blomstra.search.schemas');
/** @var array|string[] $seeders */
$seeders = $this->container->tagged('blomstra.search.seeders');
/** @var Dispatcher $events */
$events = resolve(Dispatcher::class);
@ -54,14 +62,14 @@ class Provider extends AbstractServiceProvider
/** @var Queue $queue */
$queue = resolve(Queue::class);
/** @var Schema $schema */
foreach ($schemas as $schema) {
$schema::savingOn($events, function ($model) use ($schema, $queue) {
$queue->push(new SavingJob($schema::model(), Collection::make([$model])));
/** @var string|Seeders\Seeder $seeder */
foreach ($seeders as $seeder) {
$seeder::savingOn($events, function ($model) use ($queue) {
$queue->push(new SavingJob(Collection::make([$model])));
});
$schema::deletingOn($events, function ($model) use ($schema, $queue) {
$queue->push(new DeletingJob($schema::model(), Collection::make([$model])));
$seeder::deletingOn($events, function ($model) use ($queue) {
$queue->push(new DeletingJob(Collection::make([$model])));
});
}
}

1
src/Schemas/CommentPostSchema.php
View File

@ -3,7 +3,6 @@
namespace Blomstra\Search\Schemas;
use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Api\Serializer\PostSerializer;
use Flarum\Discussion\Discussion;
use Flarum\Post\CommentPost;
use Flarum\Post\Event\Deleted;

39
src/Seeders/CommentSeeder.php Normal file
View File

@ -0,0 +1,39 @@
<?php
namespace Blomstra\Search\Seeders;
use Flarum\Api\Serializer\PostSerializer;
use Flarum\Post\CommentPost;
use Flarum\Post\Event\Deleted;
use Flarum\Post\Event\Posted;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder;
class CommentSeeder extends Seeder
{
public function type(): string
{
return resolve(PostSerializer::class)->type;
}
public function query(): Builder
{
return CommentPost::query()
->where('type', CommentPost::$type)
->with('discussion');
}
public static function savingOn(Dispatcher $events, callable $callable)
{
$events->listen(Posted::class, function (Posted $event) use ($callable) {
$callable($event->post);
});
}
public static function deletingOn(Dispatcher $events, callable $callable)
{
$events->listen(Deleted::class, function (Deleted $event) use ($callable) {
$callable($event->post);
});
}
}

39
src/Seeders/DiscussionSeeder.php Normal file
View File

@ -0,0 +1,39 @@
<?php
namespace Blomstra\Search\Seeders;
use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Discussion\Discussion;
use Flarum\Discussion\Event\Deleted;
use Flarum\Discussion\Event\Hidden;
use Flarum\Discussion\Event\Restored;
use Flarum\Discussion\Event\Started;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder;
class DiscussionSeeder extends Seeder
{
public function type(): string
{
return resolve(DiscussionSerializer::class)->type;
}
public function query(): Builder
{
return Discussion::query();
}
public static function savingOn(Dispatcher $events, callable $callable)
{
$events->listen([Started::class, Restored::class], function ($event) use ($callable) {
return $callable($event->discussion);
});
}
public static function deletingOn(Dispatcher $events, callable $callable)
{
$events->listen([Deleted::class, Hidden::class], function ($event) use ($callable) {
return $callable($event->discussion);
});
}
}

16
src/Seeders/Seeder.php Normal file
View File

@ -0,0 +1,16 @@
<?php
namespace Blomstra\Search\Seeders;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder;
abstract class Seeder
{
abstract public function type(): string;
abstract public function query(): Builder;
abstract public static function savingOn(Dispatcher $events, callable $callable);
abstract public static function deletingOn(Dispatcher $events, callable $callable);
}