diff --git a/src/Api/Controllers/SearchController.php b/src/Api/Controllers/SearchController.php index a59997f..d9d00a1 100644 --- a/src/Api/Controllers/SearchController.php +++ b/src/Api/Controllers/SearchController.php @@ -48,8 +48,7 @@ class SearchController extends ListDiscussionsController ]; public function __construct(protected Client $elastic, protected UrlGenerator $uri) - { - } + {} protected function data(ServerRequestInterface $request, Document $document) { diff --git a/src/Commands/BuildCommand.php b/src/Commands/BuildCommand.php index 3935da6..4fcf31b 100644 --- a/src/Commands/BuildCommand.php +++ b/src/Commands/BuildCommand.php @@ -15,30 +15,37 @@ namespace Blomstra\Search\Commands; use Blomstra\Search\Jobs\Job; use Blomstra\Search\Jobs\SavingJob; use Blomstra\Search\Seeders\Seeder; +use Carbon\Carbon; use Elasticsearch\Client; use Flarum\Settings\SettingsRepositoryInterface; use Illuminate\Console\Command; use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Queue\Queue; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Support\Arr; +use Spatie\ElasticsearchQueryBuilder\Builder; +use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery; +use Spatie\ElasticsearchQueryBuilder\Queries\RangeQuery; +use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery; class BuildCommand extends Command { protected $signature = 'blomstra:search:index {--max-id= : Limits for each object the number of items to seed} - {--chunk-size= : Size of the chunks to dispatch into jobs} {--throttle= : Number of seconds to wait between pushing to the queue} {--only= : type to run seeder for, eg discussions or posts} {--recreate : create or recreate the index} {--mapping : recreate the mapping} - {--continue : continue each object type where you left off}'; + {--continue : continue each object type where you left off} + {--seed-missing : attempt to seed objects that are missing in the index}'; protected $description = 'Rebuilds the complete search server with its documents.'; public function handle(Container $container) { + /** @var string $index */ $index = $container->make('blomstra.search.elastic_index'); - /** @var array $seeders */ + /** @var array|string[] $seeders */ $seeders = $container->tagged('blomstra.search.seeders'); /** @var Queue $queue */ @@ -53,11 +60,13 @@ class BuildCommand extends Command $properties = [ 'properties' => [ 'content' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'], + 'rawId' => ['type' => 'integer'], 'created_at' => ['type' => 'date'], 'updated_at' => ['type' => 'date'], 'is_private' => ['type' => 'boolean'], 'is_sticky' => ['type' => 'boolean'], 'groups' => ['type' => 'integer'], + 'tags' => ['type' => 'integer'], 'recipient_groups' => ['type' => 'integer'], 'recipient_users' => ['type' => 'integer'], 'comment_count' => ['type' => 'integer'], @@ -126,7 +135,25 @@ class BuildCommand extends Command ? ($this->continueAt($seeder->type()) ?? $seeder->query()->max('id')) : $seeder->query()->max('id'); + $seeded = null; + while ($continueAt !== null) { + if ($this->option('seed-missing')) { + $response = (new Builder($client)) + ->index($index) + ->size(1000) + ->addQuery((new BoolQuery) + ->add((new RangeQuery('rawId')) + ->gte($continueAt - 1000) + ->lte($continueAt)) + ->add(TermQuery::create('type', $seeder->type())) + + ) + ->search(); + + $seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId'); + } + /** @var Collection $collection */ $collection = $seeder->query() ->latest('id') @@ -134,6 +161,7 @@ class BuildCommand extends Command ->when($this->option('max-id'), function ($query, $id) { $query->where('id', '<=', $id); }) + ->when($seeded, fn ($query, $seeded) => $query->whereNotIn('id', $seeded)) ->get(); $min = $collection->min('id'); diff --git a/src/Jobs/SavingJob.php b/src/Jobs/SavingJob.php index 9aec901..578cdb8 100644 --- a/src/Jobs/SavingJob.php +++ b/src/Jobs/SavingJob.php @@ -35,7 +35,7 @@ class SavingJob extends Job ]; }) ->flatten(1); - +dump($body); $response = $client->bulk([ 'index' => $this->index, 'body' => $body->toArray(), diff --git a/src/Seeders/CommentSeeder.php b/src/Seeders/CommentSeeder.php index cb4e055..55e50b4 100644 --- a/src/Seeders/CommentSeeder.php +++ b/src/Seeders/CommentSeeder.php @@ -71,6 +71,7 @@ class CommentSeeder extends Seeder $document = new Document([ 'type' => $this->type(), 'id' => $this->type().':'.$model->id, + 'rawId' => $model->id, 'content' => $model->content, 'content_partial' => $model->content, 'created_at' => $model->created_at?->toAtomString(), @@ -81,6 +82,10 @@ class CommentSeeder extends Seeder 'comment_count' => $model->discussion->comment_count, ]); + if ($this->extensionEnabled('flarum-tags')) { + $document['tags'] = $model->discussion->tags->pluck('id')->toArray(); + } + if ($this->extensionEnabled('fof-byobu')) { $document['recipient_users'] = $model->discussion->recipientUsers->pluck('id')->toArray(); $document['recipient_groups'] = $model->discussion->recipientGroups->pluck('id')->toArray(); diff --git a/src/Seeders/DiscussionSeeder.php b/src/Seeders/DiscussionSeeder.php index 1dfd31f..8558c50 100644 --- a/src/Seeders/DiscussionSeeder.php +++ b/src/Seeders/DiscussionSeeder.php @@ -72,6 +72,7 @@ class DiscussionSeeder extends Seeder $document = new Document([ 'type' => $this->type(), 'id' => $this->type().':'.$model->id, + 'rawId' => $model->id, 'content' => $model->title, 'content_partial' => $model->title, 'created_at' => $model->created_at?->toAtomString(), @@ -82,6 +83,10 @@ class DiscussionSeeder extends Seeder 'comment_count' => $model->comment_count, ]); + if ($this->extensionEnabled('flarum-tags')) { + $document['tags'] = $model->tags->pluck('id')->toArray(); + } + if ($this->extensionEnabled('fof-byobu')) { $document['recipient_users'] = $model->recipientUsers->pluck('id')->toArray(); $document['recipient_groups'] = $model->recipientGroups->pluck('id')->toArray(); diff --git a/src/Seeders/Seeder.php b/src/Seeders/Seeder.php index ff0b890..a2698a9 100644 --- a/src/Seeders/Seeder.php +++ b/src/Seeders/Seeder.php @@ -47,7 +47,6 @@ abstract class Seeder /** @var Collection $tags */ $tags = $discussion->tags; - $filters['tags'] = $tags->pluck('id')->toArray(); $tagPermissions = Permission::query() ->whereIn( 'permission',