Compare commits

..

1 Commits

Author SHA1 Message Date
flarum-bot bf19eed20f Bundled output for commit 2fc66b9bf7
Includes transpiled JS/TS, and Typescript declaration files (typings).

[skip ci]
2026-04-13 20:22:53 +00:00
15 changed files with 62 additions and 187 deletions

View File

@ -34,6 +34,5 @@ return [
(new Flarum\Settings())
->default('blomstra-search.search-discussion-subjects', true)
->default('blomstra-search.search-post-bodies', true)
->default('blomstra-search.min-search-length', Commands\BuildCommand::DEFAULT_MIN_SEARCH_LENGTH)
->serializeToForum('blomstraSearchMinLength', 'blomstra-search.min-search-length', 'intval'),
->default('blomstra-search.min-search-length', Commands\BuildCommand::DEFAULT_MIN_SEARCH_LENGTH),
];

2
js/dist/admin.js vendored
View File

@ -1,2 +1,2 @@
(()=>{var t={n:a=>{var e=a&&a.__esModule?()=>a.default:()=>a;return t.d(e,{a:e}),e},d:(a,e)=>{for(var r in e)t.o(e,r)&&!t.o(a,r)&&Object.defineProperty(a,r,{enumerable:!0,get:e[r]})},o:(t,a)=>Object.prototype.hasOwnProperty.call(t,a)};(()=>{"use strict";function a(t,e){return a=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,a){return t.__proto__=a,t},a(t,e)}const e=flarum.core.compat["admin/app"];var r=t.n(e);const n=flarum.core.compat["common/extend"],s=flarum.core.compat["admin/components/DashboardPage"];var i=t.n(s);const o=flarum.core.compat["admin/components/DashboardWidget"];var l=t.n(o);const c=flarum.core.compat["common/components/Alert"];var h=t.n(c),d=function(t){var e,n;function s(){return t.apply(this,arguments)||this}n=t,(e=s).prototype=Object.create(n.prototype),e.prototype.constructor=e,a(e,n);var i=s.prototype;return i.className=function(){return"ReindexWarningWidget"},i.content=function(){return m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle",title:r().translator.trans("blomstra-search.admin.reindex-required.title")},r().translator.trans("blomstra-search.admin.reindex-required.detail"))},s}(l());r().initializers.add("blomstra-search",function(){var t=r().data.settings["blomstra-search.active-index"],a=r().data.settings["blomstra-search.index-compatible"];t&&"v2"!==a&&(0,n.extend)(i().prototype,"availableWidgets",function(t){t.add("blomstra-search-reindex",m(d),110)});var e=new Map;["arabic","armenian","basque","bengali","brazilian","bulgarian","catalan","cjk","czech","danish","dutch","english","estonian","finnish","french","galician","german","greek","hindi","hungarian","indonesian","irish","italian","latvian","lithuanian","norwegian","persian","portuguese","romanian","russian","sorani","spanish","swedish","turkish","thai"].forEach(function(t){e.set(t,t)}),r().extensionData.for("blomstra-search").registerSetting(function(){var a=r().data.settings["blomstra-search.indexed-analyzer"];if(!t||!a)return null;var e=this.setting("blomstra-search.analyzer-language")()||"english",n=this.setting("blomstra-search.min-search-length")()||String(3),s=String(r().data.settings["blomstra-search.indexed-min-search-length"]||3);return e===a&&n===s?null:m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle"},r().translator.trans("blomstra-search.admin.index-settings-changed"))}).registerSetting({setting:"blomstra-search.elastic-endpoint",label:r().translator.trans("blomstra-search.admin.elastic-endpoint"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-username",label:r().translator.trans("blomstra-search.admin.elastic-username"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-password",label:r().translator.trans("blomstra-search.admin.elastic-password"),type:"password"}).registerSetting({setting:"blomstra-search.elastic-index",label:r().translator.trans("blomstra-search.admin.elastic-index"),default:"flarum",type:"input"}).registerSetting({setting:"blomstra-search.analyzer-language",label:r().translator.trans("blomstra-search.admin.analyzer.label"),help:r().translator.trans("blomstra-search.admin.analyzer.help"),type:"select",options:Object.fromEntries(e.entries()),default:"english"}).registerSetting({setting:"blomstra-search.search-discussion-subjects",label:r().translator.trans("blomstra-search.admin.search-discussion-subjects"),type:"switch"}).registerSetting({setting:"blomstra-search.search-post-bodies",label:r().translator.trans("blomstra-search.admin.search-post-bodies"),type:"switch"}).registerSetting({setting:"blomstra-search.min-search-length",label:r().translator.trans("blomstra-search.admin.min-search-length.label"),help:r().translator.trans("blomstra-search.admin.min-search-length.help"),type:"select",options:{1:"1",2:"2",3:"3",4:"4"},default:String(3)})})})(),module.exports={}})();
(()=>{var t={n:a=>{var e=a&&a.__esModule?()=>a.default:()=>a;return t.d(e,{a:e}),e},d:(a,e)=>{for(var r in e)t.o(e,r)&&!t.o(a,r)&&Object.defineProperty(a,r,{enumerable:!0,get:e[r]})},o:(t,a)=>Object.prototype.hasOwnProperty.call(t,a)};(()=>{"use strict";function a(t,e){return a=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,a){return t.__proto__=a,t},a(t,e)}const e=flarum.core.compat["admin/app"];var r=t.n(e);const n=flarum.core.compat["common/extend"],s=flarum.core.compat["admin/components/DashboardPage"];var i=t.n(s);const o=flarum.core.compat["admin/components/DashboardWidget"];var l=t.n(o);const c=flarum.core.compat["common/components/Alert"];var h=t.n(c),d=function(t){var e,n;function s(){return t.apply(this,arguments)||this}n=t,(e=s).prototype=Object.create(n.prototype),e.prototype.constructor=e,a(e,n);var i=s.prototype;return i.className=function(){return"ReindexWarningWidget"},i.content=function(){return m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle",title:r().translator.trans("blomstra-search.admin.reindex-required.title")},r().translator.trans("blomstra-search.admin.reindex-required.detail"))},s}(l());r().initializers.add("blomstra-search",function(){var t=r().data.settings["blomstra-search.active-index"],a=r().data.settings["blomstra-search.index-compatible"];t&&"v2"!==a&&(0,n.extend)(i().prototype,"availableWidgets",function(t){t.add("blomstra-search-reindex",m(d),110)});var e=new Map;["arabic","armenian","basque","bengali","brazilian","bulgarian","catalan","cjk","czech","danish","dutch","english","estonian","finnish","french","galician","german","greek","hindi","hungarian","indonesian","irish","italian","latvian","lithuanian","norwegian","persian","portuguese","romanian","russian","sorani","spanish","swedish","turkish","thai"].forEach(function(t){e.set(t,t)}),r().extensionData.for("blomstra-search").registerSetting(function(){var a=r().data.settings["blomstra-search.indexed-analyzer"];if(!t||!a)return null;var e=this.setting("blomstra-search.analyzer-language")()||"english",n=this.setting("blomstra-search.min-search-length")(),s=String(r().data.settings["blomstra-search.indexed-min-search-length"]||r().data.settings["blomstra-search.min-search-length"]);return e===a&&n===s?null:m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle"},r().translator.trans("blomstra-search.admin.index-settings-changed"))}).registerSetting({setting:"blomstra-search.elastic-endpoint",label:r().translator.trans("blomstra-search.admin.elastic-endpoint"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-username",label:r().translator.trans("blomstra-search.admin.elastic-username"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-password",label:r().translator.trans("blomstra-search.admin.elastic-password"),type:"password"}).registerSetting({setting:"blomstra-search.elastic-index",label:r().translator.trans("blomstra-search.admin.elastic-index"),default:"flarum",type:"input"}).registerSetting({setting:"blomstra-search.analyzer-language",label:r().translator.trans("blomstra-search.admin.analyzer.label"),help:r().translator.trans("blomstra-search.admin.analyzer.help"),type:"select",options:Object.fromEntries(e.entries()),default:"english"}).registerSetting({setting:"blomstra-search.search-discussion-subjects",label:r().translator.trans("blomstra-search.admin.search-discussion-subjects"),type:"switch"}).registerSetting({setting:"blomstra-search.search-post-bodies",label:r().translator.trans("blomstra-search.admin.search-post-bodies"),type:"switch"}).registerSetting({setting:"blomstra-search.min-search-length",label:r().translator.trans("blomstra-search.admin.min-search-length.label"),help:r().translator.trans("blomstra-search.admin.min-search-length.help"),type:"select",options:{1:"1",2:"2",3:"3",4:"4"},default:r().data.settings["blomstra-search.min-search-length"]})})})(),module.exports={}})();
//# sourceMappingURL=admin.js.map

File diff suppressed because one or more lines are too long

2
js/dist/forum.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -17,12 +17,6 @@ export default function extendDiscussionState() {
...params.page,
};
// Default to latest sort when searching without an explicit sort parameter.
// Matches the backend default and keeps the dropdown label in sync.
if (!params.sort) {
params.sort = '-lastPostedAt';
}
if (Array.isArray(params.include)) {
params.include = params.include.join(',');
}

View File

@ -66,7 +66,7 @@ export default class DiscussionsSearchSource implements SearchSource {
return [
<li className="Dropdown-header">{app.translator.trans('core.forum.search.discussions_heading')}</li>,
<li>
<LinkButton icon="fas fa-search" href={app.route('index', { q: query, sort: 'latest' })}>
<LinkButton icon="fas fa-search" href={app.route('index', { q: query })}>
{app.translator.trans('core.forum.search.all_discussions_button', { query })}
</LinkButton>
</li>,

View File

@ -9,14 +9,16 @@ import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource';
import extendDiscussionState from './PaginatedListStates/extendDiscussionState';
app.initializers.add('blomstra-search', () => {
extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) {
// app.forum is not available during initializers (it is set after they run),
// so read the setting lazily here, at first render time.
const minLength = (app.forum.attribute('blomstraSearchMinLength') as number) || Search.MIN_SEARCH_LEN;
if (minLength !== Search.MIN_SEARCH_LEN) {
(Search as any).MIN_SEARCH_LEN = minLength;
}
const minLength = parseInt(app.data.settings['blomstra-search.min-search-length'] || String(Search.MIN_SEARCH_LEN), 10);
if (minLength !== Search.MIN_SEARCH_LEN) {
// Flarum provides no extension point for MIN_SEARCH_LEN, so we overwrite the
// static property directly. TypeScript `readonly` is compile-time only — at
// runtime this is a plain property assignment and is safe as long as no code
// reads MIN_SEARCH_LEN before this initializer runs.
(Search as any).MIN_SEARCH_LEN = minLength;
}
extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) {
items.replace('discussions', new DiscussionsSearchSource());
});
});

View File

@ -37,7 +37,7 @@ use Illuminate\Support\Str;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Log\LoggerInterface;
use Spatie\ElasticsearchQueryBuilder\Builder;
use Blomstra\Search\Elasticsearch\BoolQuery;
use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;
use Spatie\ElasticsearchQueryBuilder\Sorts\Sort;
use Tobscure\JsonApi\Document;
@ -78,13 +78,27 @@ class SearchController extends ListDiscussionsController
$offset = $this->extractOffset($request);
$include = array_merge($this->extractInclude($request), ['state']);
$query = BoolQuery::create()
// Always restrict to discussion documents; posts are only searched via has_child.
->add(TermQuery::create('join_field', 'discussion'), 'filter');
if (!empty($search)) {
$query->add($this->buildTextQuery($search, $actor));
}
$this->addFilters($query, $actor, $filters);
$builder = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->size($limit + 1)
->from($offset)
->addQuery($query);
$knownSortFields = array_merge(array_values($this->translateSort), ['rawId']);
$logger = resolve(LoggerInterface::class);
$phpSortField = null;
$phpSortDir = 'desc';
$needsScoring = true;
$sorts = [];
foreach ($this->extractSort($request) as $field => $direction) {
$translated = $this->translateSort[$field] ?? $field;
@ -94,8 +108,7 @@ class SearchController extends ListDiscussionsController
continue;
}
$sorts[] = new Sort($translated, $direction);
$needsScoring = false;
$builder->addSort(new Sort($translated, $direction));
if ($phpSortField === null && $translated !== 'rawId') {
$phpSortField = $translated;
@ -103,45 +116,7 @@ class SearchController extends ListDiscussionsController
}
}
// Default to latest when no explicit sort is requested. This lets has_child use
// score_mode:none, which skips child scoring entirely and lets ES short-circuit
// early on large corpora. Relevance is still available via sort=relevant if added.
if (empty($sorts)) {
$needsScoring = false;
$phpSortField = 'updated_at';
$phpSortDir = 'desc';
$sorts[] = new Sort('updated_at', 'desc');
}
$query = BoolQuery::create()
// Always restrict to discussion documents; posts are only searched via has_child.
->add(TermQuery::create('join_field', 'discussion'), 'filter');
if (!empty($search)) {
$query->add($this->buildTextQuery($search, $actor, $needsScoring));
}
$this->addFilters($query, $actor, $filters);
$builder = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->addQuery($query);
foreach ($sorts as $sort) {
$builder->addSort($sort);
}
// track_total_hits: false lets ES stop counting once it has collected
// enough results in sort order, avoiding a full-index count on every query.
$payload = $builder->getPayload();
$payload['track_total_hits'] = false;
$response = $this->elastic->search([
'index' => resolve('blomstra.search.elastic_index'),
'size' => $limit + 1,
'from' => $offset,
'body' => $payload,
]);
$response = $builder->search();
Discussion::setStateUser($actor);
@ -221,16 +196,12 @@ class SearchController extends ListDiscussionsController
* Build the text-matching portion of the query.
*
* Discussion titles are matched directly (filtered to join_field=discussion).
* Post bodies are matched via has_child. When $needsScoring is true (no explicit
* sort, i.e. relevance ordering), score_mode=sum accumulates child scores onto the
* parent so that discussions with many strongly-matching posts rank higher. When
* false (any explicit field sort), score_mode=none skips scoring entirely ES only
* checks whether a matching child exists, which is significantly cheaper on large
* corpora.
* inner_hits returns the best-matching post for use as mostRelevantPost.
* Post bodies are matched via has_child with score_mode=sum so discussions
* with many matching posts score higher than those with a single strong match.
* inner_hits returns the best-scoring post for use as mostRelevantPost.
* Hidden posts are only included in matching for users with post.hide permission.
*/
protected function buildTextQuery(string $search, User $actor, bool $needsScoring = false): BoolQuery
protected function buildTextQuery(string $search, User $actor): BoolQuery
{
$textQuery = BoolQuery::create();
@ -243,15 +214,11 @@ class SearchController extends ListDiscussionsController
// Guests and non-moderators may not see hidden posts; exclude them from child matching.
if ($actor->isGuest() || !$actor->hasPermission('post.hide')) {
$postQuery->add(TermQuery::create('is_hidden', 'false'), 'filter');
$postQuery->add(TermQuery::create('is_hidden', false), 'filter');
}
// Without minimum_should_match, ES default MSM is 0 when a filter clause is present,
// causing has_child to score every non-hidden post instead of only matching ones.
$postQuery->minimumShouldMatch(1);
$textQuery->add(
HasChildQuery::create('post', $postQuery, $needsScoring ? 'sum' : 'none')->withInnerHits(),
HasChildQuery::create('post', $postQuery)->withInnerHits(),
'should'
);
}
@ -268,6 +235,7 @@ class SearchController extends ListDiscussionsController
}
if ($this->matchWords) {
$should->add((new MatchQuery('content', $search))->operator('and')->boost(1.8 * $boost), 'should');
$should->add((new MatchQuery('content', $search))->operator('or')->boost(0.8 * $boost), 'should');
}
return $should;
@ -333,12 +301,8 @@ class SearchController extends ListDiscussionsController
$search = Arr::get($filters, 'q');
if ($search) {
// Strip Flarum gambit operators (tag:foo, author:bar, is:private, etc.)
// before passing to ES. These are structural filters handled separately;
// leaving them in causes operator:and to require the gambit tokens to
// appear literally in post content, producing zero results.
$q = collect(explode(' ', $search))
->filter(fn (string $part) => !preg_match('/^\w+:/', $part))
->filter(fn (string $part) => $part !== 'is:private')
->filter()
->join(' ');

View File

@ -16,7 +16,6 @@ use Blomstra\Search\Jobs\Job;
use Blomstra\Search\Jobs\UpdateSearchJob;
use Blomstra\Search\Seeders\Seeder;
use Elasticsearch\Client;
use Elasticsearch\Common\Exceptions\ElasticsearchException;
use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Console\Command;
use Illuminate\Contracts\Container\Container;
@ -155,10 +154,8 @@ HELP;
$this->warn("An in-progress build already exists: $staging");
$this->line('');
$this->line('Choose one of:');
$this->line(' blomstra:search:index build --resume Continue from where it left off');
$this->line(' blomstra:search:index build --resume --staging Continue and keep in staging when done');
$this->line(' blomstra:search:index build --fresh Drop this build and start completely fresh');
$this->line(' blomstra:search:index build --fresh --staging Start fresh and keep in staging when done');
$this->line(' blomstra:search:index build --resume Continue from where it left off');
$this->line(' blomstra:search:index build --fresh Drop this build and start completely fresh');
return;
}
@ -362,7 +359,17 @@ HELP;
$rangeTo = $continueAt;
if ($seedMissing) {
$seeded = $this->queryIndexedIds($client, $targetIndex, $seeder->joinRelation(), $rangeFrom, $rangeTo);
$response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $seeder->joinRelation()))
)
->search();
$seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
}
/** @var Collection $collection */
@ -562,48 +569,6 @@ HELP;
];
}
/**
* Query ES for rawIds already indexed in $targetIndex for the given $joinRelation and ID range.
* Retries on transient ES failures (NoNodesAvailableException / other ElasticsearchException)
* by sleeping past the StaticNoPingConnectionPool dead-node timeout (default 60 s) before
* each retry, giving the pool a chance to resurface the node.
*/
protected function queryIndexedIds(
Client $client,
string $targetIndex,
string $joinRelation,
int $rangeFrom,
int $rangeTo,
int $maxRetries = 10
): array {
$attempt = 0;
while (true) {
try {
$response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $joinRelation))
)
->search();
return Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} catch (ElasticsearchException $e) {
$attempt++;
if ($attempt >= $maxRetries) {
throw $e;
}
$this->warn("ES error on range {$rangeFrom}{$rangeTo} (attempt {$attempt}/{$maxRetries}): {$e->getMessage()}. Waiting 65 s before retry…");
sleep(65); // outlast the StaticNoPingConnectionPool dead-node window (default 60 s)
}
}
}
protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int
{
$raw = $settings->get("blomstra-search.continued-at.$type");

View File

@ -1,41 +0,0 @@
<?php
/*
* This file is part of blomstra/search.
*
* Copyright (c) 2022 Blomstra Ltd.
*
* For the full copyright and license information, please view the LICENSE.md
* file that was distributed with this source code.
*
*/
namespace Blomstra\Search\Elasticsearch;
class BoolQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery
{
protected ?int $minimumShouldMatch = null;
public static function create(): static
{
return new self();
}
public function minimumShouldMatch(int $minimum): static
{
$this->minimumShouldMatch = $minimum;
return $this;
}
public function toArray(): array
{
$array = parent::toArray();
if ($this->minimumShouldMatch !== null) {
$array['bool']['minimum_should_match'] = $this->minimumShouldMatch;
}
return $array;
}
}

View File

@ -14,7 +14,7 @@ namespace Blomstra\Search\Jobs;
use Blomstra\Search\Seeders\Seeder;
use Flarum\Queue\AbstractJob;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Collection;
abstract class Job extends AbstractJob
{

View File

@ -14,7 +14,6 @@ namespace Blomstra\Search\Jobs;
use Blomstra\Search\Exceptions\SeedingException;
use Elasticsearch\Client;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr;
@ -51,11 +50,10 @@ class UpdateSearchJob extends Job
$items = Arr::get($response, 'items');
$failed = array_filter($items, fn ($item) => isset($item['index']['error']));
$error = Arr::get(Arr::first($failed), 'index.error.reason', 'unknown error');
$error = Arr::get(Arr::first($items), 'index.error.reason');
throw new SeedingException(
"Failed to seed: $error (" . count($failed) . '/' . count($items) . ' items failed)',
"Failed to seed: $error",
$items
);
}

View File

@ -26,7 +26,7 @@ use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Contracts\Container\Container;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Contracts\Queue\Queue;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Collection;
use Laminas\Stratigility\MiddlewarePipe;
use Psr\Log\LoggerInterface;
@ -47,13 +47,7 @@ class Provider extends AbstractServiceProvider
$this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) {
$builder = ClientBuilder::create()
->setHosts([$settings->get('blomstra-search.elastic-endpoint')])
->setConnectionParams([
'client' => [
'connect_timeout' => 2, // fail fast if ES is unreachable
'timeout' => 10, // allow time for complex queries
],
]);
->setHosts([$settings->get('blomstra-search.elastic-endpoint')]);
if ($config->inDebugMode()) {
$builder->setLogger($container->make(LoggerInterface::class));

View File

@ -24,7 +24,7 @@ use FoF\Byobu\Events as Byobu;
use FoF\DiscussionViews\Events\DiscussionWasViewed;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
class DiscussionSeeder extends Seeder
@ -70,7 +70,7 @@ class DiscussionSeeder extends Seeder
{
$events->listen([
// flarum/core events
Core\Started::class, Core\Restored::class, Core\Renamed::class,
Core\Started::class, Core\Restored::class,
// fof/byobu discussion recipients events.
Byobu\DiscussionMadePublic::class, Byobu\RemovedSelf::class, Byobu\RecipientsChanged::class,
], function ($event) use ($callable) {