Compare commits

..

No commits in common. "115c06e9bfbbb661c945e7db669d3b042827198f" and "bf19eed20f211b030607de148d74cb159f66e06e" have entirely different histories.

38 changed files with 224 additions and 597 deletions

1
.gitignore vendored
View File

@ -2,4 +2,3 @@ js/node_modules
js/dist js/dist
vendor/ vendor/
composer.lock composer.lock
analysis/*

View File

@ -1,5 +1,5 @@
{ {
"name": "sbp/flarum-search", "name": "blomstra/search",
"description": "Replaces Flarum search with one powered by an elastic search server.", "description": "Replaces Flarum search with one powered by an elastic search server.",
"keywords": [ "keywords": [
"extension", "extension",
@ -11,20 +11,16 @@
"elasticsearch" "elasticsearch"
], ],
"support": { "support": {
"email": "mueh+git@sbp.de", "email": "helpdesk@blomstra.net",
"forum": "https://blomstra.community/t/ext-search" "forum": "https://blomstra.community/t/ext-search"
}, },
"type": "flarum-extension", "type": "flarum-extension",
"license": "MIT", "license": "MIT",
"authors": [ "authors": [
{
"name": "sbp",
"email": "info@sbp.de"
},
{ {
"name": "Team Blomstra", "name": "Team Blomstra",
"email": "development@blomstra.net", "email": "development@blomstra.net",
"role": "Original Developer" "role": "Developer"
} }
], ],
"require": { "require": {

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search; namespace Blomstra\Search;
use Flarum\Extend as Flarum; use Flarum\Extend as Flarum;
@ -26,16 +26,13 @@ return [
new Flarum\Locales(__DIR__.'/resources/locale'), new Flarum\Locales(__DIR__.'/resources/locale'),
(new Flarum\Routes('api')) (new Flarum\Routes('api'))
->get('/sbp/search/{type}', 'sbp.search', Api\Controllers\SearchController::class) ->get('/blomstra/search/{type}', 'blomstra.search', Api\Controllers\SearchController::class)
->put('/sbp/search/index', 'sbp.search.index', Api\Controllers\IndexController::class), ->put('/blomstra/search/index', 'blomstra.search.index', Api\Controllers\IndexController::class),
(new Flarum\Console()) (new Flarum\Console())
->command(Commands\BuildCommand::class), ->command(Commands\BuildCommand::class),
(new Flarum\Settings()) (new Flarum\Settings())
->default('blomstra-search.search-discussion-subjects', true) ->default('blomstra-search.search-discussion-subjects', true)
->default('blomstra-search.search-post-bodies', true) ->default('blomstra-search.search-post-bodies', true)
->default('blomstra-search.min-search-length', Commands\BuildCommand::DEFAULT_MIN_SEARCH_LENGTH) ->default('blomstra-search.min-search-length', Commands\BuildCommand::DEFAULT_MIN_SEARCH_LENGTH),
->default('blomstra-search.stem-exclusion', '')
->serializeToForum('blomstraSearchMinLength', 'blomstra-search.min-search-length', 'intval')
->serializeToForum('blomstraSearchPostBodies', 'blomstra-search.search-post-bodies', fn ($v) => (bool) $v, true),
]; ];

2
js/dist/admin.js vendored
View File

@ -1,2 +1,2 @@
(()=>{var a={n:t=>{var e=t&&t.__esModule?()=>t.default:()=>t;return a.d(e,{a:e}),e},d:(t,e)=>{for(var n in e)a.o(e,n)&&!a.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:e[n]})},o:(a,t)=>Object.prototype.hasOwnProperty.call(a,t)};(()=>{"use strict";function t(a,e){return t=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(a,t){return a.__proto__=t,a},t(a,e)}const e=flarum.core.compat["admin/app"];var n=a.n(e);const r=flarum.core.compat["common/extend"],s=flarum.core.compat["admin/components/DashboardPage"];var i=a.n(s);const l=flarum.core.compat["admin/components/DashboardWidget"];var o=a.n(l);const c=flarum.core.compat["common/components/Alert"];var h=a.n(c),d=function(a){var e,r;function s(){return a.apply(this,arguments)||this}r=a,(e=s).prototype=Object.create(r.prototype),e.prototype.constructor=e,t(e,r);var i=s.prototype;return i.className=function(){return"ReindexWarningWidget"},i.content=function(){return m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle",title:n().translator.trans("blomstra-search.admin.reindex-required.title")},n().translator.trans("blomstra-search.admin.reindex-required.detail"))},s}(o());n().initializers.add("sbp-search",function(){var a=n().data.settings["blomstra-search.active-index"],t=n().data.settings["blomstra-search.index-compatible"];a&&"v3"!==t&&(0,r.extend)(i().prototype,"availableWidgets",function(a){a.add("blomstra-search-reindex",m(d),110)}),n().extensionData.for("blomstra-search").registerSetting(function(){var t=n().data.settings["blomstra-search.indexed-analyzer"];if(!a||!t)return null;var e=this.setting("blomstra-search.analyzer-language")()||"english",r=this.setting("blomstra-search.stem-exclusion")()||"",s=n().data.settings["blomstra-search.indexed-stem-exclusion"]||"";return e===t&&r===s?null:m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle"},n().translator.trans("blomstra-search.admin.index-settings-changed"))}).registerSetting({setting:"blomstra-search.elastic-endpoint",label:n().translator.trans("blomstra-search.admin.elastic-endpoint"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-username",label:n().translator.trans("blomstra-search.admin.elastic-username"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-password",label:n().translator.trans("blomstra-search.admin.elastic-password"),type:"password"}).registerSetting({setting:"blomstra-search.elastic-index",label:n().translator.trans("blomstra-search.admin.elastic-index"),default:"flarum",type:"input"}).registerSetting({setting:"blomstra-search.search-discussion-subjects",label:n().translator.trans("blomstra-search.admin.search-discussion-subjects"),type:"switch"}).registerSetting({setting:"blomstra-search.search-post-bodies",label:n().translator.trans("blomstra-search.admin.search-post-bodies"),type:"switch"}).registerSetting({setting:"blomstra-search.analyzer-language",label:n().translator.trans("blomstra-search.admin.analyzer.label"),help:n().translator.trans("blomstra-search.admin.analyzer.help"),type:"select",options:{arabic:"Arabic",armenian:"Armenian",basque:"Basque",bengali:"Bengali",brazilian:"Brazilian Portuguese",bulgarian:"Bulgarian",catalan:"Catalan",cjk:"CJK (Chinese, Japanese, Korean)",czech:"Czech",danish:"Danish",dutch:"Dutch",english:"English",estonian:"Estonian",finnish:"Finnish",french:"French",galician:"Galician",german:"German",greek:"Greek",hindi:"Hindi",hungarian:"Hungarian",indonesian:"Indonesian",irish:"Irish",italian:"Italian",latvian:"Latvian",lithuanian:"Lithuanian",norwegian:"Norwegian",persian:"Persian",portuguese:"Portuguese",romanian:"Romanian",russian:"Russian",sorani:"Sorani (Kurdish)",spanish:"Spanish",standard:"Standard (generic)",swedish:"Swedish",turkish:"Turkish",thai:"Thai"},default:"english"}).registerSetting(function(){var a="cjk"===(this.setting("blomstra-search.analyzer-language")()||"english");return this.buildSettingComponent({setting:"blomstra-search.stem-exclusion",type:"textarea",label:n().translator.trans("blomstra-search.admin.settings.stem-exclusion.label"),help:n().translator.trans("blomstra-search.admin.settings.stem-exclusion.help"),disabled:a})}).registerSetting(function(){var a="cjk"===(this.setting("blomstra-search.analyzer-language")()||"english");return this.buildSettingComponent({setting:"blomstra-search.min-search-length",label:n().translator.trans("blomstra-search.admin.min-search-length.label"),help:n().translator.trans("blomstra-search.admin.min-search-length.help"),type:"select",options:{1:"1",2:"2",3:"3",4:"4"},default:n().data.settings["blomstra-search.min-search-length"],disabled:a})})})})(),module.exports={}})(); (()=>{var t={n:a=>{var e=a&&a.__esModule?()=>a.default:()=>a;return t.d(e,{a:e}),e},d:(a,e)=>{for(var r in e)t.o(e,r)&&!t.o(a,r)&&Object.defineProperty(a,r,{enumerable:!0,get:e[r]})},o:(t,a)=>Object.prototype.hasOwnProperty.call(t,a)};(()=>{"use strict";function a(t,e){return a=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,a){return t.__proto__=a,t},a(t,e)}const e=flarum.core.compat["admin/app"];var r=t.n(e);const n=flarum.core.compat["common/extend"],s=flarum.core.compat["admin/components/DashboardPage"];var i=t.n(s);const o=flarum.core.compat["admin/components/DashboardWidget"];var l=t.n(o);const c=flarum.core.compat["common/components/Alert"];var h=t.n(c),d=function(t){var e,n;function s(){return t.apply(this,arguments)||this}n=t,(e=s).prototype=Object.create(n.prototype),e.prototype.constructor=e,a(e,n);var i=s.prototype;return i.className=function(){return"ReindexWarningWidget"},i.content=function(){return m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle",title:r().translator.trans("blomstra-search.admin.reindex-required.title")},r().translator.trans("blomstra-search.admin.reindex-required.detail"))},s}(l());r().initializers.add("blomstra-search",function(){var t=r().data.settings["blomstra-search.active-index"],a=r().data.settings["blomstra-search.index-compatible"];t&&"v2"!==a&&(0,n.extend)(i().prototype,"availableWidgets",function(t){t.add("blomstra-search-reindex",m(d),110)});var e=new Map;["arabic","armenian","basque","bengali","brazilian","bulgarian","catalan","cjk","czech","danish","dutch","english","estonian","finnish","french","galician","german","greek","hindi","hungarian","indonesian","irish","italian","latvian","lithuanian","norwegian","persian","portuguese","romanian","russian","sorani","spanish","swedish","turkish","thai"].forEach(function(t){e.set(t,t)}),r().extensionData.for("blomstra-search").registerSetting(function(){var a=r().data.settings["blomstra-search.indexed-analyzer"];if(!t||!a)return null;var e=this.setting("blomstra-search.analyzer-language")()||"english",n=this.setting("blomstra-search.min-search-length")(),s=String(r().data.settings["blomstra-search.indexed-min-search-length"]||r().data.settings["blomstra-search.min-search-length"]);return e===a&&n===s?null:m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle"},r().translator.trans("blomstra-search.admin.index-settings-changed"))}).registerSetting({setting:"blomstra-search.elastic-endpoint",label:r().translator.trans("blomstra-search.admin.elastic-endpoint"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-username",label:r().translator.trans("blomstra-search.admin.elastic-username"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-password",label:r().translator.trans("blomstra-search.admin.elastic-password"),type:"password"}).registerSetting({setting:"blomstra-search.elastic-index",label:r().translator.trans("blomstra-search.admin.elastic-index"),default:"flarum",type:"input"}).registerSetting({setting:"blomstra-search.analyzer-language",label:r().translator.trans("blomstra-search.admin.analyzer.label"),help:r().translator.trans("blomstra-search.admin.analyzer.help"),type:"select",options:Object.fromEntries(e.entries()),default:"english"}).registerSetting({setting:"blomstra-search.search-discussion-subjects",label:r().translator.trans("blomstra-search.admin.search-discussion-subjects"),type:"switch"}).registerSetting({setting:"blomstra-search.search-post-bodies",label:r().translator.trans("blomstra-search.admin.search-post-bodies"),type:"switch"}).registerSetting({setting:"blomstra-search.min-search-length",label:r().translator.trans("blomstra-search.admin.min-search-length.label"),help:r().translator.trans("blomstra-search.admin.min-search-length.help"),type:"select",options:{1:"1",2:"2",3:"3",4:"4"},default:r().data.settings["blomstra-search.min-search-length"]})})})(),module.exports={}})();
//# sourceMappingURL=admin.js.map //# sourceMappingURL=admin.js.map

File diff suppressed because one or more lines are too long

2
js/dist/forum.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -4,7 +4,7 @@ import DashboardPage from 'flarum/admin/components/DashboardPage';
import DashboardWidget from 'flarum/admin/components/DashboardWidget'; import DashboardWidget from 'flarum/admin/components/DashboardWidget';
import Alert from 'flarum/common/components/Alert'; import Alert from 'flarum/common/components/Alert';
const REQUIRED_INDEX_COMPAT = 'v3'; const REQUIRED_INDEX_COMPAT = 'v2';
class ReindexWarningWidget extends DashboardWidget { class ReindexWarningWidget extends DashboardWidget {
className() { className() {
@ -25,7 +25,7 @@ class ReindexWarningWidget extends DashboardWidget {
} }
} }
app.initializers.add('sbp-search', () => { app.initializers.add('blomstra-search', () => {
const activeIndex = app.data.settings['blomstra-search.active-index']; const activeIndex = app.data.settings['blomstra-search.active-index'];
const compatVersion = app.data.settings['blomstra-search.index-compatible']; const compatVersion = app.data.settings['blomstra-search.index-compatible'];
@ -35,44 +35,46 @@ app.initializers.add('sbp-search', () => {
}); });
} }
const languages: Record<string, string> = { const languages = new Map();
arabic: 'Arabic', [
armenian: 'Armenian', 'arabic',
basque: 'Basque', 'armenian',
bengali: 'Bengali', 'basque',
brazilian: 'Brazilian Portuguese', 'bengali',
bulgarian: 'Bulgarian', 'brazilian',
catalan: 'Catalan', 'bulgarian',
cjk: 'CJK (Chinese, Japanese, Korean)', 'catalan',
czech: 'Czech', 'cjk',
danish: 'Danish', 'czech',
dutch: 'Dutch', 'danish',
english: 'English', 'dutch',
estonian: 'Estonian', 'english',
finnish: 'Finnish', 'estonian',
french: 'French', 'finnish',
galician: 'Galician', 'french',
german: 'German', 'galician',
greek: 'Greek', 'german',
hindi: 'Hindi', 'greek',
hungarian: 'Hungarian', 'hindi',
indonesian: 'Indonesian', 'hungarian',
irish: 'Irish', 'indonesian',
italian: 'Italian', 'irish',
latvian: 'Latvian', 'italian',
lithuanian: 'Lithuanian', 'latvian',
norwegian: 'Norwegian', 'lithuanian',
persian: 'Persian', 'norwegian',
portuguese: 'Portuguese', 'persian',
romanian: 'Romanian', 'portuguese',
russian: 'Russian', 'romanian',
sorani: 'Sorani (Kurdish)', 'russian',
spanish: 'Spanish', 'sorani',
standard: 'Standard (generic)', 'spanish',
swedish: 'Swedish', 'swedish',
turkish: 'Turkish', 'turkish',
thai: 'Thai', 'thai',
}; ].forEach((language) => {
languages.set(language, language);
});
app.extensionData app.extensionData
.for('blomstra-search') .for('blomstra-search')
@ -82,10 +84,12 @@ app.initializers.add('sbp-search', () => {
if (!activeIndex || !indexedAnalyzer) return null; if (!activeIndex || !indexedAnalyzer) return null;
const currentAnalyzer = this.setting('blomstra-search.analyzer-language')() || 'english'; const currentAnalyzer = this.setting('blomstra-search.analyzer-language')() || 'english';
const currentStemExclusion = this.setting('blomstra-search.stem-exclusion')() || ''; const currentMinLength = this.setting('blomstra-search.min-search-length')();
const indexedStemExclusion = app.data.settings['blomstra-search.indexed-stem-exclusion'] || ''; const indexedMinLength = String(
app.data.settings['blomstra-search.indexed-min-search-length'] || app.data.settings['blomstra-search.min-search-length']
);
if (currentAnalyzer === indexedAnalyzer && currentStemExclusion === indexedStemExclusion) return null; if (currentAnalyzer === indexedAnalyzer && currentMinLength === indexedMinLength) return null;
return m( return m(
Alert, Alert,
@ -114,6 +118,14 @@ app.initializers.add('sbp-search', () => {
default: 'flarum', default: 'flarum',
type: 'input', type: 'input',
}) })
.registerSetting({
setting: 'blomstra-search.analyzer-language',
label: app.translator.trans('blomstra-search.admin.analyzer.label'),
help: app.translator.trans('blomstra-search.admin.analyzer.help'),
type: 'select',
options: Object.fromEntries(languages.entries()),
default: 'english',
})
.registerSetting({ .registerSetting({
setting: 'blomstra-search.search-discussion-subjects', setting: 'blomstra-search.search-discussion-subjects',
label: app.translator.trans('blomstra-search.admin.search-discussion-subjects'), label: app.translator.trans('blomstra-search.admin.search-discussion-subjects'),
@ -125,33 +137,11 @@ app.initializers.add('sbp-search', () => {
type: 'switch', type: 'switch',
}) })
.registerSetting({ .registerSetting({
setting: 'blomstra-search.analyzer-language',
label: app.translator.trans('blomstra-search.admin.analyzer.label'),
help: app.translator.trans('blomstra-search.admin.analyzer.help'),
type: 'select',
options: languages,
default: 'english',
})
.registerSetting(function (this: any) {
const isCjk = (this.setting('blomstra-search.analyzer-language')() || 'english') === 'cjk';
return this.buildSettingComponent({
setting: 'blomstra-search.stem-exclusion',
type: 'textarea',
label: app.translator.trans('blomstra-search.admin.settings.stem-exclusion.label'),
help: app.translator.trans('blomstra-search.admin.settings.stem-exclusion.help'),
disabled: isCjk,
});
})
.registerSetting(function (this: any) {
const isCjk = (this.setting('blomstra-search.analyzer-language')() || 'english') === 'cjk';
return this.buildSettingComponent({
setting: 'blomstra-search.min-search-length', setting: 'blomstra-search.min-search-length',
label: app.translator.trans('blomstra-search.admin.min-search-length.label'), label: app.translator.trans('blomstra-search.admin.min-search-length.label'),
help: app.translator.trans('blomstra-search.admin.min-search-length.help'), help: app.translator.trans('blomstra-search.admin.min-search-length.help'),
type: 'select', type: 'select',
options: { '1': '1', '2': '2', '3': '3', '4': '4' }, options: { '1': '1', '2': '2', '3': '3', '4': '4' },
default: app.data.settings['blomstra-search.min-search-length'], default: app.data.settings['blomstra-search.min-search-length'],
disabled: isCjk,
});
}); });
}); });

View File

@ -6,9 +6,10 @@ import DiscussionListState from 'flarum/forum/states/DiscussionListState';
export default function extendDiscussionState() { export default function extendDiscussionState() {
override(DiscussionListState.prototype, 'loadPage', async function (this: DiscussionListState, original, page: number = 1) { override(DiscussionListState.prototype, 'loadPage', async function (this: DiscussionListState, original, page: number = 1) {
const preloaded = app.data.apiDocument || null;
// If existing payload is given or no search is made, fallback on native page. // If existing payload is given or no search is made, fallback on native page.
if (!this.requestParams()?.filter?.q) return original.call(this, page); if (preloaded || !this.requestParams()?.filter?.q) return original.call(this, page);
const params = this.requestParams(); const params = this.requestParams();
params.page = { params.page = {
@ -16,13 +17,12 @@ export default function extendDiscussionState() {
...params.page, ...params.page,
}; };
if (Array.isArray(params.include)) { if (Array.isArray(params.include)) {
params.include = params.include.join(','); params.include = params.include.join(',');
} }
// Construct API search URI // Construct API search URI
const url = `${app.forum.attribute('apiUrl')}/sbp/search/${this.type}`; const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`;
// Make API GET request // Make API GET request
const results = await app.request({ params, url, method: 'GET' }); const results = await app.request({ params, url, method: 'GET' });

View File

@ -21,21 +21,18 @@ export default class DiscussionsSearchSource implements SearchSource {
private type = 'discussions'; private type = 'discussions';
async search(query: string): Promise<void> { async search(query: string): Promise<void> {
// Suppress ES instant results when "search within comments" is ON;
// only usernames are shown in that mode.
query = query.toLowerCase(); query = query.toLowerCase();
this.results.set(query, []); this.results.set(query, []);
const params = { const params = {
filter: { q: query, autocomplete: 1 }, filter: { q: query },
page: { limit: 3 }, page: { limit: 3 },
include: 'mostRelevantPost', include: 'mostRelevantPost',
}; };
// Construct API search URI // Construct API search URI
const url = `${app.forum.attribute('apiUrl')}/sbp/search/${this.type}`; const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`;
// Make API GET request // Make API GET request
const results = await app.request({ params, url, method: 'GET' }); const results = await app.request({ params, url, method: 'GET' });

View File

@ -1,26 +0,0 @@
import { extend } from 'flarum/common/extend';
import DiscussionListItem from 'flarum/forum/components/DiscussionListItem';
import TerminalPost from 'flarum/forum/components/TerminalPost';
export default function extendDiscussionListItem() {
extend(DiscussionListItem.prototype, 'infoItems', function (items) {
const params = this.attrs.params;
if (!params.q) return;
const hasFieldSort = params.sort && params.sort !== 'relevance';
if (hasFieldSort) {
// Field sort active (latest, oldest, top, …): replace excerpt with TerminalPost.
// An excerpt is meaningless when results are ordered by date/count rather than relevance.
items.remove('excerpt');
if (!items.has('terminalPost')) {
items.add('terminalPost', <TerminalPost discussion={this.attrs.discussion} lastPost={!this.showFirstPost()} />);
}
} else if (!items.has('excerpt')) {
// Relevance mode but no excerpt (mostRelevantPost was null or non-comment type).
// Fall back to TerminalPost so the info section is never silently empty.
items.add('terminalPost', <TerminalPost discussion={this.attrs.discussion} lastPost={true} />);
}
});
}

View File

@ -7,26 +7,26 @@ import ItemList from 'flarum/common/utils/ItemList';
import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource'; import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource';
import extendDiscussionState from './PaginatedListStates/extendDiscussionState'; import extendDiscussionState from './PaginatedListStates/extendDiscussionState';
import extendDiscussionListItem from './extendDiscussionListItem';
app.initializers.add('sbp-search', () => { app.initializers.add('blomstra-search', () => {
extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) { const minLength = parseInt(app.data.settings['blomstra-search.min-search-length'] || String(Search.MIN_SEARCH_LEN), 10);
// app.forum is not available during initializers (it is set after they run),
// so read the setting lazily here, at first render time.
const minLength = (app.forum.attribute('blomstraSearchMinLength') as number) || Search.MIN_SEARCH_LEN;
if (minLength !== Search.MIN_SEARCH_LEN) { if (minLength !== Search.MIN_SEARCH_LEN) {
// Flarum provides no extension point for MIN_SEARCH_LEN, so we overwrite the
// static property directly. TypeScript `readonly` is compile-time only — at
// runtime this is a plain property assignment and is safe as long as no code
// reads MIN_SEARCH_LEN before this initializer runs.
(Search as any).MIN_SEARCH_LEN = minLength; (Search as any).MIN_SEARCH_LEN = minLength;
} }
extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) {
items.replace('discussions', new DiscussionsSearchSource()); items.replace('discussions', new DiscussionsSearchSource());
}); });
}); });
app.initializers.add( app.initializers.add(
'sbp-search-early', 'blomstra-search-early',
() => { () => {
extendDiscussionState(); extendDiscussionState();
extendDiscussionListItem();
}, },
999999 999999
); );

View File

@ -1,14 +0,0 @@
diff --git a/src/Commands/BuildCommand.php b/src/Commands/BuildCommand.php
index 9024e1e..9024e1e 100644
--- a/src/Commands/BuildCommand.php
+++ b/src/Commands/BuildCommand.php
@@ -38,6 +38,6 @@ class BuildCommand extends Command
/** Matches Flarum's Search::MIN_SEARCH_LEN — the default minimum query length. */
public const DEFAULT_MIN_SEARCH_LENGTH = 3;
/** N-gram bounds for the title autocomplete subfield (not admin-configurable). */
- private const TITLE_MIN_GRAM = 2;
- private const TITLE_MAX_GRAM = 15;
+ private const TITLE_MIN_GRAM = 3;
+ private const TITLE_MAX_GRAM = 20;
protected $signature = 'blomstra:search:index

View File

@ -17,12 +17,9 @@ blomstra-search:
search-post-bodies: Search inside comments search-post-bodies: Search inside comments
match-sentences: Match search term against full sentence match-sentences: Match search term against full sentence
match-words: Match search term against full words match-words: Match search term against full words
settings:
stem-exclusion:
label: "Protected words (stem exclusion)"
help: "One word per line. These words will not be stemmed during search (e.g. brand names, proper nouns). Not applicable for CJK or Thai languages. Requires a full re-index to take effect."
min-search-length: min-search-length:
label: Minimum search query length label: Minimum search query length
help: | help: |
Minimum number of characters required before a search is triggered. Lower this to 1 or 2 Minimum number of characters required before a search is triggered. Lower this to 1 or 2
for CJK (Chinese, Japanese, Korean) communities where single characters carry full meaning. for CJK (Chinese, Japanese, Korean) communities where single characters carry full meaning.
Changing this requires a full index rebuild to take effect.

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Api; namespace Blomstra\Search\Api;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ResponseInterface;

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Api\Controllers; namespace Blomstra\Search\Api\Controllers;
use Sbp\Search\Commands\BuildCommand; use Blomstra\Search\Commands\BuildCommand;
use Laminas\Diactoros\Response\EmptyResponse; use Laminas\Diactoros\Response\EmptyResponse;
use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Message\ServerRequestInterface;

View File

@ -10,15 +10,15 @@
* *
*/ */
namespace Sbp\Search\Api\Controllers; namespace Blomstra\Search\Api\Controllers;
use Sbp\Search\Elasticsearch\HasChildQuery; use Blomstra\Search\Elasticsearch\HasChildQuery;
use Sbp\Search\Elasticsearch\MatchPhraseQuery; use Blomstra\Search\Elasticsearch\MatchPhraseQuery;
use Sbp\Search\Elasticsearch\MatchQuery; use Blomstra\Search\Elasticsearch\MatchQuery;
use Sbp\Search\Elasticsearch\TermsQuery; use Blomstra\Search\Elasticsearch\TermsQuery;
use Sbp\Search\Searchers\CommentPostSearcher; use Blomstra\Search\Searchers\CommentPostSearcher;
use Sbp\Search\Searchers\DiscussionSearcher; use Blomstra\Search\Searchers\DiscussionSearcher;
use Sbp\Search\Searchers\Searcher; use Blomstra\Search\Searchers\Searcher;
use Elasticsearch\Client; use Elasticsearch\Client;
use Flarum\Api\Controller\ListDiscussionsController; use Flarum\Api\Controller\ListDiscussionsController;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;
@ -37,7 +37,7 @@ use Illuminate\Support\Str;
use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Message\ServerRequestInterface;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
use Spatie\ElasticsearchQueryBuilder\Builder; use Spatie\ElasticsearchQueryBuilder\Builder;
use Sbp\Search\Elasticsearch\BoolQuery; use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery; use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;
use Spatie\ElasticsearchQueryBuilder\Sorts\Sort; use Spatie\ElasticsearchQueryBuilder\Sorts\Sort;
use Tobscure\JsonApi\Document; use Tobscure\JsonApi\Document;
@ -78,18 +78,27 @@ class SearchController extends ListDiscussionsController
$offset = $this->extractOffset($request); $offset = $this->extractOffset($request);
$include = array_merge($this->extractInclude($request), ['state']); $include = array_merge($this->extractInclude($request), ['state']);
// Autocomplete path: fast prefix match on title.autocomplete only. $query = BoolQuery::create()
if (!empty($filters['autocomplete']) && !empty($search)) { // Always restrict to discussion documents; posts are only searched via has_child.
return $this->handleAutocomplete($search, $actor, $limit, $offset, $include, $document, $request); ->add(TermQuery::create('join_field', 'discussion'), 'filter');
if (!empty($search)) {
$query->add($this->buildTextQuery($search, $actor));
} }
$this->addFilters($query, $actor, $filters);
$builder = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->size($limit + 1)
->from($offset)
->addQuery($query);
$knownSortFields = array_merge(array_values($this->translateSort), ['rawId']); $knownSortFields = array_merge(array_values($this->translateSort), ['rawId']);
$logger = resolve(LoggerInterface::class); $logger = resolve(LoggerInterface::class);
$phpSortField = null; $phpSortField = null;
$phpSortDir = 'desc'; $phpSortDir = 'desc';
$needsScoring = true;
$sorts = [];
foreach ($this->extractSort($request) as $field => $direction) { foreach ($this->extractSort($request) as $field => $direction) {
$translated = $this->translateSort[$field] ?? $field; $translated = $this->translateSort[$field] ?? $field;
@ -99,8 +108,7 @@ class SearchController extends ListDiscussionsController
continue; continue;
} }
$sorts[] = new Sort($translated, $direction); $builder->addSort(new Sort($translated, $direction));
$needsScoring = false;
if ($phpSortField === null && $translated !== 'rawId') { if ($phpSortField === null && $translated !== 'rawId') {
$phpSortField = $translated; $phpSortField = $translated;
@ -108,45 +116,7 @@ class SearchController extends ListDiscussionsController
} }
} }
// Default to latest when no explicit sort is requested. This lets has_child use $response = $builder->search();
// score_mode:none, which skips child scoring entirely and lets ES short-circuit
// early on large corpora. Relevance is still available via sort=relevant if added.
if (empty($sorts)) {
$needsScoring = false;
$phpSortField = 'updated_at';
$phpSortDir = 'desc';
$sorts[] = new Sort('updated_at', 'desc');
}
$query = BoolQuery::create()
// Always restrict to discussion documents; posts are only searched via has_child.
->add(TermQuery::create('join_field', 'discussion'), 'filter');
if (!empty($search)) {
$query->add($this->buildTextQuery($search, $actor, $needsScoring));
}
$this->addFilters($query, $actor, $filters);
$builder = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->addQuery($query);
foreach ($sorts as $sort) {
$builder->addSort($sort);
}
// track_total_hits: false lets ES stop counting once it has collected
// enough results in sort order, avoiding a full-index count on every query.
$payload = $builder->getPayload();
$payload['track_total_hits'] = false;
$response = $this->elastic->search([
'index' => resolve('blomstra.search.elastic_index'),
'size' => $limit + 1,
'from' => $offset,
'body' => $payload,
]);
Discussion::setStateUser($actor); Discussion::setStateUser($actor);
@ -222,145 +192,18 @@ class SearchController extends ListDiscussionsController
return $discussions; return $discussions;
} }
/**
* Fast autocomplete path: match prefix n-grams on title.autocomplete only.
* No has_child, no phrase scoring a single MatchQuery suffices.
*/
protected function handleAutocomplete(
string $search,
User $actor,
int $limit,
int $offset,
array $include,
Document $document,
ServerRequestInterface $request
): mixed {
$query = BoolQuery::create()
->add(TermQuery::create('join_field', 'discussion'), 'filter');
$query->add($this->buildAutocompleteQuery($search), 'must');
$this->addFilters($query, $actor, []);
$payload = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->addQuery($query)
->addSort(new Sort('updated_at', 'desc'))
->getPayload();
$response = $this->elastic->search([
'index' => resolve('blomstra.search.elastic_index'),
'size' => $limit + 1,
'from' => $offset,
'body' => $payload,
]);
Discussion::setStateUser($actor);
$results = Collection::make(Arr::get($response, 'hits.hits'))
->map(fn ($hit) => [
'discussion_id' => Str::after($hit['_id'], 'discussions:'),
'most_relevant_post_id' => null,
'weight' => Arr::get($hit, 'sort.0', 0),
]);
$document->addPaginationLinks(
$this->uri->to('api')->route('blomstra.search', ['type' => 'discussions']),
$request->getQueryParams(),
$offset,
$limit,
$results->count() > $limit ? null : 0
);
$results = $results->take($limit);
$discussions = Discussion::query()
->when(
$actor->isGuest() || !$actor->hasPermission('discussion.hide'),
fn ($q) => $q->whereNull('hidden_at')
)
->whereIn('id', $results->pluck('discussion_id')->filter())
->get()
->each(function (Discussion $discussion) use ($results) {
$result = $results->firstWhere('discussion_id', $discussion->id);
$discussion->most_relevant_post_id = $result['most_relevant_post_id'] ?? $discussion->first_post_id;
$discussion->weight = $result['weight'] ?? 0;
})
->sortByDesc('updated_at')
->unique();
$this->loadRelations($discussions, $include);
return $discussions;
}
protected function buildAutocompleteQuery(string $search): BoolQuery
{
$query = BoolQuery::create();
// Title prefix match (stronger)
$query->add(
(new MatchQuery('title.autocomplete', $search))
->operator('and')
->boost(1.0),
'should'
);
// Content prefix match (weaker)
$query->add(
(new MatchQuery('content.autocomplete', $search))
->operator('and')
->boost(0.4),
'should'
);
$query->minimumShouldMatch(1);
return $query;
}
/** /**
* Build the text-matching portion of the query. * Build the text-matching portion of the query.
* *
* Discussion titles are matched directly (filtered to join_field=discussion). * Discussion titles are matched directly (filtered to join_field=discussion).
* Post bodies are matched via has_child. When $needsScoring is true (no explicit * Post bodies are matched via has_child with score_mode=sum so discussions
* sort, i.e. relevance ordering), score_mode=sum accumulates child scores onto the * with many matching posts score higher than those with a single strong match.
* parent so that discussions with many strongly-matching posts rank higher. When * inner_hits returns the best-scoring post for use as mostRelevantPost.
* false (any explicit field sort), score_mode=none skips scoring entirely ES only
* checks whether a matching child exists, which is significantly cheaper on large
* corpora.
* inner_hits returns the best-matching post for use as mostRelevantPost.
* Hidden posts are only included in matching for users with post.hide permission. * Hidden posts are only included in matching for users with post.hide permission.
*/ */
protected function buildTextQuery(string $search, User $actor, bool $needsScoring = false): BoolQuery protected function buildTextQuery(string $search, User $actor): BoolQuery
{ {
$textQuery = BoolQuery::create(); $textQuery = BoolQuery::create();
// Strong direct title match for the full search page.
$textQuery->add(
(new MatchQuery('title', $search))
->operator('and')
->boost(4.0),
'should'
);
// Low-weight prefix match on title.autocomplete for partial title hits.
$textQuery->add(
(new MatchQuery('title.autocomplete', $search))
->operator('and')
->boost(0.35),
'should'
);
// Low-weight prefix match on title.autocomplete for the full search page.
// Exact / normal title matches should still rank higher than prefix-only hits.
$textQuery->add(
(new MatchQuery('title.autocomplete', $search))
->operator('and')
->boost(0.35),
'should'
);
if ($this->discussionSearcher?->enabled()) { if ($this->discussionSearcher?->enabled()) {
$textQuery->add($this->buildShouldClauses($search, $this->discussionSearcher->boost()), 'should'); $textQuery->add($this->buildShouldClauses($search, $this->discussionSearcher->boost()), 'should');
@ -371,15 +214,11 @@ class SearchController extends ListDiscussionsController
// Guests and non-moderators may not see hidden posts; exclude them from child matching. // Guests and non-moderators may not see hidden posts; exclude them from child matching.
if ($actor->isGuest() || !$actor->hasPermission('post.hide')) { if ($actor->isGuest() || !$actor->hasPermission('post.hide')) {
$postQuery->add(TermQuery::create('is_hidden', 'false'), 'filter'); $postQuery->add(TermQuery::create('is_hidden', false), 'filter');
} }
// Without minimum_should_match, ES default MSM is 0 when a filter clause is present,
// causing has_child to score every non-hidden post instead of only matching ones.
$postQuery->minimumShouldMatch(1);
$textQuery->add( $textQuery->add(
HasChildQuery::create('post', $postQuery, $needsScoring ? 'sum' : 'none')->withInnerHits(), HasChildQuery::create('post', $postQuery)->withInnerHits(),
'should' 'should'
); );
} }
@ -396,7 +235,7 @@ class SearchController extends ListDiscussionsController
} }
if ($this->matchWords) { if ($this->matchWords) {
$should->add((new MatchQuery('content', $search))->operator('and')->boost(1.8 * $boost), 'should'); $should->add((new MatchQuery('content', $search))->operator('and')->boost(1.8 * $boost), 'should');
$should->add((new MatchQuery('content.autocomplete', $search))->operator('and')->boost(0.30 * $boost), 'should'); $should->add((new MatchQuery('content', $search))->operator('or')->boost(0.8 * $boost), 'should');
} }
return $should; return $should;
@ -462,12 +301,8 @@ class SearchController extends ListDiscussionsController
$search = Arr::get($filters, 'q'); $search = Arr::get($filters, 'q');
if ($search) { if ($search) {
// Strip Flarum gambit operators (tag:foo, author:bar, is:private, etc.)
// before passing to ES. These are structural filters handled separately;
// leaving them in causes operator:and to require the gambit tokens to
// appear literally in post content, producing zero results.
$q = collect(explode(' ', $search)) $q = collect(explode(' ', $search))
->filter(fn (string $part) => !preg_match('/^\w+:/', $part)) ->filter(fn (string $part) => $part !== 'is:private')
->filter() ->filter()
->join(' '); ->join(' ');

View File

@ -10,13 +10,12 @@
* *
*/ */
namespace Sbp\Search\Commands; namespace Blomstra\Search\Commands;
use Sbp\Search\Jobs\Job; use Blomstra\Search\Jobs\Job;
use Sbp\Search\Jobs\UpdateSearchJob; use Blomstra\Search\Jobs\UpdateSearchJob;
use Sbp\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Elasticsearch\Client; use Elasticsearch\Client;
use Elasticsearch\Common\Exceptions\ElasticsearchException;
use Flarum\Settings\SettingsRepositoryInterface; use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Container\Container;
@ -36,15 +35,11 @@ class BuildCommand extends Command
* read back into blomstra-search.index-compatible by saveIndexedConfig so the * read back into blomstra-search.index-compatible by saveIndexedConfig so the
* value always reflects the live index including after a promote/rollback. * value always reflects the live index including after a promote/rollback.
*/ */
public const INDEX_COMPAT_VERSION = 'v3'; public const INDEX_COMPAT_VERSION = 'v2';
/** Matches Flarum's Search::MIN_SEARCH_LEN — the default minimum query length. */ /** Matches Flarum's Search::MIN_SEARCH_LEN — the default minimum query length. */
public const DEFAULT_MIN_SEARCH_LENGTH = 3; public const DEFAULT_MIN_SEARCH_LENGTH = 3;
/** N-gram bounds for the title autocomplete subfield (not admin-configurable). */
private const TITLE_MIN_GRAM = 3;
private const TITLE_MAX_GRAM = 20;
protected $signature = 'blomstra:search:index protected $signature = 'blomstra:search:index
{action? : build | promote | rollback | discard | mapping | fill} {action? : build | promote | rollback | discard | mapping | fill}
{--fresh} {--fresh}
@ -160,9 +155,7 @@ HELP;
$this->line(''); $this->line('');
$this->line('Choose one of:'); $this->line('Choose one of:');
$this->line(' blomstra:search:index build --resume Continue from where it left off'); $this->line(' blomstra:search:index build --resume Continue from where it left off');
$this->line(' blomstra:search:index build --resume --staging Continue and keep in staging when done');
$this->line(' blomstra:search:index build --fresh Drop this build and start completely fresh'); $this->line(' blomstra:search:index build --fresh Drop this build and start completely fresh');
$this->line(' blomstra:search:index build --fresh --staging Start fresh and keep in staging when done');
return; return;
} }
@ -366,7 +359,17 @@ HELP;
$rangeTo = $continueAt; $rangeTo = $continueAt;
if ($seedMissing) { if ($seedMissing) {
$seeded = $this->queryIndexedIds($client, $targetIndex, $seeder->joinRelation(), $rangeFrom, $rangeTo); $response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $seeder->joinRelation()))
)
->search();
$seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} }
/** @var Collection $collection */ /** @var Collection $collection */
@ -496,65 +499,46 @@ HELP;
$analysis = Arr::get($settingsResponse, "$indexName.settings.index.analysis", []); $analysis = Arr::get($settingsResponse, "$indexName.settings.index.analysis", []);
$analyzer = Arr::get($analysis, 'analyzer.flarum_analyzer.type', 'english'); $analyzer = Arr::get($analysis, 'analyzer.flarum_analyzer.type', 'english');
$stemExclusion = Arr::get($analysis, 'analyzer.flarum_analyzer.stem_exclusion', []); $minGram = (int) Arr::get($analysis, 'filter.partial_search_filter.min_gram', self::DEFAULT_MIN_SEARCH_LENGTH);
$mappingResponse = $client->indices()->getMapping(['index' => $indexName]); $mappingResponse = $client->indices()->getMapping(['index' => $indexName]);
$compatVersion = Arr::get($mappingResponse, "$indexName.mappings._meta.index_compat_version"); $compatVersion = Arr::get($mappingResponse, "$indexName.mappings._meta.index_compat_version");
$settings->set('blomstra-search.indexed-analyzer', $analyzer); $settings->set('blomstra-search.indexed-analyzer', $analyzer);
$settings->set('blomstra-search.indexed-stem-exclusion', implode("\n", $stemExclusion)); $settings->set('blomstra-search.indexed-min-search-length', $minGram);
$settings->set('blomstra-search.index-compatible', $compatVersion); $settings->set('blomstra-search.index-compatible', $compatVersion);
} }
protected function buildIndexSettings(SettingsRepositoryInterface $settings): array protected function buildIndexSettings(SettingsRepositoryInterface $settings): array
{ {
$language = $settings->get('blomstra-search.analyzer-language') ?: 'english'; $language = $settings->get('blomstra-search.analyzer-language') ?: 'english';
$minGram = max(1, (int) ($settings->get('blomstra-search.min-search-length') ?: self::DEFAULT_MIN_SEARCH_LENGTH));
$maxGram = 10;
$raw = $settings->get('blomstra-search.stem-exclusion', ''); if ($minGram >= $maxGram) {
$stemExclusion = array_values(array_filter(array_map('trim', explode("\n", $raw)))); $this->error("min_gram ($minGram) must be less than max_gram ($maxGram). Using default.");
$minGram = self::DEFAULT_MIN_SEARCH_LENGTH;
}
if ($language === 'cjk') {
// CJK uses the built-in bigram analyzer; no autocomplete subfield.
return [ return [
'index.max_ngram_diff' => $maxGram - $minGram,
'analysis' => [ 'analysis' => [
'analyzer' => [ 'analyzer' => [
'flarum_analyzer' => ['type' => 'cjk'], 'flarum_analyzer' => [
'type' => $language,
],
'flarum_analyzer_partial' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => ['lowercase', 'partial_search_filter'],
], ],
], ],
];
}
$analyzerConfig = ['type' => $language];
if (!empty($stemExclusion)) {
$analyzerConfig['stem_exclusion'] = $stemExclusion;
}
return [
'index.max_ngram_diff' => self::TITLE_MAX_GRAM - self::TITLE_MIN_GRAM,
'analysis' => [
'filter' => [ 'filter' => [
'title_autocomplete_filter' => [ 'partial_search_filter' => [
'type' => 'edge_ngram', 'type' => 'ngram',
'min_gram' => self::TITLE_MIN_GRAM, 'min_gram' => $minGram,
'max_gram' => self::TITLE_MAX_GRAM, 'max_gram' => $maxGram,
'token_chars' => ['letter', 'digit'], 'token_chars' => ['letter', 'digit', 'symbol'],
],
],
'analyzer' => [
// Symmetric language analyzer: used for BOTH index and search on content + title.
'flarum_analyzer' => $analyzerConfig,
// Title autocomplete index-time: prefix n-grams, no stemming.
'flarum_title_autocomplete' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => ['lowercase', 'title_autocomplete_filter'],
],
// Title autocomplete search-time: just lowercase — predictable on partial words.
'flarum_title_search' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => ['lowercase'],
], ],
], ],
], ],
@ -563,101 +547,27 @@ HELP;
protected function mappingProperties(): array protected function mappingProperties(): array
{ {
$language = resolve(SettingsRepositoryInterface::class)->get('blomstra-search.analyzer-language') ?: 'english';
$titleMapping = [
'type' => 'text',
'analyzer' => 'flarum_analyzer',
];
// Only add the autocomplete subfield for non-CJK languages.
if ($language !== 'cjk') {
$titleMapping['fields'] = [
'autocomplete' => [
'type' => 'text',
'analyzer' => 'flarum_title_autocomplete',
'search_analyzer' => 'flarum_title_search',
],
];
}
$contentMapping = [
'type' => 'text',
'analyzer' => 'flarum_analyzer',
];
// Only add the autocomplete subfield for non-CJK languages.
if ($language !== 'cjk') {
$contentMapping['fields'] = [
'autocomplete' => [
'type' => 'text',
'analyzer' => 'flarum_title_autocomplete',
'search_analyzer' => 'flarum_title_search',
],
];
}
return [ return [
'_meta' => ['index_compat_version' => self::INDEX_COMPAT_VERSION], '_meta' => ['index_compat_version' => self::INDEX_COMPAT_VERSION],
'properties' => [ 'properties' => [
'join_field' => ['type' => 'join', 'relations' => ['discussion' => 'post']], 'join_field' => ['type' => 'join', 'relations' => ['discussion' => 'post']],
'discussion_id' => ['type' => 'integer'], 'discussion_id' => ['type' => 'integer'],
'content' => $contentMapping, 'content' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'],
'title' => $titleMapping,
'rawId' => ['type' => 'integer'], 'rawId' => ['type' => 'integer'],
'created_at' => ['type' => 'date'], 'created_at' => ['type' => 'date'],
'updated_at' => ['type' => 'date'], 'updated_at' => ['type' => 'date'],
'is_private' => ['type' => 'boolean'], 'is_private' => ['type' => 'boolean'],
'user_id' => ['type' => 'integer'],
'groups' => ['type' => 'keyword'],
'comment_count' => ['type' => 'integer'],
'tags' => ['type' => 'keyword'],
'is_sticky' => ['type' => 'boolean'], 'is_sticky' => ['type' => 'boolean'],
'groups' => ['type' => 'integer'],
'tags' => ['type' => 'integer'],
'recipient_groups' => ['type' => 'integer'],
'recipient_users' => ['type' => 'integer'],
'comment_count' => ['type' => 'integer'],
'view_count' => ['type' => 'integer'],
'is_hidden' => ['type' => 'boolean'], 'is_hidden' => ['type' => 'boolean'],
], ],
]; ];
} }
/**
* Query ES for rawIds already indexed in $targetIndex for the given $joinRelation and ID range.
* Retries on transient ES failures (NoNodesAvailableException / other ElasticsearchException)
* by sleeping past the StaticNoPingConnectionPool dead-node timeout (default 60 s) before
* each retry, giving the pool a chance to resurface the node.
*/
protected function queryIndexedIds(
Client $client,
string $targetIndex,
string $joinRelation,
int $rangeFrom,
int $rangeTo,
int $maxRetries = 10
): array {
$attempt = 0;
while (true) {
try {
$response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $joinRelation))
)
->search();
return Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} catch (ElasticsearchException $e) {
$attempt++;
if ($attempt >= $maxRetries) {
throw $e;
}
$this->warn("ES error on range {$rangeFrom}{$rangeTo} (attempt {$attempt}/{$maxRetries}): {$e->getMessage()}. Waiting 65 s before retry…");
sleep(65); // outlast the StaticNoPingConnectionPool dead-node window (default 60 s)
}
}
}
protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int
{ {

View File

@ -1,41 +0,0 @@
<?php
/*
* This file is part of blomstra/search.
*
* Copyright (c) 2022 Blomstra Ltd.
*
* For the full copyright and license information, please view the LICENSE.md
* file that was distributed with this source code.
*
*/
namespace Sbp\Search\Elasticsearch;
class BoolQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery
{
protected ?int $minimumShouldMatch = null;
public static function create(): static
{
return new self();
}
public function minimumShouldMatch(int $minimum): static
{
$this->minimumShouldMatch = $minimum;
return $this;
}
public function toArray(): array
{
$array = parent::toArray();
if ($this->minimumShouldMatch !== null) {
$array['bool']['minimum_should_match'] = $this->minimumShouldMatch;
}
return $array;
}
}

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\Query;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery; use Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
class MatchQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery class MatchQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery
{ {
@ -62,10 +62,6 @@ class MatchQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery
$query['match'][$this->field]['boost'] = $this->boost; $query['match'][$this->field]['boost'] = $this->boost;
$query['match'][$this->field]['zero_terms_query'] = $this->zeroTerms ? 'all' : 'none'; $query['match'][$this->field]['zero_terms_query'] = $this->zeroTerms ? 'all' : 'none';
if ($this->analyzer !== null) {
$query['match'][$this->field]['analyzer'] = $this->analyzer;
}
return $query; return $query;
} }
} }

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\Query;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\Query;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
class WildcardQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\WildcardQuery class WildcardQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\WildcardQuery
{ {

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Exceptions; namespace Blomstra\Search\Exceptions;
use Throwable; use Throwable;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Elasticsearch\Client; use Elasticsearch\Client;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;

View File

@ -10,11 +10,11 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Sbp\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Flarum\Queue\AbstractJob; use Flarum\Queue\AbstractJob;
use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Collection;
abstract class Job extends AbstractJob abstract class Job extends AbstractJob
{ {

View File

@ -10,11 +10,10 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Sbp\Search\Exceptions\SeedingException; use Blomstra\Search\Exceptions\SeedingException;
use Elasticsearch\Client; use Elasticsearch\Client;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
@ -51,11 +50,10 @@ class UpdateSearchJob extends Job
$items = Arr::get($response, 'items'); $items = Arr::get($response, 'items');
$failed = array_filter($items, fn ($item) => isset($item['index']['error'])); $error = Arr::get(Arr::first($items), 'index.error.reason');
$error = Arr::get(Arr::first($failed), 'index.error.reason', 'unknown error');
throw new SeedingException( throw new SeedingException(
"Failed to seed: $error (" . count($failed) . '/' . count($items) . ' items failed)', "Failed to seed: $error",
$items $items
); );
} }

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Elasticsearch\Client; use Elasticsearch\Client;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;

View File

@ -10,12 +10,12 @@
* *
*/ */
namespace Sbp\Search; namespace Blomstra\Search;
use Sbp\Search\Jobs\DeletingJob; use Blomstra\Search\Jobs\DeletingJob;
use Sbp\Search\Jobs\Job; use Blomstra\Search\Jobs\Job;
use Sbp\Search\Jobs\UpdateSearchJob; use Blomstra\Search\Jobs\UpdateSearchJob;
use Sbp\Search\Jobs\ViewsSearchJob; use Blomstra\Search\Jobs\ViewsSearchJob;
use Elasticsearch\Client as Elastic; use Elasticsearch\Client as Elastic;
use Elasticsearch\ClientBuilder; use Elasticsearch\ClientBuilder;
use Flarum\Api\Client; use Flarum\Api\Client;
@ -26,7 +26,7 @@ use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Container\Container;
use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Contracts\Queue\Queue; use Illuminate\Contracts\Queue\Queue;
use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Collection;
use Laminas\Stratigility\MiddlewarePipe; use Laminas\Stratigility\MiddlewarePipe;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
@ -47,13 +47,7 @@ class Provider extends AbstractServiceProvider
$this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) { $this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) {
$builder = ClientBuilder::create() $builder = ClientBuilder::create()
->setHosts([$settings->get('blomstra-search.elastic-endpoint')]) ->setHosts([$settings->get('blomstra-search.elastic-endpoint')]);
->setConnectionParams([
'client' => [
'connect_timeout' => 2, // fail fast if ES is unreachable
'timeout' => 10, // allow time for complex queries
],
]);
if ($config->inDebugMode()) { if ($config->inDebugMode()) {
$builder->setLogger($container->make(LoggerInterface::class)); $builder->setLogger($container->make(LoggerInterface::class));

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Save; namespace Blomstra\Search\Save;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Searchers; namespace Blomstra\Search\Searchers;
use Sbp\Search\Seeders\CommentSeeder; use Blomstra\Search\Seeders\CommentSeeder;
class CommentPostSearcher extends Searcher class CommentPostSearcher extends Searcher
{ {

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Searchers; namespace Blomstra\Search\Searchers;
use Sbp\Search\Seeders\DiscussionSeeder; use Blomstra\Search\Seeders\DiscussionSeeder;
class DiscussionSearcher extends Searcher class DiscussionSearcher extends Searcher
{ {

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Searchers; namespace Blomstra\Search\Searchers;
use Sbp\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Flarum\Settings\SettingsRepositoryInterface; use Flarum\Settings\SettingsRepositoryInterface;
abstract class Searcher abstract class Searcher

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Seeders; namespace Blomstra\Search\Seeders;
use Sbp\Search\Save\Document; use Blomstra\Search\Save\Document;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Api\Serializer\PostSerializer; use Flarum\Api\Serializer\PostSerializer;
use Flarum\Discussion\Discussion; use Flarum\Discussion\Discussion;

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Seeders; namespace Blomstra\Search\Seeders;
use Sbp\Search\Save\Document; use Blomstra\Search\Save\Document;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Discussion\Discussion; use Flarum\Discussion\Discussion;
use Flarum\Discussion\Event as Core; use Flarum\Discussion\Event as Core;
@ -24,7 +24,7 @@ use FoF\Byobu\Events as Byobu;
use FoF\DiscussionViews\Events\DiscussionWasViewed; use FoF\DiscussionViews\Events\DiscussionWasViewed;
use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection; use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
class DiscussionSeeder extends Seeder class DiscussionSeeder extends Seeder
@ -70,7 +70,7 @@ class DiscussionSeeder extends Seeder
{ {
$events->listen([ $events->listen([
// flarum/core events // flarum/core events
Core\Started::class, Core\Restored::class, Core\Renamed::class, Core\Started::class, Core\Restored::class,
// fof/byobu discussion recipients events. // fof/byobu discussion recipients events.
Byobu\DiscussionMadePublic::class, Byobu\RemovedSelf::class, Byobu\RecipientsChanged::class, Byobu\DiscussionMadePublic::class, Byobu\RemovedSelf::class, Byobu\RecipientsChanged::class,
], function ($event) use ($callable) { ], function ($event) use ($callable) {
@ -121,7 +121,6 @@ class DiscussionSeeder extends Seeder
'id' => $this->type().':'.$model->id, 'id' => $this->type().':'.$model->id,
'rawId' => $model->id, 'rawId' => $model->id,
'content' => $model->title, 'content' => $model->title,
'title' => $model->title,
'created_at' => $model->created_at?->toAtomString(), 'created_at' => $model->created_at?->toAtomString(),
'updated_at' => $model->last_posted_at?->toAtomString(), 'updated_at' => $model->last_posted_at?->toAtomString(),
'is_private' => $model->is_private, 'is_private' => $model->is_private,

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Seeders; namespace Blomstra\Search\Seeders;
use Sbp\Search\Save\Document; use Blomstra\Search\Save\Document;
use Flarum\Extension\ExtensionManager; use Flarum\Extension\ExtensionManager;
use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Builder;