Compare commits

..

No commits in common. "main" and "0.2.0" have entirely different histories.
main ... 0.2.0

40 changed files with 226 additions and 623 deletions

1
.gitignore vendored
View File

@ -2,4 +2,3 @@ js/node_modules
js/dist js/dist
vendor/ vendor/
composer.lock composer.lock
analysis/*

View File

@ -18,32 +18,9 @@ fulltext search with one that is completely relying on the proven elasticsearch
## Installation ## Installation
```bash ```bash
composer require sbp/flarum-search composer require blomstra/search:*
``` ```
## Notes
This fork is now maintained as `sbp/flarum-search`.
Intentional compatibility leftovers:
- settings keys remain `blomstra-search.*`
- some translation keys remain `blomstra-search.*`
- CLI command names may still remain `blomstra:*` until a later migration
Search behavior:
- title autocomplete enabled
- fuzzy title search enabled
- exact + fuzzy content search on full search page
- no content autocomplete in dropdown (performance decision)
``
Repository can be found here: https://gitea.muehlberger.net/sbp-jm/sbp-flarum-search
Enable the extension in the admin area and configure the Elasticsearch endpoint and index name in the extension settings. Enable the extension in the admin area and configure the Elasticsearch endpoint and index name in the extension settings.
## Setting up the index ## Setting up the index

View File

@ -1,5 +1,5 @@
{ {
"name": "sbp/flarum-search", "name": "blomstra/search",
"description": "Replaces Flarum search with one powered by an elastic search server.", "description": "Replaces Flarum search with one powered by an elastic search server.",
"keywords": [ "keywords": [
"extension", "extension",
@ -11,20 +11,16 @@
"elasticsearch" "elasticsearch"
], ],
"support": { "support": {
"email": "mueh+git@sbp.de", "email": "helpdesk@blomstra.net",
"forum": "https://blomstra.community/t/ext-search" "forum": "https://blomstra.community/t/ext-search"
}, },
"type": "flarum-extension", "type": "flarum-extension",
"license": "MIT", "license": "MIT",
"authors": [ "authors": [
{
"name": "sbp",
"email": "info@sbp.de"
},
{ {
"name": "Team Blomstra", "name": "Team Blomstra",
"email": "development@blomstra.net", "email": "development@blomstra.net",
"role": "Original Developer" "role": "Developer"
} }
], ],
"require": { "require": {
@ -47,7 +43,7 @@
}, },
"autoload": { "autoload": {
"psr-4": { "psr-4": {
"Sbp\\Search\\": "src/" "Blomstra\\Search\\": "src/"
} }
} }
} }

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search; namespace Blomstra\Search;
use Flarum\Extend as Flarum; use Flarum\Extend as Flarum;
@ -26,16 +26,13 @@ return [
new Flarum\Locales(__DIR__.'/resources/locale'), new Flarum\Locales(__DIR__.'/resources/locale'),
(new Flarum\Routes('api')) (new Flarum\Routes('api'))
->get('/sbp/search/{type}', 'sbp.search', Api\Controllers\SearchController::class) ->get('/blomstra/search/{type}', 'blomstra.search', Api\Controllers\SearchController::class)
->put('/sbp/search/index', 'sbp.search.index', Api\Controllers\IndexController::class), ->put('/blomstra/search/index', 'blomstra.search.index', Api\Controllers\IndexController::class),
(new Flarum\Console()) (new Flarum\Console())
->command(Commands\BuildCommand::class), ->command(Commands\BuildCommand::class),
(new Flarum\Settings()) (new Flarum\Settings())
->default('blomstra-search.search-discussion-subjects', true) ->default('blomstra-search.search-discussion-subjects', true)
->default('blomstra-search.search-post-bodies', true) ->default('blomstra-search.search-post-bodies', true)
->default('blomstra-search.min-search-length', Commands\BuildCommand::DEFAULT_MIN_SEARCH_LENGTH) ->default('blomstra-search.min-search-length', Commands\BuildCommand::DEFAULT_MIN_SEARCH_LENGTH),
->default('blomstra-search.stem-exclusion', '')
->serializeToForum('blomstraSearchMinLength', 'blomstra-search.min-search-length', 'intval')
->serializeToForum('blomstraSearchPostBodies', 'blomstra-search.search-post-bodies', fn ($v) => (bool) $v, true),
]; ];

2
js/dist/admin.js vendored
View File

@ -1,2 +1,2 @@
(()=>{var a={n:t=>{var e=t&&t.__esModule?()=>t.default:()=>t;return a.d(e,{a:e}),e},d:(t,e)=>{for(var n in e)a.o(e,n)&&!a.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:e[n]})},o:(a,t)=>Object.prototype.hasOwnProperty.call(a,t)};(()=>{"use strict";function t(a,e){return t=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(a,t){return a.__proto__=t,a},t(a,e)}const e=flarum.core.compat["admin/app"];var n=a.n(e);const r=flarum.core.compat["common/extend"],s=flarum.core.compat["admin/components/DashboardPage"];var i=a.n(s);const l=flarum.core.compat["admin/components/DashboardWidget"];var o=a.n(l);const c=flarum.core.compat["common/components/Alert"];var h=a.n(c),d=function(a){var e,r;function s(){return a.apply(this,arguments)||this}r=a,(e=s).prototype=Object.create(r.prototype),e.prototype.constructor=e,t(e,r);var i=s.prototype;return i.className=function(){return"ReindexWarningWidget"},i.content=function(){return m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle",title:n().translator.trans("blomstra-search.admin.reindex-required.title")},n().translator.trans("blomstra-search.admin.reindex-required.detail"))},s}(o());n().initializers.add("sbp-search",function(){var a=n().data.settings["blomstra-search.active-index"],t=n().data.settings["blomstra-search.index-compatible"];a&&"v3"!==t&&(0,r.extend)(i().prototype,"availableWidgets",function(a){a.add("blomstra-search-reindex",m(d),110)}),n().extensionData.for("sbp-search").registerSetting(function(){var t=n().data.settings["blomstra-search.indexed-analyzer"];if(!a||!t)return null;var e=this.setting("blomstra-search.analyzer-language")()||"english",r=this.setting("blomstra-search.stem-exclusion")()||"",s=n().data.settings["blomstra-search.indexed-stem-exclusion"]||"";return e===t&&r===s?null:m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle"},n().translator.trans("blomstra-search.admin.index-settings-changed"))}).registerSetting({setting:"blomstra-search.elastic-endpoint",label:n().translator.trans("blomstra-search.admin.elastic-endpoint"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-username",label:n().translator.trans("blomstra-search.admin.elastic-username"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-password",label:n().translator.trans("blomstra-search.admin.elastic-password"),type:"password"}).registerSetting({setting:"blomstra-search.elastic-index",label:n().translator.trans("blomstra-search.admin.elastic-index"),default:"flarum",type:"input"}).registerSetting({setting:"blomstra-search.search-discussion-subjects",label:n().translator.trans("blomstra-search.admin.search-discussion-subjects"),type:"switch"}).registerSetting({setting:"blomstra-search.search-post-bodies",label:n().translator.trans("blomstra-search.admin.search-post-bodies"),type:"switch"}).registerSetting({setting:"blomstra-search.analyzer-language",label:n().translator.trans("blomstra-search.admin.analyzer.label"),help:n().translator.trans("blomstra-search.admin.analyzer.help"),type:"select",options:{arabic:"Arabic",armenian:"Armenian",basque:"Basque",bengali:"Bengali",brazilian:"Brazilian Portuguese",bulgarian:"Bulgarian",catalan:"Catalan",cjk:"CJK (Chinese, Japanese, Korean)",czech:"Czech",danish:"Danish",dutch:"Dutch",english:"English",estonian:"Estonian",finnish:"Finnish",french:"French",galician:"Galician",german:"German",greek:"Greek",hindi:"Hindi",hungarian:"Hungarian",indonesian:"Indonesian",irish:"Irish",italian:"Italian",latvian:"Latvian",lithuanian:"Lithuanian",norwegian:"Norwegian",persian:"Persian",portuguese:"Portuguese",romanian:"Romanian",russian:"Russian",sorani:"Sorani (Kurdish)",spanish:"Spanish",standard:"Standard (generic)",swedish:"Swedish",turkish:"Turkish",thai:"Thai"},default:"english"}).registerSetting(function(){var a="cjk"===(this.setting("blomstra-search.analyzer-language")()||"english");return this.buildSettingComponent({setting:"blomstra-search.stem-exclusion",type:"textarea",label:n().translator.trans("blomstra-search.admin.settings.stem-exclusion.label"),help:n().translator.trans("blomstra-search.admin.settings.stem-exclusion.help"),disabled:a})}).registerSetting(function(){var a="cjk"===(this.setting("blomstra-search.analyzer-language")()||"english");return this.buildSettingComponent({setting:"blomstra-search.min-search-length",label:n().translator.trans("blomstra-search.admin.min-search-length.label"),help:n().translator.trans("blomstra-search.admin.min-search-length.help"),type:"select",options:{1:"1",2:"2",3:"3",4:"4"},default:n().data.settings["blomstra-search.min-search-length"],disabled:a})})})})(),module.exports={}})(); (()=>{var t={n:a=>{var e=a&&a.__esModule?()=>a.default:()=>a;return t.d(e,{a:e}),e},d:(a,e)=>{for(var r in e)t.o(e,r)&&!t.o(a,r)&&Object.defineProperty(a,r,{enumerable:!0,get:e[r]})},o:(t,a)=>Object.prototype.hasOwnProperty.call(t,a)};(()=>{"use strict";function a(t,e){return a=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,a){return t.__proto__=a,t},a(t,e)}const e=flarum.core.compat["admin/app"];var r=t.n(e);const n=flarum.core.compat["common/extend"],s=flarum.core.compat["admin/components/DashboardPage"];var i=t.n(s);const o=flarum.core.compat["admin/components/DashboardWidget"];var l=t.n(o);const c=flarum.core.compat["common/components/Alert"];var h=t.n(c),d=function(t){var e,n;function s(){return t.apply(this,arguments)||this}n=t,(e=s).prototype=Object.create(n.prototype),e.prototype.constructor=e,a(e,n);var i=s.prototype;return i.className=function(){return"ReindexWarningWidget"},i.content=function(){return m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle",title:r().translator.trans("blomstra-search.admin.reindex-required.title")},r().translator.trans("blomstra-search.admin.reindex-required.detail"))},s}(l());r().initializers.add("blomstra-search",function(){var t=r().data.settings["blomstra-search.active-index"],a=r().data.settings["blomstra-search.index-compatible"];t&&"v2"!==a&&(0,n.extend)(i().prototype,"availableWidgets",function(t){t.add("blomstra-search-reindex",m(d),110)});var e=new Map;["arabic","armenian","basque","bengali","brazilian","bulgarian","catalan","cjk","czech","danish","dutch","english","estonian","finnish","french","galician","german","greek","hindi","hungarian","indonesian","irish","italian","latvian","lithuanian","norwegian","persian","portuguese","romanian","russian","sorani","spanish","swedish","turkish","thai"].forEach(function(t){e.set(t,t)}),r().extensionData.for("blomstra-search").registerSetting(function(){var a=r().data.settings["blomstra-search.indexed-analyzer"];if(!t||!a)return null;var e=this.setting("blomstra-search.analyzer-language")()||"english",n=this.setting("blomstra-search.min-search-length")()||String(3),s=String(r().data.settings["blomstra-search.indexed-min-search-length"]||3);return e===a&&n===s?null:m(h(),{type:"warning",dismissible:!1,icon:"fas fa-exclamation-triangle"},r().translator.trans("blomstra-search.admin.index-settings-changed"))}).registerSetting({setting:"blomstra-search.elastic-endpoint",label:r().translator.trans("blomstra-search.admin.elastic-endpoint"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-username",label:r().translator.trans("blomstra-search.admin.elastic-username"),type:"input"}).registerSetting({setting:"blomstra-search.elastic-password",label:r().translator.trans("blomstra-search.admin.elastic-password"),type:"password"}).registerSetting({setting:"blomstra-search.elastic-index",label:r().translator.trans("blomstra-search.admin.elastic-index"),default:"flarum",type:"input"}).registerSetting({setting:"blomstra-search.analyzer-language",label:r().translator.trans("blomstra-search.admin.analyzer.label"),help:r().translator.trans("blomstra-search.admin.analyzer.help"),type:"select",options:Object.fromEntries(e.entries()),default:"english"}).registerSetting({setting:"blomstra-search.search-discussion-subjects",label:r().translator.trans("blomstra-search.admin.search-discussion-subjects"),type:"switch"}).registerSetting({setting:"blomstra-search.search-post-bodies",label:r().translator.trans("blomstra-search.admin.search-post-bodies"),type:"switch"}).registerSetting({setting:"blomstra-search.min-search-length",label:r().translator.trans("blomstra-search.admin.min-search-length.label"),help:r().translator.trans("blomstra-search.admin.min-search-length.help"),type:"select",options:{1:"1",2:"2",3:"3",4:"4"},default:String(3)})})})(),module.exports={}})();
//# sourceMappingURL=admin.js.map //# sourceMappingURL=admin.js.map

File diff suppressed because one or more lines are too long

2
js/dist/forum.js vendored

File diff suppressed because one or more lines are too long

View File

@ -1 +0,0 @@
/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/babel/babel/blob/main/packages/babel-helpers/LICENSE */

File diff suppressed because one or more lines are too long

View File

@ -4,7 +4,7 @@ import DashboardPage from 'flarum/admin/components/DashboardPage';
import DashboardWidget from 'flarum/admin/components/DashboardWidget'; import DashboardWidget from 'flarum/admin/components/DashboardWidget';
import Alert from 'flarum/common/components/Alert'; import Alert from 'flarum/common/components/Alert';
const REQUIRED_INDEX_COMPAT = 'v3'; const REQUIRED_INDEX_COMPAT = 'v2';
class ReindexWarningWidget extends DashboardWidget { class ReindexWarningWidget extends DashboardWidget {
className() { className() {
@ -25,7 +25,7 @@ class ReindexWarningWidget extends DashboardWidget {
} }
} }
app.initializers.add('sbp-search', () => { app.initializers.add('blomstra-search', () => {
const activeIndex = app.data.settings['blomstra-search.active-index']; const activeIndex = app.data.settings['blomstra-search.active-index'];
const compatVersion = app.data.settings['blomstra-search.index-compatible']; const compatVersion = app.data.settings['blomstra-search.index-compatible'];
@ -35,57 +35,61 @@ app.initializers.add('sbp-search', () => {
}); });
} }
const languages: Record<string, string> = { const languages = new Map();
arabic: 'Arabic', [
armenian: 'Armenian', 'arabic',
basque: 'Basque', 'armenian',
bengali: 'Bengali', 'basque',
brazilian: 'Brazilian Portuguese', 'bengali',
bulgarian: 'Bulgarian', 'brazilian',
catalan: 'Catalan', 'bulgarian',
cjk: 'CJK (Chinese, Japanese, Korean)', 'catalan',
czech: 'Czech', 'cjk',
danish: 'Danish', 'czech',
dutch: 'Dutch', 'danish',
english: 'English', 'dutch',
estonian: 'Estonian', 'english',
finnish: 'Finnish', 'estonian',
french: 'French', 'finnish',
galician: 'Galician', 'french',
german: 'German', 'galician',
greek: 'Greek', 'german',
hindi: 'Hindi', 'greek',
hungarian: 'Hungarian', 'hindi',
indonesian: 'Indonesian', 'hungarian',
irish: 'Irish', 'indonesian',
italian: 'Italian', 'irish',
latvian: 'Latvian', 'italian',
lithuanian: 'Lithuanian', 'latvian',
norwegian: 'Norwegian', 'lithuanian',
persian: 'Persian', 'norwegian',
portuguese: 'Portuguese', 'persian',
romanian: 'Romanian', 'portuguese',
russian: 'Russian', 'romanian',
sorani: 'Sorani (Kurdish)', 'russian',
spanish: 'Spanish', 'sorani',
standard: 'Standard (generic)', 'spanish',
swedish: 'Swedish', 'swedish',
turkish: 'Turkish', 'turkish',
thai: 'Thai', 'thai',
}; ].forEach((language) => {
languages.set(language, language);
});
app.extensionData app.extensionData
.for('sbp-search') .for('blomstra-search')
.registerSetting(function (this: any) { .registerSetting(function (this: any) {
// No index yet, or first build predates this tracking — stay silent. // No index yet, or first build predates this tracking — stay silent.
const indexedAnalyzer = app.data.settings['blomstra-search.indexed-analyzer']; const indexedAnalyzer = app.data.settings['blomstra-search.indexed-analyzer'];
if (!activeIndex || !indexedAnalyzer) return null; if (!activeIndex || !indexedAnalyzer) return null;
const currentAnalyzer = this.setting('blomstra-search.analyzer-language')() || 'english'; const currentAnalyzer = this.setting('blomstra-search.analyzer-language')() || 'english';
const currentStemExclusion = this.setting('blomstra-search.stem-exclusion')() || ''; const currentMinLength = this.setting('blomstra-search.min-search-length')();
const indexedStemExclusion = app.data.settings['blomstra-search.indexed-stem-exclusion'] || ''; const indexedMinLength = String(
app.data.settings['blomstra-search.indexed-min-search-length'] || app.data.settings['blomstra-search.min-search-length']
);
if (currentAnalyzer === indexedAnalyzer && currentStemExclusion === indexedStemExclusion) return null; if (currentAnalyzer === indexedAnalyzer && currentMinLength === indexedMinLength) return null;
return m( return m(
Alert, Alert,
@ -114,6 +118,14 @@ app.initializers.add('sbp-search', () => {
default: 'flarum', default: 'flarum',
type: 'input', type: 'input',
}) })
.registerSetting({
setting: 'blomstra-search.analyzer-language',
label: app.translator.trans('blomstra-search.admin.analyzer.label'),
help: app.translator.trans('blomstra-search.admin.analyzer.help'),
type: 'select',
options: Object.fromEntries(languages.entries()),
default: 'english',
})
.registerSetting({ .registerSetting({
setting: 'blomstra-search.search-discussion-subjects', setting: 'blomstra-search.search-discussion-subjects',
label: app.translator.trans('blomstra-search.admin.search-discussion-subjects'), label: app.translator.trans('blomstra-search.admin.search-discussion-subjects'),
@ -125,33 +137,11 @@ app.initializers.add('sbp-search', () => {
type: 'switch', type: 'switch',
}) })
.registerSetting({ .registerSetting({
setting: 'blomstra-search.analyzer-language', setting: 'blomstra-search.min-search-length',
label: app.translator.trans('blomstra-search.admin.analyzer.label'), label: app.translator.trans('blomstra-search.admin.min-search-length.label'),
help: app.translator.trans('blomstra-search.admin.analyzer.help'), help: app.translator.trans('blomstra-search.admin.min-search-length.help'),
type: 'select', type: 'select',
options: languages, options: { '1': '1', '2': '2', '3': '3', '4': '4' },
default: 'english', default: app.data.settings['blomstra-search.min-search-length'],
})
.registerSetting(function (this: any) {
const isCjk = (this.setting('blomstra-search.analyzer-language')() || 'english') === 'cjk';
return this.buildSettingComponent({
setting: 'blomstra-search.stem-exclusion',
type: 'textarea',
label: app.translator.trans('blomstra-search.admin.settings.stem-exclusion.label'),
help: app.translator.trans('blomstra-search.admin.settings.stem-exclusion.help'),
disabled: isCjk,
});
})
.registerSetting(function (this: any) {
const isCjk = (this.setting('blomstra-search.analyzer-language')() || 'english') === 'cjk';
return this.buildSettingComponent({
setting: 'blomstra-search.min-search-length',
label: app.translator.trans('blomstra-search.admin.min-search-length.label'),
help: app.translator.trans('blomstra-search.admin.min-search-length.help'),
type: 'select',
options: { '1': '1', '2': '2', '3': '3', '4': '4' },
default: app.data.settings['blomstra-search.min-search-length'],
disabled: isCjk,
});
}); });
}); });

View File

@ -6,9 +6,10 @@ import DiscussionListState from 'flarum/forum/states/DiscussionListState';
export default function extendDiscussionState() { export default function extendDiscussionState() {
override(DiscussionListState.prototype, 'loadPage', async function (this: DiscussionListState, original, page: number = 1) { override(DiscussionListState.prototype, 'loadPage', async function (this: DiscussionListState, original, page: number = 1) {
const preloaded = app.data.apiDocument || null;
// If existing payload is given or no search is made, fallback on native page. // If existing payload is given or no search is made, fallback on native page.
if (!this.requestParams()?.filter?.q) return original.call(this, page); if (preloaded || !this.requestParams()?.filter?.q) return original.call(this, page);
const params = this.requestParams(); const params = this.requestParams();
params.page = { params.page = {
@ -16,13 +17,12 @@ export default function extendDiscussionState() {
...params.page, ...params.page,
}; };
if (Array.isArray(params.include)) { if (Array.isArray(params.include)) {
params.include = params.include.join(','); params.include = params.include.join(',');
} }
// Construct API search URI // Construct API search URI
const url = `${app.forum.attribute('apiUrl')}/sbp/search/${this.type}`; const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`;
// Make API GET request // Make API GET request
const results = await app.request({ params, url, method: 'GET' }); const results = await app.request({ params, url, method: 'GET' });

View File

@ -21,21 +21,18 @@ export default class DiscussionsSearchSource implements SearchSource {
private type = 'discussions'; private type = 'discussions';
async search(query: string): Promise<void> { async search(query: string): Promise<void> {
// Suppress ES instant results when "search within comments" is ON;
// only usernames are shown in that mode.
query = query.toLowerCase(); query = query.toLowerCase();
this.results.set(query, []); this.results.set(query, []);
const params = { const params = {
filter: { q: query, autocomplete: 1 }, filter: { q: query },
page: { limit: 3 }, page: { limit: 3 },
include: 'mostRelevantPost', include: 'mostRelevantPost',
}; };
// Construct API search URI // Construct API search URI
const url = `${app.forum.attribute('apiUrl')}/sbp/search/${this.type}`; const url = `${app.forum.attribute('apiUrl')}/blomstra/search/${this.type}`;
// Make API GET request // Make API GET request
const results = await app.request({ params, url, method: 'GET' }); const results = await app.request({ params, url, method: 'GET' });

View File

@ -1,26 +0,0 @@
import { extend } from 'flarum/common/extend';
import DiscussionListItem from 'flarum/forum/components/DiscussionListItem';
import TerminalPost from 'flarum/forum/components/TerminalPost';
export default function extendDiscussionListItem() {
extend(DiscussionListItem.prototype, 'infoItems', function (items) {
const params = this.attrs.params;
if (!params.q) return;
const hasFieldSort = params.sort && params.sort !== 'relevance';
if (hasFieldSort) {
// Field sort active (latest, oldest, top, …): replace excerpt with TerminalPost.
// An excerpt is meaningless when results are ordered by date/count rather than relevance.
items.remove('excerpt');
if (!items.has('terminalPost')) {
items.add('terminalPost', <TerminalPost discussion={this.attrs.discussion} lastPost={!this.showFirstPost()} />);
}
} else if (!items.has('excerpt')) {
// Relevance mode but no excerpt (mostRelevantPost was null or non-comment type).
// Fall back to TerminalPost so the info section is never silently empty.
items.add('terminalPost', <TerminalPost discussion={this.attrs.discussion} lastPost={true} />);
}
});
}

View File

@ -7,17 +7,18 @@ import ItemList from 'flarum/common/utils/ItemList';
import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource'; import DiscussionsSearchSource from './SearchSources/DiscussionsSearchSource';
import extendDiscussionState from './PaginatedListStates/extendDiscussionState'; import extendDiscussionState from './PaginatedListStates/extendDiscussionState';
import extendDiscussionListItem from './extendDiscussionListItem';
app.initializers.add('blomstra-search', () => { app.initializers.add('blomstra-search', () => {
extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) { const minLength = parseInt(app.data.settings['blomstra-search.min-search-length'] || String(Search.MIN_SEARCH_LEN), 10);
// app.forum is not available during initializers (it is set after they run), if (minLength !== Search.MIN_SEARCH_LEN) {
// so read the setting lazily here, at first render time. // Flarum provides no extension point for MIN_SEARCH_LEN, so we overwrite the
const minLength = (app.forum.attribute('blomstraSearchMinLength') as number) || Search.MIN_SEARCH_LEN; // static property directly. TypeScript `readonly` is compile-time only — at
if (minLength !== Search.MIN_SEARCH_LEN) { // runtime this is a plain property assignment and is safe as long as no code
(Search as any).MIN_SEARCH_LEN = minLength; // reads MIN_SEARCH_LEN before this initializer runs.
} (Search as any).MIN_SEARCH_LEN = minLength;
}
extend(Search.prototype, 'sourceItems', function (this: Search<SearchAttrs>, items: ItemList<SearchSource>) {
items.replace('discussions', new DiscussionsSearchSource()); items.replace('discussions', new DiscussionsSearchSource());
}); });
}); });
@ -26,7 +27,6 @@ app.initializers.add(
'blomstra-search-early', 'blomstra-search-early',
() => { () => {
extendDiscussionState(); extendDiscussionState();
extendDiscussionListItem();
}, },
999999 999999
); );

View File

@ -1,14 +0,0 @@
diff --git a/src/Commands/BuildCommand.php b/src/Commands/BuildCommand.php
index 9024e1e..9024e1e 100644
--- a/src/Commands/BuildCommand.php
+++ b/src/Commands/BuildCommand.php
@@ -38,6 +38,6 @@ class BuildCommand extends Command
/** Matches Flarum's Search::MIN_SEARCH_LEN — the default minimum query length. */
public const DEFAULT_MIN_SEARCH_LENGTH = 3;
/** N-gram bounds for the title autocomplete subfield (not admin-configurable). */
- private const TITLE_MIN_GRAM = 2;
- private const TITLE_MAX_GRAM = 15;
+ private const TITLE_MIN_GRAM = 3;
+ private const TITLE_MAX_GRAM = 20;
protected $signature = 'blomstra:search:index

View File

@ -17,12 +17,9 @@ blomstra-search:
search-post-bodies: Search inside comments search-post-bodies: Search inside comments
match-sentences: Match search term against full sentence match-sentences: Match search term against full sentence
match-words: Match search term against full words match-words: Match search term against full words
settings:
stem-exclusion:
label: "Protected words (stem exclusion)"
help: "One word per line. These words will not be stemmed during search (e.g. brand names, proper nouns). Not applicable for CJK or Thai languages. Requires a full re-index to take effect."
min-search-length: min-search-length:
label: Minimum search query length label: Minimum search query length
help: | help: |
Minimum number of characters required before a search is triggered. Lower this to 1 or 2 Minimum number of characters required before a search is triggered. Lower this to 1 or 2
for CJK (Chinese, Japanese, Korean) communities where single characters carry full meaning. for CJK (Chinese, Japanese, Korean) communities where single characters carry full meaning.
Changing this requires a full index rebuild to take effect.

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Api; namespace Blomstra\Search\Api;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ResponseInterface;

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Api\Controllers; namespace Blomstra\Search\Api\Controllers;
use Sbp\Search\Commands\BuildCommand; use Blomstra\Search\Commands\BuildCommand;
use Laminas\Diactoros\Response\EmptyResponse; use Laminas\Diactoros\Response\EmptyResponse;
use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Message\ServerRequestInterface;

View File

@ -10,15 +10,15 @@
* *
*/ */
namespace Sbp\Search\Api\Controllers; namespace Blomstra\Search\Api\Controllers;
use Sbp\Search\Elasticsearch\HasChildQuery; use Blomstra\Search\Elasticsearch\HasChildQuery;
use Sbp\Search\Elasticsearch\MatchPhraseQuery; use Blomstra\Search\Elasticsearch\MatchPhraseQuery;
use Sbp\Search\Elasticsearch\MatchQuery; use Blomstra\Search\Elasticsearch\MatchQuery;
use Sbp\Search\Elasticsearch\TermsQuery; use Blomstra\Search\Elasticsearch\TermsQuery;
use Sbp\Search\Searchers\CommentPostSearcher; use Blomstra\Search\Searchers\CommentPostSearcher;
use Sbp\Search\Searchers\DiscussionSearcher; use Blomstra\Search\Searchers\DiscussionSearcher;
use Sbp\Search\Searchers\Searcher; use Blomstra\Search\Searchers\Searcher;
use Elasticsearch\Client; use Elasticsearch\Client;
use Flarum\Api\Controller\ListDiscussionsController; use Flarum\Api\Controller\ListDiscussionsController;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;
@ -37,7 +37,7 @@ use Illuminate\Support\Str;
use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Message\ServerRequestInterface;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
use Spatie\ElasticsearchQueryBuilder\Builder; use Spatie\ElasticsearchQueryBuilder\Builder;
use Sbp\Search\Elasticsearch\BoolQuery; use Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery;
use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery; use Spatie\ElasticsearchQueryBuilder\Queries\TermQuery;
use Spatie\ElasticsearchQueryBuilder\Sorts\Sort; use Spatie\ElasticsearchQueryBuilder\Sorts\Sort;
use Tobscure\JsonApi\Document; use Tobscure\JsonApi\Document;
@ -78,18 +78,27 @@ class SearchController extends ListDiscussionsController
$offset = $this->extractOffset($request); $offset = $this->extractOffset($request);
$include = array_merge($this->extractInclude($request), ['state']); $include = array_merge($this->extractInclude($request), ['state']);
// Autocomplete path: fast prefix match on title.autocomplete only. $query = BoolQuery::create()
if (!empty($filters['autocomplete']) && !empty($search)) { // Always restrict to discussion documents; posts are only searched via has_child.
return $this->handleAutocomplete($search, $actor, $limit, $offset, $include, $document, $request); ->add(TermQuery::create('join_field', 'discussion'), 'filter');
if (!empty($search)) {
$query->add($this->buildTextQuery($search, $actor));
} }
$this->addFilters($query, $actor, $filters);
$builder = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->size($limit + 1)
->from($offset)
->addQuery($query);
$knownSortFields = array_merge(array_values($this->translateSort), ['rawId']); $knownSortFields = array_merge(array_values($this->translateSort), ['rawId']);
$logger = resolve(LoggerInterface::class); $logger = resolve(LoggerInterface::class);
$phpSortField = null; $phpSortField = null;
$phpSortDir = 'desc'; $phpSortDir = 'desc';
$needsScoring = true;
$sorts = [];
foreach ($this->extractSort($request) as $field => $direction) { foreach ($this->extractSort($request) as $field => $direction) {
$translated = $this->translateSort[$field] ?? $field; $translated = $this->translateSort[$field] ?? $field;
@ -99,8 +108,7 @@ class SearchController extends ListDiscussionsController
continue; continue;
} }
$sorts[] = new Sort($translated, $direction); $builder->addSort(new Sort($translated, $direction));
$needsScoring = false;
if ($phpSortField === null && $translated !== 'rawId') { if ($phpSortField === null && $translated !== 'rawId') {
$phpSortField = $translated; $phpSortField = $translated;
@ -108,45 +116,7 @@ class SearchController extends ListDiscussionsController
} }
} }
// Default to latest when no explicit sort is requested. This lets has_child use $response = $builder->search();
// score_mode:none, which skips child scoring entirely and lets ES short-circuit
// early on large corpora. Relevance is still available via sort=relevant if added.
if (empty($sorts)) {
$needsScoring = false;
$phpSortField = 'updated_at';
$phpSortDir = 'desc';
$sorts[] = new Sort('updated_at', 'desc');
}
$query = BoolQuery::create()
// Always restrict to discussion documents; posts are only searched via has_child.
->add(TermQuery::create('join_field', 'discussion'), 'filter');
if (!empty($search)) {
$query->add($this->buildTextQuery($search, $actor, $needsScoring));
}
$this->addFilters($query, $actor, $filters);
$builder = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->addQuery($query);
foreach ($sorts as $sort) {
$builder->addSort($sort);
}
// track_total_hits: false lets ES stop counting once it has collected
// enough results in sort order, avoiding a full-index count on every query.
$payload = $builder->getPayload();
$payload['track_total_hits'] = false;
$response = $this->elastic->search([
'index' => resolve('blomstra.search.elastic_index'),
'size' => $limit + 1,
'from' => $offset,
'body' => $payload,
]);
Discussion::setStateUser($actor); Discussion::setStateUser($actor);
@ -176,7 +146,7 @@ class SearchController extends ListDiscussionsController
}); });
$document->addPaginationLinks( $document->addPaginationLinks(
$this->uri->to('api')->route('sbp.search', ['type' => 'discussions']), $this->uri->to('api')->route('blomstra.search', ['type' => 'discussions']),
$request->getQueryParams(), $request->getQueryParams(),
$offset, $offset,
$limit, $limit,
@ -222,145 +192,18 @@ class SearchController extends ListDiscussionsController
return $discussions; return $discussions;
} }
/**
* Fast autocomplete path: match prefix n-grams on title.autocomplete only.
* No has_child, no phrase scoring a single MatchQuery suffices.
*/
protected function handleAutocomplete(
string $search,
User $actor,
int $limit,
int $offset,
array $include,
Document $document,
ServerRequestInterface $request
): mixed {
$query = BoolQuery::create()
->add(TermQuery::create('join_field', 'discussion'), 'filter');
$query->add($this->buildAutocompleteQuery($search), 'must');
$this->addFilters($query, $actor, []);
$payload = (new Builder($this->elastic))
->index(resolve('blomstra.search.elastic_index'))
->addQuery($query)
->addSort(new Sort('updated_at', 'desc'))
->getPayload();
$response = $this->elastic->search([
'index' => resolve('blomstra.search.elastic_index'),
'size' => $limit + 1,
'from' => $offset,
'body' => $payload,
]);
Discussion::setStateUser($actor);
$results = Collection::make(Arr::get($response, 'hits.hits'))
->map(fn ($hit) => [
'discussion_id' => Str::after($hit['_id'], 'discussions:'),
'most_relevant_post_id' => null,
'weight' => Arr::get($hit, 'sort.0', 0),
]);
$document->addPaginationLinks(
$this->uri->to('api')->route('sbp.search', ['type' => 'discussions']),
$request->getQueryParams(),
$offset,
$limit,
$results->count() > $limit ? null : 0
);
$results = $results->take($limit);
$discussions = Discussion::query()
->when(
$actor->isGuest() || !$actor->hasPermission('discussion.hide'),
fn ($q) => $q->whereNull('hidden_at')
)
->whereIn('id', $results->pluck('discussion_id')->filter())
->get()
->each(function (Discussion $discussion) use ($results) {
$result = $results->firstWhere('discussion_id', $discussion->id);
$discussion->most_relevant_post_id = $result['most_relevant_post_id'] ?? $discussion->first_post_id;
$discussion->weight = $result['weight'] ?? 0;
})
->sortByDesc('updated_at')
->unique();
$this->loadRelations($discussions, $include);
return $discussions;
}
protected function buildAutocompleteQuery(string $search): BoolQuery
{
$query = BoolQuery::create();
// Title prefix match (stronger)
$query->add(
(new MatchQuery('title.autocomplete', $search))
->operator('and')
->boost(1.0),
'should'
);
// Content prefix match (weaker)
$query->add(
(new MatchQuery('content.autocomplete', $search))
->operator('and')
->boost(0.4),
'should'
);
$query->minimumShouldMatch(1);
return $query;
}
/** /**
* Build the text-matching portion of the query. * Build the text-matching portion of the query.
* *
* Discussion titles are matched directly (filtered to join_field=discussion). * Discussion titles are matched directly (filtered to join_field=discussion).
* Post bodies are matched via has_child. When $needsScoring is true (no explicit * Post bodies are matched via has_child with score_mode=sum so discussions
* sort, i.e. relevance ordering), score_mode=sum accumulates child scores onto the * with many matching posts score higher than those with a single strong match.
* parent so that discussions with many strongly-matching posts rank higher. When * inner_hits returns the best-scoring post for use as mostRelevantPost.
* false (any explicit field sort), score_mode=none skips scoring entirely ES only
* checks whether a matching child exists, which is significantly cheaper on large
* corpora.
* inner_hits returns the best-matching post for use as mostRelevantPost.
* Hidden posts are only included in matching for users with post.hide permission. * Hidden posts are only included in matching for users with post.hide permission.
*/ */
protected function buildTextQuery(string $search, User $actor, bool $needsScoring = false): BoolQuery protected function buildTextQuery(string $search, User $actor): BoolQuery
{ {
$textQuery = BoolQuery::create(); $textQuery = BoolQuery::create();
// Strong direct title match for the full search page.
$textQuery->add(
(new MatchQuery('title', $search))
->operator('and')
->boost(4.0),
'should'
);
// Low-weight prefix match on title.autocomplete for partial title hits.
$textQuery->add(
(new MatchQuery('title.autocomplete', $search))
->operator('and')
->boost(0.35),
'should'
);
// Low-weight prefix match on title.autocomplete for the full search page.
// Exact / normal title matches should still rank higher than prefix-only hits.
$textQuery->add(
(new MatchQuery('title.autocomplete', $search))
->operator('and')
->boost(0.35),
'should'
);
if ($this->discussionSearcher?->enabled()) { if ($this->discussionSearcher?->enabled()) {
$textQuery->add($this->buildShouldClauses($search, $this->discussionSearcher->boost()), 'should'); $textQuery->add($this->buildShouldClauses($search, $this->discussionSearcher->boost()), 'should');
@ -371,15 +214,11 @@ class SearchController extends ListDiscussionsController
// Guests and non-moderators may not see hidden posts; exclude them from child matching. // Guests and non-moderators may not see hidden posts; exclude them from child matching.
if ($actor->isGuest() || !$actor->hasPermission('post.hide')) { if ($actor->isGuest() || !$actor->hasPermission('post.hide')) {
$postQuery->add(TermQuery::create('is_hidden', 'false'), 'filter'); $postQuery->add(TermQuery::create('is_hidden', false), 'filter');
} }
// Without minimum_should_match, ES default MSM is 0 when a filter clause is present,
// causing has_child to score every non-hidden post instead of only matching ones.
$postQuery->minimumShouldMatch(1);
$textQuery->add( $textQuery->add(
HasChildQuery::create('post', $postQuery, $needsScoring ? 'sum' : 'none')->withInnerHits(), HasChildQuery::create('post', $postQuery)->withInnerHits(),
'should' 'should'
); );
} }
@ -394,10 +233,10 @@ class SearchController extends ListDiscussionsController
if ($this->matchSentences) { if ($this->matchSentences) {
$should->add((new MatchPhraseQuery('content', $search))->boost(2 * $boost), 'should'); $should->add((new MatchPhraseQuery('content', $search))->boost(2 * $boost), 'should');
} }
if ($this->matchWords) { if ($this->matchWords) {
$should->add((new MatchQuery('content', $search))->operator('and')->boost(1.8 * $boost), 'should'); $should->add((new MatchQuery('content', $search))->operator('and')->boost(1.8 * $boost), 'should');
$should->add((new MatchQuery('content.autocomplete', $search))->operator('and')->boost(0.30 * $boost), 'should'); $should->add((new MatchQuery('content', $search))->operator('or')->boost(0.8 * $boost), 'should');
} }
return $should; return $should;
} }
@ -462,12 +301,8 @@ class SearchController extends ListDiscussionsController
$search = Arr::get($filters, 'q'); $search = Arr::get($filters, 'q');
if ($search) { if ($search) {
// Strip Flarum gambit operators (tag:foo, author:bar, is:private, etc.)
// before passing to ES. These are structural filters handled separately;
// leaving them in causes operator:and to require the gambit tokens to
// appear literally in post content, producing zero results.
$q = collect(explode(' ', $search)) $q = collect(explode(' ', $search))
->filter(fn (string $part) => !preg_match('/^\w+:/', $part)) ->filter(fn (string $part) => $part !== 'is:private')
->filter() ->filter()
->join(' '); ->join(' ');

View File

@ -10,13 +10,12 @@
* *
*/ */
namespace Sbp\Search\Commands; namespace Blomstra\Search\Commands;
use Sbp\Search\Jobs\Job; use Blomstra\Search\Jobs\Job;
use Sbp\Search\Jobs\UpdateSearchJob; use Blomstra\Search\Jobs\UpdateSearchJob;
use Sbp\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Elasticsearch\Client; use Elasticsearch\Client;
use Elasticsearch\Common\Exceptions\ElasticsearchException;
use Flarum\Settings\SettingsRepositoryInterface; use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Container\Container;
@ -36,15 +35,11 @@ class BuildCommand extends Command
* read back into blomstra-search.index-compatible by saveIndexedConfig so the * read back into blomstra-search.index-compatible by saveIndexedConfig so the
* value always reflects the live index including after a promote/rollback. * value always reflects the live index including after a promote/rollback.
*/ */
public const INDEX_COMPAT_VERSION = 'v3'; public const INDEX_COMPAT_VERSION = 'v2';
/** Matches Flarum's Search::MIN_SEARCH_LEN — the default minimum query length. */ /** Matches Flarum's Search::MIN_SEARCH_LEN — the default minimum query length. */
public const DEFAULT_MIN_SEARCH_LENGTH = 3; public const DEFAULT_MIN_SEARCH_LENGTH = 3;
/** N-gram bounds for the title autocomplete subfield (not admin-configurable). */
private const TITLE_MIN_GRAM = 3;
private const TITLE_MAX_GRAM = 20;
protected $signature = 'blomstra:search:index protected $signature = 'blomstra:search:index
{action? : build | promote | rollback | discard | mapping | fill} {action? : build | promote | rollback | discard | mapping | fill}
{--fresh} {--fresh}
@ -159,10 +154,8 @@ HELP;
$this->warn("An in-progress build already exists: $staging"); $this->warn("An in-progress build already exists: $staging");
$this->line(''); $this->line('');
$this->line('Choose one of:'); $this->line('Choose one of:');
$this->line(' blomstra:search:index build --resume Continue from where it left off'); $this->line(' blomstra:search:index build --resume Continue from where it left off');
$this->line(' blomstra:search:index build --resume --staging Continue and keep in staging when done'); $this->line(' blomstra:search:index build --fresh Drop this build and start completely fresh');
$this->line(' blomstra:search:index build --fresh Drop this build and start completely fresh');
$this->line(' blomstra:search:index build --fresh --staging Start fresh and keep in staging when done');
return; return;
} }
@ -366,7 +359,17 @@ HELP;
$rangeTo = $continueAt; $rangeTo = $continueAt;
if ($seedMissing) { if ($seedMissing) {
$seeded = $this->queryIndexedIds($client, $targetIndex, $seeder->joinRelation(), $rangeFrom, $rangeTo); $response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $seeder->joinRelation()))
)
->search();
$seeded = Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} }
/** @var Collection $collection */ /** @var Collection $collection */
@ -495,170 +498,77 @@ HELP;
$settingsResponse = $client->indices()->getSettings(['index' => $indexName]); $settingsResponse = $client->indices()->getSettings(['index' => $indexName]);
$analysis = Arr::get($settingsResponse, "$indexName.settings.index.analysis", []); $analysis = Arr::get($settingsResponse, "$indexName.settings.index.analysis", []);
$analyzer = Arr::get($analysis, 'analyzer.flarum_analyzer.type', 'english'); $analyzer = Arr::get($analysis, 'analyzer.flarum_analyzer.type', 'english');
$stemExclusion = Arr::get($analysis, 'analyzer.flarum_analyzer.stem_exclusion', []); $minGram = (int) Arr::get($analysis, 'filter.partial_search_filter.min_gram', self::DEFAULT_MIN_SEARCH_LENGTH);
$mappingResponse = $client->indices()->getMapping(['index' => $indexName]); $mappingResponse = $client->indices()->getMapping(['index' => $indexName]);
$compatVersion = Arr::get($mappingResponse, "$indexName.mappings._meta.index_compat_version"); $compatVersion = Arr::get($mappingResponse, "$indexName.mappings._meta.index_compat_version");
$settings->set('blomstra-search.indexed-analyzer', $analyzer); $settings->set('blomstra-search.indexed-analyzer', $analyzer);
$settings->set('blomstra-search.indexed-stem-exclusion', implode("\n", $stemExclusion)); $settings->set('blomstra-search.indexed-min-search-length', $minGram);
$settings->set('blomstra-search.index-compatible', $compatVersion); $settings->set('blomstra-search.index-compatible', $compatVersion);
} }
protected function buildIndexSettings(SettingsRepositoryInterface $settings): array protected function buildIndexSettings(SettingsRepositoryInterface $settings): array
{ {
$language = $settings->get('blomstra-search.analyzer-language') ?: 'english'; $language = $settings->get('blomstra-search.analyzer-language') ?: 'english';
$minGram = max(1, (int) ($settings->get('blomstra-search.min-search-length') ?: self::DEFAULT_MIN_SEARCH_LENGTH));
$maxGram = 10;
$raw = $settings->get('blomstra-search.stem-exclusion', ''); if ($minGram >= $maxGram) {
$stemExclusion = array_values(array_filter(array_map('trim', explode("\n", $raw)))); $this->error("min_gram ($minGram) must be less than max_gram ($maxGram). Using default.");
$minGram = self::DEFAULT_MIN_SEARCH_LENGTH;
if ($language === 'cjk') {
// CJK uses the built-in bigram analyzer; no autocomplete subfield.
return [
'analysis' => [
'analyzer' => [
'flarum_analyzer' => ['type' => 'cjk'],
],
],
];
}
$analyzerConfig = ['type' => $language];
if (!empty($stemExclusion)) {
$analyzerConfig['stem_exclusion'] = $stemExclusion;
} }
return [ return [
'index.max_ngram_diff' => self::TITLE_MAX_GRAM - self::TITLE_MIN_GRAM, 'index.max_ngram_diff' => $maxGram - $minGram,
'analysis' => [ 'analysis' => [
'filter' => [
'title_autocomplete_filter' => [
'type' => 'edge_ngram',
'min_gram' => self::TITLE_MIN_GRAM,
'max_gram' => self::TITLE_MAX_GRAM,
'token_chars' => ['letter', 'digit'],
],
],
'analyzer' => [ 'analyzer' => [
// Symmetric language analyzer: used for BOTH index and search on content + title. 'flarum_analyzer' => [
'flarum_analyzer' => $analyzerConfig, 'type' => $language,
// Title autocomplete index-time: prefix n-grams, no stemming.
'flarum_title_autocomplete' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => ['lowercase', 'title_autocomplete_filter'],
], ],
// Title autocomplete search-time: just lowercase — predictable on partial words. 'flarum_analyzer_partial' => [
'flarum_title_search' => [
'type' => 'custom', 'type' => 'custom',
'tokenizer' => 'standard', 'tokenizer' => 'standard',
'filter' => ['lowercase'], 'filter' => ['lowercase', 'partial_search_filter'],
],
],
'filter' => [
'partial_search_filter' => [
'type' => 'ngram',
'min_gram' => $minGram,
'max_gram' => $maxGram,
'token_chars' => ['letter', 'digit', 'symbol'],
], ],
], ],
], ],
]; ];
} }
protected function mappingProperties(): array protected function mappingProperties(): array
{ {
$language = resolve(SettingsRepositoryInterface::class)->get('blomstra-search.analyzer-language') ?: 'english'; return [
'_meta' => ['index_compat_version' => self::INDEX_COMPAT_VERSION],
$titleMapping = [ 'properties' => [
'type' => 'text', 'join_field' => ['type' => 'join', 'relations' => ['discussion' => 'post']],
'analyzer' => 'flarum_analyzer', 'discussion_id' => ['type' => 'integer'],
]; 'content' => ['type' => 'text', 'analyzer' => 'flarum_analyzer_partial', 'search_analyzer' => 'flarum_analyzer'],
'rawId' => ['type' => 'integer'],
// Only add the autocomplete subfield for non-CJK languages. 'created_at' => ['type' => 'date'],
if ($language !== 'cjk') { 'updated_at' => ['type' => 'date'],
$titleMapping['fields'] = [ 'is_private' => ['type' => 'boolean'],
'autocomplete' => [ 'is_sticky' => ['type' => 'boolean'],
'type' => 'text', 'groups' => ['type' => 'integer'],
'analyzer' => 'flarum_title_autocomplete', 'tags' => ['type' => 'integer'],
'search_analyzer' => 'flarum_title_search', 'recipient_groups' => ['type' => 'integer'],
'recipient_users' => ['type' => 'integer'],
'comment_count' => ['type' => 'integer'],
'view_count' => ['type' => 'integer'],
'is_hidden' => ['type' => 'boolean'],
], ],
]; ];
} }
$contentMapping = [
'type' => 'text',
'analyzer' => 'flarum_analyzer',
];
// Only add the autocomplete subfield for non-CJK languages.
if ($language !== 'cjk') {
$contentMapping['fields'] = [
'autocomplete' => [
'type' => 'text',
'analyzer' => 'flarum_title_autocomplete',
'search_analyzer' => 'flarum_title_search',
],
];
}
return [
'_meta' => ['index_compat_version' => self::INDEX_COMPAT_VERSION],
'properties' => [
'join_field' => ['type' => 'join', 'relations' => ['discussion' => 'post']],
'discussion_id' => ['type' => 'integer'],
'content' => $contentMapping,
'title' => $titleMapping,
'rawId' => ['type' => 'integer'],
'created_at' => ['type' => 'date'],
'updated_at' => ['type' => 'date'],
'is_private' => ['type' => 'boolean'],
'user_id' => ['type' => 'integer'],
'groups' => ['type' => 'keyword'],
'comment_count' => ['type' => 'integer'],
'tags' => ['type' => 'keyword'],
'is_sticky' => ['type' => 'boolean'],
'is_hidden' => ['type' => 'boolean'],
],
];
}
/**
* Query ES for rawIds already indexed in $targetIndex for the given $joinRelation and ID range.
* Retries on transient ES failures (NoNodesAvailableException / other ElasticsearchException)
* by sleeping past the StaticNoPingConnectionPool dead-node timeout (default 60 s) before
* each retry, giving the pool a chance to resurface the node.
*/
protected function queryIndexedIds(
Client $client,
string $targetIndex,
string $joinRelation,
int $rangeFrom,
int $rangeTo,
int $maxRetries = 10
): array {
$attempt = 0;
while (true) {
try {
$response = (new Builder($client))
->index($targetIndex)
->size(2500)
->addQuery(
(new BoolQuery())
->add((new RangeQuery('rawId'))->gte($rangeFrom)->lte($rangeTo))
->add(TermQuery::create('join_field', $joinRelation))
)
->search();
return Arr::pluck(Arr::get($response, 'hits.hits'), '_source.rawId');
} catch (ElasticsearchException $e) {
$attempt++;
if ($attempt >= $maxRetries) {
throw $e;
}
$this->warn("ES error on range {$rangeFrom}{$rangeTo} (attempt {$attempt}/{$maxRetries}): {$e->getMessage()}. Waiting 65 s before retry…");
sleep(65); // outlast the StaticNoPingConnectionPool dead-node window (default 60 s)
}
}
}
protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int protected function getContinueAt(SettingsRepositoryInterface $settings, string $type): ?int
{ {
$raw = $settings->get("blomstra-search.continued-at.$type"); $raw = $settings->get("blomstra-search.continued-at.$type");

View File

@ -1,41 +0,0 @@
<?php
/*
* This file is part of blomstra/search.
*
* Copyright (c) 2022 Blomstra Ltd.
*
* For the full copyright and license information, please view the LICENSE.md
* file that was distributed with this source code.
*
*/
namespace Sbp\Search\Elasticsearch;
class BoolQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\BoolQuery
{
protected ?int $minimumShouldMatch = null;
public static function create(): static
{
return new self();
}
public function minimumShouldMatch(int $minimum): static
{
$this->minimumShouldMatch = $minimum;
return $this;
}
public function toArray(): array
{
$array = parent::toArray();
if ($this->minimumShouldMatch !== null) {
$array['bool']['minimum_should_match'] = $this->minimumShouldMatch;
}
return $array;
}
}

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\Query;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery; use Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
class MatchQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery class MatchQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery
{ {
@ -62,10 +62,6 @@ class MatchQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\MatchQuery
$query['match'][$this->field]['boost'] = $this->boost; $query['match'][$this->field]['boost'] = $this->boost;
$query['match'][$this->field]['zero_terms_query'] = $this->zeroTerms ? 'all' : 'none'; $query['match'][$this->field]['zero_terms_query'] = $this->zeroTerms ? 'all' : 'none';
if ($this->analyzer !== null) {
$query['match'][$this->field]['analyzer'] = $this->analyzer;
}
return $query; return $query;
} }
} }

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\Query;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
use Spatie\ElasticsearchQueryBuilder\Queries\Query; use Spatie\ElasticsearchQueryBuilder\Queries\Query;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Elasticsearch; namespace Blomstra\Search\Elasticsearch;
class WildcardQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\WildcardQuery class WildcardQuery extends \Spatie\ElasticsearchQueryBuilder\Queries\WildcardQuery
{ {

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Exceptions; namespace Blomstra\Search\Exceptions;
use Throwable; use Throwable;

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Elasticsearch\Client; use Elasticsearch\Client;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;

View File

@ -10,11 +10,11 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Sbp\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Flarum\Queue\AbstractJob; use Flarum\Queue\AbstractJob;
use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Collection;
abstract class Job extends AbstractJob abstract class Job extends AbstractJob
{ {

View File

@ -10,11 +10,10 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Sbp\Search\Exceptions\SeedingException; use Blomstra\Search\Exceptions\SeedingException;
use Elasticsearch\Client; use Elasticsearch\Client;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
@ -51,11 +50,10 @@ class UpdateSearchJob extends Job
$items = Arr::get($response, 'items'); $items = Arr::get($response, 'items');
$failed = array_filter($items, fn ($item) => isset($item['index']['error'])); $error = Arr::get(Arr::first($items), 'index.error.reason');
$error = Arr::get(Arr::first($failed), 'index.error.reason', 'unknown error');
throw new SeedingException( throw new SeedingException(
"Failed to seed: $error (" . count($failed) . '/' . count($items) . ' items failed)', "Failed to seed: $error",
$items $items
); );
} }

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Jobs; namespace Blomstra\Search\Jobs;
use Elasticsearch\Client; use Elasticsearch\Client;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;

View File

@ -10,12 +10,12 @@
* *
*/ */
namespace Sbp\Search; namespace Blomstra\Search;
use Sbp\Search\Jobs\DeletingJob; use Blomstra\Search\Jobs\DeletingJob;
use Sbp\Search\Jobs\Job; use Blomstra\Search\Jobs\Job;
use Sbp\Search\Jobs\UpdateSearchJob; use Blomstra\Search\Jobs\UpdateSearchJob;
use Sbp\Search\Jobs\ViewsSearchJob; use Blomstra\Search\Jobs\ViewsSearchJob;
use Elasticsearch\Client as Elastic; use Elasticsearch\Client as Elastic;
use Elasticsearch\ClientBuilder; use Elasticsearch\ClientBuilder;
use Flarum\Api\Client; use Flarum\Api\Client;
@ -26,7 +26,7 @@ use Flarum\Settings\SettingsRepositoryInterface;
use Illuminate\Contracts\Container\Container; use Illuminate\Contracts\Container\Container;
use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Contracts\Queue\Queue; use Illuminate\Contracts\Queue\Queue;
use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Collection;
use Laminas\Stratigility\MiddlewarePipe; use Laminas\Stratigility\MiddlewarePipe;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
@ -47,13 +47,7 @@ class Provider extends AbstractServiceProvider
$this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) { $this->container->singleton(Elastic::class, function (Container $container) use ($settings, $config) {
$builder = ClientBuilder::create() $builder = ClientBuilder::create()
->setHosts([$settings->get('blomstra-search.elastic-endpoint')]) ->setHosts([$settings->get('blomstra-search.elastic-endpoint')]);
->setConnectionParams([
'client' => [
'connect_timeout' => 2, // fail fast if ES is unreachable
'timeout' => 10, // allow time for complex queries
],
]);
if ($config->inDebugMode()) { if ($config->inDebugMode()) {
$builder->setLogger($container->make(LoggerInterface::class)); $builder->setLogger($container->make(LoggerInterface::class));

View File

@ -10,7 +10,7 @@
* *
*/ */
namespace Sbp\Search\Save; namespace Blomstra\Search\Save;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Searchers; namespace Blomstra\Search\Searchers;
use Sbp\Search\Seeders\CommentSeeder; use Blomstra\Search\Seeders\CommentSeeder;
class CommentPostSearcher extends Searcher class CommentPostSearcher extends Searcher
{ {

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Searchers; namespace Blomstra\Search\Searchers;
use Sbp\Search\Seeders\DiscussionSeeder; use Blomstra\Search\Seeders\DiscussionSeeder;
class DiscussionSearcher extends Searcher class DiscussionSearcher extends Searcher
{ {

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Searchers; namespace Blomstra\Search\Searchers;
use Sbp\Search\Seeders\Seeder; use Blomstra\Search\Seeders\Seeder;
use Flarum\Settings\SettingsRepositoryInterface; use Flarum\Settings\SettingsRepositoryInterface;
abstract class Searcher abstract class Searcher

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Seeders; namespace Blomstra\Search\Seeders;
use Sbp\Search\Save\Document; use Blomstra\Search\Save\Document;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Api\Serializer\PostSerializer; use Flarum\Api\Serializer\PostSerializer;
use Flarum\Discussion\Discussion; use Flarum\Discussion\Discussion;

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Seeders; namespace Blomstra\Search\Seeders;
use Sbp\Search\Save\Document; use Blomstra\Search\Save\Document;
use Flarum\Api\Serializer\DiscussionSerializer; use Flarum\Api\Serializer\DiscussionSerializer;
use Flarum\Discussion\Discussion; use Flarum\Discussion\Discussion;
use Flarum\Discussion\Event as Core; use Flarum\Discussion\Event as Core;
@ -24,7 +24,7 @@ use FoF\Byobu\Events as Byobu;
use FoF\DiscussionViews\Events\DiscussionWasViewed; use FoF\DiscussionViews\Events\DiscussionWasViewed;
use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection; use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
class DiscussionSeeder extends Seeder class DiscussionSeeder extends Seeder
@ -70,7 +70,7 @@ class DiscussionSeeder extends Seeder
{ {
$events->listen([ $events->listen([
// flarum/core events // flarum/core events
Core\Started::class, Core\Restored::class, Core\Renamed::class, Core\Started::class, Core\Restored::class,
// fof/byobu discussion recipients events. // fof/byobu discussion recipients events.
Byobu\DiscussionMadePublic::class, Byobu\RemovedSelf::class, Byobu\RecipientsChanged::class, Byobu\DiscussionMadePublic::class, Byobu\RemovedSelf::class, Byobu\RecipientsChanged::class,
], function ($event) use ($callable) { ], function ($event) use ($callable) {
@ -121,7 +121,6 @@ class DiscussionSeeder extends Seeder
'id' => $this->type().':'.$model->id, 'id' => $this->type().':'.$model->id,
'rawId' => $model->id, 'rawId' => $model->id,
'content' => $model->title, 'content' => $model->title,
'title' => $model->title,
'created_at' => $model->created_at?->toAtomString(), 'created_at' => $model->created_at?->toAtomString(),
'updated_at' => $model->last_posted_at?->toAtomString(), 'updated_at' => $model->last_posted_at?->toAtomString(),
'is_private' => $model->is_private, 'is_private' => $model->is_private,

View File

@ -10,9 +10,9 @@
* *
*/ */
namespace Sbp\Search\Seeders; namespace Blomstra\Search\Seeders;
use Sbp\Search\Save\Document; use Blomstra\Search\Save\Document;
use Flarum\Extension\ExtensionManager; use Flarum\Extension\ExtensionManager;
use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Builder;