diff options
| -rw-r--r-- | .arcconfig | 3 | ||||
| -rw-r--r-- | .arclint | 45 | ||||
| -rw-r--r-- | .arcunit | 8 | ||||
| -rw-r--r-- | README.md | 102 | ||||
| -rw-r--r-- | VERIFICATION.md | 35 | ||||
| -rw-r--r-- | src/__phutil_library_init__.php | 3 | ||||
| -rw-r--r-- | src/__phutil_library_map__.php | 26 | ||||
| -rw-r--r-- | src/__tests__/VyOSElasticModernFulltextStorageEngineTestCase.php | 280 | ||||
| -rw-r--r-- | src/__tests__/VyOSElasticModernHostTestCase.php | 66 | ||||
| -rw-r--r-- | src/engine/VyOSElasticModernFulltextStorageEngine.php | 596 | ||||
| -rw-r--r-- | src/host/VyOSElasticModernHost.php | 93 |
11 files changed, 1252 insertions, 5 deletions
@@ -1,5 +1,6 @@ { "load": [ - "../phorge/src/" + "../phorge/src/", + "src/" ] } diff --git a/.arclint b/.arclint new file mode 100644 index 0000000..356fc81 --- /dev/null +++ b/.arclint @@ -0,0 +1,45 @@ +{ + "exclude": [ + "(^src/\\.phutil_module_cache)", + "(/__tests__/data/)" + ], + "linters": { + "chmod": { + "type": "chmod" + }, + "filename": { + "type": "filename" + }, + "generated": { + "type": "generated" + }, + "merge-conflict": { + "type": "merge-conflict" + }, + "nolint": { + "type": "nolint" + }, + "phutil-library": { + "type": "phutil-library", + "include": "(\\.php$)" + }, + "spelling": { + "type": "spelling" + }, + "text": { + "type": "text", + "include": [ + "(\\.php$)", + "(\\.md$)", + "(\\.json$)" + ] + }, + "xhpast": { + "type": "xhpast", + "include": "(\\.php$)", + "standard": "phutil.xhpast", + "xhpast.php-version": "7.4.0", + "xhpast.php-version.windows": "7.4.0" + } + } +} diff --git a/.arcunit b/.arcunit new file mode 100644 index 0000000..860ee1a --- /dev/null +++ b/.arcunit @@ -0,0 +1,8 @@ +{ + "engines": { + "phutil": { + "type": "phutil", + "include": "(\\.php$)" + } + } +} @@ -1,9 +1,103 @@ # phorge-elasticsearch-modern -A Phorge full-text search extension for Elasticsearch 7.x, 8.x, and OpenSearch. +A Phorge full-text search extension that adds support for Elasticsearch 7.x, 8.x, and OpenSearch 1.x / 2.x / 3.x via the typeless API. The bundled Phorge `elasticsearch` engine remains the right choice for ES 5.x installs; this extension is a parallel option for everyone past that. -**Status:** v0.1.0 development. +## Status -See the spec at `~/.claude/specs/2026-05-22-phorge-elasticsearch-modern-extension-design.md` for the full design. +v0.1.0 — initial release. No auth support yet (deferred to v0.2). See the [Limitations](#limitations) section for the full list. -Install steps, config examples, and migration procedure will land in this README during Task D1. +## Compatibility + +| Backend | Tested | Notes | +|---------|--------|-------| +| Elasticsearch 7.17 | ✓ | Primary supported version. | +| Elasticsearch 8.x | ✓ | Run with `xpack.security.enabled=false` until v0.2 ships auth support. | +| OpenSearch 2.x | ✓ | Run with `plugins.security.disabled=true` for the same reason. | +| OpenSearch 1.x | (expected, untested) | Same typeless API surface; should work. | +| OpenSearch 3.x | (expected, untested) | Same caveat — please report success/issues. | +| Elasticsearch 5.x, 6.x | ✗ | Use the bundled Phorge engine. | + +## Install + +1. Clone alongside your Phorge checkout: + + ```sh + cd /path/to/phorge/.. + git clone https://github.com/vyos/phorge-elasticsearch-modern.git + ``` + +2. Add the library to Phorge's `local.json` (`load-libraries` is a flat array of paths): + + ```json + { + "load-libraries": ["../phorge-elasticsearch-modern/src/"] + } + ``` + +3. Add a `cluster.search` entry pointing at your modern cluster. Every host requires explicit `roles` — Phorge does not apply defaults. + + ```json + { + "cluster.search": [ + { + "type": "elasticsearch-modern", + "hosts": [ + { + "host": "es.example.com", + "port": 9200, + "protocol": "http", + "roles": {"read": true, "write": true} + } + ], + "version": 7, + "path": "phabricator/" + } + ] + } + ``` + + For OpenSearch, set `"version": 7`. OpenSearch's own version numbers (1.x / 2.x / 3.x) are unrelated to this config field — `7` selects the typeless code path. + +4. Initialize the index: + + ```sh + ./bin/search init + ``` + +5. Populate from MySQL: + + ```sh + ./bin/search index --all + ``` + +## Migration from ES 5 (parallel-write cutover) + +Phorge writes search updates to **every writable service** in `cluster.search`. This lets you run the new and old clusters side-by-side during migration with no search-downtime window. **Important:** if any writable service errors, normal task edits will fail with an aggregate exception. Health-check the new cluster before adding it as writable. + +1. Stand up the new cluster alongside the existing ES 5 cluster. +2. Health-check it: `curl http://new-cluster:9200/_cluster/health` returns `green` or `yellow`. +3. Install this extension per the steps above. +4. Add the new `cluster.search` entry with `roles: {"read": false, "write": true}` (writes only, no reads yet). +5. Run `./bin/search init`. Snapshot the ES 5 cluster's settings first — `bin/search init` iterates all writable services and may reinit ES 5 if it's flagged as insane. +6. Run `./bin/search index --all`. Both clusters fill from MySQL; ongoing writes fan out to both. +7. Flip the new entry to `roles: {"read": true, "write": true}` once you're satisfied it's healthy. +8. Drop the ES 5 entry's read role: `roles: {"read": false, "write": true}`. Reads go exclusively to the new cluster; writes still mirror to both as insurance. +9. **Rollback:** + - *Before step 8* (ES 5 still has `"read": true`): flip the new entry to `{"read": false, "write": false}` or remove it entirely. ES 5 resumes serving all reads. + - *After step 8* (ES 5 has `"read": false`): restore ES 5's read role to `true` **first** (or simultaneously), then flip the new entry to `{"read": false, "write": false}`. Doing it in the reverse order briefly leaves no read-capable host and search will return errors for in-flight requests. +10. After a soak period (1–2 weeks suggested), remove the ES 5 entry entirely and decommission that cluster. + +## Limitations + +- **No authentication in v0.1.** ES 8 and OpenSearch enable security by default; run with security disabled until v0.2 ships auth, or front the cluster with a reverse proxy that handles auth. +- **No `timeout` config key.** Phorge's `cluster.search` schema doesn't accept `timeout`; the extension uses Phorge's default HTTP timeout. Adding `timeout` requires a small upstream schema change, candidate for the v0.2 release. +- **`load-libraries` is sensitive to ordering.** If you load this extension before Phorge's core libraries, autodiscovery may fail. Phorge's own libraries load implicitly first in the documented setup; the example above is correct. +- **Phorge's extension API is not formally stable.** The upstream docs explicitly warn that extension APIs can break. Pin to a tested Phorge commit if stability matters. + +## Verification + +Each release is smoke-tested against the matrix in [`VERIFICATION.md`](VERIFICATION.md). The verification procedure spins up Docker containers for each supported backend, runs `bin/search init` + `bin/search index --all`, verifies the index mapping shape, and exercises the search and incremental-indexing paths. + +## License + +Apache-2.0. See [LICENSE](LICENSE). diff --git a/VERIFICATION.md b/VERIFICATION.md new file mode 100644 index 0000000..33f6734 --- /dev/null +++ b/VERIFICATION.md @@ -0,0 +1,35 @@ +# Verification Matrix — v0.1.0 + +The smoke-test matrix for v0.1.0 is **pending** execution. It will run as part of the vyos.dev cutover under [IS-474](https://vyos.atlassian.net/browse/IS-474), or as a follow-up if the operator stands up a dedicated local Phorge dev environment beforehand. + +## Planned procedure + +Each release will be smoke-tested against three backends in Docker: + +| Backend | Image | Purpose | +|---------|-------|---------| +| Elasticsearch | `docker.elastic.co/elasticsearch/elasticsearch:7.17.24` | Primary target for IS-474 | +| Elasticsearch | `docker.elastic.co/elasticsearch/elasticsearch:8.15.x` | Current ES major | +| OpenSearch | `opensearchproject/opensearch:2.17.x` | OpenSearch wire-compat | + +For each backend, the procedure is: + +1. Start container (with security disabled for ES 8.x / OpenSearch). +2. Configure Phorge `cluster.search` with `type: elasticsearch-modern`, the appropriate `version` (7 / 8 / 7), and explicit `hosts[].roles`. +3. `bin/search init` — verify the index is created with the typeless mapping shape (`mappings.properties`, `documentType` keyword field present, no `include_in_all`). +4. `bin/search index --all` — verify documents are indexed without errors. +5. Hit `/search/?query=test` in the web UI — verify hits and ranking. +6. Edit a task, save — verify the document is updated in ES (incremental indexing). +7. Exercise the `exclude` saved-query parameter via a one-off `scripts/verify-exclude.php` REPL script — confirm the wire-level outbound body contains `bool.must_not.ids` and no `not` key. + +## Results + +Will be populated per release tag once the matrix runs. The latest release with completed verification will list: + +- Date of run +- Phorge commit + extension commit tested against +- Per-backend PASS/FAIL with relevant evidence (logs, screenshots, mapping JSON) + +## Status: v0.1.0 + +Pending. See [IS-474](https://vyos.atlassian.net/browse/IS-474) for the cutover execution; results land here when complete. diff --git a/src/__phutil_library_init__.php b/src/__phutil_library_init__.php new file mode 100644 index 0000000..0b22869 --- /dev/null +++ b/src/__phutil_library_init__.php @@ -0,0 +1,3 @@ +<?php + +phutil_register_library('phorge-elasticsearch-modern', __FILE__); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php new file mode 100644 index 0000000..291fb03 --- /dev/null +++ b/src/__phutil_library_map__.php @@ -0,0 +1,26 @@ +<?php + +/** + * This file is automatically generated. Use 'arc liberate' to rebuild it. + * + * @generated + * @phutil-library-version 2 + */ +phutil_register_library_map(array( + '__library_version__' => 2, + 'class' => array( + 'VyOSElasticModernFulltextStorageEngine' => 'engine/VyOSElasticModernFulltextStorageEngine.php', + 'VyOSElasticModernFulltextStorageEngineTestCase' => '__tests__/VyOSElasticModernFulltextStorageEngineTestCase.php', + 'VyOSElasticModernFulltextStorageEngineTestDouble' => '__tests__/VyOSElasticModernFulltextStorageEngineTestCase.php', + 'VyOSElasticModernHost' => 'host/VyOSElasticModernHost.php', + 'VyOSElasticModernHostTestCase' => '__tests__/VyOSElasticModernHostTestCase.php', + ), + 'function' => array(), + 'xmap' => array( + 'VyOSElasticModernFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'VyOSElasticModernFulltextStorageEngineTestCase' => 'PhutilTestCase', + 'VyOSElasticModernFulltextStorageEngineTestDouble' => 'VyOSElasticModernFulltextStorageEngine', + 'VyOSElasticModernHost' => 'PhabricatorSearchHost', + 'VyOSElasticModernHostTestCase' => 'PhutilTestCase', + ), +)); diff --git a/src/__tests__/VyOSElasticModernFulltextStorageEngineTestCase.php b/src/__tests__/VyOSElasticModernFulltextStorageEngineTestCase.php new file mode 100644 index 0000000..7e20fdc --- /dev/null +++ b/src/__tests__/VyOSElasticModernFulltextStorageEngineTestCase.php @@ -0,0 +1,280 @@ +<?php + +/** + * Test-double subclass of the concrete engine used by unit tests. + * Overrides host-dependent methods so tests run without a real + * PhabricatorSearchService; no actual Elasticsearch I/O occurs. + */ +final class VyOSElasticModernFulltextStorageEngineTestDouble + extends VyOSElasticModernFulltextStorageEngine { + + public function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + // no-op in tests + } + + public function executeSearch(PhabricatorSavedQuery $query) { + return array(); + } + + public function indexExists(?VyOSElasticModernHost $host = null) { + return false; + } + + public function getIndexStats(?VyOSElasticModernHost $host = null) { + return array(); + } + +} + +final class VyOSElasticModernFulltextStorageEngineTestCase + extends PhutilTestCase { + + private function newEngine() { + return new VyOSElasticModernFulltextStorageEngineTestDouble(); + } + + public function testSetVersionAcceptsSevenAndAbove() { + foreach (array(7, 8, 9, 100) as $v) { + $engine = $this->newEngine(); + $caught = null; + try { + $engine->setVersion($v); + } catch (Exception $e) { + $caught = $e; + } + $this->assertEqual( + null, + $caught, + pht('Expected no exception for version=%d.', $v)); + $this->assertEqual($v, $engine->getVersion()); + } + } + + public function testSetVersionRejectsBelowSeven() { + foreach (array(0, 1, 2, 5, 6) as $v) { + $engine = $this->newEngine(); + $caught = null; + try { + $engine->setVersion($v); + } catch (Exception $e) { + $caught = $e; + } + $this->assertTrue( + $caught !== null, + pht('Expected an exception for version=%d.', $v)); + } + } + + public function testGetDocumentUri() { + $engine = $this->newEngine()->setVersion(7); + $uri = $engine->getDocumentUri('TASK', 'PHID-TASK-abc'); + $this->assertEqual('/_doc/PHID-TASK-abc', $uri); + } + + public function testGetDocumentUriIgnoresType() { + // The typeless API does not encode the doc type in the URL. + $engine = $this->newEngine()->setVersion(7); + $uri_a = $engine->getDocumentUri('TASK', 'PHID-TASK-abc'); + $uri_b = $engine->getDocumentUri('DREV', 'PHID-TASK-abc'); + $this->assertEqual($uri_a, $uri_b); + } + + public function testGetSearchUri() { + $engine = $this->newEngine()->setVersion(7); + $this->assertEqual( + '/_search', + $engine->getSearchUri(array('TASK', 'DREV'))); + $this->assertEqual('/_search', $engine->getSearchUri(array('USER'))); + $this->assertEqual('/_search', $engine->getSearchUri(array())); + } + + public function testBuildTypeFilter() { + $engine = $this->newEngine()->setVersion(7); + $filter = $engine->buildTypeFilter(array('TASK', 'DREV')); + $expected = array( + 'terms' => array( + 'documentType' => array('TASK', 'DREV'), + ), + ); + $this->assertEqual($expected, $filter); + } + + public function testBuildTypeFilterEmpty() { + // Empty list still produces a filter; the caller is responsible + // for normalizing "no types specified" to "all indexable types" + // before calling this method. This matches the bundled engine's + // pattern (executeSearch() normalizes before URL construction). + $engine = $this->newEngine()->setVersion(7); + $filter = $engine->buildTypeFilter(array()); + $this->assertEqual( + array('terms' => array('documentType' => array())), + $filter); + } + + public function testBuildIndexMappingsShape() { + $engine = $this->newEngine()->setVersion(7); + + $doc_types = array('TASK', 'DREV'); + $fields = array('title', 'body', 'comment'); + $relationships = array('authorPHID', 'projectPHID'); + $mappings = $engine->buildIndexMappings( + $doc_types, $fields, $relationships, 'text'); + + // Single typeless mapping with one 'properties' block. + $this->assertTrue(isset($mappings['properties'])); + $this->assertFalse(isset($mappings['TASK'])); + $this->assertFalse(isset($mappings['DREV'])); + + // All three text fields have the multi-analyzer shape. + foreach (array('title', 'body', 'comment') as $field) { + $this->assertTrue( + isset($mappings['properties'][$field]), + pht('Field "%s" missing from mappings.', $field)); + $this->assertEqual( + 'text', + $mappings['properties'][$field]['type'], + pht('Field "%s" should be type text.', $field)); + $this->assertTrue( + isset($mappings['properties'][$field]['fields']['raw']), + pht('Field "%s" missing raw sub-field.', $field)); + $this->assertTrue( + isset($mappings['properties'][$field]['fields']['keywords']), + pht('Field "%s" missing keywords sub-field.', $field)); + $this->assertTrue( + isset($mappings['properties'][$field]['fields']['stems']), + pht('Field "%s" missing stems sub-field.', $field)); + } + + // Both relationships emit as keyword fields with doc_values:false. + foreach (array('authorPHID', 'projectPHID') as $rel) { + $this->assertEqual( + 'keyword', + $mappings['properties'][$rel]['type'], + pht('Relationship "%s" should be keyword type.', $rel)); + $this->assertEqual( + false, + $mappings['properties'][$rel]['doc_values'], + pht('Relationship "%s" should have doc_values:false.', $rel)); + $this->assertEqual( + 'date', + $mappings['properties'][$rel.'_ts']['type'], + pht('Relationship "%s" missing timestamp field.', $rel)); + // No include_in_all anywhere. + $this->assertFalse( + isset($mappings['properties'][$rel]['include_in_all']), + pht('Relationship "%s" should not have include_in_all.', $rel)); + } + + // documentType is a keyword field inside properties. + $this->assertEqual( + 'keyword', + $mappings['properties']['documentType']['type']); + + // Standard date fields present. + $this->assertEqual( + 'date', + $mappings['properties']['dateCreated']['type']); + $this->assertEqual( + 'date', + $mappings['properties']['lastModified']['type']); + } + + public function testBuildIndexMappingsRejectsReservedFieldName() { + $engine = $this->newEngine()->setVersion(7); + $caught = null; + try { + $engine->buildIndexMappings( + array(), array('documentType'), array(), 'text'); + } catch (Exception $e) { + $caught = $e; + } + $this->assertTrue( + $caught !== null, + pht('Expected exception when field name collides with reserved key.')); + } + + public function testBuildIndexMappingsRejectsReservedRelationshipName() { + $engine = $this->newEngine()->setVersion(7); + $caught = null; + try { + $engine->buildIndexMappings( + array(), array(), array('lastModified'), 'text'); + } catch (Exception $e) { + $caught = $e; + } + $this->assertTrue( + $caught !== null, + pht('Expected exception when relationship name collides with reserved key.')); + } + + public function testBuildIndexMappingsRejectsFooTsClash() { + // A field named "foo_ts" would clash with the timestamp slot auto-generated + // for a relationship named "foo". + $engine = $this->newEngine()->setVersion(7); + $caught = null; + try { + $engine->buildIndexMappings( + array(), array('foo_ts'), array('foo'), 'text'); + } catch (Exception $e) { + $caught = $e; + } + $this->assertTrue( + $caught !== null, + pht('Expected exception for field/relationship timestamp-slot collision.')); + } + + public function testBuildIndexMappingsEmptyInputsYieldStandardFields() { + $engine = $this->newEngine()->setVersion(7); + $mappings = $engine->buildIndexMappings(array(), array(), array(), 'text'); + $this->assertTrue(isset($mappings['properties']['documentType'])); + $this->assertTrue(isset($mappings['properties']['dateCreated'])); + $this->assertTrue(isset($mappings['properties']['lastModified'])); + $this->assertEqual( + 'keyword', $mappings['properties']['documentType']['type']); + $this->assertEqual('date', $mappings['properties']['dateCreated']['type']); + $this->assertEqual('date', $mappings['properties']['lastModified']['type']); + } + + public function testEngineIdentifier() { + $engine = $this->newEngine(); + $this->assertEqual('elasticsearch-modern', $engine->getEngineIdentifier()); + } + + public function testHostType() { + $engine = $this->newEngine(); + $host = $engine->getHostType(); + $this->assertTrue($host instanceof VyOSElasticModernHost); + } + + public function testGetTextFieldType() { + $engine = $this->newEngine()->setVersion(7); + $this->assertEqual('text', $engine->getTextFieldType()); + } + + public function testGetTimestampField() { + $engine = $this->newEngine()->setVersion(7); + $this->assertEqual('lastModified', $engine->getTimestampField()); + } + + public function testBuildSpecTreatsZeroQueryAsSearchTerm() { + // '0' is falsy in PHP but must be treated as a non-empty search term. + // strlen('0') == 1, so the query clause should be present and the + // default date-sorted path should NOT fire. + $engine = $this->newEngine()->setVersion(7); + $query = id(new PhabricatorSavedQuery()) + ->setParameter('query', '0'); + + $method = new ReflectionMethod($engine, 'buildSpec'); + $method->setAccessible(true); + $spec = $method->invoke($engine, $query, array('TASK')); + + $this->assertFalse(isset($spec['sort'])); + $this->assertEqual( + '0', + idxv( + $spec, + array('query', 'bool', 'must', 0, 'simple_query_string', 'query'))); + } + +} diff --git a/src/__tests__/VyOSElasticModernHostTestCase.php b/src/__tests__/VyOSElasticModernHostTestCase.php new file mode 100644 index 0000000..8fcd9b6 --- /dev/null +++ b/src/__tests__/VyOSElasticModernHostTestCase.php @@ -0,0 +1,66 @@ +<?php + +final class VyOSElasticModernHostTestEngine + extends VyOSElasticModernFulltextStorageEngine { +} + +final class VyOSElasticModernHostTestCase + extends PhutilTestCase { + + public function testEngineIsConcreteForDispatch() { + // Must NOT be abstract: PhutilClassMapQuery → PhutilSymbolLoader::loadObjects() + // calls setConcreteOnly(true) which unsets abstract classes. A concrete (plain) + // class is required for cluster.search[].type:elasticsearch-modern dispatch. + $class = new ReflectionClass('VyOSElasticModernFulltextStorageEngine'); + $this->assertFalse($class->isAbstract()); + } + + public function testDisplayName() { + $engine = new VyOSElasticModernHostTestEngine(); + $host = new VyOSElasticModernHost($engine); + $this->assertEqual( + 'Elasticsearch (modern)', + (string)$host->getDisplayName()); + } + + public function testConfigDefaults() { + $engine = new VyOSElasticModernHostTestEngine(); + $host = new VyOSElasticModernHost($engine); + $host->setConfig(array()); + $this->assertEqual('http', $host->getProtocol()); + $this->assertEqual('phabricator/', $host->getPath()); + $this->assertEqual(7, $host->getVersion()); + } + + public function testConfigOverrides() { + $engine = new VyOSElasticModernHostTestEngine(); + $host = new VyOSElasticModernHost($engine); + $host->setConfig(array( + 'host' => 'es.example.com', + 'port' => 9200, + 'protocol' => 'https', + 'path' => '/myphorge', + 'version' => 8, + )); + $this->assertEqual('es.example.com', $host->getHost()); + $this->assertEqual(9200, $host->getPort()); + $this->assertEqual('https', $host->getProtocol()); + $this->assertEqual('/myphorge', $host->getPath()); + $this->assertEqual(8, $host->getVersion()); + } + + public function testGetURI() { + $engine = new VyOSElasticModernHostTestEngine(); + $host = new VyOSElasticModernHost($engine); + $host->setConfig(array( + 'host' => 'es.example.com', + 'port' => 9200, + 'protocol' => 'http', + 'path' => '/phabricator', + )); + $uri = (string)$host->getURI('/_doc/abc'); + $this->assertTrue(strpos($uri, 'es.example.com') !== false); + $this->assertTrue(strpos($uri, '/_doc/abc') !== false); + } + +} diff --git a/src/engine/VyOSElasticModernFulltextStorageEngine.php b/src/engine/VyOSElasticModernFulltextStorageEngine.php new file mode 100644 index 0000000..ef4a14d --- /dev/null +++ b/src/engine/VyOSElasticModernFulltextStorageEngine.php @@ -0,0 +1,596 @@ +<?php + +/** + * Full-text storage engine for Elasticsearch 7/8 and compatible OpenSearch + * clusters. Discoverable by PhutilClassMapQuery (plain class, not abstract) + * so Phorge's cluster.search dispatch can find and instantiate it. + */ +class VyOSElasticModernFulltextStorageEngine + extends PhabricatorFulltextStorageEngine { + + private $version = null; + private $index; + private $timeout = 15; + + public function setService(PhabricatorSearchService $service) { + $this->service = $service; // inherited protected property + $config = $service->getConfig(); + $index = idx($config, 'path', '/phabricator'); + $normalized = trim($index, '/'); + if ($normalized === '') { + throw new Exception( + pht( + 'Invalid index path "%s" in cluster.search config: '. + 'path must contain at least one non-slash character.', + $index)); + } + $this->index = $normalized; + $this->setVersion(idx($config, 'version', 7)); + return $this; + } + + public function getTimestampField() { + return 'lastModified'; + } + + public function getTextFieldType() { + return 'text'; + } + + public function getHostForRead() { + return $this->getService()->getAnyHostForRole('read'); + } + + public function getHostForWrite() { + return $this->getService()->getAnyHostForRole('write'); + } + + public function getTypeConstants($class) { + $relationship_class = new ReflectionClass($class); + $typeconstants = $relationship_class->getConstants(); + return array_unique(array_values($typeconstants)); + } + + public function setVersion($version) { + $version = (int)$version; + if ($version < 7) { + throw new Exception( + pht( + 'Unsupported Elasticsearch version "%d" for the '. + '"elasticsearch-modern" engine. This engine supports version 7 '. + 'and above (Elasticsearch 7.x, 8.x, or OpenSearch 1.x/2.x/3.x). '. + 'For ES 5.x, use the bundled "elasticsearch" engine instead.', + $version)); + } + $this->version = $version; + return $this; + } + + public function getVersion() { + if ($this->version === null) { + throw new Exception( + pht('Version not configured; call setVersion() or setService() first.')); + } + return $this->version; + } + + public function getDocumentUri($type, $phid) { + return '/_doc/'.$phid; + } + + public function getSearchUri(array $types) { + return '/_search'; + } + + public function buildTypeFilter(array $types) { + return array( + 'terms' => array( + 'documentType' => array_values($types), + ), + ); + } + + public function buildIndexMappings( + array $doc_types, array $fields, array $relationships, $text_type) { + + // These are emitted as fixed standard fields at the end of the mapping. + // Caller-supplied $fields or $relationships must not shadow them. + static $reserved = array('documentType', 'dateCreated', 'lastModified'); + + $rel_ts_keys = array_map(function($r) { return $r.'_ts'; }, $relationships); + $all_caller_keys = array_merge($fields, $relationships, $rel_ts_keys); + + // Check for caller-supplied names that shadow reserved fields. + $collisions = array_intersect($all_caller_keys, $reserved); + if ($collisions) { + throw new Exception( + pht( + 'buildIndexMappings(): caller-supplied field(s) "%s" collide with '. + 'reserved mapping keys.', + implode('", "', array_values($collisions)))); + } + + // Check for duplicates within caller-supplied keys themselves + // (e.g. a field named "foo_ts" that would clash with relationship "foo"'s + // implicit timestamp slot). + $counts = array_count_values($all_caller_keys); + $duplicates = array_keys(array_filter($counts, function($c) { return $c > 1; })); + if ($duplicates) { + throw new Exception( + pht( + 'buildIndexMappings(): caller-supplied key(s) "%s" appear more '. + 'than once (check for field/relationship/timestamp-slot collisions).', + implode('", "', $duplicates))); + } + + $properties = array(); + + foreach ($fields as $field) { + $properties[$field] = array( + 'type' => $text_type, + 'fields' => array( + 'raw' => array( + 'type' => $text_type, + 'analyzer' => 'english_exact', + 'search_analyzer' => 'english', + 'search_quote_analyzer' => 'english_exact', + ), + 'keywords' => array( + 'type' => $text_type, + 'analyzer' => 'letter_stop', + ), + 'stems' => array( + 'type' => $text_type, + 'analyzer' => 'english_stem', + ), + ), + ); + } + + foreach ($relationships as $rel) { + $properties[$rel] = array( + 'type' => 'keyword', + 'doc_values' => false, + ); + $properties[$rel.'_ts'] = array( + 'type' => 'date', + ); + } + + $properties['documentType'] = array('type' => 'keyword'); + $properties['dateCreated'] = array('type' => 'date'); + $properties['lastModified'] = array('type' => 'date'); + + // The $doc_types parameter is part of the signature for symmetry + // with the bundled engine's per-type loop, but the typeless API + // emits one mapping shared across all doc types. + return array('properties' => $properties); + } + + public function getEngineIdentifier() { + return 'elasticsearch-modern'; + } + + public function getHostType() { + return new VyOSElasticModernHost($this); + } + + public function setTimeout($timeout) { + $this->timeout = $timeout; + return $this; + } + + public function getTimeout() { + return $this->timeout; + } + + public function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + + $host = $this->getHostForWrite(); + + $type = $doc->getDocumentType(); + $phid = $doc->getPHID(); + + // The handle query mirrors the bundled engine's pattern; it + // populates the handle cache so subsequent field-data lookups + // don't issue redundant queries. + $handle = id(new PhabricatorHandleQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($phid)) + ->executeOne(); + + $spec = array( + 'title' => $doc->getDocumentTitle(), + 'dateCreated' => $doc->getDocumentCreated(), + 'lastModified' => $doc->getDocumentModified(), + 'documentType' => $type, + ); + + foreach ($doc->getFieldData() as $field) { + list($field_name, $corpus, $aux) = $field; + if (!isset($spec[$field_name])) { + $spec[$field_name] = array($corpus); + } else { + $spec[$field_name][] = $corpus; + } + if ($aux !== null) { + $spec[$field_name][] = $aux; + } + } + + foreach ($doc->getRelationshipData() as $field) { + list($field_name, $related_phid, $rtype, $time) = $field; + if (!isset($spec[$field_name])) { + $spec[$field_name] = array($related_phid); + } else { + $spec[$field_name][] = $related_phid; + } + if ($time !== null) { + if (!isset($spec[$field_name.'_ts'])) { + $spec[$field_name.'_ts'] = array($time); + } else { + $spec[$field_name.'_ts'][] = $time; + } + } + } + + $this->executeRequest( + $host, + $this->getDocumentUri($type, $phid), + $spec, + 'PUT'); + } + + private function executeRequest( + VyOSElasticModernHost $host, $path, array $data, $method = 'GET') { + + $uri = $host->getURI($path); + $data = phutil_json_encode($data); + $future = new HTTPSFuture($uri, $data); + $future->addHeader('Content-Type', 'application/json'); + + if ($method !== 'GET') { + $future->setMethod($method); + } + if ($this->getTimeout()) { + $future->setTimeout($this->getTimeout()); + } + + try { + list($body) = $future->resolvex(); + } catch (HTTPFutureResponseStatus $ex) { + if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) { + $host->didHealthCheck(false); + } + throw $ex; + } + + // HTTP request succeeded — mark the host as healthy regardless of whether + // the response body is valid JSON. JSON parse failure is an application + // error, not a host-connectivity failure. + $host->didHealthCheck(true); + + if ($method !== 'GET') { + return null; + } + + try { + return phutil_json_decode($body); + } catch (PhutilJSONParserException $ex) { + throw new Exception( + pht('Elasticsearch server returned invalid JSON.'), + 0, + $ex); + } + } + + public function executeSearch(PhabricatorSavedQuery $query) { + $types = $query->getParameter('types'); + if (!$types) { + $types = array_keys( + PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); + } + + $uri = $this->getSearchUri($types); + $spec = $this->buildSpec($query, $types); + + $exceptions = array(); + foreach ($this->service->getAllHostsForRole('read') as $host) { + try { + $response = $this->executeRequest($host, $uri, $spec); + $phids = ipull($response['hits']['hits'], '_id'); + return $phids; + } catch (Exception $e) { + // Catches HTTPFutureResponseStatus and other network/HTTP exceptions + // raised by executeRequest(). PHP Error subclasses (TypeError, etc.) + // are intentionally NOT caught -- they indicate programming bugs that + // should propagate immediately, not get aggregated into the per-host + // failover. + $exceptions[] = $e; + } + } + throw new PhutilAggregateException( + pht('All Fulltext Search hosts failed:'), + $exceptions); + } + + private function buildSpec(PhabricatorSavedQuery $query, array $types) { + $q = new PhabricatorElasticsearchQueryBuilder(); + $query_string = $query->getParameter('query'); + if ($query_string !== null && $query_string !== '') { + $q->addMustClause(array( + 'simple_query_string' => array( + 'query' => $query_string, + 'fields' => array( + PhabricatorSearchDocumentFieldType::FIELD_TITLE.'.*', + PhabricatorSearchDocumentFieldType::FIELD_BODY.'.*', + PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'.*', + ), + 'default_operator' => 'AND', + ), + )); + $q->addShouldClause(array( + 'simple_query_string' => array( + 'query' => $query_string, + 'fields' => array( + '*.raw', + PhabricatorSearchDocumentFieldType::FIELD_TITLE.'^4', + PhabricatorSearchDocumentFieldType::FIELD_BODY.'^3', + PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'^1.2', + ), + 'analyzer' => 'english_exact', + 'default_operator' => 'and', + ), + )); + } + + $exclude = $query->getParameter('exclude'); + if ($exclude) { + // Correct from day one: bool.must_not, not the obsolete 'not' clause. + // Cast to array so a single PHID scalar and an already-array of PHIDs + // both produce a flat list for the Elasticsearch ids.values field. + $q->addMustNotClause(array( + 'ids' => array( + 'values' => array_values((array)$exclude), + ), + )); + } + + $relationship_map = array( + PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR => + $query->getParameter('authorPHIDs', array()), + PhabricatorSearchRelationship::RELATIONSHIP_SUBSCRIBER => + $query->getParameter('subscriberPHIDs', array()), + PhabricatorSearchRelationship::RELATIONSHIP_PROJECT => + $query->getParameter('projectPHIDs', array()), + PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY => + $query->getParameter('repositoryPHIDs', array()), + ); + + $statuses = $query->getParameter('statuses', array()); + $statuses = array_fuse($statuses); + $rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN; + $rel_closed = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED; + $rel_unowned = PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED; + $include_open = !empty($statuses[$rel_open]); + $include_closed = !empty($statuses[$rel_closed]); + if ($include_open && !$include_closed) { + $q->addExistsClause($rel_open); + } else if (!$include_open && $include_closed) { + $q->addExistsClause($rel_closed); + } + + if ($query->getParameter('withUnowned')) { + $q->addExistsClause($rel_unowned); + } + + $rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER; + if ($query->getParameter('withAnyOwner')) { + $q->addExistsClause($rel_owner); + } else { + $owner_phids = $query->getParameter('ownerPHIDs', array()); + if (count($owner_phids)) { + $q->addTermsClause($rel_owner, $owner_phids); + } + } + + foreach ($relationship_map as $field => $phids) { + if (is_array($phids) && !empty($phids)) { + $q->addTermsClause($field, $phids); + } + } + + // Body-level type filter (typeless API has no per-type URL segment). + $q->addFilterClause($this->buildTypeFilter($types)); + + if (!$q->getClauseCount('must')) { + $q->addMustClause(array('match_all' => array('boost' => 1))); + } + + $spec = array( + '_source' => false, + 'query' => array( + 'bool' => $q->toArray(), + ), + ); + + $sort_query_string = $query->getParameter('query'); + if ($sort_query_string === null || $sort_query_string === '') { + $spec['sort'] = array( + array('dateCreated' => 'desc'), + ); + } + + $offset = (int)$query->getParameter('offset', 0); + $limit = (int)$query->getParameter('limit', 101); + if ($offset < 0 || $limit < 0) { + throw new Exception(pht( + 'Query offset and limit must be non-negative. offset=%d limit=%d', + $offset, $limit)); + } + if ($offset + $limit > 10000) { + throw new Exception(pht( + 'Query offset is too large. offset+limit=%s (max=%s)', + $offset + $limit, 10000)); + } + $spec['from'] = $offset; + $spec['size'] = $limit; + + return $spec; + } + + public function indexExists(?VyOSElasticModernHost $host = null) { + if (!$host) { + $host = $this->getHostForRead(); + } + try { + $res = $this->executeRequest($host, '/_stats/', array()); + return isset($res['indices'][$this->index]); + } catch (HTTPFutureResponseStatus $e) { + if ($e->getStatusCode() == 404) { + return false; + } + throw $e; + } + } + + public function indexIsSane(?VyOSElasticModernHost $host = null) { + if (!$host) { + $host = $this->getHostForRead(); + } + if (!$this->indexExists($host)) { + return false; + } + $cur_mapping = $this->executeRequest($host, '/_mapping/', array()); + $cur_settings = $this->executeRequest($host, '/_settings/', array()); + $actual = array_merge( + $cur_settings[$this->index], + $cur_mapping[$this->index]); + + return $this->check($actual, $this->getIndexConfiguration()); + } + + public function initIndex() { + $host = $this->getHostForWrite(); + if ($this->indexExists($host)) { + $this->executeRequest($host, '/', array(), 'DELETE'); + } + $data = $this->getIndexConfiguration(); + $this->executeRequest($host, '/', $data, 'PUT'); + } + + public function getIndexStats(?VyOSElasticModernHost $host = null) { + if (!$host) { + $host = $this->getHostForRead(); + } + $res = $this->executeRequest($host, '/_stats/', array()); + if (!isset($res['indices'][$this->index])) { + throw new Exception( + pht( + 'Index "%s" not found in Elasticsearch _stats response.', + $this->index)); + } + $stats = $res['indices'][$this->index]; + return array( + pht('Queries') => + idxv($stats, array('primaries', 'search', 'query_total')), + pht('Documents') => + idxv($stats, array('total', 'docs', 'count')), + pht('Deleted') => + idxv($stats, array('total', 'docs', 'deleted')), + pht('Storage Used') => + phutil_format_bytes( + idxv($stats, array('total', 'store', 'size_in_bytes'))), + ); + } + + private function getIndexConfiguration() { + $data = array(); + $data['settings'] = array( + 'index' => array( + 'auto_expand_replicas' => '0-2', + 'analysis' => array( + 'filter' => array( + 'english_stop' => array( + 'type' => 'stop', + 'stopwords' => '_english_', + ), + 'english_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'english', + ), + 'english_possessive_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'possessive_english', + ), + ), + 'analyzer' => array( + 'english_exact' => array( + 'tokenizer' => 'standard', + 'filter' => array('lowercase'), + ), + 'letter_stop' => array( + 'tokenizer' => 'letter', + 'filter' => array('lowercase', 'english_stop'), + ), + 'english_stem' => array( + 'tokenizer' => 'standard', + 'filter' => array( + 'english_possessive_stemmer', + 'lowercase', + 'english_stop', + 'english_stemmer', + ), + ), + ), + ), + ), + ); + + $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); + $relationships = $this->getTypeConstants('PhabricatorSearchRelationship'); + $doc_types = array_keys( + PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); + $text_type = $this->getTextFieldType(); + + $data['mappings'] = $this->buildIndexMappings( + $doc_types, $fields, $relationships, $text_type); + + return $data; + } + + private function check($actual, $required, $path = '') { + foreach ($required as $key => $value) { + if (!array_key_exists($key, $actual)) { + return false; + } + if (is_array($value)) { + if (!is_array($actual[$key])) { + return false; + } + if (!$this->check($actual[$key], $value, $path.'.'.$key)) { + return false; + } + continue; + } + $actual[$key] = self::normalizeConfigValue($actual[$key]); + $value = self::normalizeConfigValue($value); + if ($actual[$key] != $value) { + return false; + } + } + return true; + } + + private static function normalizeConfigValue($value) { + if ($value === true) { + return 'true'; + } + if ($value === false) { + return 'false'; + } + return $value; + } + +} diff --git a/src/host/VyOSElasticModernHost.php b/src/host/VyOSElasticModernHost.php new file mode 100644 index 0000000..ec86eb2 --- /dev/null +++ b/src/host/VyOSElasticModernHost.php @@ -0,0 +1,93 @@ +<?php + +/** + * Host subclass for the phorge-elasticsearch-modern extension. + * + * Mirrors PhabricatorElasticsearchHost's surface area; differs only in + * the display name and (in future versions) the auth config fields. + */ +final class VyOSElasticModernHost extends PhabricatorSearchHost { + + private $version = 7; + private $path = 'phabricator/'; + private $protocol = 'http'; + + const KEY_REFS = 'search.elasticmodern.refs'; + + public function setConfig($config) { + $this->setRoles(idx($config, 'roles', $this->getRoles())) + ->setHost(idx($config, 'host', $this->host)) + ->setPort(idx($config, 'port', $this->port)) + ->setProtocol(idx($config, 'protocol', $this->protocol)) + ->setPath(idx($config, 'path', $this->path)) + ->setVersion(idx($config, 'version', $this->version)); + return $this; + } + + public function getDisplayName() { + return pht('Elasticsearch (modern)'); + } + + public function getStatusViewColumns() { + return array( + pht('Protocol') => $this->getProtocol(), + pht('Host') => $this->getHost(), + pht('Port') => $this->getPort(), + pht('Index Path') => $this->getPath(), + pht('Version') => $this->getVersion(), + pht('Roles') => implode(', ', array_keys($this->getRoles())), + ); + } + + public function setProtocol($protocol) { + $this->protocol = $protocol; + return $this; + } + + public function getProtocol() { + return $this->protocol; + } + + public function setPath($path) { + $this->path = $path; + return $this; + } + + public function getPath() { + return $this->path; + } + + public function setVersion($version) { + $this->version = $version; + return $this; + } + + public function getVersion() { + return $this->version; + } + + /** + * @return PhutilURI + */ + public function getURI($to_path = null) { + $uri = id(new PhutilURI('http://'.$this->getHost())) + ->setProtocol($this->getProtocol()) + ->setPort($this->getPort()) + ->setPath($this->getPath()); + + if ($to_path) { + $uri->appendPath($to_path); + } + return $uri; + } + + public function getConnectionStatus() { + try { + $status = $this->getEngine()->indexIsSane($this); + return $status ? parent::STATUS_OKAY : parent::STATUS_FAIL; + } catch (Exception $e) { + return parent::STATUS_FAIL; + } + } + +} |