diff --git a/CHANGELOG.md b/CHANGELOG.md index 047431f465a39..0da3a49856c71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823)) - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842)) - Handle dependencies between analyzers ([#19248](https://github.com/opensearch-project/OpenSearch/pull/19248)) +- Fix `_field_caps` returning empty results and corrupted field names for `disable_objects: true` mappings ([#20800](https://github.com/opensearch-project/OpenSearch/pull/20800)) ### Dependencies - Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) diff --git a/gradle/run.gradle b/gradle/run.gradle index 1b3c6f12bf514..3a5478848ed72 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -60,7 +60,9 @@ testClusters { for (String p : installedPlugins) { // check if its a local plugin first if (project.findProject(':plugins:' + p) != null) { - plugin('plugins:' + p) + plugin(':plugins:' + p) + } else if (project.findProject(':sandbox:plugins:' + p) != null) { + plugin(':sandbox:plugins:' + p) } else { // attempt to fetch it from maven project.repositories.mavenLocal() diff --git a/release-notes/opensearch.release-notes-3.6.0.md b/release-notes/opensearch.release-notes-3.6.0.md new file mode 100644 index 0000000000000..6474cc68d0e4f --- /dev/null +++ b/release-notes/opensearch.release-notes-3.6.0.md @@ -0,0 +1,70 @@ +## Version 3.6.0 Release Notes + +Compatible with OpenSearch and OpenSearch Dashboards version 3.6.0 + +### Added +- Add Scroll API support for Workload Management rule-based autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) +- Add intra-segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) +- Add warmup phase for pull-based ingestion ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) +- Add custom search settings support for Workload Management groups ([#20536](https://github.com/opensearch-project/OpenSearch/pull/20536)) +- Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) +- Add bitmap64 query support ([#20606](https://github.com/opensearch-project/OpenSearch/pull/20606)) +- Add IndexWarmer support for replica shards with segment replication enabled ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650)) +- Support Docker distribution builds for ppc64le, arm64, and s390x ([#20678](https://github.com/opensearch-project/OpenSearch/pull/20678)) +- Fall back to Netty client when AWS CRT client is unavailable on the target platform ([#20698](https://github.com/opensearch-project/OpenSearch/pull/20698)) +- Add TLS certificate hot-reload for Arrow Flight transport ([#20700](https://github.com/opensearch-project/OpenSearch/pull/20700)) +- Add mapper_settings support and field_mapping mapper type for pull-based ingestion ([#20722](https://github.com/opensearch-project/OpenSearch/pull/20722), [#20729](https://github.com/opensearch-project/OpenSearch/pull/20729)) +- Add ref_path support for package-based Hunspell dictionary loading ([#20840](https://github.com/opensearch-project/OpenSearch/pull/20840)) +- Add node-level JVM and CPU runtime metrics ([#20844](https://github.com/opensearch-project/OpenSearch/pull/20844)) + +### Changed +- Choose the best performing node when writing with append-only indices ([#20065](https://github.com/opensearch-project/OpenSearch/pull/20065)) +- Prevent criteria update for context-aware indices ([#20250](https://github.com/opensearch-project/OpenSearch/pull/20250)) +- Support expected remote cluster name in cross-cluster search sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) +- Expose wrapped scorer in ProfileScorer for scorer tree visibility ([#20607](https://github.com/opensearch-project/OpenSearch/pull/20607)) +- Harden detection of HTTP/3 support by verifying QUIC native library availability ([#20680](https://github.com/opensearch-project/OpenSearch/pull/20680)) +- Leverage segment-global ordinal mapping for more efficient terms aggregation ([#20683](https://github.com/opensearch-project/OpenSearch/pull/20683)) +- Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) + +### Removed +- Remove experimental tag for pull-based ingestion ([#20704](https://github.com/opensearch-project/OpenSearch/pull/20704)) + +### Fixed +- Fix synonym_graph filter failure with word_delimiter_graph when using whitespace or classic tokenizer ([#19248](https://github.com/opensearch-project/OpenSearch/pull/19248)) +- Fix CriteriaBasedCodec to work with delegate codec ([#20442](https://github.com/opensearch-project/OpenSearch/pull/20442)) +- Relax updatedAt validation for Workload Management workload group creation ([#20486](https://github.com/opensearch-project/OpenSearch/pull/20486)) +- Fix listBlobsByPrefixInSortedOrder in EncryptedBlobContainer to adhere to limit parameter ([#20514](https://github.com/opensearch-project/OpenSearch/pull/20514)) +- Add range validations in query builder and field mapper ([#20518](https://github.com/opensearch-project/OpenSearch/pull/20518)) +- Fix copy_to functionality for geo_point fields with object and array values ([#20542](https://github.com/opensearch-project/OpenSearch/pull/20542)) +- Fix segment replication infinite retry due to stale metadata checkpoint ([#20551](https://github.com/opensearch-project/OpenSearch/pull/20551)) +- Fix SecurityException when changing opensearch.cgroups.hierarchy.override ([#20565](https://github.com/opensearch-project/OpenSearch/pull/20565)) +- Implement batched deletions of stale ClusterMetadataManifests in remote store ([#20566](https://github.com/opensearch-project/OpenSearch/pull/20566)) +- Fix SLF4J component error ([#20587](https://github.com/opensearch-project/OpenSearch/pull/20587)) +- Fix service startup failure on Windows with OpenJDK by updating procrun to 1.5.1 ([#20615](https://github.com/opensearch-project/OpenSearch/pull/20615)) +- Fix regression in terms aggregation optimization ([#20623](https://github.com/opensearch-project/OpenSearch/pull/20623)) +- Handle ShardSearchFailure properly in gRPC transport ([#20641](https://github.com/opensearch-project/OpenSearch/pull/20641)) +- Add accessUnixDomainSocket permission for gRPC transport ([#20649](https://github.com/opensearch-project/OpenSearch/pull/20649)) +- Fix collision of index patterns ([#20702](https://github.com/opensearch-project/OpenSearch/pull/20702)) +- Fix stream transport TLS cert hot-reload by using live SSLContext from SecureTransportSettingsProvider ([#20734](https://github.com/opensearch-project/OpenSearch/pull/20734)) +- Show heap percent threshold in search task cancellation message ([#20779](https://github.com/opensearch-project/OpenSearch/pull/20779)) +- Fix JSON escaping in task details log metadata ([#20802](https://github.com/opensearch-project/OpenSearch/pull/20802)) +- Fix field_caps returning empty results for disable_objects mappings ([#20814](https://github.com/opensearch-project/OpenSearch/pull/20814)) +- Fix terms lookup subquery to use cluster max_clause_count setting ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823)) +- Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842)) + +### Dependencies +- Bump Apache Lucene from 10.3.2 to 10.4.0 ([#20735](https://github.com/opensearch-project/OpenSearch/pull/20735)) +- Bump Netty to 4.2.10.Final ([#20586](https://github.com/opensearch-project/OpenSearch/pull/20586)) +- Bump Project Reactor to 3.8.4 and Reactor Netty to 1.3.4 ([#20589](https://github.com/opensearch-project/OpenSearch/pull/20589), [#20834](https://github.com/opensearch-project/OpenSearch/pull/20834)) +- Bump OpenTelemetry to 1.60.1 and OpenTelemetry Semconv to 1.40.0 ([#20737](https://github.com/opensearch-project/OpenSearch/pull/20737), [#20797](https://github.com/opensearch-project/OpenSearch/pull/20797)) +- Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) +- Bump org.jruby.joni:joni from 2.2.3 to 2.2.7 ([#20714](https://github.com/opensearch-project/OpenSearch/pull/20714), [#20759](https://github.com/opensearch-project/OpenSearch/pull/20759)) +- Bump org.jruby.jcodings:jcodings from 1.0.63 to 1.0.64 ([#20713](https://github.com/opensearch-project/OpenSearch/pull/20713)) +- Bump org.tukaani:xz from 1.11 to 1.12 ([#20760](https://github.com/opensearch-project/OpenSearch/pull/20760)) +- Bump com.netflix.nebula.ospackage-base from 12.2.0 to 12.3.0 ([#20799](https://github.com/opensearch-project/OpenSearch/pull/20799)) +- Bump com.netflix.nebula:gradle-info-plugin to 16.2.1 ([#20825](https://github.com/opensearch-project/OpenSearch/pull/20825)) +- Bump com.nimbusds:nimbus-jose-jwt from 10.7 to 10.8 ([#20715](https://github.com/opensearch-project/OpenSearch/pull/20715)) +- Bump org.apache.commons:commons-text from 1.14.0 to 1.15.0 ([#20576](https://github.com/opensearch-project/OpenSearch/pull/20576)) +- Bump ch.qos.logback:logback-core from 1.5.24 to 1.5.27 ([#20525](https://github.com/opensearch-project/OpenSearch/pull/20525)) +- Bump ch.qos.logback:logback-classic from 1.5.27 to 1.5.32 ([#20761](https://github.com/opensearch-project/OpenSearch/pull/20761)) +- Bump org.jline:jline from 3.30.6 to 4.0.0 ([#20800](https://github.com/opensearch-project/OpenSearch/pull/20800)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/40_disable_objects.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/40_disable_objects.yml new file mode 100644 index 0000000000000..c33786b04a580 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/40_disable_objects.yml @@ -0,0 +1,139 @@ +--- +"Field caps with disable_objects returns leaf fields": + + - skip: + version: " - 3.4.99" + reason: "disable_objects was introduced in 3.5.0" + + - do: + indices.create: + index: test_disable_objects + body: + mappings: + properties: + attributes: + type: object + disable_objects: true + + - do: + index: + index: test_disable_objects + id: "1" + body: + attributes: + foo: + bar: baz + + - do: + indices.refresh: + index: test_disable_objects + + - do: + field_caps: + index: test_disable_objects + fields: "*" + + # Leaf field must exist + - is_true: fields.attributes\.foo\.bar + + # Parent object field must have type object + - match: { fields.attributes.object.type: object } + + # Intermediate path must not exist (since disable_objects is set) + - is_false: fields.attributes\.foo + +--- +"Field caps with disable_objects does not corrupt field names after second document indexed": + + - skip: + version: " - 3.4.99" + reason: "disable_objects was introduced in 3.5.0" + + - do: + indices.create: + index: test_disable_objects_merge + body: + mappings: + properties: + attributes: + type: object + disable_objects: true + + # doc 1: creates the disable_objects leaf field + - do: + index: + index: test_disable_objects_merge + id: "1" + body: + attributes: + foo: + bar: baz + + # doc 2: unrelated - must not corrupt the existing mapping + - do: + index: + index: test_disable_objects_merge + id: "2" + body: + key: 1 + + - do: + indices.refresh: + index: test_disable_objects_merge + + - do: + field_caps: + index: test_disable_objects_merge + fields: "*" + + # Leaf field must exist + - is_true: fields.attributes\.foo\.bar + + # Corrupted field name must not exist + - is_false: fields.attributes\.foo\.foo\.bar + + # The unrelated field must also exist + - is_true: fields.key + +--- +"Field caps with normal nested object returns intermediate paths": + + - do: + indices.create: + index: test_normal_nested + body: + mappings: + properties: + user: + type: object + properties: + address: + type: object + properties: + city: + type: keyword + + - do: + index: + index: test_normal_nested + id: "1" + body: + user: + address: + city: Chemnitz + + - do: + indices.refresh: + index: test_normal_nested + + - do: + field_caps: + index: test_normal_nested + fields: "*" + + # Leaf field must exist + - is_true: fields.user\.address\.city + + # Intermediate object paths must also exist + - match: { fields.user\.address.object.type: object } + - match: { fields.user.object.type: object } diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 73cb878d192f8..d6bb43a65319d 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -180,6 +180,12 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd if (mapperService.fieldType(parentField) == null) { // no field type, it must be an object field ObjectMapper mapper = mapperService.getObjectMapper(parentField); + if (mapper == null) { + // parentField is part of a literal dotted field name under a disable_objects=true parent + // No ObjectMapper exists for this intermediate path so skip it and continue up the chain + dotIndex = parentField.lastIndexOf('.'); + continue; + } String type = mapper.nested().isNested() ? "nested" : "object"; IndexFieldCapabilities fieldCap = new IndexFieldCapabilities( parentField, diff --git a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java index db2b3c0deea88..c957be8561571 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java @@ -127,12 +127,14 @@ public ParametrizedFieldMapper merge(Mapper mergeWith) { Conflicts conflicts = new Conflicts(name()); builder.merge((FieldMapper) mergeWith, conflicts); conflicts.check(); - return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name()))); + return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name(), simpleName()))); } - private static ContentPath parentPath(String name) { - int endPos = name.lastIndexOf("."); - if (endPos == -1) { + private static ContentPath parentPath(String name, String simpleName) { + // Use simpleName to compute the parent path so that fields whose simpleName contains dots + // (because of disable_objects) get the correct parent path + int endPos = name.length() - simpleName.length() - 1; + if (endPos < 0) { return new ContentPath(0); } return new ContentPath(name.substring(0, endPos)); @@ -619,7 +621,7 @@ private void merge(FieldMapper in, Conflicts conflicts) { param.merge(in, conflicts); } for (Mapper newSubField : in.multiFields) { - multiFieldsBuilder.update(newSubField, parentPath(newSubField.name())); + multiFieldsBuilder.update(newSubField, parentPath(newSubField.name(), newSubField.simpleName())); } this.copyTo.reset(in.copyTo); validate(); diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexActionTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexActionTests.java new file mode 100644 index 0000000000000..aec7adf26f01a --- /dev/null +++ b/server/src/test/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexActionTests.java @@ -0,0 +1,156 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.fieldcaps; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.util.Map; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class TransportFieldCapabilitiesIndexActionTests extends OpenSearchSingleNodeTestCase { + + // With disable_objects=true, {"attributes": {"foo": {"bar": "baz"}}} is flattened into the leaf + // field "attributes.foo.bar". The intermediate path "attributes.foo" has no ObjectMapper and + // _field_caps must not silently return empty results when walking the parent chain. + public void testFieldCapsDisableObjects() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("attributes") + .field("type", "object") + .field("disable_objects", true) + .endObject() + .endObject() + .endObject() + .toString(); + + assertAcked(client().admin().indices().prepareCreate("test").setMapping(mapping)); + + client().prepareIndex("test") + .setId("1") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject("attributes") + .startObject("foo") + .field("bar", "baz") + .endObject() + .endObject() + .endObject() + ) + .get(); + + client().admin().indices().prepareRefresh("test").get(); + + FieldCapabilitiesResponse response = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet(); + + Map> fields = response.get(); + + assertTrue("expected attributes.foo.bar in field caps", fields.containsKey("attributes.foo.bar")); + assertTrue("expected attributes in field caps", fields.containsKey("attributes")); + assertEquals("object", fields.get("attributes").values().iterator().next().getType()); + // phantom intermediate path has no ObjectMapper and must not exist + assertFalse("attributes.foo should not exist in field caps", fields.containsKey("attributes.foo")); + } + + // Indexing a second document after a disable_objects field must not corrupt the flattened field + // name. Previously "attributes.foo.bar" became "attributes.foo.foo.bar" after a mapping merge + // triggered by an unrelated document. + public void testFieldCapsDisableObjectsAfterUnrelatedDocument() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("attributes") + .field("type", "object") + .field("disable_objects", true) + .endObject() + .endObject() + .endObject() + .toString(); + + assertAcked(client().admin().indices().prepareCreate("test").setMapping(mapping)); + + // doc 1: creates the disable_objects leaf field + client().prepareIndex("test") + .setId("1") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject("attributes") + .startObject("foo") + .field("bar", "baz") + .endObject() + .endObject() + .endObject() + ) + .get(); + + // doc 2: unrelated - must not corrupt the existing mapping + client().prepareIndex("test").setId("2").setSource("{\"key\": 1}", MediaTypeRegistry.JSON).get(); + + client().admin().indices().prepareRefresh("test").get(); + + FieldCapabilitiesResponse response = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet(); + + Map> fields = response.get(); + + assertTrue("expected attributes.foo.bar in field caps", fields.containsKey("attributes.foo.bar")); + assertTrue("expected attributes.foo.bar.keyword in field caps", fields.containsKey("attributes.foo.bar.keyword")); + assertFalse("attributes.foo.foo.bar must not exist", fields.containsKey("attributes.foo.foo.bar")); + assertTrue("expected attributes in field caps", fields.containsKey("attributes")); + assertTrue("expected key in field caps", fields.containsKey("key")); + } + + // Regression test: the parentPath fix in ParametrizedFieldMapper must not break normal object + // field handling. Regular nested object fields (without disable_objects) must still produce the + // correct parent entries in _field_caps. + public void testFieldCapsNormalNestedObjectUnaffected() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("user") + .field("type", "object") + .startObject("properties") + .startObject("address") + .field("type", "object") + .startObject("properties") + .startObject("city") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + assertAcked(client().admin().indices().prepareCreate("test").setMapping(mapping)); + + client().prepareIndex("test") + .setId("1") + .setSource("{\"user\":{\"address\":{\"city\":\"Chemnitz\"}}}", MediaTypeRegistry.JSON) + .get(); + + client().admin().indices().prepareRefresh("test").get(); + + FieldCapabilitiesResponse response = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet(); + + Map> fields = response.get(); + + assertTrue("expected user.address.city", fields.containsKey("user.address.city")); + assertTrue("expected user.address as object", fields.containsKey("user.address")); + assertEquals("object", fields.get("user.address").values().iterator().next().getType()); + assertTrue("expected user as object", fields.containsKey("user")); + assertEquals("object", fields.get("user").values().iterator().next().getType()); + } +}