diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java index 4c65621b0fc..21ce1698e05 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java @@ -20,6 +20,8 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -93,6 +95,19 @@ public boolean canAccess(byte[] expression) { } } + public static BytesEvaluator newEvaluator(Collection authsSet) { + Collection> convertedAuths = new ArrayList<>(); + for (Authorizations auths : authsSet) { + List bytesAuths = auths.getAuthorizations(); + Set stringAuths = new HashSet<>(bytesAuths.size()); + for (var auth : bytesAuths) { + stringAuths.add(new String(auth, ISO_8859_1)); + } + convertedAuths.add(stringAuths); + } + return new BytesEvaluator(ACCESS.newEvaluator(convertedAuths)); + } + public static BytesEvaluator newEvaluator(Authorizations auths) { List bytesAuths = auths.getAuthorizations(); Set stringAuths = new HashSet<>(bytesAuths.size()); diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java new file mode 100644 index 00000000000..b31e5cb07ab --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators.user; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; + +import org.apache.accumulo.access.InvalidAccessExpressionException; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.clientImpl.access.BytesAccess; +import org.apache.accumulo.core.data.ArrayByteSequence; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.OptionDescriber; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections4.map.LRUMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public class MultiAuthVisibilityFilter extends Filter implements OptionDescriber { + private BytesAccess.BytesEvaluator accessEvaluator; + protected Map cache; + private final ArrayByteSequence testVis = new ArrayByteSequence(new byte[0]); + + private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class); + + private static final String NUM_AUTHS = "numAuths"; + private static final String AUTH_PREFIX = "auth_"; + private static final String FILTER_INVALID_ONLY = "filterInvalid"; + + private boolean filterInvalid; + + @Override + public void init(SortedKeyValueIterator source, Map options, + IteratorEnvironment env) throws IOException { + super.init(source, options, env); + validateOptions(options); + this.filterInvalid = Boolean.parseBoolean(options.get(FILTER_INVALID_ONLY)); + + if (!filterInvalid) { + String numAuthsParameter = options.get(NUM_AUTHS); + Objects.requireNonNull(numAuthsParameter, "NUM_AUTHS option not set."); + int numAuths = Integer.parseInt(numAuthsParameter); + Preconditions.checkArgument(numAuths >= 0, NUM_AUTHS + " must be a positive integer"); + + Collection authSet = new ArrayList<>(); + if (numAuths == 0) { + authSet.add(new Authorizations()); + } else { + for (int idx = 0; idx < numAuths; idx++) { + String auths = options.get(AUTH_PREFIX + idx); + Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations() + : new Authorizations(auths.getBytes(UTF_8)); + authSet.add(authObj); + } + String auths = options.get(AUTH_PREFIX + numAuths); + Preconditions.checkArgument(auths == null, + "NUM_AUTHS is set incorrectly, should be at least: " + NUM_AUTHS + 1); + } + this.accessEvaluator = BytesAccess.newEvaluator(authSet); + } + this.cache = new LRUMap<>(1000); + } + + @Override + public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { + MultiAuthVisibilityFilter result = (MultiAuthVisibilityFilter) super.deepCopy(env); + result.filterInvalid = this.filterInvalid; + result.accessEvaluator = this.accessEvaluator; + result.cache = this.cache; + return result; + } + + @Override + public boolean accept(Key k, Value v) { + // The following call will replace the contents of testVis + // with the bytes for the column visibility for k. Any cached + // version of testVis needs to be a copy to avoid modifying + // the cached version. + k.getColumnVisibilityData(testVis); + if (filterInvalid) { + Boolean b = cache.get(testVis); + if (b != null) { + return b; + } + final ArrayByteSequence copy = new ArrayByteSequence(testVis); + try { + BytesAccess.validate(copy.toArray()); + // cache a copy of testVis + cache.put(copy, true); + return true; + } catch (InvalidAccessExpressionException e) { + // cache a copy of testVis + cache.put(copy, false); + return false; + } + } else { + if (testVis.length() == 0) { + return true; + } + + Boolean b = cache.get(testVis); + if (b != null) { + return b; + } + + final ArrayByteSequence copy = new ArrayByteSequence(testVis); + try { + boolean bb = accessEvaluator.canAccess(copy.toArray()); + // cache a copy of testVis + cache.put(copy, bb); + return bb; + } catch (InvalidAccessExpressionException e) { + log.error("Parse Error with visibility of Key: {}", k, e); + return false; + } + } + } + + @Override + public IteratorOptions describeOptions() { + IteratorOptions io = super.describeOptions(); + io.setName("multiAuthVisibilityFilter"); + io.setDescription("The MultiAuthVisibilityFilter allows you to filter for key/value" + + " pairs by a collection of sets of authorizations or filter invalid labels from corrupt files."); + io.addNamedOption(FILTER_INVALID_ONLY, + "if 'true', the iterator is instructed to ignore the authorizations and" + + " only filter invalid visibility labels (default: false)"); + io.addNamedOption(NUM_AUTHS, + "The number of serialized authorizations to filter against (default 0)"); + io.addUnnamedOption(AUTH_PREFIX + + "N, where the value is a serialized set of authorizations. N must be between zero and NUM_AUTHS."); + return io; + } + + public static void setAuthorizations(IteratorSetting setting, Collection auths) { + setting.addOption(NUM_AUTHS, Integer.toString(auths.size())); + int idx = 0; + for (Authorizations auth : auths) { + setting.addOption(AUTH_PREFIX + idx, auth.serialize()); + idx++; + } + } + + public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) { + setting.addOption(FILTER_INVALID_ONLY, Boolean.toString(featureEnabled)); + } + +} diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java new file mode 100644 index 00000000000..8b4b1a217b6 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators.user; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.Test; + +public class MultiAuthVisibilityFilterTest { + + private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); + + private static final Text BAD = new Text("bad"); + private static final Text GOOD = new Text("good"); + private static final Text EMPTY_VIS = new Text(""); + private static final Text GOOD_VIS = new Text("abc|def"); + private static final Text HIDDEN_VIS = new Text("abc&def&ghi"); + private static final Text BAD_VIS = new Text("&"); + private static final Value EMPTY_VALUE = new Value(); + + private TreeMap createUnprotectedSource(int numPublic, int numHidden) { + TreeMap source = new TreeMap<>(); + for (int i = 0; i < numPublic; i++) { + source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, EMPTY_VIS), EMPTY_VALUE); + } + for (int i = 0; i < numHidden; i++) { + source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, GOOD_VIS), EMPTY_VALUE); + } + return source; + } + + private TreeMap createPollutedSource(int numGood, int numBad) { + TreeMap source = new TreeMap<>(); + for (int i = 0; i < numGood; i++) { + source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, GOOD_VIS), EMPTY_VALUE); + } + for (int i = 0; i < numBad; i++) { + source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, BAD_VIS), EMPTY_VALUE); + } + return source; + } + + private TreeMap createSourceWithHiddenData(int numViewable, int numHidden) { + TreeMap source = new TreeMap<>(); + for (int i = 0; i < numViewable; i++) { + source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, GOOD_VIS), EMPTY_VALUE); + } + for (int i = 0; i < numHidden; i++) { + source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, HIDDEN_VIS), EMPTY_VALUE); + } + return source; + } + + private void verify(TreeMap source, int expectedSourceSize, Map options, + Text expectedCF, Text expectedCQ, Text expectedCV, int expectedFinalCount) + throws IOException { + assertEquals(expectedSourceSize, source.size()); + + Filter filter = new MultiAuthVisibilityFilter(); + filter.init(new SortedMapIterator(source), options, null); + filter.seek(new Range(), EMPTY_COL_FAMS, false); + + int count = 0; + while (filter.hasTop()) { + count++; + // System.out.println(DefaultFormatter.formatEntry( + // Collections.singletonMap(filter.getTopKey(), + // filter.getTopValue()).entrySet().iterator().next(), + // false)); + assertEquals(expectedCF, filter.getTopKey().getColumnFamily()); + assertEquals(expectedCQ, filter.getTopKey().getColumnQualifier()); + assertEquals(expectedCV, filter.getTopKey().getColumnVisibility()); + filter.next(); + } + assertEquals(expectedFinalCount, count); + } + + @Test + public void testAllowValidLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, true); + + TreeMap source = createPollutedSource(1, 2); + verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + + source = createPollutedSource(30, 500); + verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); + + source = createPollutedSource(1000, 500); + verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testAllowBadLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setNegate(is, true); + MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, true); + + TreeMap source = createPollutedSource(1, 2); + verify(source, 3, is.getOptions(), BAD, BAD, BAD_VIS, 2); + + source = createPollutedSource(30, 500); + verify(source, 530, is.getOptions(), BAD, BAD, BAD_VIS, 500); + + source = createPollutedSource(1000, 500); + verify(source, 1500, is.getOptions(), BAD, BAD, BAD_VIS, 500); + } + + @Test + public void testAllowAuthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testAllowUnauthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setNegate(is, true); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + } + + @Test + public void testNoLabels() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setNegate(is, false); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations())); + + TreeMap source = createUnprotectedSource(5, 2); + verify(source, 7, is.getOptions(), GOOD, GOOD, EMPTY_VIS, 5); + + MultiAuthVisibilityFilter.setNegate(is, true); + verify(source, 7, is.getOptions(), BAD, BAD, GOOD_VIS, 2); + } + + @Test + public void testFilterUnauthorizedAndBad() throws IOException { + /* + * if not explicitly filtering bad labels, they will still be filtered while validating against + * authorizations, but it will be very verbose in the logs + */ + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 5); + for (Entry entry : createPollutedSource(0, 1).entrySet()) { + source.put(entry.getKey(), entry.getValue()); + } + + verify(source, 7, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + } + + @Test + public void testCommaSeparatedAuthorizations() throws IOException { + Map options = Map.of("numAuths", "1", "auth_0", "x,def,y"); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, options, GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, options, GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testSerializedAuthorizations() throws IOException { + Map options = + Map.of("numAuths", "1", "auth_0", new Authorizations("x", "def", "y").serialize()); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, options, GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, options, GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testStaticConfigurators() { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, false); + MultiAuthVisibilityFilter.setNegate(is, true); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("abc", "def"))); + + Map opts = is.getOptions(); + assertEquals("false", opts.get("filterInvalid")); + assertEquals("true", opts.get("negate")); + assertEquals("1", opts.get("numAuths")); + assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auth_0")); + } + + @Test + public void testDeepCopyAfterInit() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("abc"))); + Map opts = is.getOptions(); + Filter filter = new MultiAuthVisibilityFilter(); + TreeMap source = new TreeMap<>(); + filter.init(new SortedMapIterator(source), opts, null); + Filter copyFilter = (Filter) filter.deepCopy(null); + Key k = new Key("row", "cf", "cq", "abc"); + assertTrue(copyFilter.accept(k, new Value())); + } + +} diff --git a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java index 96cc472f8c7..32ea3ea5c8c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java @@ -37,12 +37,14 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.MultiAuthVisibilityFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.ByteArraySet; @@ -92,6 +94,7 @@ public void run() throws Exception { insertData(c, table); queryData(c, table); + queryDataMultiAuth(c, table); deleteData(c, table); insertDefaultData(c, table2); @@ -222,6 +225,48 @@ private void queryData(AccumuloClient c, String tableName) throws Exception { queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected); } + /** + * Configures Scanners with the users default authorizations, then it adds a + * MultiAuthVisibilityFilter with different sets of Authorizations + */ + private void queryDataMultiAuth(AccumuloClient c, String tableName) throws Exception { + + c.securityOperations().changeUserAuthorizations(getAdminPrincipal(), + new Authorizations("A", "B", "FOO", "L", "M", "Z")); + + Authorizations userAuths = c.securityOperations().getUserAuthorizations(c.whoami()); + + Set expectedUserAuths = + Set.of("v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13"); + try (Scanner scanner = c.createScanner(tableName, userAuths); + BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) { + verify(scanner.iterator(), expectedUserAuths.toArray(new String[] {})); + + bs.setRanges(Collections.singleton(new Range())); + verify(bs.iterator(), expectedUserAuths.toArray(new String[] {})); + } + + Authorizations entity1 = new Authorizations("A", "B", "FOO", "L", "M"); + Authorizations entity2 = new Authorizations("B", "FOO", "Z"); + // should only see entries with no column visibility, B and/or FOO + Set expectedAuths = Set.of("v1", "v3", "v11"); + + IteratorSetting is = new IteratorSetting(100, "userAuths", MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2)); + + try (Scanner scanner = c.createScanner(tableName, userAuths); + BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) { + + scanner.addScanIterator(is); + verify(scanner.iterator(), expectedAuths.toArray(new String[] {})); + + bs.setRanges(Collections.singleton(new Range())); + bs.addScanIterator(is); + verify(bs.iterator(), expectedAuths.toArray(new String[] {})); + } + + } + private void queryData(AccumuloClient c, String tableName, Set allAuths, Set userAuths, Map,Set> expected) throws Exception {