Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[6.8] Improve circular reference detection in grok processor (#74581) #74772

Merged
merged 2 commits into from Jun 30, 2021
Merged

[6.8] Improve circular reference detection in grok processor (#74581) #74772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
765cdb8
Improve circular reference detection in grok processor (#74581)
danhermann Jun 30, 2021
9da6601
fix compile error
danhermann Jun 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
33 changes: 26 additions & 7 deletions libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
View file
Open in desktop
Expand Up @@ -114,7 +114,21 @@ private Grok(Map<String, String> patternBank, String grokPattern, boolean namedC
* check for a circular reference.
*/
private void forbidCircularReferences(String patternName, List<String> path, String pattern) {
if (pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":")) {
// first ensure that the pattern bank contains no simple circular references (i.e., any pattern
// containing an immediate reference to itself) as those can cause the remainder of this algorithm
// to recurse infinitely
for (Map.Entry<String, String> entry : patternBank.entrySet()) {
if (patternReferencesItself(entry.getValue(), entry.getKey())) {
throw new IllegalArgumentException("circular reference in pattern [" + entry.getKey() + "][" + entry.getValue() + "]");
}
}

// next recursively check any other pattern names referenced in the pattern
innerForbidCircularReferences(patternName, path, pattern);
}

private void innerForbidCircularReferences(String patternName, List<String> path, String pattern) {
if (patternReferencesItself(pattern, patternName)) {
String message;
if (path.isEmpty()) {
message = "circular reference in pattern [" + patternName + "][" + pattern + "]";
Expand All @@ -129,17 +143,18 @@ private void forbidCircularReferences(String patternName, List<String> path, Str
throw new IllegalArgumentException(message);
}

// next check any other pattern names found in the pattern
for (int i = pattern.indexOf("%{"); i != -1; i = pattern.indexOf("%{", i + 1)) {
int begin = i + 2;
int brackedIndex = pattern.indexOf('}', begin);
int bracketIndex = pattern.indexOf('}', begin);
int columnIndex = pattern.indexOf(':', begin);
int end;
if (brackedIndex != -1 && columnIndex == -1) {
end = brackedIndex;
} else if (columnIndex != -1 && brackedIndex == -1) {
if (bracketIndex != -1 && columnIndex == -1) {
end = bracketIndex;
} else if (columnIndex != -1 && bracketIndex == -1) {
end = columnIndex;
} else if (brackedIndex != -1 && columnIndex != -1) {
end = Math.min(brackedIndex, columnIndex);
} else if (bracketIndex != -1 && columnIndex != -1) {
end = Math.min(bracketIndex, columnIndex);
} else {
throw new IllegalArgumentException("pattern [" + pattern + "] has circular references to other pattern definitions");
}
Expand All @@ -149,6 +164,10 @@ private void forbidCircularReferences(String patternName, List<String> path, Str
}
}

private static boolean patternReferencesItself(String pattern, String patternName) {
return pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":");
}

public String groupMatch(String name, Region region, String pattern) {
try {
int number = GROK_PATTERN_REGEX.nameToBackrefNumber(name.getBytes(StandardCharsets.UTF_8), 0,
Expand Down
20 changes: 16 additions & 4 deletions libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
View file
Open in desktop
Expand Up @@ -244,8 +244,7 @@ public void testCircularReference() {
String pattern = "%{NAME1}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME3][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2]",
e.getMessage());
assertEquals("circular reference in pattern [NAME3][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2]", e.getMessage());

e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new TreeMap<>();
Expand All @@ -257,8 +256,21 @@ public void testCircularReference() {
String pattern = "%{NAME1}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME5][!!!%{NAME1}!!!] back to pattern [NAME1] " +
"via patterns [NAME2=>NAME3=>NAME4]", e.getMessage());
assertEquals(
"circular reference in pattern [NAME5][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2=>NAME3=>NAME4]",
e.getMessage()
);
}

public void testCircularSelfReference() {
Exception e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new HashMap<>();
bank.put("ANOTHER", "%{INT}");
bank.put("INT", "%{INT}");
String pattern = "does_not_matter";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [INT][%{INT}]", e.getMessage());
}

public void testBooleanCaptures() {
Expand Down