Updates to JSON schema and Data Prepper documentation. (#5019)

Support @JsonValue for determining enumeration values in the JSON Schema. Provide a JSON schema type of string for EventKey objects. Documentation wording improvements to the mutate event and mutate string processors.

Signed-off-by: David Venable <[email protected]>

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java

@@ -95,7 +95,7 @@ public String getTypeName() {
     }
 
     @JsonCreator
-    static DataType fromTypeName(final String option) {
+    public static DataType fromTypeName(final String option) {
         return TYPES_MAP.get(option);
     }
 

data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java

@@ -6,14 +6,21 @@
 package org.opensearch.dataprepper.model.event;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.EnumSource;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyString;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
@@ -39,26 +46,55 @@ void test_isSameType(Object object, String type, boolean expectedResult) {
         assertThat(DataType.isSameType(object, type), equalTo(expectedResult));
     }
 
+    @ParameterizedTest
+    @EnumSource(DataType.class)
+    void getTypeName_returns_non_empty_string_for_all_types(final DataType dataType) {
+        assertThat(dataType.getTypeName(), notNullValue());
+        assertThat(dataType.getTypeName(), not(emptyString()));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(DataTypeToKnownString.class)
+    void getTypeName_returns_expected_name(final DataType dataType, final String expectedString) {
+        assertThat(dataType.getTypeName(), equalTo(expectedString));
+    }
+
     private static Stream<Arguments> getSameTypeTestData() {
         int[] testArray = {1,2};
         List<Integer> testList = new ArrayList<>();
         return Stream.of(
-            Arguments.of(2, "integer", true),
-            Arguments.of("testString", "string", true),
-            Arguments.of(2L, "long", true),
-            Arguments.of(2.0, "double", true),
-            Arguments.of(BigDecimal.valueOf(2.34567), "big_decimal", true),
-            Arguments.of(true, "boolean", true),
-            Arguments.of(Map.of("k","v"), "map", true),
-            Arguments.of(testArray, "array", true),
-            Arguments.of(testList, "array", true),
-            Arguments.of(2.0, "integer", false),
-            Arguments.of(2, "string", false),
-            Arguments.of("testString", "long", false),
-            Arguments.of("testString", "double", false),
-            Arguments.of(2, "boolean", false),
-            Arguments.of(2L, "map", false),
-            Arguments.of(2, "array", false)
+            arguments(2, "integer", true),
+            arguments("testString", "string", true),
+            arguments(2L, "long", true),
+            arguments(2.0, "double", true),
+            arguments(BigDecimal.valueOf(2.34567), "big_decimal", true),
+            arguments(true, "boolean", true),
+            arguments(Map.of("k","v"), "map", true),
+            arguments(testArray, "array", true),
+            arguments(testList, "array", true),
+            arguments(2.0, "integer", false),
+            arguments(2, "string", false),
+            arguments("testString", "long", false),
+            arguments("testString", "double", false),
+            arguments(2, "boolean", false),
+            arguments(2L, "map", false),
+            arguments(2, "array", false)
         );
     }
+
+    static class DataTypeToKnownString implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(final ExtensionContext extensionContext) {
+            return Stream.of(
+                    arguments(DataType.STRING, "string"),
+                    arguments(DataType.BOOLEAN, "boolean"),
+                    arguments(DataType.INTEGER, "integer"),
+                    arguments(DataType.LONG, "long"),
+                    arguments(DataType.DOUBLE, "double"),
+                    arguments(DataType.BIG_DECIMAL, "big_decimal"),
+                    arguments(DataType.MAP, "map"),
+                    arguments(DataType.ARRAY, "array")
+            );
+        }
+    }
 }

data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java

@@ -23,6 +23,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static com.github.victools.jsonschema.module.jackson.JacksonOption.FLATTENED_ENUMS_FROM_JSONVALUE;
 import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_ORDER;
 import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED;
 
@@ -52,7 +53,7 @@ public static void main(String[] args) {
     @Override
     public void run() {
         final List<Module> modules = List.of(
-                new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED, RESPECT_JSONPROPERTY_ORDER),
+                new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED, RESPECT_JSONPROPERTY_ORDER, FLATTENED_ENUMS_FROM_JSONVALUE),
                 new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED,
                         JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS)
         );

data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java

@@ -1,5 +1,7 @@
 package org.opensearch.dataprepper.schemas;
 
+import com.fasterxml.classmate.TypeBindings;
+import com.fasterxml.classmate.types.ResolvedObjectType;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -12,12 +14,14 @@
 import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart;
 import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart;
 import com.github.victools.jsonschema.generator.SchemaVersion;
+import org.opensearch.dataprepper.model.event.EventKey;
 import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
 import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin;
 import org.opensearch.dataprepper.plugin.PluginProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -44,6 +48,7 @@ public ObjectNode convertIntoJsonSchema(
         overrideTargetTypeWithUsesDataPrepperPlugin(scopeSchemaGeneratorConfigPart);
         resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart);
         overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset);
+        resolveDataPrepperTypes(scopeSchemaGeneratorConfigPart);
 
         final SchemaGeneratorConfig config = configBuilder.build();
         final SchemaGenerator generator = new SchemaGenerator(config);
@@ -103,4 +108,13 @@ private void resolveDefaultValueFromJsonProperty(
             return annotation == null || annotation.defaultValue().isEmpty() ? null : annotation.defaultValue();
         });
     }
+
+    private void resolveDataPrepperTypes(final SchemaGeneratorConfigPart<FieldScope> scopeSchemaGeneratorConfigPart) {
+        scopeSchemaGeneratorConfigPart.withTargetTypeOverridesResolver(field -> {
+            if(field.getType().getErasedType().equals(EventKey.class)) {
+                return Collections.singletonList(ResolvedObjectType.create(String.class, TypeBindings.emptyBindings(), null, null));
+            }
+            return Collections.singletonList(field.getType());
+        });
+    }
 }

data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java

@@ -10,6 +10,7 @@
 import com.github.victools.jsonschema.generator.OptionPreset;
 import com.github.victools.jsonschema.generator.SchemaVersion;
 import org.junit.jupiter.api.Test;
+import org.opensearch.dataprepper.model.event.EventKey;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
@@ -22,6 +23,7 @@
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 
 @ExtendWith(MockitoExtension.class)
@@ -64,6 +66,18 @@ void testConvertIntoJsonSchemaWithCustomJacksonModule() throws JsonProcessingExc
         assertThat(propertiesNode.has("custom_test_attribute"), is(true));
     }
 
+    @Test
+    void testConvertIntoJsonSchemaWithEventKey() throws JsonProcessingException {
+        final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList(), pluginProvider);
+        final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema(
+                SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class);
+        final JsonNode propertiesNode = jsonSchemaNode.at("/properties");
+        assertThat(propertiesNode, instanceOf(ObjectNode.class));
+        assertThat(propertiesNode.has("testAttributeEventKey"), is(equalTo(true)));
+        assertThat(propertiesNode.get("testAttributeEventKey"), is(notNullValue()));
+        assertThat(propertiesNode.get("testAttributeEventKey").get("type"), is(equalTo(TextNode.valueOf("string"))));
+    }
+
     @JsonClassDescription("test config")
     static class TestConfig {
         private String testAttributeWithGetter;
@@ -77,5 +91,7 @@ static class TestConfig {
         public String getTestAttributeWithGetter() {
             return testAttributeWithGetter;
         }
+
+        private EventKey testAttributeEventKey;
     }
 }

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java

@@ -18,7 +18,7 @@
 import java.util.stream.Stream;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `add_entries` processor adds entries to an event.")
+@JsonClassDescription("The <code>add_entries</code> processor adds entries to an event.")
 public class AddEntryProcessorConfig {
     public static class Entry {
 
@@ -29,7 +29,7 @@ public static class Entry {
 
         @JsonProperty("metadata_key")
         @JsonPropertyDescription("The key for the new metadata attribute. The argument must be a literal string key " +
-                "and not a JSON Pointer. Either one string key or <code>metadata_key</code> is required.")
+                "and not a JSON Pointer. Either one of <code>key</code> or <code>metadata_key</code> is required.")
         private String metadataKey;
 
         @JsonPropertyDescription("The value of the new entry to be added, which can be used with any of the " +

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java

@@ -15,30 +15,30 @@
 import java.util.Optional;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `convert_entry_type` processor converts a value type associated with the specified key in " +
-        "a event to the specified type. It is a casting processor that changes the types of some fields in events.")
+@JsonClassDescription("The <code>convert_entry_type</code> processor converts a value associated with the specified key in " +
+        "a event to the specified type. It is a casting processor that changes the types of specified fields in events.")
 public class ConvertEntryTypeProcessorConfig implements ConverterArguments {
     @JsonProperty("key")
     @JsonPropertyDescription("Key whose value needs to be converted to a different type.")
     private String key;
 
     @JsonProperty("keys")
-    @JsonPropertyDescription("List of keys whose value needs to be converted to a different type.")
+    @JsonPropertyDescription("List of keys whose values needs to be converted to a different type.")
     private List<String> keys;
 
     @JsonProperty("type")
-    @JsonPropertyDescription("Target type for the key-value pair. Possible values are integer, long, double, big_decimal, string, and boolean. Default value is integer.")
+    @JsonPropertyDescription("Target type for the values. Default value is <code>integer.</code>")
     private TargetType type = TargetType.INTEGER;
 
     /**
      * Optional scale value used only in the case of BigDecimal converter
      */
     @JsonProperty("scale")
-    @JsonPropertyDescription("Modifies the scale of the big_decimal when converting to a big_decimal. The default value is 0.")
+    @JsonPropertyDescription("Modifies the scale of the <code>big_decimal</code> when converting to a <code>big_decimal</code>. The default value is 0.")
     private int scale = 0;
 
     @JsonProperty("convert_when")
-    @JsonPropertyDescription("Specifies a condition using a <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">Data Prepper expression</a> for performing the convert_entry_type operation. If specified, the convert_entry_type operation runs only when the expression evaluates to true.")
+    @JsonPropertyDescription("Specifies a condition using a <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> for performing the <code>convert_entry_type</code> operation. If specified, the <code>convert_entry_type</code> operation runs only when the expression evaluates to true. Example: <code>/mykey != \"---\"</code>")
     private String convertWhen;
 
     @JsonProperty("null_values")

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java

@@ -17,8 +17,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `copy_values` processor copies values within an event and is a [mutate event]" +
-        "(https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/mutate-event/) processor.")
+@JsonClassDescription("The <code>copy_values</code> processor copies values within an event to other fields within the event.")
 public class CopyValueProcessorConfig {
     public static class Entry {
         @NotEmpty

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java

@@ -18,20 +18,20 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `delete_entries` processor deletes entries, such as key-value pairs, from an event. " +
-        "You can define the keys you want to delete in the `with-keys` field following `delete_entries` in the YAML " +
-        "configuration file. Those keys and their values are deleted.")
+@JsonClassDescription("The <code>delete_entries</code> processor deletes fields from events. " +
+        "You can define the keys you want to delete in the <code>with_keys</code> configuration." +
+        "Those keys and their values are deleted from events.")
 public class DeleteEntryProcessorConfig {
     @NotEmpty
     @NotNull
     @JsonProperty("with_keys")
     @EventKeyConfiguration(EventKeyFactory.EventAction.DELETE)
-    @JsonPropertyDescription("An array of keys for the entries to be deleted.")
+    @JsonPropertyDescription("A list of keys to be deleted.")
     private List<@NotNull @NotEmpty EventKey> withKeys;
 
     @JsonProperty("delete_when")
     @JsonPropertyDescription("Specifies under what condition the <code>delete_entries</code> processor should perform deletion. " +
-            "Default is no condition.")
+            "By default, keys are always deleted. Example: <code>/mykey == \"---\"</code>")
     private String deleteWhen;
 
     public List<EventKey> getWithKeys() {

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java

@@ -19,8 +19,8 @@
 import java.util.stream.Collectors;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `list_to_map` processor converts a list of objects from an event, " +
-        "where each object contains a `key` field, into a map of target keys.")
+@JsonClassDescription("The <code>list_to_map</code> processor converts a list of objects from an event, " +
+        "where each object contains a <code>key</code> field, into a map of target keys.")
 public class ListToMapProcessorConfig {
     enum FlattenedElement {
         FIRST("first"),
@@ -89,9 +89,9 @@ static FlattenedElement fromOptionValue(final String option) {
     private FlattenedElement flattenedElement = FlattenedElement.FIRST;
 
     @JsonProperty("list_to_map_when")
-    @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
             "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will be " +
-            "run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
+            "run on the event. By default, all events will be processed unless otherwise stated.")
     private String listToMapWhen;
 
     @JsonProperty("tags_on_failure")

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorConfig.java

@@ -16,7 +16,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `map_to_list` processor converts a map of key-value pairs to a list of objects. " +
+@JsonClassDescription("The <code>map_to_list</code> processor converts a map of key-value pairs to a list of objects. " +
         "Each object contains the key and value in separate fields.")
 public class MapToListProcessorConfig {
     private static final String DEFAULT_KEY_NAME = "key";
@@ -45,9 +45,9 @@ public class MapToListProcessorConfig {
     private String valueName = DEFAULT_VALUE_NAME;
 
     @JsonProperty("map_to_list_when")
-    @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
             "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will " +
-            "be run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
+            "be run on the event. By default, all events will be processed unless otherwise stated.")
     private String mapToListWhen;
 
     @JsonProperty("exclude_keys")

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java

@@ -19,7 +19,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `rename_keys` processor renames keys in an event.")
+@JsonClassDescription("The <code>rename_keys</code> processor renames keys in an event.")
 public class RenameKeyProcessorConfig {
     public static class Entry {
         @NotEmpty
@@ -41,9 +41,9 @@ public static class Entry {
         private boolean overwriteIfToKeyExists = false;
 
         @JsonProperty("rename_when")
-        @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+        @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
                 "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will be " +
-                "run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
+                "run on the event. By default, all events will be processed unless otherwise stated.")
         private String renameWhen;
 
         public EventKey getFromKey() {

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/SelectEntriesProcessorConfig.java

@@ -15,7 +15,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `select_entries` processor selects entries from a Data Prepper event.")
+@JsonClassDescription("The <code>select_entries</code> processor selects entries from an event.")
 public class SelectEntriesProcessorConfig {
     @NotEmpty
     @NotNull
@@ -24,7 +24,7 @@ public class SelectEntriesProcessorConfig {
     private List<String> includeKeys;
 
     @JsonProperty("select_when")
-    @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
             "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will be " +
             "run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
     private String selectWhen;