Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion config/site/support/doctest_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
require "elastic_graph/apollo/schema_definition/api_extension"
require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names"
require "elastic_graph/schema_definition/api"
require "elastic_graph/schema_definition/extension_module_support"
require "elastic_graph/schema_definition/schema_artifact_manager"
require "elastic_graph/warehouse/schema_definition/api_extension"
require "rspec/mocks"
Expand Down Expand Up @@ -60,7 +61,7 @@ module ElasticGraph
@api = SchemaDefinition::API.new(
SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}),
true,
extension_modules: extension_modules
extension_modules: SchemaDefinition::ExtensionModuleSupport.default_extension_modules + extension_modules
)

# This is required in all schemas, but we don't want to have to put in all our examples,
Expand Down Expand Up @@ -96,6 +97,7 @@ module ElasticGraph
# `schema.json_schema_version` raises an error when the version is set more than once.
# By default we set it above. Here we clear it to allow our example to set it.
schema.state.json_schema_version = nil
schema.state.json_schema_version_setter_location = nil
end
end

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ def self.with_both_casing_forms(&block)
end

def define_schema(&block)
extension_modules = [SchemaDefinition::APIExtension]
extension_modules = ::ElasticGraph::SchemaDefinition::ExtensionModuleSupport.default_extension_modules + [SchemaDefinition::APIExtension]
super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1431,7 +1431,9 @@ def expect_identifiable_type_tagging_of_token(&type_def_for)
end

def define_schema(with_apollo: true, &block)
extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : []
# Always include the JSON ingestion default so `Results#json_schemas_for` is available in both modes.
extension_modules = ::ElasticGraph::SchemaDefinition::ExtensionModuleSupport.default_extension_modules
extension_modules += [SchemaDefinition::APIExtension] if with_apollo
super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def json_schema_indexing_field_types_by_name
end
.sort_by(&:name)
.to_h do |type|
# @type var indexing_field_type: ElasticGraph::SchemaDefinition::Indexing::_FieldType
# @type var indexing_field_type: Indexing::_JSONFieldType
indexing_field_type = _ = type.to_indexing_field_type
[type.name, indexing_field_type]
end
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ module ElasticGraph

attr_reader unused_deprecated_elements: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]

def initialize: (ElasticGraph::SchemaDefinition::Results) -> void
def initialize: ((ElasticGraph::SchemaDefinition::Results & ResultsExtension)) -> void
def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata

private
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ module ElasticGraph
@json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]]?
@current_public_json_schema: ::Hash[::String, untyped]?
@json_schema_with_metadata_merger: Indexing::JSONSchemaWithMetadata::Merger?
@json_schema_indexing_field_types_by_name: ::Hash[::String, ::ElasticGraph::SchemaDefinition::Indexing::_FieldType]?
@json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_JSONFieldType]?

def json_ingestion_state: () -> (::ElasticGraph::SchemaDefinition::State & StateExtension)
def json_schema_with_metadata_merger: () -> Indexing::JSONSchemaWithMetadata::Merger
def build_public_json_schema: () -> ::Hash[::String, untyped]
def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, ::ElasticGraph::SchemaDefinition::Indexing::_FieldType]
def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_JSONFieldType]
end
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ module ElasticGraph

private

@json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]
@json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]?

def json_ingestion_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension)
def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ Gem::Specification.new do |spec|

spec.add_dependency "elasticgraph-graphql", ElasticGraph::VERSION # needed since we validate that scalar `coerce_with` options are valid (which loads scalar coercion adapters)
spec.add_dependency "elasticgraph-indexer", ElasticGraph::VERSION # needed since we validate that scalar `prepare_for_indexing_with` options are valid (which loads indexing preparer adapters)
# `elasticgraph-json_ingestion` is the default ingestion serializer extension. It's a soft dep here
# so `default_extension_modules` can fall back to `[]` when the gem isn't installed (e.g. for apps
# that supply a different serializer via `Gemfile-custom`), but most apps want it.
spec.add_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
spec.add_dependency "elasticgraph-schema_artifacts", ElasticGraph::VERSION
spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,15 @@
require "elastic_graph/schema_artifacts/runtime_metadata/extension"
require "elastic_graph/schema_artifacts/runtime_metadata/graphql_resolver"
require "elastic_graph/schema_definition/extension_module_support"
require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect"
require "elastic_graph/schema_definition/results"
require "elastic_graph/schema_definition/state"

# :nocov: -- only loaded on JRuby
require "elastic_graph/schema_definition/jruby_patches" if RUBY_ENGINE == "jruby"
# :nocov:

require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect"
require "elastic_graph/schema_definition/results"
require "elastic_graph/schema_definition/state"

module ElasticGraph
# The main entry point for schema definition from ElasticGraph applications.
#
Expand Down Expand Up @@ -295,7 +296,7 @@ def union_type(name, &block)
# ElasticGraph.define_schema do |schema|
# schema.scalar_type "URL" do |t|
# t.mapping type: "keyword"
# t.json_schema type: "string", format: "uri"
# t.json_schema type: "string"
# end
# end
def scalar_type(name, &block)
Expand Down Expand Up @@ -454,67 +455,23 @@ def results
@results ||= @factory.new_results
end

# Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema
# artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique
# version number. The publisher will then include this version number in published events to identify the version of the schema it
# was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync.
# Records the JSON schema version of this schema definition.
#
# @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly
# have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this
# on every JSON schema change by setting `enforce_json_schema_version` to `false` in your `Rakefile`.
# The default implementation is a no-op so that callers (and the test helpers) can invoke this
# uniformly regardless of whether an ingestion-serializer extension is loaded. When
# `elasticgraph-json_ingestion` is loaded, its `APIExtension` overrides this with real behavior.
#
# @param version [Integer] current version number of the JSON schema artifact
# @param version [Integer] the JSON schema version
# @return [void]
# @see Local::RakeTasks#enforce_json_schema_version
#
# @example Set the JSON schema version to 1
# ElasticGraph.define_schema do |schema|
# schema.json_schema_version 1
# end
def json_schema_version(version)
if !version.is_a?(Integer) || version < 1
raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}"
end

if @state.json_schema_version
raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.json_schema_version}"
end

@state.json_schema_version = version
@state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first
nil
end

# Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the
# publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to
# configure this behavior.
#
# @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events.
# @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events.
# @return [void]
# Records JSON schema strictness configuration.
#
# @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled
# field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher
# accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId`
# is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of
# these to `true` (or none).
# The default implementation is a no-op; overridden by the JSON ingestion extension.
#
# @example Allow omitted fields and disallow extra fields
# ElasticGraph.define_schema do |schema|
# schema.json_schema_strictness allow_omitted_fields: true, allow_extra_fields: false
# end
# @return [void]
def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true)
unless [true, false].include?(allow_omitted_fields)
raise Errors::SchemaError, "`allow_omitted_fields` must be true or false"
end

unless [true, false].include?(allow_extra_fields)
raise Errors::SchemaError, "`allow_extra_fields` must be true or false"
end

@state.allow_omitted_json_schema_fields = allow_omitted_fields
@state.allow_extra_json_schema_fields = allow_extra_fields
nil
end

# Registers a customization callback that will be applied to every built-in type automatically provided by ElasticGraph. Provides
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,22 @@ module SchemaDefinition
#
# @private
module ExtensionModuleSupport
# Default extension modules applied to {API} when none are explicitly specified.
# Default extension modules applied to {API} when none are explicitly specified. Currently this is
# just `JSONIngestion::SchemaDefinition::APIExtension` (when `elasticgraph-json_ingestion` is installed).
#
# @return [Array<Module>] default extension modules
# @return [Array<Module>] default extension modules, or empty when no default ingestion serializer is installed.
def self.default_extension_modules
require "elastic_graph/json_ingestion/schema_definition/api_extension"
[JSONIngestion::SchemaDefinition::APIExtension]
# :nocov: -- only reached in bundles that exclude `elasticgraph-json_ingestion` (e.g. via
# `Gemfile-custom`); not exercised by the standard test suite, where the gem is always present.
rescue ::LoadError
# `elasticgraph-json_ingestion` is an optional gem. When it isn't installed, fall back to no
# default extension modules so that `elasticgraph-schema_definition` remains usable on its own.
# The umbrella gems (e.g. `elasticgraph-local`) declare a runtime dep on `elasticgraph-json_ingestion`
# to preserve backward compatibility for existing users.
[]
# :nocov:
end
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
# frozen_string_literal: true

require "elastic_graph/constants"
require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata"
require "elastic_graph/schema_definition/indexing/list_counts_mapping"
require "elastic_graph/support/hash_util"
require "elastic_graph/support/memoizable_data"
Expand All @@ -22,28 +21,13 @@ class Field < Support::MemoizableData.define(
:name,
:name_in_index,
:type,
:json_schema_layers,
:indexing_field_type,
:accuracy_confidence,
:json_schema_customizations,
:mapping_customizations,
:source,
:runtime_field_script,
:doc_comment
)
# JSON schema overrides that automatically apply to specific mapping types so that the JSON schema
# validation will reject values which cannot be indexed into fields of a specific mapping type.
#
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation
# @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type).
# @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values.
JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = {
"byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1},
"short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1},
"keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH},
"text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH}
}

# @return [Hash<String, Object>] the mapping for this field. The returned hash should be composed entirely
# of Ruby primitives that, when converted to a JSON string, match the structure required by
# [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html).
Expand All @@ -63,23 +47,6 @@ def mapping
end
end

# @return [Hash<String, Object>] the JSON schema definition for this field. The returned object should
# be composed entirely of Ruby primitives that, when converted to a JSON string, match the
# requirements of [the JSON schema spec](https://json-schema.org/).
def json_schema
json_schema_layers
.reverse # resolve layers from innermost to outermost wrappings
.reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) }
.merge(outer_json_schema_customizations)
.merge({"description" => doc_comment}.compact)
.then { |h| Support::HashUtil.stringify_keys(h) }
end

# @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field.
def json_schema_metadata
::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index)
end

# Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the
# datastore does so that consistency checks between our index configuration and what's in the datastore
# work properly.
Expand Down Expand Up @@ -107,80 +74,6 @@ def self.normalized_mapping_hash_for(fields)

mapping_hash
end

def nullable?
json_schema_layers.include?(:nullable)
end

private

def inner_json_schema
user_specified_customizations =
if user_specified_json_schema_customizations_go_on_outside?
{} # : ::Hash[::String, untyped]
else
Support::HashUtil.stringify_keys(json_schema_customizations)
end

customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {}
customizations = customizations_from_mapping.merge(user_specified_customizations)
customizations = indexing_field_type.format_field_json_schema_customizations(customizations)

ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"}
return ref if customizations.empty?

# Combine any customizations with type ref under an "allOf" subschema:
# All of these properties must hold true for the type to be valid.
#
# Note that if we simply combine the customizations with the `$ref`
# at the same level, it will not work, because other subschema
# properties are ignored when they are in the same object as a `$ref`:
# https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168
{"allOf" => [ref, customizations]}
end

def outer_json_schema_customizations
return {} unless user_specified_json_schema_customizations_go_on_outside?
Support::HashUtil.stringify_keys(json_schema_customizations)
end

# Indicates if the user-specified JSON schema customizations should go on the inside
# (where they normally go) or on the outside. They only go on the outside when it's
# an array field, because then they apply to the array itself instead of the items in the
# array.
def user_specified_json_schema_customizations_go_on_outside?
json_schema_layers.include?(:array)
end

def process_layer(layer, schema)
case layer
when :nullable
make_nullable(schema)
when :array
make_array(schema)
else
# :nocov: - layer is only ever `:nullable` or `:array` so we never get here
schema
# :nocov:
end
end

def make_nullable(schema)
# Here we use "anyOf" to ensure that JSON can either match the schema OR null.
#
# (Using "oneOf" would mean that if we had a schema that also allowed null,
# null would never be allowed, since "oneOf" must match exactly one subschema).
{
"anyOf" => [
schema,
{"type" => "null"}
]
}
end

def make_array(schema)
{"type" => "array", "items" => schema}
end
end
end
end
Expand Down
Loading
Loading