Disallow extra fields other than "@context"

[candleindark]

This PR closes #75. It addresses #75 in the follow manner.

1. Make the models disallow data instances that have extra fields other than the "@context" field in both Pydantic and JSON level.
2. The Pydantic models continue to not have a context field.
3. Data instances with an "@context" field can be validated against a Pydantic model but ignored.

The solution implemented in this PR is based on the following script.

from typing import Any
import json

from pydantic import BaseModel, ConfigDict, model_validator
from pydantic.json_schema import JsonDict, JsonValue
from jsonschema import validate, Draft202012Validator
import jsonschema


def get_dict_without_context(d: Any) -> Any:
    """
    If a given object is a dictionary, return a copy of it without the
    `@context` key. Otherwise, return the input object as is.

    :param d: The given object
    :return: If the object is a dictionary, a copy of it without the `@context` key;
             otherwise, the input object as is.
    """
    if isinstance(d, dict):
        return {k: v for k, v in d.items() if k != "@context"}
    return d


def add_context(json_schema: JsonDict) -> None:
    """
    Add the `@context` key to the given JSON schema as a required key represented as a
    dictionary.

    :param json_schema: The dictionary representing the JSON schema

    raises: ValueError if the `@context` key is already present in the given dictionary
    """
    context_key = "@context"
    context_key_title = "@Context"
    properties: JsonDict = json_schema.get("properties", {})
    required: list[JsonValue] = json_schema.get("required", [])

    if context_key in properties or context_key in required:
        msg = f"The '{context_key}' key is already present in the given JSON schema."
        raise ValueError(msg)

    properties[context_key] = {
        "format": "uri",
        "minLength": 1,
        "title": context_key_title,
        "type": "string",
    }
    required.append(context_key)

    # Update the schema
    # This is needed to handle the case in which the keys are newly created
    json_schema["properties"] = properties
    json_schema["required"] = required


class Foo(BaseModel):
    x: int

    # Model validator to remove the `"@context"` key from data instance before
    # "base" validation is performed.
    _remove_context_key = model_validator(mode="before")(get_dict_without_context)

    model_config = ConfigDict(extra="forbid", json_schema_extra=add_context)


json_schema_ = Foo.model_json_schema()
print(json.dumps(json_schema_, indent=2))
"""
{
  "additionalProperties": false,
  "properties": {
    "x": {
      "title": "X",
      "type": "integer"
    },
    "@context": {
      "format": "uri",
      "minLength": 1,
      "title": "@Context",
      "type": "string"
    }
  },
  "required": [
    "x",
    "@context"
  ],
  "title": "Foo",
  "type": "object"
}
"""

instance_json_str = '{"x": 1}'
instance_json_str_with_context = '{"@context": "not a valid URI", "x": 1}'
instance_json_str_with_extra = '{"x": 1, "e": 42}'

vv = Foo.model_validate_json(instance_json_str)
print("\n====================================")
print(f"vv: {vv!r}")
"vv: Foo(x=1)"

# Ignore the context field in Pydantic level
vv_with_context = Foo.model_validate_json(instance_json_str_with_context)
print("\n====================================")
print(f"vv_with_context: {vv_with_context!r}")
"vv_with_context: Foo(x=1)"

# Disallow other extra fields in Pydantic level
try:
    Foo.model_validate_json(instance_json_str_with_extra)
except ValueError as e:
    print("\n====================================")
    print(e)
    """
    1 validation error for Foo
    e
      Extra inputs are not permitted [type=extra_forbidden, input_value=42, input_type=int]
        For further information visit https://errors.pydantic.dev/2.9/v/extra_forbidden
    """

instance = {"@context": "https://schema.org", "x": 1}
instance_with_invalid_context = {"@context": "invalid context", "x": 1}
instance_missing_context = {"x": 1}
instance_with_extra = {"@context": "https://schema.org", "x": 1, "e": 42}

# Validate an instance with valid context and x field
validate(instance, json_schema_, format_checker=Draft202012Validator.FORMAT_CHECKER)

# Instance with invalid context fails validation
try:
    validate(
        instance_with_invalid_context,
        json_schema_,
        format_checker=Draft202012Validator.FORMAT_CHECKER,
    )
except jsonschema.exceptions.ValidationError as e:
    print("\n====================================")
    print(e)
    """
    'invalid context' is not a 'uri'
    Failed validating 'format' in schema['properties']['@context']:
        {'format': 'uri', 'minLength': 1, 'title': '@Context', 'type': 'string'}
    On instance['@context']:
        'invalid context'
    """

# Instance missing context fails validation
try:
    validate(
        instance_missing_context,
        json_schema_,
        format_checker=Draft202012Validator.FORMAT_CHECKER,
    )
except jsonschema.exceptions.ValidationError as e:
    print("\n====================================")
    print(e)
    """
    '@context' is a required property
    Failed validating 'required' in schema:
        {'additionalProperties': False,
         'properties': {'x': {'title': 'X', 'type': 'integer'},
                        '@context': {'format': 'uri',
                                     'minLength': 1,
                                     'title': '@Context',
                                     'type': 'string'}},
         'required': ['x', '@context'],
         'title': 'Foo',
         'type': 'object'}
    On instance:
        {'x': 1}
    """

# Instance with extra field fails validation
try:
    validate(
        instance_with_extra,
        json_schema_,
        format_checker=Draft202012Validator.FORMAT_CHECKER,
    )
except jsonschema.exceptions.ValidationError as e:
    print("\n====================================")
    print(e)
    """
    Additional properties are not allowed ('e' was unexpected)
    Failed validating 'additionalProperties' in schema:
        {'additionalProperties': False,
         'properties': {'x': {'title': 'X', 'type': 'integer'},
                        '@context': {'format': 'uri',
                                     'minLength': 1,
                                     'title': '@Context',
                                     'type': 'string'}},
         'required': ['x', '@context'],
         'title': 'Foo',
         'type': 'object'}
    On instance:
        {'@context': 'https://schema.org', 'x': 1, 'e': 42}
    """

TODOs

• ensure that tests pass
  • @yarikoptic: let's for now make @context optional and then discuss
• run against all dandisets metadata (manifests) and see if any gets (more) broken.
  • we have them already fetched to drogon -- just run there. */*/dandiset.jsonld files under /mnt/backup/dandi/dandiset-manifests-s3cmd .
  • code for checking is currently is in https://github.com/candleindark/dandi-schema/tree/against-manifests
• check meditor

[codecov]

Codecov Report

Attention: Patch coverage is 85.00000% with 3 lines in your changes missing coverage. Please review.

Project coverage is 92.21%. Comparing base (9d068b9) to head (391aa1a).
Report is 4 commits behind head on master.

Files with missing lines	Patch %	Lines
dandischema/models.py	85.00%	3 Missing ⚠️

❗ There is a different number of reports uploaded between BASE (9d068b9) and HEAD (391aa1a). Click for more details.

HEAD has 127 uploads less than BASE

Flag	BASE (9d068b9)	HEAD (391aa1a)
unittests	151	24

Additional details and impacted files

@@            Coverage Diff             @@
##           master     #266      +/-   ##
==========================================
- Coverage   97.71%   92.21%   -5.51%     
==========================================
  Files          16       16              
  Lines        1753     1772      +19     
==========================================
- Hits         1713     1634      -79     
- Misses         40      138      +98     

Flag	Coverage Δ
unittests	92.21% <85.00%> (-5.51%)	⬇️

Flags with carried forward coverage won't be shown. Click here to find out more.

☔ View full report in Codecov by Sentry.
📢 Have feedback on the report? Share it here.

[candleindark]

@satra @mvandenburgh This solution will generated JSON schemas that have "@context" as a property. Currently, I made "@context" as a required property, but I can make it optional. Will the additional "@context" property, required or optional, be a problem for the web UI? Can we make the web UI ignore this property if needed?

[yarikoptic]

are failing tests expected?

@mvandenburgh please point @candleindark to where to change URL to the json schema so some alternative one could be tested.

[mvandenburgh]

@candleindark the json schema used by the frontend is set here - https://github.com/dandi/dandi-archive/blob/master/dandiapi/api/views/info.py#L11-L14
If you want to use a custom schema, you could serve out the json file locally and point it at that url.

[candleindark]

are failing tests expected?

Yes. Because I made "@context" as a required property in the JSON schemas, the tests need to be updated.

[yarikoptic]

look here for an example on how we add extra to json schema.

May be you could add extra for your @context which would state something like "includeInUI": False

[candleindark]

@candleindark the json schema used by the frontend is set here - https://github.com/dandi/dandi-archive/blob/master/dandiapi/api/views/info.py#L11-L14 If you want to use a custom schema, you could serve out the json file locally and point it at that url.

@mvandenburgh I completed the initial setup, started the API server using docker compose up, and brought up the web app using yarn run dev. Both the API server and the web app seemed to be running normally. However, whenever I modified the value of schema_url in https://github.com/dandi/dandi-archive/blob/dc454f44b9e814b8a09781e9b798662e6cb8528f/dandiapi/api/views/info.py#L11-L14 to a URL of a locally served schema I got the "Connection to server failed." error. This was true even for the unmodified dandiset schema at 0.6.8, which seems to be the current default version (as schema_url = "http://localhost:9999/0.6.8/dandiset.json" or schema_url = "https://localhost:8443/0.6.8/dandiset.json").

On a related note, I was wondering if the web app/UI depends on the published-dandiset.json, asset.json, published-asset.json as well as the dandiset.json?

[candleindark]

I just tried the native development setup with dandiset.json hosted at a public URL using ngrok ( and set schema_url = "https://4ed0-2600-6c50-60f0-6500-9dec-627c-49b1-f7a0.ngrok-free.app/0.6.8/dandiset.json"). I got the same result as I mentioned in the last post.

[mvandenburgh]

On a related note, I was wondering if the web app/UI depends on the published-dandiset.json, asset.json, published-asset.json as well as the dandiset.json

The web UI only depends on dandiset.json.

@mvandenburgh I completed the initial setup, started the API server using docker compose up, and brought up the web app using yarn run dev. Both the API server and the web app seemed to be running normally. However, whenever I modified the value of schema_url in dandi/dandi-archive@dc454f4/dandiapi/api/views/info.py#L11-L14 to a URL of a locally served schema I got the "Connection to server failed." error. This was true even for the unmodified dandiset schema at 0.6.8, which seems to be the current default version (as schema_url = "http://localhost:9999/0.6.8/dandiset.json" or schema_url = "https://localhost:8443/0.6.8/dandiset.json").

On a related note, I was wondering if the web app/UI depends on the published-dandiset.json, asset.json, published-asset.json as well as the dandiset.json?

Does the web UI always display that "Connection to server failed" message, or just when you modify schema_url? If the former, then that indicates something is going wrong with the local Django server.

[candleindark]

Does the web UI always display that "Connection to server failed" message, or just when you modify schema_url? If the former, then that indicates something is going wrong with the local Django server.

It is the latter. I got the attached landing page if nothing no code is modified. In fact, there is no problem even if I hardcode the schema_url to a particular version. For example,

schema_url = (
    'https://raw.githubusercontent.com/dandi/schema/master/'
    f'releases/0.6.8/dandiset.json'
)

or even

schema_url = (
    'https://raw.githubusercontent.com/dandi/schema/master/'
    f'releases/0.6.7/dandiset.json'
)

The problem occurred when schema_url was set to the URLs I mentioned in previous posts (though the localhost ones are problematic if I run the Django service in a container).