diff --git a/apps/transport/lib/transport_web/api/controllers/aom_controller.ex b/apps/transport/lib/transport_web/api/controllers/aom_controller.ex index 68cf642ea9..8d8113f0fd 100644 --- a/apps/transport/lib/transport_web/api/controllers/aom_controller.ex +++ b/apps/transport/lib/transport_web/api/controllers/aom_controller.ex @@ -16,8 +16,7 @@ defmodule TransportWeb.API.AomController do def by_coordinates_operation, do: %Operation{ tags: ["aom"], - summary: "Show AOM by coordinates", - description: "Show covered regions", + summary: "Show first AOM containing a point with given coordinates", operationId: "API.AOMController.by_coordinates", parameters: [ Operation.parameter(:lon, :query, :number, "Longitude"), @@ -43,9 +42,8 @@ defmodule TransportWeb.API.AomController do @spec by_insee_operation :: OpenApiSpex.Operation.t() def by_insee_operation, do: %Operation{ - tags: ["insee"], - summary: "Show AOM by INSEE", - description: "Show covered regions", + tags: ["aom"], + summary: "Search AOM by INSEE code", operationId: "API.AOMController.by_insee_operation", parameters: [ Operation.parameter(:insee, :path, :string, "INSEE") @@ -92,9 +90,8 @@ defmodule TransportWeb.API.AomController do @spec geojson_operation :: OpenApiSpex.Operation.t() def geojson_operation, do: %Operation{ - tags: ["geojson"], - summary: "Show geojson of AOM", - description: "Show covered regions", + tags: ["aom"], + summary: "Show GeoJSON of all available AOMs (Autorités Organisatrices de la Mobilité)", operationId: "API.AOMController.geojson_operation", parameters: [], responses: %{ diff --git a/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex b/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex index eccc6f4953..926de2f8e5 100644 --- a/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex +++ b/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex @@ -4,7 +4,7 @@ defmodule TransportWeb.API.DatasetController do alias Helpers alias OpenApiSpex.Operation alias DB.{AOM, Dataset, Repo, Resource} - alias TransportWeb.API.Schemas.{DatasetsResponse, GeoJSONResponse} + alias TransportWeb.API.Schemas.{DatasetDetails, DatasetsResponse, GeoJSONResponse} alias Geo.{JSON, MultiPolygon} # The default (one minute) felt a bit too high for someone doing scripted operations @@ -19,12 +19,15 @@ defmodule TransportWeb.API.DatasetController do def datasets_operation, do: %Operation{ tags: ["datasets"], - summary: "Show datasets and its resources", - description: "For every dataset, show its associated resources, url and validity date", + summary: "List all datasets (with their resources)", + description: ~s"This call returns (in a single, non-paginated response) the list of all the + datasets referenced on the site, along with their associated resources. The datasets + and resources are here provided in summarized form (without history & conversions). + You can call `/api/datasets/:id` for each dataset to get extra data (history & conversions)", operationId: "API.DatasetController.datasets", parameters: [], responses: %{ - 200 => Operation.response("Dataset", "application/json", DatasetsResponse) + 200 => Operation.response("DatasetsResponse", "application/json", DatasetsResponse) } } @@ -50,12 +53,13 @@ defmodule TransportWeb.API.DatasetController do def by_id_operation, do: %Operation{ tags: ["datasets"], - summary: "Show given dataset and its resources", - description: "For one dataset, show its associated resources, url and validity date", + summary: "Return the details of a given dataset and its resources", + description: + ~s"Returns the detailed version of a dataset, showing its resources, the resources history & conversions.", operationId: "API.DatasetController.datasets_by_id", - parameters: [Operation.parameter(:id, :path, :string, "id")], + parameters: [Operation.parameter(:id, :path, :string, "datagouv id of the dataset you want to retrieve")], responses: %{ - 200 => Operation.response("Dataset", "application/json", DatasetsResponse) + 200 => Operation.response("DatasetDetails", "application/json", DatasetDetails) } } @@ -63,8 +67,8 @@ defmodule TransportWeb.API.DatasetController do def geojson_by_id_operation, do: %Operation{ tags: ["datasets"], - summary: "Show given dataset geojson", - description: "For one dataset, show its associated geojson", + summary: "Show given dataset GeoJSON", + description: "For one dataset, show its associated GeoJSON.", operationId: "API.DatasetController.datasets_geojson_by_id", parameters: [Operation.parameter(:id, :path, :string, "id")], responses: %{ @@ -195,6 +199,7 @@ defmodule TransportWeb.API.DatasetController do ) end + # NOTE: only added in detailed dataset view defp add_conversions(%{"resources" => resources} = data, %Dataset{} = dataset) do conversions = dataset diff --git a/apps/transport/lib/transport_web/api/controllers/places_controller.ex b/apps/transport/lib/transport_web/api/controllers/places_controller.ex index 86e486b132..74fbdecfe0 100644 --- a/apps/transport/lib/transport_web/api/controllers/places_controller.ex +++ b/apps/transport/lib/transport_web/api/controllers/places_controller.ex @@ -79,7 +79,7 @@ defmodule TransportWeb.API.PlacesController do %Operation{ tags: ["datasets"], summary: "Autocomplete search for datasets", - description: "Given a search input, return potentialy corresponding results with the associated url", + description: "Given a search input, return potentially corresponding results with the associated search URL", operationId: "API.DatasetController.datasets_autocomplete", parameters: [Operation.parameter(:q, :query, :string, "query")], responses: %{ diff --git a/apps/transport/lib/transport_web/api/controllers/stats_controller.ex b/apps/transport/lib/transport_web/api/controllers/stats_controller.ex index 71cfecdd95..c9d999b6e5 100644 --- a/apps/transport/lib/transport_web/api/controllers/stats_controller.ex +++ b/apps/transport/lib/transport_web/api/controllers/stats_controller.ex @@ -13,7 +13,7 @@ defmodule TransportWeb.API.StatsController do @spec regions_operation() :: Operation.t() def regions_operation, do: %Operation{ - tags: ["regions"], + tags: ["stats"], summary: "Show regions", description: "Show covered french administrative regions", operationId: "API.StatsController.regions", @@ -26,7 +26,7 @@ defmodule TransportWeb.API.StatsController do @spec index_operation() :: Operation.t() def index_operation, do: %Operation{ - tags: ["index"], + tags: ["stats"], summary: "Show regions", description: "Show covered french administrative regions", operationId: "API.StatsController.index", @@ -39,7 +39,7 @@ defmodule TransportWeb.API.StatsController do @spec bike_scooter_sharing_operation() :: Operation.t() def bike_scooter_sharing_operation, do: %Operation{ - tags: ["bike-scooter-sharing"], + tags: ["stats"], summary: "Show bike and scooter sharing stats", description: "Show bike and scooter sharing stats", operationId: "API.StatsController.bike_scooter_sharing", @@ -52,7 +52,7 @@ defmodule TransportWeb.API.StatsController do @spec quality_operation() :: Operation.t() def quality_operation, do: %Operation{ - tags: ["quality"], + tags: ["stats"], summary: "Show data quality stats", description: "Show data quality stats", operationId: "API.StatsController.quality", @@ -66,6 +66,7 @@ defmodule TransportWeb.API.StatsController do def geojson(features), do: %{ "type" => "FeatureCollection", + # This is now completely incorrect! "name" => "Autorités organisatrices de Mobiltés", "features" => features } @@ -376,6 +377,9 @@ defmodule TransportWeb.API.StatsController do %{ "geometry" => r.geometry |> JSON.encode!(), "type" => "Feature", + # NOTE: there is a bug here - the key is an atom. + # I won't change it now because it would mean more changes somewhere else, maybe. + # `Map.reject(fn({k,v}) -> k == :geometry end)` will do it. "properties" => Map.take(r, Enum.filter(Map.keys(r), fn k -> k != "geometry" end)) } end) diff --git a/apps/transport/lib/transport_web/api/router.ex b/apps/transport/lib/transport_web/api/router.ex index 3bc4a62469..89abf5dd23 100644 --- a/apps/transport/lib/transport_web/api/router.ex +++ b/apps/transport/lib/transport_web/api/router.ex @@ -60,7 +60,7 @@ defmodule TransportWeb.API.Router do do: %{ info: %{ version: "1.0", - title: "Transport.data.gouv.fr API" + title: "transport.data.gouv.fr API" } } end diff --git a/apps/transport/lib/transport_web/api/schemas.ex b/apps/transport/lib/transport_web/api/schemas.ex index 33babb5fee..15c40250d7 100644 --- a/apps/transport/lib/transport_web/api/schemas.ex +++ b/apps/transport/lib/transport_web/api/schemas.ex @@ -1,6 +1,14 @@ defmodule TransportWeb.API.Schemas do @moduledoc """ OpenAPI schema defintions + + Useful documentation: + - https://json-schema.org/understanding-json-schema/reference/array.html + - https://json-schema.org/understanding-json-schema/reference/object.html + - https://json-schema.org/understanding-json-schema/reference/string.html + + A good chunk of our GeoJSON responses do not pass our OpenAPI specs. It would need more work. + """ require OpenApiSpex alias OpenApiSpex.{ExternalDocumentation, Schema} @@ -14,13 +22,24 @@ defmodule TransportWeb.API.Schemas do type: :object, description: "GeoJSon geometry", required: [:type], - externalDocs: %ExternalDocumentation{url: "http://geojson.org/geojson-spec.html#geometry-objects"}, + externalDocs: %ExternalDocumentation{ + url: "http://geojson.org/geojson-spec.html#geometry-objects" + }, properties: %{ type: %Schema{ type: :string, - enum: ["Point", "LineString", "Polygon", "MultiPoint", "MultiLineString", "MultiPolygon"] + enum: [ + "Point", + "LineString", + "Polygon", + "MultiPoint", + "MultiLineString", + "MultiPolygon" + ] } - } + }, + # allow extra properties since this is used as a composable base + additionalProperties: true }) end @@ -66,7 +85,8 @@ defmodule TransportWeb.API.Schemas do coordinates: %Schema{type: :array, items: Point2D} } } - ] + ], + additionalProperties: false }) end @@ -77,17 +97,27 @@ defmodule TransportWeb.API.Schemas do OpenApiSpex.schema(%{ type: :object, title: "Polygon", - description: "GeoJSon geometry", + description: "GeoJSON geometry", externalDocs: %ExternalDocumentation{url: "http://geojson.org/geojson-spec.html#id4"}, allOf: [ GeometryBase.schema(), %Schema{ type: :object, properties: %{ - coordinates: %Schema{type: :array, items: %Schema{type: :array, items: Point2D}} + coordinates: %Schema{ + type: :array, + items: %Schema{ + type: :array, + items: %Schema{ + type: :array, + items: Point2D + } + } + } } } - ] + ], + additionalProperties: false }) end @@ -108,7 +138,8 @@ defmodule TransportWeb.API.Schemas do coordinates: %Schema{type: :array, items: Point2D} } } - ] + ], + additionalProperties: false }) end @@ -129,7 +160,8 @@ defmodule TransportWeb.API.Schemas do coordinates: %Schema{type: :array, items: %Schema{type: :array, items: Point2D}} } } - ] + ], + additionalProperties: false }) end @@ -153,7 +185,8 @@ defmodule TransportWeb.API.Schemas do } } } - ] + ], + additionalProperties: false }) end @@ -171,7 +204,8 @@ defmodule TransportWeb.API.Schemas do MultiPoint.schema(), MultiLineString.schema(), MultiPolygon.schema() - ] + ], + additionalProperties: false }) end @@ -187,11 +221,16 @@ defmodule TransportWeb.API.Schemas do properties: %{ type: %Schema{type: :string, enum: ["Feature"]}, geometry: Geometry, - properties: %Schema{type: :object}, - id: %Schema{ - oneOf: [%Schema{type: :string}, %Schema{type: :number}] + properties: %Schema{ + type: :object, + properties: %{ + id: %Schema{ + oneOf: [%Schema{type: :string}, %Schema{type: :number}] + } + } } - } + }, + additionalProperties: false }) end @@ -204,8 +243,11 @@ defmodule TransportWeb.API.Schemas do title: "FeatureCollection", description: "FeatureCollection object", properties: %{ - features: %Schema{type: :array, items: Feature} - } + features: %Schema{type: :array, items: Feature}, + type: %Schema{type: :string, enum: ["FeatureCollection"], required: true}, + name: %Schema{type: :string} + }, + additionalProperties: false }) end @@ -213,17 +255,176 @@ defmodule TransportWeb.API.Schemas do @moduledoc false require OpenApiSpex + @properties %{ + siren: %Schema{type: :string, nullable: true}, + nom: %Schema{type: :string}, + insee_commune_principale: %Schema{type: :string}, + forme_juridique: %Schema{type: :string}, + departement: %Schema{type: :string} + } + + OpenApiSpex.schema(%{ + title: "AOMResponse", + description: "AOM object, as returned from AOMs endpoints", + type: :object, + properties: @properties, + required: @properties |> Map.keys(), + additionalProperties: false + }) + end + + defmodule AOMShortRef do + @moduledoc false + require OpenApiSpex + OpenApiSpex.schema(%{ - title: "AOM", - description: "AOM object", + title: "AOMShortRef", + description: + "AOM object, as embedded in datasets (short version - DEPRECATED, only there for retrocompatibility, use covered_area instead)", type: :object, + required: [:name], properties: %{ - siren: %Schema{type: :string}, - nom: %Schema{type: :string}, - insee_commune_principale: %Schema{type: :string}, - forme_juridique: %Schema{type: :string}, - departement: %Schema{type: :string} - } + # nullable because we saw it null in actual production data + # probably exactly what's described in https://github.com/etalab/transport-site/issues/3422 + siren: %Schema{type: :string, nullable: true}, + name: %Schema{type: :string, nullable: true} + }, + additionalProperties: false + }) + end + + defmodule CoveredArea.Country do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "CoveredArea.Country", + type: :object, + required: [ + :country, + :name, + :type + ], + properties: %{ + name: %Schema{type: :string}, + type: %Schema{type: :string, enum: ["country"], required: true}, + country: %Schema{ + type: :object, + required: [:name], + properties: %{ + name: %Schema{type: :string} + }, + additionalProperties: false + } + }, + additionalProperties: false + }) + end + + # Very similar to `AOMShortRef` but ultimately only CoveredAreas should be kept (?) + defmodule CoveredArea.AOM do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "CoveredArea.AOM", + type: :object, + required: [ + :aom, + :name, + :type + ], + properties: %{ + name: %Schema{type: :string}, + type: %Schema{type: :string, enum: ["aom"], required: true}, + aom: %Schema{ + type: :object, + required: [:name, :siren], + properties: %{ + name: %Schema{type: :string}, + siren: %Schema{type: :string} + }, + additionalProperties: false + } + }, + additionalProperties: false + }) + end + + defmodule CoveredArea.Cities do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "CoveredArea.Cities", + type: :object, + required: [ + :cities, + :name, + :type + ], + properties: %{ + name: %Schema{type: :string}, + type: %Schema{type: :string, enum: ["cities"], required: true}, + cities: %Schema{ + type: :array, + items: %Schema{ + type: :object, + required: [:name, :insee], + properties: %{ + name: %Schema{type: :string}, + insee: %Schema{type: :string} + }, + additionalProperties: false + } + } + }, + additionalProperties: false + }) + end + + defmodule CoveredArea.Region do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "CoveredArea.Region", + type: :object, + required: [ + :region, + :name, + :type + ], + properties: %{ + name: %Schema{type: :string}, + type: %Schema{type: :string, enum: ["region"], required: true}, + region: %Schema{ + type: :object, + required: [:name], + properties: %{ + name: %Schema{type: :string} + }, + additionalProperties: false + } + }, + additionalProperties: false + }) + end + + defmodule CoveredArea do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "CoveredArea", + type: :object, + oneOf: [ + CoveredArea.Country.schema(), + CoveredArea.AOM.schema(), + CoveredArea.Region.schema(), + CoveredArea.Cities.schema() + ], + additionalProperties: false }) end @@ -239,35 +440,93 @@ defmodule TransportWeb.API.Schemas do Geometry.schema(), Feature.schema(), FeatureCollection.schema() - ] + ], + additionalProperties: false + }) + end + + defmodule Publisher do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "Publisher", + description: "Publisher", + type: :object, + properties: %{ + # as seen in production data + name: %Schema{type: :string, nullable: true}, + type: %Schema{type: :string} + }, + additionalProperties: false }) end - defmodule Utils do + defmodule ResourceUtils do @moduledoc false def get_resource_prop(conversions: false), do: %{ - url: %Schema{type: :string, description: "Stable URL of the file"}, - original_url: %Schema{type: :string, description: "Direct URL of the file"}, - title: %Schema{type: :string, description: "Title of the resource"}, - updated: %Schema{type: :string, description: "Last update date-time"}, - end_calendar_validity: %Schema{ + datagouv_id: %Schema{ type: :string, - description: - "The last day of the validity period of the file (read from the calendars for the GTFS). null if the file couldn’t be read" + description: "Data gouv id of the resource" + }, + id: %Schema{ + type: :integer, + description: "transport.data.gouv.fr's ID" + }, + format: %Schema{ + type: :string, + description: "The format of the resource (GTFS, NeTEx, etc.)" + }, + is_available: %Schema{ + type: :boolean, + description: "Availability of the resource" }, - start_calendar_validity: %Schema{ + original_url: %Schema{ type: :string, - description: - "The first day of the validity period of the file (read from the calendars for the GTFS). null if the file couldn’t be read" + description: "Direct URL of the file" + }, + url: %Schema{type: :string, description: "Stable URL of the file"}, + page_url: %Schema{ + type: :string, + description: "URL of the resource on transport.data.gouv.fr" + }, + features: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: "Features" + }, + title: %Schema{type: :string, description: "Title of the resource"}, + filesize: %Schema{ + type: :integer, + description: "Size of the resource in bytes" }, - format: %Schema{type: :string, description: "The format of the resource (GTFS, NeTEx, etc.)"}, metadata: %Schema{ type: :object, description: "Some metadata about the resource" + }, + type: %Schema{type: :string, description: "Category of the data"}, + modes: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: "Types of transportation" + }, + updated: %Schema{ + type: :string, + format: "date-time", + description: "Last update date-time" + }, + schema_name: %Schema{ + type: :string, + description: "Data schema followed by the resource" + }, + schema_version: %Schema{ + type: :string, + description: "Version of the data schema followed by the resource" } } + # conversions are only shown in the detailed dataset view def get_resource_prop(conversions: true), do: [conversions: false] @@ -279,16 +538,52 @@ defmodule TransportWeb.API.Schemas do GeoJSON: %Schema{ type: :object, description: "Conversion to the GeoJSON format", - properties: conversion_properties() + required: conversion_properties() |> Map.keys(), + properties: conversion_properties(), + additionalProperties: false }, NeTEx: %Schema{ type: :object, description: "Conversion to the NeTEx format", - properties: conversion_properties() + required: conversion_properties() |> Map.keys(), + properties: conversion_properties(), + additionalProperties: false } } }) + def get_community_resource_prop do + [conversions: false] + |> ResourceUtils.get_resource_prop() + |> Map.put(:community_resource_publisher, %Schema{ + type: :string, + description: "Name of the producer of the community resource" + }) + |> Map.put( + :original_resource_url, + %Schema{ + type: :string, + description: """ + Some community resources have been generated from another dataset (like the generated NeTEx / GeoJSON). + Those resources have a `original_resource_url` equals to the original resource's `original_url` + """ + } + ) + end + + # DRYing keys here + def get_resource_optional_properties_keys do + [ + :features, + :filesize, + :metadata, + :modes, + :original_resource_url, + :schema_name, + :schema_version + ] + end + defp conversion_properties, do: %{ filesize: %Schema{type: :integer, description: "File size in bytes"}, @@ -301,14 +596,35 @@ defmodule TransportWeb.API.Schemas do } end - defmodule Resource do + defmodule DetailedResource do + @moduledoc false + require OpenApiSpex + + @properties ResourceUtils.get_resource_prop(conversions: true) + @optional_properties ResourceUtils.get_resource_optional_properties_keys() + + OpenApiSpex.schema(%Schema{ + type: :object, + description: "A single resource (including conversions)", + required: (@properties |> Map.keys()) -- @optional_properties, + properties: @properties, + additionalProperties: false + }) + end + + defmodule SummarizedResource do @moduledoc false require OpenApiSpex + @properties ResourceUtils.get_resource_prop(conversions: false) + @optional_properties ResourceUtils.get_resource_optional_properties_keys() + OpenApiSpex.schema(%Schema{ type: :object, - description: "A single resource", - properties: Utils.get_resource_prop(conversions: true) + description: "A single resource (summarized version)", + required: (@properties |> Map.keys()) -- @optional_properties, + properties: @properties, + additionalProperties: false }) end @@ -316,51 +632,161 @@ defmodule TransportWeb.API.Schemas do @moduledoc false require OpenApiSpex + @properties ResourceUtils.get_community_resource_prop() + @optional_properties ResourceUtils.get_resource_optional_properties_keys() + OpenApiSpex.schema(%Schema{ type: :object, description: "A single community resource", - properties: - [conversions: false] - |> Utils.get_resource_prop() - |> Map.put(:community_resource_publisher, %Schema{ - type: :string, - description: "Name of the producer of the community resource" - }) - |> Map.put(:original_resource_url, %Schema{ - type: :string, - description: """ - some community resources have been generated from another dataset (like the generated NeTEx / GeoJson). - Those resources have a `original_resource_url` equals to the original resource's `original_url` - """ - }) + required: (@properties |> Map.keys()) -- @optional_properties, + properties: @properties, + additionalProperties: false }) end - defmodule DatasetsResponse do + defmodule ResourceHistory do @moduledoc false require OpenApiSpex - OpenApiSpex.schema(%{ - title: "Dataset", - description: "A dataset is a composed of at least one GTFS resource", + @properties %{ + inserted_at: %Schema{type: :string, format: "date-time"}, + updated_at: %Schema{type: :string, format: "date-time"}, + last_up_to_date_at: %Schema{type: :string, format: "date-time", nullable: true}, + payload: %Schema{type: :object, description: "Payload (loosely specified at the moment)"}, + latest_schema_version_to_date: %Schema{type: :string}, + permanent_url: %Schema{type: :string}, + resource_latest_url: %Schema{type: :string}, + resource_url: %Schema{type: :string}, + # NOTE: apparently, can be nil sometimes! This should be investigated + resource_id: %Schema{type: :integer, nullable: true}, + schema_name: %Schema{type: :string}, + schema_version: %Schema{type: :string}, + title: %Schema{type: :string}, + uuid: %Schema{type: :string} + } + @optional_properties [ + :latest_schema_version_to_date, + :permanent_url, + :resource_latest_url, + :resource_url, + :schema_name, + :schema_version, + :uuid, + :title + ] + + OpenApiSpex.schema(%Schema{ type: :object, - properties: %{ - updated: %Schema{type: :string, description: "The last update of any resource of that dataset"}, - name: %Schema{type: :string}, - licence: %Schema{type: :string, description: "The licence of the dataset"}, - created_at: %Schema{type: :string, format: :date, description: "Date of creation of the dataset"}, - aom: %Schema{type: :string, description: "Transit authority responsible of this authority"}, + description: "A resource version", + required: (@properties |> Map.keys()) -- @optional_properties, + properties: @properties, + additionalProperties: false + }) + end + + defmodule DatasetUtils do + @moduledoc false + + def get_dataset_prop(details: details) do + # base resource comes in 2 flavors + resource_type = if details == true, do: DetailedResource, else: SummarizedResource + + base = %{ + datagouv_id: %Schema{ + type: :string, + description: "Data gouv id for this dataset" + }, + id: %Schema{type: :string, description: "Same as datagouv_id"}, + updated: %Schema{ + type: :string, + format: :"date-time", + description: "The last update of any resource of that dataset (`null` if the dataset has no resources)", + nullable: true + }, + page_url: %Schema{ + type: :string, + description: "transport.data.gouv.fr page for this dataset" + }, + publisher: Publisher.schema(), + slug: %Schema{type: :string, description: "unique dataset slug"}, + title: %Schema{type: :string}, + type: %Schema{type: :string}, + licence: %Schema{ + type: :string, + description: "The licence of the dataset" + }, + created_at: %Schema{ + type: :string, + format: :date, + description: "Date of creation of the dataset" + }, + # Obsolete, to be removed (see https://github.com/etalab/transport-site/issues/3422) + aom: AOMShortRef.schema(), resources: %Schema{ type: :array, - description: "All the resources (files) associated with the dataset", - items: Resource + description: "All the resources associated with the dataset", + # NOTE: community resources will have to be removed from here + # https://github.com/etalab/transport-site/issues/3407 + items: %Schema{anyOf: [resource_type, CommunityResource]} }, community_resources: %Schema{ type: :array, - description: "All the community resources (files published by the community) associated with the dataset", + description: "All the community resources (published by the community) associated with the dataset", items: CommunityResource - } + }, + covered_area: CoveredArea.schema() } + + if details do + base + |> Map.put(:history, %Schema{type: :array, items: ResourceHistory}) + else + base + end + end + end + + defmodule DatasetSummary do + @moduledoc false + require OpenApiSpex + + @properties DatasetUtils.get_dataset_prop(details: false) + + OpenApiSpex.schema(%{ + title: "DatasetSummary", + description: "A dataset is a composed of one or more resources (summarized version)", + type: :object, + required: @properties |> Map.keys(), + properties: @properties, + additionalProperties: false + }) + end + + defmodule DatasetDetails do + @moduledoc false + require OpenApiSpex + + @properties DatasetUtils.get_dataset_prop(details: true) + + OpenApiSpex.schema(%{ + title: "DatasetDetails", + description: + "A dataset is a composed of one or more resources (detailed version, including history & conversions).", + type: :object, + required: @properties |> Map.keys(), + properties: @properties, + additionalProperties: false + }) + end + + defmodule DatasetsResponse do + @moduledoc false + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "DatasetsResponse", + type: :array, + items: DatasetSummary.schema() }) end @@ -368,15 +794,19 @@ defmodule TransportWeb.API.Schemas do @moduledoc false require OpenApiSpex + @properties %{ + url: %Schema{type: :string, description: "URL of the Resource"}, + type: %Schema{type: :string, description: "type of the resource (commune, region, aom)"}, + name: %Schema{type: :string, description: "name of the resource"} + } + OpenApiSpex.schema(%{ title: "Autocomplete result", description: "One result of the autocomplete", type: :object, - properties: %{ - url: %Schema{type: :string, description: "URL of the Resource"}, - type: %Schema{type: :string, description: "type of the resource (commune, region, aom)"}, - name: %Schema{type: :string, description: "name of the resource"} - } + required: @properties |> Map.keys(), + properties: @properties, + additionalProperties: false }) end @@ -385,7 +815,7 @@ defmodule TransportWeb.API.Schemas do require OpenApiSpex OpenApiSpex.schema(%{ - title: "Autocomplete results", + title: "AutocompleteResponse", description: "An array of matching results", type: :array, items: AutocompleteItem diff --git a/apps/transport/lib/transport_web/api/spec.ex b/apps/transport/lib/transport_web/api/spec.ex index b50a4f5af5..ca46605746 100644 --- a/apps/transport/lib/transport_web/api/spec.ex +++ b/apps/transport/lib/transport_web/api/spec.ex @@ -2,14 +2,26 @@ defmodule TransportWeb.API.Spec do @moduledoc """ OpenAPI specifications """ - alias OpenApiSpex.{Info, OpenApi, Paths} + alias OpenApiSpex.{Contact, Info, OpenApi, Paths} @spec spec :: OpenApiSpex.OpenApi.t() def spec do %OpenApi{ + # # https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md#info-object info: %Info{ - title: "Transport.data.gouv.fr API", - version: "1.0" + title: "transport.data.gouv.fr API", + version: "1.0", + description: ~s""" + Extra documentation. + + The structure of the returned data is detailed at the bottom (see `Schemas`) and on each query (click on `Schema` near `Example Value`). + + To create a query, add the domain `https://transport.data.gouv.fr` and the path (e.g. `/api/datasets`). + """, + contact: %Contact{ + name: "API email support", + email: Application.fetch_env!(:transport, :contact_email) + } }, paths: Paths.from_router(TransportWeb.API.Router) } diff --git a/apps/transport/lib/transport_web/router.ex b/apps/transport/lib/transport_web/router.ex index 973c1d75bb..3d509f9c0c 100644 --- a/apps/transport/lib/transport_web/router.ex +++ b/apps/transport/lib/transport_web/router.ex @@ -45,7 +45,14 @@ defmodule TransportWeb.Router do scope "/", OpenApiSpex.Plug do pipe_through(:browser_no_csp) - get("/swaggerui", SwaggerUI, path: "/api/openapi") + + # NOTE: version of SwaggerUI is currently hardcoded by the Elixir package. + # See: https://github.com/open-api-spex/open_api_spex/issues/559 + get("/swaggerui", SwaggerUI, + path: "/api/openapi", + # See: https://github.com/etalab/transport-site/issues/3421 + syntax_highlight: false + ) end scope "/", TransportWeb do diff --git a/apps/transport/lib/transport_web/templates/dataset/index.html.heex b/apps/transport/lib/transport_web/templates/dataset/index.html.heex index bdc826bfc3..b483ecb67c 100644 --- a/apps/transport/lib/transport_web/templates/dataset/index.html.heex +++ b/apps/transport/lib/transport_web/templates/dataset/index.html.heex @@ -234,9 +234,12 @@ diff --git a/apps/transport/mix.exs b/apps/transport/mix.exs index bd693cace4..d106e7dc31 100644 --- a/apps/transport/mix.exs +++ b/apps/transport/mix.exs @@ -132,7 +132,8 @@ defmodule Transport.Mixfile do {:appsignal, "~> 2.0"}, {:appsignal_phoenix, "~> 2.0"}, {:vega_lite, "~> 0.1.7"}, - {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false}, + {:req, "~> 0.3.11", only: :dev}, + {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false} ] end end diff --git a/apps/transport/test/db/db_dataset_test.exs b/apps/transport/test/db/db_dataset_test.exs index 9b2a2fd9d1..3ba5344e68 100644 --- a/apps/transport/test/db/db_dataset_test.exs +++ b/apps/transport/test/db/db_dataset_test.exs @@ -473,8 +473,8 @@ defmodule DB.DatasetDBTest do insert(:dataset, datagouv_id: datagouv_id = Ecto.UUID.generate()) # we test a random change to check if the changeset is valid without an organization type specified - assert {:ok, %Ecto.Changeset{changes: %{licence: "lov2"}}} = - Dataset.changeset(%{"datagouv_id" => datagouv_id, "licence" => "lov2"}) + assert {:ok, %Ecto.Changeset{changes: %{licence: "fr-lo"}}} = + Dataset.changeset(%{"datagouv_id" => datagouv_id, "licence" => "fr-lo"}) end test "incorrect organization type" do diff --git a/apps/transport/test/support/factory.ex b/apps/transport/test/support/factory.ex index b9be31b653..69264f48bb 100644 --- a/apps/transport/test/support/factory.ex +++ b/apps/transport/test/support/factory.ex @@ -18,6 +18,7 @@ defmodule DB.Factory do %DB.AOM{ insee_commune_principale: "38185", nom: "Grenoble", + siren: "253800825", region: build(:region), # The value must be unique, ExFactory helps us with a named sequence composition_res_id: 1000 + sequence("composition_res_id", & &1) @@ -29,12 +30,15 @@ defmodule DB.Factory do created_at: DateTime.utc_now(), last_update: DateTime.utc_now(), datagouv_title: "Hello", + custom_title: "Hello", slug: sequence(:slug, fn i -> "dataset_slug_#{i}" end), datagouv_id: sequence(:datagouv_id, fn i -> "dataset_datagouv_id_#{i}" end), organization_id: sequence(:organization_id, fn i -> "dataset_organization_id_#{i}" end), + licence: "lov2", # NOTE: need to figure out how to pass aom/region together with changeset checks here aom: build(:aom), - tags: [] + tags: [], + type: "public-transit" } end @@ -43,7 +47,11 @@ defmodule DB.Factory do last_import: DateTime.utc_now(), last_update: DateTime.utc_now(), title: "GTFS.zip", - latest_url: "url" + # NOTE: we should use real urls here (but something safe on localhost?) + latest_url: "url", + url: "url", + type: "main", + datagouv_id: Ecto.UUID.generate() } end diff --git a/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs b/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs new file mode 100644 index 0000000000..312e2b3653 --- /dev/null +++ b/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs @@ -0,0 +1,46 @@ +defmodule TransportWeb.API.AomControllerTest do + use TransportWeb.ConnCase, async: true + import DB.Factory + import OpenApiSpex.TestAssertions + + setup do + Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + test "GET /api/aoms (by_coordinates)", %{conn: conn} do + geom = %Geo.Polygon{ + coordinates: [ + [ + {55.0, 3.0}, + {60.0, 3.0}, + {60.0, 5.0}, + {55.0, 5.0}, + {55.0, 3.0} + ] + ], + srid: 4326, + properties: %{} + } + + insert(:aom, + geom: geom, + departement: "75", + forme_juridique: "Communauté de communes", + insee_commune_principale: "38185", + siren: "247400690" + ) + + conn = conn |> get(TransportWeb.API.Router.Helpers.aom_path(conn, :by_coordinates, lon: 56.0, lat: 4.0)) + json = json_response(conn, 200) + + assert_response_schema(json, "AOMResponse", TransportWeb.API.Spec.spec()) + + assert json == %{ + "departement" => "75", + "forme_juridique" => "Communauté de communes", + "insee_commune_principale" => "38185", + "nom" => "Grenoble", + "siren" => "247400690" + } + end +end diff --git a/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs b/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs index 8a1f4dcd5b..8077795fde 100644 --- a/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs @@ -4,6 +4,7 @@ defmodule TransportWeb.API.DatasetControllerTest do alias TransportWeb.API.Router.Helpers import DB.Factory import Mox + import OpenApiSpex.TestAssertions setup :verify_on_exit! @@ -12,15 +13,19 @@ defmodule TransportWeb.API.DatasetControllerTest do conn = conn |> get(path) [etag] = conn |> get_resp_header("etag") - json_response(conn, 200) + json = json_response(conn, 200) assert etag assert conn |> get_resp_header("cache-control") == ["max-age=60, public, must-revalidate"] # Passing the previous `ETag` value in a new HTTP request returns a 304 conn |> recycle() |> put_req_header("if-none-match", etag) |> get(path) |> response(304) + + api_spec = TransportWeb.API.Spec.spec() + assert_schema(json, "DatasetsResponse", api_spec) end - test "GET /api/datasets *with* history, multi_validation and resource_metadata", %{conn: conn} do + test "GET /api/datasets then /api/datasets/:id *with* history, multi_validation and resource_metadata", + %{conn: conn} do dataset = insert(:dataset, custom_title: "title", @@ -161,7 +166,9 @@ defmodule TransportWeb.API.DatasetControllerTest do |> DateTime.to_iso8601() } - assert [dataset_res] == conn |> get(path) |> json_response(200) + assert json = conn |> get(path) |> json_response(200) + assert [dataset_res] == json + assert_schema(json, "DatasetsResponse", TransportWeb.API.Spec.spec()) # check the result is in line with a query on this dataset # only difference: individual dataset adds information about history and conversions @@ -176,7 +183,9 @@ defmodule TransportWeb.API.DatasetControllerTest do |> Map.merge(%{"history" => []}) |> Map.put("resources", Enum.map(dataset_res["resources"], &Map.put(&1, "conversions", %{}))) - assert dataset_res == conn |> get(Helpers.dataset_path(conn, :by_id, datagouv_id)) |> json_response(200) + json = conn |> get(Helpers.dataset_path(conn, :by_id, datagouv_id)) |> json_response(200) + assert dataset_res == json + assert_schema(json, "DatasetDetails", TransportWeb.API.Spec.spec()) end test "GET /api/datasets *without* history, multi_validation and resource_metadata", %{conn: conn} do @@ -201,6 +210,8 @@ defmodule TransportWeb.API.DatasetControllerTest do path = Helpers.dataset_path(conn, :datasets) + json = conn |> get(path) |> json_response(200) + assert [ %{ "aom" => %{"name" => "Angers Métropole", "siren" => "siren"}, @@ -235,7 +246,9 @@ defmodule TransportWeb.API.DatasetControllerTest do "type" => "public-transit", "updated" => resource.last_update |> DateTime.to_iso8601() } - ] == conn |> get(path) |> json_response(200) + ] == json + + assert_schema(json, "DatasetsResponse", TransportWeb.API.Spec.spec()) end test "GET /api/datasets/:id *without* history, multi_validation and resource_metadata", %{conn: conn} do @@ -256,16 +269,19 @@ defmodule TransportWeb.API.DatasetControllerTest do datagouv_id: "1", type: "main", format: "GTFS", - filesize: 42 + filesize: 42, + title: "The title" }, %DB.Resource{ last_import: DateTime.utc_now(), last_update: last_update_geojson = DateTime.utc_now() |> DateTime.add(-1, :hour), url: "http://link.to/file.zip?foo=bar", + latest_url: "http://static.data.gouv.fr/?foo=bar", datagouv_id: "2", type: "main", format: "geojson", - schema_name: "etalab/schema-zfe" + schema_name: "etalab/schema-zfe", + title: "The other title" } ], created_at: ~U[2021-12-23 13:30:40.000000Z], @@ -283,6 +299,8 @@ defmodule TransportWeb.API.DatasetControllerTest do path = Helpers.dataset_path(conn, :by_id, dataset.datagouv_id) + json = conn |> get(path) |> json_response(200) + assert %{ "aom" => %{"name" => "Angers Métropole", "siren" => "siren"}, "community_resources" => [], @@ -309,7 +327,8 @@ defmodule TransportWeb.API.DatasetControllerTest do "original_url" => "https://link.to/file.zip", "updated" => last_update_gtfs |> DateTime.to_iso8601(), "url" => "https://static.data.gouv.fr/foo", - "conversions" => %{} + "conversions" => %{}, + "title" => "The title" }, %{ "is_available" => true, @@ -321,7 +340,9 @@ defmodule TransportWeb.API.DatasetControllerTest do "original_url" => "http://link.to/file.zip?foo=bar", "schema_name" => "etalab/schema-zfe", "updated" => last_update_geojson |> DateTime.to_iso8601(), - "conversions" => %{} + "url" => "http://static.data.gouv.fr/?foo=bar", + "conversions" => %{}, + "title" => "The other title" } ], "slug" => "slug-1", @@ -329,7 +350,9 @@ defmodule TransportWeb.API.DatasetControllerTest do "type" => "public-transit", "licence" => "lov2", "updated" => [last_update_gtfs, last_update_geojson] |> Enum.max(DateTime) |> DateTime.to_iso8601() - } == conn |> get(path) |> json_response(200) + } == json + + assert_schema(json, "DatasetDetails", TransportWeb.API.Spec.spec()) end test "GET /api/datasets/:id *with* history, conversions, multi_validation and resource_metadata", %{conn: conn} do @@ -398,6 +421,8 @@ defmodule TransportWeb.API.DatasetControllerTest do path = Helpers.dataset_path(conn, :by_id, dataset.datagouv_id) + json = conn |> get(path) |> json_response(200) + assert %{ "aom" => %{"name" => "Angers Métropole", "siren" => "siren"}, "community_resources" => [], @@ -456,7 +481,9 @@ defmodule TransportWeb.API.DatasetControllerTest do "type" => "public-transit", "updated" => [resource, gbfs_resource] |> Enum.map(& &1.last_update) |> Enum.max(DateTime) |> DateTime.to_iso8601() - } == conn |> get(path) |> json_response(200) + } == json + + assert_schema(json, "DatasetDetails", TransportWeb.API.Spec.spec()) end test "gtfs-rt features are filled", %{conn: conn} do @@ -474,7 +501,11 @@ defmodule TransportWeb.API.DatasetControllerTest do # call to specific dataset path = Helpers.dataset_path(conn, :by_id, datagouv_id_1) - %{"resources" => [%{"features" => features}]} = conn |> get(path) |> json_response(200) + dataset_response = conn |> get(path) |> json_response(200) + %{"resources" => [%{"features" => features}]} = dataset_response + + assert_schema(dataset_response, "DatasetDetails", TransportWeb.API.Spec.spec()) + assert features |> Enum.sort() == ["a", "b", "c"] # add another dataset @@ -485,6 +516,7 @@ defmodule TransportWeb.API.DatasetControllerTest do # call for all datasets path = Helpers.dataset_path(conn, :datasets) datasets = conn |> get(path) |> json_response(200) + assert_schema(datasets, "DatasetsResponse", TransportWeb.API.Spec.spec()) assert ["a", "b", "c"] == datasets diff --git a/apps/transport/test/transport_web/controllers/api/places_controller_test.exs b/apps/transport/test/transport_web/controllers/api/places_controller_test.exs index 73da29265e..de92a9e3d1 100644 --- a/apps/transport/test/transport_web/controllers/api/places_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/api/places_controller_test.exs @@ -3,6 +3,7 @@ defmodule TransportWeb.API.PlacesControllerTest do use TransportWeb.ConnCase, async: false alias TransportWeb.API.Router.Helpers alias DB.{AOM, Commune, Dataset, Region, Repo} + import OpenApiSpex.TestAssertions defp cleanup(value) do # we cannot compare the urls as they can contain internal unstable db id @@ -37,21 +38,23 @@ defmodule TransportWeb.API.PlacesControllerTest do ]) [etag] = conn |> get_resp_header("etag") - json_response(conn, 200) + json = json_response(conn, 200) assert etag assert conn |> get_resp_header("cache-control") == ["max-age=60, public, must-revalidate"] # Passing the previous `ETag` value in a new HTTP request returns a 304 conn |> recycle() |> put_req_header("if-none-match", etag) |> get(path) |> response(304) + + assert_response_schema(json, "AutocompleteResponse", TransportWeb.API.Spec.spec()) end test "Search a place with accent", %{conn: conn} do - r = + json = conn |> get(Helpers.places_path(conn, :autocomplete, q: "cha")) |> json_response(200) - assert sort_and_clean(r) == + assert sort_and_clean(json) == Enum.sort([ %{ "name" => "Châteauroux (36044)", @@ -69,15 +72,17 @@ defmodule TransportWeb.API.PlacesControllerTest do "url" => "/datasets/commune/:id" } ]) + + assert_response_schema(json, "AutocompleteResponse", TransportWeb.API.Spec.spec()) end test "Search a place with multiple word", %{conn: conn} do - r = + json = conn |> get(Helpers.places_path(conn, :autocomplete, q: "ile de fr")) |> json_response(200) - assert sort_and_clean(r) == + assert sort_and_clean(json) == Enum.sort([ %{ "name" => "Île-de-France Mobilités", @@ -90,14 +95,18 @@ defmodule TransportWeb.API.PlacesControllerTest do "url" => "/datasets/region/:id" } ]) + + assert_response_schema(json, "AutocompleteResponse", TransportWeb.API.Spec.spec()) end test "Search a unknown place", %{conn: conn} do - r = + json = conn |> get(Helpers.places_path(conn, :autocomplete, q: "pouet")) |> json_response(200) - assert sort_and_clean(r) == [] + assert sort_and_clean(json) == [] + + assert_response_schema(json, "AutocompleteResponse", TransportWeb.API.Spec.spec()) end end diff --git a/apps/transport/test/transport_web/controllers/api/schemas_test.exs b/apps/transport/test/transport_web/controllers/api/schemas_test.exs new file mode 100644 index 0000000000..9a9445da87 --- /dev/null +++ b/apps/transport/test/transport_web/controllers/api/schemas_test.exs @@ -0,0 +1,16 @@ +defmodule TransportWeb.API.SchemasTest do + use ExUnit.Case, async: true + + test "make sure we get a warning because this helps keeping specs in sync with API output" do + api_spec = TransportWeb.API.Spec.spec() + + api_spec.components.schemas + |> Enum.filter(fn {_name, schema} -> schema.type == :object end) + # composable type + |> Enum.reject(fn {name, _schema} -> name == "GeometryBase" end) + |> Enum.each(fn {name, schema} -> + assert schema.additionalProperties == false, + ~s("#{name}" OpenAPI spec declaration lacks additionalProperties: false") + end) + end +end diff --git a/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs b/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs index 04b4a76a57..a65c290b1d 100644 --- a/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs @@ -3,6 +3,7 @@ defmodule TransportWeb.API.StatsControllerTest do use TransportWeb.ConnCase import Mock import DB.Factory + import OpenApiSpex.TestAssertions @cached_features_routes [ {"/api/stats", "api-stats-aoms"}, @@ -109,6 +110,8 @@ defmodule TransportWeb.API.StatsControllerTest do # the aom status is outdated assert %{"features" => [%{"properties" => %{"quality" => %{"expired_from" => %{"status" => "outdated"}}}}]} = res + + assert_schema(res, "FeatureCollection", TransportWeb.API.Spec.spec()) end describe("aom quality features") do diff --git a/mix.lock b/mix.lock index ebd0c09f8d..3aa7ec848f 100644 --- a/mix.lock +++ b/mix.lock @@ -101,6 +101,7 @@ "rambo": {:hex, :rambo, "0.3.4", "8962ac3bd1a633ee9d0e8b44373c7913e3ce3d875b4151dcd060886092d2dce7", [:mix], [], "hexpm", "0cc54ed089fbbc84b65f4b8a774224ebfe60e5c80186fafc7910b3e379ad58f1"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "recon": {:hex, :recon, "2.5.3", "739107b9050ea683c30e96de050bc59248fd27ec147696f79a8797ff9fa17153", [:mix, :rebar3], [], "hexpm", "6c6683f46fd4a1dfd98404b9f78dcabc7fcd8826613a89dcb984727a8c3099d7"}, + "req": {:hex, :req, "0.3.11", "462315e50db6c6e1f61c45e8c0b267b0d22b6bd1f28444c136908dfdca8d515a", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0e4b331627fedcf90b29aa8064cd5a95619ef6134d5ab13919b6e1c4d7cccd4b"}, "saxy": {:hex, :saxy, "1.5.0", "0141127f2d042856f135fb2d94e0beecda7a2306f47546dbc6411fc5b07e28bf", [:mix], [], "hexpm", "ea7bb6328fbd1f2aceffa3ec6090bfb18c85aadf0f8e5030905e84235861cf89"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, diff --git a/scripts/api/.gitignore b/scripts/api/.gitignore new file mode 100644 index 0000000000..9112f0efd9 --- /dev/null +++ b/scripts/api/.gitignore @@ -0,0 +1 @@ +cache-dir \ No newline at end of file diff --git a/scripts/api/spec_check.exs b/scripts/api/spec_check.exs new file mode 100755 index 0000000000..f660feded9 --- /dev/null +++ b/scripts/api/spec_check.exs @@ -0,0 +1,58 @@ +#! mix run +Code.require_file(__DIR__ <> "/../irve/req_custom_cache.exs") + +ExUnit.start() + +defmodule Query do + def cache_dir, do: Path.join(__ENV__.file, "../cache-dir") |> Path.expand() + + def cached_get!(url) do + req = Req.new() |> CustomCache.attach() + Req.get!(req, url: url, custom_cache_dir: cache_dir()) + end +end + +defmodule TestSuite do + use ExUnit.Case + import OpenApiSpex.TestAssertions + + @host "https://transport.data.gouv.fr" + @index_url Path.join(@host, "/api/datasets") + + def api_spec, do: TransportWeb.API.Spec.spec() + + test "/api/datasets passes our OpenAPI specification" do + url = @index_url + %{status: 200, body: json} = Query.cached_get!(url) + assert_schema(json, "DatasetsResponse", api_spec()) + end + + test "each /api/datasets/:id passes our OpenAPI specification" do + url = @index_url + %{status: 200, body: json} = Query.cached_get!(url) + + task = fn id -> + # IO.puts("Processing #{id}") + url = Path.join(@host, "/api/datasets/#{id}") + %{status: 200, body: json} = Query.cached_get!(url) + json + end + + datasets = + json + |> Enum.map(& &1["id"]) + |> Task.async_stream( + task, + max_concurrency: 25, + on_timeout: :kill_task, + timeout: 15_000 + ) + |> Enum.map(fn {:ok, result} -> result end) + |> Enum.into([]) + + datasets + |> Enum.each(fn d -> + assert_schema(d, "DatasetDetails", api_spec()) + end) + end +end