From c90a702581e3b97a48df91f595783320f192db39 Mon Sep 17 00:00:00 2001 From: Javi Date: Thu, 17 Apr 2025 16:47:15 +0200 Subject: [PATCH 1/6] new endpoint observations/places_counts --- lib/controllers/v1/observations_controller.js | 65 +++++++++++++++++++ lib/inaturalist_api.js | 3 + lib/views/swagger_v1.yml.ejs | 38 +++++++++++ schema/fixtures.js | 1 + test/integration/v1/observations.js | 39 +++++++++++ 5 files changed, 146 insertions(+) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index 77069926..a73f9e2e 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -363,6 +363,11 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { } else if ( req.inat && req.inat.impliedBounds ) { response.total_bounds = req.inat.impliedBounds; } + if (data.aggregations?.by_place) { + response.aggregations = { + by_place: data.aggregations.by_place + }; + } response.page = Number( req.elastic_query.page ); response.per_page = Number( req.elastic_query.per_page ); response.results = obs; @@ -406,11 +411,71 @@ ObservationsController.speciesCountsCacheWrapper = async req => ( "ObservationsController.speciesCounts" ) ); +ObservationsController.placesCountsCacheWrapper = async req => ( + ObservationsController.methodCacheWrapper( req, + ObservationsController.placesCounts, + "ObservationsController.placesCounts" ) +); + ObservationsController.speciesCounts = async req => { const leafCounts = await ObservationsController.leafCounts( req ); return TaxaController.speciesCountsResponse( req, leafCounts ); }; +ObservationsController.placesCounts = async (req, options = {}) => { + + const page = parseInt(req.query.page || 1, 10); + const perPage = parseInt(req.query.per_page || 30, 10); + const offset = (page - 1) * perPage; + const order = req.query.order === "asc" ? "asc" : "desc"; + + req.query.aggs = { + by_place: { + terms: { + field: "place_ids", + size: 1000, + order: { "_count": order } + } + } + }; + req.query.per_page = 0; + + const data = await ObservationsController.resultsForRequest(req, options); + const buckets = data.aggregations?.by_place?.buckets || []; + + const places = await Promise.all( + buckets.map(bucket => + Place.findByID(bucket.key, { + fields: ["id", "name", "display_name"] + }) + ) + ); + + const results = places + .map((place, i) => { + if (!place) return null; + return { + count: buckets[i].doc_count, + place: { + id: place.id, + name: place.name, + display_name: place.display_name + } + }; + }) + .filter(Boolean); + + const paginated = results.slice(offset, offset + perPage); + + return { + total_results: results.length, + page, + per_page: perPage, + results: paginated + }; +}; + + ObservationsController.taxa = async req => { if ( !req.query.user_id ) { throw new Error( 422 ); diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js index b51fd2ad..24dfa6f8 100644 --- a/lib/inaturalist_api.js +++ b/lib/inaturalist_api.js @@ -298,6 +298,9 @@ InaturalistAPI.server = async ( ) => { dfault( "get", "/v1/observations/species_counts", ObservationsController.speciesCountsCacheWrapper, { setTTL: true } ); + dfault( "get", "/v1/observations/places_counts", ObservationsController.placesCountsCacheWrapper, { + setTTL: true + } ); dfault( "get", "/v1/observations/taxa_counts_by_month", ObservationsController.taxaCountsByMonth, { setTTL: true } ); diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index e233caac..6a5f0b92 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1046,6 +1046,29 @@ paths: description: Unexpected error schema: $ref: "#/definitions/Error" + /observations/places_counts: + get: + summary: Observation Place Counts + description: | + Given zero to many of the following parameters, returns the number of observations matching + the search criteria, grouped by place. Each result includes the place and the count of + associated observations. This endpoint works similarly to `/observations`, but instead of + returning individual observations, it returns aggregated results per place. A + maximum of 1000 results will be returned + parameters: + <%- include( "_observation_search_params_v1.yml.ejs", { type: "index" } ) %> + tags: + - Observations + responses: + 200: + description: | + Returns an object with metadata and an array of taxa + schema: + $ref: "#/definitions/PlacesCountsResponse" + default: + description: Unexpected error + schema: + $ref: "#/definitions/Error" /observations/popular_field_values: get: summary: Observation Popular Field Values @@ -4699,6 +4722,21 @@ definitions: type: integer taxon: $ref: "#/definitions/ShowTaxon" + PlacesCountsResponse: + allOf: + - $ref: "#/definitions/BaseResponse" + - required: + - results + properties: + results: + type: array + items: + type: object + properties: + count: + type: integer + place: + $ref: "#/definitions/CorePlace" PlacesResponse: allOf: - $ref: "#/definitions/BaseResponse" diff --git a/schema/fixtures.js b/schema/fixtures.js index d75370e9..41342b37 100644 --- a/schema/fixtures.js +++ b/schema/fixtures.js @@ -99,6 +99,7 @@ "name": "United States", "slug": "united-states", "display_name_autocomplete": "United States", + "display_name": "United States", "location": "48.8907012939,-116.9820022583", "admin_level": 0, "bbox_area": 5500, diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 7e03ad8a..715a63cc 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1409,6 +1409,45 @@ describe( "Observations", ( ) => { } ); } ); + describe.only( "place_counts", ( ) => { + it( "returns JSON", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ) + .expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + + it( "sorts by count desc by default", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + expect( res.body.results.length ).to.be.greaterThan( 1 ); + expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); + } ).expect( 200, done ); + } ); + + it( "can sort by count asc", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=asc&order_by=created_at" ).expect( res => { + expect( res.body.results.length ).to.be.greaterThan( 1 ); + expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); + } ).expect( 200, done ); + } ); + + it( "supports pagination", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { + expect( res.body.page ).to.eq( 2 ); + expect( res.body.per_page ).to.eq( 1 ); + } ).expect( 200, done ); + } ); + + it( "returns results places counts with expected fields", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + const result = res.body.results[0]; + expect( result ).to.have.property( "count" ); + expect( result.place ).to.have.property( "id" ); + expect( result.place ).to.have.property( "name" ); + expect( result.place ).to.have.property( "display_name" ); + } ).expect( 200, done ); + } ); + } ); + describe( "iconic_taxa_counts", ( ) => { it( "returns json", function ( done ) { request( this.app ).get( "/v1/observations/iconic_taxa_counts" ) From bd4409ca960acaa5e619b450cb44401eebefba2e Mon Sep 17 00:00:00 2001 From: Javi Date: Thu, 17 Apr 2025 17:10:13 +0200 Subject: [PATCH 2/6] fix eslint --- lib/controllers/v1/observations_controller.js | 46 +++++++++---------- test/integration/v1/observations.js | 8 ++-- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index a73f9e2e..b4f6b671 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -363,7 +363,7 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { } else if ( req.inat && req.inat.impliedBounds ) { response.total_bounds = req.inat.impliedBounds; } - if (data.aggregations?.by_place) { + if ( data.aggregations?.by_place ) { response.aggregations = { by_place: data.aggregations.by_place }; @@ -422,11 +422,10 @@ ObservationsController.speciesCounts = async req => { return TaxaController.speciesCountsResponse( req, leafCounts ); }; -ObservationsController.placesCounts = async (req, options = {}) => { - - const page = parseInt(req.query.page || 1, 10); - const perPage = parseInt(req.query.per_page || 30, 10); - const offset = (page - 1) * perPage; +ObservationsController.placesCounts = async ( req, options = {} ) => { + const page = parseInt( req.query.page || 1, 10 ); + const perPage = parseInt( req.query.per_page || 30, 10 ); + const offset = ( page - 1 ) * perPage; const order = req.query.order === "asc" ? "asc" : "desc"; req.query.aggs = { @@ -434,26 +433,24 @@ ObservationsController.placesCounts = async (req, options = {}) => { terms: { field: "place_ids", size: 1000, - order: { "_count": order } + order: { _count: order } } } }; req.query.per_page = 0; - const data = await ObservationsController.resultsForRequest(req, options); + const data = await ObservationsController.resultsForRequest( req, options ); const buckets = data.aggregations?.by_place?.buckets || []; const places = await Promise.all( - buckets.map(bucket => - Place.findByID(bucket.key, { - fields: ["id", "name", "display_name"] - }) - ) + buckets.map( bucket => Place.findByID( bucket.key, { + fields: ["id", "name", "display_name"] + } ) ) ); const results = places - .map((place, i) => { - if (!place) return null; + .map( ( place, i ) => { + if ( !place ) return null; return { count: buckets[i].doc_count, place: { @@ -462,20 +459,19 @@ ObservationsController.placesCounts = async (req, options = {}) => { display_name: place.display_name } }; - }) - .filter(Boolean); + } ) + .filter( Boolean ); - const paginated = results.slice(offset, offset + perPage); + const paginated = results.slice( offset, offset + perPage ); - return { - total_results: results.length, - page, - per_page: perPage, - results: paginated - }; + return { + total_results: results.length, + page, + per_page: perPage, + results: paginated + }; }; - ObservationsController.taxa = async req => { if ( !req.query.user_id ) { throw new Error( 422 ); diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 715a63cc..367bca83 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1415,28 +1415,28 @@ describe( "Observations", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); - + it( "sorts by count desc by default", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); } ).expect( 200, done ); } ); - + it( "can sort by count asc", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=asc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); } ).expect( 200, done ); } ); - + it( "supports pagination", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { expect( res.body.page ).to.eq( 2 ); expect( res.body.per_page ).to.eq( 1 ); } ).expect( 200, done ); } ); - + it( "returns results places counts with expected fields", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { const result = res.body.results[0]; From f25573581e2bfaf83e05858612864c5811150aa9 Mon Sep 17 00:00:00 2001 From: Javi Date: Thu, 17 Apr 2025 17:10:33 +0200 Subject: [PATCH 3/6] fix test --- test/integration/v1/observations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 367bca83..b5a31ae2 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1409,7 +1409,7 @@ describe( "Observations", ( ) => { } ); } ); - describe.only( "place_counts", ( ) => { + describe( "place_counts", ( ) => { it( "returns JSON", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ) .expect( "Content-Type", /json/ ) From 96c4663e77949b9a2c7a02d90434a12d40ffd5ae Mon Sep 17 00:00:00 2001 From: Javi Date: Sun, 11 May 2025 21:51:51 +0200 Subject: [PATCH 4/6] change places to place --- lib/inaturalist_api.js | 2 +- lib/views/swagger_v1.yml.ejs | 2 +- test/integration/v1/observations.js | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js index 24dfa6f8..127880f0 100644 --- a/lib/inaturalist_api.js +++ b/lib/inaturalist_api.js @@ -298,7 +298,7 @@ InaturalistAPI.server = async ( ) => { dfault( "get", "/v1/observations/species_counts", ObservationsController.speciesCountsCacheWrapper, { setTTL: true } ); - dfault( "get", "/v1/observations/places_counts", ObservationsController.placesCountsCacheWrapper, { + dfault( "get", "/v1/observations/place_counts", ObservationsController.placesCountsCacheWrapper, { setTTL: true } ); dfault( "get", "/v1/observations/taxa_counts_by_month", ObservationsController.taxaCountsByMonth, { diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index 6a5f0b92..60f84556 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1046,7 +1046,7 @@ paths: description: Unexpected error schema: $ref: "#/definitions/Error" - /observations/places_counts: + /observations/place_counts: get: summary: Observation Place Counts description: | diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index b5a31ae2..9bf83e86 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1411,34 +1411,34 @@ describe( "Observations", ( ) => { describe( "place_counts", ( ) => { it( "returns JSON", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ) + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ) .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); it( "sorts by count desc by default", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); } ).expect( 200, done ); } ); it( "can sort by count asc", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=asc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=asc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); } ).expect( 200, done ); } ); it( "supports pagination", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { expect( res.body.page ).to.eq( 2 ); expect( res.body.per_page ).to.eq( 1 ); } ).expect( 200, done ); } ); it( "returns results places counts with expected fields", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ).expect( res => { const result = res.body.results[0]; expect( result ).to.have.property( "count" ); expect( result.place ).to.have.property( "id" ); From 4752b2c20b57700482fede31c41ec888fcae4872 Mon Sep 17 00:00:00 2001 From: Kevin Chen Date: Wed, 15 Oct 2025 00:36:37 -0400 Subject: [PATCH 5/6] Implements query param to specify place id buckets for counts. Includes tests --- lib/controllers/v1/observations_controller.js | 39 ++++++++++------ lib/inaturalist_api.js | 2 +- lib/views/swagger_v1.yml.ejs | 20 +++++++-- schema/fixtures.js | 44 +++++++++++++++++++ test/integration/v1/observations.js | 24 +++++++++- 5 files changed, 109 insertions(+), 20 deletions(-) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index b4f6b671..b8c953d4 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -363,9 +363,13 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { } else if ( req.inat && req.inat.impliedBounds ) { response.total_bounds = req.inat.impliedBounds; } - if ( data.aggregations?.by_place ) { + if ( data.aggregations?.count_by_place ) { response.aggregations = { - by_place: data.aggregations.by_place + count_by_place: data.aggregations.count_by_place + }; + } else if ( data.aggregations?.filtered_count_by_place?.aggregations?.count_by_place ) { + response.aggregations = { + count_by_place: data.aggregations.filtered_count_by_place.aggregations.count_by_place }; } response.page = Number( req.elastic_query.page ); @@ -411,36 +415,43 @@ ObservationsController.speciesCountsCacheWrapper = async req => ( "ObservationsController.speciesCounts" ) ); -ObservationsController.placesCountsCacheWrapper = async req => ( - ObservationsController.methodCacheWrapper( req, - ObservationsController.placesCounts, - "ObservationsController.placesCounts" ) -); - ObservationsController.speciesCounts = async req => { const leafCounts = await ObservationsController.leafCounts( req ); return TaxaController.speciesCountsResponse( req, leafCounts ); }; -ObservationsController.placesCounts = async ( req, options = {} ) => { +ObservationsController.placeCountsCacheWrapper = async req => ( + ObservationsController.methodCacheWrapper( req, + ObservationsController.placeCounts, + "ObservationsController.placeCounts" ) +); + +ObservationsController.placeCounts = async ( req, options = {} ) => { const page = parseInt( req.query.page || 1, 10 ); const perPage = parseInt( req.query.per_page || 30, 10 ); const offset = ( page - 1 ) * perPage; const order = req.query.order === "asc" ? "asc" : "desc"; - - req.query.aggs = { - by_place: { + const idsToFilter = util.paramArray( req.query.count_place_id ); + const shortenedIdsToFilter = _.isArray( idsToFilter ) + ? req.query.count_place_id.slice( 0, 1000 ) + : []; + const countByPlaceTermsAgg = { + count_by_place: { terms: { - field: "place_ids", + field: "place_ids.keyword", size: 1000, order: { _count: order } } } }; + if ( !_.isEmpty( shortenedIdsToFilter ) ) { + countByPlaceTermsAgg.count_by_place.terms.include = idsToFilter; + } + req.query.aggs = countByPlaceTermsAgg; req.query.per_page = 0; const data = await ObservationsController.resultsForRequest( req, options ); - const buckets = data.aggregations?.by_place?.buckets || []; + const buckets = data.aggregations?.count_by_place?.buckets || []; const places = await Promise.all( buckets.map( bucket => Place.findByID( bucket.key, { diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js index 127880f0..c97148e4 100644 --- a/lib/inaturalist_api.js +++ b/lib/inaturalist_api.js @@ -298,7 +298,7 @@ InaturalistAPI.server = async ( ) => { dfault( "get", "/v1/observations/species_counts", ObservationsController.speciesCountsCacheWrapper, { setTTL: true } ); - dfault( "get", "/v1/observations/place_counts", ObservationsController.placesCountsCacheWrapper, { + dfault( "get", "/v1/observations/place_counts", ObservationsController.placeCountsCacheWrapper, { setTTL: true } ); dfault( "get", "/v1/observations/taxa_counts_by_month", ObservationsController.taxaCountsByMonth, { diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index 60f84556..7893cc2b 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1051,12 +1051,14 @@ paths: summary: Observation Place Counts description: | Given zero to many of the following parameters, returns the number of observations matching - the search criteria, grouped by place. Each result includes the place and the count of - associated observations. This endpoint works similarly to `/observations`, but instead of + the search criteria grouped by place. Each result includes the place and the count of + associated observations. This endpoint works similarly to `/observations`, but instead of returning individual observations, it returns aggregated results per place. A - maximum of 1000 results will be returned + maximum of 1000 place-count entries will be returned. Use the parameter + `count_place_id` to specify the places for which to return counts. parameters: - <%- include( "_observation_search_params_v1.yml.ejs", { type: "index" } ) %> + <%- include( "_observation_search_params_v1.yml.ejs", { type: "index" } ) %> + - $ref: "#/parameters/count_place_id" tags: - Observations responses: @@ -2191,6 +2193,16 @@ parameters: type: boolean in: query description: Captive or cultivated observations + count_place_id: + type: array + items: + type: integer + in: query + description: | + Places where from aggregate observation counts are returned. + If more than 1,000 id's are specified, the first 1,000 are used. + If `count_place_id` is not specified, results will contain any places included in observations returned from your query; + note that therefore it may include other places you have not included in the `place_id` parameter if specified endemic: name: endemic type: boolean diff --git a/schema/fixtures.js b/schema/fixtures.js index 41342b37..0bfe3787 100644 --- a/schema/fixtures.js +++ b/schema/fixtures.js @@ -253,6 +253,20 @@ ] ] } + }, + { + "id": 2025101508, + "name": "Testing Place Counts Island" + }, + { + "id": 2025101509, + "name": "Forest In Testing Place Counts Island", + "ancestor_place_ids": [2025101508, 2025101509] + }, + { + "id": 2025101510, + "name": "Lake In Testing Place Counts Island", + "ancestor_place_ids": [2025101508, 2025101510] } ] }, @@ -1223,6 +1237,24 @@ } ], "sounds_count": 1 + }, + { + "id": 2025101501, + "uuid": "testing-obsv-places-count1", + "place_ids": [2025101508, 2025101509], + "private_place_ids": [2025101508, 2025101509] + }, + { + "id": 2025101502, + "uuid": "testing-obsv-places-count2", + "place_ids": [2025101508, 2025101510], + "private_place_ids": [2025101508, 2025101510] + }, + { + "id": 2025101503, + "uuid": "testing-obsv-places-count3", + "place_ids": [2025101508, 2025101510], + "private_place_ids": [2025101508, 2025101510] } ] }, @@ -2764,6 +2796,18 @@ { "id": 2025012202, "uuid": "537d9290-df3c-404f-a26d-ffe07156908c" + }, + { + "id": 2025101501, + "uuid": "testing-obsv-places-count1" + }, + { + "id": 2025101502, + "uuid": "testing-obsv-places-count2" + }, + { + "id": 2025101503, + "uuid": "testing-obsv-places-count3" } ], "observation_photos": [ diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 9bf83e86..ac62beb3 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1438,7 +1438,7 @@ describe( "Observations", ( ) => { } ); it( "returns results places counts with expected fields", function ( done ) { - request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts" ).expect( res => { const result = res.body.results[0]; expect( result ).to.have.property( "count" ); expect( result.place ).to.have.property( "id" ); @@ -1446,6 +1446,28 @@ describe( "Observations", ( ) => { expect( result.place ).to.have.property( "display_name" ); } ).expect( 200, done ); } ); + + it( "returns counts from all places filtered in observations when place_id is specified", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?place_id=2025101508" ).expect( res => { + expect( res.body.results.length ).to.be.eq( 3 ); + expect( res.body.results[0].place.id ).to.be.eq( 2025101508 ); + expect( res.body.results[0].count ).to.be.eq( 3 ); + expect( res.body.results[1].place.id ).to.be.eq( 2025101510 ); + expect( res.body.results[1].count ).to.be.eq( 2 ); + expect( res.body.results[2].place.id ).to.be.eq( 2025101509 ); + expect( res.body.results[2].count ).to.be.eq( 1 ); + } ).expect( 200, done ); + } ); + + it( "returns counts only from count_place_id when both it and place_id are specified", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?place_id=2025101508&count_place_id=2025101510%2C2025101509" ).expect( res => { + expect( res.body.results.length ).to.be.eq( 2 ); + expect( res.body.results[0].place.id ).to.be.eq( 2025101510 ); + expect( res.body.results[0].count ).to.be.eq( 2 ); + expect( res.body.results[1].place.id ).to.be.eq( 2025101509 ); + expect( res.body.results[1].count ).to.be.eq( 1 ); + } ).expect( 200, done ); + } ); } ); describe( "iconic_taxa_counts", ( ) => { From 51b5b46fb2b067f247bbf2d70a40e02a26d41b2c Mon Sep 17 00:00:00 2001 From: Kevin Chen Date: Wed, 15 Oct 2025 02:35:43 -0400 Subject: [PATCH 6/6] Ensures order_by makes sense for obsv/place_counts endpoint --- lib/controllers/v1/observations_controller.js | 10 ++++------ lib/views/swagger_v1.yml.ejs | 18 ++++++++++++++++-- test/integration/v1/observations.js | 16 ++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index b8c953d4..57c64ca4 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -367,10 +367,6 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { response.aggregations = { count_by_place: data.aggregations.count_by_place }; - } else if ( data.aggregations?.filtered_count_by_place?.aggregations?.count_by_place ) { - response.aggregations = { - count_by_place: data.aggregations.filtered_count_by_place.aggregations.count_by_place - }; } response.page = Number( req.elastic_query.page ); response.per_page = Number( req.elastic_query.per_page ); @@ -430,7 +426,8 @@ ObservationsController.placeCounts = async ( req, options = {} ) => { const page = parseInt( req.query.page || 1, 10 ); const perPage = parseInt( req.query.per_page || 30, 10 ); const offset = ( page - 1 ) * perPage; - const order = req.query.order === "asc" ? "asc" : "desc"; + const orderAscDesc = req.query.order === "asc" ? "asc" : "desc"; + const order = req.query.order_by === "id" ? { _key: orderAscDesc } : { _count: orderAscDesc }; const idsToFilter = util.paramArray( req.query.count_place_id ); const shortenedIdsToFilter = _.isArray( idsToFilter ) ? req.query.count_place_id.slice( 0, 1000 ) @@ -440,7 +437,8 @@ ObservationsController.placeCounts = async ( req, options = {} ) => { terms: { field: "place_ids.keyword", size: 1000, - order: { _count: order } + // according to Elastic docs, _count "asc" can return inaccurate results due to sharding + order } } }; diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index 7893cc2b..6c594882 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1054,11 +1054,16 @@ paths: the search criteria grouped by place. Each result includes the place and the count of associated observations. This endpoint works similarly to `/observations`, but instead of returning individual observations, it returns aggregated results per place. A - maximum of 1000 place-count entries will be returned. Use the parameter + maximum of 1000 place-count entries can be aggregated for a given query. Use the parameter `count_place_id` to specify the places for which to return counts. parameters: - <%- include( "_observation_search_params_v1.yml.ejs", { type: "index" } ) %> + <%- include( "_observation_search_params_v1.yml.ejs" ) %> - $ref: "#/parameters/count_place_id" + - $ref: "#/parameters/page" + - $ref: "#/parameters/per_page" + - $ref: "#/parameters/order" + - $ref: "#/parameters/place_counts_order_by" + - $ref: "#/parameters/only_id" tags: - Observations responses: @@ -2861,6 +2866,15 @@ parameters: in: query description: | Number of results to return in a `page`. The maximum value is 500 + place_counts_order_by: + name: order_by + type: string + in: query + description: Sort field. We strongly discourage count when combined with order = 'asc' as results may be inaccurate. + default: count + enum: + - count + - id q: name: q type: string diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index ac62beb3..e353e4f8 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1417,21 +1417,21 @@ describe( "Observations", ( ) => { } ); it( "sorts by count desc by default", function ( done ) { - request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); } ).expect( 200, done ); } ); it( "can sort by count asc", function ( done ) { - request( this.app ).get( "/v1/observations/place_counts?order=asc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=asc&order_by=count" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); } ).expect( 200, done ); } ); it( "supports pagination", function ( done ) { - request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=count&per_page=1&page=2" ).expect( res => { expect( res.body.page ).to.eq( 2 ); expect( res.body.per_page ).to.eq( 1 ); } ).expect( 200, done ); @@ -1460,12 +1460,12 @@ describe( "Observations", ( ) => { } ); it( "returns counts only from count_place_id when both it and place_id are specified", function ( done ) { - request( this.app ).get( "/v1/observations/place_counts?place_id=2025101508&count_place_id=2025101510%2C2025101509" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?place_id=2025101508&count_place_id=2025101510%2C2025101509&order=asc&order_by=id" ).expect( res => { expect( res.body.results.length ).to.be.eq( 2 ); - expect( res.body.results[0].place.id ).to.be.eq( 2025101510 ); - expect( res.body.results[0].count ).to.be.eq( 2 ); - expect( res.body.results[1].place.id ).to.be.eq( 2025101509 ); - expect( res.body.results[1].count ).to.be.eq( 1 ); + expect( res.body.results[0].place.id ).to.be.eq( 2025101509 ); + expect( res.body.results[0].count ).to.be.eq( 1 ); + expect( res.body.results[1].place.id ).to.be.eq( 2025101510 ); + expect( res.body.results[1].count ).to.be.eq( 2 ); } ).expect( 200, done ); } ); } );