diff --git a/app/components/map/layers/mobile/mobile-box-layer.tsx b/app/components/map/layers/mobile/mobile-box-layer.tsx index cd7d02b3..a2bcc13e 100644 --- a/app/components/map/layers/mobile/mobile-box-layer.tsx +++ b/app/components/map/layers/mobile/mobile-box-layer.tsx @@ -1,24 +1,16 @@ -import bbox from '@turf/bbox' -import { - featureCollection, - lineString, - multiLineString, - point, -} from '@turf/helpers' -import { type MultiLineString, type Point } from 'geojson' -import { - type CircleLayerSpecification, - type LineLayerSpecification, -} from 'maplibre-gl' +import { featureCollection, point } from '@turf/helpers' +import { type Point } from 'geojson' +import { type CircleLayerSpecification } from 'maplibre-gl' import { createContext, useContext, useEffect, useMemo } from 'react' import { Layer, Popup, Source, useMap } from 'react-map-gl/maplibre' import { HIGH_COLOR, LOW_COLOR, createPalette } from './color-palette' import { type Sensor } from '~/db/schema' +import { validLngLat } from '~/lib/location' interface CustomGeoJsonProperties { locationId: number value: number - createdAt: Date + time: string color: string } @@ -36,27 +28,30 @@ export default function MobileBoxLayer({ maxColor = HIGH_COLOR, }: { sensor: Sensor - minColor?: - | NonNullable['circle-color'] - | NonNullable['line-color'] - maxColor?: - | NonNullable['circle-color'] - | NonNullable['line-color'] + minColor?: NonNullable['circle-color'] + maxColor?: NonNullable['circle-color'] }) { const { hoveredPoint, setHoveredPoint } = useContext(HoveredPointContext) const { osem: mapRef } = useMap() const sourceData = useMemo(() => { const sensorData = (sensor.data ?? []) as unknown as { - value: string - location: { x: number; y: number; id: number } - createdAt: Date + value: number | string | null + location: { x: number; y: number; id: number } | null + time: Date | string | null }[] + const mappableData = sensorData.filter( + (measurement) => + measurement.location !== null && + validLngLat(measurement.location.x, measurement.location.y) && + measurement.value !== null && + Number.isFinite(Number(measurement.value)), + ) - if (sensorData.length === 0) return null + if (mappableData.length === 0) return null - const minValue = Math.min(...sensorData.map((d) => Number(d.value))) - const maxValue = Math.max(...sensorData.map((d) => Number(d.value))) + const minValue = Math.min(...mappableData.map((d) => Number(d.value))) + const maxValue = Math.max(...mappableData.map((d) => Number(d.value))) const palette = createPalette( minValue, maxValue, @@ -64,19 +59,16 @@ export default function MobileBoxLayer({ maxColor as string, ) - const points = sensorData.map((measurement) => - point([measurement.location.x, measurement.location.y], { + const points = mappableData.map((measurement) => + point([measurement.location!.x, measurement.location!.y], { value: Number(measurement.value), - createdAt: new Date(measurement.createdAt), + time: measurement.time ? new Date(measurement.time).toISOString() : '', color: palette(Number(measurement.value)).hex(), - locationId: measurement.location.id, + locationId: measurement.location!.id, }), ) - const line = lineString(points.map((p) => p.geometry.coordinates)) - const lines = multiLineString([line.geometry.coordinates]) - - return featureCollection([...points, lines]) + return featureCollection(points) }, [maxColor, minColor, sensor.data]) const hoveredFeature = useMemo(() => { @@ -92,26 +84,6 @@ export default function MobileBoxLayer({ ) }, [hoveredPoint, sourceData]) - useEffect(() => { - if (!mapRef || !sourceData) return - - const bounds = bbox(sourceData).slice(0, 4) as [ - number, - number, - number, - number, - ] - - mapRef.fitBounds(bounds, { - padding: { - top: 100, - bottom: 400, - left: 500, - right: 100, - }, - }) - }, [mapRef, sourceData]) - useEffect(() => { if (!mapRef) return @@ -152,17 +124,6 @@ export default function MobileBoxLayer({ return ( <> - +
{sensors.map((sensor, index) => ( -
+
{index === 1 && sensors.length === 2 && ( - {showOriginalColors ? ( -

- We have tried to organise your data into trips. This may not - be accurate. -

- ) : ( -

- You are viewing raw data right now. Activate to see trips. -

- )} +

+ {t('tripExplanation', { count: MOBILE_TRIP_LIMIT })} +

+
+ + +
{showOriginalColors && ( -
    - {uniqueColors.map((item, index) => ( -
  • onLegendItemHover(item?.color ?? null)} - onMouseLeave={() => onLegendItemHover(null)} // Reset highlight on mouse leave - > -
    +
      + {items.map((item, index) => ( +
    • +
      • ))}
    diff --git a/app/db/schema/measurement.ts b/app/db/schema/measurement.ts index e75f788b..301222fb 100644 --- a/app/db/schema/measurement.ts +++ b/app/db/schema/measurement.ts @@ -47,7 +47,7 @@ export const measurementRelations = relations(measurement, ({ one }) => ({ */ export const measurements10minView = pgMaterializedView('measurement_10min', { sensorId: text('sensor_id'), - time: timestamp('time', { precision: 3, withTimezone: true }), + time: timestamp('time', { precision: 3, withTimezone: true }).notNull(), value: doublePrecision('avg_value'), total_values: integer('total_values'), min_value: doublePrecision('min_value'), @@ -56,7 +56,7 @@ export const measurements10minView = pgMaterializedView('measurement_10min', { export const measurements1hourView = pgMaterializedView('measurement_1hour', { sensorId: text('sensor_id'), - time: timestamp('time', { precision: 3, withTimezone: true }), + time: timestamp('time', { precision: 3, withTimezone: true }).notNull(), value: doublePrecision('avg_value'), total_values: integer('total_values'), min_value: doublePrecision('min_value'), @@ -65,7 +65,7 @@ export const measurements1hourView = pgMaterializedView('measurement_1hour', { export const measurements1dayView = pgMaterializedView('measurement_1day', { sensorId: text('sensor_id'), - time: timestamp('time', { precision: 3, withTimezone: true }), + time: timestamp('time', { precision: 3, withTimezone: true }).notNull(), value: doublePrecision('avg_value'), total_values: integer('total_values'), min_value: doublePrecision('min_value'), @@ -74,7 +74,7 @@ export const measurements1dayView = pgMaterializedView('measurement_1day', { export const measurements1monthView = pgMaterializedView('measurement_1month', { sensorId: text('sensor_id'), - time: timestamp('time', { precision: 3, withTimezone: true }), + time: timestamp('time', { precision: 3, withTimezone: true }).notNull(), value: doublePrecision('avg_value'), total_values: integer('total_values'), min_value: doublePrecision('min_value'), @@ -83,7 +83,7 @@ export const measurements1monthView = pgMaterializedView('measurement_1month', { export const measurements1yearView = pgMaterializedView('measurement_1year', { sensorId: text('sensor_id'), - time: timestamp('time', { precision: 3, withTimezone: true }), + time: timestamp('time', { precision: 3, withTimezone: true }).notNull(), value: doublePrecision('avg_value'), total_values: integer('total_values'), min_value: doublePrecision('min_value'), diff --git a/app/lib/mobile-box-helper.ts b/app/lib/mobile-box-helper.ts index 3e532021..23c2a540 100644 --- a/app/lib/mobile-box-helper.ts +++ b/app/lib/mobile-box-helper.ts @@ -1,3 +1,6 @@ +export const MOBILE_TRIP_GAP_SECONDS = 60 +export const MOBILE_TRIP_LIMIT = 5 + export interface LocationPoint { geometry: { x: number @@ -6,106 +9,72 @@ export interface LocationPoint { time: string } -interface Trip { +export interface Trip { points: LocationPoint[] startTime: string endTime: string } +/** + * Split location points into chronological trips. A gap greater than the + * threshold starts a new trip. + */ export function categorizeIntoTrips( dataPoints: LocationPoint[], - timeThreshold: number, // in seconds, time threshold for a new trip + timeThreshold = MOBILE_TRIP_GAP_SECONDS, ): Trip[] { - const trips: Trip[] = [] - let currentTrip: LocationPoint[] = [] + if (dataPoints.length === 0) return [] - // Pre-sort data by time to ensure order - dataPoints.sort( + const sortedPoints = [...dataPoints].sort( (a, b) => new Date(a.time).getTime() - new Date(b.time).getTime(), ) + const trips: Trip[] = [] + let currentTrip: LocationPoint[] = [sortedPoints[0]] - for (let i = 1; i < dataPoints.length; i++) { - const previousPoint = dataPoints[i - 1] - const currentPoint = dataPoints[i] - - // Calculate time difference in seconds + for (let i = 1; i < sortedPoints.length; i++) { + const previousPoint = sortedPoints[i - 1] + const currentPoint = sortedPoints[i] const timeDifference = (new Date(currentPoint.time).getTime() - new Date(previousPoint.time).getTime()) / 1000 - // Check if a new trip should start based solely on the time difference - const isNewTrip = timeDifference > timeThreshold - - if (isNewTrip) { - if (currentTrip.length > 0) { - trips.push({ - points: currentTrip, - startTime: currentTrip[0].time, - endTime: currentTrip[currentTrip.length - 1].time, - }) - } - currentTrip = [] + if (timeDifference > timeThreshold) { + trips.push(toTrip(currentTrip)) + currentTrip = [currentPoint] + } else { + currentTrip.push(currentPoint) } - currentTrip.push(currentPoint) } - // Add the final trip - if (currentTrip.length > 0) { - trips.push({ - points: currentTrip, - startTime: currentTrip[0].time, - endTime: currentTrip[currentTrip.length - 1].time, - }) - } - - // Optionally merge small trips (can be removed if not needed) - return mergeSmallTrips(trips, timeThreshold) + trips.push(toTrip(currentTrip)) + return trips } -function mergeSmallTrips(trips: Trip[], timeThreshold: number): Trip[] { - if (trips.length <= 1) return trips - - const mergedTrips: Trip[] = [] - let currentTrip: Trip | null = null - - for (const trip of trips) { - // If a trip is too small (in terms of points or duration), merge it with the current trip - const tripDuration = - (new Date(trip.endTime).getTime() - new Date(trip.startTime).getTime()) / - 1000 - - if (tripDuration >= timeThreshold) { - if (currentTrip) { - mergedTrips.push(currentTrip) - currentTrip = null - } - mergedTrips.push(trip) - } else { - if (!currentTrip) { - currentTrip = { points: [], startTime: '', endTime: '' } - } - currentTrip.points.push(...trip.points) +/** Return the newest trips while preserving chronological display order. */ +export function getLatestTrips( + dataPoints: LocationPoint[], + limit = MOBILE_TRIP_LIMIT, + timeThreshold = MOBILE_TRIP_GAP_SECONDS, +): Trip[] { + if (limit <= 0) return [] + return categorizeIntoTrips(dataPoints, timeThreshold).slice(-limit) +} - // Recompute start and end times - if (currentTrip.points.length > 0) { - currentTrip.startTime = currentTrip.points[0].time - currentTrip.endTime = - currentTrip.points[currentTrip.points.length - 1].time - } - } - } +export function getLatestTripPoints( + dataPoints: LocationPoint[], + limit = MOBILE_TRIP_LIMIT, + timeThreshold = MOBILE_TRIP_GAP_SECONDS, +): LocationPoint[] { + return getLatestTrips(dataPoints, limit, timeThreshold).flatMap( + (trip) => trip.points, + ) +} - // Add any remaining combined trip - if (currentTrip && currentTrip.points.length > 0) { - mergedTrips.push(currentTrip) +function toTrip(points: LocationPoint[]): Trip { + return { + points, + startTime: points[0].time, + endTime: points[points.length - 1].time, } - - // Post-process to sort all trips by time - return mergedTrips.map((trip) => { - trip.points.sort( - (a, b) => new Date(a.time).getTime() - new Date(b.time).getTime(), - ) - return trip - }) } diff --git a/app/routes/explore.$deviceId.$sensorId.$.tsx b/app/routes/explore.$deviceId.$sensorId.$.tsx index 40f6ed03..640de7a4 100644 --- a/app/routes/explore.$deviceId.$sensorId.$.tsx +++ b/app/routes/explore.$deviceId.$sensorId.$.tsx @@ -8,7 +8,7 @@ import { getMeasurement } from '~/db/models/measurement.query.server' import { getSensor } from '~/db/models/sensor.server' import { type SensorWithMeasurementData } from '~/db/schema' import { - categorizeIntoTrips, + getLatestTripPoints, type LocationPoint, } from '~/lib/mobile-box-helper' @@ -16,6 +16,65 @@ interface SensorWithColor extends SensorWithMeasurementData { color: string } +type RawMeasurement = { + sensorId: string + locationId: bigint | null + time: Date | null + value: number | null + location: { + id: bigint + x: number + y: number + } | null +} + +function prepareSensorData( + measurements: RawMeasurement[], + sensorId: string, + limitToLatestTrips: boolean, +): SensorWithMeasurementData['data'] { + const normalizedData = measurements.map((measurement) => ({ + ...measurement, + sensorId, + locationId: + measurement.locationId === null ? null : Number(measurement.locationId), + location: measurement.location + ? { + ...measurement.location, + id: Number(measurement.location.id), + } + : null, + })) + + if (!limitToLatestTrips) return normalizedData + + const locationPoints: LocationPoint[] = normalizedData.flatMap( + (measurement) => { + if (measurement.location == null || measurement.time === null) return [] + + return [ + { + geometry: { + x: measurement.location.x, + y: measurement.location.y, + }, + time: measurement.time.toISOString(), + }, + ] + }, + ) + const latestPointTimes = new Set( + getLatestTripPoints(locationPoints).map((point) => point.time), + ) + + return normalizedData.filter( + (measurement) => + measurement.location != null && + measurement.time !== null && + latestPointTimes.has(measurement.time.toISOString()), + ) +} + export async function loader({ params, request }: Route.LoaderArgs) { const { deviceId, sensorId } = params const sensorId2 = params['*'] @@ -47,77 +106,12 @@ export async function loader({ params, request }: Route.LoaderArgs) { endDate ? addDays(new Date(endDate), 1) : undefined, ) - const normalizedSensor1Data = ( - sensor1Data as { - sensorId: string - locationId: bigint | null - time: Date - value: number | null - location: { - id: bigint - x: number - y: number - } - }[] - ).map((d) => ({ - ...d, - locationId: Number(d.locationId), - location: d.location - ? { - ...d.location, - id: Number(d.location.id), - } - : null, - })) - - // If device exposure is 'mobile', process trips - if (device.exposure === 'mobile' && !startDate) { - // Categorize data into trips - const dataPoints: LocationPoint[] = normalizedSensor1Data - .filter((d) => d.location !== null) - .map((d) => ({ - // null locations cannot be shown on the map and have been filtered above - // hence the ! operator is fine here - geometry: { x: d.location!.x, y: d.location!.y }, - time: d.time.toISOString(), // Ensure the time is in ISO format - })) - - const trips = categorizeIntoTrips(dataPoints, 600) // 600 seconds (10 minutes) as the time threshold - - // Get the latest 5 trips - const latestTrips = trips.slice(0, 1) - - // Calculate the time range of the latest 5 trips - const latestTripTimeRange = { - startTime: latestTrips[0].startTime, - endTime: latestTrips[latestTrips.length - 1].endTime, - } - - // Filter sensor data to include only the points within the time range of the latest 5 trips - const filteredData = normalizedSensor1Data.filter((point) => { - const pointTime = point.time.getTime() - const tripStartTime = new Date(latestTripTimeRange.startTime).getTime() - const tripEndTime = new Date(latestTripTimeRange.endTime).getTime() - - // Keep only the points within the time range of the latest trips - return pointTime >= tripStartTime && pointTime <= tripEndTime - }) - - // Update sensor1 data with the filtered data - sensor1.data = filteredData.map((d) => ({ - ...d, - sensorId: sensorId, // Set the sensorId to match - locationId: d.locationId ?? null, // Retain the locationId if available - location: d.location, - time: d.time, // Keep the timestamp - value: d.value ?? 0, // Set value to the actual value or default it to 0 - })) - - sensor1.color = sensor1.color || '#8da0cb' - } else { - sensor1.data = normalizedSensor1Data - sensor1.color = sensor1.color || '#8da0cb' - } + sensor1.data = prepareSensorData( + sensor1Data as RawMeasurement[], + sensorId, + device.exposure === 'mobile' && !startDate, + ) + sensor1.color = sensor1.color || '#8da0cb' let sensor2: SensorWithColor | null = null @@ -130,75 +124,12 @@ export async function loader({ params, request }: Route.LoaderArgs) { endDate ? addDays(new Date(endDate), 1) : undefined, ) - const normalizedSensor2Data = ( - sensor2Data as { - sensorId: string - locationId: bigint | null - time: Date - value: number | null - location: { - id: bigint - x: number - y: number - } - }[] - ).map((d) => ({ - ...d, - locationId: Number(d.locationId), - location: d.location - ? { - ...d.location, - id: Number(d.location.id), - } - : null, - })) - - if (device.exposure === 'mobile') { - // Categorize data into trips - const dataPoints: LocationPoint[] = normalizedSensor2Data - .filter((d) => d.location !== null) - .map((d) => ({ - // null locations cannot be shown on the map and have been filtered above - // hence the ! operator is fine here - geometry: { x: d.location!.x, y: d.location!.y }, - time: d.time.toISOString(), // Ensure the time is in ISO format - })) - - const trips = categorizeIntoTrips(dataPoints, 600) // 600 seconds (10 minutes) as the time threshold - - // Get the latest trip --- slice to get more trips if needed - const latestTrips = trips.slice(0, 1) - - // Calculate the time range of the latest 5 trips - const latestTripTimeRange = { - startTime: latestTrips[0].startTime, - endTime: latestTrips[latestTrips.length - 1].endTime, - } - - // Filter sensor data to include only the points within the time range of the latest 5 trips - const filteredData = normalizedSensor2Data.filter((point) => { - const pointTime = point.time.getTime() - const tripStartTime = new Date(latestTripTimeRange.startTime).getTime() - const tripEndTime = new Date(latestTripTimeRange.endTime).getTime() - - // Keep only the points within the time range of the latest trips - return pointTime >= tripStartTime && pointTime <= tripEndTime - }) - - // Update sensor2 data with the filtered data - sensor2.data = filteredData.map((d) => ({ - ...d, - sensorId: sensorId2, // Set the sensorId to match - locationId: d.locationId ?? null, // Retain the locationId if available - location: d.location, - time: d.time, // Keep the timestamp - value: d.value ?? 0, // Set value to the actual value or default it to 0 - })) - sensor2.color = sensor2.color || '#fc8d62' - } else { - sensor2.data = normalizedSensor2Data - sensor2.color = sensor2.color || '#fc8d62' - } + sensor2.data = prepareSensorData( + sensor2Data as RawMeasurement[], + sensorId2, + device.exposure === 'mobile' && !startDate, + ) + sensor2.color = sensor2.color || '#fc8d62' } return { diff --git a/app/routes/explore.$deviceId.tsx b/app/routes/explore.$deviceId.tsx index 73016ed3..01b28512 100644 --- a/app/routes/explore.$deviceId.tsx +++ b/app/routes/explore.$deviceId.tsx @@ -1,12 +1,12 @@ import { useState } from 'react' -import { Outlet, useLoaderData, useMatches } from 'react-router' +import { Outlet, useLoaderData } from 'react-router' import { type Route } from './+types/explore.$deviceId' import DeviceDetailBox from '~/components/device-detail/device-detail-box' import { HoveredPointContext } from '~/components/map/layers/mobile/mobile-box-layer' import MobileOverviewLayer from '~/components/map/layers/mobile/mobile-overview-layer' import { getDevice } from '~/db/models/device.server' import { getSensorsWithLastMeasurement } from '~/db/models/sensor.server' -import { categorizeIntoTrips } from '~/lib/mobile-box-helper' +import { getLatestTripPoints } from '~/lib/mobile-box-helper' import { getDeviceImageUrl } from '~/lib/s3.server' import { getLocale } from '~/middleware/i18next' @@ -24,27 +24,19 @@ export async function loader({ context, params, request }: Route.LoaderArgs) { params.deviceId, ) - // get only locations from the last 5 trips + // Keep the payload and map readable by showing the latest mobile trips. if (device?.exposure === 'mobile' && device?.locations) { - // Convert each location's time to ISO string format (explicitly cast time to string) const formattedLocations = device.locations.map((location) => ({ - time: String(location.time), // Force it to be a string + time: String(location.time), geometry: location.geometry, })) + const latestPointTimes = new Set( + getLatestTripPoints(formattedLocations).map((location) => location.time), + ) - // Now you can safely pass the formattedLocations to categorizeIntoTrips - const filteredLocations = categorizeIntoTrips(formattedLocations, 60) // 60 seconds as time threshold - - // get the last time of the 5th trip - const lastTime = - filteredLocations[4]?.points[filteredLocations[4].points.length - 1]?.time - // cut all locations from the device to the last time of the 5th trip - const cutLocations = device.locations.filter((location) => { - const locationTime = String(location.time) // Ensure time is treated as a string - return locationTime <= lastTime - }) - // set the locations to the device - device.locations = cutLocations + device.locations = device.locations.filter((location) => + latestPointTimes.has(String(location.time)), + ) } // Find all sensors from the device response that have the same id as one of the sensor array value @@ -81,10 +73,6 @@ export async function loader({ context, params, request }: Route.LoaderArgs) { export default function DeviceId() { // Retrieving the data returned by the loader using the useLoaderData hook const data = useLoaderData() - const matches = useMatches() - const isSensorView = matches[matches.length - 1].params.sensorId - ? true - : false const [hoveredPoint, setHoveredPoint] = useState(null) const setHoveredPointDebug = (point: any) => { @@ -100,9 +88,8 @@ export default function DeviceId() { - {/* If the box is mobile, iterate over selected sensors and show trajectory */} + {/* Keep the canonical device trips visible while sensors are selected. */} {data.device?.exposure === 'mobile' && - !isSensorView && Array.isArray(data.device?.locations) && data.device.locations.length > 0 && (