diff --git a/app/db.server.ts b/app/db.server.ts index cd399be0..82cd8c35 100644 --- a/app/db.server.ts +++ b/app/db.server.ts @@ -1,4 +1,10 @@ -import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js' +import { + drizzle, + type PostgresJsDatabase, + type PostgresJsQueryResultHKT, +} from 'drizzle-orm/postgres-js' +import { type ExtractTablesWithRelations } from 'drizzle-orm' +import { type PgTransaction } from 'drizzle-orm/pg-core' import postgres, { type Sql } from 'postgres' import invariant from 'tiny-invariant' import * as schema from './db/schema' @@ -57,3 +63,9 @@ function parsePoolSize(value: string | undefined): number { } export { drizzleClient, pg } + +export type DatabaseTransaction = PgTransaction< + PostgresJsQueryResultHKT, + typeof schema, + ExtractTablesWithRelations +> diff --git a/app/db/models/device.server.ts b/app/db/models/device.server.ts index 0001975b..0c6c85dc 100644 --- a/app/db/models/device.server.ts +++ b/app/db/models/device.server.ts @@ -29,7 +29,7 @@ import { type Sensor, } from '~/db/schema' import type * as schema from '~/db/schema/index' -import { drizzleClient } from '~/db.server' +import { drizzleClient, type DatabaseTransaction } from '~/db.server' import BaseNewDeviceEmail, { messages as BaseNewDeviceMessages, } from '~/emails/base-new-device' @@ -155,56 +155,66 @@ export type DeviceForSingleMeasurementWrite = Awaited< ReturnType > -export function getDeviceForMeasurementWrite({ id }: Pick) { - return drizzleClient.query.device.findFirst({ - where: (device, { eq }) => eq(device.id, id), - columns: { - id: true, - archivedAt: true, - useAuth: true, - apiKey: true, - }, - with: { - sensors: { - columns: { - id: true, - title: true, - sensorType: true, - }, - }, - }, - }) +export async function getDeviceForMeasurementWrite( + { id }: Pick, + tx: DatabaseTransaction, +) { + const currentDevice = await lockDeviceForMeasurementWrite(id, tx) + if (!currentDevice) return undefined + + const sensors = await tx + .select({ + id: sensor.id, + title: sensor.title, + sensorType: sensor.sensorType, + }) + .from(sensor) + .where(eq(sensor.deviceId, id)) + .orderBy(sensor.id) + .for('update') + + return { ...currentDevice, sensors } } -export async function getDeviceForSingleMeasurementWrite({ - id, - sensorId, -}: Pick & { sensorId: string }) { - const [row] = await drizzleClient +export async function getDeviceForSingleMeasurementWrite( + { id, sensorId }: Pick & { sensorId: string }, + tx: DatabaseTransaction, +) { + const currentDevice = await lockDeviceForMeasurementWrite(id, tx) + if (!currentDevice) return undefined + + const [currentSensor] = await tx + .select({ + id: sensor.id, + }) + .from(sensor) + .where(and(eq(sensor.deviceId, currentDevice.id), eq(sensor.id, sensorId))) + .limit(1) + .for('update') + + return { + ...currentDevice, + sensors: currentSensor ? [currentSensor] : [], + } +} + +async function lockDeviceForMeasurementWrite( + id: Device['id'], + tx: DatabaseTransaction, +) { + const [currentDevice] = await tx .select({ id: device.id, archivedAt: device.archivedAt, useAuth: device.useAuth, apiKey: device.apiKey, - sensorId: sensor.id, }) .from(device) - .leftJoin( - sensor, - and(eq(sensor.deviceId, device.id), eq(sensor.id, sensorId)), - ) .where(eq(device.id, id)) .limit(1) + .for('share') - if (!row) return undefined - - return { - id: row.id, - archivedAt: row.archivedAt, - useAuth: row.useAuth, - apiKey: row.apiKey, - sensors: row.sensorId ? [{ id: row.sensorId }] : [], - } + return currentDevice } export function getUserDevice({ id, userId }: Pick) { diff --git a/app/db/models/measurement.server.ts b/app/db/models/measurement.server.ts index 4a4862b7..3c89a82a 100644 --- a/app/db/models/measurement.server.ts +++ b/app/db/models/measurement.server.ts @@ -1,5 +1,4 @@ import { and, desc, eq, gt, gte, inArray, lt, lte, sql } from 'drizzle-orm' -import { ArchivedDeviceError } from './device.server' import { type LastMeasurement, location, @@ -9,9 +8,8 @@ import { measurements1hourView, measurements1monthView, measurements1yearView, - device, } from '~/db/schema' -import { drizzleClient } from '~/db.server' +import { drizzleClient, type DatabaseTransaction } from '~/db.server' import { type MinimalDevice, type MeasurementWithLocation, @@ -178,11 +176,11 @@ export function getMeasurement( } export async function saveMeasurements( + tx: DatabaseTransaction, minimalDevice: MinimalDevice, measurements: MeasurementWithLocation[], timing?: MeasurementTiming | null, ): Promise { - if (!device) throw new Error('No device given!') if (!Array.isArray(measurements)) throw new Error('Array expected') const sensorIds = new Set(minimalDevice.sensors.map((s: any) => s.id)) @@ -234,57 +232,35 @@ export async function saveMeasurements( locationUpdateCount: deviceLocationUpdates.length, }) - await drizzleClient.transaction(async (tx) => { - const [currentDevice] = await tx - .select({ - id: device.id, - archivedAt: device.archivedAt, - }) - .from(device) - .where(eq(device.id, minimalDevice.id)) - .limit(1) - timing?.mark('transactionDeviceLookup') - - if (!currentDevice) { - const error = new Error('Device not found') - error.name = 'NotFoundError' - throw error - } - - if (currentDevice.archivedAt) { - throw new ArchivedDeviceError(currentDevice.id) - } - - const locations = - deviceLocationUpdates.length > 0 - ? await findOrCreateLocations(deviceLocationUpdates) - : [] - timing?.mark('findOrCreateLocations', { - locationCount: locations.length, - }) - - if (deviceLocationUpdates.length > 0) { - await addLocationUpdates( - deviceLocationUpdates, - minimalDevice.id, - locations, - ) - } - timing?.mark('addLocationUpdates') + const locations = + deviceLocationUpdates.length > 0 + ? await findOrCreateLocations(deviceLocationUpdates, tx) + : [] + timing?.mark('findOrCreateLocations', { + locationCount: locations.length, + }) - await insertMeasurementsWithLocation( - measurements, - locations, + if (deviceLocationUpdates.length > 0) { + await addLocationUpdates( + deviceLocationUpdates, minimalDevice.id, + locations, tx, - { shouldReturn: false }, - timing, ) - timing?.mark('insertMeasurements') - await updateLastMeasurements(lastMeasurements, tx, timing) - timing?.mark('updateLastMeasurements') - }) - timing?.mark('transaction') + } + timing?.mark('addLocationUpdates') + + await insertMeasurementsWithLocation( + measurements, + locations, + minimalDevice.id, + tx, + { shouldReturn: false }, + timing, + ) + timing?.mark('insertMeasurements') + await updateLastMeasurements(lastMeasurements, tx, timing) + timing?.mark('updateLastMeasurements') } export async function insertMeasurements(measurements: any[]): Promise { diff --git a/app/lib/measurement-server-helper.ts b/app/lib/measurement-server-helper.ts index 8967d033..6b34a712 100644 --- a/app/lib/measurement-server-helper.ts +++ b/app/lib/measurement-server-helper.ts @@ -7,7 +7,7 @@ import { measurement, sensor, } from '~/db/schema' -import { drizzleClient } from '~/db.server' +import { type DatabaseTransaction } from '~/db.server' import { type MeasurementTiming } from '~/lib/measurement-timing.server' export interface MeasurementWithLocation { @@ -66,72 +66,69 @@ export function getLocationUpdates( */ export async function findOrCreateLocations( locationUpdates: DeviceLocationUpdate[], + tx: DatabaseTransaction, ): Promise { const newLocations = locationUpdates.map((update) => update.location) - let foundLocations: LocationWithId[] = [] - - await drizzleClient.transaction(async (tx) => { - const existingLocations = await tx - .select({ id: location.id, location: location.location }) - .from(location) - .where( - or( - ...newLocations.map( - (newLocation) => - sql`ST_EQUALS( + const existingLocations = await tx + .select({ id: location.id, location: location.location }) + .from(location) + .where( + or( + ...newLocations.map( + (newLocation) => + sql`ST_EQUALS( ${location.location}, ST_SetSRID(ST_MakePoint(${newLocation.lng}, ${newLocation.lat}), 4326) )`, - ), ), - ) + ), + ) - foundLocations = existingLocations.map((location) => { - return { - lng: location.location.x, - lat: location.location.y, - height: undefined, - id: location.id, - } - }) + const foundLocations = existingLocations.map((location) => { + return { + lng: location.location.x, + lat: location.location.y, + height: undefined, + id: location.id, + } + }) - const toInsert = newLocations.filter( - (newLocation) => !foundLocationsContain(foundLocations, newLocation), - ) - const uniqueToInsert = toInsert.filter( - (newLocation, index, arr) => - arr.findIndex( - (candidate) => - candidate.lng === newLocation.lng && - candidate.lat === newLocation.lat, - ) === index, - ) + const toInsert = newLocations.filter( + (newLocation) => !foundLocationsContain(foundLocations, newLocation), + ) + const uniqueToInsert = toInsert.filter( + (newLocation, index, arr) => + arr.findIndex( + (candidate) => + candidate.lng === newLocation.lng && + candidate.lat === newLocation.lat, + ) === index, + ) - const inserted = - uniqueToInsert.length > 0 - ? await tx - .insert(location) - .values( - uniqueToInsert.map((newLocation) => { - return { - location: sql`ST_SetSRID(ST_MakePoint(${newLocation.lng}, ${newLocation.lat}), 4326)`, - } - }), - ) - .onConflictDoNothing() - .returning() - : [] + const inserted = + uniqueToInsert.length > 0 + ? await tx + .insert(location) + .values( + uniqueToInsert.map((newLocation) => { + return { + location: sql`ST_SetSRID(ST_MakePoint(${newLocation.lng}, ${newLocation.lat}), 4326)`, + } + }), + ) + .onConflictDoNothing() + .returning() + : [] - inserted.forEach((value) => - foundLocations.push({ - lng: value.location.x, - lat: value.location.y, - height: undefined, - id: value.id, - }), - ) - }) + inserted.forEach((value) => + foundLocations.push({ + lng: value.location.x, + lat: value.location.y, + height: undefined, + id: value.id, + }), + ) return foundLocations } @@ -171,38 +168,34 @@ export async function addLocationUpdates( deviceLocationUpdates: DeviceLocationUpdate[], deviceId: string, locations: LocationWithId[], + tx: DatabaseTransaction, ) { - await drizzleClient.transaction(async (tx) => { - let filteredUpdates = await filterLocationUpdates( - deviceLocationUpdates, - deviceId, - tx, - ) + const filteredUpdates = await filterLocationUpdates( + deviceLocationUpdates, + deviceId, + tx, + ) - filteredUpdates - .filter((update) => !foundLocationsContain(locations, update.location)) - .forEach((update) => { - throw new Error(`Location ID for location ${update.location} not found, + filteredUpdates + .filter((update) => !foundLocationsContain(locations, update.location)) + .forEach((update) => { + throw new Error(`Location ID for location ${update.location} not found, even though it should've been inserted`) - }) + }) - if (filteredUpdates.length > 0) - await tx - .insert(deviceToLocation) - .values( - filteredUpdates.map((update) => { - return { - deviceId: deviceId, - locationId: foundLocationsGet( - locations, - update.location, - ) as bigint, - time: update.time, - } - }), - ) - .onConflictDoNothing() - }) + if (filteredUpdates.length > 0) + await tx + .insert(deviceToLocation) + .values( + filteredUpdates.map((update) => { + return { + deviceId: deviceId, + locationId: foundLocationsGet(locations, update.location) as bigint, + time: update.time, + } + }), + ) + .onConflictDoNothing() } /** @@ -212,7 +205,7 @@ export async function addLocationUpdates( export async function filterLocationUpdates( deviceLocationUpdates: DeviceLocationUpdate[], deviceId: string, - tx: any, + tx: DatabaseTransaction, ): Promise { const currentLatestLocation = await tx .select({ time: deviceToLocation.time }) @@ -240,7 +233,7 @@ export async function insertMeasurementsWithLocation( measurements: MeasurementWithLocation[], locations: LocationWithId[], deviceId: string, - tx: any, + tx: DatabaseTransaction, options: { shouldReturn?: boolean } = {}, timing?: MeasurementTiming | null, ): Promise { @@ -288,7 +281,7 @@ export async function insertMeasurementsWithLocation( */ export async function updateLastMeasurements( lastMeasurements: Record>, - tx: any, + tx: DatabaseTransaction, timing?: MeasurementTiming | null, ) { const sqlChunks: SQL[] = [ diff --git a/app/services/measurement-service.server.ts b/app/services/measurement-service.server.ts index 44dcb850..3d38dcd0 100644 --- a/app/services/measurement-service.server.ts +++ b/app/services/measurement-service.server.ts @@ -13,6 +13,7 @@ import { getSensorWithLastMeasurement, } from '~/db/models/sensor.server' import { type SensorWithLatestMeasurement } from '~/db/schema' +import { drizzleClient } from '~/db.server' import { decodeMeasurements, hasDecoder, @@ -137,36 +138,38 @@ export const postNewMeasurements = async ( throw new Error('UnsupportedMediaTypeError: Unsupported content-type.') } - const device = await getDeviceForMeasurementWrite({ id: deviceId }) - if (!device) { - throw new Error('NotFoundError: Device not found') - } + await drizzleClient.transaction(async (tx) => { + const device = await getDeviceForMeasurementWrite({ id: deviceId }, tx) + if (!device) { + throw new Error('NotFoundError: Device not found') + } - assertDeviceIsWritable(device) + assertDeviceIsWritable(device) - if (device.useAuth && !isTrustedService) { - if (device.apiKey !== authorization) { - const error = new Error('Device access token not valid!') - error.name = 'UnauthorizedError' - throw error + if (device.useAuth && !isTrustedService) { + if (device.apiKey !== authorization) { + const error = new Error('Device access token not valid!') + error.name = 'UnauthorizedError' + throw error + } } - } - const measurements = await decodeMeasurements(body, { - contentType, - sensors: device.sensors, - }) + const measurements = await decodeMeasurements(body, { + contentType, + sensors: device.sensors, + }) - for (const m of measurements) { - const locationData: LocationData | null = m.location ?? null - if (locationData && !validLngLat(locationData.lng, locationData.lat)) { - const error = new Error('Invalid location coordinates') - error.name = 'UnprocessableEntityError' - throw error + for (const m of measurements) { + const locationData: LocationData | null = m.location ?? null + if (locationData && !validLngLat(locationData.lng, locationData.lat)) { + const error = new Error('Invalid location coordinates') + error.name = 'UnprocessableEntityError' + throw error + } } - } - await saveMeasurements(device, measurements) + await saveMeasurements(tx, device, measurements) + }) } export const postSingleMeasurement = async ( @@ -185,42 +188,6 @@ export const postSingleMeasurement = async ( } timing?.mark('validateBody') - const device = await getDeviceForSingleMeasurementWrite({ - id: deviceId, - sensorId, - }) - timing?.mark('deviceLookup', { - deviceFound: Boolean(device), - sensorCount: device?.sensors.length ?? 0, - }) - - if (!device) { - const error = new Error('Device not found') - error.name = 'NotFoundError' - throw error - } - - assertDeviceIsWritable(device) - - if (device.sensors.length === 0) { - const error = new Error('Sensor not found on device') - error.name = 'NotFoundError' - throw error - } - timing?.mark('validateDeviceAndSensor') - - if (device.useAuth && !isTrustedService) { - if (device.apiKey !== authorization) { - const error = new Error('Device access token not valid!') - error.name = 'UnauthorizedError' - throw error - } - } - timing?.mark('authorizeDevice', { - deviceUsesAuth: Boolean(device.useAuth), - isTrustedService: Boolean(isTrustedService), - }) - let timestamp: Date | undefined if (body.createdAt) { timestamp = new Date(body.createdAt) @@ -257,7 +224,47 @@ export const postSingleMeasurement = async ( ] timing?.mark('buildMeasurements') - await saveMeasurements(device, measurements, timing) + await drizzleClient.transaction(async (tx) => { + timing?.mark('transactionAcquire') + const device = await getDeviceForSingleMeasurementWrite( + { id: deviceId, sensorId }, + tx, + ) + timing?.mark('deviceLookup', { + deviceFound: Boolean(device), + sensorCount: device?.sensors.length ?? 0, + }) + + if (!device) { + const error = new Error('Device not found') + error.name = 'NotFoundError' + throw error + } + + assertDeviceIsWritable(device) + + if (device.sensors.length === 0) { + const error = new Error('Sensor not found on device') + error.name = 'NotFoundError' + throw error + } + timing?.mark('validateDeviceAndSensor') + + if (device.useAuth && !isTrustedService) { + if (device.apiKey !== authorization) { + const error = new Error('Device access token not valid!') + error.name = 'UnauthorizedError' + throw error + } + } + timing?.mark('authorizeDevice', { + deviceUsesAuth: Boolean(device.useAuth), + isTrustedService: Boolean(isTrustedService), + }) + + await saveMeasurements(tx, device, measurements, timing) + }) + timing?.mark('transaction') timing?.mark('saveMeasurements') } catch (error) { if (