diff --git a/src/app/(admin)/actions.ts b/src/app/(admin)/actions.ts index 3c8d0b6..df01909 100644 --- a/src/app/(admin)/actions.ts +++ b/src/app/(admin)/actions.ts @@ -55,3 +55,24 @@ export async function adminResetMonthlyUsage(data: unknown) { }); revalidatePath("/admin/users"); } + +export async function adminTriggerScrape(): Promise<{ + ok: boolean; + message: string; +}> { + await requireAdmin(); + + const url = new URL("/api/scrape", env.NEXT_PUBLIC_APP_URL); + const res = await fetch(url.toString(), { + method: "POST", + headers: { Authorization: `Bearer ${env.CRON_SECRET}` }, + }); + + if (!res.ok) { + return { ok: false, message: `Scrape failed: ${res.status} ${res.statusText}` }; + } + + const body = (await res.json()) as { poolNew?: number }; + revalidatePath("/admin/system"); + return { ok: true, message: `Scrape complete — ${body.poolNew ?? 0} new to pool` }; +} diff --git a/src/app/(admin)/admin/system/page.tsx b/src/app/(admin)/admin/system/page.tsx index 44e924e..2f5bb2e 100644 --- a/src/app/(admin)/admin/system/page.tsx +++ b/src/app/(admin)/admin/system/page.tsx @@ -4,6 +4,7 @@ import { formatDistanceToNow } from "date-fns"; import { getSystemHealth, getRecentScrapeRuns } from "@/lib/admin-queries"; import { AdminStatCard } from "@/components/admin/AdminStatCard"; +import { TriggerScrapeButton } from "@/components/admin/TriggerScrapeButton"; export const metadata: Metadata = { title: "System Health" }; @@ -53,13 +54,16 @@ export default async function AdminSystemPage() { return (
{/* Header */} -
-

- System Health -

-

- Scraper status, failure tracking, and AI spend -

+
+
+

+ System Health +

+

+ Scraper status, failure tracking, and AI spend +

+
+
{/* Cron status */} diff --git a/src/app/api/scrape/route.ts b/src/app/api/scrape/route.ts index 2eccbc2..7f8a72e 100644 --- a/src/app/api/scrape/route.ts +++ b/src/app/api/scrape/route.ts @@ -17,7 +17,7 @@ import { } from "@/lib/normalize"; import type { ScraperSource } from "@prisma/client"; -export const maxDuration = 60; +export const maxDuration = 300; export async function POST(req: Request) { const authHeader = req.headers.get("authorization"); diff --git a/src/components/admin/TriggerScrapeButton.tsx b/src/components/admin/TriggerScrapeButton.tsx new file mode 100644 index 0000000..fa4f068 --- /dev/null +++ b/src/components/admin/TriggerScrapeButton.tsx @@ -0,0 +1,45 @@ +"use client"; + +import { useState, useTransition } from "react"; + +import { adminTriggerScrape } from "@/app/(admin)/actions"; + +export function TriggerScrapeButton() { + const [isPending, startTransition] = useTransition(); + const [result, setResult] = useState<{ + ok: boolean; + message: string; + } | null>(null); + + function handleClick() { + setResult(null); + startTransition(async () => { + const res = await adminTriggerScrape(); + setResult(res); + }); + } + + return ( +
+ + {result && ( + + {result.message} + + )} +
+ ); +} diff --git a/src/config/companies.ts b/src/config/companies.ts index 0f9903b..1fb6dbb 100644 --- a/src/config/companies.ts +++ b/src/config/companies.ts @@ -157,25 +157,60 @@ export const LEVER_COMPANIES: CompanyConfig[] = [ // ── USAJobs ────────────────────────────────────────────────────────────────── // data.usajobs.gov/api/search — requires API key + registered email +// No rate limit. Empty keyword = all listings posted in the last 7 days. +// Pool dedup (skipDuplicates) prevents redundant DB rows across runs. export const USAJOBS_SEARCHES: USAJobsSearchConfig[] = [ - { keyword: "Software Engineer", location: "Washington, DC" }, - { keyword: "IT Specialist", location: "" }, - { keyword: "Human Resources", location: "" }, - { keyword: "Administrative", location: "" }, - { keyword: "Banking", location: "" }, + // Catch-all: fetches every federal listing posted in the last 7 days + { keyword: "", location: "" }, ]; // ── Adzuna ──────────────────────────────────────────────────────────────────── // api.adzuna.com/v1/api/jobs/{country}/search/{page} // Requires ADZUNA_APP_ID and ADZUNA_APP_KEY env vars +// Broad coverage across major English + European markets. export const ADZUNA_SEARCHES: AdzunaSearchConfig[] = [ - { country: "us", keyword: "Software Engineer", location: "" }, - { country: "us", keyword: "Human Resources", location: "" }, - { country: "us", keyword: "IT Specialist", location: "" }, - { country: "us", keyword: "Bank Teller", location: "" }, - { country: "us", keyword: "Administrative Assistant", location: "" }, - { country: "de", keyword: "Software Engineer", location: "Berlin" }, - { country: "gb", keyword: "Software Engineer", location: "London" }, + // United States — broad categories + { country: "us", keyword: "Software Engineer", location: "" }, + { country: "us", keyword: "Data Scientist", location: "" }, + { country: "us", keyword: "Product Manager", location: "" }, + { country: "us", keyword: "Human Resources", location: "" }, + { country: "us", keyword: "Accountant", location: "" }, + { country: "us", keyword: "Marketing Manager", location: "" }, + { country: "us", keyword: "Administrative", location: "" }, + { country: "us", keyword: "Sales", location: "" }, + { country: "us", keyword: "Project Manager", location: "" }, + { country: "us", keyword: "Nurse", location: "" }, + // United Kingdom + { country: "gb", keyword: "Software Engineer", location: "" }, + { country: "gb", keyword: "Data Analyst", location: "" }, + { country: "gb", keyword: "Product Manager", location: "" }, + { country: "gb", keyword: "Marketing", location: "" }, + { country: "gb", keyword: "Finance", location: "" }, + // Germany + { country: "de", keyword: "Software Engineer", location: "" }, + { country: "de", keyword: "Data Engineer", location: "" }, + { country: "de", keyword: "Product Manager", location: "" }, + // Canada + { country: "ca", keyword: "Software Engineer", location: "" }, + { country: "ca", keyword: "Data Analyst", location: "" }, + { country: "ca", keyword: "Project Manager", location: "" }, + // Australia + { country: "au", keyword: "Software Engineer", location: "" }, + { country: "au", keyword: "Data Analyst", location: "" }, + // Netherlands + { country: "nl", keyword: "Software Engineer", location: "" }, + { country: "nl", keyword: "Data", location: "" }, + // France + { country: "fr", keyword: "Software Engineer", location: "" }, + // Poland + { country: "pl", keyword: "Software Engineer", location: "" }, + // India + { country: "in", keyword: "Software Engineer", location: "" }, + { country: "in", keyword: "Data Scientist", location: "" }, + // Singapore + { country: "sg", keyword: "Software Engineer", location: "" }, + { country: "sg", keyword: "Data", location: "" }, + { country: "sg", keyword: "Product Manager", location: "" }, ]; diff --git a/src/lib/normalize.ts b/src/lib/normalize.ts index f1e2847..8130713 100644 --- a/src/lib/normalize.ts +++ b/src/lib/normalize.ts @@ -36,6 +36,7 @@ export function normalizeGreenhouseForPool( description: raw.content ? htmlToMarkdown(raw.content) : "", postedAt: raw.updated_at ? new Date(raw.updated_at) : null, rawData: rawMeta as unknown as Prisma.InputJsonValue, + skills: [], country, region, }; @@ -89,6 +90,7 @@ export function normalizeLeverForPool( jobType: LEVER_JOB_TYPE[commitment] ?? null, postedAt: raw.createdAt ? new Date(raw.createdAt) : null, rawData: raw as unknown as Prisma.InputJsonValue, + skills: [], country, region, }; @@ -138,6 +140,7 @@ export function normalizeAshbyForPool( jobType: ASHBY_JOB_TYPE[empType] ?? null, postedAt: raw.publishedAt ? new Date(raw.publishedAt) : null, rawData: rawMeta as unknown as Prisma.InputJsonValue, + skills: [], country, region, }; @@ -224,6 +227,7 @@ export function normalizeUSAJobsForPool( salaryMax, currency: "USD", rawData: rawMeta as unknown as Prisma.InputJsonValue, + skills: [], country: "US", region: raw.PositionLocation?.[0]?.LocationName ? parseLocation(raw.PositionLocation[0].LocationName).region @@ -259,6 +263,7 @@ export function normalizeAdzunaForPool( description: raw.description ?? "", postedAt: raw.created ? new Date(raw.created) : null, rawData: rawMeta as unknown as Prisma.InputJsonValue, + skills: [], salaryMin: raw.salary_min ? Math.round(raw.salary_min) : null, salaryMax: raw.salary_max ? Math.round(raw.salary_max) : null, currency: currencyForCountry(country), diff --git a/src/lib/scrapers/adzuna.ts b/src/lib/scrapers/adzuna.ts index 5ab37b3..730a97e 100644 --- a/src/lib/scrapers/adzuna.ts +++ b/src/lib/scrapers/adzuna.ts @@ -96,7 +96,7 @@ async function fetchPage( // ── Main scraper ──────────────────────────────────────────────────────────── -const MAX_PAGES = 3; // 50 results/page * 3 = 150 max per search +const MAX_PAGES = 5; // 50 results/page * 5 = 250 max per search /** * Scrapes job listings from the Adzuna API for all configured searches. @@ -114,6 +114,7 @@ export async function scrapeAdzuna( for (const search of searches) { let page = 1; + let searchFetched = 0; while (page <= MAX_PAGES) { const data = await fetchPage(search, page, appId, appKey); @@ -126,9 +127,10 @@ export async function scrapeAdzuna( keyword: search.keyword, }); } + searchFetched += data.results.length; - // Stop if we've fetched all available results - if (results.length >= data.count || data.results.length < 50) break; + // Stop if we've fetched all available results for this search + if (searchFetched >= data.count || data.results.length < 50) break; page++; } } diff --git a/src/lib/scrapers/usajobs.ts b/src/lib/scrapers/usajobs.ts index d0e0ec6..d534cbb 100644 --- a/src/lib/scrapers/usajobs.ts +++ b/src/lib/scrapers/usajobs.ts @@ -78,7 +78,6 @@ async function fetchSearch( if (!apiKey || !email) return null; const params = new URLSearchParams({ - Keyword: search.keyword, ResultsPerPage: String(RESULTS_PER_PAGE), Page: String(page), DatePosted: "7", @@ -86,6 +85,10 @@ async function fetchSearch( SortDirection: "Desc", }); + if (search.keyword) { + params.set("Keyword", search.keyword); + } + if (search.location) { params.set("LocationName", search.location); }