Fix null skills constraint in pool normalizers

src/app/(admin)/actions.ts

@@ -55,3 +55,24 @@ export async function adminResetMonthlyUsage(data: unknown) {
   });
   revalidatePath("/admin/users");
 }
+
+export async function adminTriggerScrape(): Promise<{
+  ok: boolean;
+  message: string;
+}> {
+  await requireAdmin();
+
+  const url = new URL("/api/scrape", env.NEXT_PUBLIC_APP_URL);
+  const res = await fetch(url.toString(), {
+    method: "POST",
+    headers: { Authorization: `Bearer ${env.CRON_SECRET}` },
+  });
+
+  if (!res.ok) {
+    return { ok: false, message: `Scrape failed: ${res.status} ${res.statusText}` };
+  }
+
+  const body = (await res.json()) as { poolNew?: number };
+  revalidatePath("/admin/system");
+  return { ok: true, message: `Scrape complete — ${body.poolNew ?? 0} new to pool` };
+}

src/app/(admin)/admin/system/page.tsx

@@ -4,6 +4,7 @@ import { formatDistanceToNow } from "date-fns";
 
 import { getSystemHealth, getRecentScrapeRuns } from "@/lib/admin-queries";
 import { AdminStatCard } from "@/components/admin/AdminStatCard";
+import { TriggerScrapeButton } from "@/components/admin/TriggerScrapeButton";
 
 export const metadata: Metadata = { title: "System Health" };
 
@@ -53,13 +54,16 @@ export default async function AdminSystemPage() {
   return (
     <div className="space-y-8">
       {/* Header */}
-      <div>
-        <h1 className="text-xl font-semibold text-[var(--text)]">
-          System Health
-        </h1>
-        <p className="mt-1 text-sm text-[var(--text-muted)]">
-          Scraper status, failure tracking, and AI spend
-        </p>
+      <div className="flex items-start justify-between gap-4">
+        <div>
+          <h1 className="text-xl font-semibold text-[var(--text)]">
+            System Health
+          </h1>
+          <p className="mt-1 text-sm text-[var(--text-muted)]">
+            Scraper status, failure tracking, and AI spend
+          </p>
+        </div>
+        <TriggerScrapeButton />
       </div>
 
       {/* Cron status */}

src/app/api/scrape/route.ts

@@ -17,7 +17,7 @@ import {
 } from "@/lib/normalize";
 import type { ScraperSource } from "@prisma/client";
 
-export const maxDuration = 60;
+export const maxDuration = 300;
 
 export async function POST(req: Request) {
   const authHeader = req.headers.get("authorization");

src/components/admin/TriggerScrapeButton.tsx

@@ -0,0 +1,45 @@
+"use client";
+
+import { useState, useTransition } from "react";
+
+import { adminTriggerScrape } from "@/app/(admin)/actions";
+
+export function TriggerScrapeButton() {
+  const [isPending, startTransition] = useTransition();
+  const [result, setResult] = useState<{
+    ok: boolean;
+    message: string;
+  } | null>(null);
+
+  function handleClick() {
+    setResult(null);
+    startTransition(async () => {
+      const res = await adminTriggerScrape();
+      setResult(res);
+    });
+  }
+
+  return (
+    <div className="flex items-center gap-3">
+      <button
+        type="button"
+        onClick={handleClick}
+        disabled={isPending}
+        className="rounded-lg border border-[var(--border)] bg-[var(--bg-card)] px-3 py-1.5 text-sm font-medium text-[var(--text)] transition-colors hover:bg-[var(--bg-subtle)] disabled:opacity-50"
+      >
+        {isPending ? "Scraping…" : "Trigger Scrape"}
+      </button>
+      {result && (
+        <span
+          className={`text-sm ${
+            result.ok
+              ? "text-emerald-600 dark:text-emerald-400"
+              : "text-red-600 dark:text-red-400"
+          }`}
+        >
+          {result.message}
+        </span>
+      )}
+    </div>
+  );
+}

src/config/companies.ts

@@ -157,25 +157,60 @@ export const LEVER_COMPANIES: CompanyConfig[] = [
 
 // ── USAJobs ──────────────────────────────────────────────────────────────────
 // data.usajobs.gov/api/search — requires API key + registered email
+// No rate limit. Empty keyword = all listings posted in the last 7 days.
+// Pool dedup (skipDuplicates) prevents redundant DB rows across runs.
 
 export const USAJOBS_SEARCHES: USAJobsSearchConfig[] = [
-  { keyword: "Software Engineer", location: "Washington, DC" },
-  { keyword: "IT Specialist", location: "" },
-  { keyword: "Human Resources", location: "" },
-  { keyword: "Administrative", location: "" },
-  { keyword: "Banking", location: "" },
+  // Catch-all: fetches every federal listing posted in the last 7 days
+  { keyword: "", location: "" },
 ];
 
 // ── Adzuna ────────────────────────────────────────────────────────────────────
 // api.adzuna.com/v1/api/jobs/{country}/search/{page}
 // Requires ADZUNA_APP_ID and ADZUNA_APP_KEY env vars
+// Broad coverage across major English + European markets.
 
 export const ADZUNA_SEARCHES: AdzunaSearchConfig[] = [
-  { country: "us", keyword: "Software Engineer",        location: "" },
-  { country: "us", keyword: "Human Resources",          location: "" },
-  { country: "us", keyword: "IT Specialist",            location: "" },
-  { country: "us", keyword: "Bank Teller",              location: "" },
-  { country: "us", keyword: "Administrative Assistant", location: "" },
-  { country: "de", keyword: "Software Engineer",        location: "Berlin" },
-  { country: "gb", keyword: "Software Engineer",        location: "London" },
+  // United States — broad categories
+  { country: "us", keyword: "Software Engineer",    location: "" },
+  { country: "us", keyword: "Data Scientist",       location: "" },
+  { country: "us", keyword: "Product Manager",      location: "" },
+  { country: "us", keyword: "Human Resources",      location: "" },
+  { country: "us", keyword: "Accountant",           location: "" },
+  { country: "us", keyword: "Marketing Manager",    location: "" },
+  { country: "us", keyword: "Administrative",       location: "" },
+  { country: "us", keyword: "Sales",                location: "" },
+  { country: "us", keyword: "Project Manager",      location: "" },
+  { country: "us", keyword: "Nurse",                location: "" },
+  // United Kingdom
+  { country: "gb", keyword: "Software Engineer",    location: "" },
+  { country: "gb", keyword: "Data Analyst",         location: "" },
+  { country: "gb", keyword: "Product Manager",      location: "" },
+  { country: "gb", keyword: "Marketing",            location: "" },
+  { country: "gb", keyword: "Finance",              location: "" },
+  // Germany
+  { country: "de", keyword: "Software Engineer",    location: "" },
+  { country: "de", keyword: "Data Engineer",        location: "" },
+  { country: "de", keyword: "Product Manager",      location: "" },
+  // Canada
+  { country: "ca", keyword: "Software Engineer",    location: "" },
+  { country: "ca", keyword: "Data Analyst",         location: "" },
+  { country: "ca", keyword: "Project Manager",      location: "" },
+  // Australia
+  { country: "au", keyword: "Software Engineer",    location: "" },
+  { country: "au", keyword: "Data Analyst",         location: "" },
+  // Netherlands
+  { country: "nl", keyword: "Software Engineer",    location: "" },
+  { country: "nl", keyword: "Data",                 location: "" },
+  // France
+  { country: "fr", keyword: "Software Engineer",    location: "" },
+  // Poland
+  { country: "pl", keyword: "Software Engineer",    location: "" },
+  // India
+  { country: "in", keyword: "Software Engineer",    location: "" },
+  { country: "in", keyword: "Data Scientist",       location: "" },
+  // Singapore
+  { country: "sg", keyword: "Software Engineer",    location: "" },
+  { country: "sg", keyword: "Data",                 location: "" },
+  { country: "sg", keyword: "Product Manager",      location: "" },
 ];

src/lib/normalize.ts

@@ -24,7 +24,7 @@
 ): Prisma.JobPoolUncheckedCreateInput {
   // Exclude HTML content from rawData — it's already stored as markdown in description
   // and the full HTML can be 50–100KB per job, making bulk inserts impractical.
   const { content: _content, ...rawMeta } = raw;
   const { country, region } = parseLocation(raw.location?.name ?? null);
   return {
     externalId:  `greenhouse-${slug}-${raw.id}`,
@@ -36,6 +36,7 @@
     description: raw.content ? htmlToMarkdown(raw.content) : "",
     postedAt:    raw.updated_at ? new Date(raw.updated_at) : null,
     rawData:     rawMeta as unknown as Prisma.InputJsonValue,
+    skills:      [],
     country,
     region,
   };
@@ -89,6 +90,7 @@
     jobType:      LEVER_JOB_TYPE[commitment] ?? null,
     postedAt:     raw.createdAt ? new Date(raw.createdAt) : null,
     rawData:      raw as unknown as Prisma.InputJsonValue,
+    skills:       [],
     country,
     region,
   };
@@ -123,7 +125,7 @@
     : (raw.descriptionPlain ?? "");
 
   // Exclude large HTML/text description fields from rawData
   const { descriptionHtml: _html, descriptionPlain: _plain, ...rawMeta } = raw;
   const { country, region } = parseLocation(raw.location ?? null);
 
   return {
@@ -138,6 +140,7 @@
     jobType:      ASHBY_JOB_TYPE[empType] ?? null,
     postedAt:     raw.publishedAt ? new Date(raw.publishedAt) : null,
     rawData:      rawMeta as unknown as Prisma.InputJsonValue,
+    skills:       [],
     country,
     region,
   };
@@ -207,7 +210,7 @@
 
   // Exclude QualificationSummary and MajorDuties from rawData since they are
   // already stored as structured description text
   const { QualificationSummary: _qs, ...rawMeta } = raw;
 
   return {
     externalId:   `usajobs-${raw.PositionID}`,
@@ -224,6 +227,7 @@
     salaryMax,
     currency:     "USD",
     rawData:      rawMeta as unknown as Prisma.InputJsonValue,
+    skills:       [],
     country:      "US",
     region:       raw.PositionLocation?.[0]?.LocationName
       ? parseLocation(raw.PositionLocation[0].LocationName).region
@@ -246,7 +250,7 @@
   const contractTime = raw.contract_time?.toLowerCase() ?? "";
 
   // Exclude description from rawData — already stored as top-level field
   const { description: _desc, ...rawMeta } = raw;
   const { region } = parseLocation(raw.location?.display_name ?? null);
 
   return {
@@ -259,6 +263,7 @@
     description: raw.description ?? "",
     postedAt:    raw.created ? new Date(raw.created) : null,
     rawData:     rawMeta as unknown as Prisma.InputJsonValue,
+    skills:      [],
     salaryMin:   raw.salary_min ? Math.round(raw.salary_min) : null,
     salaryMax:   raw.salary_max ? Math.round(raw.salary_max) : null,
     currency:    currencyForCountry(country),
@@ -290,7 +295,7 @@
   raw: ArbeitnowJob,
 ): Prisma.JobPoolUncheckedCreateInput {
   // Exclude HTML description from rawData — it's already stored as markdown
   const { description: _desc, ...rawMeta } = raw;
 
   const locationType: LocationType | null = raw.remote ? "REMOTE" : null;
   const { country, region } = parseLocation(raw.location || null);

src/lib/scrapers/adzuna.ts

@@ -96,7 +96,7 @@ async function fetchPage(
 
 // ── Main scraper ────────────────────────────────────────────────────────────
 
-const MAX_PAGES = 3; // 50 results/page * 3 = 150 max per search
+const MAX_PAGES = 5; // 50 results/page * 5 = 250 max per search
 
 /**
  * Scrapes job listings from the Adzuna API for all configured searches.
@@ -114,6 +114,7 @@ export async function scrapeAdzuna(
 
   for (const search of searches) {
     let page = 1;
+    let searchFetched = 0;
 
     while (page <= MAX_PAGES) {
       const data = await fetchPage(search, page, appId, appKey);
@@ -126,9 +127,10 @@ export async function scrapeAdzuna(
           keyword: search.keyword,
         });
       }
+      searchFetched += data.results.length;
 
-      // Stop if we've fetched all available results
-      if (results.length >= data.count || data.results.length < 50) break;
+      // Stop if we've fetched all available results for this search
+      if (searchFetched >= data.count || data.results.length < 50) break;
       page++;
     }
   }

src/lib/scrapers/usajobs.ts

@@ -78,14 +78,17 @@ async function fetchSearch(
   if (!apiKey || !email) return null;
 
   const params = new URLSearchParams({
-    Keyword: search.keyword,
     ResultsPerPage: String(RESULTS_PER_PAGE),
     Page: String(page),
     DatePosted: "7",
     SortField: "DatePosted",
     SortDirection: "Desc",
   });
 
+  if (search.keyword) {
+    params.set("Keyword", search.keyword);
+  }
+
   if (search.location) {
     params.set("LocationName", search.location);
   }