Add reusable dataset metadata fields

scripts/sqd-network-metadata/README.md

@@ -14,4 +14,28 @@ pip install pyyaml rich
 python scripts/sqd-network-metadata/__main__.py -h
 python scripts/sqd-network-metadata/__main__.py add
 python scripts/sqd-network-metadata/__main__.py sort
+python scripts/sqd-network-metadata/__main__.py validate
 ```
+
+## Metadata fields
+
+Each dataset has its own record. Chain-level fields are repeated on every
+network in the same `ecosystem` so consumers do not need another lookup.
+
+| Field | Meaning |
+| --- | --- |
+| `display_name` | Human-readable network name. |
+| `ecosystem` | Canonical chain grouping. Mainnets and testnets for one chain use the same value. |
+| `kind` | Data model or VM, such as `evm`, `substrate`, `solana`, or `bitcoin`. |
+| `type` | Network class: `mainnet`, `testnet`, or `devnet`. |
+| `logo_url` | Network or chain logo. |
+| `logo_bg` | Optional rendering hint. `white` adds a white background behind a dark logo. |
+| `website` | Official chain website. |
+| `docs` | Official developer documentation. |
+| `explorer` | Official or primary block explorer when reviewed. |
+| `tier` | SQD support tier: `core`, `partner`, or `frontier`. This is consistent across an ecosystem. |
+| `private` | `true` when access requires a private or commercial arrangement. |
+
+Run `validate` before opening a PR. It checks required fields on every metadata
+record, confirms every declared dataset has metadata, and verifies that
+chain-level fields agree within an ecosystem.

scripts/sqd-network-metadata/__main__.py

@@ -2,6 +2,7 @@
 
 import add_command
 import sort_command
+import validate_command
 
 
 if __name__ == "__main__":
@@ -11,5 +12,6 @@
     subparser = program.add_subparsers()
     add_command.update_parser(subparser.add_parser("add"))
     sort_command.update_parser(subparser.add_parser("sort"))
+    validate_command.update_parser(subparser.add_parser("validate"))
     parsed_args = program.parse_args()
     parsed_args.func(parsed_args)

scripts/sqd-network-metadata/add_command.py

@@ -9,6 +9,7 @@
 
 METADATA_PATH = Path(__file__).resolve().parent.parent.parent / "src/sqd-network/mainnet/metadata.yml"
 TYPE_CHOICES = ["testnet", "mainnet", "devnet"]
+TIER_CHOICES = ["core", "partner", "frontier"]
 
 
 def update_parser(parser: argparse.ArgumentParser):
@@ -31,17 +32,45 @@ def _parse_chain_id(chain_id_raw: str):
     return int(chain_id_raw)
 
 
-def _build_entry(kind: str, display_name: str, logo_url_raw: str, chain_type: str, chain_id_raw: str):
+def _optional_value(value: str):
+    return None if value == "null" else value
+
+
+def _build_entry(
+    kind: str,
+    display_name: str,
+    ecosystem: str,
+    logo_url: str,
+    chain_type: str,
+    chain_id_raw: str,
+    website: str,
+    docs: str,
+    explorer_raw: str,
+    tier: str,
+    private: bool,
+    logo_bg_raw: str,
+):
     chain_id = _parse_chain_id(chain_id_raw)
 
     meta = {
         "kind": kind,
         "display_name": display_name,
+        "ecosystem": ecosystem,
         "type": chain_type,
+        "logo_url": logo_url,
+        "website": website,
+        "docs": docs,
+        "tier": tier,
+        "private": private,
     }
 
-    if logo_url_raw != "null":
-        meta["logo_url"] = logo_url_raw
+    explorer = _optional_value(explorer_raw)
+    if explorer is not None:
+        meta["explorer"] = explorer
+
+    logo_bg = _optional_value(logo_bg_raw)
+    if logo_bg is not None:
+        meta["logo_bg"] = logo_bg
 
     if chain_id is not None:
         meta["evm"] = {"chain_id": chain_id}
@@ -68,13 +97,37 @@ def _run(parsed_args):
     display_name = Prompt.ask("display_name").strip()
     assert display_name, "display_name must not be empty"
 
-    logo_url_raw = Prompt.ask("logo_url", default="null").strip()
-    assert logo_url_raw, "logo_url must not be empty (use 'null' for missing value)"
+    ecosystem = Prompt.ask("ecosystem", default=dataset_key).strip()
+    assert ecosystem, "ecosystem must not be empty"
+
+    logo_url = Prompt.ask("logo_url").strip()
+    assert logo_url, "logo_url must not be empty"
 
     chain_type = Prompt.ask("type", default="mainnet", choices=TYPE_CHOICES).strip()
     chain_id_raw = Prompt.ask("chain_id", default="null").strip()
-
-    entry = _build_entry(kind, display_name, logo_url_raw, chain_type, chain_id_raw)
+    website = Prompt.ask("website").strip()
+    assert website, "website must not be empty"
+    docs = Prompt.ask("docs").strip()
+    assert docs, "docs must not be empty"
+    explorer_raw = Prompt.ask("explorer", default="null").strip()
+    tier = Prompt.ask("tier", default="frontier", choices=TIER_CHOICES).strip()
+    private = Confirm.ask("private", default=False)
+    logo_bg_raw = Prompt.ask("logo_bg", default="null", choices=["null", "white"]).strip()
+
+    entry = _build_entry(
+        kind,
+        display_name,
+        ecosystem,
+        logo_url,
+        chain_type,
+        chain_id_raw,
+        website,
+        docs,
+        explorer_raw,
+        tier,
+        private,
+        logo_bg_raw,
+    )
     syntax = Syntax(yaml.safe_dump(entry, sort_keys=False), "yaml", theme="monokai", line_numbers=True)
     console.print(f"\nFollowing entry will be added as datasets.{dataset_key}:")
     console.print(syntax)

scripts/sqd-network-metadata/validate_command.py

@@ -0,0 +1,76 @@
+import argparse
+from collections import defaultdict
+from pathlib import Path
+
+import yaml
+from rich.console import Console
+
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+METADATA_PATH = ROOT / "src/sqd-network/mainnet/metadata.yml"
+DATASETS_PATH = ROOT / "src/sqd-network/datasets.yml"
+REQUIRED_FIELDS = ("kind", "display_name", "ecosystem", "logo_url", "website", "docs", "tier", "private")
+TIER_CHOICES = {"core", "partner", "frontier"}
+TYPE_CHOICES = {"mainnet", "testnet", "devnet"}
+LOGO_BG_CHOICES = {"white"}
+
+
+def update_parser(parser: argparse.ArgumentParser):
+    parser.description = "Validate dataset metadata"
+    parser.set_defaults(func=_run)
+
+
+def _load_yaml(path: Path):
+    with open(path, "r", encoding="utf-8") as handle:
+        return yaml.safe_load(handle)
+
+
+def _run(parsed_args):
+    del parsed_args
+    console = Console()
+    metadata = _load_yaml(METADATA_PATH)["datasets"]
+    active = [entry["name"] for entry in _load_yaml(DATASETS_PATH)["sqd-network-datasets"]]
+    errors = []
+    ecosystems = defaultdict(lambda: defaultdict(set))
+
+    for dataset in active:
+        if dataset not in metadata:
+            errors.append(f"{dataset}: missing metadata entry")
+
+    for dataset, record in metadata.items():
+        fields = record.get("metadata") or {}
+        for field in REQUIRED_FIELDS:
+            if field not in fields or fields[field] in (None, ""):
+                errors.append(f"{dataset}: missing metadata.{field}")
+        if fields.get("tier") not in TIER_CHOICES:
+            errors.append(f"{dataset}: invalid metadata.tier {fields.get('tier')!r}")
+        if "type" in fields and fields["type"] not in TYPE_CHOICES:
+            errors.append(f"{dataset}: invalid metadata.type {fields['type']!r}")
+        if not isinstance(fields.get("private"), bool):
+            errors.append(f"{dataset}: metadata.private must be a boolean")
+        if "logo_bg" in fields and fields["logo_bg"] not in LOGO_BG_CHOICES:
+            errors.append(f"{dataset}: invalid metadata.logo_bg {fields['logo_bg']!r}")
+        for field in ("website", "docs", "explorer", "logo_url"):
+            if field in fields and not str(fields[field]).startswith(("https://", "http://")):
+                errors.append(f"{dataset}: metadata.{field} must be an HTTP(S) URL")
+        ecosystem = fields.get("ecosystem")
+        if ecosystem:
+            for field in ("website", "docs", "tier", "private"):
+                if field in fields:
+                    ecosystems[ecosystem][field].add(fields[field])
+
+    for ecosystem, fields in ecosystems.items():
+        for field, values in fields.items():
+            if len(values) > 1:
+                errors.append(f"{ecosystem}: networks disagree on metadata.{field}: {sorted(values)!r}")
+
+    if errors:
+        console.print("Metadata validation failed:", style="bold red")
+        for error in errors:
+            console.print(f"- {error}")
+        raise SystemExit(1)
+
+    console.print(
+        f"Validated {len(metadata)} metadata records and {len(active)} declared datasets.",
+        style="bold green",
+    )