app-platform: generate catalog from app manifests
This commit is contained in:
archipelago
173
scripts/check-app-catalog-drift.py
Normal file
173
scripts/check-app-catalog-drift.py
Normal file
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Report drift between app-catalog/catalog.json and apps/*/manifest.yml."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
INTERNAL_MANIFEST_IDS = {
|
||||
"aiui",
|
||||
"archy-btcpay-db",
|
||||
"archy-mempool-db",
|
||||
"archy-mempool-web",
|
||||
"archy-nbxplorer",
|
||||
"bitcoin-ui",
|
||||
"core-lightning",
|
||||
"did-wallet",
|
||||
"electrs-ui",
|
||||
"lightning-stack",
|
||||
"lnd-ui",
|
||||
"mempool-api",
|
||||
"morphos-server",
|
||||
"onlyoffice",
|
||||
"router",
|
||||
"strfry",
|
||||
"web5-dwn",
|
||||
}
|
||||
|
||||
LEGACY_STACK_CATALOG_IDS = {
|
||||
"immich",
|
||||
"netbird",
|
||||
"saleor",
|
||||
"tailscale",
|
||||
}
|
||||
|
||||
|
||||
def load_catalog(path: Path) -> dict[str, dict[str, Any]]:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
apps = data.get("apps", [])
|
||||
if not isinstance(apps, list):
|
||||
raise ValueError(f"{path}: expected .apps to be a list")
|
||||
return {str(app.get("id", "")): app for app in apps if isinstance(app, dict) and app.get("id")}
|
||||
|
||||
|
||||
def load_manifests(apps_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
manifests: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(apps_dir.glob("*/manifest.yml")):
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
if not isinstance(data, dict) or not isinstance(data.get("app"), dict):
|
||||
continue
|
||||
app = data["app"]
|
||||
app_id = app.get("id")
|
||||
if app_id:
|
||||
manifests[str(app_id)] = {"path": str(path), "app": app}
|
||||
return manifests
|
||||
|
||||
|
||||
def metadata(app: dict[str, Any]) -> dict[str, Any]:
|
||||
value = app.get("metadata")
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def manifest_value(app: dict[str, Any], field: str) -> Any:
|
||||
meta = metadata(app)
|
||||
container = app.get("container") if isinstance(app.get("container"), dict) else {}
|
||||
match field:
|
||||
case "title":
|
||||
return app.get("name")
|
||||
case "version":
|
||||
return str(app.get("version", ""))
|
||||
case "description":
|
||||
return app.get("description")
|
||||
case "dockerImage":
|
||||
return container.get("image")
|
||||
case "category":
|
||||
return app.get("category") or meta.get("category")
|
||||
case "tier":
|
||||
return meta.get("tier")
|
||||
case "icon":
|
||||
return meta.get("icon")
|
||||
case "repoUrl":
|
||||
return meta.get("repo") or meta.get("repoUrl")
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def normalize(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--catalog", default="app-catalog/catalog.json")
|
||||
parser.add_argument("--apps-dir", default="apps")
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="exit non-zero when missing entries or metadata drift are found",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--release",
|
||||
action="store_true",
|
||||
help="suppress known internal/legacy-stack entries so output is release-actionable",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
catalog = load_catalog(Path(args.catalog))
|
||||
manifests = load_manifests(Path(args.apps_dir))
|
||||
|
||||
catalog_ids = set(catalog)
|
||||
manifest_ids = set(manifests)
|
||||
missing_manifests = sorted(catalog_ids - manifest_ids)
|
||||
missing_catalog = sorted(manifest_ids - catalog_ids)
|
||||
if args.release:
|
||||
missing_manifests = [app_id for app_id in missing_manifests if app_id not in LEGACY_STACK_CATALOG_IDS]
|
||||
missing_catalog = [app_id for app_id in missing_catalog if app_id not in INTERNAL_MANIFEST_IDS]
|
||||
|
||||
compared_fields = [
|
||||
"title",
|
||||
"version",
|
||||
"description",
|
||||
"dockerImage",
|
||||
"category",
|
||||
"tier",
|
||||
"icon",
|
||||
"repoUrl",
|
||||
]
|
||||
drift: list[str] = []
|
||||
for app_id in sorted(catalog_ids & manifest_ids):
|
||||
catalog_app = catalog[app_id]
|
||||
manifest_app = manifests[app_id]["app"]
|
||||
for field in compared_fields:
|
||||
catalog_val = normalize(catalog_app.get(field))
|
||||
manifest_val = normalize(manifest_value(manifest_app, field))
|
||||
if catalog_val and manifest_val and catalog_val != manifest_val:
|
||||
drift.append(f"{app_id}: {field}: catalog={catalog_val!r} manifest={manifest_val!r}")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"catalog_apps": len(catalog),
|
||||
"manifest_apps": len(manifests),
|
||||
"missing_manifests": len(missing_manifests),
|
||||
"missing_catalog": len(missing_catalog),
|
||||
"metadata_drift": len(drift),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
for app_id in missing_manifests:
|
||||
print(f"MISSING_MANIFEST {app_id}")
|
||||
for app_id in missing_catalog:
|
||||
print(f"MISSING_CATALOG {app_id}")
|
||||
for item in drift:
|
||||
print(f"DRIFT {item}")
|
||||
|
||||
if args.strict and (missing_manifests or missing_catalog or drift):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
205
scripts/generate-app-catalog.py
Normal file
205
scripts/generate-app-catalog.py
Normal file
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync public app catalog metadata from apps/*/manifest.yml.
|
||||
|
||||
Manifests are the source of truth for fields the runtime already needs
|
||||
(`name`, `version`, `description`, container image, category, tier, icon,
|
||||
repo URL). The catalog still owns presentation-only fields that manifests do
|
||||
not carry yet, such as `author`, `requires`, `featured`, and rich
|
||||
`containerConfig` notes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
SYNC_FIELDS = ("title", "version", "description", "dockerImage", "category", "tier", "icon", "repoUrl")
|
||||
|
||||
|
||||
def load_manifests(apps_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
manifests: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(apps_dir.glob("*/manifest.yml")):
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
if not isinstance(data, dict) or not isinstance(data.get("app"), dict):
|
||||
continue
|
||||
app = data["app"]
|
||||
app_id = app.get("id")
|
||||
if app_id:
|
||||
manifests[str(app_id)] = app
|
||||
return manifests
|
||||
|
||||
|
||||
def metadata(app: dict[str, Any]) -> dict[str, Any]:
|
||||
value = app.get("metadata")
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def manifest_catalog_values(app: dict[str, Any]) -> dict[str, str]:
|
||||
meta = metadata(app)
|
||||
container = app.get("container") if isinstance(app.get("container"), dict) else {}
|
||||
values = {
|
||||
"title": app.get("name"),
|
||||
"version": app.get("version"),
|
||||
"description": app.get("description"),
|
||||
"dockerImage": container.get("image"),
|
||||
"category": app.get("category") or meta.get("category"),
|
||||
"tier": meta.get("tier"),
|
||||
"icon": meta.get("icon"),
|
||||
"repoUrl": meta.get("repo") or meta.get("repoUrl") or meta.get("source"),
|
||||
}
|
||||
return {key: str(value) for key, value in values.items() if value is not None and str(value).strip()}
|
||||
|
||||
|
||||
def manifest_launch_port(app: dict[str, Any]) -> int | None:
|
||||
"""Return the manifest-owned public UI port, when it is unambiguous."""
|
||||
interfaces = app.get("interfaces")
|
||||
if isinstance(interfaces, dict):
|
||||
main = interfaces.get("main")
|
||||
if isinstance(main, dict) and main.get("type") == "ui":
|
||||
port = main.get("port")
|
||||
if isinstance(port, int):
|
||||
return port
|
||||
if isinstance(port, str) and port.isdigit():
|
||||
return int(port)
|
||||
|
||||
ports = app.get("ports")
|
||||
if not isinstance(ports, list):
|
||||
return None
|
||||
tcp_ports = [
|
||||
item.get("host")
|
||||
for item in ports
|
||||
if isinstance(item, dict) and str(item.get("protocol", "tcp")).lower() == "tcp"
|
||||
]
|
||||
if len(tcp_ports) != 1:
|
||||
return None
|
||||
port = tcp_ports[0]
|
||||
if isinstance(port, int):
|
||||
return port
|
||||
if isinstance(port, str) and port.isdigit():
|
||||
return int(port)
|
||||
return None
|
||||
|
||||
|
||||
def manifest_opens_in_new_tab(app: dict[str, Any]) -> bool:
|
||||
"""Return whether manifest launch metadata opts the app out of iframe launch."""
|
||||
launch = metadata(app).get("launch")
|
||||
if not isinstance(launch, dict):
|
||||
return False
|
||||
return launch.get("open_in_new_tab") is True
|
||||
|
||||
|
||||
def ts_string(value: str) -> str:
|
||||
return json.dumps(value, ensure_ascii=True)
|
||||
|
||||
|
||||
def render_app_session_config(manifests: dict[str, dict[str, Any]]) -> str:
|
||||
ports: dict[str, int] = {}
|
||||
titles: dict[str, str] = {}
|
||||
new_tab_apps: list[str] = []
|
||||
for app_id, app in sorted(manifests.items()):
|
||||
name = app.get("name")
|
||||
if isinstance(name, str) and name.strip():
|
||||
titles[app_id] = name.strip()
|
||||
port = manifest_launch_port(app)
|
||||
if port:
|
||||
ports[app_id] = port
|
||||
if manifest_opens_in_new_tab(app):
|
||||
new_tab_apps.append(app_id)
|
||||
|
||||
lines = [
|
||||
"/** Generated by scripts/generate-app-catalog.py. Do not edit manually. */",
|
||||
"",
|
||||
"export const GENERATED_APP_PORTS: Record<string, number> = {",
|
||||
]
|
||||
for app_id, port in ports.items():
|
||||
lines.append(f" {ts_string(app_id)}: {port},")
|
||||
lines.extend([
|
||||
"}",
|
||||
"",
|
||||
"export const GENERATED_APP_TITLES: Record<string, string> = {",
|
||||
])
|
||||
for app_id, title in titles.items():
|
||||
lines.append(f" {ts_string(app_id)}: {ts_string(title)},")
|
||||
lines.extend([
|
||||
"}",
|
||||
"",
|
||||
"export const GENERATED_NEW_TAB_APPS = new Set<string>([",
|
||||
])
|
||||
for app_id in new_tab_apps:
|
||||
lines.append(f" {ts_string(app_id)},")
|
||||
lines.extend(["])", ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def sync_catalog(path: Path, manifests: dict[str, dict[str, Any]]) -> int:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
catalog = json.load(fh)
|
||||
apps = catalog.get("apps")
|
||||
if not isinstance(apps, list):
|
||||
raise ValueError(f"{path}: expected .apps to be a list")
|
||||
|
||||
changed = 0
|
||||
for catalog_app in apps:
|
||||
if not isinstance(catalog_app, dict):
|
||||
continue
|
||||
app_id = catalog_app.get("id")
|
||||
if not app_id or str(app_id) not in manifests:
|
||||
continue
|
||||
values = manifest_catalog_values(manifests[str(app_id)])
|
||||
for field in SYNC_FIELDS:
|
||||
if field not in values:
|
||||
continue
|
||||
old = catalog_app.get(field)
|
||||
new = values[field]
|
||||
if old != new:
|
||||
catalog_app[field] = new
|
||||
changed += 1
|
||||
|
||||
path.write_text(json.dumps(catalog, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return changed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apps-dir", default="apps")
|
||||
parser.add_argument(
|
||||
"--catalog",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Catalog JSON path to update. May be passed multiple times.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--app-session-config",
|
||||
default="neode-ui/src/views/appSession/generatedAppSessionConfig.ts",
|
||||
help="Generated TypeScript app-session metadata path. Pass an empty string to skip.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
catalogs = args.catalog or ["app-catalog/catalog.json", "neode-ui/public/catalog.json"]
|
||||
manifests = load_manifests(Path(args.apps_dir))
|
||||
total = 0
|
||||
for catalog in catalogs:
|
||||
changed = sync_catalog(Path(catalog), manifests)
|
||||
total += changed
|
||||
print(f"{catalog}: updated {changed} fields")
|
||||
if args.app_session_config:
|
||||
path = Path(args.app_session_config)
|
||||
content = render_app_session_config(manifests)
|
||||
old = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
if old != content:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
print(f"{path}: updated")
|
||||
else:
|
||||
print(f"{path}: updated 0 fields")
|
||||
print(f"total_updated={total}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user