feat(pipelines): add pipelines poc

packages/pipelines/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@ucdjs/pipelines",
+  "version": "1.0.0",
+  "type": "module",
+  "author": {
+    "name": "Lucas Norgaard",
+    "email": "[email protected]",
+    "url": "https://luxass.dev"
+  },
+  "packageManager": "[email protected]",
+  "license": "MIT",
+  "homepage": "https://github.com/ucdjs/ucd",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ucdjs/ucd.git",
+    "directory": "packages/pipelines"
+  },
+  "bugs": {
+    "url": "https://github.com/ucdjs/ucd/issues"
+  },
+  "exports": {
+    ".": "./dist/index.mjs",
+    "./package.json": "./package.json"
+  },
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22.18"
+  },
+  "scripts": {
+    "build": "tsdown --tsconfig=./tsconfig.build.json",
+    "dev": "tsdown --watch",
+    "clean": "git clean -xdf dist node_modules",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit -p tsconfig.build.json"
+  },
+  "dependencies": {
+    "@ucdjs-internal/shared": "workspace:*",
+    "picomatch": "catalog:prod"
+  },
+  "devDependencies": {
+    "@luxass/eslint-config": "catalog:linting",
+    "@types/picomatch": "catalog:types",
+    "@ucdjs-tooling/tsconfig": "workspace:*",
+    "@ucdjs-tooling/tsdown-config": "workspace:*",
+    "eslint": "catalog:linting",
+    "publint": "catalog:dev",
+    "tsdown": "catalog:dev",
+    "tsx": "catalog:dev",
+    "typescript": "catalog:dev"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/pipelines/src/artifact.ts

@@ -0,0 +1,31 @@
+import type { ParseContext, ParsedRow, PipelineFilter } from "./types";
+
+export interface ArtifactBuildContext {
+  version: string;
+}
+
+export interface PipelineArtifactDefinition<
+  TId extends string = string,
+  TValue = unknown,
+> {
+  id: TId;
+  filter?: PipelineFilter;
+  parser?: (ctx: ParseContext) => AsyncIterable<ParsedRow>;
+  build: (ctx: ArtifactBuildContext, rows?: AsyncIterable<ParsedRow>) => Promise<TValue>;
+}
+
+export function definePipelineArtifact<
+  const TId extends string,
+  TValue,
+>(
+  definition: PipelineArtifactDefinition<TId, TValue>,
+): PipelineArtifactDefinition<TId, TValue> {
+  return definition;
+}
+
+export type InferArtifactId<T> = T extends PipelineArtifactDefinition<infer TId, unknown> ? TId : never;
+export type InferArtifactValue<T> = T extends PipelineArtifactDefinition<string, infer TValue> ? TValue : never;
+
+export type InferArtifactsMap<T extends readonly PipelineArtifactDefinition[]> = {
+  [K in T[number] as InferArtifactId<K>]: InferArtifactValue<K>;
+};

packages/pipelines/src/events.ts

@@ -0,0 +1,165 @@
+import type { FileContext } from "./types";
+
+export type PipelineEventType =
+  | "pipeline:start"
+  | "pipeline:end"
+  | "version:start"
+  | "version:end"
+  | "artifact:start"
+  | "artifact:end"
+  | "file:matched"
+  | "file:skipped"
+  | "file:fallback"
+  | "parse:start"
+  | "parse:end"
+  | "resolve:start"
+  | "resolve:end"
+  | "error";
+
+export type PipelineStartEvent = {
+  type: "pipeline:start";
+  versions: string[];
+  timestamp: number;
+};
+
+export type PipelineEndEvent = {
+  type: "pipeline:end";
+  durationMs: number;
+  timestamp: number;
+};
+
+export type VersionStartEvent = {
+  type: "version:start";
+  version: string;
+  timestamp: number;
+};
+
+export type VersionEndEvent = {
+  type: "version:end";
+  version: string;
+  durationMs: number;
+  timestamp: number;
+};
+
+export type ArtifactStartEvent = {
+  type: "artifact:start";
+  artifactId: string;
+  version: string;
+  timestamp: number;
+};
+
+export type ArtifactEndEvent = {
+  type: "artifact:end";
+  artifactId: string;
+  version: string;
+  durationMs: number;
+  timestamp: number;
+};
+
+export type FileMatchedEvent = {
+  type: "file:matched";
+  file: FileContext;
+  routeId: string;
+  timestamp: number;
+};
+
+export type FileSkippedEvent = {
+  type: "file:skipped";
+  file: FileContext;
+  reason: "no-match" | "filtered";
+  timestamp: number;
+};
+
+export type FileFallbackEvent = {
+  type: "file:fallback";
+  file: FileContext;
+  timestamp: number;
+};
+
+export type ParseStartEvent = {
+  type: "parse:start";
+  file: FileContext;
+  routeId: string;
+  timestamp: number;
+};
+
+export type ParseEndEvent = {
+  type: "parse:end";
+  file: FileContext;
+  routeId: string;
+  rowCount: number;
+  durationMs: number;
+  timestamp: number;
+};
+
+export type ResolveStartEvent = {
+  type: "resolve:start";
+  file: FileContext;
+  routeId: string;
+  timestamp: number;
+};
+
+export type ResolveEndEvent = {
+  type: "resolve:end";
+  file: FileContext;
+  routeId: string;
+  outputCount: number;
+  durationMs: number;
+  timestamp: number;
+};
+
+export type PipelineErrorEvent = {
+  type: "error";
+  error: PipelineError;
+  timestamp: number;
+};
+
+export type PipelineEvent =
+  | PipelineStartEvent
+  | PipelineEndEvent
+  | VersionStartEvent
+  | VersionEndEvent
+  | ArtifactStartEvent
+  | ArtifactEndEvent
+  | FileMatchedEvent
+  | FileSkippedEvent
+  | FileFallbackEvent
+  | ParseStartEvent
+  | ParseEndEvent
+  | ResolveStartEvent
+  | ResolveEndEvent
+  | PipelineErrorEvent;
+
+export type PipelineErrorScope = "pipeline" | "version" | "file" | "route" | "artifact";
+
+export interface PipelineError {
+  scope: PipelineErrorScope;
+  message: string;
+  error?: unknown;
+  file?: FileContext;
+  routeId?: string;
+  artifactId?: string;
+  version?: string;
+}
+
+export type PipelineGraphNodeType = "source" | "file" | "route" | "artifact" | "output";
+
+export type PipelineGraphNode =
+  | { id: string; type: "source"; version: string }
+  | { id: string; type: "file"; file: FileContext }
+  | { id: string; type: "route"; routeId: string }
+  | { id: string; type: "artifact"; artifactId: string }
+  | { id: string; type: "output"; outputIndex: number; property?: string };
+
+export type PipelineGraphEdgeType = "provides" | "matched" | "parsed" | "resolved" | "uses-artifact";
+
+export interface PipelineGraphEdge {
+  from: string;
+  to: string;
+  type: PipelineGraphEdgeType;
+}
+
+export interface PipelineGraph {
+  nodes: PipelineGraphNode[];
+  edges: PipelineGraphEdge[];
+}

packages/pipelines/src/filters.ts

@@ -0,0 +1,58 @@
+import type { FileContext, PipelineFilter } from "./types";
+import picomatch from "picomatch";
+
+export function byName(name: string): PipelineFilter {
+  return (ctx) => ctx.file.name === name;
+}
+
+export function byDir(dir: FileContext["dir"]): PipelineFilter {
+  return (ctx) => ctx.file.dir === dir;
+}
+
+export function byExt(ext: string): PipelineFilter {
+  // Handle empty extension case (files without extension like "Makefile")
+  if (ext === "") {
+    return (ctx) => ctx.file.ext === "";
+  }
+  const normalizedExt = ext.startsWith(".") ? ext : `.${ext}`;
+  return (ctx) => ctx.file.ext === normalizedExt;
+}
+
+export function byGlob(pattern: string): PipelineFilter {
+  const matcher = picomatch(pattern);
+  return (ctx) => matcher(ctx.file.path);
+}
+
+export function byPath(pathPattern: string | RegExp): PipelineFilter {
+  if (typeof pathPattern === "string") {
+    return (ctx) => ctx.file.path === pathPattern;
+  }
+  return (ctx) => pathPattern.test(ctx.file.path);
+}
+
+export function byProp(pattern: string | RegExp): PipelineFilter {
+  if (typeof pattern === "string") {
+    return (ctx) => ctx.row?.property === pattern;
+  }
+  return (ctx) => !!ctx.row?.property && pattern.test(ctx.row.property);
+}
+
+export function and(...filters: PipelineFilter[]): PipelineFilter {
+  return (ctx) => filters.every((f) => f(ctx));
+}
+
+export function or(...filters: PipelineFilter[]): PipelineFilter {
+  return (ctx) => filters.some((f) => f(ctx));
+}
+
+export function not(filter: PipelineFilter): PipelineFilter {
+  return (ctx) => !filter(ctx);
+}
+
+export function always(): PipelineFilter {
+  return () => true;
+}
+
+export function never(): PipelineFilter {
+  return () => false;
+}