140 lines
4.4 KiB
JavaScript
140 lines
4.4 KiB
JavaScript
import { describe, it, expect } from 'vitest';
|
|
import * as yaml from 'js-yaml';
|
|
|
|
/**
|
|
* Helper to serialize canvas topology to Airflow YAML specification.
|
|
* Directly mirrors StudioPage.jsx generateCanvasYaml implementation.
|
|
*/
|
|
function generateCanvasYaml({ nodes, edges, pipelineId, schedule, bundleName, description }) {
|
|
const taskNodes = nodes.filter((n) => n.type !== 'noteNode');
|
|
if (taskNodes.length === 0) {
|
|
return 'dag_id: ' + (pipelineId || 'example') + '\ntasks: []';
|
|
}
|
|
|
|
const tasks = taskNodes.map((n) => {
|
|
const nodeType = n.data?.type || n.type || 'source_db';
|
|
const config = n.data?.params || n.data?.config || {};
|
|
const targetEdges = edges.filter((e) => (e.target || e[1]) === n.id);
|
|
const dependencies = targetEdges.map((e) => e.source || e[0]);
|
|
const isDisabled = n.data?.disabled === true || n.data?.enabled === false;
|
|
|
|
return {
|
|
task_id: n.id,
|
|
type: isDisabled ? 'empty' : (n.data?.yamlType || nodeType),
|
|
label: n.data?.label || n.data?.title || n.id,
|
|
enabled: !isDisabled,
|
|
...(dependencies.length > 0 ? { dependencies } : {}),
|
|
config: isDisabled ? {} : config,
|
|
};
|
|
});
|
|
|
|
const pipelineSpec = {
|
|
dag_id: pipelineId || 'example_pipeline',
|
|
schedule: schedule || '@hourly',
|
|
bundle: bundleName || 'dlh-core-ingestion',
|
|
description: description || '',
|
|
tasks,
|
|
};
|
|
|
|
return yaml.dump(pipelineSpec);
|
|
}
|
|
|
|
describe('yamlCodec.test.js - Airflow YAML Generator and Codec', () => {
|
|
it('1. Generates clean YAML with dag_id, schedule, and tasks from canvas graph', () => {
|
|
const nodes = [
|
|
{
|
|
id: 'node-db',
|
|
type: 'customNode',
|
|
data: {
|
|
label: 'Source Database',
|
|
type: 'source_db',
|
|
yamlType: 'source_db',
|
|
params: { conn: 'Postgres', table: 'permits' },
|
|
},
|
|
},
|
|
{
|
|
id: 'node-dbt',
|
|
type: 'customNode',
|
|
data: {
|
|
label: 'dbt Transform',
|
|
type: 'dbt_transform',
|
|
yamlType: 'dbt_transform',
|
|
params: { sel: 'gold', tgt: 'postgres' },
|
|
},
|
|
},
|
|
];
|
|
|
|
const edges = [{ source: 'node-db', target: 'node-dbt' }];
|
|
|
|
const yamlOutput = generateCanvasYaml({
|
|
nodes,
|
|
edges,
|
|
pipelineId: 'permits_pipeline',
|
|
schedule: '0 */2 * * *',
|
|
});
|
|
|
|
expect(yamlOutput).toContain('dag_id: permits_pipeline');
|
|
expect(yamlOutput).toMatch(/schedule:\s*"?0 \*\/2 \* \* \*"?/);
|
|
expect(yamlOutput).toContain('task_id: node-db');
|
|
expect(yamlOutput).toContain('type: source_db');
|
|
expect(yamlOutput).toContain('task_id: node-dbt');
|
|
expect(yamlOutput).toContain('type: dbt_transform');
|
|
expect(yamlOutput).toContain('dependencies:');
|
|
expect(yamlOutput).toContain('- node-db');
|
|
|
|
// Parse back to confirm valid YAML syntax
|
|
const parsed = yaml.load(yamlOutput);
|
|
expect(parsed.dag_id).toBe('permits_pipeline');
|
|
expect(parsed.tasks).toHaveLength(2);
|
|
expect(parsed.tasks[1].dependencies).toContain('node-db');
|
|
expect(parsed.tasks[0].config.table).toBe('permits');
|
|
});
|
|
|
|
it('2. Maps disabled task to type: empty and enabled: false per Decision #1', () => {
|
|
const nodes = [
|
|
{
|
|
id: 'node-disabled',
|
|
type: 'customNode',
|
|
data: {
|
|
label: 'Disabled Task',
|
|
type: 'ge_quality',
|
|
yamlType: 'ge_quality',
|
|
disabled: true,
|
|
enabled: false,
|
|
params: { suite: 'bronze_suite' },
|
|
},
|
|
},
|
|
];
|
|
|
|
const yamlOutput = generateCanvasYaml({ nodes, edges: [], pipelineId: 'disabled_test' });
|
|
const parsed = yaml.load(yamlOutput);
|
|
|
|
expect(parsed.tasks[0].type).toBe('empty');
|
|
expect(parsed.tasks[0].enabled).toBe(false);
|
|
expect(parsed.tasks[0].config).toEqual({});
|
|
});
|
|
|
|
it('3. Strictly omits note nodes from YAML output per data-model.md', () => {
|
|
const nodes = [
|
|
{
|
|
id: 'note-1',
|
|
type: 'noteNode',
|
|
data: { text: 'Architecture notes\nInternal details' },
|
|
},
|
|
{
|
|
id: 'node-task',
|
|
type: 'customNode',
|
|
data: { label: 'Real Task', type: 'source_file', params: { path: 'data/' } },
|
|
},
|
|
];
|
|
|
|
const yamlOutput = generateCanvasYaml({ nodes, edges: [], pipelineId: 'notes_test' });
|
|
const parsed = yaml.load(yamlOutput);
|
|
|
|
expect(parsed.tasks).toHaveLength(1);
|
|
expect(parsed.tasks[0].task_id).toBe('node-task');
|
|
expect(yamlOutput).not.toContain('note-1');
|
|
expect(yamlOutput).not.toContain('Architecture notes');
|
|
});
|
|
});
|