You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
188 lines
4.3 KiB
188 lines
4.3 KiB
{ |
|
"nodes": [ |
|
{ |
|
"id": "data-source", |
|
"type": "webserver", |
|
"position": { "x": 100, "y": 200 }, |
|
"props": { |
|
"label": "Data Ingestion API", |
|
"rpsCapacity": 500 |
|
} |
|
}, |
|
{ |
|
"id": "raw-data-queue", |
|
"type": "messageQueue", |
|
"position": { "x": 300, "y": 200 }, |
|
"props": { |
|
"label": "Raw Data Queue", |
|
"queueCapacity": 10000, |
|
"retentionSeconds": 3600, |
|
"processingRate": 200 |
|
} |
|
}, |
|
{ |
|
"id": "etl-pipeline-1", |
|
"type": "data pipeline", |
|
"position": { "x": 500, "y": 150 }, |
|
"props": { |
|
"label": "Data Cleansing Pipeline", |
|
"batchSize": 100, |
|
"transformation": "validate" |
|
} |
|
}, |
|
{ |
|
"id": "etl-pipeline-2", |
|
"type": "data pipeline", |
|
"position": { "x": 500, "y": 250 }, |
|
"props": { |
|
"label": "Data Transformation Pipeline", |
|
"batchSize": 50, |
|
"transformation": "aggregate" |
|
} |
|
}, |
|
{ |
|
"id": "ml-pipeline", |
|
"type": "data pipeline", |
|
"position": { "x": 700, "y": 150 }, |
|
"props": { |
|
"label": "ML Feature Pipeline", |
|
"batchSize": 200, |
|
"transformation": "enrich" |
|
} |
|
}, |
|
{ |
|
"id": "analytics-pipeline", |
|
"type": "data pipeline", |
|
"position": { "x": 700, "y": 250 }, |
|
"props": { |
|
"label": "Analytics Pipeline", |
|
"batchSize": 500, |
|
"transformation": "join" |
|
} |
|
}, |
|
{ |
|
"id": "cache-1", |
|
"type": "cache", |
|
"position": { "x": 900, "y": 150 }, |
|
"props": { |
|
"label": "Feature Cache", |
|
"cacheTTL": 300, |
|
"maxEntries": 50000, |
|
"evictionPolicy": "LRU" |
|
} |
|
}, |
|
{ |
|
"id": "data-warehouse", |
|
"type": "database", |
|
"position": { "x": 900, "y": 250 }, |
|
"props": { |
|
"label": "Data Warehouse", |
|
"replication": 3, |
|
"maxRPS": 1000, |
|
"baseLatencyMs": 50 |
|
} |
|
}, |
|
{ |
|
"id": "monitoring-1", |
|
"type": "monitoring/alerting", |
|
"position": { "x": 500, "y": 350 }, |
|
"props": { |
|
"label": "Pipeline Monitor", |
|
"tool": "Datadog", |
|
"alertMetric": "latency", |
|
"thresholdValue": 1000, |
|
"thresholdUnit": "ms" |
|
} |
|
}, |
|
{ |
|
"id": "compression-pipeline", |
|
"type": "data pipeline", |
|
"position": { "x": 300, "y": 350 }, |
|
"props": { |
|
"label": "Data Compression", |
|
"batchSize": 1000, |
|
"transformation": "compress" |
|
} |
|
} |
|
], |
|
"connections": [ |
|
{ |
|
"source": "data-source", |
|
"target": "raw-data-queue", |
|
"label": "Raw Data Stream", |
|
"protocol": "http" |
|
}, |
|
{ |
|
"source": "raw-data-queue", |
|
"target": "etl-pipeline-1", |
|
"label": "Data Validation", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "raw-data-queue", |
|
"target": "etl-pipeline-2", |
|
"label": "Data Transformation", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "etl-pipeline-1", |
|
"target": "ml-pipeline", |
|
"label": "Clean Data", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "etl-pipeline-2", |
|
"target": "analytics-pipeline", |
|
"label": "Transformed Data", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "ml-pipeline", |
|
"target": "cache-1", |
|
"label": "ML Features", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "analytics-pipeline", |
|
"target": "data-warehouse", |
|
"label": "Analytics Data", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "etl-pipeline-1", |
|
"target": "monitoring-1", |
|
"label": "Pipeline Metrics", |
|
"protocol": "http" |
|
}, |
|
{ |
|
"source": "etl-pipeline-2", |
|
"target": "monitoring-1", |
|
"label": "Pipeline Metrics", |
|
"protocol": "http" |
|
}, |
|
{ |
|
"source": "ml-pipeline", |
|
"target": "monitoring-1", |
|
"label": "Pipeline Metrics", |
|
"protocol": "http" |
|
}, |
|
{ |
|
"source": "analytics-pipeline", |
|
"target": "monitoring-1", |
|
"label": "Pipeline Metrics", |
|
"protocol": "http" |
|
}, |
|
{ |
|
"source": "raw-data-queue", |
|
"target": "compression-pipeline", |
|
"label": "Archive Stream", |
|
"protocol": "tcp" |
|
}, |
|
{ |
|
"source": "compression-pipeline", |
|
"target": "data-warehouse", |
|
"label": "Compressed Archive", |
|
"protocol": "tcp" |
|
} |
|
] |
|
}
|
|
|