#!/usr/bin/env python3 """ update-workflows.py Generate GitHub Actions workflow files for worker deployment. Worker definitions are loaded from config/workers.json (single source of truth). Authors: Mark Smith Copyright (c) 2022-2025 by Dreamwidth Studios, LLC. This program is free software; you may redistribute it and/or modify it under the same terms as Perl itself. For a copy of the license, please reference 'perldoc perlartistic' or 'perldoc perlgpl'. """ import json from pathlib import Path # Paths script_dir = Path(__file__).parent config_file = script_dir / "workers.json" workflows_dir = script_dir.parent / ".github" / "workflows" tasks_dir = workflows_dir / "tasks" # Load worker definitions from JSON with open(config_file) as f: config = json.load(f) workers = config["workers"] worker_names = sorted(workers.keys()) print(f"Loaded {len(workers)} workers from {config_file}") def generate_workflow(image_name, workflow_name): """Generate a worker deploy workflow file.""" # Build options list options = ["- ALL WORKERS (*)"] for name in worker_names: options.append(f"- {name}") options_str = "\n ".join(options) # Build render steps render_steps = [] for name in worker_names: render_steps.append(f""" - name: ({name}) Render Amazon ECS task definition if: github.event.inputs.service == 'ALL WORKERS (*)' || github.event.inputs.service == '{name}' id: render-worker-container-{name} uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: ".github/workflows/tasks/worker-{name}-service.json" container-name: ${{{{ env.CONTAINER_NAME }}}} image: "${{{{ env.IMAGE_BASE }}}}@${{{{ github.event.inputs.tag }}}}" """) # Build deploy steps deploy_steps = [] for name in worker_names: deploy_steps.append(f""" - name: ({name}) Deploy to Amazon ECS service if: github.event.inputs.service == 'ALL WORKERS (*)' || github.event.inputs.service == '{name}' uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: task-definition: ${{{{ steps.render-worker-container-{name}.outputs.task-definition }}}} cluster: ${{{{ env.ECS_CLUSTER }}}} service: "worker-{name}-service" """) workflow = f"""# # AUTO-GENERATED by config/update-workflows.py. DO NOT EDIT. # name: (deploy) {workflow_name} on: workflow_dispatch: inputs: service: type: choice description: Which service to deploy options: {options_str} tag: type: string description: SHA256 to deploy (include "sha256:" prefix) required: true env: REGION: us-east-1 ECS_CLUSTER: dreamwidth CONTAINER_NAME: worker IMAGE_BASE: ghcr.io/dreamwidth/{image_name} jobs: deploy: if: github.repository == 'dreamwidth/dreamwidth' runs-on: ubuntu-latest steps: - name: Checkout Code uses: actions/checkout@v3 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{{{ secrets.AWS_ACCESS_KEY_ID }}}} aws-secret-access-key: ${{{{ secrets.AWS_SECRET_ACCESS_KEY }}}} aws-region: ${{{{ env.REGION }}}} {"".join(render_steps).rstrip()} {"".join(deploy_steps).rstrip()} - name: Notify Discord uses: sarisia/actions-status-discord@v1 if: always() with: title: "${{{{ github.event.inputs.service }}}} DEPLOY STARTED" description: "Deploying `${{{{ github.event.inputs.tag }}}}` to `${{{{ github.event.inputs.service }}}}`\\n\\nClick the header above to watch the deployment progress." url: "https://${{{{ env.REGION }}}}.console.aws.amazon.com/ecs/v2/clusters/dreamwidth/services?region=${{{{ env.REGION }}}}" webhook: ${{{{ secrets.DISCORD_WEBHOOK }}}} nocontext: true """ return workflow def generate_task_definition(name, cpu, memory): """Generate a task definition JSON file for a worker.""" task = { "containerDefinitions": [ { "name": "worker", "image": "ghcr.io/dreamwidth/worker:latest", "cpu": 0, "portMappings": [], "essential": True, "command": [ "bash", "/opt/startup-prod.sh", f"bin/worker/{name}", "-v" ], "environment": [], "mountPoints": [ { "sourceVolume": "dw-config", "containerPath": "/dw/etc", "readOnly": True } ], "volumesFrom": [], "linuxParameters": { "initProcessEnabled": True }, "logConfiguration": { "logDriver": "awslogs", "options": { "awslogs-create-group": "true", "awslogs-group": f"/dreamwidth/worker/{name}", "awslogs-region": "us-east-1", "awslogs-stream-prefix": "worker" } } } ], "family": f"worker-{name}", "taskRoleArn": "arn:aws:iam::194396987458:role/dreamwidth-ecsTaskRole", "executionRoleArn": "arn:aws:iam::194396987458:role/dreamwidth-ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { "name": "dw-config", "efsVolumeConfiguration": { "fileSystemId": "fs-f9f3e04d", "rootDirectory": "/etc-workers", "transitEncryption": "DISABLED" } } ], "requiresCompatibilities": ["FARGATE"], "cpu": str(cpu), "memory": str(memory) } return json.dumps(task, indent=4) # Generate deployment workflows for image_name, workflow_name, filename in [ ("worker22", "workers (22.04)", "worker22-deploy.yml"), ]: print(f"Generating {workflows_dir / filename}...") workflow_content = generate_workflow(image_name, workflow_name) with open(workflows_dir / filename, "w") as f: f.write(workflow_content) print(f" Written {len(worker_names)} worker entries") # Generate task definitions print(f"Generating task definitions in {tasks_dir}...") tasks_dir.mkdir(exist_ok=True) for name in worker_names: w = workers[name] task_content = generate_task_definition(name, w["cpu"], w["memory"]) with open(tasks_dir / f"worker-{name}-service.json", "w") as f: f.write(task_content) f.write("\n") print(f" Written {len(worker_names)} task definition files") print("\nDone!")