Feat: Adds base project
This commit is contained in:
59
infra/ecs_alb/Pulumi.ifsp-assistente-matricula-ecs-alb.yaml
Normal file
59
infra/ecs_alb/Pulumi.ifsp-assistente-matricula-ecs-alb.yaml
Normal file
@@ -0,0 +1,59 @@
|
||||
config:
|
||||
aws:region: us-east-1
|
||||
app-ecs:account_id: "305427701314" # dnxbrasil-nonprod
|
||||
app-ecs:project_name: assistente-analitico
|
||||
app-ecs:environment: dev
|
||||
# app-ecs:bedrock_api_key:
|
||||
# secure: you-can-put-your-pulumi-encrypted-secure-string-here
|
||||
app-ecs:tags:
|
||||
project: assistente-analitico-db-dev
|
||||
env: dev # dev, test, stage, prod
|
||||
account: nonprod # prod, nonprod, dataScience
|
||||
costCenter: AI # AWSGeneral, AI, data, productName
|
||||
owner: AI # team or a preson responsible
|
||||
app-ecs:network:
|
||||
vpc_id: vpc-17ceb96c
|
||||
alb_internal: false
|
||||
alb_subnet_ids: # 2+ private subnets if alb_internal else public subnets in the same region and vpc
|
||||
- subnet-0de9f056635629827
|
||||
- subnet-09cda74f27c543521
|
||||
alb_allow_ingress_cidr:
|
||||
- 3.14.44.224/32
|
||||
ecs_subnet_ids:
|
||||
- subnet-0f50f25a2fbb054d4
|
||||
- subnet-043a427630309c2f4
|
||||
app-ecs:ecs:
|
||||
- task_name: assisnte-analitico-db-dev
|
||||
ecr_repo_name: assistente-analitico-db-dev
|
||||
ecr_image_tag: latest
|
||||
ecr_image_digest: sha256:0bd3a927df4367ba29dbd173e0414d884e973c37599a3f6241341e8d190e827b
|
||||
cpu: 256
|
||||
memory: 512
|
||||
desired_count: 1
|
||||
sgs_allowing_ingress: {}
|
||||
use_load_balancer: true
|
||||
auto_scaling:
|
||||
min_capacity: 1
|
||||
max_capacity: 3
|
||||
target_value: 60.0
|
||||
lb_configs:
|
||||
- name: streamlit
|
||||
listener_port: 8501
|
||||
target_port: 8501
|
||||
container_port: 8501
|
||||
- name: api
|
||||
listener_port: 8000
|
||||
target_port: 8000
|
||||
container_port: 8000
|
||||
env_variables:
|
||||
LANGFUSE_HOST: http://172.31.252.176:3000
|
||||
TABLE: poc_dnx_monthly_summary
|
||||
REGION: us-east-1
|
||||
AWS_ACCOUNT: "305427701314"
|
||||
SECRET_NAME: assistente-db-secrets-manager
|
||||
# SECRET_NAME: dev/ai-pge-doc-classification
|
||||
# BEDROCK_REGION: us-east-1
|
||||
# LANGCHAIN_TRACING_V2: "true"
|
||||
# LANGCHAIN_PROJECT: pge-doc-classification-dev
|
||||
app-ecs:cloudwatch:
|
||||
log_group_name: assistente-analitico-db-dev
|
||||
6
infra/ecs_alb/Pulumi.yaml
Normal file
6
infra/ecs_alb/Pulumi.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
name: app-ecs
|
||||
runtime:
|
||||
name: python
|
||||
options:
|
||||
virtualenv: venv
|
||||
description: AWS ECS application deploy
|
||||
71
infra/ecs_alb/README.md
Normal file
71
infra/ecs_alb/README.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# GenAI project infratructure with Pulumi
|
||||
|
||||
Nesta documentação apresentamos o setup da IaC do projeto de GenAI.
|
||||
|
||||
|
||||
## 🚀 Deploying Infrastructure to AWS
|
||||
|
||||
Para deployar a infraestrutura referente o projeto de GenAI, é utilizado uma stack IaC Pulumi.
|
||||
|
||||
Para isso, o desenvolvedor deve seguir os passos:
|
||||
|
||||
1. Dentro do diretório raiz do projeto, crie um ambiente virtual pelos comandos:
|
||||
```shell
|
||||
# pip install virtualenv
|
||||
python3.11 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
2. (opcional) Configure o seu Pulumi Token:
|
||||
```shell
|
||||
export PULUMI_ACCESS_TOKEN="your-access-token"
|
||||
```
|
||||
|
||||
3. Pulumi Login:
|
||||
```shell
|
||||
pulumi login
|
||||
```
|
||||
|
||||
4. (opcional) Select a stack to work on:
|
||||
```shell
|
||||
pulumi stack select
|
||||
```
|
||||
|
||||
5. Suba a stack do pulumi:
|
||||
```shell
|
||||
pulumi up #selecione o stack se não tiver selecionado
|
||||
```
|
||||
|
||||
Após estes passo, os serviços e dependências do projeto serão criados/atualizados, gerando a fundação para a aplicação.
|
||||
|
||||
#### Observação 1:
|
||||
|
||||
Lembre-se de configurar o usuário IAM registrado como profile no aws-cli, para detectar a conta AWS a ser utilizada:
|
||||
|
||||
```shell
|
||||
export AWS_PROFILE=myorg-nonprod
|
||||
```
|
||||
Se não tiver configurado o profile o aws-cli, exporte as seguintes variáveis:
|
||||
|
||||
```shell
|
||||
export AWS_DEFAULT_REGION=
|
||||
export AWS_ACCESS_KEY_ID=
|
||||
export AWS_SECRET_ACCESS_KEY=
|
||||
```
|
||||
|
||||
#### Observação 2: Registro de imagem Docker no ECR
|
||||
|
||||
Para que a infraestrutura projetada anteriormente consiga referenciar as imagens de cada projeto, é necessário indicar o nome do repositório no argumento `ecr_repo_name` no arquivo YAML.
|
||||
|
||||
Como a versão da imagem a ser deployada, indique ou `ecr_image_tag` ou `ecr_image_digest`, sem indicar ambos.
|
||||
|
||||
Para registrar corretamente a imagem Docker da sua aplicação no ECR, siga os passos descritos na seguinte documentação:
|
||||
|
||||
- https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html
|
||||
|
||||
Para incluir variáveis de ambiente no container, informe-as como dicionário (chave-valor) em `env_variables`.
|
||||
|
||||
## 🤝 Agradecimentos
|
||||
|
||||
* Agradecemos por toda a parceria durante o projeto. Conte conosco! 📢
|
||||
60
infra/ecs_alb/__main__.py
Normal file
60
infra/ecs_alb/__main__.py
Normal file
@@ -0,0 +1,60 @@
|
||||
import pulumi
|
||||
import pulumi_aws as aws
|
||||
import conf as config
|
||||
import iam
|
||||
import ecs
|
||||
|
||||
|
||||
# ECS Cluster Setup
|
||||
app_ecs_cluster = aws.ecs.Cluster(f"{config.project_name}-ecs-cluster",
|
||||
configuration=aws.ecs.ClusterConfigurationArgs(
|
||||
execute_command_configuration=aws.ecs.ClusterConfigurationExecuteCommandConfigurationArgs(
|
||||
logging="DEFAULT",
|
||||
),
|
||||
),
|
||||
settings=[aws.ecs.ClusterSettingArgs(
|
||||
name="containerInsights",
|
||||
value="disabled",
|
||||
)],
|
||||
tags={"Name": f"{config.project_name}-{config.stack_name}"},
|
||||
)
|
||||
|
||||
ecs_cluster_capacity_providers = aws.ecs.ClusterCapacityProviders(f"{config.project_name}-cluster-capacity-providers",
|
||||
cluster_name=app_ecs_cluster.name,
|
||||
capacity_providers=["FARGATE", "FARGATE_SPOT"],
|
||||
)
|
||||
|
||||
# Security Group Setup
|
||||
alb_security_group = aws.ec2.SecurityGroup(f"{config.project_name}-security-group",
|
||||
vpc_id=config.network["vpc_id"],
|
||||
ingress=[aws.ec2.SecurityGroupIngressArgs(
|
||||
protocol="-1",
|
||||
from_port=0,
|
||||
to_port=0,
|
||||
cidr_blocks=config.network["alb_allow_ingress_cidr"],
|
||||
),
|
||||
],
|
||||
egress=[aws.ec2.SecurityGroupEgressArgs(
|
||||
protocol="-1",
|
||||
from_port=0,
|
||||
to_port=0,
|
||||
cidr_blocks=["0.0.0.0/0"],
|
||||
)],
|
||||
)
|
||||
|
||||
# Load Balancer Setup
|
||||
app_load_balancer = aws.lb.LoadBalancer(
|
||||
f"alb-{config.project_name}",
|
||||
load_balancer_type="application",
|
||||
security_groups=[alb_security_group.id],
|
||||
subnets=config.network["alb_subnet_ids"],
|
||||
idle_timeout=(1200),
|
||||
internal=config.network['alb_internal'],
|
||||
)
|
||||
|
||||
for ecs_app in config.ecs:
|
||||
ecs.deploy_app(ecs_app, app_ecs_cluster, alb_security_group, app_load_balancer.arn)
|
||||
|
||||
# Export the ALB DNS Name
|
||||
pulumi.export("url", app_load_balancer.dns_name.apply(lambda dns_name: f"http://{dns_name}"))
|
||||
|
||||
13
infra/ecs_alb/autotag/autotag.py
Normal file
13
infra/ecs_alb/autotag/autotag.py
Normal file
@@ -0,0 +1,13 @@
|
||||
import pulumi
|
||||
from autotag.taggable import is_taggable
|
||||
|
||||
# registerAutoTags registers a global stack transformation that merges a set
|
||||
# of tags with whatever was also explicitly added to the resource definition.
|
||||
def register_auto_tags(auto_tags):
|
||||
pulumi.runtime.register_stack_transformation(lambda args: auto_tag(args, auto_tags))
|
||||
|
||||
# auto_tag applies the given tags to the resource properties if applicable.
|
||||
def auto_tag(args, auto_tags):
|
||||
if is_taggable(args.type_):
|
||||
args.props['tags'] = {**(args.props['tags'] or {}), **auto_tags}
|
||||
return pulumi.ResourceTransformationResult(args.props, args.opts)
|
||||
12
infra/ecs_alb/autotag/policy-config.json
Normal file
12
infra/ecs_alb/autotag/policy-config.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"all": "mandatory",
|
||||
"check-required-tags": {
|
||||
"requiredTags": [
|
||||
"user:project",
|
||||
"user:env",
|
||||
"user:account",
|
||||
"user:costCenter",
|
||||
"user:owner"
|
||||
]
|
||||
}
|
||||
}
|
||||
234
infra/ecs_alb/autotag/taggable.py
Normal file
234
infra/ecs_alb/autotag/taggable.py
Normal file
@@ -0,0 +1,234 @@
|
||||
# isTaggable returns true if the given resource type is an AWS resource that supports tags.
|
||||
def is_taggable(t):
|
||||
return t in taggable_resource_types
|
||||
|
||||
# taggable_resource_types is a list of known AWS type tokens that are taggable.
|
||||
taggable_resource_types = [
|
||||
'aws:accessanalyzer/analyzer:Analyzer',
|
||||
'aws:acm/certificate:Certificate',
|
||||
'aws:acmpca/certificateAuthority:CertificateAuthority',
|
||||
'aws:alb/loadBalancer:LoadBalancer',
|
||||
'aws:alb/targetGroup:TargetGroup',
|
||||
'aws:apigateway/apiKey:ApiKey',
|
||||
'aws:apigateway/clientCertificate:ClientCertificate',
|
||||
'aws:apigateway/domainName:DomainName',
|
||||
'aws:apigateway/restApi:RestApi',
|
||||
'aws:apigateway/stage:Stage',
|
||||
'aws:apigateway/usagePlan:UsagePlan',
|
||||
'aws:apigateway/vpcLink:VpcLink',
|
||||
'aws:applicationloadbalancing/loadBalancer:LoadBalancer',
|
||||
'aws:applicationloadbalancing/targetGroup:TargetGroup',
|
||||
'aws:appmesh/mesh:Mesh',
|
||||
'aws:appmesh/route:Route',
|
||||
'aws:appmesh/virtualNode:VirtualNode',
|
||||
'aws:appmesh/virtualRouter:VirtualRouter',
|
||||
'aws:appmesh/virtualService:VirtualService',
|
||||
'aws:appsync/graphQLApi:GraphQLApi',
|
||||
'aws:athena/workgroup:Workgroup',
|
||||
'aws:autoscaling/group:Group',
|
||||
'aws:backup/plan:Plan',
|
||||
'aws:backup/vault:Vault',
|
||||
'aws:cfg/aggregateAuthorization:AggregateAuthorization',
|
||||
'aws:cfg/configurationAggregator:ConfigurationAggregator',
|
||||
'aws:cfg/rule:Rule',
|
||||
'aws:cloudformation/stack:Stack',
|
||||
'aws:cloudformation/stackSet:StackSet',
|
||||
'aws:cloudfront/distribution:Distribution',
|
||||
'aws:cloudhsmv2/cluster:Cluster',
|
||||
'aws:cloudtrail/trail:Trail',
|
||||
'aws:cloudwatch/eventRule:EventRule',
|
||||
'aws:cloudwatch/logGroup:LogGroup',
|
||||
'aws:cloudwatch/metricAlarm:MetricAlarm',
|
||||
'aws:codebuild/project:Project',
|
||||
'aws:codecommit/repository:Repository',
|
||||
'aws:codepipeline/pipeline:Pipeline',
|
||||
'aws:codepipeline/webhook:Webhook',
|
||||
'aws:codestarnotifications/notificationRule:NotificationRule',
|
||||
'aws:cognito/identityPool:IdentityPool',
|
||||
'aws:cognito/userPool:UserPool',
|
||||
'aws:datapipeline/pipeline:Pipeline',
|
||||
'aws:datasync/agent:Agent',
|
||||
'aws:datasync/efsLocation:EfsLocation',
|
||||
'aws:datasync/locationSmb:LocationSmb',
|
||||
'aws:datasync/nfsLocation:NfsLocation',
|
||||
'aws:datasync/s3Location:S3Location',
|
||||
'aws:datasync/task:Task',
|
||||
'aws:dax/cluster:Cluster',
|
||||
'aws:directconnect/connection:Connection',
|
||||
'aws:directconnect/hostedPrivateVirtualInterfaceAccepter:HostedPrivateVirtualInterfaceAccepter',
|
||||
'aws:directconnect/hostedPublicVirtualInterfaceAccepter:HostedPublicVirtualInterfaceAccepter',
|
||||
'aws:directconnect/hostedTransitVirtualInterfaceAcceptor:HostedTransitVirtualInterfaceAcceptor',
|
||||
'aws:directconnect/linkAggregationGroup:LinkAggregationGroup',
|
||||
'aws:directconnect/privateVirtualInterface:PrivateVirtualInterface',
|
||||
'aws:directconnect/publicVirtualInterface:PublicVirtualInterface',
|
||||
'aws:directconnect/transitVirtualInterface:TransitVirtualInterface',
|
||||
'aws:directoryservice/directory:Directory',
|
||||
'aws:dlm/lifecyclePolicy:LifecyclePolicy',
|
||||
'aws:dms/endpoint:Endpoint',
|
||||
'aws:dms/replicationInstance:ReplicationInstance',
|
||||
'aws:dms/replicationSubnetGroup:ReplicationSubnetGroup',
|
||||
'aws:dms/replicationTask:ReplicationTask',
|
||||
'aws:docdb/cluster:Cluster',
|
||||
'aws:docdb/clusterInstance:ClusterInstance',
|
||||
'aws:docdb/clusterParameterGroup:ClusterParameterGroup',
|
||||
'aws:docdb/subnetGroup:SubnetGroup',
|
||||
'aws:dynamodb/table:Table',
|
||||
'aws:ebs/snapshot:Snapshot',
|
||||
'aws:ebs/snapshotCopy:SnapshotCopy',
|
||||
'aws:ebs/volume:Volume',
|
||||
'aws:ec2/ami:Ami',
|
||||
'aws:ec2/amiCopy:AmiCopy',
|
||||
'aws:ec2/amiFromInstance:AmiFromInstance',
|
||||
'aws:ec2/capacityReservation:CapacityReservation',
|
||||
'aws:ec2/customerGateway:CustomerGateway',
|
||||
'aws:ec2/defaultNetworkAcl:DefaultNetworkAcl',
|
||||
'aws:ec2/defaultRouteTable:DefaultRouteTable',
|
||||
'aws:ec2/defaultSecurityGroup:DefaultSecurityGroup',
|
||||
'aws:ec2/defaultSubnet:DefaultSubnet',
|
||||
'aws:ec2/defaultVpc:DefaultVpc',
|
||||
'aws:ec2/defaultVpcDhcpOptions:DefaultVpcDhcpOptions',
|
||||
'aws:ec2/eip:Eip',
|
||||
'aws:ec2/fleet:Fleet',
|
||||
'aws:ec2/instance:Instance',
|
||||
'aws:ec2/internetGateway:InternetGateway',
|
||||
'aws:ec2/keyPair:KeyPair',
|
||||
'aws:ec2/launchTemplate:LaunchTemplate',
|
||||
'aws:ec2/natGateway:NatGateway',
|
||||
'aws:ec2/networkAcl:NetworkAcl',
|
||||
'aws:ec2/networkInterface:NetworkInterface',
|
||||
'aws:ec2/placementGroup:PlacementGroup',
|
||||
'aws:ec2/routeTable:RouteTable',
|
||||
'aws:ec2/securityGroup:SecurityGroup',
|
||||
'aws:ec2/spotInstanceRequest:SpotInstanceRequest',
|
||||
'aws:ec2/subnet:Subnet',
|
||||
'aws:ec2/vpc:Vpc',
|
||||
'aws:ec2/vpcDhcpOptions:VpcDhcpOptions',
|
||||
'aws:ec2/vpcEndpoint:VpcEndpoint',
|
||||
'aws:ec2/vpcEndpointService:VpcEndpointService',
|
||||
'aws:ec2/vpcPeeringConnection:VpcPeeringConnection',
|
||||
'aws:ec2/vpcPeeringConnectionAccepter:VpcPeeringConnectionAccepter',
|
||||
'aws:ec2/vpnConnection:VpnConnection',
|
||||
'aws:ec2/vpnGateway:VpnGateway',
|
||||
'aws:ec2clientvpn/endpoint:Endpoint',
|
||||
'aws:ec2transitgateway/routeTable:RouteTable',
|
||||
'aws:ec2transitgateway/transitGateway:TransitGateway',
|
||||
'aws:ec2transitgateway/vpcAttachment:VpcAttachment',
|
||||
'aws:ec2transitgateway/vpcAttachmentAccepter:VpcAttachmentAccepter',
|
||||
'aws:ecr/repository:Repository',
|
||||
'aws:ecs/capacityProvider:CapacityProvider',
|
||||
'aws:ecs/cluster:Cluster',
|
||||
'aws:ecs/service:Service',
|
||||
'aws:ecs/taskDefinition:TaskDefinition',
|
||||
'aws:efs/fileSystem:FileSystem',
|
||||
'aws:eks/cluster:Cluster',
|
||||
'aws:eks/fargateProfile:FargateProfile',
|
||||
'aws:eks/nodeGroup:NodeGroup',
|
||||
'aws:elasticache/cluster:Cluster',
|
||||
'aws:elasticache/replicationGroup:ReplicationGroup',
|
||||
'aws:elasticbeanstalk/application:Application',
|
||||
'aws:elasticbeanstalk/applicationVersion:ApplicationVersion',
|
||||
'aws:elasticbeanstalk/environment:Environment',
|
||||
'aws:elasticloadbalancing/loadBalancer:LoadBalancer',
|
||||
'aws:elasticloadbalancingv2/loadBalancer:LoadBalancer',
|
||||
'aws:elasticloadbalancingv2/targetGroup:TargetGroup',
|
||||
'aws:elasticsearch/domain:Domain',
|
||||
'aws:elb/loadBalancer:LoadBalancer',
|
||||
'aws:emr/cluster:Cluster',
|
||||
'aws:fsx/lustreFileSystem:LustreFileSystem',
|
||||
'aws:fsx/windowsFileSystem:WindowsFileSystem',
|
||||
'aws:gamelift/alias:Alias',
|
||||
'aws:gamelift/build:Build',
|
||||
'aws:gamelift/fleet:Fleet',
|
||||
'aws:gamelift/gameSessionQueue:GameSessionQueue',
|
||||
'aws:glacier/vault:Vault',
|
||||
'aws:glue/crawler:Crawler',
|
||||
'aws:glue/job:Job',
|
||||
'aws:glue/trigger:Trigger',
|
||||
'aws:iam/role:Role',
|
||||
'aws:iam/user:User',
|
||||
'aws:inspector/resourceGroup:ResourceGroup',
|
||||
'aws:kinesis/analyticsApplication:AnalyticsApplication',
|
||||
'aws:kinesis/firehoseDeliveryStream:FirehoseDeliveryStream',
|
||||
'aws:kinesis/stream:Stream',
|
||||
'aws:kms/externalKey:ExternalKey',
|
||||
'aws:kms/key:Key',
|
||||
'aws:lambda/function:Function',
|
||||
'aws:lb/loadBalancer:LoadBalancer',
|
||||
'aws:lb/targetGroup:TargetGroup',
|
||||
'aws:licensemanager/licenseConfiguration:LicenseConfiguration',
|
||||
'aws:lightsail/instance:Instance',
|
||||
'aws:mediaconvert/queue:Queue',
|
||||
'aws:mediapackage/channel:Channel',
|
||||
'aws:mediastore/container:Container',
|
||||
'aws:mq/broker:Broker',
|
||||
'aws:mq/configuration:Configuration',
|
||||
'aws:msk/cluster:Cluster',
|
||||
'aws:neptune/cluster:Cluster',
|
||||
'aws:neptune/clusterInstance:ClusterInstance',
|
||||
'aws:neptune/clusterParameterGroup:ClusterParameterGroup',
|
||||
'aws:neptune/eventSubscription:EventSubscription',
|
||||
'aws:neptune/parameterGroup:ParameterGroup',
|
||||
'aws:neptune/subnetGroup:SubnetGroup',
|
||||
'aws:opsworks/stack:Stack',
|
||||
'aws:organizations/account:Account',
|
||||
'aws:pinpoint/app:App',
|
||||
'aws:qldb/ledger:Ledger',
|
||||
'aws:ram/resourceShare:ResourceShare',
|
||||
'aws:rds/cluster:Cluster',
|
||||
'aws:rds/clusterEndpoint:ClusterEndpoint',
|
||||
'aws:rds/clusterInstance:ClusterInstance',
|
||||
'aws:rds/clusterParameterGroup:ClusterParameterGroup',
|
||||
'aws:rds/clusterSnapshot:ClusterSnapshot',
|
||||
'aws:rds/eventSubscription:EventSubscription',
|
||||
'aws:rds/instance:Instance',
|
||||
'aws:rds/optionGroup:OptionGroup',
|
||||
'aws:rds/parameterGroup:ParameterGroup',
|
||||
'aws:rds/securityGroup:SecurityGroup',
|
||||
'aws:rds/snapshot:Snapshot',
|
||||
'aws:rds/subnetGroup:SubnetGroup',
|
||||
'aws:redshift/cluster:Cluster',
|
||||
'aws:redshift/eventSubscription:EventSubscription',
|
||||
'aws:redshift/parameterGroup:ParameterGroup',
|
||||
'aws:redshift/snapshotCopyGrant:SnapshotCopyGrant',
|
||||
'aws:redshift/snapshotSchedule:SnapshotSchedule',
|
||||
'aws:redshift/subnetGroup:SubnetGroup',
|
||||
'aws:resourcegroups/group:Group',
|
||||
'aws:route53/healthCheck:HealthCheck',
|
||||
'aws:route53/resolverEndpoint:ResolverEndpoint',
|
||||
'aws:route53/resolverRule:ResolverRule',
|
||||
'aws:route53/zone:Zone',
|
||||
'aws:s3/bucket:Bucket',
|
||||
'aws:s3/bucketObject:BucketObject',
|
||||
'aws:sagemaker/endpoint:Endpoint',
|
||||
'aws:sagemaker/endpointConfiguration:EndpointConfiguration',
|
||||
'aws:sagemaker/model:Model',
|
||||
'aws:sagemaker/notebookInstance:NotebookInstance',
|
||||
'aws:secretsmanager/secret:Secret',
|
||||
'aws:servicecatalog/portfolio:Portfolio',
|
||||
'aws:sfn/activity:Activity',
|
||||
'aws:sfn/stateMachine:StateMachine',
|
||||
'aws:sns/topic:Topic',
|
||||
'aws:sqs/queue:Queue',
|
||||
'aws:ssm/activation:Activation',
|
||||
'aws:ssm/document:Document',
|
||||
'aws:ssm/maintenanceWindow:MaintenanceWindow',
|
||||
'aws:ssm/parameter:Parameter',
|
||||
'aws:ssm/patchBaseline:PatchBaseline',
|
||||
'aws:storagegateway/cachesIscsiVolume:CachesIscsiVolume',
|
||||
'aws:storagegateway/gateway:Gateway',
|
||||
'aws:storagegateway/nfsFileShare:NfsFileShare',
|
||||
'aws:storagegateway/smbFileShare:SmbFileShare',
|
||||
'aws:swf/domain:Domain',
|
||||
'aws:transfer/server:Server',
|
||||
'aws:transfer/user:User',
|
||||
'aws:waf/rateBasedRule:RateBasedRule',
|
||||
'aws:waf/rule:Rule',
|
||||
'aws:waf/ruleGroup:RuleGroup',
|
||||
'aws:waf/webAcl:WebAcl',
|
||||
'aws:wafregional/rateBasedRule:RateBasedRule',
|
||||
'aws:wafregional/rule:Rule',
|
||||
'aws:wafregional/ruleGroup:RuleGroup',
|
||||
'aws:wafregional/webAcl:WebAcl',
|
||||
'aws:workspaces/directory:Directory',
|
||||
'aws:workspaces/ipGroup:IpGroup',
|
||||
]
|
||||
31
infra/ecs_alb/conf.py
Normal file
31
infra/ecs_alb/conf.py
Normal file
@@ -0,0 +1,31 @@
|
||||
import pulumi
|
||||
import pulumi_aws as aws
|
||||
from autotag.autotag import register_auto_tags
|
||||
|
||||
config = pulumi.Config()
|
||||
|
||||
pulumi_project = pulumi.get_project()
|
||||
|
||||
project_name = config.require("project_name")
|
||||
stack_name = pulumi.get_stack()
|
||||
aws_region = aws.get_region().id
|
||||
|
||||
|
||||
current = aws.get_caller_identity()
|
||||
|
||||
current = aws.get_caller_identity_output()
|
||||
account_id1 = current.account_id
|
||||
|
||||
account_id = config.require("account_id")
|
||||
|
||||
network = config.get_object("network")
|
||||
ecs = config.get_object("ecs")
|
||||
ecr = config.get_object("ecr")
|
||||
environment = config.require("environment")
|
||||
|
||||
register_auto_tags(config.get_object('tags'))
|
||||
|
||||
def get(x):
|
||||
return config.get(x)
|
||||
def get_bool(x):
|
||||
return config.get_bool(x)
|
||||
100
infra/ecs_alb/ecr.py
Normal file
100
infra/ecs_alb/ecr.py
Normal file
@@ -0,0 +1,100 @@
|
||||
import pulumi
|
||||
import pulumi_aws as aws
|
||||
import conf as config
|
||||
import json
|
||||
|
||||
|
||||
def create_ecr_repo():
|
||||
ecr_repositories = []
|
||||
for repo in config.ecr["repos"]:
|
||||
if repo["create_ecr_repo"]:
|
||||
ecr_repository = aws.ecr.Repository(
|
||||
repo,
|
||||
name=f"{repo}",
|
||||
force_delete=True)
|
||||
|
||||
token = aws.ecr.get_authorization_token_output(registry_id=ecr_repository.registry_id)
|
||||
langserve_ecr_life_cycle_policy = aws.ecr.LifecyclePolicy(f"{repo}-ecr-life-cycle-policy",
|
||||
repository=ecr_repository.name,
|
||||
policy=json.dumps({
|
||||
"rules": [{
|
||||
"rulePriority": 1,
|
||||
"description": "Expire images when they are more than 10 available",
|
||||
"selection": {
|
||||
"tagStatus": "any",
|
||||
"countType": "imageCountMoreThan",
|
||||
"countNumber": 10,
|
||||
},
|
||||
"action": {
|
||||
"type": "expire",
|
||||
},
|
||||
}],
|
||||
}))
|
||||
|
||||
policy_ecr = aws.iam.get_policy_document(statements=[{
|
||||
"sid": "new policy",
|
||||
"effect": "Allow",
|
||||
"principals": [{
|
||||
"type": "AWS",
|
||||
"identifiers": [config.account_id],
|
||||
}],
|
||||
"actions": [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability",
|
||||
"ecr:PutImage",
|
||||
"ecr:InitiateLayerUpload",
|
||||
"ecr:UploadLayerPart",
|
||||
"ecr:CompleteLayerUpload",
|
||||
"ecr:DescribeRepositories",
|
||||
"ecr:GetRepositoryPolicy",
|
||||
"ecr:ListImages",
|
||||
"ecr:DeleteRepository",
|
||||
"ecr:BatchDeleteImage",
|
||||
"ecr:SetRepositoryPolicy",
|
||||
"ecr:DeleteRepositoryPolicy",
|
||||
],
|
||||
}])
|
||||
attach_policy = aws.ecr.RepositoryPolicy(f"{repo}-policy_ecr",
|
||||
repository=ecr_repository.name,
|
||||
policy=policy_ecr.json)
|
||||
else:
|
||||
ecr_repository = aws.ecr.get_repository_output(name=repo['name'])
|
||||
token = aws.ecr.get_authorization_token_output(registry_id=ecr_repository.registry_id)
|
||||
|
||||
repo['ecr_repo_resource'] = ecr_repository
|
||||
repo['ecr_token'] = token
|
||||
ecr_repositories.append(repo)
|
||||
|
||||
return ecr_repositories
|
||||
|
||||
def get_image(ecr_repo_name, image_tag=None, image_digest=None):
|
||||
assert (image_tag is not None) != (image_digest is not None), 'User either tag or image_digest, not both, to identify ECR image version.'
|
||||
if image_tag:
|
||||
return aws.ecr.get_image(repository_name=ecr_repo_name, image_tag=image_tag)
|
||||
elif image_digest:
|
||||
return aws.ecr.get_image(repository_name=ecr_repo_name, image_digest=image_digest)
|
||||
|
||||
def build_and_push(ecr_repositories):
|
||||
ecr_repo_images = {}
|
||||
for repo in ecr_repositories:
|
||||
ecr_repo = repo['ecr_repo_resource']
|
||||
container_context = config.get("container-context")
|
||||
if container_context is None:
|
||||
container_context = "."
|
||||
container_file = config.get("container-file")
|
||||
if container_file is None:
|
||||
container_file = "./Dockerfile"
|
||||
|
||||
assert ('tag' in repo.keys()) != ('image_digest' in repo.keys()), 'User must provide either tag or image_digest, but not both, to identify image version'
|
||||
if 'tag' in repo.keys():
|
||||
ecr_image=aws.ecr.get_image(repository_name=ecr_repo.name, image_tag=repo['tag'])
|
||||
elif 'image_digest' in repo.keys():
|
||||
ecr_image=aws.ecr.get_image(repository_name=ecr_repo.name, image_digest=repo['image_digest'])
|
||||
|
||||
repo['ecr_image'] = ecr_image
|
||||
|
||||
ecr_repo_images[repo['name']] = repo
|
||||
|
||||
#ecr_repo_images.append(repo)
|
||||
return ecr_repo_images
|
||||
240
infra/ecs_alb/ecs.py
Normal file
240
infra/ecs_alb/ecs.py
Normal file
@@ -0,0 +1,240 @@
|
||||
import pulumi
|
||||
import pulumi_aws as aws
|
||||
import conf as config
|
||||
import iam
|
||||
import ecr
|
||||
import json
|
||||
|
||||
def deploy_app(config_ecs_app, app_ecs_cluster, alb_security_group, app_load_balancer_arn):
|
||||
lb_configs = config_ecs_app["lb_configs"]
|
||||
target_groups = []
|
||||
load_balancers = []
|
||||
|
||||
for lb_config in lb_configs:
|
||||
tg = aws.lb.TargetGroup(f"app-target-group-{lb_config['listener_port']}",
|
||||
port=lb_config["target_port"],
|
||||
protocol="HTTP",
|
||||
vpc_id=config.network["vpc_id"],
|
||||
target_type="ip",
|
||||
health_check=aws.lb.TargetGroupHealthCheckArgs(
|
||||
path="/",
|
||||
protocol="HTTP",
|
||||
port="traffic-port",
|
||||
healthy_threshold=2,
|
||||
unhealthy_threshold=2,
|
||||
timeout=5,
|
||||
interval=30,
|
||||
matcher="200-499",
|
||||
),
|
||||
)
|
||||
aws.lb.Listener(f"app-listener-{lb_config['listener_port']}",
|
||||
load_balancer_arn=app_load_balancer_arn,
|
||||
port=lb_config["listener_port"],
|
||||
protocol="HTTP",
|
||||
default_actions=[aws.lb.ListenerDefaultActionArgs(
|
||||
type="forward",
|
||||
target_group_arn=tg.arn,
|
||||
)],
|
||||
)
|
||||
target_groups.append(tg)
|
||||
load_balancers.append(aws.ecs.ServiceLoadBalancerArgs(
|
||||
target_group_arn=tg.arn,
|
||||
container_name=f"{config.project_name}-{config_ecs_app['task_name']}-{config.environment}-service",
|
||||
container_port=lb_config["target_port"],
|
||||
))
|
||||
|
||||
# Build and Push ECR
|
||||
# ecr_repos = ecr.create_ecr_repo(config_ecs_app['ecr_repo_name'])
|
||||
# assert ('ecr_image_tag' in config_ecs_app.keys()) != ('ecr_image_digest' in config_ecs_app.keys()), 'User must provide either tag or image_digest, but not both, to identify image version'
|
||||
if 'ecr_image_tag' in config_ecs_app.keys():
|
||||
ecr_repo_image = ecr.get_image(config_ecs_app['ecr_repo_name'], image_tag=config_ecs_app['ecr_image_tag'])
|
||||
elif 'ecr_image_digest' in config_ecs_app.keys():
|
||||
ecr_repo_image = ecr.get_image(config_ecs_app['ecr_repo_name'], image_digest=config_ecs_app['ecr_image_digest'])
|
||||
|
||||
|
||||
# Log Group Setup #TODO move into ecs
|
||||
app_log_group = aws.cloudwatch.LogGroup(f"{config.project_name}-{config_ecs_app['task_name']}-log-group", retention_in_days=7)
|
||||
|
||||
|
||||
# if key
|
||||
# ssm_parameter, key = kms.setup_kms()
|
||||
# iam.create_execution_role_with_keys(ssm_parameter, key)
|
||||
# IAM Roles Setup
|
||||
app_execution_role = iam.create_execution_role()
|
||||
app_task_role = iam.create_task_role()
|
||||
|
||||
# summarization_repo_image = config_ecs_app['ecr_image']
|
||||
|
||||
if config_ecs_app['use_load_balancer']:
|
||||
environemnt_variables = [dict(name=k, value=v) for k,v in config_ecs_app['env_variables'].items()]
|
||||
print(environemnt_variables)
|
||||
# ECS Task Definition Setup
|
||||
app_task_definition = aws.ecs.TaskDefinition(f"{config.project_name}-{config_ecs_app['task_name']}-task-definition",
|
||||
family=f"{config.project_name}-{config_ecs_app['task_name']}-{config.environment}",
|
||||
cpu=config_ecs_app["cpu"],
|
||||
memory=config_ecs_app["memory"],
|
||||
network_mode="awsvpc",
|
||||
execution_role_arn=app_execution_role.arn,
|
||||
task_role_arn=app_task_role.arn,
|
||||
requires_compatibilities=["FARGATE"],
|
||||
container_definitions=pulumi.Output.all(ecr_repo_image.image_uri,
|
||||
# ecr_repo_image.image_digest,
|
||||
app_log_group.name,
|
||||
environemnt_variables,
|
||||
# config_ecs_app["secret_name"],
|
||||
).apply(lambda args: json.dumps([{
|
||||
"name": f"{config.project_name}-{config_ecs_app['task_name']}-{config.environment}-service",
|
||||
"image": args[0],
|
||||
"cpu": 0,
|
||||
"portMappings": [
|
||||
{
|
||||
"name": lb_cfg["name"],
|
||||
"containerPort": lb_cfg["container_port"],
|
||||
"hostPort": lb_cfg["target_port"],
|
||||
"protocol": "tcp",
|
||||
} for lb_cfg in lb_configs
|
||||
],
|
||||
"essential": True,
|
||||
"logConfiguration": {
|
||||
"logDriver": "awslogs",
|
||||
"options": {
|
||||
"awslogs-group": args[1],
|
||||
"awslogs-region": config.aws_region,
|
||||
"awslogs-stream-prefix": "pulumi-langserve",
|
||||
},
|
||||
},
|
||||
"environment": args[2],
|
||||
}])),
|
||||
)
|
||||
|
||||
# ECS Security Group Setup
|
||||
app_ecs_security_group = aws.ec2.SecurityGroup(f"{config.project_name}-{config_ecs_app['task_name']}-ecs-security-group",
|
||||
vpc_id=config.network["vpc_id"],
|
||||
ingress=[aws.ec2.SecurityGroupIngressArgs(
|
||||
protocol="-1",
|
||||
from_port=0,
|
||||
to_port=0,
|
||||
security_groups=[alb_security_group.id],
|
||||
)],
|
||||
egress=[aws.ec2.SecurityGroupEgressArgs(
|
||||
protocol="-1",
|
||||
from_port=0,
|
||||
to_port=0,
|
||||
cidr_blocks=["0.0.0.0/0"],
|
||||
)],
|
||||
)
|
||||
|
||||
# Security Group Rules for Ingress
|
||||
for sg_name, sg_id in config_ecs_app["sgs_allowing_ingress"].items():
|
||||
aws.ec2.SecurityGroupRule(f"sgr-{sg_name}-allow_in_from-{config.project_name}",
|
||||
type="ingress",
|
||||
from_port=0,
|
||||
to_port=0,
|
||||
protocol="-1",
|
||||
security_group_id=sg_id,
|
||||
source_security_group_id=app_ecs_security_group.id,
|
||||
description=f"Allow from {config.project_name} ECS SG",
|
||||
)
|
||||
|
||||
# Service Discovery Namespace Setup
|
||||
app_service_discovery_namespace = aws.servicediscovery.PrivateDnsNamespace(f"{config.project_name}-{config_ecs_app['task_name']}-service-discovery-namespace",
|
||||
name=f"{config.environment}.{config.project_name}.local",
|
||||
vpc=config.network["vpc_id"],
|
||||
)
|
||||
|
||||
# ECS Service Setup
|
||||
app_service = aws.ecs.Service(f"{config.project_name}-{config_ecs_app['task_name']}-service",
|
||||
cluster=app_ecs_cluster.arn,
|
||||
task_definition=app_task_definition.arn,
|
||||
desired_count=config_ecs_app["desired_count"],
|
||||
launch_type="FARGATE",
|
||||
network_configuration=aws.ecs.ServiceNetworkConfigurationArgs(
|
||||
assign_public_ip=True,
|
||||
security_groups=[app_ecs_security_group.id],
|
||||
subnets=config.network["ecs_subnet_ids"],
|
||||
),
|
||||
load_balancers=load_balancers,
|
||||
scheduling_strategy="REPLICA",
|
||||
service_connect_configuration=aws.ecs.ServiceServiceConnectConfigurationArgs(
|
||||
enabled=True,
|
||||
namespace=app_service_discovery_namespace.arn,
|
||||
),
|
||||
tags={"Name": f"{config.project_name}-{config_ecs_app['task_name']}-{config.environment}"},
|
||||
)
|
||||
|
||||
#defining an auto-scaling for summarization
|
||||
scalable_target = aws.appautoscaling.Target("app-svc-target",
|
||||
max_capacity=config_ecs_app['auto_scaling']['max_capacity'],
|
||||
min_capacity=config_ecs_app['auto_scaling']['min_capacity'],
|
||||
resource_id=pulumi.Output.all(app_ecs_cluster.name, app_service.name).apply(lambda args: f"service/{args[0]}/{args[1]}"),
|
||||
scalable_dimension="ecs:service:DesiredCount",
|
||||
service_namespace="ecs",
|
||||
)
|
||||
|
||||
# Define an auto-scaling policy on CPU utilization
|
||||
scaling_policy = aws.appautoscaling.Policy("app-svc-policy",
|
||||
policy_type="TargetTrackingScaling",
|
||||
resource_id=scalable_target.resource_id,
|
||||
scalable_dimension="ecs:service:DesiredCount",
|
||||
service_namespace="ecs",
|
||||
target_tracking_scaling_policy_configuration={
|
||||
"target_value": config_ecs_app['auto_scaling']['target_value'], # Target CPU utilization (30%)
|
||||
"predefined_metric_specification": {
|
||||
"predefined_metric_type": "ECSServiceAverageCPUUtilization"
|
||||
},
|
||||
},
|
||||
)
|
||||
else:
|
||||
# classification_repo_image = ecr_repo_images['ai-med-exam-classification']['ecr_image']
|
||||
# ECS without Load Balancer
|
||||
new_ecs_task_definition = aws.ecs.TaskDefinition("classification-theia-poc-task-definition",
|
||||
family="classification-theia-poc",
|
||||
cpu=config_ecs_app["cpu"],
|
||||
memory=config_ecs_app["memory"],
|
||||
network_mode="awsvpc",
|
||||
execution_role_arn=app_execution_role.arn,
|
||||
task_role_arn=app_task_role.arn,
|
||||
requires_compatibilities=["FARGATE"],
|
||||
container_definitions=pulumi.Output.all(ecr_repo_image.image_uri,
|
||||
# classification_repo_image.image_digest,
|
||||
app_log_group.name).apply(lambda args: json.dumps([{
|
||||
"name": "classification-theia-poc-service",
|
||||
"image": args[0],
|
||||
"cpu": 0,
|
||||
"portMappings": [
|
||||
{
|
||||
"name": "api",
|
||||
"containerPort": 80, # Define the container port without Load Balancer
|
||||
"hostPort": 80,
|
||||
"protocol": "tcp",
|
||||
}
|
||||
],
|
||||
"essential": True,
|
||||
"logConfiguration": {
|
||||
"logDriver": "awslogs",
|
||||
"options": {
|
||||
"awslogs-group": args[1],
|
||||
"awslogs-region": config.aws_region,
|
||||
"awslogs-stream-prefix": "pulumi-classification-theia-poc",
|
||||
},
|
||||
},
|
||||
}])),
|
||||
)
|
||||
# ecs without load balancer
|
||||
new_ecs_service = aws.ecs.Service("classification-theia-poc-service",
|
||||
cluster=app_ecs_cluster.arn,
|
||||
task_definition=new_ecs_task_definition.arn,
|
||||
desired_count=config_ecs_app["desired_count"],
|
||||
launch_type="FARGATE",
|
||||
network_configuration=aws.ecs.ServiceNetworkConfigurationArgs(
|
||||
assign_public_ip=True,
|
||||
security_groups=[app_ecs_security_group.id],
|
||||
subnets=config.network["ecs_subnet_ids"],
|
||||
),
|
||||
scheduling_strategy="REPLICA",
|
||||
service_connect_configuration=aws.ecs.ServiceServiceConnectConfigurationArgs(
|
||||
enabled=True,
|
||||
namespace=app_service_discovery_namespace.arn,
|
||||
),
|
||||
tags={"Name": "classification-theia-poc"},
|
||||
)
|
||||
284
infra/ecs_alb/iam.py
Normal file
284
infra/ecs_alb/iam.py
Normal file
@@ -0,0 +1,284 @@
|
||||
import pulumi
|
||||
import pulumi_aws as aws
|
||||
import conf as config
|
||||
import json
|
||||
|
||||
|
||||
def create_execution_role():
|
||||
execution_role = aws.iam.Role(f"{config.project_name}-execution-role",
|
||||
assume_role_policy=json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Action": "sts:AssumeRole",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": "ecs-tasks.amazonaws.com",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
inline_policies=[aws.iam.RoleInlinePolicyArgs(
|
||||
name=f"{config.project_name}-{config.stack_name}-service-secrets-policy",
|
||||
policy=json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"ecr:GetAuthorizationToken",
|
||||
"ecr:BatchCheckLayerAvailability",
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:GetRepositoryPolicy",
|
||||
"ecr:DescribeRepositories",
|
||||
"ecr:ListImages",
|
||||
"ecr:DescribeImages",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:GetLifecyclePolicy",
|
||||
"ecr:GetLifecyclePolicyPreview",
|
||||
"ecr:ListTagsForResource",
|
||||
"ecr:DescribeImageScanFindings"
|
||||
],
|
||||
"Resource": "*"
|
||||
}
|
||||
],
|
||||
}),
|
||||
)],
|
||||
managed_policy_arns=["arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"])
|
||||
return execution_role
|
||||
|
||||
def create_execution_role_with_keys(ssm_parameter, key):
|
||||
execution_role = aws.iam.Role(f"{config.project_name}-execution-role",
|
||||
assume_role_policy=json.dumps({
|
||||
"Statement": [{
|
||||
"Action": "sts:AssumeRole",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": "ecs-tasks.amazonaws.com",
|
||||
},
|
||||
}],
|
||||
"Version": "2012-10-17",
|
||||
}),
|
||||
inline_policies=[aws.iam.RoleInlinePolicyArgs(
|
||||
name=f"{config.project_name}-{config.stack_name}-service-secrets-policy",
|
||||
policy=pulumi.Output.all(ssm_parameter.arn, key.arn).apply(lambda args: json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Action": ["ssm:GetParameters"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"ssm:ResourceTag/pulumi-application": config.project_name,
|
||||
"ssm:ResourceTag/pulumi-environment": config.stack_name,
|
||||
},
|
||||
},
|
||||
"Effect": "Allow",
|
||||
"Resource": [args[0]],
|
||||
},
|
||||
{
|
||||
"Action": ["kms:Decrypt"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"aws:ResourceTag/pulumi-application": config.project_name,
|
||||
"aws:ResourceTag/pulumi-environment": config.stack_name,
|
||||
},
|
||||
},
|
||||
"Effect": "Allow",
|
||||
"Resource": [args[1]],
|
||||
"Sid": "DecryptTaggedKMSKey",
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"ecr:GetAuthorizationToken",
|
||||
"ecr:BatchCheckLayerAvailability",
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:GetRepositoryPolicy",
|
||||
"ecr:DescribeRepositories",
|
||||
"ecr:ListImages",
|
||||
"ecr:DescribeImages",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:GetLifecyclePolicy",
|
||||
"ecr:GetLifecyclePolicyPreview",
|
||||
"ecr:ListTagsForResource",
|
||||
"ecr:DescribeImageScanFindings"
|
||||
],
|
||||
"Resource": "*"
|
||||
}
|
||||
],
|
||||
})),
|
||||
)],
|
||||
managed_policy_arns=["arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"])
|
||||
return execution_role
|
||||
|
||||
|
||||
def create_task_role():
|
||||
task_role = aws.iam.Role(f"{config.project_name}-task-role",
|
||||
assume_role_policy=json.dumps({
|
||||
"Statement": [{
|
||||
"Action": "sts:AssumeRole",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": "ecs-tasks.amazonaws.com",
|
||||
},
|
||||
}],
|
||||
"Version": "2012-10-17",
|
||||
}),
|
||||
inline_policies=[
|
||||
aws.iam.RoleInlinePolicyArgs(
|
||||
name="ExecuteCommand",
|
||||
policy=json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Action": [
|
||||
"ssmmessages:CreateControlChannel",
|
||||
"ssmmessages:OpenControlChannel",
|
||||
"ssmmessages:CreateDataChannel",
|
||||
"ssmmessages:OpenDataChannel",
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "*",
|
||||
},
|
||||
{
|
||||
"Action": [
|
||||
"logs:CreateLogStream",
|
||||
"logs:DescribeLogGroups",
|
||||
"logs:DescribeLogStreams",
|
||||
"logs:PutLogEvents",
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "*",
|
||||
},{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"athena:StartQueryExecution",
|
||||
"athena:GetQueryExecution",
|
||||
"athena:GetQueryResults",
|
||||
"athena:StopQueryExecution",
|
||||
],
|
||||
"Resource": f"arn:aws:athena:us-east-1:305427701314:workgroup/iceberg-workgroup",
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"glue:GetDatabase",
|
||||
"glue:GetTable",
|
||||
"glue:GetPartitions",
|
||||
],
|
||||
"Resource": [
|
||||
f"arn:aws:glue:us-east-1:305427701314:catalog",
|
||||
f"arn:aws:glue:us-east-1:305427701314:database/dnx_warehouse",
|
||||
f"arn:aws:glue:us-east-1:305427701314:table/dnx_warehouse/*",
|
||||
],
|
||||
}, {
|
||||
"Effect" : "Allow",
|
||||
"Action" : [
|
||||
"secretsmanager:GetSecretValue",
|
||||
"secretsmanager:DescribeSecret"
|
||||
],
|
||||
"Resource" : ["arn:aws:secretsmanager:us-east-1:305427701314:secret:assistente-db-secrets-manager-mpYPMi"
|
||||
]},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"dynamodb:Scan",
|
||||
"dynamodb:GetItem",
|
||||
"dynamodb:Query",
|
||||
"dynamodb:DescribeTable"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:dynamodb:us-east-1:305427701314:table/poc_dnx_monthly_summary",
|
||||
"arn:aws:dynamodb:us-east-1:305427701314:table/poc_dnx_monthly_summary/index/*"
|
||||
]
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
aws.iam.RoleInlinePolicyArgs(
|
||||
name="DenyIAM",
|
||||
policy=json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Action": "iam:*",
|
||||
"Effect": "Deny",
|
||||
"Resource": "*",
|
||||
}],
|
||||
}),
|
||||
),
|
||||
aws.iam.RoleInlinePolicyArgs(
|
||||
name="BedrockS3SQSAccess",
|
||||
policy=json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
# S3
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:*",
|
||||
"s3-object-lambda:*"
|
||||
],
|
||||
"Resource": "*"
|
||||
},
|
||||
# SQS
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"sqs:StartMessageMoveTask",
|
||||
"sqs:DeleteMessage",
|
||||
"sqs:GetQueueUrl",
|
||||
"sqs:ListDeadLetterSourceQueues",
|
||||
"sqs:ListMessageMoveTasks",
|
||||
"sqs:PurgeQueue",
|
||||
"sqs:ReceiveMessage",
|
||||
"sqs:GetQueueAttributes",
|
||||
"sqs:ListQueueTags"
|
||||
],
|
||||
"Resource": "arn:aws:sqs:us-east-1:673991670544:ai-med-dev-queue-63cb463"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "sqs:ListQueues",
|
||||
"Resource": "*"
|
||||
},
|
||||
# Bedrock
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"bedrock:*"
|
||||
],
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"kms:DescribeKey"
|
||||
],
|
||||
"Resource": "arn:*:kms:*:*:key/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"iam:ListRoles",
|
||||
"ec2:DescribeVpcs",
|
||||
"ec2:DescribeSubnets",
|
||||
"ec2:DescribeSecurityGroups"
|
||||
],
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"iam:PassRole"
|
||||
],
|
||||
"Resource": "arn:aws:iam::*:role/*AmazonBedrock*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"iam:PassedToService": "bedrock.amazonaws.com"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
),
|
||||
])
|
||||
return task_role
|
||||
74
infra/ecs_alb/kms.py
Normal file
74
infra/ecs_alb/kms.py
Normal file
@@ -0,0 +1,74 @@
|
||||
import pulumi
|
||||
import pulumi_aws as aws
|
||||
import pulumi_docker as docker
|
||||
import conf as config
|
||||
import json
|
||||
|
||||
|
||||
def setup_kms():
|
||||
# KMS Key Setup
|
||||
app_key = aws.kms.Key(f"{config.project_name}-key",
|
||||
description="Key for encrypting secrets",
|
||||
enable_key_rotation=True,
|
||||
policy=json.dumps({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Sid": "",
|
||||
"Principal": {
|
||||
"AWS": f"arn:aws:iam::{config.account_id}:root",
|
||||
},
|
||||
"Action": [
|
||||
"kms:Create*", "kms:Describe*", "kms:Enable*", "kms:List*", "kms:Put*", "kms:Update*",
|
||||
"kms:Revoke*", "kms:Disable*", "kms:Get*", "kms:Delete*", "kms:ScheduleKeyDeletion",
|
||||
"kms:CancelKeyDeletion", "kms:Tag*", "kms:UntagResource",
|
||||
],
|
||||
"Resource": "*",
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": f"arn:aws:iam::{config.account_id}:root",
|
||||
},
|
||||
"Action": [
|
||||
"kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey",
|
||||
],
|
||||
"Resource": "*",
|
||||
},
|
||||
{
|
||||
"Sid": 'Allow access to EFS for all principals in the account that are authorized to use EFS',
|
||||
"Effect": 'Allow',
|
||||
"Principal": {"AWS": "*"},
|
||||
"Action": [
|
||||
"kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*",
|
||||
"kms:CreateGrant", "kms:DescribeKey",
|
||||
],
|
||||
"Resource": "*",
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"kms:ViaService": f"elasticfilesystem.{config.aws_region}.amazonaws.com",
|
||||
"kms:CallerAccount": config.account_id,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
tags={
|
||||
"pulumi-application": config.project_name,
|
||||
"pulumi-environment": config.stack_name,
|
||||
},
|
||||
)
|
||||
|
||||
# SSM Parameter Setup
|
||||
app_ssm_parameter = aws.ssm.Parameter(f"{config.project_name}-ssm-parameter",
|
||||
type="SecureString",
|
||||
value=config.config.require_secret("bedrock_api_key"),
|
||||
key_id=app_key.key_id,
|
||||
name=f"/{config.project_name}/{config.stack_name}/BEDROCK_API_KEY",
|
||||
tags={
|
||||
"pulumi-application": config.project_name,
|
||||
"pulumi-environment": config.stack_name,
|
||||
},
|
||||
)
|
||||
return app_ssm_parameter, app_key
|
||||
5
infra/ecs_alb/requirements.txt
Normal file
5
infra/ecs_alb/requirements.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
pulumi
|
||||
pulumi-aws
|
||||
pulumi-docker
|
||||
boto3
|
||||
setuptools
|
||||
Reference in New Issue
Block a user