Adds langfuse ec2 and api key to code

infra/ecs_alb/main.tf

@@ -202,13 +202,23 @@ resource "aws_iam_role_policy" "s3_policy" {
 
   policy = jsonencode({
     Version = "2012-10-17"
-    Statement = [{
-      Effect = "Allow"
-      Action = [
-        "s3:GetObject"
-      ]
-      Resource = "arn:aws:s3:::upflux-doc-analyzer/*"
-    }]
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:PutObject"
+        ]
+        Resource = "arn:aws:s3:::upflux-doc-analyzer/*"
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "s3:DeleteObject"
+        ]
+        Resource = "arn:aws:s3:::upflux-doc-analyzer/temp_textract/*"
+      }
+    ]
   })
 }
 
@@ -229,6 +239,23 @@ resource "aws_iam_role_policy" "textract_policy" {
     }]
   })
 }
+
+resource "aws_iam_role_policy" "secrets_manager_policy" {
+  name = "${var.app_name}-secrets-manager-policy"
+  role = aws_iam_role.ecs_task_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Action = [
+        "secretsmanager:GetSecretValue"
+      ]
+      Resource = "*"
+    }]
+  })
+}
+
 # ECS Task Definition
 resource "aws_ecs_task_definition" "app" {
   family                   = var.app_name
@@ -242,6 +269,13 @@ resource "aws_ecs_task_definition" "app" {
     name  = var.app_name
     image = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com/${var.ecr_repository_name}:${var.image_tag}"
     
+    environment = [
+      {
+        name  = "LANGFUSE_HOST"
+        value = var.langfuse_host
+      }
+    ]
+
     portMappings = [{
       containerPort = 8000
       hostPort      = 8000

infra/ecs_alb/terraform.tfvars

@@ -20,4 +20,5 @@ ecr_repository_name = "upflux-doc-analyser"
 image_tag           = "latest"
 fargate_cpu         = "256"
 fargate_memory      = "512"
-app_count           = 1
+app_count           = 1
+langfuse_host       = "http://10.0.0.12:3000"

infra/ecs_alb/variables.tf

@@ -53,4 +53,9 @@ variable "app_count" {
   description = "Number of tasks to run"
   type        = number
   default     = 1
+}
+
+variable "langfuse_host" {
+  description = "Langfuse host URL"
+  type        = string
 }

infra/langfuse-terraform/data.tf

@@ -0,0 +1,14 @@
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+}

infra/langfuse-terraform/main.tf

@@ -0,0 +1,57 @@
+# ──────────────────────────────────────────────
+# Security Group
+# ──────────────────────────────────────────────
+resource "aws_security_group" "langfuse" {
+  name        = var.sg_name
+  description = "Allow defined ports for Langfuse"
+  vpc_id      = var.vpc_id
+
+  dynamic "ingress" {
+    for_each = var.allowed_ports
+    content {
+      from_port   = ingress.value
+      to_port     = ingress.value
+      protocol    = "tcp"
+      cidr_blocks = ["3.14.44.224/32"]
+    }
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = merge(var.tags, {
+    Name = var.sg_name
+  })
+}
+
+# ──────────────────────────────────────────────
+# EC2 Instance
+# ──────────────────────────────────────────────
+resource "aws_instance" "langfuse" {
+  ami                         = data.aws_ami.ubuntu.id
+  instance_type               = var.instance_type
+  subnet_id                   = var.subnet_id
+  vpc_security_group_ids      = [aws_security_group.langfuse.id]
+  associate_public_ip_address = true
+  key_name                    = var.key_name != "" ? var.key_name : null
+
+  user_data = templatefile("${path.module}/user_data.sh.tftpl", {
+    langfuse_repo_url = var.langfuse_repo_url
+    langfuse_web_port = var.langfuse_web_port
+    ebs_device_name   = var.ebs_device_name
+  })
+
+  root_block_device {
+    volume_size           = var.root_volume_size
+    volume_type           = var.root_volume_type
+    delete_on_termination = true
+  }
+
+  tags = merge(var.tags, {
+    Name = var.instance_name
+  })
+}

infra/langfuse-terraform/outputs.tf

@@ -0,0 +1,9 @@
+output "instance_ip" {
+  description = "Public IP of the Langfuse EC2 instance"
+  value       = aws_instance.langfuse.public_ip
+}
+
+output "url" {
+  description = "Langfuse web UI URL"
+  value       = "http://${aws_instance.langfuse.public_ip}:${var.langfuse_web_port}"
+}

infra/langfuse-terraform/providers.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.aws_region
+}

infra/langfuse-terraform/terraform.tfvars

@@ -0,0 +1,36 @@
+# ──────────────────────────────────────────────
+# General
+# ──────────────────────────────────────────────
+aws_region   = "us-east-2"
+project_name = "langfuse"
+environment  = "dev"
+
+tags = {
+  project    = "doc-processor"
+  env        = "dev"
+  costCenter = "AI"
+  owner      = "ai-team"
+}
+
+# ──────────────────────────────────────────────
+# Network
+# ──────────────────────────────────────────────
+vpc_id    = "vpc-0270f02aee3bf1b8d"
+subnet_id = "subnet-088bc49c54ec8f028" # public-us-east-1a-subnet
+
+# ──────────────────────────────────────────────
+# EC2
+# ──────────────────────────────────────────────
+instance_type    = "t3.xlarge"
+instance_name    = "LangfuseEC2"
+sg_name          = "langfuse-sg"
+allowed_ports    = [22, 80, 443, 3000]
+root_volume_size = 100
+root_volume_type = "gp2"
+ebs_device_name  = "/dev/sdf"
+
+# ──────────────────────────────────────────────
+# Langfuse
+# ──────────────────────────────────────────────
+langfuse_repo_url = "https://github.com/langfuse/langfuse.git"
+langfuse_web_port = 3000

infra/langfuse-terraform/user_data.sh.tftpl

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+# Install Docker
+sudo apt-get update -y
+sudo apt-get install -y ca-certificates curl gnupg git
+sudo install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+sudo chmod a+r /etc/apt/keyrings/docker.gpg
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update -y
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+sudo groupadd docker || true
+sudo usermod -aG docker ubuntu
+sudo chmod 666 /var/run/docker.sock
+sudo systemctl enable docker
+sudo systemctl restart docker
+
+# Clone and configure Langfuse
+cd /opt
+git clone ${langfuse_repo_url}
+cd langfuse
+
+NEXTAUTH_SECRET=$(openssl rand -hex 32)
+PUBLIC_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+SALT=$(openssl rand -hex 16)
+ENCRYPTION_KEY=$(openssl rand -hex 32)
+
+cat > .env <<EOF
+NEXTAUTH_SECRET=$NEXTAUTH_SECRET
+NEXTAUTH_URL=http://$PUBLIC_IP:${langfuse_web_port}
+DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres
+CLICKHOUSE_URL=http://clickhouse:8123
+CLICKHOUSE_USER=clickhouse
+CLICKHOUSE_PASSWORD=clickhouse
+TELEMETRY_ENABLED=false
+SALT=$SALT
+ENCRYPTION_KEY=$ENCRYPTION_KEY
+REDIS_AUTH=myredissecret
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=miniosecret
+EOF
+
+sudo docker compose -f docker-compose.yml up -d
+
+# Mount additional EBS volume
+DEVICE="${ebs_device_name}"
+MOUNT_DIR="/mnt/langfuse-data"
+
+if [ -b "$DEVICE" ]; then
+  sudo mkfs -t ext4 $DEVICE
+  sudo mkdir -p $MOUNT_DIR
+  sudo mount $DEVICE $MOUNT_DIR
+  echo "$DEVICE $MOUNT_DIR ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
+else
+  echo "Volume $DEVICE not found."
+fi

infra/langfuse-terraform/variables.tf

@@ -0,0 +1,103 @@
+# ──────────────────────────────────────────────
+# General
+# ──────────────────────────────────────────────
+variable "aws_region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "project_name" {
+  description = "Project name used for resource naming"
+  type        = string
+}
+
+variable "environment" {
+  description = "Environment (dev, staging, prod)"
+  type        = string
+}
+
+variable "tags" {
+  description = "Tags applied to all resources"
+  type        = map(string)
+  default     = {}
+}
+
+# ──────────────────────────────────────────────
+# Network
+# ──────────────────────────────────────────────
+variable "vpc_id" {
+  description = "VPC ID where resources will be created"
+  type        = string
+}
+
+variable "subnet_id" {
+  description = "Subnet ID for the EC2 instance (must be public for associate_public_ip)"
+  type        = string
+}
+
+# ──────────────────────────────────────────────
+# EC2
+# ──────────────────────────────────────────────
+variable "instance_type" {
+  description = "EC2 instance type (langfuse requires at least t3.xlarge)"
+  type        = string
+  default     = "t3.xlarge"
+}
+
+variable "instance_name" {
+  description = "Name tag for the EC2 instance"
+  type        = string
+  default     = "LangfuseEC2"
+}
+
+variable "key_name" {
+  description = "EC2 key pair name for SSH access (optional)"
+  type        = string
+  default     = ""
+}
+
+variable "sg_name" {
+  description = "Security group name"
+  type        = string
+  default     = "langfuse-sg"
+}
+
+variable "allowed_ports" {
+  description = "List of TCP ports to allow inbound"
+  type        = list(number)
+  default     = [22, 80, 443, 3000]
+}
+
+variable "root_volume_size" {
+  description = "Root EBS volume size in GB"
+  type        = number
+  default     = 100
+}
+
+variable "root_volume_type" {
+  description = "Root EBS volume type"
+  type        = string
+  default     = "gp2"
+}
+
+variable "ebs_device_name" {
+  description = "Device name for the additional EBS volume"
+  type        = string
+  default     = "/dev/sdf"
+}
+
+# ──────────────────────────────────────────────
+# Langfuse
+# ──────────────────────────────────────────────
+variable "langfuse_repo_url" {
+  description = "Langfuse git repository URL"
+  type        = string
+  default     = "https://github.com/langfuse/langfuse.git"
+}
+
+variable "langfuse_web_port" {
+  description = "Langfuse web UI port"
+  type        = number
+  default     = 3000
+}