feat: implement s3 media upload features

config/config.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Any, Literal, Optional
 from pydantic import BaseModel
 import os
 import yaml
@@ -22,6 +22,16 @@ class SecurityConfig(BaseModel):
     auth_jwt_secret_key: str
 
 
+class S3ApiConfig(BaseModel):
+    bucket_name: str
+    endpoint_url: str
+
+
+class ContentDeliveryConfig(BaseModel):
+    type: Literal["filesystem", "s3api"]
+    s3api: S3ApiConfig
+
+
 class HostingConfig(BaseModel):
     domain: str
     ssl: bool
@@ -31,6 +41,7 @@ class HostingConfig(BaseModel):
     self_hosted: bool
     sentry_config: Optional[SentryConfig]
     cookie_config: CookieConfig
+    content_delivery: ContentDeliveryConfig
 
 
 class DatabaseConfig(BaseModel):
@@ -116,6 +127,33 @@ def get_learnhouse_config() -> LearnHouseConfig:
     ).get("domain")
     cookie_config = CookieConfig(domain=cookies_domain)
 
+    env_content_delivery_type = os.environ.get("LEARNHOUSE_CONTENT_DELIVERY_TYPE")
+    content_delivery_type: str = (
+        (yaml_config.get("hosting_config", {}).get("content_delivery", {}).get("type"))
+        or env_content_delivery_type
+        or "filesystem"
+    )  # default to filesystem
+
+    env_bucket_name = os.environ.get("LEARNHOUSE_S3_API_BUCKET_NAME")
+    env_endpoint_url = os.environ.get("LEARNHOUSE_S3_API_ENDPOINT_URL")
+    bucket_name = (
+        yaml_config.get("hosting_config", {})
+        .get("content_delivery", {})
+        .get("s3api", {})
+        .get("bucket_name")
+    ) or env_bucket_name
+    endpoint_url = (
+        yaml_config.get("hosting_config", {})
+        .get("content_delivery", {})
+        .get("s3api", {})
+        .get("endpoint_url")
+    ) or env_endpoint_url
+
+    content_delivery = ContentDeliveryConfig(
+        type=content_delivery_type,  # type: ignore
+        s3api=S3ApiConfig(bucket_name=bucket_name, endpoint_url=endpoint_url),  # type: ignore
+    )
+
     # Database config
     mongodb_connection_string = env_mongodb_connection_string or yaml_config.get(
         "database_config", {}
@@ -158,6 +196,7 @@ def get_learnhouse_config() -> LearnHouseConfig:
         self_hosted=bool(self_hosted),
         sentry_config=sentry_config,
         cookie_config=cookie_config,
+        content_delivery=content_delivery,
     )
     database_config = DatabaseConfig(
         mongodb_connection_string=mongodb_connection_string

config/config.yaml

@@ -17,6 +17,11 @@ hosting_config:
   cookies_config:
     domain: ".localhost"
   allowed_regexp: '\b((?:https?://)[^\s/$.?#].[^\s]*)\b'
+  content_delivery:
+    type: "filesystem" # "filesystem" or "s3api"
+    s3api:
+      bucket_name: ""
+      endpoint_url: ""
 
 database_config:
   mongodb_connection_string: mongodb://learnhouse:learnhouse@mongo:27017/

front/services/media/media.ts

@@ -1,37 +1,46 @@
 import { getBackendUrl } from "@services/config/config";
+const LEARNHOUSE_MEDIA_URL = process.env.NEXT_PUBLIC_LEARNHOUSE_MEDIA_URL;
+
+function getMediaUrl() {
+  if (LEARNHOUSE_MEDIA_URL) {
+    return LEARNHOUSE_MEDIA_URL;
+  } else {
+    return getBackendUrl();
+  }
+}
 
 export function getCourseThumbnailMediaDirectory(orgId: string, courseId: string, fileId: string) {
-  let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/thumbnails/${fileId}`;
+  let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/thumbnails/${fileId}`;
   return uri;
 }
 
 export function getActivityBlockMediaDirectory(orgId: string, courseId: string, activityId: string, blockId: any, fileId: any, type: string) {
   if (type == "pdfBlock") {
-    let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/pdfBlock/${blockId}/${fileId}`;
+    let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/pdfBlock/${blockId}/${fileId}`;
     return uri;
   }
   if (type == "videoBlock") {
-    let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/videoBlock/${blockId}/${fileId}`;
+    let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/videoBlock/${blockId}/${fileId}`;
     return uri;
   }
   if (type == "imageBlock") {
-    let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/imageBlock/${blockId}/${fileId}`;
+    let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/imageBlock/${blockId}/${fileId}`;
     return uri;
   }
 }
 
 export function getActivityMediaDirectory(orgId: string, courseId: string, activityId: string, fileId: string, activityType: string) {
   if (activityType == "video") {
-    let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/video/${fileId}`;
+    let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/video/${fileId}`;
     return uri;
   }
   if (activityType == "documentpdf") {
-    let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/documentpdf/${fileId}`;
+    let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/documentpdf/${fileId}`;
     return uri;
   }
 }
 
 export function getOrgLogoMediaDirectory(orgId: string, fileId: string) {
-  let uri = `${getBackendUrl()}content/${orgId}/logos/${fileId}`;
+  let uri = `${getMediaUrl()}content/${orgId}/logos/${fileId}`;
   return uri;
 }

requirements.txt

@@ -4,6 +4,8 @@ uvicorn==0.20.0
 pymongo==4.3.3
 motor==3.1.1
 python-multipart
+boto3
+botocore
 python-jose
 passlib
 fastapi-jwt-auth

src/services/utils/upload_content.py

@@ -1,17 +1,69 @@
+import boto3
+from botocore.exceptions import ClientError
 import os
 
+from config.config import get_learnhouse_config
+
 
 async def upload_content(
     directory: str, org_id: str, file_binary: bytes, file_and_format: str
 ):
-    # create folder for activity
+    # Get Learnhouse Config
-    if not os.path.exists(f"content/{org_id}/{directory}"):
+    learnhouse_config = get_learnhouse_config()
+
+    # Get content delivery method
+    content_delivery = learnhouse_config.hosting_config.content_delivery.type
+
+    if content_delivery == "filesystem":
         # create folder for activity
-        os.makedirs(f"content/{org_id}/{directory}")
+        if not os.path.exists(f"content/{org_id}/{directory}"):
-    # upload file to server
+            # create folder for activity
-    with open(
+            os.makedirs(f"content/{org_id}/{directory}")
-        f"content/{org_id}/{directory}/{file_and_format}",
+        # upload file to server
-        "wb",
+        with open(
-    ) as f:
+            f"content/{org_id}/{directory}/{file_and_format}",
-        f.write(file_binary)
+            "wb",
-        f.close()
+        ) as f:
+            f.write(file_binary)
+            f.close()
+
+    elif content_delivery == "s3api":
+        # Upload to server then to s3 (AWS Keys are stored in environment variables and are loaded by boto3)
+        # TODO: Improve implementation of this
+        s3 = boto3.client(
+            "s3",
+            endpoint_url=learnhouse_config.hosting_config.content_delivery.s3api.endpoint_url,
+        )
+
+        # Create folder for activity
+        if not os.path.exists(f"content/{org_id}/{directory}"):
+            # create folder for activity
+            os.makedirs(f"content/{org_id}/{directory}")
+
+        # Upload file to server
+        with open(
+            f"content/{org_id}/{directory}/{file_and_format}",
+            "wb",
+        ) as f:
+            f.write(file_binary)
+            f.close()
+
+        print("Uploading to s3 using boto3...")
+        try:
+            s3.upload_file(
+                f"content/{org_id}/{directory}/{file_and_format}",
+                "learnhouse-media",
+                f"content/{org_id}/{directory}/{file_and_format}",
+            )
+        except ClientError as e:
+            print(e)
+
+        print("Checking if file exists in s3...")
+        try:
+            s3.head_object(
+                Bucket="learnhouse-media",
+                Key=f"content/{org_id}/{directory}/{file_and_format}",
+            )
+            print("File upload successful!")
+        except Exception as e:
+            print(f"An error occurred: {str(e)}")