diff --git a/config/config.py b/config/config.py index caeead27..0884aea3 100644 --- a/config/config.py +++ b/config/config.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Any, Literal, Optional from pydantic import BaseModel import os import yaml @@ -22,6 +22,16 @@ class SecurityConfig(BaseModel): auth_jwt_secret_key: str +class S3ApiConfig(BaseModel): + bucket_name: str + endpoint_url: str + + +class ContentDeliveryConfig(BaseModel): + type: Literal["filesystem", "s3api"] + s3api: S3ApiConfig + + class HostingConfig(BaseModel): domain: str ssl: bool @@ -31,6 +41,7 @@ class HostingConfig(BaseModel): self_hosted: bool sentry_config: Optional[SentryConfig] cookie_config: CookieConfig + content_delivery: ContentDeliveryConfig class DatabaseConfig(BaseModel): @@ -116,6 +127,33 @@ def get_learnhouse_config() -> LearnHouseConfig: ).get("domain") cookie_config = CookieConfig(domain=cookies_domain) + env_content_delivery_type = os.environ.get("LEARNHOUSE_CONTENT_DELIVERY_TYPE") + content_delivery_type: str = ( + (yaml_config.get("hosting_config", {}).get("content_delivery", {}).get("type")) + or env_content_delivery_type + or "filesystem" + ) # default to filesystem + + env_bucket_name = os.environ.get("LEARNHOUSE_S3_API_BUCKET_NAME") + env_endpoint_url = os.environ.get("LEARNHOUSE_S3_API_ENDPOINT_URL") + bucket_name = ( + yaml_config.get("hosting_config", {}) + .get("content_delivery", {}) + .get("s3api", {}) + .get("bucket_name") + ) or env_bucket_name + endpoint_url = ( + yaml_config.get("hosting_config", {}) + .get("content_delivery", {}) + .get("s3api", {}) + .get("endpoint_url") + ) or env_endpoint_url + + content_delivery = ContentDeliveryConfig( + type=content_delivery_type, # type: ignore + s3api=S3ApiConfig(bucket_name=bucket_name, endpoint_url=endpoint_url), # type: ignore + ) + # Database config mongodb_connection_string = env_mongodb_connection_string or yaml_config.get( "database_config", {} @@ -158,6 +196,7 @@ def get_learnhouse_config() -> LearnHouseConfig: self_hosted=bool(self_hosted), sentry_config=sentry_config, cookie_config=cookie_config, + content_delivery=content_delivery, ) database_config = DatabaseConfig( mongodb_connection_string=mongodb_connection_string diff --git a/config/config.yaml b/config/config.yaml index b381699f..bf151243 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -17,6 +17,11 @@ hosting_config: cookies_config: domain: ".localhost" allowed_regexp: '\b((?:https?://)[^\s/$.?#].[^\s]*)\b' + content_delivery: + type: "filesystem" # "filesystem" or "s3api" + s3api: + bucket_name: "" + endpoint_url: "" database_config: mongodb_connection_string: mongodb://learnhouse:learnhouse@mongo:27017/ diff --git a/front/services/media/media.ts b/front/services/media/media.ts index 9e1e0e54..b17191a2 100644 --- a/front/services/media/media.ts +++ b/front/services/media/media.ts @@ -1,37 +1,46 @@ import { getBackendUrl } from "@services/config/config"; +const LEARNHOUSE_MEDIA_URL = process.env.NEXT_PUBLIC_LEARNHOUSE_MEDIA_URL; + +function getMediaUrl() { + if (LEARNHOUSE_MEDIA_URL) { + return LEARNHOUSE_MEDIA_URL; + } else { + return getBackendUrl(); + } +} export function getCourseThumbnailMediaDirectory(orgId: string, courseId: string, fileId: string) { - let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/thumbnails/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/thumbnails/${fileId}`; return uri; } export function getActivityBlockMediaDirectory(orgId: string, courseId: string, activityId: string, blockId: any, fileId: any, type: string) { if (type == "pdfBlock") { - let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/pdfBlock/${blockId}/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/pdfBlock/${blockId}/${fileId}`; return uri; } if (type == "videoBlock") { - let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/videoBlock/${blockId}/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/videoBlock/${blockId}/${fileId}`; return uri; } if (type == "imageBlock") { - let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/imageBlock/${blockId}/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/dynamic/blocks/imageBlock/${blockId}/${fileId}`; return uri; } } export function getActivityMediaDirectory(orgId: string, courseId: string, activityId: string, fileId: string, activityType: string) { if (activityType == "video") { - let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/video/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/video/${fileId}`; return uri; } if (activityType == "documentpdf") { - let uri = `${getBackendUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/documentpdf/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/courses/${courseId}/activities/${activityId}/documentpdf/${fileId}`; return uri; } } export function getOrgLogoMediaDirectory(orgId: string, fileId: string) { - let uri = `${getBackendUrl()}content/${orgId}/logos/${fileId}`; + let uri = `${getMediaUrl()}content/${orgId}/logos/${fileId}`; return uri; } diff --git a/requirements.txt b/requirements.txt index ee409c20..5f09ce52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ uvicorn==0.20.0 pymongo==4.3.3 motor==3.1.1 python-multipart +boto3 +botocore python-jose passlib fastapi-jwt-auth diff --git a/src/services/utils/upload_content.py b/src/services/utils/upload_content.py index 6110ed7c..81d766a2 100644 --- a/src/services/utils/upload_content.py +++ b/src/services/utils/upload_content.py @@ -1,17 +1,69 @@ +import boto3 +from botocore.exceptions import ClientError import os +from config.config import get_learnhouse_config + async def upload_content( directory: str, org_id: str, file_binary: bytes, file_and_format: str ): - # create folder for activity - if not os.path.exists(f"content/{org_id}/{directory}"): + # Get Learnhouse Config + learnhouse_config = get_learnhouse_config() + + # Get content delivery method + content_delivery = learnhouse_config.hosting_config.content_delivery.type + + if content_delivery == "filesystem": # create folder for activity - os.makedirs(f"content/{org_id}/{directory}") - # upload file to server - with open( - f"content/{org_id}/{directory}/{file_and_format}", - "wb", - ) as f: - f.write(file_binary) - f.close() + if not os.path.exists(f"content/{org_id}/{directory}"): + # create folder for activity + os.makedirs(f"content/{org_id}/{directory}") + # upload file to server + with open( + f"content/{org_id}/{directory}/{file_and_format}", + "wb", + ) as f: + f.write(file_binary) + f.close() + + elif content_delivery == "s3api": + # Upload to server then to s3 (AWS Keys are stored in environment variables and are loaded by boto3) + # TODO: Improve implementation of this + s3 = boto3.client( + "s3", + endpoint_url=learnhouse_config.hosting_config.content_delivery.s3api.endpoint_url, + ) + + # Create folder for activity + if not os.path.exists(f"content/{org_id}/{directory}"): + # create folder for activity + os.makedirs(f"content/{org_id}/{directory}") + + # Upload file to server + with open( + f"content/{org_id}/{directory}/{file_and_format}", + "wb", + ) as f: + f.write(file_binary) + f.close() + + print("Uploading to s3 using boto3...") + try: + s3.upload_file( + f"content/{org_id}/{directory}/{file_and_format}", + "learnhouse-media", + f"content/{org_id}/{directory}/{file_and_format}", + ) + except ClientError as e: + print(e) + + print("Checking if file exists in s3...") + try: + s3.head_object( + Bucket="learnhouse-media", + Key=f"content/{org_id}/{directory}/{file_and_format}", + ) + print("File upload successful!") + except Exception as e: + print(f"An error occurred: {str(e)}")