From 2d822459fba2ebcf2ee66ce28dd38a5e2dd1170c Mon Sep 17 00:00:00 2001 From: swve Date: Fri, 15 Dec 2023 21:37:22 +0100 Subject: [PATCH] feat: init migration script from mongodb --- apps/api/config/config.py | 12 +- apps/api/config/config.yaml | 1 + apps/api/src/core/events/database.py | 11 +- apps/api/src/routers/dev.py | 13 +- apps/api/src/services/dev/dev.py | 2 + .../src/services/dev/migration_from_mongo.py | 275 ++++++++++++++++++ .../activity/[activityid]/page.tsx | 1 - 7 files changed, 307 insertions(+), 8 deletions(-) create mode 100644 apps/api/src/services/dev/migration_from_mongo.py diff --git a/apps/api/config/config.py b/apps/api/config/config.py index f56728c8..30d579a8 100644 --- a/apps/api/config/config.py +++ b/apps/api/config/config.py @@ -47,6 +47,7 @@ class HostingConfig(BaseModel): class DatabaseConfig(BaseModel): sql_connection_string: Optional[str] + mongo_connection_string: Optional[str] class LearnHouseConfig(BaseModel): @@ -105,9 +106,7 @@ def get_learnhouse_config() -> LearnHouseConfig: env_allowed_origins = env_allowed_origins.split(",") env_allowed_regexp = os.environ.get("LEARNHOUSE_ALLOWED_REGEXP") env_self_hosted = os.environ.get("LEARNHOUSE_SELF_HOSTED") - env_sql_connection_string = os.environ.get( - "LEARNHOUSE_SQL_CONNECTION_STRING" - ) + env_sql_connection_string = os.environ.get("LEARNHOUSE_SQL_CONNECTION_STRING") # Sentry Config env_sentry_dsn = os.environ.get("LEARNHOUSE_SENTRY_DSN") @@ -170,6 +169,10 @@ def get_learnhouse_config() -> LearnHouseConfig: "database_config", {} ).get("sql_connection_string") + mongo_connection_string = yaml_config.get("database_config", {}).get( + "mongo_connection_string" + ) + # Sentry config # check if the sentry config is provided in the YAML file sentry_config_verif = ( @@ -210,7 +213,8 @@ def get_learnhouse_config() -> LearnHouseConfig: content_delivery=content_delivery, ) database_config = DatabaseConfig( - sql_connection_string=sql_connection_string + sql_connection_string=sql_connection_string, + mongo_connection_string=mongo_connection_string, ) # Create LearnHouseConfig object diff --git a/apps/api/config/config.yaml b/apps/api/config/config.yaml index 37c492eb..17bea379 100644 --- a/apps/api/config/config.yaml +++ b/apps/api/config/config.yaml @@ -26,3 +26,4 @@ hosting_config: database_config: sql_connection_string: postgresql://learnhouse:learnhouse@db:5432/learnhouse + mongo_connection_string: mongodb://learnhouse:learnhouse@mongo:27017/ diff --git a/apps/api/src/core/events/database.py b/apps/api/src/core/events/database.py index 58ae8433..2b709236 100644 --- a/apps/api/src/core/events/database.py +++ b/apps/api/src/core/events/database.py @@ -2,10 +2,11 @@ import logging from config.config import get_learnhouse_config from fastapi import FastAPI from sqlmodel import SQLModel, Session, create_engine +import motor.motor_asyncio learnhouse_config = get_learnhouse_config() engine = create_engine( - learnhouse_config.database_config.sql_connection_string, echo=False + learnhouse_config.database_config.sql_connection_string, echo=False # type: ignore ) SQLModel.metadata.create_all(engine) @@ -13,9 +14,15 @@ SQLModel.metadata.create_all(engine) async def connect_to_db(app: FastAPI): app.db_engine = engine # type: ignore logging.info("LearnHouse database has been started.") - SQLModel.metadata.create_all(engine) + # MongoDB for migration purposes + # mongodb + app.mongodb_client = motor.motor_asyncio.AsyncIOMotorClient( # type: ignore + app.learnhouse_config.database_config.mongo_connection_string # type: ignore + ) # type: ignore + app.db = app.mongodb_client["learnhouse"] # type: ignore + def get_db_session(): with Session(engine) as session: diff --git a/apps/api/src/routers/dev.py b/apps/api/src/routers/dev.py index 4fd3d2d5..2c8da946 100644 --- a/apps/api/src/routers/dev.py +++ b/apps/api/src/routers/dev.py @@ -1,4 +1,7 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends, Request +from sqlmodel import Session +from src.core.events.database import get_db_session +from src.services.dev.migration_from_mongo import start_migrate_from_mongo from config.config import get_learnhouse_config @@ -9,3 +12,11 @@ router = APIRouter() async def config(): config = get_learnhouse_config() return config.dict() + + +@router.get("/migrate_from_mongo") +async def migrate_from_mongo( + request: Request, + db_session: Session = Depends(get_db_session), +): + return await start_migrate_from_mongo(request, db_session) diff --git a/apps/api/src/services/dev/dev.py b/apps/api/src/services/dev/dev.py index e74b25d7..386f5c7b 100644 --- a/apps/api/src/services/dev/dev.py +++ b/apps/api/src/services/dev/dev.py @@ -16,3 +16,5 @@ def isDevModeEnabledOrRaise(): return True else: raise HTTPException(status_code=403, detail="Development mode is disabled") + + diff --git a/apps/api/src/services/dev/migration_from_mongo.py b/apps/api/src/services/dev/migration_from_mongo.py new file mode 100644 index 00000000..07b9a1b9 --- /dev/null +++ b/apps/api/src/services/dev/migration_from_mongo.py @@ -0,0 +1,275 @@ +from datetime import date +import datetime +from fastapi import Request +from sqlmodel import Session, select +from src.db.blocks import Block, BlockTypeEnum +from src.db.chapter_activities import ChapterActivity +from src.db.activities import Activity, ActivitySubTypeEnum, ActivityTypeEnum +from src.db.course_chapters import CourseChapter +from src.db.resource_authors import ResourceAuthor, ResourceAuthorshipEnum +from src.db.user_organizations import UserOrganization +from src.db.chapters import Chapter +from src.db.courses import Course +from src.db.users import User + +from src.db.organizations import Organization + + +async def start_migrate_from_mongo(request: Request, db_session: Session): + orgs = request.app.db["organizations"] + + ## ----> Organizations migration + org_db_list = await orgs.find().to_list(length=100) + + for org in org_db_list: + org_to_add = Organization( + name=org["name"], + description=org["description"], + slug=org["slug"], + logo_image=org["logo"], + email=org["email"], + org_uuid=org["org_id"], + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + db_session.add(org_to_add) + db_session.commit() + + print("Migrated organizations.") + + ## ----> Users migration + users = request.app.db["users"] + + users_db_list = await users.find().to_list(length=100) + + for user in users_db_list: + user_to_add = User( + email=user["email"], + username=user["username"], + first_name="", + last_name="", + user_uuid=user["user_id"], + password=user["password"], + creation_date=user["creation_date"], + update_date=user["update_date"], + ) + db_session.add(user_to_add) + db_session.commit() + + # Link Orgs to users and make them owners + for org in user["orgs"]: + statement = select(Organization).where( + Organization.org_uuid == org["org_id"] + ) + org_from_db = db_session.exec(statement).first() + + statement = select(User).where(User.user_uuid == user["user_id"]) + user_from_db = db_session.exec(statement).first() + + user_org_object = UserOrganization( + user_id=user_from_db.id, # type: ignore + org_id=org_from_db.id, # type: ignore + role_id=1, + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + db_session.add(user_org_object) + db_session.commit() + + print("Migrated users and linked them to orgs.") + + ## ----> Courses migration + courses = request.app.db["courses"] + + courses_db_list = await courses.find().to_list(length=300) + + for course in courses_db_list: + # Get the organization id + statement = select(Organization).where( + Organization.org_uuid == course["org_id"] + ) + org_from_db = db_session.exec(statement).first() + + course_to_add = Course( + name=course["name"], + description=course["description"], + about=course["description"], + learnings="", + course_uuid=course["course_id"], + thumbnail_image=course["thumbnail"], + tags="", + org_id=org_from_db.id, # type: ignore + public=course["public"], + creation_date=str(course["creationDate"]), + update_date=str(course["updateDate"]), + ) + db_session.add(course_to_add) + db_session.commit() + + # Get this course + statement = select(Course).where(Course.course_uuid == course["course_id"]) + course_from_db = db_session.exec(statement).first() + + # Add Authorship + authors = course["authors"] + + for author in authors: + # Get the user id + statement = select(User).where(User.user_uuid == author) + user_from_db = db_session.exec(statement).first() + + authorship = ResourceAuthor( + resource_uuid=course_from_db.course_uuid, # type: ignore + user_id=user_from_db.id, # type: ignore + authorship=ResourceAuthorshipEnum.CREATOR, + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + db_session.add(authorship) + db_session.commit() + + print("Added authorship.") + + ## ----> Chapters migration & Link + + chapter_object = course["chapters_content"] + order = 0 + for chapter in chapter_object: + chapter_to_add = Chapter( + name=chapter["name"], + description=chapter["description"], + chapter_uuid=chapter["coursechapter_id"].replace( + "coursechapter", "chapter" + ), + org_id=org_from_db.id, # type: ignore + course_id=course_from_db.id, # type: ignore + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + db_session.add(chapter_to_add) + db_session.commit() + + # Get this chapter + statement = select(Chapter).where( + Chapter.chapter_uuid + == chapter["coursechapter_id"].replace("coursechapter", "chapter") + ) + chapter_from_db = db_session.exec(statement).first() + + # Link chapter to course + coursechapter_to_add = CourseChapter( + chapter_id=chapter_from_db.id, # type: ignore + course_id=course_from_db.id, # type: ignore + order=order, + org_id=org_from_db.id, # type: ignore + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + + db_session.add(coursechapter_to_add) + db_session.commit() + + order += 1 + + ## ----> Activities migration + activities = request.app.db["activities"] + activities_db_list = await activities.find( + {"coursechapter_id": chapter["coursechapter_id"]} + ).to_list(length=100) + + activity_order = 0 + + for activity in activities_db_list: + type_to_use = ActivityTypeEnum.TYPE_CUSTOM + sub_type_to_use = ActivityTypeEnum.TYPE_CUSTOM + + if activity["type"] == "video": + type_to_use = ActivityTypeEnum.TYPE_VIDEO + sub_type_to_use = ActivitySubTypeEnum.SUBTYPE_VIDEO_HOSTED + + if activity["type"] == "external_video": + type_to_use = ActivityTypeEnum.TYPE_VIDEO + sub_type_to_use = ActivitySubTypeEnum.SUBTYPE_VIDEO_YOUTUBE + + if activity["type"] == "documentpdf": + type_to_use = ActivityTypeEnum.TYPE_DOCUMENT + sub_type_to_use = ActivitySubTypeEnum.SUBTYPE_DOCUMENT_PDF + + if activity["type"] == "dynamic": + type_to_use = ActivityTypeEnum.TYPE_DYNAMIC + sub_type_to_use = ActivitySubTypeEnum.SUBTYPE_DYNAMIC_PAGE + + activity_to_add = Activity( + name=activity["name"], + activity_uuid=activity["activity_id"], + version=1, + published_version=1, + activity_type=type_to_use, + activity_sub_type=sub_type_to_use, + chapter_id=chapter_from_db.id, # type: ignore + org_id=org_from_db.id, # type: ignore + course_id=course_from_db.id, # type: ignore + creation_date=str(activity["creationDate"]), + update_date=str(activity["updateDate"]), + ) + db_session.add(activity_to_add) + db_session.commit() + + # Link activity to chapter + statement = select(Activity).where( + Activity.activity_uuid == activity["activity_id"] + ) + + activity_from_db = db_session.exec(statement).first() + + activitychapter_to_add = ChapterActivity( + chapter_id=chapter_from_db.id, # type: ignore + activity_id=activity_from_db.id, # type: ignore + order=activity_order, + course_id=course_from_db.id, # type: ignore + org_id=org_from_db.id, # type: ignore + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + + db_session.add(activitychapter_to_add) + db_session.commit() + + activity_order += 1 + + ## ----> Blocks migration + blocks = request.app.db["blocks"] + + blocks_db_list = await blocks.find( + {"activity_id": activity["activity_id"]} + ).to_list(length=200) + + for block in blocks_db_list: + type_to_use = BlockTypeEnum.BLOCK_CUSTOM + + if block["block_type"] == "imageBlock": + type_to_use = BlockTypeEnum.BLOCK_IMAGE + + if block["block_type"] == "videoBlock": + type_to_use = BlockTypeEnum.BLOCK_VIDEO + + if block["block_type"] == "pdfBlock": + type_to_use = BlockTypeEnum.BLOCK_DOCUMENT_PDF + + print('block', block) + + block_to_add = Block( + block_uuid=block["block_id"], + content=block["block_data"], + block_type=type_to_use, + activity_id=activity_from_db.id, # type: ignore + org_id=org_from_db.id, # type: ignore + course_id=course_from_db.id, # type: ignore + chapter_id=chapter_from_db.id, # type: ignore + creation_date=str(datetime.datetime.now()), + update_date=str(datetime.datetime.now()), + ) + db_session.add(block_to_add) + db_session.commit() + + return "Migration successfull." diff --git a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/page.tsx b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/page.tsx index e7b724c2..8a12424d 100644 --- a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/page.tsx +++ b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/page.tsx @@ -41,7 +41,6 @@ export async function generateMetadata( openGraph: { title: activity.name + ` — ${course_meta.name} Course`, description: course_meta.description, - type: activity.type === 'video' ? 'video.other' : 'article', publishedTime: course_meta.creation_date, tags: course_meta.learnings, },