From 6ebac01c618c66825fd42f7246e25fcd46a7c310 Mon Sep 17 00:00:00 2001 From: swve Date: Sun, 16 Mar 2025 11:18:38 +0100 Subject: [PATCH 1/3] chore: update database connection settings with pool size and timeout configurations --- apps/api/src/core/events/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/api/src/core/events/database.py b/apps/api/src/core/events/database.py index 67a199ef..6733dd81 100644 --- a/apps/api/src/core/events/database.py +++ b/apps/api/src/core/events/database.py @@ -35,7 +35,11 @@ learnhouse_config = get_learnhouse_config() engine = create_engine( learnhouse_config.database_config.sql_connection_string, # type: ignore echo=False, - pool_pre_ping=True # type: ignore + pool_pre_ping=True, # type: ignore + pool_size=10, + max_overflow=20, + pool_recycle=300, # Recycle connections after 5 minutes + pool_timeout=30 ) # Create all tables after importing all models From 5e7ae54215742d8e2c07260f84ee23a93fa2a65d Mon Sep 17 00:00:00 2001 From: swve Date: Sun, 16 Mar 2025 11:34:41 +0100 Subject: [PATCH 2/3] refactor: optimize functions queries --- .../services/courses/activities/activities.py | 53 ++++----- apps/api/src/services/courses/courses.py | 104 ++++++++++++------ apps/api/src/services/orgs/orgs.py | 64 +++++------ 3 files changed, 115 insertions(+), 106 deletions(-) diff --git a/apps/api/src/services/courses/activities/activities.py b/apps/api/src/services/courses/activities/activities.py index 0d738530..d758358b 100644 --- a/apps/api/src/services/courses/activities/activities.py +++ b/apps/api/src/services/courses/activities/activities.py @@ -92,24 +92,21 @@ async def get_activity( current_user: PublicUser, db_session: Session, ): - statement = select(Activity).where(Activity.activity_uuid == activity_uuid) - activity = db_session.exec(statement).first() + # Optimize by joining Activity with Course in a single query + statement = ( + select(Activity, Course) + .join(Course) + .where(Activity.activity_uuid == activity_uuid) + ) + result = db_session.exec(statement).first() - if not activity: + if not result: raise HTTPException( status_code=404, detail="Activity not found", ) - - # Get course from that activity - statement = select(Course).where(Course.id == activity.course_id) - course = db_session.exec(statement).first() - - if not course: - raise HTTPException( - status_code=404, - detail="Course not found", - ) + + activity, course = result # RBAC check await rbac_check(request, course.course_uuid, current_user, "read", db_session) @@ -124,9 +121,8 @@ async def get_activity( activity_read = ActivityRead.model_validate(activity) activity_read.content = activity_read.content if has_paid_access else { "paid_access": False } - activity = activity_read - return activity + return activity_read async def get_activityby_id( request: Request, @@ -134,31 +130,26 @@ async def get_activityby_id( current_user: PublicUser, db_session: Session, ): - statement = select(Activity).where(Activity.id == activity_id) - activity = db_session.exec(statement).first() + # Optimize by joining Activity with Course in a single query + statement = ( + select(Activity, Course) + .join(Course) + .where(Activity.id == activity_id) + ) + result = db_session.exec(statement).first() - if not activity: + if not result: raise HTTPException( status_code=404, detail="Activity not found", ) - - # Get course from that activity - statement = select(Course).where(Course.id == activity.course_id) - course = db_session.exec(statement).first() - - if not course: - raise HTTPException( - status_code=404, - detail="Course not found", - ) + + activity, course = result # RBAC check await rbac_check(request, course.course_uuid, current_user, "read", db_session) - activity = ActivityRead.model_validate(activity) - - return activity + return ActivityRead.model_validate(activity) async def update_activity( diff --git a/apps/api/src/services/courses/courses.py b/apps/api/src/services/courses/courses.py index 42b9b49d..afbd53c9 100644 --- a/apps/api/src/services/courses/courses.py +++ b/apps/api/src/services/courses/courses.py @@ -27,6 +27,7 @@ from src.security.rbac.rbac import ( from src.services.courses.thumbnails import upload_thumbnail from fastapi import HTTPException, Request, UploadFile from datetime import datetime +import asyncio async def get_course( @@ -106,6 +107,7 @@ async def get_course_meta( # Avoid circular import from src.services.courses.chapters import get_course_chapters + # Get course with a single query course_statement = select(Course).where(Course.course_uuid == course_uuid) course = db_session.exec(course_statement).first() @@ -118,36 +120,51 @@ async def get_course_meta( # RBAC check await rbac_check(request, course.course_uuid, current_user, "read", db_session) - # Get course authors - authors_statement = ( - select(User) - .join(ResourceAuthor) - .where(ResourceAuthor.resource_uuid == course.course_uuid) - ) - authors = db_session.exec(authors_statement).all() - - # convert from User to UserRead - authors = [UserRead.model_validate(author) for author in authors] - - course = CourseRead(**course.model_dump(), authors=authors) - - # Get course chapters - chapters = await get_course_chapters(request, course.id, db_session, current_user) - - # Trail - trail = None - - if isinstance(current_user, AnonymousUser): - trail = None - else: - trail = await get_user_trail_with_orgid( + # Start async tasks concurrently + tasks = [] + + # Task 1: Get course authors + async def get_authors(): + authors_statement = ( + select(User) + .join(ResourceAuthor) + .where(ResourceAuthor.resource_uuid == course.course_uuid) + ) + return db_session.exec(authors_statement).all() + + # Task 2: Get course chapters + async def get_chapters(): + # Ensure course.id is not None + if course.id is None: + return [] + return await get_course_chapters(request, course.id, db_session, current_user) + + # Task 3: Get user trail (only for authenticated users) + async def get_trail(): + if isinstance(current_user, AnonymousUser): + return None + return await get_user_trail_with_orgid( request, current_user, course.org_id, db_session ) - + + # Add tasks to the list + tasks.append(get_authors()) + tasks.append(get_chapters()) + tasks.append(get_trail()) + + # Run all tasks concurrently + authors_raw, chapters, trail = await asyncio.gather(*tasks) + + # Convert authors from User to UserRead + authors = [UserRead.model_validate(author) for author in authors_raw] + + # Create course read model + course_read = CourseRead(**course.model_dump(), authors=authors) + return FullCourseReadWithTrail( - **course.model_dump(), + **course_read.model_dump(), chapters=chapters, - trail=trail if trail else None, + trail=trail, ) async def get_courses_orgslug( @@ -196,19 +213,34 @@ async def get_courses_orgslug( query = query.offset(offset).limit(limit).distinct() courses = db_session.exec(query).all() - - # Fetch authors for each course + + if not courses: + return [] + + # Get all course UUIDs + course_uuids = [course.course_uuid for course in courses] + + # Fetch all authors for all courses in a single query + authors_query = ( + select(ResourceAuthor, User) + .join(User, ResourceAuthor.user_id == User.id) # type: ignore + .where(ResourceAuthor.resource_uuid.in_(course_uuids)) # type: ignore + ) + + author_results = db_session.exec(authors_query).all() + + # Create a dictionary mapping course_uuid to list of authors + course_authors = {} + for resource_author, user in author_results: + if resource_author.resource_uuid not in course_authors: + course_authors[resource_author.resource_uuid] = [] + course_authors[resource_author.resource_uuid].append(UserRead.model_validate(user)) + + # Create CourseRead objects with authors course_reads = [] for course in courses: - authors_query = ( - select(User) - .join(ResourceAuthor, ResourceAuthor.user_id == User.id) # type: ignore - .where(ResourceAuthor.resource_uuid == course.course_uuid) - ) - authors = db_session.exec(authors_query).all() - course_read = CourseRead.model_validate(course) - course_read.authors = [UserRead.model_validate(author) for author in authors] + course_read.authors = course_authors.get(course.course_uuid, []) course_reads.append(course_read) return course_reads diff --git a/apps/api/src/services/orgs/orgs.py b/apps/api/src/services/orgs/orgs.py index 9f580b28..bdfcd156 100644 --- a/apps/api/src/services/orgs/orgs.py +++ b/apps/api/src/services/orgs/orgs.py @@ -529,39 +529,31 @@ async def get_orgs_by_user_admin( page: int = 1, limit: int = 10, ) -> list[OrganizationRead]: - + # Join Organization, UserOrganization and OrganizationConfig in a single query statement = ( - select(Organization) + select(Organization, OrganizationConfig) .join(UserOrganization) + .outerjoin(OrganizationConfig) .where( UserOrganization.user_id == user_id, UserOrganization.role_id == 1, # Only where the user is admin + UserOrganization.org_id == Organization.id, + OrganizationConfig.org_id == Organization.id ) .offset((page - 1) * limit) .limit(limit) ) - # Get organizations where the user is an admin + # Execute single query to get all data result = db_session.exec(statement) - orgs = result.all() + org_data = result.all() + # Process results in memory orgsWithConfig = [] - - for org in orgs: - - # Get org config - statement = select(OrganizationConfig).where( - OrganizationConfig.org_id == org.id - ) - result = db_session.exec(statement) - - org_config = result.first() - + for org, org_config in org_data: config = OrganizationConfig.model_validate(org_config) if org_config else {} - - org = OrganizationRead(**org.model_dump(), config=config) - - orgsWithConfig.append(org) + org_read = OrganizationRead(**org.model_dump(), config=config) + orgsWithConfig.append(org_read) return orgsWithConfig @@ -573,36 +565,30 @@ async def get_orgs_by_user( page: int = 1, limit: int = 10, ) -> list[OrganizationRead]: - + # Join Organization, UserOrganization and OrganizationConfig in a single query statement = ( - select(Organization) + select(Organization, OrganizationConfig) .join(UserOrganization) - .where(UserOrganization.user_id == user_id) + .outerjoin(OrganizationConfig) + .where( + UserOrganization.user_id == user_id, + UserOrganization.org_id == Organization.id, + OrganizationConfig.org_id == Organization.id + ) .offset((page - 1) * limit) .limit(limit) ) - # Get organizations where the user is an admin + # Execute single query to get all data result = db_session.exec(statement) - orgs = result.all() + org_data = result.all() + # Process results in memory orgsWithConfig = [] - - for org in orgs: - - # Get org config - statement = select(OrganizationConfig).where( - OrganizationConfig.org_id == org.id - ) - result = db_session.exec(statement) - - org_config = result.first() - + for org, org_config in org_data: config = OrganizationConfig.model_validate(org_config) if org_config else {} - - org = OrganizationRead(**org.model_dump(), config=config) - - orgsWithConfig.append(org) + org_read = OrganizationRead(**org.model_dump(), config=config) + orgsWithConfig.append(org_read) return orgsWithConfig From 744e372f4ebea6bb9f5b4a030beda86437d0337b Mon Sep 17 00:00:00 2001 From: swve Date: Sun, 16 Mar 2025 15:23:02 +0100 Subject: [PATCH 3/3] refactor: improve perf on the frontend --- .../activity/[activityid]/activity.tsx | 1 + .../activity/[activityid]/page.tsx | 68 +++++++++++-------- .../(withmenu)/course/[courseuuid]/course.tsx | 5 ++ .../(withmenu)/course/[courseuuid]/page.tsx | 23 +++---- .../DraggableElements/ActivityElement.tsx | 3 - .../Pages/Courses/ActivityIndicators.tsx | 2 +- apps/web/services/courses/activities.ts | 4 +- apps/web/services/courses/courses.ts | 6 +- 8 files changed, 61 insertions(+), 51 deletions(-) diff --git a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/activity.tsx b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/activity.tsx index f0728c3f..f3327327 100644 --- a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/activity.tsx +++ b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/activity/[activityid]/activity.tsx @@ -499,6 +499,7 @@ function ActivityChapterDropdown(props: { setIsOpen(false)} >
} +type Session = { + tokens?: { + access_token?: string + } +} + +// Add this function at the top level to avoid duplicate fetches +async function fetchCourseMetadata(courseuuid: string, access_token: string | null | undefined) { + return await getCourseMetadata( + courseuuid, + { revalidate: 1800, tags: ['courses'] }, + access_token || null + ) +} + export async function generateMetadata(props: MetadataProps): Promise { const params = await props.params; - const session = await getServerSession(nextAuthOptions) - const access_token = session?.tokens?.access_token + const session = await getServerSession(nextAuthOptions as any) as Session + const access_token = session?.tokens?.access_token || null // Get Org context information const org = await getOrganizationContextInfo(params.orgslug, { revalidate: 1800, tags: ['organizations'], }) - const course_meta = await getCourseMetadata( - params.courseuuid, - { revalidate: 0, tags: ['courses'] }, - access_token ? access_token : null - ) + const course_meta = await fetchCourseMetadata(params.courseuuid, access_token) const activity = await getActivityWithAuthHeader( params.activityid, - { revalidate: 0, tags: ['activities'] }, - access_token ? access_token : null + { revalidate: 1800, tags: ['activities'] }, + access_token || null ) // SEO @@ -57,32 +68,29 @@ export async function generateMetadata(props: MetadataProps): Promise } const ActivityPage = async (params: any) => { - const session = await getServerSession(nextAuthOptions) - const access_token = session?.tokens?.access_token + const session = await getServerSession(nextAuthOptions as any) as Session + const access_token = session?.tokens?.access_token || null const activityid = (await params.params).activityid const courseuuid = (await params.params).courseuuid const orgslug = (await params.params).orgslug - const course_meta = await getCourseMetadata( - courseuuid, - { revalidate: 0, tags: ['courses'] }, - access_token ? access_token : null - ) - const activity = await getActivityWithAuthHeader( - activityid, - { revalidate: 0, tags: ['activities'] }, - access_token ? access_token : null - ) + const [course_meta, activity] = await Promise.all([ + fetchCourseMetadata(courseuuid, access_token), + getActivityWithAuthHeader( + activityid, + { revalidate: 1800, tags: ['activities'] }, + access_token || null + ) + ]) + return ( - <> - - + ) } diff --git a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/course.tsx b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/course.tsx index 6bcba6dd..c3ebb3a0 100644 --- a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/course.tsx +++ b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/course.tsx @@ -223,6 +223,7 @@ const CourseClient = (props: any) => { )}` } rel="noopener noreferrer" + prefetch={false} >

{activity.name}

@@ -240,6 +241,7 @@ const CourseClient = (props: any) => { )}` } rel="noopener noreferrer" + prefetch={false} >

Page

@@ -260,6 +262,7 @@ const CourseClient = (props: any) => { )}` } rel="noopener noreferrer" + prefetch={false} >

Video

@@ -281,6 +284,7 @@ const CourseClient = (props: any) => { )}` } rel="noopener noreferrer" + prefetch={false} >

Document

@@ -302,6 +306,7 @@ const CourseClient = (props: any) => { )}` } rel="noopener noreferrer" + prefetch={false} >

Assignment

diff --git a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/page.tsx b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/page.tsx index 1bd761be..d0c721a4 100644 --- a/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/page.tsx +++ b/apps/web/app/orgs/[orgslug]/(withmenu)/course/[courseuuid]/page.tsx @@ -24,7 +24,7 @@ export async function generateMetadata(props: MetadataProps): Promise }) const course_meta = await getCourseMetadata( params.courseuuid, - { revalidate: 0, tags: ['courses'] }, + { revalidate: 1800, tags: ['courses'] }, access_token ? access_token : null ) @@ -66,24 +66,23 @@ export async function generateMetadata(props: MetadataProps): Promise } const CoursePage = async (params: any) => { - const courseuuid = (await params.params).courseuuid - const orgslug = (await params.params).orgslug const session = await getServerSession(nextAuthOptions) const access_token = session?.tokens?.access_token + + // Fetch course metadata once const course_meta = await getCourseMetadata( - courseuuid, - { revalidate: 0, tags: ['courses'] }, + params.params.courseuuid, + { revalidate: 1800, tags: ['courses'] }, access_token ? access_token : null ) return ( -
- -
+ ) } diff --git a/apps/web/components/Dashboard/Pages/Course/EditCourseStructure/DraggableElements/ActivityElement.tsx b/apps/web/components/Dashboard/Pages/Course/EditCourseStructure/DraggableElements/ActivityElement.tsx index c7a1d34f..a4b9e584 100644 --- a/apps/web/components/Dashboard/Pages/Course/EditCourseStructure/DraggableElements/ActivityElement.tsx +++ b/apps/web/components/Dashboard/Pages/Course/EditCourseStructure/DraggableElements/ActivityElement.tsx @@ -191,7 +191,6 @@ function ActivityElement(props: ActivitiyElementProps) { '' )}` } - prefetch className="p-1 px-2 sm:px-3 bg-linear-to-bl text-cyan-800 from-sky-400/50 to-cyan-200/80 border border-cyan-600/10 shadow-md rounded-md font-bold text-xs flex items-center space-x-1 transition-colors duration-200 hover:from-sky-500/50 hover:to-cyan-300/80" rel="noopener noreferrer" > @@ -296,7 +295,6 @@ const ActivityElementOptions = ({ activity, isMobile }: { activity: any; isMobil '' )}/edit` } - prefetch className={`hover:cursor-pointer p-1 ${isMobile ? 'px-2' : 'px-3'} bg-sky-700 rounded-md items-center`} target='_blank' > @@ -313,7 +311,6 @@ const ActivityElementOptions = ({ activity, isMobile }: { activity: any; isMobil getUriWithOrg(org.slug, '') + `/dash/assignments/${assignmentUUID}` } - prefetch className={`hover:cursor-pointer p-1 ${isMobile ? 'px-2' : 'px-3'} bg-teal-700 rounded-md items-center`} >
diff --git a/apps/web/components/Pages/Courses/ActivityIndicators.tsx b/apps/web/components/Pages/Courses/ActivityIndicators.tsx index 59f32a48..8fb212a3 100644 --- a/apps/web/components/Pages/Courses/ActivityIndicators.tsx +++ b/apps/web/components/Pages/Courses/ActivityIndicators.tsx @@ -65,7 +65,7 @@ function ActivityIndicators(props: Props) { key={activity.activity_uuid} >