diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3b2f533e9e17..68beeec8448c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # default code owner -* @omarithawi +* @bryanlandia # default set of reviewers -* @omarithawi @melvinsoft @thraxil @shadinaif +* @amirtds @xscrio diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ccd1c9b49c28..cbad39a52923 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,7 +18,7 @@ jobs: PULL_REQUEST_BODY: | This is an automated pull request from branch `hawthorn/main` into `hawthorn/prod` (production). Please review the changes and merge this pull request _before_ running the Tahoe production Cloud Build deployment. - PULL_REQUEST_REVIEWERS: "johnbaldwin amirtds bryanlandia" + PULL_REQUEST_REVIEWERS: "amirtds bryanlandia xscrio" hawthorn-to-juniper-sync: name: PullRequestAction runs-on: ubuntu-latest @@ -35,5 +35,5 @@ jobs: This is meant for making sure all of our Hawthorn changes gets merge into Juniper otherwise Juniper would stall. If tests passes merge this pull request. If there are merge conflicts, it needs to be resolved manually in a seperate pull request. - PULL_REQUEST_REVIEWERS: "melvinsoft shadinaif OmarIthawi thraxil" + PULL_REQUEST_REVIEWERS: "bryanlandia amirtds xscrio" diff --git a/.github/workflows/report_conflicts.yml b/.github/workflows/report_conflicts.yml index 1239066f1f87..96ff1955f55d 100644 --- a/.github/workflows/report_conflicts.yml +++ b/.github/workflows/report_conflicts.yml @@ -2,13 +2,13 @@ on: [pull_request] name: 'Merge conflicts' jobs: - report_master: - name: 'koa, lilac, maple, nutmeg and master' + report: + name: 'nutmeg and master' uses: appsembler/action-conflict-counter/.github/workflows/report-via-comment.yml@main with: local_base_branch: ${{ github.base_ref }} - upstream_repo: 'https://github.com/edx/edx-platform.git' - upstream_branches: 'open-release/koa.master,open-release/lilac.master,open-release/maple.master,open-release/nutmeg.master,master' + upstream_repo: 'https://github.com/openedx/edx-platform.git' + upstream_branches: 'open-release/nutmeg.master,master' exclude_paths: 'cms/static/js/,conf/locale/,lms/static/js/,package.json,package-lock.json,.github/' secrets: custom_github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sync_nutmeg_with_juniper.yml b/.github/workflows/sync_nutmeg_with_juniper.yml index f92283f378d1..c1dbb1130b64 100644 --- a/.github/workflows/sync_nutmeg_with_juniper.yml +++ b/.github/workflows/sync_nutmeg_with_juniper.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: pull-request-action - uses: vsoch/pull-request-action@1.0.19 + uses: vsoch/pull-request-action@1.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BRANCH_PREFIX: "main" diff --git a/.github/workflows/sync_prod_with_main.yml b/.github/workflows/sync_prod_with_main.yml index 4927eb46f65d..358f6072bf33 100644 --- a/.github/workflows/sync_prod_with_main.yml +++ b/.github/workflows/sync_prod_with_main.yml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - name: pull-request-action - uses: vsoch/pull-request-action@1.0.19 + uses: vsoch/pull-request-action@1.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BRANCH_PREFIX: "main" PULL_REQUEST_BRANCH: "prod" PULL_REQUEST_TITLE: "Update from `main` (production)" - PULL_REQUEST_REVIEWERS: "melvinsoft OmarIthawi thraxil shadinaif" + PULL_REQUEST_REVIEWERS: "VladyslavTy daniilly" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a110bf2f8fd2..b96fe22dde49 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,8 @@ jobs: - lms-1 - lms-2 - mte + - legacy-amc-tests + - db-migrations - studio steps: @@ -32,6 +34,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + env: + PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" - name: Install dependencies # TODO: Remove tox-pip-version once we upgrade to Koa+, or whenever we have addressed pip 20.3 strict issues. run: | diff --git a/Dockerfile.tutor b/Dockerfile.tutor index deb29f5c1b9f..fe47d6ea685d 100644 --- a/Dockerfile.tutor +++ b/Dockerfile.tutor @@ -17,23 +17,25 @@ RUN pip install -r ./requirements/edx/base.txt \ # Sync with `edx-configs` `appsembler/tahoe/us/juniper/prod/files/server-vars.yml` RUN echo "Installing pip packages:" \ - && pip install openedx-scorm-xblock==10.4.0 \ - && pip install xblock-launchcontainer==2.3.1 \ + && pip install xblock-launchcontainer==4.0.0 \ && pip install xblock-prismjs==0.1.4 \ && pip install xblock-problem-builder==4.1.9 \ && echo \ + && pip install https://github.com/appsembler/openedx-scorm-xblock/archive/refs/tags/v15.1.0-appsembler-tahoe-compat.tar.gz \ && pip install https://github.com/appsembler/pdfXBlock/archive/v0.3.1.tar.gz \ && pip install https://github.com/edx/xblock-free-text-response/archive/4149cc450.tar.gz \ && pip install https://github.com/pmitros/FeedbackXBlock/archive/v1.1.tar.gz \ && pip install https://github.com/ubc/ubcpi/archive/1.0.0.tar.gz \ && echo \ - && pip install course-access-groups==0.5.4 \ - && pip install figures==0.4.1 \ + && pip install course-access-groups==0.6.1 \ + && pip install figures==0.4.4 \ && pip install tahoe-figures-plugins==0.1.1 \ && pip install tahoe-lti==0.3.0 \ - && pip install tahoe-scorm==0.1.2 \ + && pip install tahoe-scorm==0.1.4 \ + && pip install xblock-grade-fetcher==0.5.0 \ + && pip install django-manage-admins==0.1.0 \ && echo \ - && pip install https://github.com/appsembler/openedx-completion-aggregator/archive/3.0.3-2021-may-18-bug-fixes.tar.gz \ + && pip install https://github.com/appsembler/openedx-completion-aggregator/archive/3.0.3-2023-mar-27-revert-use-of-task-track.tar.gz \ && echo "Finished installing pip packages." EXPOSE 8000 diff --git a/cms/djangoapps/appsembler/apps.py b/cms/djangoapps/appsembler/apps.py new file mode 100644 index 000000000000..1f7c35a3b84c --- /dev/null +++ b/cms/djangoapps/appsembler/apps.py @@ -0,0 +1,14 @@ +""" +Appsembler CMS App Configuration +""" + + +from django.apps import AppConfig + + +class CMSAppsemblerConfig(AppConfig): + """ + Application Configuration for Badges. + """ + name = u'appsembler' + plugin_app = {} diff --git a/test_root/log/.gitkeep b/cms/djangoapps/appsembler/management/__init__.py similarity index 100% rename from test_root/log/.gitkeep rename to cms/djangoapps/appsembler/management/__init__.py diff --git a/cms/djangoapps/appsembler/management/commands/__init__.py b/cms/djangoapps/appsembler/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py b/cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py new file mode 100644 index 000000000000..2c6897fff6da --- /dev/null +++ b/cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py @@ -0,0 +1,100 @@ +""" +Command to remove courses without associated organization. + +This command is intended as a follow-up step after `remove_site` but can be run independently. +""" + +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings +from opaque_keys.edx.keys import CourseKey + +from xmodule.contentstore.django import contentstore +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore + +from contentstore.utils import delete_course + +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +from openedx.core.djangoapps.appsembler.sites.deletion_utils import ( + confirm_deletion, +) + + +def get_deletable_course_keys_from_mongo(): + """ + Get keys of courses without active organization. + """ + mongodb_course_keys = {str(mongodb_course.id) for mongodb_course in modulestore().get_course_summaries()} + mysql_course_keys = {str(mysql_course_key) for mysql_course_key in CourseOverview.get_all_course_keys()} + return list(mongodb_course_keys - mysql_course_keys) + + +def delete_course_and_assets(course_key): + """ + Delete all courses without active organization. + """ + course_key_obj = CourseKey.from_string(course_key) + delete_course(course_key_obj, ModuleStoreEnum.UserID.mgmt_command, keep_instructors=False) + contentstore().delete_all_course_assets(course_key_obj) + + +def cms_remove_stray_courses(commit, limit): + """ + Remove all courses from mongodb that has no CourseOverview entry in MySQL. + """ + course_keys = get_deletable_course_keys_from_mongo() + if limit: + course_keys = course_keys[:limit] + + if not course_keys: + raise CommandError('No courses found to delete.') + + str_course_list = [str(course_key) for course_key in course_keys] + print('Preparing to delete:') + print('\n'.join(str_course_list)) + commit = confirm_deletion( + question='Do you confirm to delete the courses from CMS?', + commit=commit, + ) + + for course_key in course_keys: + if commit: + print('Deleting course: {}'.format(course_key)) + delete_course_and_assets(course_key) + else: + print('[Dry run] deleting course: {}'.format(course_key)) + + print('Finished removing deletable courses') + + +class Command(BaseCommand): + help = "Delete courses that don't belong to organization in `get_active_organizations()`." + + def add_arguments(self, parser): + parser.add_argument( + '--limit', + dest='limit', + default=1, + type=int, + help='Max courses to delete, use 0 to delete all courses.', + ) + + parser.add_argument( + '--commit', + dest='commit', + action='store_true', + help='Remove courses, otherwise only the log will be printed.', + ) + + parser.add_argument( + '--dry-run', + dest='commit', + action='store_false', + help='Do not remove courses, only print the logs.', + ) + + def handle(self, *args, **options): + if settings.ROOT_URLCONF != 'cms.urls': + raise CommandError('This command can only be run in CMS.') + + cms_remove_stray_courses(commit=options.get('commit'), limit=options['limit']) diff --git a/cms/djangoapps/appsembler/tests/test_deletion_command.py b/cms/djangoapps/appsembler/tests/test_deletion_command.py new file mode 100644 index 000000000000..cf95a801ffa2 --- /dev/null +++ b/cms/djangoapps/appsembler/tests/test_deletion_command.py @@ -0,0 +1,35 @@ +""" + +""" + +from django.core.management import call_command, CommandError + +from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + + +class DeletionCommandTestCase(ModuleStoreTestCase): + def test_cms_remove_stray_courses_command_no_courses(self): + """ + Raise CommandError if there's no courses to delete. + """ + with self.assertRaises(CommandError): + call_command('cms_remove_stray_courses') + + def test_cms_remove_stray_courses_command(self): + """ + Removes all courses that has only MongoDB entry. + """ + CourseFactory.create() + call_command('cms_remove_stray_courses') + + def test_cms_remove_stray_courses_command_non_to_delete(self): + """ + Should not remove courses from MongoDB if it has a MySQL CourseOverview entry. + """ + course = CourseFactory.create() + CourseOverviewFactory.create(id=course.id) + + with self.assertRaises(CommandError): + call_command('cms_remove_stray_courses') diff --git a/cms/djangoapps/appsembler/tests/test_studio_logout_view.py b/cms/djangoapps/appsembler/tests/test_studio_logout_view.py new file mode 100644 index 000000000000..d6c243cf13f2 --- /dev/null +++ b/cms/djangoapps/appsembler/tests/test_studio_logout_view.py @@ -0,0 +1,145 @@ +""" +Tests for APPSEMBLER_MULTI_TENANT_EMAILS in Studio logout. + +Special note: + +This test module needs to patch `cms.urls.urlpatterns` to include urlpatterns +from `cms.djangoapps.appsembler.urls`. This works by overriding the +`doango.conf.settings.ROOT_URLCONF` with `django.test.utils.override_settings` +at the TestCase class level with the `urlpatterns` list declared in the module +containing the TestCase class. + +For this test module, we've added a `urlpatterns` module level variable and +assigned it the value of `cms.urls.urlpatterns` then appended the conditionally +included urlpatterns we need to run the tests. + +Then we add `@override_settings(ROOT_URLCONF=__name__)` to the TestClass + +There are other ways to do this. However, this is simple and does not require +our code to explicitly hack `sys.modules` reloading +""" +from unittest.mock import Mock, patch + +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import auth +from django.contrib.auth.models import AnonymousUser +from django.urls import reverse +from django.test import RequestFactory, TestCase +from django.test.utils import override_settings +from rest_framework import status +from tahoe_sites.api import add_user_to_organization, create_tahoe_site + +from student.tests.factories import UserFactory +import cms.urls +from cms.djangoapps.appsembler.views import get_logout_redirect_url + + +# Set the urlpatterns we want to use for our tests in this module only +urlpatterns = cms.urls.urlpatterns + [ + url(r'', include('cms.djangoapps.appsembler.urls')) +] + + +@override_settings(ROOT_URLCONF=__name__) # the module that contains `urlpatterns` +@override_settings(LOGOUT_REDIRECT_URL='home') # ensure that we have a value for LOGOUT_REDIRECT_URL +@patch.dict('django.conf.settings.FEATURES', {'TAHOE_STUDIO_LOCAL_LOGIN': True}) +class TestStudioLogoutView(TestCase): + """ + Testing the APPSEMBLER_MULTI_TENANT_EMAILS feature when enabled in Studio. + """ + BLUE = 'blue1' + EMAIL = 'customer@example.com' + PASSWORD = 'xyz' + DOMAIN = 'testdomain.com' + SHORT_NAME = 'testdomain' + + def setUp(self): + super(TestStudioLogoutView, self).setUp() + self.url = reverse('logout') + self.user = UserFactory.create(email=self.EMAIL, password=self.PASSWORD) + add_user_to_organization( + user=self.user, + organization=create_tahoe_site(domain=self.DOMAIN, short_name=self.SHORT_NAME)['organization'] + ) + self.request = RequestFactory() + self.request.is_secure = Mock(return_value=False) + self.lms_url = 'http://{site_domain}/logout'.format(site_domain=self.DOMAIN) + + def test_logout_must_be_authenticated(self): + """ + Test logout from studio must be authenticated + """ + response = self.client.get(self.url) + assert response.status_code == status.HTTP_302_FOUND + assert not response.content + assert '?next=/logout' in response.url + + def test_logout_normal_user(self): + """ + Test logout from studio for normal users (meaning that they are linked to an organization). It is expected + that a logout then redirect to LMS will be performed + """ + self.client.login(username=self.user.username, password=self.PASSWORD) + assert auth.get_user(self.client).is_authenticated + + response = self.client.get(self.url) + assert not auth.get_user(self.client).is_authenticated + + assert response.status_code == status.HTTP_302_FOUND + assert not response.content + assert response.url == self.lms_url + + def test_logout_staff_user(self): + """ + Test logout from studio for staff users (meaning that they are not linked to any organization). It is expected + that a logout then a redirect to settings.LOGOUT_REDIRECT_URL will be performed + """ + # Not necessary to set the user as staff, the thing we need to test is when it lacks a link to an organization + user = UserFactory.create(email=self.EMAIL, password=self.PASSWORD) + + self.client.login(username=user.username, password=self.PASSWORD) + assert auth.get_user(self.client).is_authenticated + + response = self.client.get(self.url) + assert not auth.get_user(self.client).is_authenticated + + assert response.status_code == status.HTTP_302_FOUND + assert not response.content + assert response.url == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_no_request(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if the request is None + """ + assert get_logout_redirect_url(request=None) == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_no_user(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if no user is logged in + """ + assert not hasattr(self.request, 'user') + assert get_logout_redirect_url(request=self.request) == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_anonymous(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if the user is anonymous + """ + self.request.user = AnonymousUser() + assert get_logout_redirect_url(request=self.request) == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_user(self): + """ + Verify that get_logout_redirect_url will return the LMS URL related to the user + """ + self.request.user = self.user + assert get_logout_redirect_url(request=self.request) == self.lms_url + + def test_get_logout_redirect_url_staff(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if the user is not linked to + any organization (staff users and superusers) + """ + user = UserFactory.create(email=self.EMAIL, password=self.PASSWORD) + self.request.user = user + assert get_logout_redirect_url(request=self.request) == reverse(settings.LOGOUT_REDIRECT_URL) diff --git a/cms/djangoapps/appsembler/urls.py b/cms/djangoapps/appsembler/urls.py index 200913eeb344..6562882ef50c 100644 --- a/cms/djangoapps/appsembler/urls.py +++ b/cms/djangoapps/appsembler/urls.py @@ -5,13 +5,10 @@ We have this code in the Appsembler CMS app to help isolate custom code """ -from django.conf import settings from django.urls import path -from django.contrib.auth.views import LogoutView -from .views import LoginView +from .views import LoginView, StudioLogoutView urlpatterns = [ path('login/', LoginView.as_view(), name='login'), - path('logout/', LogoutView.as_view( - next_page=settings.LOGOUT_REDIRECT_URL), name='logout'), + path('logout/', StudioLogoutView.as_view(), name='logout'), ] diff --git a/cms/djangoapps/appsembler/views.py b/cms/djangoapps/appsembler/views.py index c3faf55bc4f0..77297ca02e9f 100644 --- a/cms/djangoapps/appsembler/views.py +++ b/cms/djangoapps/appsembler/views.py @@ -9,6 +9,9 @@ import logging from django.conf import settings from django.contrib.auth import authenticate, get_user_model, login +from django.contrib.auth.decorators import login_required +from django.contrib.auth.views import LogoutView as DjangoLogoutView +from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.http import HttpResponseServerError from django.shortcuts import redirect @@ -18,7 +21,11 @@ from django.views import View from django.views.decorators.clickjacking import xframe_options_deny from django.views.decorators.csrf import csrf_protect -from tahoe_sites.api import deprecated_get_admin_users_queryset_by_email +from tahoe_sites.api import ( + deprecated_get_admin_users_queryset_by_email, + get_organization_for_user, + get_site_by_organization, +) from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.user_authn.utils import is_safe_login_or_logout_redirect @@ -274,3 +281,39 @@ def log_multiple_objects_returned(self): def render_login_page_with_error(self, error_code): return render_login_page( login_error_message=self.error_messages[error_code]) + + +def get_logout_redirect_url(request): + """ + Return logout redirect url using the site related to the given user if possible. + Otherwise, return settings.LOGOUT_REDIRECT_URL + + :return: logout redirect url or settings.LOGOUT_REDIRECT_URL + """ + user = getattr(request, 'user', None) + if not user or not user.is_authenticated: + return reverse(settings.LOGOUT_REDIRECT_URL) + + try: + organization = get_organization_for_user(user=user) + except ObjectDoesNotExist: + return reverse(settings.LOGOUT_REDIRECT_URL) + site = get_site_by_organization(organization=organization) + + return '{protocol}://{site_domain}/logout'.format( + protocol='https' if request.is_secure() else 'http', + site_domain=site.domain + ) + + +class StudioLogoutView(View): + """ + Studio Logout View + """ + @method_decorator(csrf_protect) + @method_decorator(login_required) + def get(self, request): + """ + Perform logout from studio, and redirect to LMS home page + """ + return DjangoLogoutView.as_view(next_page=get_logout_redirect_url(request))(request) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index 636c3f22e2b9..98fd0e931c66 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -311,7 +311,7 @@ def async_migrate_transcript_subtask(*args, **kwargs): # pylint: disable=unused command_run=command_run, edx_video_id=edx_video_id, language_code=language_code, - transcript_content=transcript_content, + transcript_content=transcript_content.encode(), file_format=Transcript.SJSON, force_update=force_update, ) diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 5a8f75c76621..976c27b98103 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -708,7 +708,7 @@ def get_courses_accessible_to_user(request, org=None): return courses, in_process_course_actions -def _process_courses_list(courses_iter, in_process_course_actions, split_archived=False): +def _process_courses_list(courses_iter, in_process_course_actions, split_archived=False, ignore_archived=False): """ Iterates over the list of courses to be displayed to the user, and: @@ -738,6 +738,22 @@ def format_course_for_view(course): archived_courses = [] for course in courses_iter: + # BEGIN temp fix for Sounds Write + + # TODO: if a user has tons of courses like Sounds Write (571), some pages will time out. + # This is a temporary patch to prevent Sounds-Write requets timing out. The right fix + # is to optimize the DB queries. + + # This skipping step can be potentially kept since it's just used for the "prerequisite" course list + # where we do not need to list archived courses. + if course.has_ended() and ignore_archived: + continue + + if course.has_ended() and course.location.org == "sounds-write": + continue + + # END temp fix for Sounds Write + if isinstance(course, ErrorDescriptor) or (course.id in in_process_action_course_keys): continue @@ -1114,11 +1130,11 @@ def settings_handler(request, course_key_string): 'upgrade_deadline': upgrade_deadline, } if is_prerequisite_courses_enabled(): - courses, in_process_course_actions = get_courses_accessible_to_user(request) + courses, in_process_course_actions = get_courses_accessible_to_user(request, course_module.location.org) # exclude current course from the list of available courses courses = (course for course in courses if course.id != course_key) if courses: - courses, __ = _process_courses_list(courses, in_process_course_actions) + courses, __ = _process_courses_list(courses, in_process_course_actions, ignore_archived=True) settings_context.update({'possible_pre_requisite_courses': list(courses)}) if credit_eligibility_enabled: @@ -1799,3 +1815,4 @@ def _get_course_creator_status(user): course_creator_status = 'granted' return course_creator_status + diff --git a/cms/djangoapps/contentstore/views/course.py.edited b/cms/djangoapps/contentstore/views/course.py.edited new file mode 100644 index 000000000000..9a4557447021 --- /dev/null +++ b/cms/djangoapps/contentstore/views/course.py.edited @@ -0,0 +1,1804 @@ +""" +Views related to operations on course objects +""" + + +import copy +import json +import logging +import random +import re +import string +from collections import defaultdict + +import django.utils +import six +from ccx_keys.locator import CCXLocator +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied, ValidationError +from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseNotFound +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import ensure_csrf_cookie +from django.views.decorators.http import require_GET, require_http_methods +from milestones import api as milestones_api +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import BlockUsageLocator +from six import text_type +from six.moves import filter + +from contentstore.course_group_config import ( + COHORT_SCHEME, + ENROLLMENT_SCHEME, + RANDOM_SCHEME, + GroupConfiguration, + GroupConfigurationsValidationError +) +from contentstore.course_info_model import delete_course_update, get_course_updates, update_course_updates +from contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError +from contentstore.tasks import rerun_course as rerun_course_task +from contentstore.utils import ( + add_instructor, + get_lms_link_for_item, + initialize_permissions, + remove_all_instructors, + reverse_course_url, + reverse_library_url, + reverse_url, + reverse_usage_url +) +from contentstore.views.entrance_exam import create_entrance_exam, delete_entrance_exam, update_entrance_exam +from course_action_state.managers import CourseActionStateItemNotFoundError +from course_action_state.models import CourseRerunState, CourseRerunUIStateManager +from course_creators.views import add_user_with_status_unrequested, get_course_creator_status +from course_modes.models import CourseMode +from edxmako.shortcuts import render_to_response +from models.settings.course_grading import CourseGradingModel +from models.settings.course_metadata import CourseMetadata +from models.settings.encoder import CourseSettingsEncoder +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course +from openedx.core.djangoapps.credit.tasks import update_credit_course_requirements +from openedx.core.djangoapps.models.course_details import CourseDetails +from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers +from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace +from openedx.core.djangolib.js_utils import dump_js_escaped_json +from openedx.core.lib.course_tabs import CourseTabPluginManager +from openedx.core.lib.courses import course_image_url +from openedx.features.content_type_gating.models import ContentTypeGatingConfig +from openedx.features.content_type_gating.partitions import CONTENT_TYPE_GATING_SCHEME +from openedx.features.course_experience.waffle import ENABLE_COURSE_ABOUT_SIDEBAR_HTML +from openedx.features.course_experience.waffle import waffle as course_experience_waffle +from student import auth +from student.auth import has_course_author_access, has_studio_read_access, has_studio_write_access +from student.roles import CourseCreatorRole, CourseInstructorRole, CourseStaffRole, GlobalStaff, UserBasedRole +from util.course import get_link_for_about_page +from util.date_utils import get_default_time_display +from util.json_request import JsonResponse, JsonResponseBadRequest, expect_json +from util.milestones_helpers import ( + is_entrance_exams_enabled, + is_prerequisite_courses_enabled, + is_valid_course_key, + remove_prerequisite_course, + set_prerequisite_courses, + get_namespace_choices, + generate_milestone_namespace +) +from util.organizations_helpers import add_organization_course, get_organization_by_short_name, organizations_enabled +from util.string_utils import _has_non_ascii_characters +from xblock_django.api import deprecated_xblocks +from xmodule.contentstore.content import StaticContent +from xmodule.course_module import DEFAULT_START_DATE, CourseFields +from xmodule.error_module import ErrorDescriptor +from xmodule.modulestore import EdxJSONEncoder +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError +from xmodule.partitions.partitions import UserPartition +from xmodule.tabs import CourseTab, CourseTabList, InvalidTabsException + +from .component import ADVANCED_COMPONENT_TYPES +from .item import create_xblock_info +from .library import LIBRARIES_ENABLED, get_library_creator_status + +log = logging.getLogger(__name__) + + +__all__ = ['course_info_handler', 'course_handler', 'course_listing', + 'course_info_update_handler', 'course_search_index_handler', + 'course_rerun_handler', + 'settings_handler', + 'grading_handler', + 'advanced_settings_handler', + 'course_notifications_handler', + 'textbooks_list_handler', 'textbooks_detail_handler', + 'group_configurations_list_handler', 'group_configurations_detail_handler'] + +WAFFLE_NAMESPACE = 'studio_home' + + +class AccessListFallback(Exception): + """ + An exception that is raised whenever we need to `fall back` to fetching *all* courses + available to a user, rather than using a shorter method (i.e. fetching by group) + """ + pass + + +def get_course_and_check_access(course_key, user, depth=0): + """ + Internal method used to calculate and return the locator and course module + for the view functions in this file. + """ + if not has_studio_read_access(user, course_key): + raise PermissionDenied() + course_module = modulestore().get_course(course_key, depth=depth) + return course_module + + +def reindex_course_and_check_access(course_key, user): + """ + Internal method used to restart indexing on a course. + """ + if not has_course_author_access(user, course_key): + raise PermissionDenied() + return CoursewareSearchIndexer.do_course_reindex(modulestore(), course_key) + + +@login_required +def course_notifications_handler(request, course_key_string=None, action_state_id=None): + """ + Handle incoming requests for notifications in a RESTful way. + + course_key_string and action_state_id must both be set; else a HttpBadResponseRequest is returned. + + For each of these operations, the requesting user must have access to the course; + else a PermissionDenied error is returned. + + GET + json: return json representing information about the notification (action, state, etc) + DELETE + json: return json repressing success or failure of dismissal/deletion of the notification + PUT + Raises a NotImplementedError. + POST + Raises a NotImplementedError. + """ + # ensure that we have a course and an action state + if not course_key_string or not action_state_id: + return HttpResponseBadRequest() + + response_format = request.GET.get('format') or request.POST.get('format') or 'html' + + course_key = CourseKey.from_string(course_key_string) + + if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): + if not has_studio_write_access(request.user, course_key): + raise PermissionDenied() + if request.method == 'GET': + return _course_notifications_json_get(action_state_id) + elif request.method == 'DELETE': + # we assume any delete requests dismiss actions from the UI + return _dismiss_notification(request, action_state_id) + elif request.method == 'PUT': + raise NotImplementedError() + elif request.method == 'POST': + raise NotImplementedError() + else: + return HttpResponseBadRequest() + else: + return HttpResponseNotFound() + + +def _course_notifications_json_get(course_action_state_id): + """ + Return the action and the action state for the given id + """ + try: + action_state = CourseRerunState.objects.find_first(id=course_action_state_id) + except CourseActionStateItemNotFoundError: + return HttpResponseBadRequest() + + action_state_info = { + 'action': action_state.action, + 'state': action_state.state, + 'should_display': action_state.should_display + } + return JsonResponse(action_state_info) + + +def _dismiss_notification(request, course_action_state_id): + """ + Update the display of the course notification + """ + try: + action_state = CourseRerunState.objects.find_first(id=course_action_state_id) + + except CourseActionStateItemNotFoundError: + # Can't dismiss a notification that doesn't exist in the first place + return HttpResponseBadRequest() + + if action_state.state == CourseRerunUIStateManager.State.FAILED: + # We remove all permissions for this course key at this time, since + # no further access is required to a course that failed to be created. + remove_all_instructors(action_state.course_key) + + # The CourseRerunState is no longer needed by the UI; delete + action_state.delete() + + return JsonResponse({'success': True}) + + +@login_required +def course_handler(request, course_key_string=None): + """ + The restful handler for course specific requests. + It provides the course tree with the necessary information for identifying and labeling the parts. The root + will typically be a 'course' object but may not be especially as we support modules. + + GET + html: return course listing page if not given a course id + html: return html page overview for the given course if given a course id + json: return json representing the course branch's index entry as well as dag w/ all of the children + replaced w/ json docs where each doc has {'_id': , 'display_name': , 'children': } + POST + json: create a course, return resulting json + descriptor (same as in GET course/...). Leaving off /branch/draft would imply create the course w/ default + branches. Cannot change the structure contents ('_id', 'display_name', 'children') but can change the + index entry. + PUT + json: update this course (index entry not xblock) such as repointing head, changing display name, org, + course, run. Return same json as above. + DELETE + json: delete this branch from this course (leaving off /branch/draft would imply delete the course) + """ + try: + response_format = request.GET.get('format') or request.POST.get('format') or 'html' + if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): + if request.method == 'GET': + course_key = CourseKey.from_string(course_key_string) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user, depth=None) + return JsonResponse(_course_outline_json(request, course_module)) + elif request.method == 'POST': # not sure if this is only post. If one will have ids, it goes after access + return _create_or_rerun_course(request) + elif not has_studio_write_access(request.user, CourseKey.from_string(course_key_string)): + raise PermissionDenied() + elif request.method == 'PUT': + raise NotImplementedError() + elif request.method == 'DELETE': + raise NotImplementedError() + else: + return HttpResponseBadRequest() + elif request.method == 'GET': # assume html + if course_key_string is None: + return redirect(reverse('home')) + else: + return course_index(request, CourseKey.from_string(course_key_string)) + else: + return HttpResponseNotFound() + except InvalidKeyError: + raise Http404 + + +@login_required +@ensure_csrf_cookie +@require_http_methods(["GET"]) +def course_rerun_handler(request, course_key_string): + """ + The restful handler for course reruns. + GET + html: return html page with form to rerun a course for the given course id + """ + # Only global staff (PMs) are able to rerun courses during the soft launch + # Appsembler: Also course staff can do reruns + course_key = CourseKey.from_string(course_key_string) + if not CourseStaffRole(course_key).has_user(request.user): + if not CourseInstructorRole(course_key).has_user(request.user): + if not GlobalStaff().has_user(request.user): + raise PermissionDenied() + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user, depth=3) + if request.method == 'GET': + return render_to_response('course-create-rerun.html', { + 'source_course_key': course_key, + 'display_name': course_module.display_name, + 'user': request.user, + 'course_creator_status': _get_course_creator_status(request.user), + 'allow_unicode_course_id': settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID', False) + }) + + +@login_required +@ensure_csrf_cookie +@require_GET +def course_search_index_handler(request, course_key_string): + """ + The restful handler for course indexing. + GET + html: return status of indexing task + json: return status of indexing task + """ + # Only global staff (PMs) are able to index courses + if not GlobalStaff().has_user(request.user): + raise PermissionDenied() + course_key = CourseKey.from_string(course_key_string) + content_type = request.META.get('CONTENT_TYPE', None) + if content_type is None: + content_type = "application/json; charset=utf-8" + with modulestore().bulk_operations(course_key): + try: + reindex_course_and_check_access(course_key, request.user) + except SearchIndexingError as search_err: + return HttpResponse(dump_js_escaped_json({ + "user_message": search_err.error_list + }), content_type=content_type, status=500) + return HttpResponse(dump_js_escaped_json({ + "user_message": _("Course has been successfully reindexed.") + }), content_type=content_type, status=200) + + +def _course_outline_json(request, course_module): + """ + Returns a JSON representation of the course module and recursively all of its children. + """ + is_concise = request.GET.get('format') == 'concise' + include_children_predicate = lambda xblock: not xblock.category == 'vertical' + if is_concise: + include_children_predicate = lambda xblock: xblock.has_children + return create_xblock_info( + course_module, + include_child_info=True, + course_outline=False if is_concise else True, + include_children_predicate=include_children_predicate, + is_concise=is_concise, + user=request.user + ) + + +def get_in_process_course_actions(request): + """ + Get all in-process course actions + """ + return [ + course for course in + CourseRerunState.objects.find_all( + exclude_args={'state': CourseRerunUIStateManager.State.SUCCEEDED}, + should_display=True, + ) + if has_studio_read_access(request.user, course.course_key) + ] + + +def _accessible_courses_summary_iter(request, org=None): + """ + List all courses available to the logged in user by iterating through all the courses + + Arguments: + request: the request object + org (string): if not None, this value will limit the courses returned. An empty + string will result in no courses, and otherwise only courses with the + specified org will be returned. The default value is None. + """ + def course_filter(course_summary): + """ + Filter out unusable and inaccessible courses + """ + # TODO remove this condition when templates purged from db + if course_summary.location.course == 'templates': + return False + + return has_studio_read_access(request.user, course_summary.id) + if org is not None: + courses_summary = [] if org == '' else CourseOverview.get_all_courses(orgs=[org]) + else: + courses_summary = modulestore().get_course_summaries() + courses_summary = six.moves.filter(course_filter, courses_summary) + in_process_course_actions = get_in_process_course_actions(request) + return courses_summary, in_process_course_actions + + +def _accessible_courses_iter(request): + """ + List all courses available to the logged in user by iterating through all the courses. + """ + def course_filter(course): + """ + Filter out unusable and inaccessible courses + """ + if isinstance(course, ErrorDescriptor): + return False + + # Custom Courses for edX (CCX) is an edX feature for re-using course content. + # CCXs cannot be edited in Studio (aka cms) and should not be shown in this dashboard. + if isinstance(course.id, CCXLocator): + return False + + # TODO remove this condition when templates purged from db + if course.location.course == 'templates': + return False + + return has_studio_read_access(request.user, course.id) + + courses = six.moves.filter(course_filter, modulestore().get_courses()) + + in_process_course_actions = get_in_process_course_actions(request) + return courses, in_process_course_actions + + +def _accessible_courses_iter_for_tests(request): + """ + List all courses available to the logged in user by iterating through all the courses. + CourseSummary objects are used for listing purposes. + This method is only used by tests. + """ + def course_filter(course): + """ + Filter out unusable and inaccessible courses + """ + + # Custom Courses for edX (CCX) is an edX feature for re-using course content. + # CCXs cannot be edited in Studio (aka cms) and should not be shown in this dashboard. + if isinstance(course.id, CCXLocator): + return False + + # TODO remove this condition when templates purged from db + if course.location.course == 'templates': + return False + + return has_studio_read_access(request.user, course.id) + + courses = six.moves.filter(course_filter, modulestore().get_course_summaries()) + + in_process_course_actions = get_in_process_course_actions(request) + return courses, in_process_course_actions + + +def _accessible_courses_list_from_groups(request): + """ + List all courses available to the logged in user by reversing access group names + """ + def filter_ccx(course_access): + """ CCXs cannot be edited in Studio and should not be shown in this dashboard """ + return not isinstance(course_access.course_id, CCXLocator) + + instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role() + staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + all_courses = list(filter(filter_ccx, instructor_courses | staff_courses)) + courses_list = [] + course_keys = {} + + for course_access in all_courses: + if course_access.course_id is None: + raise AccessListFallback + course_keys[course_access.course_id] = course_access.course_id + + course_keys = list(course_keys.values()) + + if course_keys: + courses_list = modulestore().get_course_summaries(course_keys=course_keys) + + return courses_list, [] + + +def _accessible_libraries_iter(user, org=None): + """ + List all libraries available to the logged in user by iterating through all libraries. + + org (string): if not None, this value will limit the libraries returned. An empty + string will result in no libraries, and otherwise only libraries with the + specified org will be returned. The default value is None. + """ + if org is not None: + libraries = [] if org == '' else modulestore().get_libraries(org=org) + else: + libraries = modulestore().get_library_summaries() + # No need to worry about ErrorDescriptors - split's get_libraries() never returns them. + return (lib for lib in libraries if has_studio_read_access(user, lib.location.library_key)) + + +@login_required +@ensure_csrf_cookie +def course_listing(request): + """ + List all courses and libraries available to the logged in user + """ + + optimization_enabled = GlobalStaff().has_user(request.user) and \ + WaffleSwitchNamespace(name=WAFFLE_NAMESPACE).is_enabled(u'enable_global_staff_optimization') + + org = request.GET.get('org', '') if optimization_enabled else None + courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org) + user = request.user + libraries = _accessible_libraries_iter(request.user, org) if LIBRARIES_ENABLED else [] + + def format_in_process_course_view(uca): + """ + Return a dict of the data which the view requires for each unsucceeded course + """ + return { + u'display_name': uca.display_name, + u'course_key': six.text_type(uca.course_key), + u'org': uca.course_key.org, + u'number': uca.course_key.course, + u'run': uca.course_key.run, + u'is_failed': True if uca.state == CourseRerunUIStateManager.State.FAILED else False, + u'is_in_progress': True if uca.state == CourseRerunUIStateManager.State.IN_PROGRESS else False, + u'dismiss_link': reverse_course_url( + u'course_notifications_handler', + uca.course_key, + kwargs={ + u'action_state_id': uca.id, + }, + ) if uca.state == CourseRerunUIStateManager.State.FAILED else u'' + } + + def format_library_for_view(library): + """ + Return a dict of the data which the view requires for each library + """ + + return { + u'display_name': library.display_name, + u'library_key': six.text_type(library.location.library_key), + u'url': reverse_library_url(u'library_handler', six.text_type(library.location.library_key)), + u'org': library.display_org_with_default, + u'number': library.display_number_with_default, + u'can_edit': has_studio_write_access(request.user, library.location.library_key), + } + + split_archived = settings.FEATURES.get(u'ENABLE_SEPARATE_ARCHIVED_COURSES', False) + active_courses, archived_courses = _process_courses_list(courses_iter, in_process_course_actions, split_archived) + in_process_course_actions = [format_in_process_course_view(uca) for uca in in_process_course_actions] + + return render_to_response(u'index.html', { + u'courses': active_courses, + u'archived_courses': archived_courses, + u'in_process_course_actions': in_process_course_actions, + u'libraries_enabled': LIBRARIES_ENABLED, + u'libraries': [format_library_for_view(lib) for lib in libraries], + u'show_new_library_button': get_library_creator_status(user), + u'user': user, + u'request_course_creator_url': reverse('request_course_creator'), + u'course_creator_status': _get_course_creator_status(user), + # Appsembler: Using the course creator status instead of `GlobalStaff().has_user(user)` + # to match the Tahoe multi-tenant requirements. + u'rerun_creator_status': _get_course_creator_status(user) == 'granted', + u'allow_unicode_course_id': settings.FEATURES.get(u'ALLOW_UNICODE_COURSE_ID', False), + u'allow_course_reruns': settings.FEATURES.get(u'ALLOW_COURSE_RERUNS', True), + u'optimization_enabled': optimization_enabled + }) + + +def _get_rerun_link_for_item(course_key): + """ Returns the rerun link for the given course key. """ + return reverse_course_url('course_rerun_handler', course_key) + + +def _deprecated_blocks_info(course_module, deprecated_block_types): + """ + Returns deprecation information about `deprecated_block_types` + + Arguments: + course_module (CourseDescriptor): course object + deprecated_block_types (list): list of deprecated blocks types + + Returns: + Dict with following keys: + deprecated_enabled_block_types (list): list containing all deprecated blocks types enabled on this course + blocks (list): List of `deprecated_enabled_block_types` instances and their parent's url + advance_settings_url (str): URL to advance settings page + """ + data = { + 'deprecated_enabled_block_types': [ + block_type for block_type in course_module.advanced_modules if block_type in deprecated_block_types + ], + 'blocks': [], + 'advance_settings_url': reverse_course_url('advanced_settings_handler', course_module.id) + } + + deprecated_blocks = modulestore().get_items( + course_module.id, + qualifiers={ + 'category': re.compile('^' + '$|^'.join(deprecated_block_types) + '$') + } + ) + + for block in deprecated_blocks: + data['blocks'].append([ + reverse_usage_url('container_handler', block.parent), + block.display_name + ]) + + return data + + +@login_required +@ensure_csrf_cookie +def course_index(request, course_key): + """ + Display an editable course overview. + + org, course, name: Attributes of the Location for the item to edit + """ + # A depth of None implies the whole course. The course outline needs this in order to compute has_changes. + # A unit may not have a draft version, but one of its components could, and hence the unit itself has changes. + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user, depth=None) + if not course_module: + raise Http404 + lms_link = get_lms_link_for_item(course_module.location) + reindex_link = None + if settings.FEATURES.get('ENABLE_COURSEWARE_INDEX', False): + if GlobalStaff().has_user(request.user): + reindex_link = "/course/{course_id}/search_reindex".format(course_id=six.text_type(course_key)) + sections = course_module.get_children() + course_structure = _course_outline_json(request, course_module) + locator_to_show = request.GET.get('show', None) + + course_release_date = ( + get_default_time_display(course_module.start) + if course_module.start != DEFAULT_START_DATE + else _("Set Date") + ) + + settings_url = reverse_course_url('settings_handler', course_key) + + try: + current_action = CourseRerunState.objects.find_first(course_key=course_key, should_display=True) + except (ItemNotFoundError, CourseActionStateItemNotFoundError): + current_action = None + + deprecated_block_names = [block.name for block in deprecated_xblocks()] + deprecated_blocks_info = _deprecated_blocks_info(course_module, deprecated_block_names) + + frontend_app_publisher_url = configuration_helpers.get_value_for_org( + course_module.location.org, + 'FRONTEND_APP_PUBLISHER_URL', + settings.FEATURES.get('FRONTEND_APP_PUBLISHER_URL', False) + ) + + return render_to_response('course_outline.html', { + 'language_code': request.LANGUAGE_CODE, + 'context_course': course_module, + 'lms_link': lms_link, + 'sections': sections, + 'course_structure': course_structure, + 'initial_state': course_outline_initial_state(locator_to_show, course_structure) if locator_to_show else None, + 'rerun_notification_id': current_action.id if current_action else None, + 'course_release_date': course_release_date, + 'settings_url': settings_url, + 'reindex_link': reindex_link, + 'deprecated_blocks_info': deprecated_blocks_info, + 'notification_dismiss_url': reverse_course_url( + 'course_notifications_handler', + current_action.course_key, + kwargs={ + 'action_state_id': current_action.id, + }, + ) if current_action else None, + 'frontend_app_publisher_url': frontend_app_publisher_url, + }) + + +def get_courses_accessible_to_user(request, org=None): + """ + Try to get all courses by first reversing django groups and fallback to old method if it fails + Note: overhead of pymongo reads will increase if getting courses from django groups fails + + Arguments: + request: the request object + org (string): for global staff users ONLY, this value will be used to limit + the courses returned. A value of None will have no effect (all courses + returned), an empty string will result in no courses, and otherwise only courses with the + specified org will be returned. The default value is None. + """ + if GlobalStaff().has_user(request.user): + # user has global access so no need to get courses from django groups + courses, in_process_course_actions = _accessible_courses_summary_iter(request, org) + else: + try: + courses, in_process_course_actions = _accessible_courses_list_from_groups(request) + except AccessListFallback: + # user have some old groups or there was some error getting courses from django groups + # so fallback to iterating through all courses + courses, in_process_course_actions = _accessible_courses_summary_iter(request) + return courses, in_process_course_actions + + +def _process_courses_list(courses_iter, in_process_course_actions, split_archived=False): + """ + Iterates over the list of courses to be displayed to the user, and: + + * Removes any in-process courses from the courses list. "In-process" refers to courses + that are in the process of being generated for re-run. + * If split_archived=True, removes any archived courses and returns them in a separate list. + Archived courses have has_ended() == True. + * Formats the returned courses (in both lists) to prepare them for rendering to the view. + """ + def format_course_for_view(course): + """ + Return a dict of the data which the view requires for each course + """ + return { + 'display_name': course.display_name, + 'course_key': six.text_type(course.location.course_key), + 'url': reverse_course_url('course_handler', course.id), + 'lms_link': get_lms_link_for_item(course.location), + 'rerun_link': _get_rerun_link_for_item(course.id), + 'org': course.display_org_with_default, + 'number': course.display_number_with_default, + 'run': course.location.run + } + + in_process_action_course_keys = {uca.course_key for uca in in_process_course_actions} + active_courses = [] + archived_courses = [] + + for course in courses_iter: + # if course.has_ended(): + # continue + + if isinstance(course, ErrorDescriptor) or (course.id in in_process_action_course_keys): + continue + + formatted_course = format_course_for_view(course) + if split_archived and course.has_ended(): + archived_courses.append(formatted_course) + else: + active_courses.append(formatted_course) + + return active_courses, archived_courses + + +def course_outline_initial_state(locator_to_show, course_structure): + """ + Returns the desired initial state for the course outline view. If the 'show' request parameter + was provided, then the view's initial state will be to have the desired item fully expanded + and to scroll to see the new item. + """ + def find_xblock_info(xblock_info, locator): + """ + Finds the xblock info for the specified locator. + """ + if xblock_info['id'] == locator: + return xblock_info + children = xblock_info['child_info']['children'] if xblock_info.get('child_info', None) else None + if children: + for child_xblock_info in children: + result = find_xblock_info(child_xblock_info, locator) + if result: + return result + return None + + def collect_all_locators(locators, xblock_info): + """ + Collect all the locators for an xblock and its children. + """ + locators.append(xblock_info['id']) + children = xblock_info['child_info']['children'] if xblock_info.get('child_info', None) else None + if children: + for child_xblock_info in children: + collect_all_locators(locators, child_xblock_info) + + selected_xblock_info = find_xblock_info(course_structure, locator_to_show) + if not selected_xblock_info: + return None + expanded_locators = [] + collect_all_locators(expanded_locators, selected_xblock_info) + return { + 'locator_to_show': locator_to_show, + 'expanded_locators': expanded_locators + } + + +@expect_json +def _create_or_rerun_course(request): + """ + To be called by requests that create a new destination course (i.e., create_new_course and rerun_course) + Returns the destination course_key and overriding fields for the new course. + Raises DuplicateCourseError and InvalidKeyError + """ + if not auth.user_has_role(request.user, CourseCreatorRole()): + raise PermissionDenied() + + try: + org = request.json.get('org') + course = request.json.get('number', request.json.get('course')) + display_name = request.json.get('display_name') + # force the start date for reruns and allow us to override start via the client + start = request.json.get('start', CourseFields.start.default) + run = request.json.get('run') + + # allow/disable unicode characters in course_id according to settings + if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'): + if _has_non_ascii_characters(org) or _has_non_ascii_characters(course) or _has_non_ascii_characters(run): + return JsonResponse( + {'error': _('Special characters not allowed in organization, course number, and course run.')}, + status=400 + ) + + fields = {'start': start} + if display_name is not None: + fields['display_name'] = display_name + + # Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for + # existing xml courses this cannot be changed in CourseDescriptor. + # # TODO get rid of defining wiki slug in this org/course/run specific way and reconcile + # w/ xmodule.course_module.CourseDescriptor.__init__ + wiki_slug = u"{0}.{1}.{2}".format(org, course, run) + definition_data = {'wiki_slug': wiki_slug} + fields.update(definition_data) + + source_course_key = request.json.get('source_course_key') + if source_course_key: + source_course_key = CourseKey.from_string(source_course_key) + destination_course_key = rerun_course(request.user, source_course_key, org, course, run, fields) + return JsonResponse({ + 'url': reverse_url('course_handler'), + 'destination_course_key': six.text_type(destination_course_key) + }) + else: + try: + new_course = create_new_course(request.user, org, course, run, fields) + return JsonResponse({ + 'url': reverse_course_url('course_handler', new_course.id), + 'course_key': six.text_type(new_course.id), + }) + except ValidationError as ex: + return JsonResponse({'error': text_type(ex)}, status=400) + except DuplicateCourseError: + return JsonResponse({ + 'ErrMsg': _( + 'There is already a course defined with the same ' + 'organization and course number. Please ' + 'change either organization or course number to be unique.' + ), + 'OrgErrMsg': _( + 'Please change either the organization or ' + 'course number so that it is unique.'), + 'CourseErrMsg': _( + 'Please change either the organization or ' + 'course number so that it is unique.'), + }) + except InvalidKeyError as error: + return JsonResponse({ + "ErrMsg": _(u"Unable to create course '{name}'.\n\n{err}").format(name=display_name, err=text_type(error))} + ) + + +def create_new_course(user, org, number, run, fields): + """ + Create a new course run. + + Raises: + DuplicateCourseError: Course run already exists. + """ + org_data = get_organization_by_short_name(org) + if not org_data and organizations_enabled(): + raise ValidationError(_('You must link this course to an organization in order to continue. Organization ' + 'you selected does not exist in the system, you will need to add it to the system')) + store_for_new_course = modulestore().default_modulestore.get_modulestore_type() + new_course = create_new_course_in_store(store_for_new_course, user, org, number, run, fields) + add_organization_course(org_data, new_course.id) + return new_course + + +def create_new_course_in_store(store, user, org, number, run, fields): + """ + Create course in store w/ handling instructor enrollment, permissions, and defaulting the wiki slug. + Separated out b/c command line course creation uses this as well as the web interface. + """ + + # Set default language from settings and enable web certs + fields.update({ + 'language': getattr(settings, 'DEFAULT_COURSE_LANGUAGE', 'en'), + 'cert_html_view_enabled': True, + }) + + with modulestore().default_store(store): + # Creating the course raises DuplicateCourseError if an existing course with this org/name is found + new_course = modulestore().create_course( + org, + number, + run, + user.id, + fields=fields, + ) + + # Make sure user has instructor and staff access to the new course + add_instructor(new_course.id, user, user) + + # Initialize permissions for user in the new course + initialize_permissions(new_course.id, user) + return new_course + + +def rerun_course(user, source_course_key, org, number, run, fields, background=True): + """ + Rerun an existing course. + """ + # verify user has access to the original course + if not has_studio_write_access(user, source_course_key): + raise PermissionDenied() + + # create destination course key + store = modulestore() + with store.default_store('split'): + destination_course_key = store.make_course_key(org, number, run) + + # verify org course and run don't already exist + if store.has_course(destination_course_key, ignore_case=True): + raise DuplicateCourseError(source_course_key, destination_course_key) + + # Make sure user has instructor and staff access to the destination course + # so the user can see the updated status for that course + add_instructor(destination_course_key, user, user) + + # Mark the action as initiated + CourseRerunState.objects.initiated(source_course_key, destination_course_key, user, fields['display_name']) + + # Clear the fields that must be reset for the rerun + fields['advertised_start'] = None + fields['enrollment_start'] = None + fields['enrollment_end'] = None + fields['video_upload_pipeline'] = {} + + json_fields = json.dumps(fields, cls=EdxJSONEncoder) + args = [six.text_type(source_course_key), six.text_type(destination_course_key), user.id, json_fields] + + if background: + rerun_course_task.delay(*args) + else: + rerun_course_task(*args) + + return destination_course_key + + +@login_required +@ensure_csrf_cookie +@require_http_methods(["GET"]) +def course_info_handler(request, course_key_string): + """ + GET + html: return html for editing the course info handouts and updates. + """ + try: + course_key = CourseKey.from_string(course_key_string) + except InvalidKeyError: + raise Http404 + + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + if not course_module: + raise Http404 + if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): + return render_to_response( + 'course_info.html', + { + 'context_course': course_module, + 'updates_url': reverse_course_url('course_info_update_handler', course_key), + 'handouts_locator': course_key.make_usage_key('course_info', 'handouts'), + 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.id), + } + ) + else: + return HttpResponseBadRequest("Only supports html requests") + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT", "DELETE")) +@expect_json +def course_info_update_handler(request, course_key_string, provided_id=None): + """ + restful CRUD operations on course_info updates. + provided_id should be none if it's new (create) and index otherwise. + GET + json: return the course info update models + POST + json: create an update + PUT or DELETE + json: change an existing update + """ + if 'application/json' not in request.META.get('HTTP_ACCEPT', 'application/json'): + return HttpResponseBadRequest("Only supports json requests") + + course_key = CourseKey.from_string(course_key_string) + usage_key = course_key.make_usage_key('course_info', 'updates') + if provided_id == '': + provided_id = None + + # check that logged in user has permissions to this item (GET shouldn't require this level?) + if not has_studio_write_access(request.user, usage_key.course_key): + raise PermissionDenied() + + if request.method == 'GET': + course_updates = get_course_updates(usage_key, provided_id, request.user.id) + if isinstance(course_updates, dict) and course_updates.get('error'): + return JsonResponse(course_updates, course_updates.get('status', 400)) + else: + return JsonResponse(course_updates) + elif request.method == 'DELETE': + try: + return JsonResponse(delete_course_update(usage_key, request.json, provided_id, request.user)) + except: + return HttpResponseBadRequest( + "Failed to delete", + content_type="text/plain" + ) + # can be either and sometimes django is rewriting one to the other: + elif request.method in ('POST', 'PUT'): + try: + return JsonResponse(update_course_updates(usage_key, request.json, provided_id, request.user)) + except: + return HttpResponseBadRequest( + "Failed to save", + content_type="text/plain" + ) + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "PUT", "POST")) +@expect_json +def settings_handler(request, course_key_string): + """ + Course settings for dates and about pages + GET + html: get the page + json: get the CourseDetails model + PUT + json: update the Course and About xblocks through the CourseDetails model + """ + course_key = CourseKey.from_string(course_key_string) + credit_eligibility_enabled = settings.FEATURES.get('ENABLE_CREDIT_ELIGIBILITY', False) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET': + upload_asset_url = reverse_course_url('assets_handler', course_key) + + # see if the ORG of this course can be attributed to a defined configuration . In that case, the + # course about page should be editable in Studio + publisher_enabled = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_PUBLISHER', + settings.FEATURES.get('ENABLE_PUBLISHER', False) + ) + marketing_enabled = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_MKTG_SITE', + settings.FEATURES.get('ENABLE_MKTG_SITE', False) + ) + enable_extended_course_details = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_EXTENDED_COURSE_DETAILS', + settings.FEATURES.get('ENABLE_EXTENDED_COURSE_DETAILS', False) + ) + + about_page_editable = not publisher_enabled + enrollment_end_editable = GlobalStaff().has_user(request.user) or not publisher_enabled + short_description_editable = configuration_helpers.get_value_for_org( + course_module.location.org, + 'EDITABLE_SHORT_DESCRIPTION', + settings.FEATURES.get('EDITABLE_SHORT_DESCRIPTION', True) + ) + sidebar_html_enabled = course_experience_waffle().is_enabled(ENABLE_COURSE_ABOUT_SIDEBAR_HTML) + # self_paced_enabled = SelfPacedConfiguration.current().enabled + + verified_mode = CourseMode.verified_mode_for_course(course_key, include_expired=True) + upgrade_deadline = (verified_mode and verified_mode.expiration_datetime and + verified_mode.expiration_datetime.isoformat()) + + settings_context = { + 'context_course': course_module, + 'course_locator': course_key, + 'lms_link_for_about_page': get_link_for_about_page(course_module), + 'course_image_url': course_image_url(course_module, 'course_image'), + 'banner_image_url': course_image_url(course_module, 'banner_image'), + 'video_thumbnail_image_url': course_image_url(course_module, 'video_thumbnail_image'), + 'details_url': reverse_course_url('settings_handler', course_key), + 'about_page_editable': about_page_editable, + 'marketing_enabled': marketing_enabled, + 'short_description_editable': short_description_editable, + 'sidebar_html_enabled': sidebar_html_enabled, + 'upload_asset_url': upload_asset_url, + 'course_handler_url': reverse_course_url('course_handler', course_key), + 'language_options': settings.ALL_LANGUAGES, + 'credit_eligibility_enabled': credit_eligibility_enabled, + 'is_credit_course': False, + 'show_min_grade_warning': False, + 'enrollment_end_editable': enrollment_end_editable, + 'is_prerequisite_courses_enabled': is_prerequisite_courses_enabled(), + 'is_entrance_exams_enabled': is_entrance_exams_enabled(), + 'enable_extended_course_details': enable_extended_course_details, + 'upgrade_deadline': upgrade_deadline, + } + if is_prerequisite_courses_enabled(): + courses, in_process_course_actions = get_courses_accessible_to_user(request, course_module.location.org) + # exclude current course from the list of available courses + courses = (course for course in courses if course.id != course_key) + if courses: + courses, __ = _process_courses_list(courses, in_process_course_actions) + settings_context.update({'possible_pre_requisite_courses': list(courses)}) + + if credit_eligibility_enabled: + if is_credit_course(course_key): + # get and all credit eligibility requirements + credit_requirements = get_credit_requirements(course_key) + # pair together requirements with same 'namespace' values + paired_requirements = {} + for requirement in credit_requirements: + namespace = requirement.pop("namespace") + paired_requirements.setdefault(namespace, []).append(requirement) + + # if 'minimum_grade_credit' of a course is not set or 0 then + # show warning message to course author. + show_min_grade_warning = False if course_module.minimum_grade_credit > 0 else True + settings_context.update( + { + 'is_credit_course': True, + 'credit_requirements': paired_requirements, + 'show_min_grade_warning': show_min_grade_warning, + } + ) + + return render_to_response('settings.html', settings_context) + elif 'application/json' in request.META.get('HTTP_ACCEPT', ''): + if request.method == 'GET': + course_details = CourseDetails.fetch(course_key) + return JsonResponse( + course_details, + # encoder serializes dates, old locations, and instances + encoder=CourseSettingsEncoder + ) + # For every other possible method type submitted by the caller... + else: + # if pre-requisite course feature is enabled set pre-requisite course + if is_prerequisite_courses_enabled(): + prerequisite_course_keys = request.json.get('pre_requisite_courses', []) + if prerequisite_course_keys: + if not all(is_valid_course_key(course_key) for course_key in prerequisite_course_keys): + return JsonResponseBadRequest({"error": _("Invalid prerequisite course key")}) + set_prerequisite_courses(course_key, prerequisite_course_keys) + else: + # None is chosen, so remove the course prerequisites + course_milestones = milestones_api.get_course_milestones(course_key=course_key, relationship="requires") + for milestone in course_milestones: + ee_milestone_namespace = generate_milestone_namespace( + get_namespace_choices().get('ENTRANCE_EXAM'), + course_key + ) + if not milestone["namespace"] == ee_milestone_namespace: + remove_prerequisite_course(course_key, milestone) + + # If the entrance exams feature has been enabled, we'll need to check for some + # feature-specific settings and handle them accordingly + # We have to be careful that we're only executing the following logic if we actually + # need to create or delete an entrance exam from the specified course + if is_entrance_exams_enabled(): + course_entrance_exam_present = course_module.entrance_exam_enabled + entrance_exam_enabled = request.json.get('entrance_exam_enabled', '') == 'true' + ee_min_score_pct = request.json.get('entrance_exam_minimum_score_pct', None) + # If the entrance exam box on the settings screen has been checked... + if entrance_exam_enabled: + # Load the default minimum score threshold from settings, then try to override it + entrance_exam_minimum_score_pct = float(settings.ENTRANCE_EXAM_MIN_SCORE_PCT) + if ee_min_score_pct: + entrance_exam_minimum_score_pct = float(ee_min_score_pct) + if entrance_exam_minimum_score_pct.is_integer(): + entrance_exam_minimum_score_pct = entrance_exam_minimum_score_pct / 100 + # If there's already an entrance exam defined, we'll update the existing one + if course_entrance_exam_present: + exam_data = { + 'entrance_exam_minimum_score_pct': entrance_exam_minimum_score_pct + } + update_entrance_exam(request, course_key, exam_data) + # If there's no entrance exam defined, we'll create a new one + else: + create_entrance_exam(request, course_key, entrance_exam_minimum_score_pct) + + # If the entrance exam box on the settings screen has been unchecked, + # and the course has an entrance exam attached... + elif not entrance_exam_enabled and course_entrance_exam_present: + delete_entrance_exam(request, course_key) + + # Perform the normal update workflow for the CourseDetails model + return JsonResponse( + CourseDetails.update_from_json(course_key, request.json, request.user), + encoder=CourseSettingsEncoder + ) + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT", "DELETE")) +@expect_json +def grading_handler(request, course_key_string, grader_index=None): + """ + Course Grading policy configuration + GET + html: get the page + json no grader_index: get the CourseGrading model (graceperiod, cutoffs, and graders) + json w/ grader_index: get the specific grader + PUT + json no grader_index: update the Course through the CourseGrading model + json w/ grader_index: create or update the specific grader (create if index out of range) + """ + course_key = CourseKey.from_string(course_key_string) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + + if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET': + course_details = CourseGradingModel.fetch(course_key) + + return render_to_response('settings_graders.html', { + 'context_course': course_module, + 'course_locator': course_key, + 'course_details': course_details, + 'grading_url': reverse_course_url('grading_handler', course_key), + 'is_credit_course': is_credit_course(course_key), + }) + elif 'application/json' in request.META.get('HTTP_ACCEPT', ''): + if request.method == 'GET': + if grader_index is None: + return JsonResponse( + CourseGradingModel.fetch(course_key), + # encoder serializes dates, old locations, and instances + encoder=CourseSettingsEncoder + ) + else: + return JsonResponse(CourseGradingModel.fetch_grader(course_key, grader_index)) + elif request.method in ('POST', 'PUT'): # post or put, doesn't matter. + # update credit course requirements if 'minimum_grade_credit' + # field value is changed + if 'minimum_grade_credit' in request.json: + update_credit_course_requirements.delay(six.text_type(course_key)) + + # None implies update the whole model (cutoffs, graceperiod, and graders) not a specific grader + if grader_index is None: + return JsonResponse( + CourseGradingModel.update_from_json(course_key, request.json, request.user), + encoder=CourseSettingsEncoder + ) + else: + return JsonResponse( + CourseGradingModel.update_grader_from_json(course_key, request.json, request.user) + ) + elif request.method == "DELETE" and grader_index is not None: + CourseGradingModel.delete_grader(course_key, grader_index, request.user) + return JsonResponse() + + +def _refresh_course_tabs(request, course_module): + """ + Automatically adds/removes tabs if changes to the course require them. + + Raises: + InvalidTabsException: raised if there's a problem with the new version of the tabs. + """ + + def update_tab(tabs, tab_type, tab_enabled): + """ + Adds or removes a course tab based upon whether it is enabled. + """ + tab_panel = { + "type": tab_type.type, + } + has_tab = tab_panel in tabs + if tab_enabled and not has_tab: + tabs.append(CourseTab.from_json(tab_panel)) + elif not tab_enabled and has_tab: + tabs.remove(tab_panel) + + course_tabs = copy.copy(course_module.tabs) + + # Additionally update any tabs that are provided by non-dynamic course views + for tab_type in CourseTabPluginManager.get_tab_types(): + if not tab_type.is_dynamic and tab_type.is_default: + tab_enabled = tab_type.is_enabled(course_module, user=request.user) + update_tab(course_tabs, tab_type, tab_enabled) + + CourseTabList.validate_tabs(course_tabs) + + # Save the tabs into the course if they have been changed + if course_tabs != course_module.tabs: + course_module.tabs = course_tabs + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT")) +@expect_json +def advanced_settings_handler(request, course_key_string): + """ + Course settings configuration + GET + html: get the page + json: get the model + PUT, POST + json: update the Course's settings. The payload is a json rep of the + metadata dicts. + """ + course_key = CourseKey.from_string(course_key_string) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + + advanced_dict = CourseMetadata.fetch(course_module) + if settings.FEATURES.get('DISABLE_MOBILE_COURSE_AVAILABLE', False): + advanced_dict.get('mobile_available')['deprecated'] = True + + if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET': + publisher_enabled = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_PUBLISHER', + settings.FEATURES.get('ENABLE_PUBLISHER', False) + ) + + return render_to_response('settings_advanced.html', { + 'context_course': course_module, + 'advanced_dict': advanced_dict, + 'advanced_settings_url': reverse_course_url('advanced_settings_handler', course_key), + 'publisher_enabled': publisher_enabled, + + }) + elif 'application/json' in request.META.get('HTTP_ACCEPT', ''): + if request.method == 'GET': + return JsonResponse(CourseMetadata.fetch(course_module)) + else: + try: + # validate data formats and update the course module. + # Note: don't update mongo yet, but wait until after any tabs are changed + is_valid, errors, updated_data = CourseMetadata.validate_and_update_from_json( + course_module, + request.json, + user=request.user, + ) + + if is_valid: + try: + # update the course tabs if required by any setting changes + _refresh_course_tabs(request, course_module) + except InvalidTabsException as err: + log.exception(text_type(err)) + response_message = [ + { + 'message': _('An error occurred while trying to save your tabs'), + 'model': {'display_name': _('Tabs Exception')} + } + ] + return JsonResponseBadRequest(response_message) + + # now update mongo + modulestore().update_item(course_module, request.user.id) + + return JsonResponse(updated_data) + else: + return JsonResponseBadRequest(errors) + + # Handle all errors that validation doesn't catch + except (TypeError, ValueError, InvalidTabsException) as err: + return HttpResponseBadRequest( + django.utils.html.escape(text_type(err)), + content_type="text/plain" + ) + + +class TextbookValidationError(Exception): + "An error thrown when a textbook input is invalid" + pass + + +def validate_textbooks_json(text): + """ + Validate the given text as representing a single PDF textbook + """ + if isinstance(text, (bytes, bytearray)): # data appears as bytes + text = text.decode('utf-8') + try: + textbooks = json.loads(text) + except ValueError: + raise TextbookValidationError("invalid JSON") + if not isinstance(textbooks, (list, tuple)): + raise TextbookValidationError("must be JSON list") + for textbook in textbooks: + validate_textbook_json(textbook) + # check specified IDs for uniqueness + all_ids = [textbook["id"] for textbook in textbooks if "id" in textbook] + unique_ids = set(all_ids) + if len(all_ids) > len(unique_ids): + raise TextbookValidationError("IDs must be unique") + return textbooks + + +def validate_textbook_json(textbook): + """ + Validate the given text as representing a list of PDF textbooks + """ + if isinstance(textbook, (bytes, bytearray)): # data appears as bytes + textbook = textbook.decode('utf-8') + if isinstance(textbook, six.string_types): + try: + textbook = json.loads(textbook) + except ValueError: + raise TextbookValidationError("invalid JSON") + if not isinstance(textbook, dict): + raise TextbookValidationError("must be JSON object") + if not textbook.get("tab_title"): + raise TextbookValidationError("must have tab_title") + tid = six.text_type(textbook.get("id", "")) + if tid and not tid[0].isdigit(): + raise TextbookValidationError("textbook ID must start with a digit") + return textbook + + +def assign_textbook_id(textbook, used_ids=()): + """ + Return an ID that can be assigned to a textbook + and doesn't match the used_ids + """ + tid = BlockUsageLocator.clean(textbook["tab_title"]) + if not tid[0].isdigit(): + # stick a random digit in front + tid = random.choice(string.digits) + tid + while tid in used_ids: + # add a random ASCII character to the end + tid = tid + random.choice(string.ascii_lowercase) + return tid + + +@require_http_methods(("GET", "POST", "PUT")) +@login_required +@ensure_csrf_cookie +def textbooks_list_handler(request, course_key_string): + """ + A RESTful handler for textbook collections. + + GET + html: return textbook list page (Backbone application) + json: return JSON representation of all textbooks in this course + POST + json: create a new textbook for this course + PUT + json: overwrite all textbooks in the course with the given list + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course = get_course_and_check_access(course_key, request.user) + + if "application/json" not in request.META.get('HTTP_ACCEPT', 'text/html'): + # return HTML page + upload_asset_url = reverse_course_url('assets_handler', course_key) + textbook_url = reverse_course_url('textbooks_list_handler', course_key) + return render_to_response('textbooks.html', { + 'context_course': course, + 'textbooks': course.pdf_textbooks, + 'upload_asset_url': upload_asset_url, + 'textbook_url': textbook_url, + }) + + # from here on down, we know the client has requested JSON + if request.method == 'GET': + return JsonResponse(course.pdf_textbooks) + elif request.method == 'PUT': + try: + textbooks = validate_textbooks_json(request.body) + except TextbookValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + + tids = set(t["id"] for t in textbooks if "id" in t) + for textbook in textbooks: + if "id" not in textbook: + tid = assign_textbook_id(textbook, tids) + textbook["id"] = tid + tids.add(tid) + + if not any(tab['type'] == 'pdf_textbooks' for tab in course.tabs): + course.tabs.append(CourseTab.load('pdf_textbooks')) + course.pdf_textbooks = textbooks + store.update_item(course, request.user.id) + return JsonResponse(course.pdf_textbooks) + elif request.method == 'POST': + # create a new textbook for the course + try: + textbook = validate_textbook_json(request.body) + except TextbookValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + if not textbook.get("id"): + tids = set(t["id"] for t in course.pdf_textbooks if "id" in t) + textbook["id"] = assign_textbook_id(textbook, tids) + existing = course.pdf_textbooks + existing.append(textbook) + course.pdf_textbooks = existing + if not any(tab['type'] == 'pdf_textbooks' for tab in course.tabs): + course.tabs.append(CourseTab.load('pdf_textbooks')) + store.update_item(course, request.user.id) + resp = JsonResponse(textbook, status=201) + resp["Location"] = reverse_course_url( + 'textbooks_detail_handler', + course.id, + kwargs={'textbook_id': textbook["id"]} + ) + return resp + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT", "DELETE")) +def textbooks_detail_handler(request, course_key_string, textbook_id): + """ + JSON API endpoint for manipulating a textbook via its internal ID. + Used by the Backbone application. + + GET + json: return JSON representation of textbook + POST or PUT + json: update textbook based on provided information + DELETE + json: remove textbook + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + matching_id = [tb for tb in course_module.pdf_textbooks + if six.text_type(tb.get("id")) == six.text_type(textbook_id)] + if matching_id: + textbook = matching_id[0] + else: + textbook = None + + if request.method == 'GET': + if not textbook: + return JsonResponse(status=404) + return JsonResponse(textbook) + elif request.method in ('POST', 'PUT'): # can be either and sometimes django is rewriting one to the other + try: + new_textbook = validate_textbook_json(request.body) + except TextbookValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + new_textbook["id"] = textbook_id + if textbook: + i = course_module.pdf_textbooks.index(textbook) + new_textbooks = course_module.pdf_textbooks[0:i] + new_textbooks.append(new_textbook) + new_textbooks.extend(course_module.pdf_textbooks[i + 1:]) + course_module.pdf_textbooks = new_textbooks + else: + course_module.pdf_textbooks.append(new_textbook) + store.update_item(course_module, request.user.id) + return JsonResponse(new_textbook, status=201) + elif request.method == 'DELETE': + if not textbook: + return JsonResponse(status=404) + i = course_module.pdf_textbooks.index(textbook) + remaining_textbooks = course_module.pdf_textbooks[0:i] + remaining_textbooks.extend(course_module.pdf_textbooks[i + 1:]) + course_module.pdf_textbooks = remaining_textbooks + store.update_item(course_module, request.user.id) + return JsonResponse() + + +def remove_content_or_experiment_group(request, store, course, configuration, group_configuration_id, group_id=None): + """ + Remove content group or experiment group configuration only if it's not in use. + """ + configuration_index = course.user_partitions.index(configuration) + if configuration.scheme.name == RANDOM_SCHEME: + usages = GroupConfiguration.get_content_experiment_usage_info(store, course) + used = int(group_configuration_id) in usages + + if used: + return JsonResponse( + {"error": _("This group configuration is in use and cannot be deleted.")}, + status=400 + ) + course.user_partitions.pop(configuration_index) + elif configuration.scheme.name == COHORT_SCHEME: + if not group_id: + return JsonResponse(status=404) + + group_id = int(group_id) + usages = GroupConfiguration.get_partitions_usage_info(store, course) + used = group_id in usages[configuration.id] + + if used: + return JsonResponse( + {"error": _("This content group is in use and cannot be deleted.")}, + status=400 + ) + + matching_groups = [group for group in configuration.groups if group.id == group_id] + if matching_groups: + group_index = configuration.groups.index(matching_groups[0]) + configuration.groups.pop(group_index) + else: + return JsonResponse(status=404) + + course.user_partitions[configuration_index] = configuration + + store.update_item(course, request.user.id) + return JsonResponse(status=204) + + +@require_http_methods(("GET", "POST")) +@login_required +@ensure_csrf_cookie +def group_configurations_list_handler(request, course_key_string): + """ + A RESTful handler for Group Configurations + + GET + html: return Group Configurations list page (Backbone application) + POST + json: create new group configuration + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course = get_course_and_check_access(course_key, request.user) + + if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): + group_configuration_url = reverse_course_url('group_configurations_list_handler', course_key) + course_outline_url = reverse_course_url('course_handler', course_key) + should_show_experiment_groups = are_content_experiments_enabled(course) + if should_show_experiment_groups: + experiment_group_configurations = GroupConfiguration.get_split_test_partitions_with_usage(store, course) + else: + experiment_group_configurations = None + + all_partitions = GroupConfiguration.get_all_user_partition_details(store, course) + should_show_enrollment_track = False + has_content_groups = False + displayable_partitions = [] + for partition in all_partitions: + partition['read_only'] = getattr(UserPartition.get_scheme(partition['scheme']), 'read_only', False) + + if partition['scheme'] == COHORT_SCHEME: + has_content_groups = True + displayable_partitions.append(partition) + elif partition['scheme'] == CONTENT_TYPE_GATING_SCHEME: + # Add it to the front of the list if it should be shown. + if ContentTypeGatingConfig.current(course_key=course_key).studio_override_enabled: + displayable_partitions.append(partition) + elif partition['scheme'] == ENROLLMENT_SCHEME: + should_show_enrollment_track = len(partition['groups']) > 1 + + # Add it to the front of the list if it should be shown. + if should_show_enrollment_track: + displayable_partitions.insert(0, partition) + elif partition['scheme'] != RANDOM_SCHEME: + # Experiment group configurations are handled explicitly above. We don't + # want to display their groups twice. + displayable_partitions.append(partition) + + # Set the sort-order. Higher numbers sort earlier + scheme_priority = defaultdict(lambda: -1, { + ENROLLMENT_SCHEME: 1, + CONTENT_TYPE_GATING_SCHEME: 0 + }) + displayable_partitions.sort(key=lambda p: scheme_priority[p['scheme']], reverse=True) + # Add empty content group if there is no COHORT User Partition in the list. + # This will add ability to add new groups in the view. + if not has_content_groups: + displayable_partitions.append(GroupConfiguration.get_or_create_content_group(store, course)) + + return render_to_response('group_configurations.html', { + 'context_course': course, + 'group_configuration_url': group_configuration_url, + 'course_outline_url': course_outline_url, + 'experiment_group_configurations': experiment_group_configurations, + 'should_show_experiment_groups': should_show_experiment_groups, + 'all_group_configurations': displayable_partitions, + 'should_show_enrollment_track': should_show_enrollment_track + }) + elif "application/json" in request.META.get('HTTP_ACCEPT'): + if request.method == 'POST': + # create a new group configuration for the course + try: + new_configuration = GroupConfiguration(request.body, course).get_user_partition() + except GroupConfigurationsValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + + course.user_partitions.append(new_configuration) + response = JsonResponse(new_configuration.to_json(), status=201) + + response["Location"] = reverse_course_url( + 'group_configurations_detail_handler', + course.id, + kwargs={'group_configuration_id': new_configuration.id} + ) + store.update_item(course, request.user.id) + return response + else: + return HttpResponse(status=406) + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("POST", "PUT", "DELETE")) +def group_configurations_detail_handler(request, course_key_string, group_configuration_id, group_id=None): + """ + JSON API endpoint for manipulating a group configuration via its internal ID. + Used by the Backbone application. + + POST or PUT + json: update group configuration based on provided information + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course = get_course_and_check_access(course_key, request.user) + matching_id = [p for p in course.user_partitions + if six.text_type(p.id) == six.text_type(group_configuration_id)] + if matching_id: + configuration = matching_id[0] + else: + configuration = None + + if request.method in ('POST', 'PUT'): # can be either and sometimes django is rewriting one to the other + try: + new_configuration = GroupConfiguration(request.body, course, group_configuration_id).get_user_partition() + except GroupConfigurationsValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + + if configuration: + index = course.user_partitions.index(configuration) + course.user_partitions[index] = new_configuration + else: + course.user_partitions.append(new_configuration) + store.update_item(course, request.user.id) + configuration = GroupConfiguration.update_usage_info(store, course, new_configuration) + return JsonResponse(configuration, status=201) + + elif request.method == "DELETE": + if not configuration: + return JsonResponse(status=404) + + return remove_content_or_experiment_group( + request=request, + store=store, + course=course, + configuration=configuration, + group_configuration_id=group_configuration_id, + group_id=group_id + ) + + +def are_content_experiments_enabled(course): + """ + Returns True if content experiments have been enabled for the course. + """ + return ( + 'split_test' in ADVANCED_COMPONENT_TYPES and + 'split_test' in course.advanced_modules + ) + + +def _get_course_creator_status(user): + """ + Helper method for returning the course creator status for a particular user, + taking into account the values of DISABLE_COURSE_CREATION and ENABLE_CREATOR_GROUP. + + If the user passed in has not previously visited the index page, it will be + added with status 'unrequested' if the course creator group is in use. + """ + + if user.is_staff: + course_creator_status = 'granted' + elif settings.FEATURES.get('DISABLE_COURSE_CREATION', False): + course_creator_status = 'disallowed_for_this_site' + elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): + course_creator_status = get_course_creator_status(user) + if course_creator_status is None: + # User not grandfathered in as an existing user, has not previously visited the dashboard page. + # Add the user to the course creator admin table with status 'unrequested'. + add_user_with_status_unrequested(user) + course_creator_status = get_course_creator_status(user) + else: + course_creator_status = 'granted' + + return course_creator_status diff --git a/cms/djangoapps/contentstore/views/course.py.original b/cms/djangoapps/contentstore/views/course.py.original new file mode 100644 index 000000000000..bd194177c10f --- /dev/null +++ b/cms/djangoapps/contentstore/views/course.py.original @@ -0,0 +1,1801 @@ +""" +Views related to operations on course objects +""" + + +import copy +import json +import logging +import random +import re +import string +from collections import defaultdict + +import django.utils +import six +from ccx_keys.locator import CCXLocator +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied, ValidationError +from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseNotFound +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import ensure_csrf_cookie +from django.views.decorators.http import require_GET, require_http_methods +from milestones import api as milestones_api +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import BlockUsageLocator +from six import text_type +from six.moves import filter + +from contentstore.course_group_config import ( + COHORT_SCHEME, + ENROLLMENT_SCHEME, + RANDOM_SCHEME, + GroupConfiguration, + GroupConfigurationsValidationError +) +from contentstore.course_info_model import delete_course_update, get_course_updates, update_course_updates +from contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError +from contentstore.tasks import rerun_course as rerun_course_task +from contentstore.utils import ( + add_instructor, + get_lms_link_for_item, + initialize_permissions, + remove_all_instructors, + reverse_course_url, + reverse_library_url, + reverse_url, + reverse_usage_url +) +from contentstore.views.entrance_exam import create_entrance_exam, delete_entrance_exam, update_entrance_exam +from course_action_state.managers import CourseActionStateItemNotFoundError +from course_action_state.models import CourseRerunState, CourseRerunUIStateManager +from course_creators.views import add_user_with_status_unrequested, get_course_creator_status +from course_modes.models import CourseMode +from edxmako.shortcuts import render_to_response +from models.settings.course_grading import CourseGradingModel +from models.settings.course_metadata import CourseMetadata +from models.settings.encoder import CourseSettingsEncoder +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course +from openedx.core.djangoapps.credit.tasks import update_credit_course_requirements +from openedx.core.djangoapps.models.course_details import CourseDetails +from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers +from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace +from openedx.core.djangolib.js_utils import dump_js_escaped_json +from openedx.core.lib.course_tabs import CourseTabPluginManager +from openedx.core.lib.courses import course_image_url +from openedx.features.content_type_gating.models import ContentTypeGatingConfig +from openedx.features.content_type_gating.partitions import CONTENT_TYPE_GATING_SCHEME +from openedx.features.course_experience.waffle import ENABLE_COURSE_ABOUT_SIDEBAR_HTML +from openedx.features.course_experience.waffle import waffle as course_experience_waffle +from student import auth +from student.auth import has_course_author_access, has_studio_read_access, has_studio_write_access +from student.roles import CourseCreatorRole, CourseInstructorRole, CourseStaffRole, GlobalStaff, UserBasedRole +from util.course import get_link_for_about_page +from util.date_utils import get_default_time_display +from util.json_request import JsonResponse, JsonResponseBadRequest, expect_json +from util.milestones_helpers import ( + is_entrance_exams_enabled, + is_prerequisite_courses_enabled, + is_valid_course_key, + remove_prerequisite_course, + set_prerequisite_courses, + get_namespace_choices, + generate_milestone_namespace +) +from util.organizations_helpers import add_organization_course, get_organization_by_short_name, organizations_enabled +from util.string_utils import _has_non_ascii_characters +from xblock_django.api import deprecated_xblocks +from xmodule.contentstore.content import StaticContent +from xmodule.course_module import DEFAULT_START_DATE, CourseFields +from xmodule.error_module import ErrorDescriptor +from xmodule.modulestore import EdxJSONEncoder +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError +from xmodule.partitions.partitions import UserPartition +from xmodule.tabs import CourseTab, CourseTabList, InvalidTabsException + +from .component import ADVANCED_COMPONENT_TYPES +from .item import create_xblock_info +from .library import LIBRARIES_ENABLED, get_library_creator_status + +log = logging.getLogger(__name__) + + +__all__ = ['course_info_handler', 'course_handler', 'course_listing', + 'course_info_update_handler', 'course_search_index_handler', + 'course_rerun_handler', + 'settings_handler', + 'grading_handler', + 'advanced_settings_handler', + 'course_notifications_handler', + 'textbooks_list_handler', 'textbooks_detail_handler', + 'group_configurations_list_handler', 'group_configurations_detail_handler'] + +WAFFLE_NAMESPACE = 'studio_home' + + +class AccessListFallback(Exception): + """ + An exception that is raised whenever we need to `fall back` to fetching *all* courses + available to a user, rather than using a shorter method (i.e. fetching by group) + """ + pass + + +def get_course_and_check_access(course_key, user, depth=0): + """ + Internal method used to calculate and return the locator and course module + for the view functions in this file. + """ + if not has_studio_read_access(user, course_key): + raise PermissionDenied() + course_module = modulestore().get_course(course_key, depth=depth) + return course_module + + +def reindex_course_and_check_access(course_key, user): + """ + Internal method used to restart indexing on a course. + """ + if not has_course_author_access(user, course_key): + raise PermissionDenied() + return CoursewareSearchIndexer.do_course_reindex(modulestore(), course_key) + + +@login_required +def course_notifications_handler(request, course_key_string=None, action_state_id=None): + """ + Handle incoming requests for notifications in a RESTful way. + + course_key_string and action_state_id must both be set; else a HttpBadResponseRequest is returned. + + For each of these operations, the requesting user must have access to the course; + else a PermissionDenied error is returned. + + GET + json: return json representing information about the notification (action, state, etc) + DELETE + json: return json repressing success or failure of dismissal/deletion of the notification + PUT + Raises a NotImplementedError. + POST + Raises a NotImplementedError. + """ + # ensure that we have a course and an action state + if not course_key_string or not action_state_id: + return HttpResponseBadRequest() + + response_format = request.GET.get('format') or request.POST.get('format') or 'html' + + course_key = CourseKey.from_string(course_key_string) + + if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): + if not has_studio_write_access(request.user, course_key): + raise PermissionDenied() + if request.method == 'GET': + return _course_notifications_json_get(action_state_id) + elif request.method == 'DELETE': + # we assume any delete requests dismiss actions from the UI + return _dismiss_notification(request, action_state_id) + elif request.method == 'PUT': + raise NotImplementedError() + elif request.method == 'POST': + raise NotImplementedError() + else: + return HttpResponseBadRequest() + else: + return HttpResponseNotFound() + + +def _course_notifications_json_get(course_action_state_id): + """ + Return the action and the action state for the given id + """ + try: + action_state = CourseRerunState.objects.find_first(id=course_action_state_id) + except CourseActionStateItemNotFoundError: + return HttpResponseBadRequest() + + action_state_info = { + 'action': action_state.action, + 'state': action_state.state, + 'should_display': action_state.should_display + } + return JsonResponse(action_state_info) + + +def _dismiss_notification(request, course_action_state_id): + """ + Update the display of the course notification + """ + try: + action_state = CourseRerunState.objects.find_first(id=course_action_state_id) + + except CourseActionStateItemNotFoundError: + # Can't dismiss a notification that doesn't exist in the first place + return HttpResponseBadRequest() + + if action_state.state == CourseRerunUIStateManager.State.FAILED: + # We remove all permissions for this course key at this time, since + # no further access is required to a course that failed to be created. + remove_all_instructors(action_state.course_key) + + # The CourseRerunState is no longer needed by the UI; delete + action_state.delete() + + return JsonResponse({'success': True}) + + +@login_required +def course_handler(request, course_key_string=None): + """ + The restful handler for course specific requests. + It provides the course tree with the necessary information for identifying and labeling the parts. The root + will typically be a 'course' object but may not be especially as we support modules. + + GET + html: return course listing page if not given a course id + html: return html page overview for the given course if given a course id + json: return json representing the course branch's index entry as well as dag w/ all of the children + replaced w/ json docs where each doc has {'_id': , 'display_name': , 'children': } + POST + json: create a course, return resulting json + descriptor (same as in GET course/...). Leaving off /branch/draft would imply create the course w/ default + branches. Cannot change the structure contents ('_id', 'display_name', 'children') but can change the + index entry. + PUT + json: update this course (index entry not xblock) such as repointing head, changing display name, org, + course, run. Return same json as above. + DELETE + json: delete this branch from this course (leaving off /branch/draft would imply delete the course) + """ + try: + response_format = request.GET.get('format') or request.POST.get('format') or 'html' + if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): + if request.method == 'GET': + course_key = CourseKey.from_string(course_key_string) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user, depth=None) + return JsonResponse(_course_outline_json(request, course_module)) + elif request.method == 'POST': # not sure if this is only post. If one will have ids, it goes after access + return _create_or_rerun_course(request) + elif not has_studio_write_access(request.user, CourseKey.from_string(course_key_string)): + raise PermissionDenied() + elif request.method == 'PUT': + raise NotImplementedError() + elif request.method == 'DELETE': + raise NotImplementedError() + else: + return HttpResponseBadRequest() + elif request.method == 'GET': # assume html + if course_key_string is None: + return redirect(reverse('home')) + else: + return course_index(request, CourseKey.from_string(course_key_string)) + else: + return HttpResponseNotFound() + except InvalidKeyError: + raise Http404 + + +@login_required +@ensure_csrf_cookie +@require_http_methods(["GET"]) +def course_rerun_handler(request, course_key_string): + """ + The restful handler for course reruns. + GET + html: return html page with form to rerun a course for the given course id + """ + # Only global staff (PMs) are able to rerun courses during the soft launch + # Appsembler: Also course staff can do reruns + course_key = CourseKey.from_string(course_key_string) + if not CourseStaffRole(course_key).has_user(request.user): + if not CourseInstructorRole(course_key).has_user(request.user): + if not GlobalStaff().has_user(request.user): + raise PermissionDenied() + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user, depth=3) + if request.method == 'GET': + return render_to_response('course-create-rerun.html', { + 'source_course_key': course_key, + 'display_name': course_module.display_name, + 'user': request.user, + 'course_creator_status': _get_course_creator_status(request.user), + 'allow_unicode_course_id': settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID', False) + }) + + +@login_required +@ensure_csrf_cookie +@require_GET +def course_search_index_handler(request, course_key_string): + """ + The restful handler for course indexing. + GET + html: return status of indexing task + json: return status of indexing task + """ + # Only global staff (PMs) are able to index courses + if not GlobalStaff().has_user(request.user): + raise PermissionDenied() + course_key = CourseKey.from_string(course_key_string) + content_type = request.META.get('CONTENT_TYPE', None) + if content_type is None: + content_type = "application/json; charset=utf-8" + with modulestore().bulk_operations(course_key): + try: + reindex_course_and_check_access(course_key, request.user) + except SearchIndexingError as search_err: + return HttpResponse(dump_js_escaped_json({ + "user_message": search_err.error_list + }), content_type=content_type, status=500) + return HttpResponse(dump_js_escaped_json({ + "user_message": _("Course has been successfully reindexed.") + }), content_type=content_type, status=200) + + +def _course_outline_json(request, course_module): + """ + Returns a JSON representation of the course module and recursively all of its children. + """ + is_concise = request.GET.get('format') == 'concise' + include_children_predicate = lambda xblock: not xblock.category == 'vertical' + if is_concise: + include_children_predicate = lambda xblock: xblock.has_children + return create_xblock_info( + course_module, + include_child_info=True, + course_outline=False if is_concise else True, + include_children_predicate=include_children_predicate, + is_concise=is_concise, + user=request.user + ) + + +def get_in_process_course_actions(request): + """ + Get all in-process course actions + """ + return [ + course for course in + CourseRerunState.objects.find_all( + exclude_args={'state': CourseRerunUIStateManager.State.SUCCEEDED}, + should_display=True, + ) + if has_studio_read_access(request.user, course.course_key) + ] + + +def _accessible_courses_summary_iter(request, org=None): + """ + List all courses available to the logged in user by iterating through all the courses + + Arguments: + request: the request object + org (string): if not None, this value will limit the courses returned. An empty + string will result in no courses, and otherwise only courses with the + specified org will be returned. The default value is None. + """ + def course_filter(course_summary): + """ + Filter out unusable and inaccessible courses + """ + # TODO remove this condition when templates purged from db + if course_summary.location.course == 'templates': + return False + + return has_studio_read_access(request.user, course_summary.id) + if org is not None: + courses_summary = [] if org == '' else CourseOverview.get_all_courses(orgs=[org]) + else: + courses_summary = modulestore().get_course_summaries() + courses_summary = six.moves.filter(course_filter, courses_summary) + in_process_course_actions = get_in_process_course_actions(request) + return courses_summary, in_process_course_actions + + +def _accessible_courses_iter(request): + """ + List all courses available to the logged in user by iterating through all the courses. + """ + def course_filter(course): + """ + Filter out unusable and inaccessible courses + """ + if isinstance(course, ErrorDescriptor): + return False + + # Custom Courses for edX (CCX) is an edX feature for re-using course content. + # CCXs cannot be edited in Studio (aka cms) and should not be shown in this dashboard. + if isinstance(course.id, CCXLocator): + return False + + # TODO remove this condition when templates purged from db + if course.location.course == 'templates': + return False + + return has_studio_read_access(request.user, course.id) + + courses = six.moves.filter(course_filter, modulestore().get_courses()) + + in_process_course_actions = get_in_process_course_actions(request) + return courses, in_process_course_actions + + +def _accessible_courses_iter_for_tests(request): + """ + List all courses available to the logged in user by iterating through all the courses. + CourseSummary objects are used for listing purposes. + This method is only used by tests. + """ + def course_filter(course): + """ + Filter out unusable and inaccessible courses + """ + + # Custom Courses for edX (CCX) is an edX feature for re-using course content. + # CCXs cannot be edited in Studio (aka cms) and should not be shown in this dashboard. + if isinstance(course.id, CCXLocator): + return False + + # TODO remove this condition when templates purged from db + if course.location.course == 'templates': + return False + + return has_studio_read_access(request.user, course.id) + + courses = six.moves.filter(course_filter, modulestore().get_course_summaries()) + + in_process_course_actions = get_in_process_course_actions(request) + return courses, in_process_course_actions + + +def _accessible_courses_list_from_groups(request): + """ + List all courses available to the logged in user by reversing access group names + """ + def filter_ccx(course_access): + """ CCXs cannot be edited in Studio and should not be shown in this dashboard """ + return not isinstance(course_access.course_id, CCXLocator) + + instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role() + staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + all_courses = list(filter(filter_ccx, instructor_courses | staff_courses)) + courses_list = [] + course_keys = {} + + for course_access in all_courses: + if course_access.course_id is None: + raise AccessListFallback + course_keys[course_access.course_id] = course_access.course_id + + course_keys = list(course_keys.values()) + + if course_keys: + courses_list = modulestore().get_course_summaries(course_keys=course_keys) + + return courses_list, [] + + +def _accessible_libraries_iter(user, org=None): + """ + List all libraries available to the logged in user by iterating through all libraries. + + org (string): if not None, this value will limit the libraries returned. An empty + string will result in no libraries, and otherwise only libraries with the + specified org will be returned. The default value is None. + """ + if org is not None: + libraries = [] if org == '' else modulestore().get_libraries(org=org) + else: + libraries = modulestore().get_library_summaries() + # No need to worry about ErrorDescriptors - split's get_libraries() never returns them. + return (lib for lib in libraries if has_studio_read_access(user, lib.location.library_key)) + + +@login_required +@ensure_csrf_cookie +def course_listing(request): + """ + List all courses and libraries available to the logged in user + """ + + optimization_enabled = GlobalStaff().has_user(request.user) and \ + WaffleSwitchNamespace(name=WAFFLE_NAMESPACE).is_enabled(u'enable_global_staff_optimization') + + org = request.GET.get('org', '') if optimization_enabled else None + courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org) + user = request.user + libraries = _accessible_libraries_iter(request.user, org) if LIBRARIES_ENABLED else [] + + def format_in_process_course_view(uca): + """ + Return a dict of the data which the view requires for each unsucceeded course + """ + return { + u'display_name': uca.display_name, + u'course_key': six.text_type(uca.course_key), + u'org': uca.course_key.org, + u'number': uca.course_key.course, + u'run': uca.course_key.run, + u'is_failed': True if uca.state == CourseRerunUIStateManager.State.FAILED else False, + u'is_in_progress': True if uca.state == CourseRerunUIStateManager.State.IN_PROGRESS else False, + u'dismiss_link': reverse_course_url( + u'course_notifications_handler', + uca.course_key, + kwargs={ + u'action_state_id': uca.id, + }, + ) if uca.state == CourseRerunUIStateManager.State.FAILED else u'' + } + + def format_library_for_view(library): + """ + Return a dict of the data which the view requires for each library + """ + + return { + u'display_name': library.display_name, + u'library_key': six.text_type(library.location.library_key), + u'url': reverse_library_url(u'library_handler', six.text_type(library.location.library_key)), + u'org': library.display_org_with_default, + u'number': library.display_number_with_default, + u'can_edit': has_studio_write_access(request.user, library.location.library_key), + } + + split_archived = settings.FEATURES.get(u'ENABLE_SEPARATE_ARCHIVED_COURSES', False) + active_courses, archived_courses = _process_courses_list(courses_iter, in_process_course_actions, split_archived) + in_process_course_actions = [format_in_process_course_view(uca) for uca in in_process_course_actions] + + return render_to_response(u'index.html', { + u'courses': active_courses, + u'archived_courses': archived_courses, + u'in_process_course_actions': in_process_course_actions, + u'libraries_enabled': LIBRARIES_ENABLED, + u'libraries': [format_library_for_view(lib) for lib in libraries], + u'show_new_library_button': get_library_creator_status(user), + u'user': user, + u'request_course_creator_url': reverse('request_course_creator'), + u'course_creator_status': _get_course_creator_status(user), + # Appsembler: Using the course creator status instead of `GlobalStaff().has_user(user)` + # to match the Tahoe multi-tenant requirements. + u'rerun_creator_status': _get_course_creator_status(user) == 'granted', + u'allow_unicode_course_id': settings.FEATURES.get(u'ALLOW_UNICODE_COURSE_ID', False), + u'allow_course_reruns': settings.FEATURES.get(u'ALLOW_COURSE_RERUNS', True), + u'optimization_enabled': optimization_enabled + }) + + +def _get_rerun_link_for_item(course_key): + """ Returns the rerun link for the given course key. """ + return reverse_course_url('course_rerun_handler', course_key) + + +def _deprecated_blocks_info(course_module, deprecated_block_types): + """ + Returns deprecation information about `deprecated_block_types` + + Arguments: + course_module (CourseDescriptor): course object + deprecated_block_types (list): list of deprecated blocks types + + Returns: + Dict with following keys: + deprecated_enabled_block_types (list): list containing all deprecated blocks types enabled on this course + blocks (list): List of `deprecated_enabled_block_types` instances and their parent's url + advance_settings_url (str): URL to advance settings page + """ + data = { + 'deprecated_enabled_block_types': [ + block_type for block_type in course_module.advanced_modules if block_type in deprecated_block_types + ], + 'blocks': [], + 'advance_settings_url': reverse_course_url('advanced_settings_handler', course_module.id) + } + + deprecated_blocks = modulestore().get_items( + course_module.id, + qualifiers={ + 'category': re.compile('^' + '$|^'.join(deprecated_block_types) + '$') + } + ) + + for block in deprecated_blocks: + data['blocks'].append([ + reverse_usage_url('container_handler', block.parent), + block.display_name + ]) + + return data + + +@login_required +@ensure_csrf_cookie +def course_index(request, course_key): + """ + Display an editable course overview. + + org, course, name: Attributes of the Location for the item to edit + """ + # A depth of None implies the whole course. The course outline needs this in order to compute has_changes. + # A unit may not have a draft version, but one of its components could, and hence the unit itself has changes. + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user, depth=None) + if not course_module: + raise Http404 + lms_link = get_lms_link_for_item(course_module.location) + reindex_link = None + if settings.FEATURES.get('ENABLE_COURSEWARE_INDEX', False): + if GlobalStaff().has_user(request.user): + reindex_link = "/course/{course_id}/search_reindex".format(course_id=six.text_type(course_key)) + sections = course_module.get_children() + course_structure = _course_outline_json(request, course_module) + locator_to_show = request.GET.get('show', None) + + course_release_date = ( + get_default_time_display(course_module.start) + if course_module.start != DEFAULT_START_DATE + else _("Set Date") + ) + + settings_url = reverse_course_url('settings_handler', course_key) + + try: + current_action = CourseRerunState.objects.find_first(course_key=course_key, should_display=True) + except (ItemNotFoundError, CourseActionStateItemNotFoundError): + current_action = None + + deprecated_block_names = [block.name for block in deprecated_xblocks()] + deprecated_blocks_info = _deprecated_blocks_info(course_module, deprecated_block_names) + + frontend_app_publisher_url = configuration_helpers.get_value_for_org( + course_module.location.org, + 'FRONTEND_APP_PUBLISHER_URL', + settings.FEATURES.get('FRONTEND_APP_PUBLISHER_URL', False) + ) + + return render_to_response('course_outline.html', { + 'language_code': request.LANGUAGE_CODE, + 'context_course': course_module, + 'lms_link': lms_link, + 'sections': sections, + 'course_structure': course_structure, + 'initial_state': course_outline_initial_state(locator_to_show, course_structure) if locator_to_show else None, + 'rerun_notification_id': current_action.id if current_action else None, + 'course_release_date': course_release_date, + 'settings_url': settings_url, + 'reindex_link': reindex_link, + 'deprecated_blocks_info': deprecated_blocks_info, + 'notification_dismiss_url': reverse_course_url( + 'course_notifications_handler', + current_action.course_key, + kwargs={ + 'action_state_id': current_action.id, + }, + ) if current_action else None, + 'frontend_app_publisher_url': frontend_app_publisher_url, + }) + + +def get_courses_accessible_to_user(request, org=None): + """ + Try to get all courses by first reversing django groups and fallback to old method if it fails + Note: overhead of pymongo reads will increase if getting courses from django groups fails + + Arguments: + request: the request object + org (string): for global staff users ONLY, this value will be used to limit + the courses returned. A value of None will have no effect (all courses + returned), an empty string will result in no courses, and otherwise only courses with the + specified org will be returned. The default value is None. + """ + if GlobalStaff().has_user(request.user): + # user has global access so no need to get courses from django groups + courses, in_process_course_actions = _accessible_courses_summary_iter(request, org) + else: + try: + courses, in_process_course_actions = _accessible_courses_list_from_groups(request) + except AccessListFallback: + # user have some old groups or there was some error getting courses from django groups + # so fallback to iterating through all courses + courses, in_process_course_actions = _accessible_courses_summary_iter(request) + return courses, in_process_course_actions + + +def _process_courses_list(courses_iter, in_process_course_actions, split_archived=False): + """ + Iterates over the list of courses to be displayed to the user, and: + + * Removes any in-process courses from the courses list. "In-process" refers to courses + that are in the process of being generated for re-run. + * If split_archived=True, removes any archived courses and returns them in a separate list. + Archived courses have has_ended() == True. + * Formats the returned courses (in both lists) to prepare them for rendering to the view. + """ + def format_course_for_view(course): + """ + Return a dict of the data which the view requires for each course + """ + return { + 'display_name': course.display_name, + 'course_key': six.text_type(course.location.course_key), + 'url': reverse_course_url('course_handler', course.id), + 'lms_link': get_lms_link_for_item(course.location), + 'rerun_link': _get_rerun_link_for_item(course.id), + 'org': course.display_org_with_default, + 'number': course.display_number_with_default, + 'run': course.location.run + } + + in_process_action_course_keys = {uca.course_key for uca in in_process_course_actions} + active_courses = [] + archived_courses = [] + + for course in courses_iter: + if isinstance(course, ErrorDescriptor) or (course.id in in_process_action_course_keys): + continue + + formatted_course = format_course_for_view(course) + if split_archived and course.has_ended(): + archived_courses.append(formatted_course) + else: + active_courses.append(formatted_course) + + return active_courses, archived_courses + + +def course_outline_initial_state(locator_to_show, course_structure): + """ + Returns the desired initial state for the course outline view. If the 'show' request parameter + was provided, then the view's initial state will be to have the desired item fully expanded + and to scroll to see the new item. + """ + def find_xblock_info(xblock_info, locator): + """ + Finds the xblock info for the specified locator. + """ + if xblock_info['id'] == locator: + return xblock_info + children = xblock_info['child_info']['children'] if xblock_info.get('child_info', None) else None + if children: + for child_xblock_info in children: + result = find_xblock_info(child_xblock_info, locator) + if result: + return result + return None + + def collect_all_locators(locators, xblock_info): + """ + Collect all the locators for an xblock and its children. + """ + locators.append(xblock_info['id']) + children = xblock_info['child_info']['children'] if xblock_info.get('child_info', None) else None + if children: + for child_xblock_info in children: + collect_all_locators(locators, child_xblock_info) + + selected_xblock_info = find_xblock_info(course_structure, locator_to_show) + if not selected_xblock_info: + return None + expanded_locators = [] + collect_all_locators(expanded_locators, selected_xblock_info) + return { + 'locator_to_show': locator_to_show, + 'expanded_locators': expanded_locators + } + + +@expect_json +def _create_or_rerun_course(request): + """ + To be called by requests that create a new destination course (i.e., create_new_course and rerun_course) + Returns the destination course_key and overriding fields for the new course. + Raises DuplicateCourseError and InvalidKeyError + """ + if not auth.user_has_role(request.user, CourseCreatorRole()): + raise PermissionDenied() + + try: + org = request.json.get('org') + course = request.json.get('number', request.json.get('course')) + display_name = request.json.get('display_name') + # force the start date for reruns and allow us to override start via the client + start = request.json.get('start', CourseFields.start.default) + run = request.json.get('run') + + # allow/disable unicode characters in course_id according to settings + if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'): + if _has_non_ascii_characters(org) or _has_non_ascii_characters(course) or _has_non_ascii_characters(run): + return JsonResponse( + {'error': _('Special characters not allowed in organization, course number, and course run.')}, + status=400 + ) + + fields = {'start': start} + if display_name is not None: + fields['display_name'] = display_name + + # Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for + # existing xml courses this cannot be changed in CourseDescriptor. + # # TODO get rid of defining wiki slug in this org/course/run specific way and reconcile + # w/ xmodule.course_module.CourseDescriptor.__init__ + wiki_slug = u"{0}.{1}.{2}".format(org, course, run) + definition_data = {'wiki_slug': wiki_slug} + fields.update(definition_data) + + source_course_key = request.json.get('source_course_key') + if source_course_key: + source_course_key = CourseKey.from_string(source_course_key) + destination_course_key = rerun_course(request.user, source_course_key, org, course, run, fields) + return JsonResponse({ + 'url': reverse_url('course_handler'), + 'destination_course_key': six.text_type(destination_course_key) + }) + else: + try: + new_course = create_new_course(request.user, org, course, run, fields) + return JsonResponse({ + 'url': reverse_course_url('course_handler', new_course.id), + 'course_key': six.text_type(new_course.id), + }) + except ValidationError as ex: + return JsonResponse({'error': text_type(ex)}, status=400) + except DuplicateCourseError: + return JsonResponse({ + 'ErrMsg': _( + 'There is already a course defined with the same ' + 'organization and course number. Please ' + 'change either organization or course number to be unique.' + ), + 'OrgErrMsg': _( + 'Please change either the organization or ' + 'course number so that it is unique.'), + 'CourseErrMsg': _( + 'Please change either the organization or ' + 'course number so that it is unique.'), + }) + except InvalidKeyError as error: + return JsonResponse({ + "ErrMsg": _(u"Unable to create course '{name}'.\n\n{err}").format(name=display_name, err=text_type(error))} + ) + + +def create_new_course(user, org, number, run, fields): + """ + Create a new course run. + + Raises: + DuplicateCourseError: Course run already exists. + """ + org_data = get_organization_by_short_name(org) + if not org_data and organizations_enabled(): + raise ValidationError(_('You must link this course to an organization in order to continue. Organization ' + 'you selected does not exist in the system, you will need to add it to the system')) + store_for_new_course = modulestore().default_modulestore.get_modulestore_type() + new_course = create_new_course_in_store(store_for_new_course, user, org, number, run, fields) + add_organization_course(org_data, new_course.id) + return new_course + + +def create_new_course_in_store(store, user, org, number, run, fields): + """ + Create course in store w/ handling instructor enrollment, permissions, and defaulting the wiki slug. + Separated out b/c command line course creation uses this as well as the web interface. + """ + + # Set default language from settings and enable web certs + fields.update({ + 'language': getattr(settings, 'DEFAULT_COURSE_LANGUAGE', 'en'), + 'cert_html_view_enabled': True, + }) + + with modulestore().default_store(store): + # Creating the course raises DuplicateCourseError if an existing course with this org/name is found + new_course = modulestore().create_course( + org, + number, + run, + user.id, + fields=fields, + ) + + # Make sure user has instructor and staff access to the new course + add_instructor(new_course.id, user, user) + + # Initialize permissions for user in the new course + initialize_permissions(new_course.id, user) + return new_course + + +def rerun_course(user, source_course_key, org, number, run, fields, background=True): + """ + Rerun an existing course. + """ + # verify user has access to the original course + if not has_studio_write_access(user, source_course_key): + raise PermissionDenied() + + # create destination course key + store = modulestore() + with store.default_store('split'): + destination_course_key = store.make_course_key(org, number, run) + + # verify org course and run don't already exist + if store.has_course(destination_course_key, ignore_case=True): + raise DuplicateCourseError(source_course_key, destination_course_key) + + # Make sure user has instructor and staff access to the destination course + # so the user can see the updated status for that course + add_instructor(destination_course_key, user, user) + + # Mark the action as initiated + CourseRerunState.objects.initiated(source_course_key, destination_course_key, user, fields['display_name']) + + # Clear the fields that must be reset for the rerun + fields['advertised_start'] = None + fields['enrollment_start'] = None + fields['enrollment_end'] = None + fields['video_upload_pipeline'] = {} + + json_fields = json.dumps(fields, cls=EdxJSONEncoder) + args = [six.text_type(source_course_key), six.text_type(destination_course_key), user.id, json_fields] + + if background: + rerun_course_task.delay(*args) + else: + rerun_course_task(*args) + + return destination_course_key + + +@login_required +@ensure_csrf_cookie +@require_http_methods(["GET"]) +def course_info_handler(request, course_key_string): + """ + GET + html: return html for editing the course info handouts and updates. + """ + try: + course_key = CourseKey.from_string(course_key_string) + except InvalidKeyError: + raise Http404 + + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + if not course_module: + raise Http404 + if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): + return render_to_response( + 'course_info.html', + { + 'context_course': course_module, + 'updates_url': reverse_course_url('course_info_update_handler', course_key), + 'handouts_locator': course_key.make_usage_key('course_info', 'handouts'), + 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.id), + } + ) + else: + return HttpResponseBadRequest("Only supports html requests") + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT", "DELETE")) +@expect_json +def course_info_update_handler(request, course_key_string, provided_id=None): + """ + restful CRUD operations on course_info updates. + provided_id should be none if it's new (create) and index otherwise. + GET + json: return the course info update models + POST + json: create an update + PUT or DELETE + json: change an existing update + """ + if 'application/json' not in request.META.get('HTTP_ACCEPT', 'application/json'): + return HttpResponseBadRequest("Only supports json requests") + + course_key = CourseKey.from_string(course_key_string) + usage_key = course_key.make_usage_key('course_info', 'updates') + if provided_id == '': + provided_id = None + + # check that logged in user has permissions to this item (GET shouldn't require this level?) + if not has_studio_write_access(request.user, usage_key.course_key): + raise PermissionDenied() + + if request.method == 'GET': + course_updates = get_course_updates(usage_key, provided_id, request.user.id) + if isinstance(course_updates, dict) and course_updates.get('error'): + return JsonResponse(course_updates, course_updates.get('status', 400)) + else: + return JsonResponse(course_updates) + elif request.method == 'DELETE': + try: + return JsonResponse(delete_course_update(usage_key, request.json, provided_id, request.user)) + except: + return HttpResponseBadRequest( + "Failed to delete", + content_type="text/plain" + ) + # can be either and sometimes django is rewriting one to the other: + elif request.method in ('POST', 'PUT'): + try: + return JsonResponse(update_course_updates(usage_key, request.json, provided_id, request.user)) + except: + return HttpResponseBadRequest( + "Failed to save", + content_type="text/plain" + ) + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "PUT", "POST")) +@expect_json +def settings_handler(request, course_key_string): + """ + Course settings for dates and about pages + GET + html: get the page + json: get the CourseDetails model + PUT + json: update the Course and About xblocks through the CourseDetails model + """ + course_key = CourseKey.from_string(course_key_string) + credit_eligibility_enabled = settings.FEATURES.get('ENABLE_CREDIT_ELIGIBILITY', False) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET': + upload_asset_url = reverse_course_url('assets_handler', course_key) + + # see if the ORG of this course can be attributed to a defined configuration . In that case, the + # course about page should be editable in Studio + publisher_enabled = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_PUBLISHER', + settings.FEATURES.get('ENABLE_PUBLISHER', False) + ) + marketing_enabled = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_MKTG_SITE', + settings.FEATURES.get('ENABLE_MKTG_SITE', False) + ) + enable_extended_course_details = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_EXTENDED_COURSE_DETAILS', + settings.FEATURES.get('ENABLE_EXTENDED_COURSE_DETAILS', False) + ) + + about_page_editable = not publisher_enabled + enrollment_end_editable = GlobalStaff().has_user(request.user) or not publisher_enabled + short_description_editable = configuration_helpers.get_value_for_org( + course_module.location.org, + 'EDITABLE_SHORT_DESCRIPTION', + settings.FEATURES.get('EDITABLE_SHORT_DESCRIPTION', True) + ) + sidebar_html_enabled = course_experience_waffle().is_enabled(ENABLE_COURSE_ABOUT_SIDEBAR_HTML) + # self_paced_enabled = SelfPacedConfiguration.current().enabled + + verified_mode = CourseMode.verified_mode_for_course(course_key, include_expired=True) + upgrade_deadline = (verified_mode and verified_mode.expiration_datetime and + verified_mode.expiration_datetime.isoformat()) + + settings_context = { + 'context_course': course_module, + 'course_locator': course_key, + 'lms_link_for_about_page': get_link_for_about_page(course_module), + 'course_image_url': course_image_url(course_module, 'course_image'), + 'banner_image_url': course_image_url(course_module, 'banner_image'), + 'video_thumbnail_image_url': course_image_url(course_module, 'video_thumbnail_image'), + 'details_url': reverse_course_url('settings_handler', course_key), + 'about_page_editable': about_page_editable, + 'marketing_enabled': marketing_enabled, + 'short_description_editable': short_description_editable, + 'sidebar_html_enabled': sidebar_html_enabled, + 'upload_asset_url': upload_asset_url, + 'course_handler_url': reverse_course_url('course_handler', course_key), + 'language_options': settings.ALL_LANGUAGES, + 'credit_eligibility_enabled': credit_eligibility_enabled, + 'is_credit_course': False, + 'show_min_grade_warning': False, + 'enrollment_end_editable': enrollment_end_editable, + 'is_prerequisite_courses_enabled': is_prerequisite_courses_enabled(), + 'is_entrance_exams_enabled': is_entrance_exams_enabled(), + 'enable_extended_course_details': enable_extended_course_details, + 'upgrade_deadline': upgrade_deadline, + } + if is_prerequisite_courses_enabled(): + courses, in_process_course_actions = get_courses_accessible_to_user(request, course_module.location.org) + # exclude current course from the list of available courses + courses = (course for course in courses if course.id != course_key) + if courses: + courses, __ = _process_courses_list(courses, in_process_course_actions) + settings_context.update({'possible_pre_requisite_courses': list(courses)}) + + if credit_eligibility_enabled: + if is_credit_course(course_key): + # get and all credit eligibility requirements + credit_requirements = get_credit_requirements(course_key) + # pair together requirements with same 'namespace' values + paired_requirements = {} + for requirement in credit_requirements: + namespace = requirement.pop("namespace") + paired_requirements.setdefault(namespace, []).append(requirement) + + # if 'minimum_grade_credit' of a course is not set or 0 then + # show warning message to course author. + show_min_grade_warning = False if course_module.minimum_grade_credit > 0 else True + settings_context.update( + { + 'is_credit_course': True, + 'credit_requirements': paired_requirements, + 'show_min_grade_warning': show_min_grade_warning, + } + ) + + return render_to_response('settings.html', settings_context) + elif 'application/json' in request.META.get('HTTP_ACCEPT', ''): + if request.method == 'GET': + course_details = CourseDetails.fetch(course_key) + return JsonResponse( + course_details, + # encoder serializes dates, old locations, and instances + encoder=CourseSettingsEncoder + ) + # For every other possible method type submitted by the caller... + else: + # if pre-requisite course feature is enabled set pre-requisite course + if is_prerequisite_courses_enabled(): + prerequisite_course_keys = request.json.get('pre_requisite_courses', []) + if prerequisite_course_keys: + if not all(is_valid_course_key(course_key) for course_key in prerequisite_course_keys): + return JsonResponseBadRequest({"error": _("Invalid prerequisite course key")}) + set_prerequisite_courses(course_key, prerequisite_course_keys) + else: + # None is chosen, so remove the course prerequisites + course_milestones = milestones_api.get_course_milestones(course_key=course_key, relationship="requires") + for milestone in course_milestones: + ee_milestone_namespace = generate_milestone_namespace( + get_namespace_choices().get('ENTRANCE_EXAM'), + course_key + ) + if not milestone["namespace"] == ee_milestone_namespace: + remove_prerequisite_course(course_key, milestone) + + # If the entrance exams feature has been enabled, we'll need to check for some + # feature-specific settings and handle them accordingly + # We have to be careful that we're only executing the following logic if we actually + # need to create or delete an entrance exam from the specified course + if is_entrance_exams_enabled(): + course_entrance_exam_present = course_module.entrance_exam_enabled + entrance_exam_enabled = request.json.get('entrance_exam_enabled', '') == 'true' + ee_min_score_pct = request.json.get('entrance_exam_minimum_score_pct', None) + # If the entrance exam box on the settings screen has been checked... + if entrance_exam_enabled: + # Load the default minimum score threshold from settings, then try to override it + entrance_exam_minimum_score_pct = float(settings.ENTRANCE_EXAM_MIN_SCORE_PCT) + if ee_min_score_pct: + entrance_exam_minimum_score_pct = float(ee_min_score_pct) + if entrance_exam_minimum_score_pct.is_integer(): + entrance_exam_minimum_score_pct = entrance_exam_minimum_score_pct / 100 + # If there's already an entrance exam defined, we'll update the existing one + if course_entrance_exam_present: + exam_data = { + 'entrance_exam_minimum_score_pct': entrance_exam_minimum_score_pct + } + update_entrance_exam(request, course_key, exam_data) + # If there's no entrance exam defined, we'll create a new one + else: + create_entrance_exam(request, course_key, entrance_exam_minimum_score_pct) + + # If the entrance exam box on the settings screen has been unchecked, + # and the course has an entrance exam attached... + elif not entrance_exam_enabled and course_entrance_exam_present: + delete_entrance_exam(request, course_key) + + # Perform the normal update workflow for the CourseDetails model + return JsonResponse( + CourseDetails.update_from_json(course_key, request.json, request.user), + encoder=CourseSettingsEncoder + ) + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT", "DELETE")) +@expect_json +def grading_handler(request, course_key_string, grader_index=None): + """ + Course Grading policy configuration + GET + html: get the page + json no grader_index: get the CourseGrading model (graceperiod, cutoffs, and graders) + json w/ grader_index: get the specific grader + PUT + json no grader_index: update the Course through the CourseGrading model + json w/ grader_index: create or update the specific grader (create if index out of range) + """ + course_key = CourseKey.from_string(course_key_string) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + + if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET': + course_details = CourseGradingModel.fetch(course_key) + + return render_to_response('settings_graders.html', { + 'context_course': course_module, + 'course_locator': course_key, + 'course_details': course_details, + 'grading_url': reverse_course_url('grading_handler', course_key), + 'is_credit_course': is_credit_course(course_key), + }) + elif 'application/json' in request.META.get('HTTP_ACCEPT', ''): + if request.method == 'GET': + if grader_index is None: + return JsonResponse( + CourseGradingModel.fetch(course_key), + # encoder serializes dates, old locations, and instances + encoder=CourseSettingsEncoder + ) + else: + return JsonResponse(CourseGradingModel.fetch_grader(course_key, grader_index)) + elif request.method in ('POST', 'PUT'): # post or put, doesn't matter. + # update credit course requirements if 'minimum_grade_credit' + # field value is changed + if 'minimum_grade_credit' in request.json: + update_credit_course_requirements.delay(six.text_type(course_key)) + + # None implies update the whole model (cutoffs, graceperiod, and graders) not a specific grader + if grader_index is None: + return JsonResponse( + CourseGradingModel.update_from_json(course_key, request.json, request.user), + encoder=CourseSettingsEncoder + ) + else: + return JsonResponse( + CourseGradingModel.update_grader_from_json(course_key, request.json, request.user) + ) + elif request.method == "DELETE" and grader_index is not None: + CourseGradingModel.delete_grader(course_key, grader_index, request.user) + return JsonResponse() + + +def _refresh_course_tabs(request, course_module): + """ + Automatically adds/removes tabs if changes to the course require them. + + Raises: + InvalidTabsException: raised if there's a problem with the new version of the tabs. + """ + + def update_tab(tabs, tab_type, tab_enabled): + """ + Adds or removes a course tab based upon whether it is enabled. + """ + tab_panel = { + "type": tab_type.type, + } + has_tab = tab_panel in tabs + if tab_enabled and not has_tab: + tabs.append(CourseTab.from_json(tab_panel)) + elif not tab_enabled and has_tab: + tabs.remove(tab_panel) + + course_tabs = copy.copy(course_module.tabs) + + # Additionally update any tabs that are provided by non-dynamic course views + for tab_type in CourseTabPluginManager.get_tab_types(): + if not tab_type.is_dynamic and tab_type.is_default: + tab_enabled = tab_type.is_enabled(course_module, user=request.user) + update_tab(course_tabs, tab_type, tab_enabled) + + CourseTabList.validate_tabs(course_tabs) + + # Save the tabs into the course if they have been changed + if course_tabs != course_module.tabs: + course_module.tabs = course_tabs + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT")) +@expect_json +def advanced_settings_handler(request, course_key_string): + """ + Course settings configuration + GET + html: get the page + json: get the model + PUT, POST + json: update the Course's settings. The payload is a json rep of the + metadata dicts. + """ + course_key = CourseKey.from_string(course_key_string) + with modulestore().bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + + advanced_dict = CourseMetadata.fetch(course_module) + if settings.FEATURES.get('DISABLE_MOBILE_COURSE_AVAILABLE', False): + advanced_dict.get('mobile_available')['deprecated'] = True + + if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET': + publisher_enabled = configuration_helpers.get_value_for_org( + course_module.location.org, + 'ENABLE_PUBLISHER', + settings.FEATURES.get('ENABLE_PUBLISHER', False) + ) + + return render_to_response('settings_advanced.html', { + 'context_course': course_module, + 'advanced_dict': advanced_dict, + 'advanced_settings_url': reverse_course_url('advanced_settings_handler', course_key), + 'publisher_enabled': publisher_enabled, + + }) + elif 'application/json' in request.META.get('HTTP_ACCEPT', ''): + if request.method == 'GET': + return JsonResponse(CourseMetadata.fetch(course_module)) + else: + try: + # validate data formats and update the course module. + # Note: don't update mongo yet, but wait until after any tabs are changed + is_valid, errors, updated_data = CourseMetadata.validate_and_update_from_json( + course_module, + request.json, + user=request.user, + ) + + if is_valid: + try: + # update the course tabs if required by any setting changes + _refresh_course_tabs(request, course_module) + except InvalidTabsException as err: + log.exception(text_type(err)) + response_message = [ + { + 'message': _('An error occurred while trying to save your tabs'), + 'model': {'display_name': _('Tabs Exception')} + } + ] + return JsonResponseBadRequest(response_message) + + # now update mongo + modulestore().update_item(course_module, request.user.id) + + return JsonResponse(updated_data) + else: + return JsonResponseBadRequest(errors) + + # Handle all errors that validation doesn't catch + except (TypeError, ValueError, InvalidTabsException) as err: + return HttpResponseBadRequest( + django.utils.html.escape(text_type(err)), + content_type="text/plain" + ) + + +class TextbookValidationError(Exception): + "An error thrown when a textbook input is invalid" + pass + + +def validate_textbooks_json(text): + """ + Validate the given text as representing a single PDF textbook + """ + if isinstance(text, (bytes, bytearray)): # data appears as bytes + text = text.decode('utf-8') + try: + textbooks = json.loads(text) + except ValueError: + raise TextbookValidationError("invalid JSON") + if not isinstance(textbooks, (list, tuple)): + raise TextbookValidationError("must be JSON list") + for textbook in textbooks: + validate_textbook_json(textbook) + # check specified IDs for uniqueness + all_ids = [textbook["id"] for textbook in textbooks if "id" in textbook] + unique_ids = set(all_ids) + if len(all_ids) > len(unique_ids): + raise TextbookValidationError("IDs must be unique") + return textbooks + + +def validate_textbook_json(textbook): + """ + Validate the given text as representing a list of PDF textbooks + """ + if isinstance(textbook, (bytes, bytearray)): # data appears as bytes + textbook = textbook.decode('utf-8') + if isinstance(textbook, six.string_types): + try: + textbook = json.loads(textbook) + except ValueError: + raise TextbookValidationError("invalid JSON") + if not isinstance(textbook, dict): + raise TextbookValidationError("must be JSON object") + if not textbook.get("tab_title"): + raise TextbookValidationError("must have tab_title") + tid = six.text_type(textbook.get("id", "")) + if tid and not tid[0].isdigit(): + raise TextbookValidationError("textbook ID must start with a digit") + return textbook + + +def assign_textbook_id(textbook, used_ids=()): + """ + Return an ID that can be assigned to a textbook + and doesn't match the used_ids + """ + tid = BlockUsageLocator.clean(textbook["tab_title"]) + if not tid[0].isdigit(): + # stick a random digit in front + tid = random.choice(string.digits) + tid + while tid in used_ids: + # add a random ASCII character to the end + tid = tid + random.choice(string.ascii_lowercase) + return tid + + +@require_http_methods(("GET", "POST", "PUT")) +@login_required +@ensure_csrf_cookie +def textbooks_list_handler(request, course_key_string): + """ + A RESTful handler for textbook collections. + + GET + html: return textbook list page (Backbone application) + json: return JSON representation of all textbooks in this course + POST + json: create a new textbook for this course + PUT + json: overwrite all textbooks in the course with the given list + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course = get_course_and_check_access(course_key, request.user) + + if "application/json" not in request.META.get('HTTP_ACCEPT', 'text/html'): + # return HTML page + upload_asset_url = reverse_course_url('assets_handler', course_key) + textbook_url = reverse_course_url('textbooks_list_handler', course_key) + return render_to_response('textbooks.html', { + 'context_course': course, + 'textbooks': course.pdf_textbooks, + 'upload_asset_url': upload_asset_url, + 'textbook_url': textbook_url, + }) + + # from here on down, we know the client has requested JSON + if request.method == 'GET': + return JsonResponse(course.pdf_textbooks) + elif request.method == 'PUT': + try: + textbooks = validate_textbooks_json(request.body) + except TextbookValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + + tids = set(t["id"] for t in textbooks if "id" in t) + for textbook in textbooks: + if "id" not in textbook: + tid = assign_textbook_id(textbook, tids) + textbook["id"] = tid + tids.add(tid) + + if not any(tab['type'] == 'pdf_textbooks' for tab in course.tabs): + course.tabs.append(CourseTab.load('pdf_textbooks')) + course.pdf_textbooks = textbooks + store.update_item(course, request.user.id) + return JsonResponse(course.pdf_textbooks) + elif request.method == 'POST': + # create a new textbook for the course + try: + textbook = validate_textbook_json(request.body) + except TextbookValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + if not textbook.get("id"): + tids = set(t["id"] for t in course.pdf_textbooks if "id" in t) + textbook["id"] = assign_textbook_id(textbook, tids) + existing = course.pdf_textbooks + existing.append(textbook) + course.pdf_textbooks = existing + if not any(tab['type'] == 'pdf_textbooks' for tab in course.tabs): + course.tabs.append(CourseTab.load('pdf_textbooks')) + store.update_item(course, request.user.id) + resp = JsonResponse(textbook, status=201) + resp["Location"] = reverse_course_url( + 'textbooks_detail_handler', + course.id, + kwargs={'textbook_id': textbook["id"]} + ) + return resp + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT", "DELETE")) +def textbooks_detail_handler(request, course_key_string, textbook_id): + """ + JSON API endpoint for manipulating a textbook via its internal ID. + Used by the Backbone application. + + GET + json: return JSON representation of textbook + POST or PUT + json: update textbook based on provided information + DELETE + json: remove textbook + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course_module = get_course_and_check_access(course_key, request.user) + matching_id = [tb for tb in course_module.pdf_textbooks + if six.text_type(tb.get("id")) == six.text_type(textbook_id)] + if matching_id: + textbook = matching_id[0] + else: + textbook = None + + if request.method == 'GET': + if not textbook: + return JsonResponse(status=404) + return JsonResponse(textbook) + elif request.method in ('POST', 'PUT'): # can be either and sometimes django is rewriting one to the other + try: + new_textbook = validate_textbook_json(request.body) + except TextbookValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + new_textbook["id"] = textbook_id + if textbook: + i = course_module.pdf_textbooks.index(textbook) + new_textbooks = course_module.pdf_textbooks[0:i] + new_textbooks.append(new_textbook) + new_textbooks.extend(course_module.pdf_textbooks[i + 1:]) + course_module.pdf_textbooks = new_textbooks + else: + course_module.pdf_textbooks.append(new_textbook) + store.update_item(course_module, request.user.id) + return JsonResponse(new_textbook, status=201) + elif request.method == 'DELETE': + if not textbook: + return JsonResponse(status=404) + i = course_module.pdf_textbooks.index(textbook) + remaining_textbooks = course_module.pdf_textbooks[0:i] + remaining_textbooks.extend(course_module.pdf_textbooks[i + 1:]) + course_module.pdf_textbooks = remaining_textbooks + store.update_item(course_module, request.user.id) + return JsonResponse() + + +def remove_content_or_experiment_group(request, store, course, configuration, group_configuration_id, group_id=None): + """ + Remove content group or experiment group configuration only if it's not in use. + """ + configuration_index = course.user_partitions.index(configuration) + if configuration.scheme.name == RANDOM_SCHEME: + usages = GroupConfiguration.get_content_experiment_usage_info(store, course) + used = int(group_configuration_id) in usages + + if used: + return JsonResponse( + {"error": _("This group configuration is in use and cannot be deleted.")}, + status=400 + ) + course.user_partitions.pop(configuration_index) + elif configuration.scheme.name == COHORT_SCHEME: + if not group_id: + return JsonResponse(status=404) + + group_id = int(group_id) + usages = GroupConfiguration.get_partitions_usage_info(store, course) + used = group_id in usages[configuration.id] + + if used: + return JsonResponse( + {"error": _("This content group is in use and cannot be deleted.")}, + status=400 + ) + + matching_groups = [group for group in configuration.groups if group.id == group_id] + if matching_groups: + group_index = configuration.groups.index(matching_groups[0]) + configuration.groups.pop(group_index) + else: + return JsonResponse(status=404) + + course.user_partitions[configuration_index] = configuration + + store.update_item(course, request.user.id) + return JsonResponse(status=204) + + +@require_http_methods(("GET", "POST")) +@login_required +@ensure_csrf_cookie +def group_configurations_list_handler(request, course_key_string): + """ + A RESTful handler for Group Configurations + + GET + html: return Group Configurations list page (Backbone application) + POST + json: create new group configuration + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course = get_course_and_check_access(course_key, request.user) + + if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): + group_configuration_url = reverse_course_url('group_configurations_list_handler', course_key) + course_outline_url = reverse_course_url('course_handler', course_key) + should_show_experiment_groups = are_content_experiments_enabled(course) + if should_show_experiment_groups: + experiment_group_configurations = GroupConfiguration.get_split_test_partitions_with_usage(store, course) + else: + experiment_group_configurations = None + + all_partitions = GroupConfiguration.get_all_user_partition_details(store, course) + should_show_enrollment_track = False + has_content_groups = False + displayable_partitions = [] + for partition in all_partitions: + partition['read_only'] = getattr(UserPartition.get_scheme(partition['scheme']), 'read_only', False) + + if partition['scheme'] == COHORT_SCHEME: + has_content_groups = True + displayable_partitions.append(partition) + elif partition['scheme'] == CONTENT_TYPE_GATING_SCHEME: + # Add it to the front of the list if it should be shown. + if ContentTypeGatingConfig.current(course_key=course_key).studio_override_enabled: + displayable_partitions.append(partition) + elif partition['scheme'] == ENROLLMENT_SCHEME: + should_show_enrollment_track = len(partition['groups']) > 1 + + # Add it to the front of the list if it should be shown. + if should_show_enrollment_track: + displayable_partitions.insert(0, partition) + elif partition['scheme'] != RANDOM_SCHEME: + # Experiment group configurations are handled explicitly above. We don't + # want to display their groups twice. + displayable_partitions.append(partition) + + # Set the sort-order. Higher numbers sort earlier + scheme_priority = defaultdict(lambda: -1, { + ENROLLMENT_SCHEME: 1, + CONTENT_TYPE_GATING_SCHEME: 0 + }) + displayable_partitions.sort(key=lambda p: scheme_priority[p['scheme']], reverse=True) + # Add empty content group if there is no COHORT User Partition in the list. + # This will add ability to add new groups in the view. + if not has_content_groups: + displayable_partitions.append(GroupConfiguration.get_or_create_content_group(store, course)) + + return render_to_response('group_configurations.html', { + 'context_course': course, + 'group_configuration_url': group_configuration_url, + 'course_outline_url': course_outline_url, + 'experiment_group_configurations': experiment_group_configurations, + 'should_show_experiment_groups': should_show_experiment_groups, + 'all_group_configurations': displayable_partitions, + 'should_show_enrollment_track': should_show_enrollment_track + }) + elif "application/json" in request.META.get('HTTP_ACCEPT'): + if request.method == 'POST': + # create a new group configuration for the course + try: + new_configuration = GroupConfiguration(request.body, course).get_user_partition() + except GroupConfigurationsValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + + course.user_partitions.append(new_configuration) + response = JsonResponse(new_configuration.to_json(), status=201) + + response["Location"] = reverse_course_url( + 'group_configurations_detail_handler', + course.id, + kwargs={'group_configuration_id': new_configuration.id} + ) + store.update_item(course, request.user.id) + return response + else: + return HttpResponse(status=406) + + +@login_required +@ensure_csrf_cookie +@require_http_methods(("POST", "PUT", "DELETE")) +def group_configurations_detail_handler(request, course_key_string, group_configuration_id, group_id=None): + """ + JSON API endpoint for manipulating a group configuration via its internal ID. + Used by the Backbone application. + + POST or PUT + json: update group configuration based on provided information + """ + course_key = CourseKey.from_string(course_key_string) + store = modulestore() + with store.bulk_operations(course_key): + course = get_course_and_check_access(course_key, request.user) + matching_id = [p for p in course.user_partitions + if six.text_type(p.id) == six.text_type(group_configuration_id)] + if matching_id: + configuration = matching_id[0] + else: + configuration = None + + if request.method in ('POST', 'PUT'): # can be either and sometimes django is rewriting one to the other + try: + new_configuration = GroupConfiguration(request.body, course, group_configuration_id).get_user_partition() + except GroupConfigurationsValidationError as err: + return JsonResponse({"error": text_type(err)}, status=400) + + if configuration: + index = course.user_partitions.index(configuration) + course.user_partitions[index] = new_configuration + else: + course.user_partitions.append(new_configuration) + store.update_item(course, request.user.id) + configuration = GroupConfiguration.update_usage_info(store, course, new_configuration) + return JsonResponse(configuration, status=201) + + elif request.method == "DELETE": + if not configuration: + return JsonResponse(status=404) + + return remove_content_or_experiment_group( + request=request, + store=store, + course=course, + configuration=configuration, + group_configuration_id=group_configuration_id, + group_id=group_id + ) + + +def are_content_experiments_enabled(course): + """ + Returns True if content experiments have been enabled for the course. + """ + return ( + 'split_test' in ADVANCED_COMPONENT_TYPES and + 'split_test' in course.advanced_modules + ) + + +def _get_course_creator_status(user): + """ + Helper method for returning the course creator status for a particular user, + taking into account the values of DISABLE_COURSE_CREATION and ENABLE_CREATOR_GROUP. + + If the user passed in has not previously visited the index page, it will be + added with status 'unrequested' if the course creator group is in use. + """ + + if user.is_staff: + course_creator_status = 'granted' + elif settings.FEATURES.get('DISABLE_COURSE_CREATION', False): + course_creator_status = 'disallowed_for_this_site' + elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): + course_creator_status = get_course_creator_status(user) + if course_creator_status is None: + # User not grandfathered in as an existing user, has not previously visited the dashboard page. + # Add the user to the course creator admin table with status 'unrequested'. + add_user_with_status_unrequested(user) + course_creator_status = get_course_creator_status(user) + else: + course_creator_status = 'granted' + + return course_creator_status diff --git a/cms/envs/common.py b/cms/envs/common.py index 3e77bc28d3b4..95e57cd3abf6 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1461,6 +1461,9 @@ # CMS specific user task handling 'cms_user_tasks.apps.CmsUserTasksConfig', + # Appsembler customization app for CMS + 'appsembler.apps.CMSAppsemblerConfig', + # Unusual migrations 'database_fixups', diff --git a/cms/static/js/certificates/models/certificate.js b/cms/static/js/certificates/models/certificate.js index a440d569d606..d0e6ae6e9f35 100644 --- a/cms/static/js/certificates/models/certificate.js +++ b/cms/static/js/certificates/models/certificate.js @@ -43,7 +43,7 @@ define([ initialize: function(attributes, options) { // Set up the initial state of the attributes set for this model instance this.canBeEmpty = options && options.canBeEmpty; - if (options.add) { + if (options.add && !attributes.signatories) { // Ensure at least one child Signatory model is defined for any new Certificate model attributes.signatories = new SignatoryModel({certificate: this}); } diff --git a/cms/templates/js/group-configuration-details.underscore b/cms/templates/js/group-configuration-details.underscore index 8310a2a657bf..f3fd77971827 100644 --- a/cms/templates/js/group-configuration-details.underscore +++ b/cms/templates/js/group-configuration-details.underscore @@ -31,9 +31,9 @@ <% if(showGroups) { %> <% allocation = Math.floor(100 / groups.length) %> -
    +
      <% groups.each(function(group, groupIndex) { %> -
    1. +
    2. <%- group.get('name') %> <%- allocation %>%
      3. diff --git a/cms/templates/js/maintenance/force-published-course-response.underscore b/cms/templates/js/maintenance/force-published-course-response.underscore index d77059ca5311..7441413dd47c 100644 --- a/cms/templates/js/maintenance/force-published-course-response.underscore +++ b/cms/templates/js/maintenance/force-published-course-response.underscore @@ -3,7 +3,7 @@ <%- gettext('You have done a dry run of force publishing the course. Nothing has changed. Had you run it, the following course versions would have been change.') %>
      - <%= StringUtils.interpolate( + <%- StringUtils.interpolate( gettext('The published branch version, {published}, was reset to the draft branch version, {draft}.'), { published: current_versions['published-branch'], diff --git a/cms/templates/js/mock/mock-xmodule-editor.underscore b/cms/templates/js/mock/mock-xmodule-editor.underscore index cb9a59c6f5b2..d65921f3891a 100644 --- a/cms/templates/js/mock/mock-xmodule-editor.underscore +++ b/cms/templates/js/mock/mock-xmodule-editor.underscore @@ -16,13 +16,13 @@ diff --git a/common/djangoapps/student/helpers.py b/common/djangoapps/student/helpers.py index 3c47510b5af1..354c9a328d1a 100644 --- a/common/djangoapps/student/helpers.py +++ b/common/djangoapps/student/helpers.py @@ -6,6 +6,7 @@ import json import logging import mimetypes +import re import urllib.parse from collections import OrderedDict from datetime import datetime @@ -65,6 +66,8 @@ EMAIL_EXISTS_MSG_FMT = _("An account with the Email '{email}' already exists.") USERNAME_EXISTS_MSG_FMT = _("An account with the Public Username '{username}' already exists.") +COURSE_URL_PATTERN = re.compile(r'courses/course-v1:[^/]+/course') + log = logging.getLogger(__name__) @@ -224,7 +227,8 @@ def check_verify_status_by_course(user, course_enrollments): # Query string parameters that can be passed to the "finish_auth" view to manage # things like auto-enrollment. -POST_AUTH_PARAMS = ('course_id', 'enrollment_action', 'course_mode', 'email_opt_in', 'purchase_workflow') +POST_AUTH_PARAMS = ('course_id', 'enrollment_action', 'course_mode', 'email_opt_in', 'purchase_workflow', + 'hide_elements') def get_next_url_for_login_page(request): @@ -301,6 +305,7 @@ def _get_redirect_to(request_host, request_headers, request_params, request_is_h redirect url if safe else None """ redirect_to = request_params.get('next') + redirect_to = sanitize_next_parameter(redirect_to) header_accept = request_headers.get('HTTP_ACCEPT', '') accepts_text_html = any( mime_type in header_accept @@ -700,3 +705,37 @@ def get_resume_urls_for_enrollments(user, enrollments): url_to_block = '' resume_course_urls[enrollment.course_id] = url_to_block return resume_course_urls + + +def sanitize_next_parameter(next_param): + """ + Check the next parameter pattern and update the + symbol to its ASCII equivalent. + """ + if not next_param: + return next_param + + if COURSE_URL_PATTERN.match(next_param): + # Sometimes the course id received with incorrect encoding/decoding, we need to + # replace it to the correct pattern: + # course-v1:test-sandbox PREP-CORE C -> course-v1:test-sandbox+PREP-CORE+C + # + # Note: We are not expect to have any spaces in course URL + + if ' ' in next_param: + next_param = next_param.replace(' ', '+') + elif '%20' in next_param: + next_param = next_param.replace('%20', '+') + + sanitized_next_parameter = re.sub(r'\+', '%2B', next_param) + + log.info( + u"The course-like next parameter was detected '%(next_param)s'" + u" this will be replaced with sanitized version: '%(sanitized_next_parameter)s'", + { + "next_param": next_param, + "sanitized_next_parameter": sanitized_next_parameter, + } + ) + return sanitized_next_parameter + + return next_param diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 37b208311ae2..9db611db6d7c 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -1656,7 +1656,10 @@ def unenroll_by_email(cls, email, course_id): RequestCache('get_enrollment').clear() try: - user = User.objects.get(email=email) + if settings.FEATURES.get('APPSEMBLER_MULTI_TENANT_EMAILS', False): + user = CourseEnrollment.get_user_by_email_within_organization(email) + else: + user = User.objects.get(email=email) return cls.unenroll(user, course_id) except User.DoesNotExist: log.error( diff --git a/common/djangoapps/student/tests/test_helpers.py b/common/djangoapps/student/tests/test_helpers.py index 655d9b763b36..f069910296f8 100644 --- a/common/djangoapps/student/tests/test_helpers.py +++ b/common/djangoapps/student/tests/test_helpers.py @@ -14,7 +14,7 @@ from testfixtures import LogCapture from openedx.core.djangoapps.site_configuration.tests.test_util import with_site_configuration_context -from student.helpers import get_next_url_for_login_page +from student.helpers import get_next_url_for_login_page, sanitize_next_parameter LOGGER_NAME = "student.helpers" @@ -156,3 +156,49 @@ def test_custom_tahoe_site_redirect_lms(self): 'LOGIN_REDIRECT_URL': '' # Falsy or empty URLs should not be used }): assert '/dashboard' == get_next_url_for_login_page(request), 'Falsy url should default to dashboard' + + def test_sanitize_next_param(self): + # Valid URL with plus - change the plus symbol to ASCII code + next_param = 'courses/course-v1:abc-sandbox+ACC-PTF+C/course' + expected_result = 'courses/course-v1:abc-sandbox%2BACC-PTF%2BC/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + # Valid URL without plus - keep the next_param as it is + next_param = 'courses/course-v1:abc-sandbox/course' + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # Empty string - keep the next_param as it is + next_param = '' + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # None input - keep the next_param as it is + next_param = None + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # Invalid pattern - keep the next_param as it is + next_param = 'some/other/path' + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # Invalid URL with space - replace the ' ' with '+' and encode it + expected_result = 'courses/course-v1:abc-sandbox%2BACC-PTF%2BC/course' + + next_param = 'courses/course-v1:abc-sandbox ACC-PTF C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox ACC-PTF+C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox+ACC-PTF C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + # Invalid URL with encoded space - replace the '%20' with '+' and encode it + expected_result = 'courses/course-v1:abc-sandbox%2BACC-PTF%2BC/course' + + next_param = 'courses/course-v1:abc-sandbox%20ACC-PTF%20C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox%20ACC-PTF+C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox+ACC-PTF%20C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) diff --git a/common/djangoapps/third_party_auth/settings.py b/common/djangoapps/third_party_auth/settings.py index 234dba9857d0..09d6a9cef812 100644 --- a/common/djangoapps/third_party_auth/settings.py +++ b/common/djangoapps/third_party_auth/settings.py @@ -57,7 +57,9 @@ def apply_settings(django_settings): 'third_party_auth.pipeline.get_username', 'third_party_auth.pipeline.set_pipeline_timeout', 'third_party_auth.pipeline.ensure_user_information', - 'social_core.pipeline.user.create_user', + # wrap social_core.pipeline.user.create_user so we can test selectively disabling + 'openedx.core.djangoapps.appsembler.tahoe_idp.tpa_pipeline.wrapped_social_core_create_user', + # 'social_core.pipeline.user.create_user', 'social_core.pipeline.social_auth.associate_user', 'social_core.pipeline.social_auth.load_extra_data', 'social_core.pipeline.user.user_details', diff --git a/common/djangoapps/track/views/segmentio.py b/common/djangoapps/track/views/segmentio.py index 82aa2923752e..a61c5fdf5a5e 100644 --- a/common/djangoapps/track/views/segmentio.py +++ b/common/djangoapps/track/views/segmentio.py @@ -289,8 +289,8 @@ def send_event(request, method, **params): segment_key = helpers.get_current_site_configuration().get_secret_value('SEGMENT_KEY') if segment_key: data['writeKey'] = segment_key - data['messageId'] = 'ajs-' + uuid.uuid4().hex - site_response = requests.post(url, json=data) + data['messageId'] = 'ajs-next-' + uuid.uuid4().hex + site_response = requests.post(url, json=data) # noqa: F841 return HttpResponse( main_response.content, status=main_response.status_code, diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index 6051cc428621..98adaf6af182 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -90,6 +90,7 @@ function(VideoPlayer, i18n, moment, _) { _youtubeApiDeferred = null, _oldOnYouTubeIframeAPIReady; + const setupOnYouTubeIframeAPIReadyMaxCalls=5; Initialize.prototype = methodsDict; @@ -163,14 +164,16 @@ function(VideoPlayer, i18n, moment, _) { // so that it resolves our Deferred object, which will call all of the // OnYouTubeIframeAPIReady callbacks. // - // If this global function is already defined, we store it first, and make - // sure that it gets executed when our Deferred object is resolved. - setupOnYouTubeIframeAPIReady = function() { + + let setupOnYouTubeIframeAPIReady = function() { + + // If this global function is already defined, we store it first, and make + // sure that it gets executed when our Deferred object is resolved. _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; window.onYouTubeIframeAPIReady = function() { - window.onYouTubeIframeAPIReady.resolve(); - }; + _youtubeApiDeferred.resolve(); + } window.onYouTubeIframeAPIReady.resolve = _youtubeApiDeferred.resolve; window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; @@ -201,6 +204,7 @@ function(VideoPlayer, i18n, moment, _) { window.YT.ready(onYTApiReady); }); } + } else { video = VideoPlayer(state); diff --git a/common/lib/xmodule/xmodule/video_module/video_module.py b/common/lib/xmodule/xmodule/video_module/video_module.py index 1884a1388c9b..eea74231c91c 100644 --- a/common/lib/xmodule/xmodule/video_module/video_module.py +++ b/common/lib/xmodule/xmodule/video_module/video_module.py @@ -63,6 +63,8 @@ from .video_utils import create_youtube_string, format_xml_exception_message, get_poster, rewrite_video_url from .video_xfields import VideoFields +from openedx.core.djangoapps.appsembler.sites import utils as appsembler_site_utils + # The following import/except block for edxval is temporary measure until # edxval is a proper XBlock Runtime Service. # @@ -393,6 +395,10 @@ def get_html(self, view=STUDENT_VIEW): # it anymore; therefore we force-disable it in this case (when controls aren't visible). autoadvance_this_video = self.auto_advance and autoadvance_enabled + scheme = "https" if settings.HTTPS == "on" else "http" + lms_base_url = appsembler_site_utils.get_lms_link_from_course_key(settings.LMS_ROOT_URL, self.course_id) + lms_root_url = "{}://{}".format(scheme, lms_base_url) if "http" not in lms_base_url else lms_base_url + metadata = { 'saveStateEnabled': view != PUBLIC_VIEW, 'saveStateUrl': self.ajax_url + '/save_user_state', @@ -418,7 +424,7 @@ def get_html(self, view=STUDENT_VIEW): 'transcriptLanguages': sorted_languages, 'ytTestTimeout': settings.YOUTUBE['TEST_TIMEOUT'], 'ytApiUrl': settings.YOUTUBE['API'], - 'lmsRootURL': settings.LMS_ROOT_URL, + 'lmsRootURL': lms_root_url, 'ytMetadataEndpoint': ( # In the new runtime, get YouTube metadata via a handler. The handler supports anonymous users and # can work in sandboxed iframes. In the old runtime, the JS will call the LMS's yt_video_metadata diff --git a/common/test/data/badges/good.png b/common/test/data/badges/good.png deleted file mode 100644 index 0c8fa7206fa3..000000000000 Binary files a/common/test/data/badges/good.png and /dev/null differ diff --git a/common/test/data/badges/large.png b/common/test/data/badges/large.png deleted file mode 100644 index cd261a518c83..000000000000 Binary files a/common/test/data/badges/large.png and /dev/null differ diff --git a/common/test/data/badges/unbalanced.png b/common/test/data/badges/unbalanced.png deleted file mode 100644 index 31e0924d4ca8..000000000000 Binary files a/common/test/data/badges/unbalanced.png and /dev/null differ diff --git a/conf/locale/ar/LC_MESSAGES/django.mo b/conf/locale/ar/LC_MESSAGES/django.mo index c05c00998def..a73163947863 100644 Binary files a/conf/locale/ar/LC_MESSAGES/django.mo and b/conf/locale/ar/LC_MESSAGES/django.mo differ diff --git a/conf/locale/ar/LC_MESSAGES/django.po b/conf/locale/ar/LC_MESSAGES/django.po index 75cc4f5f81b2..1a154de88d00 100644 --- a/conf/locale/ar/LC_MESSAGES/django.po +++ b/conf/locale/ar/LC_MESSAGES/django.po @@ -16825,7 +16825,7 @@ msgstr "استعراض عملية التقييم في Studio" #: lms/templates/courseware/progress.html msgid "Course Progress for '{username}' ({email})" -msgstr "تقدم الدورة ل '{اسم المستخدم}' ({عنوان البريد الالكتروني})" +msgstr "تقدم الدورة ل '{username}' ({email})" #: lms/templates/courseware/progress.html msgid "View Certificate" diff --git a/lms/djangoapps/courseware/models.py b/lms/djangoapps/courseware/models.py index d6dae57db4c5..09d64eb9fa63 100644 --- a/lms/djangoapps/courseware/models.py +++ b/lms/djangoapps/courseware/models.py @@ -37,6 +37,21 @@ log = logging.getLogger("edx.courseware") +def should_update_student_module_modified_on_save(): + """ + Fixes RED-3616 to avoid updating StudentModule.modified after regrade or any celery automated updates. + + The modified field is relied upon for Monthly Active Users calculations, so any celery task updates it would be + confused with leaner activity. + + Hack: This duplicates the `common.djangoapps.track.shim:is_celery_worker` helper function, but hopefully we'll + be removing this soon enough that code duplication won't be a problem. + """ + is_celery_worker = getattr(settings, 'IS_CELERY_WORKER', False) + is_hack_enabled = settings.FEATURES.get('TAHOE_STUDENT_MODULES_DISABLE_MODIFIED_IN_CELERY', True) + return not (is_celery_worker and is_hack_enabled) + + def chunks(items, chunk_size): """ Yields the values from items in chunks of size chunk_size @@ -119,7 +134,12 @@ class Meta(object): done = models.CharField(max_length=8, choices=DONE_TYPES, default='na') created = models.DateTimeField(auto_now_add=True, db_index=True) - modified = models.DateTimeField(auto_now=True, db_index=True) + + if should_update_student_module_modified_on_save(): + modified = models.DateTimeField(auto_now=True, db_index=True) + else: + # Fixes RED-3616 to modified on celery. + modified = models.DateTimeField(auto_now_add=True, db_index=True) @classmethod def all_submitted_problems_read_only(cls, course_id): diff --git a/lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py b/lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py new file mode 100644 index 000000000000..7ea880d33060 --- /dev/null +++ b/lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py @@ -0,0 +1,84 @@ +""" +Tests for the RED-3616 hack/fix for MAU calculations depending on StudentModule.modified. +""" + +from importlib import reload + +import pytest +from freezegun import freeze_time + + +def import_fresh_models(): + """ + Import `lms.djangoapps.courseware.models` and reload it to react to features. + """ + from lms.djangoapps.courseware import models as courseware_models + from lms.djangoapps.courseware.tests import factories as courseware_factories + reload(courseware_models) + reload(courseware_factories) + return { + 'courseware_models': courseware_models, + 'courseware_factories': courseware_factories, + } + + +def test_is_untouched_by_default_in_celery(settings): + settings.IS_CELERY_WORKER = True + courseware_models = import_fresh_models()['courseware_models'] + assert not courseware_models.should_update_student_module_modified_on_save(), 'Should be enabled for celery' + + +def test_in_updated_by_default_in_http_requests(settings): + settings.IS_CELERY_WORKER = False + courseware_models = import_fresh_models()['courseware_models'] + assert courseware_models.should_update_student_module_modified_on_save(), 'Should be disabled in http requests' + + +def test_can_be_updated_in_celery_if_needed(settings): + """ + TAHOE_STUDENT_MODULES_DISABLE_MODIFIED_IN_CELERY is on by default but can be turned off via lms FEATURES. + """ + settings.IS_CELERY_WORKER = True + settings.FEATURES = { + **settings.FEATURES, + 'TAHOE_STUDENT_MODULES_DISABLE_MODIFIED_IN_CELERY': False, + } + courseware_models = import_fresh_models()['courseware_models'] + assert courseware_models.should_update_student_module_modified_on_save(), 'The feature is configurable' + + +@pytest.mark.django_db +def test_new_student_module_with_in_celery(settings): + """ + Ensure that `StudentModule.modified` isn't updated when saving from within a celery task. + """ + settings.IS_CELERY_WORKER = True + courseware_factories = import_fresh_models()['courseware_factories'] + + with freeze_time('2012-01-14'): + student_module = courseware_factories.StudentModuleFactory.create() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2012 + + with freeze_time('2020-12-20'): + student_module.save() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2012, 'Should not touch `modified` during in celery' + + +@pytest.mark.django_db +def test_new_student_module_with_in_http_requests(settings): + """ + Ensure that `StudentModule.modified` _is updated_ when saving from within an HTTP request. + """ + courseware_factories = import_fresh_models()['courseware_factories'] + + with freeze_time('2012-01-14'): + student_module = courseware_factories.StudentModuleFactory.create() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2012 + + with freeze_time('2020-12-20'): + student_module.save() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2020, 'Should update `modified` outside celery' diff --git a/lms/static/css/vendor/images/treeview-default-line.gif b/lms/static/css/vendor/images/treeview-default-line.gif deleted file mode 100644 index e88b0d4d61f4..000000000000 Binary files a/lms/static/css/vendor/images/treeview-default-line.gif and /dev/null differ diff --git a/lms/static/css/vendor/images/treeview-default.gif b/lms/static/css/vendor/images/treeview-default.gif deleted file mode 100644 index fa3d8239eab1..000000000000 Binary files a/lms/static/css/vendor/images/treeview-default.gif and /dev/null differ diff --git a/lms/static/css/vendor/indicator.gif b/lms/static/css/vendor/indicator.gif deleted file mode 100755 index 7ecfa15cec35..000000000000 Binary files a/lms/static/css/vendor/indicator.gif and /dev/null differ diff --git a/lms/static/css/vendor/jquery.autocomplete.css b/lms/static/css/vendor/jquery.autocomplete.css deleted file mode 100644 index 4301a633629a..000000000000 --- a/lms/static/css/vendor/jquery.autocomplete.css +++ /dev/null @@ -1,36 +0,0 @@ -.acInput { - width: 200px; -} - -.acResults { - padding: 0px; - border: 1px solid WindowFrame; - background-color: Window; - overflow: hidden; -} - -.acResults ul { - margin: 0px; - padding: 0px; - list-style-position: outside; - list-style: none; -} - -.acResults ul li { - margin: 0px; - padding: 2px 5px; - cursor: pointer; - display: block; - font: menu; - font-size: 12px; - overflow: hidden; -} - -.acLoading { - background : url('indicator.gif') right center no-repeat; -} - -.acSelect { - background-color: Highlight; - color: HighlightText; -} diff --git a/lms/static/css/vendor/jquery.treeview.css b/lms/static/css/vendor/jquery.treeview.css deleted file mode 100644 index 798f2ac004bc..000000000000 --- a/lms/static/css/vendor/jquery.treeview.css +++ /dev/null @@ -1,49 +0,0 @@ -.treeview, .treeview ul { - padding: 0; - margin: 0; - list-style: none; -} - -.treeview ul { - background-color: white; - margin-top: 4px; -} - -.treeview .hitarea { - background: url(images/treeview-default.gif) -64px -25px no-repeat; - height: 16px; - width: 16px; - margin-left: -16px; - float: left; - cursor: pointer; -} -/* fix for IE6 */ -* html .hitarea { - display: inline; - float:none; -} - -.treeview li { - margin: 0; - padding: 3px 0pt 3px 16px; -} - -.treeview a.selected { - background-color: #eee; -} - -#treecontrol { margin: 1em 0; display: none; } - -.treeview .hover { color: red; cursor: pointer; } - -.treeview li { background: url(images/treeview-default-line.gif) 0 0 no-repeat; } -.treeview li.collapsable, .treeview li.expandable { background-position: 0 -176px; } - -.treeview .expandable-hitarea { background-position: -80px -3px; } - -.treeview li.last { background-position: 0 -1766px } -.treeview li.lastCollapsable, .treeview li.lastExpandable { background-image: url(images/treeview-default.gif); } -.treeview li.lastCollapsable { background-position: 0 -111px } -.treeview li.lastExpandable { background-position: -32px -67px } - -.treeview div.lastCollapsable-hitarea, .treeview div.lastExpandable-hitarea { background-position: 0; } diff --git a/lms/static/js/discovery/models/search_state.js b/lms/static/js/discovery/models/search_state.js index cd37f938ea91..e727938cd81f 100644 --- a/lms/static/js/discovery/models/search_state.js +++ b/lms/static/js/discovery/models/search_state.js @@ -11,7 +11,7 @@ return Backbone.Model.extend({ page: 0, - pageSize: 20, + pageSize: 100, // Tahoe: fix a bug related to Course Access Groups - RED-3598 searchTerm: '', terms: {}, jqhxr: null, diff --git a/lms/static/js/student_account/components/StudentAccountDeletion.jsx b/lms/static/js/student_account/components/StudentAccountDeletion.jsx index 30713689bb12..0ed88a0f9f98 100644 --- a/lms/static/js/student_account/components/StudentAccountDeletion.jsx +++ b/lms/static/js/student_account/components/StudentAccountDeletion.jsx @@ -112,18 +112,10 @@ export class StudentAccountDeletion extends React.Component { {bodyDeletion} {bodyDeletion2}

      -

      -

      )} diff --git a/lms/static/js/student_account/views/FinishAuthView.js b/lms/static/js/student_account/views/FinishAuthView.js index 870c56b61591..461516b6e12e 100644 --- a/lms/static/js/student_account/views/FinishAuthView.js +++ b/lms/static/js/student_account/views/FinishAuthView.js @@ -51,7 +51,8 @@ courseId: $.url('?course_id'), courseMode: $.url('?course_mode'), emailOptIn: $.url('?email_opt_in'), - purchaseWorkflow: $.url('?purchase_workflow') + purchaseWorkflow: $.url('?purchase_workflow'), + hideElements: $.url('?hide_elements') }; for (var key in queryParams) { if (queryParams[key]) { @@ -64,6 +65,7 @@ this.emailOptIn = queryParams.emailOptIn; this.nextUrl = this.urls.defaultNextUrl; this.purchaseWorkflow = queryParams.purchaseWorkflow; + this.hideElements = queryParams.hideElements; if (queryParams.next) { // Ensure that the next URL is internal for security reasons if (! window.isExternal(queryParams.next)) { @@ -74,6 +76,9 @@ render: function() { try { + if (this.hideElements) { + document.cookie = 'hideElements=' + this.hideElements + '; path=/'; + } var next = _.bind(this.enrollment, this); this.checkEmailOptIn(next); } catch (err) { diff --git a/lms/static/js/student_account/views/account_settings_factory.js b/lms/static/js/student_account/views/account_settings_factory.js index 670f039914f4..2340c480eb31 100644 --- a/lms/static/js/student_account/views/account_settings_factory.js +++ b/lms/static/js/student_account/views/account_settings_factory.js @@ -117,7 +117,10 @@ required: true, title: gettext('Country or Region of Residence'), valueAttribute: 'country', - options: fieldsData.country.options, + groupOptions: [{ + selectOptions: fieldsData.country.options, + nullValueOptionLabel: gettext('(Not Specified)') + }], persistChanges: true, helpMessage: gettext('The country or region where you live.') }; @@ -180,7 +183,10 @@ gettext('The language used throughout this site. This site is currently available in a limited number of languages. Changing the value of this field will cause the page to refresh.'), // eslint-disable-line max-len {platform_name: platformName} ), - options: fieldsData.language.options, + groupOptions: [{ + nullValueOptionLabel: gettext('Default (English)'), + selectOptions: fieldsData.language.options + }], persistChanges: true, focusNextID: '#u-field-select-country' }) @@ -211,7 +217,10 @@ model: userAccountModel, title: gettext('Education Completed'), valueAttribute: 'level_of_education', - options: fieldsData.level_of_education.options, + groupOptions: [{ + selectOptions: fieldsData.level_of_education.options, + nullValueOptionLabel: gettext('(Not Specified)') + }], persistChanges: true }) }, @@ -220,7 +229,10 @@ model: userAccountModel, title: gettext('Gender'), valueAttribute: 'gender', - options: fieldsData.gender.options, + groupOptions: [{ + selectOptions: fieldsData.gender.options, + nullValueOptionLabel: gettext('(Not Specified)') + }], persistChanges: true }) }, @@ -229,7 +241,10 @@ model: userAccountModel, title: gettext('Year of Birth'), valueAttribute: 'year_of_birth', - options: fieldsData.year_of_birth.options, + groupOptions: [{ + selectOptions: fieldsData.year_of_birth.options, + nullValueOptionLabel: gettext('(Not Specified)') + }], persistChanges: true }) }, @@ -238,7 +253,10 @@ model: userAccountModel, title: gettext('Preferred Language'), valueAttribute: 'language_proficiencies', - options: fieldsData.preferred_language.options, + groupOptions: [{ + selectOptions: fieldsData.preferred_language.options, + nullValueOptionLabel: gettext('(Not Specified)') + }], persistChanges: true }) } diff --git a/lms/static/lms/js/iframe-render.js b/lms/static/lms/js/iframe-render.js new file mode 100644 index 000000000000..f5b5bce6b20a --- /dev/null +++ b/lms/static/lms/js/iframe-render.js @@ -0,0 +1,31 @@ +// List of the classes to hide while rendered in an iframe +const classesToHide = ['.global-header', '.wrapper-course-material', '.a--footer']; + +// Function to get a cookie by name +function getCookieByName(name) { + let cname = name + "="; + let decodedCookie = decodeURIComponent(document.cookie); + let cookies = decodedCookie.split(';'); + for (let i = 0; i < cookies.length; i++) { + let c = cookies[i]; + while (c.charAt(0) == ' ') { + c = c.substring(1); + } + if (c.indexOf(name) == 0) { + return c.substring(name.length, c.length); + } + } + return ""; +} + +document.addEventListener('DOMContentLoaded', function () { + const hideElements = getCookieByName('hideElements'); + + if (hideElements) { + classesToHide.forEach(function (className) { + document.querySelectorAll(className).forEach(function (element) { + element.classList.add('hidden-element'); + }); + }); + } +}); diff --git a/lms/static/sass/shared-v2/_base.scss b/lms/static/sass/shared-v2/_base.scss index 584cf5799faf..450a1aa51b75 100644 --- a/lms/static/sass/shared-v2/_base.scss +++ b/lms/static/sass/shared-v2/_base.scss @@ -24,3 +24,8 @@ @extend .sr-only; } +// Hide element when rendered in iFrame +.hidden-element { + display: none !important; +} + diff --git a/lms/templates/main.html b/lms/templates/main.html index 48edc767a132..af1d51fe9cbc 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -123,6 +123,7 @@ }).call(this, require || RequireJS.require); + <%block name="js_overrides"> ${render_require_js_path_overrides(settings.REQUIRE_JS_PATH_OVERRIDES) | n, decode.utf8} diff --git a/lms/templates/student_account/account_settings.html b/lms/templates/student_account/account_settings.html index 83bb822327b3..9a1446d93c83 100644 --- a/lms/templates/student_account/account_settings.html +++ b/lms/templates/student_account/account_settings.html @@ -77,7 +77,7 @@ + + // Check if resource is a string (a URL) + if (typeof resource === 'string') { + return replaceUrl(resource); + } else if (resource instanceof Request) { + // If resource is a Request object, create a new Request with a replaced URL + const newUrl = replaceUrl(resource.url); + return new Request(newUrl, { + method: resource.method, + headers: resource.headers, + body: resource.body, + mode: resource.mode, + credentials: resource.credentials, + cache: resource.cache, + redirect: resource.redirect, + referrer: resource.referrer, + integrity: resource.integrity, + keepalive: resource.keepalive, + signal: resource.signal + }); + } else if (resource instanceof URL) { + // If resource is a URL object, convert it to a string and replace the URL + return replaceUrl(resource.href); + } else { + // If it's neither a string nor a Request object, log a warning or handle as needed + console.warn('replaceFetchResourceForSegmentSite was called with an unexpected argument type:', typeof resource, resource); + return resource; + } + } + // Override the fetch function to use the replaceFetchResourceForSegmentSite function + const originalFetch = window.fetch; + window.fetch = async (...args) => { + args[0] = replaceFetchResourceForSegmentSite(args[0]); + return originalFetch.apply(window, args); + }; + }(); + ## Appsembler: end Segment Site ## end Copy % endif diff --git a/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py b/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py index 0c7061d02288..4bc844c62bef 100644 --- a/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py +++ b/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py @@ -16,6 +16,8 @@ import ddt import mock +from six import text_type +from opaque_keys.edx.keys import CourseKey from tahoe_sites.api import update_admin_role_in_organization from openedx.core.djangoapps.waffle_utils.testutils import override_waffle_flag @@ -41,7 +43,7 @@ OrganizationFactory, OrganizationCourseFactory, ) - +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview APPSEMBLER_API_VIEWS_MODULE = 'openedx.core.djangoapps.appsembler.api.v1.views' @@ -382,6 +384,73 @@ def test_enroll_learner_by_username(self): assert 'invalidIdentifier' not in response.content.decode(), message assert CourseEnrollment.is_enrolled(registered_user, co.id), 'Enrollment is successful by username' + def test_enrollment_with_bad_course_id(self): + """ + Sometimes, the API receives the course_key in the wrong letters-case + For example, (course-v1:org+name+number_number) instead of (course-v1:org+Name+Number_number) + + This will still be evaluated correctly when we use (CourseOverview.get_from_id), but we must also ensure that + the API is creating the enrollment with the correct string. Because (CourseKey.from_string) cannot fix + wrong IDs saved in (CourseEnrollment.course_id) + + The test is a bit complicated because django with SQLite cannot filter case-insensitive. Therefore, we have + to do some tricks with mocks to mimic MySQL behavior + """ + # Prepare course for testing + course = CourseFactory.create() + course_overview = CourseOverviewFactory(id=course.id) + OrganizationCourseFactory(organization=self.my_site_org, course_id=text_type(course.id)) + lowercase_key = CourseKey.from_string(text_type(course.id).lower()) + registered_user = UserFactory() + create_organization_mapping(user=registered_user, organization=self.my_site_org) + payload = { + 'action': 'enroll', + 'auto_enroll': True, + 'identifiers': [registered_user.username], + 'email_learners': True, + 'courses': [lowercase_key], + } + + # Double check that course.id and lowercase_key are not identical + self.assertNotEqual(course.id, lowercase_key) + self.assertEqual(text_type(course.id).lower(), text_type(lowercase_key).lower()) + + # CourseOverview.get_from_id will find the course even if the given key has the wrong letters-case + # because MySQL can do that. But in SQLite tests, it will fail + self.assertEqual(CourseOverview.get_from_id(course.id), course_overview) + with self.assertRaises(CourseOverview.DoesNotExist): + # This will fail because of SQLite limitations https://www.sqlite.org/faq.html#q18 + CourseOverview.get_from_id(lowercase_key) + + # As a workaround, we can force (get_from_id) to return the course by mocking (load_from_module_store) + with mock.patch( + 'openedx.core.djangoapps.content.course_overviews.models.CourseOverview.load_from_module_store', + return_value=course_overview, + ): + self.assertEqual(CourseOverview.get_from_id(lowercase_key), course_overview) + + # For the same reasons described about (get_from_id), we must mock (get_site_for_course) and (get_course_by_id) + with mock.patch( + 'openedx.core.djangoapps.content.course_overviews.models.CourseOverview.load_from_module_store', + return_value=course_overview, + ): + with mock.patch( + 'openedx.core.djangoapps.appsembler.api.sites.get_site_for_course', + return_value=self.my_site + ): + with mock.patch( + 'openedx.core.djangoapps.appsembler.api.v1.views.get_course_by_id', + return_value=course + ): + response = self.call_enrollment_api('post', self.my_site, self.caller, {'data': payload}) + assert response.status_code == status.HTTP_201_CREATED, response.content + assert 'invalidIdentifier' not in response.content.decode() + + # Finally, use SQLite limitation to verify that enrollment was saved using the correct letters case regardless + # of the fact that we sent a wrong one to the API + assert CourseEnrollment.objects.filter(course_id=course.id).count() == 1 + assert CourseEnrollment.objects.filter(course_id=lowercase_key).count() == 0 + @ddt.ddt @mock.patch(APPSEMBLER_API_VIEWS_MODULE + '.EnrollmentViewSet.throttle_classes', []) diff --git a/openedx/core/djangoapps/appsembler/api/v1/views.py b/openedx/core/djangoapps/appsembler/api/v1/views.py index f39f9e76d39b..edb7371789a3 100644 --- a/openedx/core/djangoapps/appsembler/api/v1/views.py +++ b/openedx/core/djangoapps/appsembler/api/v1/views.py @@ -345,7 +345,15 @@ def create(self, request, *args, **kwargs): results = [] for course_id in serializer.data.get('courses'): - course_key = as_course_key(course_id) + try: + # To avoid running into MongoDB vs. MySQL case sensitivity issues, and to avoid having + # CourseEnrollment.course_id returning the ID with the wrong letters case; we convert + # the key to the correct letter case one. RED-3540 + course_key = CourseOverview.get_from_id(course_id).id + except CourseOverview.DoesNotExist: + # We allow enrollments to non-existence courses! + course_key = as_course_key(course_id) + # TODO: The two checks below deserve a refactor to make it clearer or a v2 API that works on a # single course and use `instructor/views/api.py:students_update_enrollment` directly. # Ensuring the course is linked to an organization. It's somewhat a legacy code, keeping diff --git a/openedx/core/djangoapps/appsembler/auth/course_roles.py b/openedx/core/djangoapps/appsembler/auth/course_roles.py deleted file mode 100644 index 01202df93356..000000000000 --- a/openedx/core/djangoapps/appsembler/auth/course_roles.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Tahoe Authentication helpers for managing course related roles. -""" - -from common.djangoapps.student.roles import CourseCreatorRole, OrgStaffRole - - -def update_organization_staff_roles(user, organization_short_name, set_as_organization_staff=False): - """ - Update the organization-wide OrgStaffRole/CourseCreatorRole for using Studio and instructor dashboards. - """ - assert user, 'Parameter `user` is required.' - assert organization_short_name, 'Parameter `organization_short_name` is required.' - - organization_role = OrgStaffRole(organization_short_name) - creator_role = CourseCreatorRole() - - if set_as_organization_staff: - organization_role.add_users(user) - creator_role.add_users(user) - else: - organization_role.remove_users(user) - creator_role.remove_users(user) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py index c6e99723bcf7..adf8efe33663 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py @@ -7,15 +7,36 @@ So, don't add imports to this that will fail before Django has fully loaded. """ +import inspect import os import sys -def is_not_lms(): - """Utility function: return False if not running in the LMS.""" - return os.getenv("SERVICE_VARIANT") != 'lms' +def is_lms(): + """Utility function: return True if running in the LMS. And not a test.""" + return os.getenv("SERVICE_VARIANT") == 'lms' + + +def is_test(): + return 'pytest ' in ' '.join(sys.argv) + + +def is_self_test(): + """ + Utility function: return True if this is in an LMS test from within the + openedx.core.djangoapps.appsembler.eventtracking.test_tahoeusermetadata module. + + It's ugly but needed to only run in its own tests, to keep SQL query counts as expected + in other tests. + """ + callstack = inspect.stack() + stack_filenames = [fi.filename for fi in callstack] + is_own_package_test = any( + ['appsembler/eventtracking/tests/' in fi for fi in stack_filenames] + ) + return is_own_package_test def is_not_runserver(): - """Utility function: return False if not runserver command.""" + """Utility function: return True if not a runserver command.""" return 'runserver' not in sys.argv diff --git a/openedx/core/djangoapps/appsembler/eventtracking/apps.py b/openedx/core/djangoapps/appsembler/eventtracking/apps.py index 23a2889b3d32..2250e639b214 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/apps.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/apps.py @@ -63,10 +63,11 @@ def ready(self): # only want to prefill the cache on lms runserver... if ( app_variant.is_not_runserver() or - app_variant.is_not_lms() or + app_variant.is_lms() or is_celery_worker() ): logger.debug("Not initializing metadatacache. This is Studio, Celery, other command.") return else: - tahoeusermetadata.prefetch_tahoe_usermetadata_cache.delay(metadatacache) + pass + # tahoeusermetadata.prefetch_tahoe_usermetadata_cache.delay(metadatacache) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 9bdf5bff93db..1e04453f06cc 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -8,11 +8,10 @@ import logging from celery import task -from crum import get_current_user from django.core.cache import caches from django.core.cache.backends.base import InvalidCacheBackendError -from . import app_variant +from . import app_variant, utils logger = logging.getLogger(__name__) @@ -119,6 +118,21 @@ def _get_reg_metadata_from_cache(self, user_id): else: return {} + def _get_idp_metadata_from_tpa_pipeline(self): + """Check ThirdPartyAuth pipeline for details containing IdP metadata.""" + import crum + from third_party_auth import pipeline + try: + request = crum.get_current_request() + tpa_running = pipeline.running(request) + if not tpa_running: + return None + tpa = pipeline.get(request) + details = tpa['kwargs'].get('details') + return details.get('tahoe_idp_metadata', {}) + except: + return None + def _get_custom_registration_metadata(self, user_id): """ Get any custom registration field data for the User. @@ -137,10 +151,18 @@ def _get_custom_registration_metadata(self, user_id): try: profile = UserProfile.objects.get(user__id=user_id) except UserProfile.DoesNotExist: - logger.info("User {user_id} has no UserProfile".format(user_id=user_id)) return {} - meta = profile.get_meta() - idp_metadata = meta.get("tahoe_idp_metadata", {}) + else: + meta = profile.get_meta() + idp_metadata = meta.get("tahoe_idp_metadata", {}) + if not idp_metadata: + logger.info("User {user_id} has no IDP metadata yet".format(user_id=user_id)) + # We could be processing an event (e.g., course enrollment) on very first User.save() + # This can happen before UserProfile has been updated via tahoe_idp TPA step. + # Try getting it direclty from TPA pipeline. + idp_metadata = self._get_idp_metadata_from_tpa_pipeline() + if not idp_metadata: + return {} custom_reg_data = idp_metadata.get("registration_additional") userprofile_metadata_cache.set_by_user_id(user_id, idp_metadata) return custom_reg_data @@ -166,21 +188,40 @@ def __call__(self, event): # WARNING: # We have to be careful to not add SQL queries that would require updating upstream tests # which count SQL queries; e.g., `cms.djangoapps.contentstore.views.tests.test_course_index) - # currently we can do this by only enabling the event processor for LMS - if app_variant.is_not_lms(): # we don't care about user metadata for Studio, at this point - return event + # We should not let this run in any CMS tests and any LMS tests other than from + # within openedx/core/djangoapps/eventtracking/ Ugh. + # We don't care about user metadata for Studio, at this point. + # Allow to run in LMS or it's own LMS env tests. + if not app_variant.is_lms(): # this returns False if a test in LMS + if app_variant.is_test(): + if not app_variant.is_self_test(): # expensive, make sure it's a test first. + return event + else: + return event + + # eventtracking Processors are loaded before apps are ready + from django.contrib.auth.models import User + + # Don't try to get the user from the request: it could be an instructor doing + # a bulk enrollment or exception certificate triggering the event. Only use the + # user from event itself. - user = get_current_user() - if not user or not user.pk: - # should be an AnonymousUser or in tests + try: + user_id = utils.get_user_id_from_event(event) + except AttributeError: + logger.debug( + "TahoeUserMetadataProcessor passed invalid type to " + "get_user_id_from_event: {}. Likely innocuous. " + "Logging and continuing.".format(event) + ) + else: + if user_id: + # Add any Tahoe metadata context + tahoe_user_metadata = self._get_user_tahoe_metadata(user_id) + if tahoe_user_metadata: + event['context']['tahoe_user_metadata'] = tahoe_user_metadata + finally: return event - # Add any Tahoe metadata context - tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) - if tahoe_user_metadata: - event['context']['tahoe_user_metadata'] = tahoe_user_metadata - - return event - userprofile_metadata_cache = TahoeUserProfileMetadataCache() diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py index 92ee2c877c89..af3a178c8caf 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py @@ -1,6 +1,8 @@ """Test the appsembler.eventtracking.tahoeusermetadata module.""" +from copy import deepcopy import factory +import json from mock import MagicMock, patch import pytest @@ -17,20 +19,32 @@ "name": "event_name", "time": "2022-08-29T15:42:50.636766+00:00", "context": {}, + "event": {}, "data": {} } +TAHOE_USER_METADATA_CONTEXT = { + "tahoe_user_metadata": { + "registration_extra": {"custom_reg_field": "value1"} + } +} + class UserProfileWithMetadataFactory(UserProfileFactory): """Factory for UserProfile sequence with some tahoe_user_metadata.""" - # TODO: a Sequence is a bit of a silly way to set things up. - meta = factory.Sequence(lambda n: { - "tahoe_user_metadata": { - "some_other_key": "some_other_val", - "registration_extra": {"custom_reg_field": "value{n}"} - } - } if n == 1 else {"tahoe_user_metadata": {}} - ) + + def _meta_val(n): + """Return a JSON meta value for Sequence member""" + reg_field_value = "value{}".format(n % 2) + meta_dict = { + "tahoe_idp_metadata": { + "registration_additional": {"custom_reg_field": reg_field_value} + } + } if n % 2 == 1 else {"tahoe_idp_metadata": {}} + + return json.dumps(meta_dict) + + meta = factory.Sequence(_meta_val) class UserWithTahoeMetadataFactory(UserFactory): @@ -39,6 +53,7 @@ class UserWithTahoeMetadataFactory(UserFactory): @pytest.fixture(autouse=True) def users(): + UserProfileFactory.reset_sequence(0) return [UserWithTahoeMetadataFactory() for i in range(2)] @@ -55,22 +70,19 @@ def processor(): @pytest.mark.django_db def test_for_metadata_no_cache(users, base_event, processor): """Test happy path, Processor returns the event with user metadata in `context`.""" - with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = users[0] - base_event.update(context={ - "tahoe_user_metadata": { - "some_other_key": "some_other_val", - "registration_extra": {"custom_reg_field": "value0"} - } - }) + event_with_metadata = deepcopy(base_event) + event_with_metadata.update(context=TAHOE_USER_METADATA_CONTEXT) + + with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.utils.get_user_id_from_event', MagicMock()) as mocked: + mocked.return_value = users[1].id event = processor(base_event) - assert event == base_event + assert event == event_with_metadata @pytest.mark.django_db def test_no_context_added_if_no_metadata_of_interest(users, base_event, processor): """Test happy path, Processor returns the event with user metadata in `context`.""" - with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = users[1] + with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.utils.get_user_id_from_event', MagicMock()) as mocked: + mocked.return_value = users[0].id event = processor(base_event) assert event == base_event diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py index e78cc52bae2c..d9db5943b82a 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py @@ -9,7 +9,7 @@ EventProcessingError, ) from openedx.core.djangoapps.appsembler.eventtracking.utils import ( - get_site_config_for_event, + get_site_config_for_event, get_user_id_from_event ) from openedx.core.djangoapps.site_configuration.tests.factories import ( @@ -102,3 +102,55 @@ def test_event_raises_exception_on_no_course_id_found(caplog): with pytest.raises(EventProcessingError): get_site_config_for_event(dict(course_id='no-course-id')) assert 'get_site_config_for_event: Cannot get site config for event' in caplog.text, 'Should log the exception' + + +TEST_EVENT_FOR_USER_IDS_ONE = { + "user_id": None, + "context": { + "course_id": "course-v1:org+course+run", + "path": "/user_api/v1/account/registration/", + "user_id": 1, # for example an Instructor + "org_id": "org" + }, + "event_type": "edx.course.enrollment.activated", + "username": "", + "host": "host.tld", + "event": { + "course_id": "course-v1:org+course+run", + "user_id": 2, + "context": { + "user_id": 3 + } + }, + "referer": "https://host.tld/register" +} + +TEST_EVENT_FOR_USER_IDS_TWO = { + "user_id": 1, + "context": { + "course_id": "course-v1:org+course+run", + "path": "/user_api/v1/account/registration/", + "user_id": 3, # for example an Instructor + "org_id": "org" + }, + "event_type": "edx.course.enrollment.activated", + "username": "", + "host": "host.tld", + "event": { + "course_id": "course-v1:org+course+run", + "user_id": "", # not sure if this would ever occur, but let's test + }, + "referer": "https://host.tld/register" +} + +TEST_EVENTS_FOR_USER_IDS = [TEST_EVENT_FOR_USER_IDS_ONE, TEST_EVENT_FOR_USER_IDS_TWO] + + +@pytest.mark.parametrize('event', TEST_EVENTS_FOR_USER_IDS) +def test_get_user_id_from_event(event): + """ + Test getting user_id from event properties. + In some cases a user_id may be in context, in others in event.context, or event.context.event. + """ + # 3 is the id of the deepest valid user_id + assert get_user_id_from_event(event) == 3 diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 684c66a66149..e13441eabf0a 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -16,6 +16,8 @@ `get_site_config_for_event` is specific to sites. """ + +from collections.abc import MutableMapping import logging from django.core.exceptions import MultipleObjectsReturned @@ -76,3 +78,44 @@ def get_site_config_for_event(event_props): log.exception('get_site_config_for_event: Cannot get site config for event. props=`%s`', repr(event_props)) raise EventProcessingError(e) return site_configuration + + +def get_user_id_from_event(event_props): + """ + Get a user id from event properties, preferring deepest-nested user_id value. + + Use in favor of trying to get the user_id from the request (django_crum-based). + Needed for all events emitted without a request, e.g., an event emitted + by a Celery worker, e.g., `edx.bi.completion.*` or `.grade_calculated` events. + Some events are also emitted with a user in the request which is an instructor or + other initiating user that is not the actual user tied to the event itself. + """ + + user_id = None + + # ... typically the most interior object will have a good user_id + # search event props to find the deepest valid user_id :\ + + def _flatten_dict(d, parent_key='', sep='.'): + def _flatten_dict_gen(d, parent_key, sep): + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + yield from _flatten_dict(v, new_key, sep=sep).items() + else: + yield new_key, v + + return dict(_flatten_dict_gen(d, parent_key, sep)) + + user_id_props = { + key: int(val) for (key, val) in _flatten_dict(event_props).items() + if 'user_id' in key and val is not None and bool(val) and int(val) + } + deepest_user_id_prop = sorted(user_id_props.keys(), key=lambda x: x.count('.'), reverse=True) + prefer_event_over_context = sorted(deepest_user_id_prop, key=lambda x: 'event' in x, reverse=True) + try: + best_user_id_prop = prefer_event_over_context[0] + user_id = user_id_props[best_user_id_prop] + except IndexError: + pass + return user_id diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py index c9bc31fe9710..2527f9587aca 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py @@ -1,3 +1,9 @@ +""" +Tests for the LMS part of the deprecated AMC trial signup. +""" +# TODO: RED-2845 Remove after migrating to Tahoe 2.0 + +import unittest import json from mock import patch, Mock import uuid @@ -8,6 +14,7 @@ from rest_framework.test import APITestCase from tahoe_sites.api import get_organization_for_user, is_active_admin_on_organization +from tahoe_sites.zd_helpers import should_site_use_org_models from .test_utils import lms_multi_tenant_test, with_organization_context @@ -98,6 +105,7 @@ def register_new_amc_admin(self, color, email): assert site_response.status_code == status.HTTP_201_CREATED, '{}: {}'.format(color, site_response.content) return user_response, site_response + @unittest.skipUnless(should_site_use_org_models(), 'RED-2845 Remove with AMC') def test_new_admin_with_learner(self, mock_add_creator): """ Test happy scenario regardless of APPSEMBLER_MULTI_TENANT_EMAILS. @@ -110,6 +118,7 @@ def test_new_admin_with_learner(self, mock_add_creator): with with_organization_context(site_color=red_site): self.register_learner('learner@example.com', 'learner') + @unittest.skipUnless(should_site_use_org_models(), 'RED-2845 Remove with AMC') def test_learner_registers_for_trial(self, mock_add_creator): """ Test learner registers for a new Tahoe trial signup when APPSEMBLER_MULTI_TENANT_EMAILS is enabled. diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py index d16511648486..6c977e5eaff8 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py @@ -12,6 +12,8 @@ create_org_user, with_organization_context, ) +from student.tests.factories import CourseEnrollmentFactory + User = get_user_model() @@ -21,7 +23,7 @@ def test_enroll_by_email_single_tenant(settings): """ Ensure `enroll_by_email` works as upstream intended if APPSEMBLER_MULTI_TENANT_EMAILS is disabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': False} + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': False} course = CourseOverviewFactory.create() course_key = course.id @@ -43,7 +45,7 @@ def test_enroll_by_email_multi_tenant(settings): """ Ensure `enroll_by_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': True} + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': True} course = CourseOverviewFactory.create() course_key = course.id @@ -58,3 +60,27 @@ def test_enroll_by_email_multi_tenant(settings): assert not CourseEnrollment.enroll_by_email(blue_user.email, course_key), 'Should not enroll in other sites' assert not CourseEnrollment.is_enrolled(blue_user, course_key), 'Should not enroll in other sites' + + +@pytest.mark.django_db +def test_unenroll_by_email_multi_tenant(settings): + """ + Ensure `unenroll_by_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. + """ + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': True} + course = CourseOverviewFactory.create() + course_key = course.id + + with with_organization_context(site_color='blue1') as blue_org: + blue_user = create_org_user(blue_org) + CourseEnrollmentFactory(user=blue_user, course_id=course_key) + CourseEnrollment.enroll_by_email(blue_user.email, course_key) + + with with_organization_context(site_color='red1') as red_org: + red_user = create_org_user(red_org) + CourseEnrollmentFactory(user=red_user, course_id=course_key) + CourseEnrollment.unenroll_by_email(red_user.email, course_key) + assert not CourseEnrollment.is_enrolled(red_user, course_key), 'Should unenroll in same sites' + + CourseEnrollment.unenroll_by_email(blue_user.email, course_key) + assert CourseEnrollment.is_enrolled(blue_user, course_key), 'Should not unenroll in other sites' diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py index c3487c746790..12fe7f5ff417 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py @@ -20,7 +20,7 @@ def test_get_user_by_username_or_email_single_tenant(settings): """ Ensure `get_user_by_username_or_email` works as upstream intended if APPSEMBLER_MULTI_TENANT_EMAILS is disabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': False} + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': False} with with_organization_context(site_color='blue1') as blue_org: blue_user = create_org_user(blue_org) @@ -44,7 +44,7 @@ def test_get_user_by_username_or_email_multi_tenant(settings): """ Ensure `get_user_by_username_or_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': True} + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': True} with with_organization_context(site_color='blue1') as blue_org: blue_user = create_org_user(blue_org) diff --git a/openedx/core/djangoapps/appsembler/settings/settings/common.py b/openedx/core/djangoapps/appsembler/settings/settings/common.py index b3b06fe77268..1eb9a34bad88 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/common.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/common.py @@ -89,3 +89,6 @@ def plugin_settings(settings): # Off by default. See the `site_configuration.tahoe_organization_helpers.py` module. settings.FEATURES['TAHOE_SITE_CONFIG_CLIENT_ORGANIZATIONS_SUPPORT'] = False + + # Give a little more time for YT API to load in video_module (default was 1500 ms) + settings.YOUTUBE['TEST_TIMEOUT'] = 2500 diff --git a/openedx/core/djangoapps/appsembler/settings/settings/production_common.py b/openedx/core/djangoapps/appsembler/settings/settings/production_common.py index 20f544f0dc83..bdd911e8a286 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/production_common.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/production_common.py @@ -48,7 +48,22 @@ def plugin_settings(settings): "MANDRILL_API_KEY": settings.MANDRILL_API_KEY, } settings.INSTALLED_APPS += ['anymail'] - + # Mandrill Subaccount Support + settings.MANDRILL_SUBACCOUNT = settings.ENV_TOKENS.get("MANDRILL_SUBACCOUNT") + if settings.MANDRILL_SUBACCOUNT: + subaccount_settings = { + "MANDRILL_SEND_DEFAULTS": { + "esp_extra": { + "message": { + "subaccount": settings.MANDRILL_SUBACCOUNT + } + } + } + } + if settings.ANYMAIL: + settings.ANYMAIL.update(subaccount_settings) + else: + settings.ANYMAIL = subaccount_settings # Sentry settings.SENTRY_DSN = settings.AUTH_TOKENS.get('SENTRY_DSN', False) if settings.SENTRY_DSN: diff --git a/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py b/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py index 26b2ea10f272..a6bb8dbc01db 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py @@ -45,6 +45,7 @@ def plugin_settings(settings): # from the redirect mechanics. settings.MAIN_SITE_REDIRECT_ALLOWLIST = [ '/api/', + '/user_api/', # still used by EdxRestAPIClient in integrations '/admin', 'oauth', # TODO: Add slashes during Nutmeg upgrade since this requires a lot of QA 'status', # TODO: Add slashes during Nutmeg upgrade since this requires a lot of QA @@ -63,6 +64,13 @@ def plugin_settings(settings): tpa_admin_app_name, ] + settings.INSTALLED_APPS += [ + 'user_tasks', # Release upgrade note: This line can be removed if it causes errors, + # but the `remove_site` must be tested afterwards + # `user_tasks` is a CMS-only app, but adding it in LMS to fix an error with `remove_site` command + # `user_tasks` helps to manage of user-triggered async tasks (course import/export, etc.) + ] + settings.CORS_ORIGIN_ALLOW_ALL = True settings.CORS_ALLOW_HEADERS = ( diff --git a/openedx/core/djangoapps/appsembler/settings/settings/test_common.py b/openedx/core/djangoapps/appsembler/settings/settings/test_common.py index 1be1ce7b5fb6..b4fa3d749efe 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/test_common.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/test_common.py @@ -26,6 +26,14 @@ def plugin_settings(settings): settings.TAHOE_ALWAYS_SKIP_TEST = True settings.CMS_UPDATE_SEARCH_INDEX_JOB_QUEUE = 'edx.cms.core.default' + # TODO: Remove when AMC is removed: RED-2845 + settings.FEATURES['TAHOE_SITES_USE_ORGS_MODELS'] = getenv('TEST_TAHOE_SITES_USE_ORGS_MODELS', 'true') == 'true' + + if getenv('TEST_ENABLE_TIERS_APP', 'false') == 'true': + settings.INSTALLED_APPS += [ + 'tiers', + ] + if settings.FEATURES.get('APPSEMBLER_MULTI_TENANT_EMAILS', False): settings.INSTALLED_APPS += [ 'openedx.core.djangoapps.appsembler.multi_tenant_emails', diff --git a/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py b/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py index aec9fbd148d3..e8f136bb8201 100644 --- a/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py +++ b/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py @@ -23,6 +23,7 @@ def fake_production_settings(settings): settings.AUTH_TOKENS = {} settings.CELERY_QUEUES = {} settings.ALTERNATE_QUEUE_ENVS = [] + settings.INSTALLED_APPS = settings.INSTALLED_APPS.copy() # Prevent polluting the original list settings.FEATURES = settings.FEATURES.copy() # Prevent polluting other tests. settings.ENV_TOKENS = { 'LMS_BASE': 'fake-lms-base', diff --git a/openedx/core/djangoapps/appsembler/sites/api.py b/openedx/core/djangoapps/appsembler/sites/api.py index 0c46491c4a29..f64f96d49e35 100644 --- a/openedx/core/djangoapps/appsembler/sites/api.py +++ b/openedx/core/djangoapps/appsembler/sites/api.py @@ -30,11 +30,11 @@ RegistrationSerializer, AlternativeDomainSerializer, ) -from openedx.core.djangoapps.appsembler.sites.utils import ( - delete_site, +from .utils import ( get_customer_files_storage, to_safe_file_name, ) +from .deletion_utils import delete_site log = logging.Logger(__name__) diff --git a/openedx/core/djangoapps/appsembler/sites/deletion_utils.py b/openedx/core/djangoapps/appsembler/sites/deletion_utils.py new file mode 100644 index 000000000000..e7402c1ba6b0 --- /dev/null +++ b/openedx/core/djangoapps/appsembler/sites/deletion_utils.py @@ -0,0 +1,184 @@ +""" +Site and courses deletion utils. +""" + +import beeline + +from django.apps import apps +from django.core.management import CommandError +from django.db import transaction + +import tahoe_sites.api +from organizations.models import OrganizationCourse + + +from opaque_keys.edx.django.models import CourseKeyField, LearningContextKeyField + +from common.djangoapps.util.organizations_helpers import get_organization_courses + + +from ...content.course_overviews.models import CourseOverview +from organizations.api import get_organization_courses + + +def confirm_deletion(commit, question): + """ + Utility for yes/no interactive confirmation if `commit` is `None`. + """ + if commit is None: + result = input('%s [type yes or no] ' % question) + while not result or result.lower() not in ['yes', 'no']: + result = input('Please answer yes or no: ') + return result == 'yes' + return commit + + +def remove_course_creator_role(users): + """ + Remove course creator role to fix `delete_site` issue. + + This will fail in when running tests from within the LMS because the CMS migrations + don't run during tests. Patch this function to avoid such errors. + TODO: RED-2853 Remove this helper when AMC is removed + This helper is being replaced by `update_course_creator_role_for_cms` which has unit tests. + """ + from cms.djangoapps.course_creators.models import CourseCreator # Fix LMS->CMS imports. + from student.roles import CourseAccessRole # Avoid circular import. + CourseCreator.objects.filter(user__in=users).delete() + CourseAccessRole.objects.filter(user__in=users).delete() + + +@beeline.traced(name="delete_site") +def delete_site(site): + """ + Delete site with all related objects except for MongoDB course files. + """ + from third_party_auth.models import SAMLConfiguration # local import to avoid import-time errors + + print('Deleting SiteConfiguration of', site) + site.configuration.delete() + + print('Deleting theme of', site) + site.themes.all().delete() + + organization = tahoe_sites.api.get_organization_by_site(site) + + print('Deleting users of', site) + users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) + remove_course_creator_role(users) + + # Prepare removing users by avoiding on_delete=models.PROTECT error + # SAMLConfiguration will be deleted with `site.delete()` + SAMLConfiguration.objects.filter(changed_by__in=users).update(changed_by=None) + + users.delete() + + print('Deleting courses of', site) + delete_organization_courses(organization) + + print('Deleting organization', organization) + organization.delete() + + print('Deleting site', site) + site.delete() + + +def get_models_using_course_key(): + """ + Get all course related model classes. + """ + course_key_field_names = { + 'course_key', + 'course_id', + } + + models_with_course_key = { + (CourseOverview, 'id'), # The CourseKeyField with a `id` name. Hard-coding it for simplicity. + (OrganizationCourse, 'course_id'), # course_id is CharField + } + + model_classes = apps.get_models() + for model_class in model_classes: + for field_name in course_key_field_names: + field_object = getattr(model_class, field_name, None) + if field_object: + field_definition = getattr(field_object, 'field', None) + if field_definition and isinstance(field_definition, (CourseKeyField, LearningContextKeyField)): + models_with_course_key.add( + (model_class, field_name,) + ) + + return models_with_course_key + + +def delete_organization_courses(organization): + """ + Delete all course related model instances. + """ + course_keys = [] + + for course in get_organization_courses({'id': organization.id}): + course_keys.append(course['course_id']) + + delete_related_models_of_courses(course_keys) + + +def delete_related_models_of_courses(course_keys): + model_classes = get_models_using_course_key() + + print('Deleting course related models:', ', '.join([ + '{model}.{field}'.format(model=model_class.__name__, field=field_name) + for model_class, field_name in model_classes + ])) + + for model_class, field_name in model_classes: + objects_to_delete = model_class.objects.filter(**{ + '{field_name}__in'.format(field_name=field_name): course_keys, + }) + objects_to_delete.delete() + + +def get_courses_keys_without_organization_linked(limit=None, only_active_links=True): + """ + Get keys of stray courses. + """ + course_links = OrganizationCourse.objects.all() + if only_active_links: + course_links = course_links.filter(active=True) + + queryset = CourseOverview.objects.exclude( + id__in=course_links.values_list('course_id', flat=True), + ) + + course_keys = queryset.values_list( + 'id', flat=True + ) + + course_keys_list = [str(course_key) for course_key in course_keys] + if limit: + course_keys_list = course_keys_list[:limit] + + return course_keys_list + + +def remove_stray_courses_from_mysql(limit, commit=None, print_func=print): + """ + Removes courses without linked organization from LMS MySQL database. + + The MongoDB courses won't be removed with this command. + """ + course_keys = get_courses_keys_without_organization_linked(limit=limit) + if not course_keys: + raise CommandError('No courses to delete.') + + print_func('Preparing to delete:') + print_func('\n'.join(course_keys)) + + commit = confirm_deletion(commit=commit, question='Do you confirm to delete those courses from the LMS?') + + with transaction.atomic(): + delete_related_models_of_courses(course_keys) + print_func('Finished [commit={}] courses.'.format(commit)) + + if not commit: + transaction.set_rollback(True) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py b/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py index 75c338454da9..6e20cc7d7cf7 100644 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py @@ -1,7 +1,8 @@ -import hashlib import inspect import json -import uuid + +from openedx.core.djangoapps.appsembler.sites.serializers_v2 import TahoeSiteCreationSerializer +from tahoe_sites.api import add_user_to_organization from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User @@ -10,16 +11,11 @@ from django.db import transaction from openedx.core.djangoapps.appsembler.sites.serializers import RegistrationSerializer -from openedx.core.djangoapps.appsembler.sites.utils import reset_amc_tokens -from student.models import UserProfile -from student.roles import CourseCreatorRole class Command(BaseCommand): """ - Create the demo something.localhost:18000 site for devstack. - - Needs the corresponding `create_devstack_site` AMC command to be run as well. + Create a Tahoe 2.0 demo something.localhost:18000 site for devstack. """ def add_arguments(self, parser): @@ -46,17 +42,14 @@ def congrats(self, **kwargs): """ Congrats, Your site is ready! - Username: "{name}" - Email: "{email}" - Password: "{password}" - Site URL: "http://{site}/" Please add the following entry to your /etc/hosts file: 127.0.0.1 {domain} - Remember to run the corresponding AMC command. + You can login via FusionAuth via a Learner or an Administrator depending + on the user.data.platform_role you chose. Enjoy! """.format(**kwargs) @@ -83,52 +76,32 @@ def _handle_with_atmoic(self, *args, **options): domain = '{name}.{base_domain}'.format(name=name, base_domain=base_domain) site_name = '{domain}:18000'.format(domain=domain) - user = User.objects.create_user( - username=name, - email='{}@example.com'.format(name), - password=name, - ) - CourseCreatorRole().add_users(user) - UserProfile.objects.create(user=user, name=name) - - # Calculated access tokens to the AMC devstack can have them without needing to communicate with the LMS. - # Just making it easier to automate this without having cross-dependency in devstack - fake_token = hashlib.md5(user.username.encode('utf-8')).hexdigest() - reset_amc_tokens(user, access_token=fake_token, refresh_token=fake_token) - - data = { - 'site': { - 'domain': site_name, - 'name': site_name, - }, - 'username': user.username, - 'organization': { - 'name': name, - 'short_name': name, - 'edx_uuid': uuid.uuid4(), # TODO: RED-2845 Remove this line when AMC is migrated - }, - 'initial_values': { - 'SITE_NAME': site_name, - 'platform_name': '{} Academy'.format(name), - 'logo_positive': None, - 'logo_negative': None, - 'font': 'Roboto', - 'accent-font': 'Delius Unicase', - 'primary_brand_color': '#F00', - 'base_text_color': '#000', - 'cta_button_bg': '#00F', - } - } - serializer = RegistrationSerializer(data=data) + serializer = TahoeSiteCreationSerializer(data={ + 'short_name': name, + 'domain': site_name, + }) if not serializer.is_valid(): raise CommandError('Something went wrong with the process: \n{errors}'.format( errors=json.dumps(serializer.errors, indent=4) )) - serializer.save() + site_data = serializer.save() + + # This admin cannot login without FusionAuth, but it's added for simulation purposes + # in testing. + fake_admin_user = User.objects.create_user( + username=name, + email='{}@example.com'.format(name), + password=name, + ) + add_user_to_organization( + user=fake_admin_user, + organization=site_data['organization'], + is_admin=True, + ) self.congrats( - name=user.username, - email=user.email, + name=fake_admin_user.username, + email=fake_admin_user.email, password=name, site=site_name, domain=domain, diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py b/openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py deleted file mode 100644 index f39c5a2b0f87..000000000000 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py +++ /dev/null @@ -1,57 +0,0 @@ -from django.core.management import BaseCommand, CommandError - -from openedx.core.djangoapps.appsembler.sites.utils import get_active_sites - - -class Command(BaseCommand): - help = "DANGEROUS: Renames all domains for production/staging candidate. Remove Google Tag Manager and Segment keys. Do not run during on production!" - - def add_arguments(self, parser): - parser.add_argument( - 'from', - help='The production/staging domain e.g. "tahoe.appsembler.com"', - type=str, - ) - - parser.add_argument( - 'to', - help='The candidate domain e.g. "tahoe-us-juniper-prod.appsembler.com"', - type=str, - ) - - def handle(self, *args, **options): - sites_with_configs = get_active_sites().filter(configuration__isnull=False) - - has_errors = False - for site in sites_with_configs: - self.stdout.write('FROM {}'.format(site.domain)) - site.domain = site.domain.replace('.{}'.format(options['from']), '.{}'.format(options['to'])) - self.stdout.write('TO {}'.format(site.domain)) - site.save() - - site.configuration.site_values['SITE_NAME'] = site.domain - - try: - del site.configuration.site_values['SEGMENT_KEY'] - self.stdout.write('deleted SEGMENT_KEY') - except KeyError: - self.stdout.write('no SEGMENT_KEY') - pass - - try: - del site.configuration.site_values['customer_gtm_id'] - self.stdout.write('deleted customer_gtm_id') - except KeyError: - self.stdout.write('no customer_gtm_id') - pass - try: - site.configuration.save() - except Exception as e: - has_errors = True - self.stdout.write(e) - self.stdout.write('---') - - if has_errors: - msg = 'Some sites have failed, please review this command output for more information.' - self.stdout.write(msg) - raise CommandError(msg) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py b/openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py new file mode 100644 index 000000000000..5ac0b1f721ad --- /dev/null +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py @@ -0,0 +1,48 @@ +""" +Remove stray courses from LMS. +""" + +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +from ...deletion_utils import remove_stray_courses_from_mysql + + +class Command(BaseCommand): + """ + Bulk removal of courses without an organization linked (aka stray courses). + + This only works for MySQL database. + """ + + def add_arguments(self, parser): + parser.add_argument( + '--limit', + help='Max courses to delete, use 0 to delete all courses.', + default=1, + type=int, + ) + + parser.add_argument( + '--commit', + help='Otherwise, the transaction would be rolled back.', + action='store_true', + dest='commit', + ) + + parser.add_argument( + '--dry-run', + help='Dry run the deletion process without removing the courses.', + action='store_false', + dest='commit', + ) + + def handle(self, *args, **options): + if settings.ROOT_URLCONF != 'lms.urls': + raise CommandError('This command can only be run in LMS.') + + remove_stray_courses_from_mysql( + limit=options['limit'], + commit=options.get('commit'), + print_func=self.stdout.write, + ) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py index 0bfb41813757..376f76e9774a 100644 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py @@ -1,8 +1,10 @@ -from django.core.management.base import BaseCommand, CommandError +import traceback + +from django.core.management.base import BaseCommand from django.contrib.sites.models import Site from django.db import transaction -from openedx.core.djangoapps.appsembler.sites.utils import delete_site +from ...deletion_utils import delete_site class Command(BaseCommand): @@ -13,11 +15,6 @@ class Command(BaseCommand): """ def add_arguments(self, parser): - parser.add_argument( - 'domain', - help='The domain of the organization to be deleted.', - type=str, - ) parser.add_argument( '--commit', default=False, @@ -26,33 +23,42 @@ def add_arguments(self, parser): action='store_true', ) + parser.add_argument( + 'domain', + help='The domain of the organization to be deleted.', + nargs='+', + type=str, + ) + def handle(self, *args, **options): - organization_domain = options['domain'] - - self.stdout.write('Removing "%s" in progress...' % organization_domain) - organization = self._get_site(organization_domain) - - with transaction.atomic(): - delete_site(organization) - - if not options['commit']: - transaction.set_rollback(True) - - self.stdout.write(self.style.SUCCESS( - '{message} removed site "{domain}"'.format( - message='Successfully' if options['commit'] else 'Dry run', - domain=organization_domain, - ) - )) - - def _get_site(self, domain): - """ - Locates the site to be deleted and return its instance. - - :param domain: The domain of the site to be returned. - :return: Returns the site object that has the given domain. - """ - try: - return Site.objects.get(domain=domain) - except Site.DoesNotExist: - raise CommandError('Cannot find "%s" in Sites!' % domain) + domains = options['domain'] + + for domain in domains: + self.stdout.write('Removing "%s" in progress...' % domain) + + try: + site = Site.objects.filter(domain=domain).first() + if not site: + self.stderr.write(self.style.ERROR('Cannot find "{domain}"'.format(domain=domain))) + continue + + with transaction.atomic(): + delete_site(site) + + if not options['commit']: + transaction.set_rollback(True) + except Exception: # noqa + self.stderr.write(self.style.ERROR( + 'Failed to remove site "{domain}" error: \n {error}'.format( + domain=domain, + error=traceback.format_exc(), + ) + )) + traceback.format_exc() + else: + self.stdout.write(self.style.SUCCESS( + '{message} removed site "{domain}"'.format( + message='Successfully' if options['commit'] else 'Dry run', + domain=domain, + ) + )) diff --git a/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py b/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py index 7c41e436ec8a..3a3df3891039 100644 --- a/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py +++ b/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py @@ -1,7 +1,7 @@ """ Integration helpers for SiteConfig Client adapter. """ - +import beeline import logging from uuid import UUID @@ -16,7 +16,7 @@ from site_config_client.openedx.adapter import SiteConfigAdapter from site_config_client.exceptions import SiteConfigurationError -from tiers.tier_info import TierInfo +from ..tahoe_tiers.tier_info import TierInfo from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers @@ -29,22 +29,25 @@ # TODO: Move these helpers into the `site_config_client.openedx.api` module +@beeline.traced('site_config_client_helpers.is_enabled_for_site') def is_enabled_for_site(site): """ Checks if the SiteConfiguration client is enabled for a specific organization. """ from django.conf import settings # Local import to avoid AppRegistryNotReady error - if site.id == settings.SITE_ID: - # Disable the SiteConfig service on main site. - return False - - try: - uuid = tahoe_sites.api.get_uuid_by_site(site) - except ObjectDoesNotExist: - # Return sane result in case of malformed data - return False - return is_feature_enabled_for_site(uuid) + is_enabled = False + if site.id != settings.SITE_ID: # Disable the SiteConfig service on main site. + try: + uuid = tahoe_sites.api.get_uuid_by_site(site) + except ObjectDoesNotExist: + # Act as if disabled in case of malformed data + is_enabled = False + else: + is_enabled = is_feature_enabled_for_site(uuid) + + beeline.add_trace_field('site_config.enabled', is_enabled) + return is_enabled def enable_for_site(site, note=''): diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py index 5e35d4a5d5a2..0ffd0348eca0 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py @@ -1,6 +1,5 @@ -import hashlib import os -from mock import patch, mock_open +from unittest.mock import patch, mock_open, Mock from io import StringIO from django.conf import settings @@ -9,11 +8,13 @@ from django.core.management import call_command from django.core.management.base import CommandError from django.test import override_settings, TestCase + from tahoe_sites.api import ( create_tahoe_site_by_link, get_organization_for_user, get_users_of_organization, get_uuid_by_organization, + get_organization_by_site, ) from tahoe_sites.tests.utils import create_organization_mapping @@ -55,11 +56,8 @@ UserStandingFactory, ) -from organizations.models import Organization, OrganizationCourse - -from oauth2_provider.models import AccessToken, RefreshToken, Application - -from student.roles import CourseCreatorRole +from organizations.models import OrganizationCourse, Organization +from oauth2_provider.models import Application @override_settings( @@ -100,57 +98,15 @@ def test_create_devstack_site(self): with patch.object(Command, 'congrats') as mock_congrats: call_command('create_devstack_site', self.name, 'localhost') - mock_congrats.assert_called_once() # Ensure that congrats message is printed + assert mock_congrats.called # Ensure that congrats message is printed + assert mock_congrats.call_count == 1 # Ensure that congrats message is printed once # Ensure objects are created correctly. assert Site.objects.get(domain=self.site_name) organization = Organization.objects.get(name=self.name) user = get_user_model().objects.get() - assert user.check_password(self.name) - assert user.profile.name == self.name assert get_organization_for_user(user=user) == organization - assert CourseCreatorRole().has_user(user), 'User should be a course creator' - - fake_token = hashlib.md5(user.username.encode('utf-8')).hexdigest() # Using a fake token so AMC devstack can guess it - assert fake_token == '80bfa968ffad007c79bfc603f3670c99', 'Ensure hash is identical to AMC' - assert AccessToken.objects.get(user=user).token == fake_token, 'Access token is needed' - assert RefreshToken.objects.get(user=user).token == fake_token, 'Refresh token is needed' - - -@override_settings( - DEBUG=True, -) -class TestCandidateSitesCleanupCommand(TestCase): - """ - Tests for the `danger_candidate_sites_cleanup` management command. - """ - def setUp(self): - Application.objects.create(client_id=settings.AMC_APP_OAUTH2_CLIENT_ID, - client_type=Application.CLIENT_CONFIDENTIAL) - call_command('create_devstack_site', 'blue', 'oldlocalhost') - site_config = self.get_site().configuration - site_config.site_values.update({ - 'SEGMENT_KEY': 'test1', - 'customer_gtm_id': 'test2', - }) - site_config.save() - - def get_site(self): - return Site.objects.get(domain__startswith='blue.') - - def test_run(self): - assert self.get_site().domain == 'blue.oldlocalhost:18000' - active_orgs = Organization.objects.all() - active_orgs_function_path = 'openedx.core.djangoapps.appsembler.sites.utils.get_active_organizations' - with patch(active_orgs_function_path, return_value=active_orgs): - # Side-step the `Tier` model. - call_command('danger_candidate_sites_cleanup', 'oldlocalhost:18000', 'newlocalhost:18000') - assert self.get_site().domain == 'blue.newlocalhost:18000' - assert not self.get_site().configuration.get_value('customer_gtm_id') - assert not self.get_site().configuration.get_value('SEGMENT_KEY') - assert self.get_site().configuration.get_value('SITE_NAME') == self.get_site().domain - @override_settings( DEBUG=True, @@ -160,6 +116,10 @@ def test_run(self): 'DISABLE_COURSE_CREATION': False, 'ENABLE_CREATOR_GROUP': True, }) +@patch( # Avoid CMS-related import issues in tests + 'openedx.core.djangoapps.appsembler.sites.deletion_utils.remove_course_creator_role', + Mock() +) class RemoveSiteCommandTestCase(TestCase): """ Test ./manage.py lms remove_site mysite @@ -182,17 +142,22 @@ def test_remove_devstack_site_commit(self): """ deleted_domain = '{}.localhost:18000'.format(self.to_be_deleted) remained_domain = '{}.localhost:18000'.format(self.shall_remain) - + assert Site.objects.filter(domain__endswith='.localhost:18000').count() == 2, 'there are two sites' + remained_site = Site.objects.get(domain=remained_domain) + + # TODO: Re-produce the error we face in staging + to_delete_site = Site.objects.get(domain=deleted_domain) + to_delete_organization = get_organization_by_site(to_delete_site) + users = get_users_of_organization(to_delete_organization) + assert len(users), 'Ensure the site has users' call_command('remove_site', deleted_domain, commit=True) # Ensure objects are removed correctly. assert not Site.objects.filter(domain=deleted_domain).exists() - site = Site.objects.get(domain=remained_domain) - - assert SiteConfiguration.objects.count() == 1 - assert SiteConfiguration.objects.get(site=site) - assert SiteTheme.objects.filter(site=site).count() == site.themes.count() + assert Site.objects.filter(domain__endswith='.localhost:18000').count() == 1, 'One site is deleted' + remained_site.refresh_from_db() # remained_domain site config is kept + assert SiteTheme.objects.filter(site=remained_site).count() == remained_site.themes.count() def test_remove_devstack_site_rollback(self): """ diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py index 887665aea96f..0ebd5078f5f1 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py @@ -1,28 +1,39 @@ +from unittest.mock import patch + import pytest import tahoe_sites.api from django.contrib.auth import get_user_model from django.contrib.sites.models import Site -from django.core.management import call_command +from django.core.management import call_command, CommandError from oauth2_provider.models import Application from organizations.models import OrganizationCourse from status.models import CourseMessage from student.models import AnonymousUserId +from lms.djangoapps.courseware.models import StudentModule from openedx.core.djangoapps.appsembler.api.tests.factories import ( CourseOverviewFactory, OrganizationCourseFactory, ) from openedx.core.djangoapps.content.course_overviews.models import CourseOverview -User = get_user_model() - - -from openedx.core.djangoapps.appsembler.sites.utils import ( +from openedx.core.djangoapps.appsembler.sites.deletion_utils import ( + delete_organization_courses, delete_site, get_models_using_course_key, - delete_organization_courses, + remove_stray_courses_from_mysql, ) +User = get_user_model() + + +def delete_site_with_patched_cms_imports(red_site): + """ + Delete a site without running the CMS-related code. + """ + with patch('openedx.core.djangoapps.appsembler.sites.deletion_utils.remove_course_creator_role'): + delete_site(red_site) + @pytest.fixture @pytest.mark.django_db @@ -49,7 +60,7 @@ def test_delete_site(make_site): Test `delete_site` happy path. """ red_site = make_site('red') - delete_site(red_site) + delete_site_with_patched_cms_imports(red_site) with pytest.raises(User.DoesNotExist): User.objects.get(username='red') @@ -63,7 +74,7 @@ def test_delete_one_site_keeps_another(make_site): red_site = make_site('red') make_site('blue') - delete_site(red_site) + delete_site_with_patched_cms_imports(red_site) with pytest.raises(User.DoesNotExist): User.objects.get(username='red') @@ -87,6 +98,7 @@ def test_get_models_using_course_key(): assert AnonymousUserId in classes, 'Should include AnonymousUserId due to course_id field' assert CourseMessage in classes, 'Should include CourseMessage due to course_key field' assert OrganizationCourse in classes, 'Should include OrganizationCourse' + assert StudentModule in classes, 'Should include models with LearningContextKeyField' @pytest.mark.django_db @@ -125,3 +137,23 @@ def test_delete_course_related_models(make_site): # Should delete the course-related models with pytest.raises(model_class.DoesNotExist): model_class.objects.get() + + +@pytest.mark.django_db +def test_mysql_remove_stray_courses(capsys): + """ + Tests for the remove_stray_courses_from_mysql with and without courses. + """ + with pytest.raises(CommandError, match='No courses to delete.'): + remove_stray_courses_from_mysql(limit=0, commit=False) + + course_key = CourseOverviewFactory.create().id + assert course_key in CourseOverview.get_all_course_keys(), 'Stray course has been created' + + remove_stray_courses_from_mysql(limit=0, commit=False) + assert course_key in CourseOverview.get_all_course_keys(), 'Commit=False do not delete the course' + assert str(course_key) in capsys.readouterr()[0] + + remove_stray_courses_from_mysql(limit=0, commit=True) + assert course_key not in CourseOverview.get_all_course_keys(), 'Stray course is removed' + assert str(course_key) in capsys.readouterr()[0] diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index e85ca237e2e6..cef2bdce7755 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -3,13 +3,10 @@ A lot of this module should be migrated into more specific modules such as `tahoe-sites`. """ -import tahoe_sites.api -from django.apps import apps from datetime import timedelta import beeline -from opaque_keys.edx.django.models import CourseKeyField from urllib.parse import urlparse @@ -31,9 +28,9 @@ from organizations import api as org_api from organizations import models as org_models -from organizations.models import OrganizationCourse from organizations.models import Organization + from tahoe_sites.api import ( add_user_to_organization, create_tahoe_site_by_link, @@ -44,7 +41,6 @@ update_admin_role_in_organization, ) -from common.djangoapps.util.organizations_helpers import get_organization_courses from openedx.core.lib.api.api_key_permissions import is_request_has_valid_api_key from openedx.core.lib.log_utils import audit_log from openedx.core.djangoapps.theming.helpers import get_current_request, get_current_site @@ -52,7 +48,6 @@ from ..tahoe_tiers.legacy_amc_helpers import get_active_tiers_uuids_from_amc_postgres from .site_config_client_helpers import get_active_site_uuids_from_site_config_service -from openedx.core.djangoapps.content.course_overviews.models import CourseOverview @beeline.traced(name="get_lms_link_from_course_key") @@ -93,9 +88,11 @@ def get_active_organizations(): TODO: This helper should live in a future Tahoe Sites package. """ - active_tiers_uuids = get_active_organizations_uuids() - - return get_organizations_from_uuids(uuids=active_tiers_uuids) + if settings.FEATURES.get('ENABLE_TIERS_APP', False): + active_tiers_uuids = get_active_organizations_uuids() + return get_organizations_from_uuids(uuids=active_tiers_uuids) + else: + return Organization.objects.all() def get_active_sites(order_by='domain'): @@ -480,70 +477,6 @@ def bootstrap_site(site, org_data=None, username=None): return organization, site, user -def get_models_using_course_key(): - course_key_field_names = { - 'course_key', - 'course_id', - } - - models_with_course_key = { - (CourseOverview, 'id'), # The CourseKeyField with a `id` name. Hard-coding it for simplicity. - (OrganizationCourse, 'course_id'), # course_id is CharField - } - - model_classes = apps.get_models() - for model_class in model_classes: - for field_name in course_key_field_names: - field_object = getattr(model_class, field_name, None) - if field_object: - field_definition = getattr(field_object, 'field', None) - if field_definition and isinstance(field_definition, CourseKeyField): - models_with_course_key.add( - (model_class, field_name,) - ) - - return models_with_course_key - - -def delete_organization_courses(organization): - course_keys = [] - - for course in get_organization_courses({'id': organization.id}): - course_keys.append(course['course_id']) - - for model_class, field_name in get_models_using_course_key(): - print('Deleting models of', model_class.__name__, 'with field', field_name) - objects_to_delete = model_class.objects.filter(**{ - '{field_name}__in'.format(field_name=field_name): course_keys, - }) - objects_to_delete.delete() - - -@beeline.traced(name="delete_site") -def delete_site(site): - print('Deleting SiteConfiguration of', site) - site.configuration.delete() - - print('Deleting theme of', site) - site.themes.all().delete() - - organization = tahoe_sites.api.get_organization_by_site(site) - - users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) - - print('Deleting users of', site) - users.delete() - - print('Deleting courses of', site) - delete_organization_courses(organization) - - print('Deleting organization', organization) - organization.delete() - - print('Deleting site', site) - site.delete() - - @beeline.traced(name="add_course_creator_role") def add_course_creator_role(user): """ diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py b/openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py new file mode 100644 index 000000000000..e44aa9f058de --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py @@ -0,0 +1,56 @@ +""" +Tahoe Authentication helpers for managing course related roles. +""" + +from common.djangoapps.student.roles import ( + CourseCreatorRole, + OrgRole, + OrgStaffRole, + register_access_role, +) + + +def update_organization_staff_roles( + user, + organization_short_name, + set_as_course_author=False, + set_as_organization_staff=False, +): + """ + Update the organization-wide OrgStaffRole/CourseCreatorRole for using Studio and instructor dashboards. + """ + assert user, 'Parameter `user` is required.' + assert organization_short_name, 'Parameter `organization_short_name` is required.' + + organization_role = OrgStaffRole(organization_short_name) + course_author_role = TahoeCourseAuthorRole(organization_short_name) + creator_role = CourseCreatorRole() + + if set_as_organization_staff or set_as_course_author: + # Both org-wide staff and limited course author can create courses. + creator_role.add_users(user) + else: + creator_role.remove_users(user) + + if set_as_organization_staff: + organization_role.add_users(user) + else: + organization_role.remove_users(user) + + if set_as_course_author: + course_author_role.add_users(user) + else: + course_author_role.remove_users(user) + + +@register_access_role +class TahoeCourseAuthorRole(OrgRole): + """ + A limited course access role to allow Studio access without having a course. + + A user with this role needs to be explicitly invited to a course. + """ + ROLE = 'tahoe_course_author' + + def __init__(self, *args, **kwargs): + super().__init__(self.ROLE, *args, **kwargs) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py b/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py index c34884a472d2..0a3d3bc20625 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py @@ -4,23 +4,25 @@ - https://github.com/appsembler/tahoe-idp/ """ +import re +from urllib import parse from collections import OrderedDict from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse from django.utils.http import urlencode from site_config_client.openedx import api as config_client_api -from tahoe_sites.api import is_active_admin_on_organization, get_organization_for_user +from organizations.models import Organization +from tahoe_sites.api import ( + is_active_admin_on_organization, + get_organization_for_user, + get_site_by_organization, +) import third_party_auth from third_party_auth.pipeline import running as pipeline_running - -from .constants import ( - TAHOE_IDP_BACKEND_NAME, - TAHOE_IDP_PROVIDER_NAME, -) - from student.roles import ( CourseAccessRole, CourseCreatorRole, @@ -33,6 +35,28 @@ from tahoe_idp import api as tahoe_idp_api +from .constants import ( + TAHOE_IDP_BACKEND_NAME, + TAHOE_IDP_PROVIDER_NAME, +) + +from .course_roles import TahoeCourseAuthorRole + +ALLOWED_KEY_CHAR = r'[\w\-~.:%]' +KEY_PARTS = '(?P{ALLOWED_KEY_CHAR}+)\\+(?P{ALLOWED_KEY_CHAR}+)\\+(?P{ALLOWED_KEY_CHAR}+)' \ + .format(ALLOWED_KEY_CHAR=ALLOWED_KEY_CHAR) +VALID_SEPARATOR = '[/@&\\-\\?\\+]' +VALID_LOCATOR = '(\\bcourse|\\bblock)-v1:{KEY_PARTS}'.format(KEY_PARTS=KEY_PARTS) +VALID_PRE_KEY = '(.*{VALID_SEPARATOR})|({VALID_SEPARATOR})'.format(VALID_SEPARATOR=VALID_SEPARATOR) +VALID_POST_KEY = '({VALID_SEPARATOR}.*|({VALID_SEPARATOR}))'.format(VALID_SEPARATOR=VALID_SEPARATOR) +VALID_URL = '(?P{VALID_PRE_KEY})?(?P{VALID_LOCATOR})(?P{VALID_POST_KEY})?'.format( + VALID_LOCATOR=VALID_LOCATOR, + VALID_PRE_KEY=VALID_PRE_KEY, + VALID_POST_KEY=VALID_POST_KEY, +) +URL_WITH_LOCATOR_REGEX = re.compile(VALID_URL, re.UNICODE) + + def is_tahoe_idp_enabled(): """ Tahoe: Check if tahoe-idp package is enabled for the current site (or cluster-wide). @@ -148,7 +172,7 @@ def is_studio_allowed_for_user(user, organization=None): OR the user is staff user OR the user is an admin on the organization OR the user has deprecated_has_course_specific_role() - OR the user has (OrgStaffRole) or (OrgInstructorRole) role + OR the user has (OrgStaffRole) or (OrgInstructorRole) or (TahoeCourseAuthorRole) role :param user: the user in question :param organization: the user's organization. If the user is not super admin or staff, this value will be used @@ -168,9 +192,12 @@ def is_studio_allowed_for_user(user, organization=None): return True short_name = organization.short_name - has_org_wide_role = OrgStaffRole(short_name).has_user(user) or OrgInstructorRole(short_name).has_user(user) - return has_org_wide_role + for org_wide_role in [OrgStaffRole, OrgInstructorRole, TahoeCourseAuthorRole]: + if org_wide_role(short_name).has_user(user): + return True + + return False def is_studio_login_form_overridden(): @@ -180,3 +207,52 @@ def is_studio_login_form_overridden(): if settings.FEATURES.get('TAHOE_IDP_STUDIO_LOGIN_FORM_OVERRIDE', None): return True return False + + +def extract_organization_from_url(url): + """ + Extracts the organization from the given url + + :param url: source uri to extract the course_id from + :return: organization if found, None otherwise + """ + url = url or '' + organization = None + + match = re.search(URL_WITH_LOCATOR_REGEX, url) + if match: + try: + organization = Organization.objects.get( + short_name=match.group('org_short_name') + ) + except ObjectDoesNotExist: + pass + return organization + + +def get_redirect_to_lms_login_url(request): + """ + Get organization site from course id if found in (next) argument of (request.get_full_path()). Then return + the appropriate URL for to studio Magic Link authentication. + + :param request: full path from the request to be processed + :return: redirect url if a valid course key found, otherwise return empty string + """ + if not (request and request.GET.get('next')): + return '' + + next_url = request.GET['next'] + organization = extract_organization_from_url(next_url) + + if organization: + site = get_site_by_organization(organization=organization) + + protocol = 'https' if request.is_secure() else 'http' + redirect_url = '{protocol}://{site_domain}/studio/?next={quoted_next}'.format( + protocol=protocol, + site_domain=site.domain, + quoted_next=parse.quote_plus(next_url), + ) + return redirect_url + + return '' diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/patches.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/patches.py new file mode 100644 index 000000000000..0fd4d15f2b20 --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/patches.py @@ -0,0 +1,10 @@ +""" +patching helpers +""" + +from mock import Mock + + +# Keep Signal receivers in tahoe_idp from executing fully when not explicitly testing +dummy_receivers_idp_not_enabled = Mock() +dummy_receivers_idp_not_enabled.return_value = False diff --git a/openedx/core/djangoapps/appsembler/auth/tests/test_course_roles.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_course_roles.py similarity index 97% rename from openedx/core/djangoapps/appsembler/auth/tests/test_course_roles.py rename to openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_course_roles.py index 5be3ed86094b..8e457fcf9a8b 100644 --- a/openedx/core/djangoapps/appsembler/auth/tests/test_course_roles.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_course_roles.py @@ -7,7 +7,7 @@ from student.tests.factories import UserFactory -from .. import course_roles +from openedx.core.djangoapps.appsembler.tahoe_idp import course_roles @pytest.mark.django_db diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_account_deletion.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_account_deletion.py index 513cf3169fa4..80b50bcfdd50 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_account_deletion.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_account_deletion.py @@ -8,6 +8,7 @@ from django.urls import reverse from django.contrib.auth import get_user_model + from rest_framework.test import APITestCase from rest_framework import status @@ -18,6 +19,8 @@ setup_retirement_states, # pylint: disable=unused-import ) +from . import patches + @patch.dict('django.conf.settings.FEATURES', {'SKIP_EMAIL_VALIDATION': True}) @pytest.mark.usefixtures("setup_retirement_states") @@ -64,6 +67,7 @@ def deactivate_user(self, color, username=None): }) return response + @patch('tahoe_idp.receivers.helpers.is_tahoe_idp_enabled', new=patches.dummy_receivers_idp_not_enabled) @patch('tahoe_idp.api.get_tahoe_idp_id_by_user') @patch('tahoe_idp.api.deactivate_user') @patch.dict('django.conf.settings.FEATURES', {'ENABLE_TAHOE_IDP': True}) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_email_change.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_email_change.py index a240b4db7f32..0228f3e47c79 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_email_change.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_email_change.py @@ -12,6 +12,8 @@ from student.models import PendingEmailChange from student.tests.factories import PendingEmailChangeFactory, UserFactory +from . import patches + @skip_unless_lms class EmailChangeWithIdpTests(TestCase): @@ -37,11 +39,13 @@ def test_successful_email_change_without_idp(self, mock_update_user_email): 'Should not use idp unless explicitly enabled via ENABLE_TAHOE_IDP' ) + @patch('tahoe_idp.receivers.helpers.is_tahoe_idp_enabled', new=patches.dummy_receivers_idp_not_enabled) @patch('tahoe_idp.api.update_user_email') def test_successful_email_change_with_idp(self, mock_update_user_email): """ Test `confirm_email_change` with ENABLE_TAHOE_IDP = True. """ + with patch.dict(settings.FEATURES, {'ENABLE_TAHOE_IDP': True}): response = self.client.get(reverse('confirm_email_change', args=[self.key])) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py index 2393b29cd8aa..868795c30ea0 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py @@ -2,8 +2,10 @@ Tests for `tahoe_idp.helpers`. """ from unittest.mock import patch, Mock +from urllib import parse from django.conf import settings +from django.test import RequestFactory import pytest from organizations.tests.factories import OrganizationFactory @@ -38,6 +40,17 @@ def user_with_org(): return learner, organization +@pytest.fixture +def valid_request(): + """ + :return: a request that can be used in our tests here + """ + request = RequestFactory() + request.GET = {} + request.is_secure = Mock(return_value=False) + return request + + @pytest.mark.parametrize('global_flags,site_flags,should_be_enabled,message', [ ({}, {'ENABLE_TAHOE_IDP': True}, True, 'site-flag should enable it'), ({'ENABLE_TAHOE_IDP': True}, {}, True, 'cluster-wide flag should enable it'), @@ -232,3 +245,96 @@ def test_is_studio_login_form_overridden_flag_available(flag_value, expected_res """ with patch.dict('django.conf.settings.FEATURES', {'TAHOE_IDP_STUDIO_LOGIN_FORM_OVERRIDE': flag_value}): assert helpers.is_studio_login_form_overridden() is expected_result + + +@pytest.mark.parametrize('url', [ + 'course-v1:ninja_org+course+2022', + '/course-v1:ninja_org+course+2022', + 'bla_bla_bla/course-v1:ninja_org+course+2022/', + 'course-v1:ninja_org+course+2022/bla_bla_bla', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla-v1:ninja_org+course+2022', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla/course-v1:ninja_org+course+2022', + 'bla_bla_bla/course-v1:unexpected_other_org+course+2022/bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla', +]) +@pytest.mark.django_db +def test_extract_organization_from_url_success(url): + """ + Verify that extract_organization_from_url returns the expected organization or None according to the given URL + """ + organization = OrganizationFactory.create(short_name='ninja_org', name='ninja_org_long_name') + assert helpers.extract_organization_from_url(url) == organization + url = url.replace('course-v1', 'block-v1') + assert helpers.extract_organization_from_url(url) == organization + + +@pytest.mark.parametrize('url', [ + 'course-v1:ninja_org+course+', + 'course-v1:ninja_org+course', + 'course-v2:ninja_org+course+2022', + 'courses-v1:ninja_org+course+2022', + 'asdasdcourse-v1:ninja_org+course+2022', + 'asdasdcourse-v1:ninja_org+course+2022asdasdad', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla/course-v1:unexpected_other_org+course+2022/bla_bla_bla', +]) +@pytest.mark.django_db +def test_extract_organization_from_url_not_found(url): + """ + Verify that extract_organization_from_url returns the expected organization or None according to the given URL + """ + # just to verify that the function doesn't return (None) because of a missing organization + OrganizationFactory.create(short_name='ninja_org', name='ninja_org_long_name') + + assert helpers.extract_organization_from_url(url) is None + + +def test_get_redirect_to_lms_login_url_no_request(): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + assert helpers.get_redirect_to_lms_login_url(None) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_no_next(valid_request): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + assert helpers.get_redirect_to_lms_login_url(valid_request) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_next_with_no_course(valid_request): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + valid_request.GET['next'] = 'bla_bla' + assert helpers.get_redirect_to_lms_login_url(valid_request) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_next_with_invalid_course(valid_request): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + valid_request.GET['next'] = 'course/course-v1:ORG+DoesNotExist' + assert helpers.get_redirect_to_lms_login_url(valid_request) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_next_with_valid_course(valid_request, user_with_org): + """ + Verify that get_redirect_to_lms_login_url will return the expected URL is a valid course_id is provided + """ + _, organization = user_with_org + site = tahoe_sites_apis.get_site_by_organization(organization=organization) + + next_url = 'container/block-v1:{org}+course+run'.format(org=organization.short_name) + encoded_next = parse.quote_plus(next_url) + expected_url = 'http://{site_domain}/studio/?next={encoded_next}'.format( + site_domain=site.domain, + encoded_next=encoded_next + ) + + valid_request.GET['next'] = next_url + assert helpers.get_redirect_to_lms_login_url(valid_request) == expected_url diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py index c7be2563ec61..05874cbaf5f3 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py @@ -8,6 +8,7 @@ import tahoe_sites.api from common.djangoapps.student.roles import CourseCreatorRole, OrgStaffRole +from ..course_roles import TahoeCourseAuthorRole from ..tpa_pipeline import tahoe_idp_user_updates from openedx.core.djangoapps.site_configuration.tests.factories import SiteFactory @@ -37,40 +38,58 @@ def test_tahoe_idp_step_in_settings(): assert idp_step_index == force_sync_step_index + 1, 'Tahoe IdP step should be right after `user_details_force_sync`' -@pytest.mark.parametrize('user_details,should_be_admin,should_be_staff,message', [ - ( - { +@pytest.mark.parametrize('test_case', [ + { + 'user_details': { 'tahoe_idp_is_organization_admin': False, 'tahoe_idp_is_organization_staff': False, + 'tahoe_idp_is_course_author': False, 'tahoe_idp_metadata': {'field': 'some value'}, }, - False, - False, - 'Check for learner', - ), - ( - { + 'should_be_admin': False, + 'should_be_staff': False, + 'should_be_author': False, + 'message': 'Check for learner', + }, + { + 'user_details': { 'tahoe_idp_is_organization_admin': False, 'tahoe_idp_is_organization_staff': True, + 'tahoe_idp_is_course_author': False, 'tahoe_idp_metadata': {'field': 'some value'}, }, - False, - True, - 'Check for Studio', - ), - ( - { + 'should_be_admin': False, + 'should_be_staff': True, + 'should_be_author': False, + 'message': 'Check for Studio', + }, + { + 'user_details': { 'tahoe_idp_is_organization_admin': True, 'tahoe_idp_is_organization_staff': True, + 'tahoe_idp_is_course_author': False, 'tahoe_idp_metadata': {'field': 'some value'}, }, - True, - True, - 'Check for Admins', - ), + 'should_be_admin': True, + 'should_be_staff': True, + 'should_be_author': False, + 'message': 'Check for Admins', + }, + { + 'user_details': { + 'tahoe_idp_is_organization_admin': False, + 'tahoe_idp_is_organization_staff': False, + 'tahoe_idp_is_course_author': True, + 'tahoe_idp_metadata': {'field': 'some value'}, + }, + 'should_be_admin': False, + 'should_be_staff': False, + 'should_be_author': True, + 'message': 'Check for Course Authors', + }, ]) @pytest.mark.django_db -def test_tahoe_idp_roles_step_roles(user_details, should_be_admin, should_be_staff, message): +def test_tahoe_idp_roles_step_roles(test_case): """ Tests for happy scenarios of the `tahoe_idp_user_updates` step. """ @@ -88,15 +107,18 @@ def test_tahoe_idp_roles_step_roles(user_details, should_be_admin, should_be_sta tahoe_idp_user_updates( auth_entry=None, strategy=strategy, - details=user_details, + details=test_case['user_details'], user=user, ) - org_role = OrgStaffRole(organization.short_name) + tahoe_author_role = TahoeCourseAuthorRole(organization.short_name) creator_role = CourseCreatorRole() - assert org_role.has_user(user) == should_be_staff, message - assert creator_role.has_user(user) == should_be_staff, message - assert tahoe_sites.api.is_active_admin_on_organization(user, organization) == should_be_admin, message + message = test_case['message'] + should_be_course_creator = test_case['should_be_staff'] or test_case['should_be_author'] + assert org_role.has_user(user) == test_case['should_be_staff'], message + assert creator_role.has_user(user) == should_be_course_creator, message + assert tahoe_author_role.has_user(user) == test_case['should_be_author'], message + assert tahoe_sites.api.is_active_admin_on_organization(user, organization) == test_case['should_be_admin'], message assert user.profile.get_meta() == {'tahoe_idp_metadata': {'field': 'some value'}} mock_update_tahoe_user_id.assert_called_once_with(user) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_reset_password.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_reset_password.py index b2ab7fe5264d..aedb06bd0a16 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_reset_password.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_reset_password.py @@ -13,9 +13,12 @@ from student.tests.factories import UserFactory +from . import patches + @ddt.ddt @skip_unless_lms +@patch('tahoe_idp.receivers.helpers.is_tahoe_idp_enabled', new=patches.dummy_receivers_idp_not_enabled) @patch('tahoe_idp.api.request_password_reset') class TahoeIdpResetPasswordTests(TestCase): """ diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_registration_api.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_registration_api.py index 4cabca20b698..9ad24c211019 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_registration_api.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_registration_api.py @@ -12,6 +12,8 @@ from ...multi_tenant_emails.tests.test_utils import with_organization_context +from . import patches + APPSEMBLER_API_VIEWS_MODULE = 'openedx.core.djangoapps.appsembler.api.v1.views' @@ -51,6 +53,7 @@ def test_api_without_tahoe_idp(self, url): content = response.content.decode('utf-8') assert response.status_code == status.HTTP_200_OK, '{} {}'.format(color1, content) + @patch('tahoe_idp.receivers.helpers.is_tahoe_idp_enabled', new=patches.dummy_receivers_idp_not_enabled) @patch.dict('django.conf.settings.FEATURES', {'ENABLE_TAHOE_IDP': True}) @ddt.data( reverse_lazy('tahoe-api:v1:registrations-list'), @@ -60,6 +63,7 @@ def test_api_wit_tahoe_idp(self, url): """ Both v1 and v2 API shouldn't work with Tahoe IdP. """ + color1 = 'red1' with with_organization_context(site_color=color1): response = self.register_user(url, 'red_learner') diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py index ee663cd32844..1616a5225873 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py @@ -6,12 +6,13 @@ import beeline import tahoe_sites.api -from openedx.core.djangoapps.appsembler.auth import course_roles +from openedx.core.djangoapps.appsembler import waffle as appsembler_waffle +from social_core.pipeline.user import create_user as social_core_create_user from tahoe_idp import api as tahoe_idp_api +from . import course_roles from .helpers import store_idp_metadata_in_user_profile - from .constants import TAHOE_IDP_BACKEND_NAME log = logging.getLogger(__name__) @@ -38,6 +39,7 @@ def tahoe_idp_user_updates(auth_entry, strategy, details, user=None, *args, **kw if user and backend_name == TAHOE_IDP_BACKEND_NAME: set_as_admin = details['tahoe_idp_is_organization_admin'] set_as_organization_staff = details['tahoe_idp_is_organization_staff'] + set_as_course_author = details['tahoe_idp_is_course_author'] organization = tahoe_sites.api.get_current_organization(strategy.request) @@ -53,6 +55,7 @@ def tahoe_idp_user_updates(auth_entry, strategy, details, user=None, *args, **kw course_roles.update_organization_staff_roles( user=user, organization_short_name=organization_short_name, + set_as_course_author=set_as_course_author, set_as_organization_staff=set_as_organization_staff, ) @@ -60,3 +63,14 @@ def tahoe_idp_user_updates(auth_entry, strategy, details, user=None, *args, **kw # TODO: Directly call `tahoe_idp.api` function may not be a good idea, find a better signal or hook instead. tahoe_idp_api.update_tahoe_user_id(user) + + +def wrapped_social_core_create_user(strategy, details, backend, user=None, *args, **kwargs): + """ + Wrapped social_core.pipeline.create_user + Check to disable based on Waffle Flag. + """ + if appsembler_waffle.disable_tpa_create_user_step(strategy.request): # effectively disable + return {'is_new': False} + else: + social_core_create_user(strategy, details, backend, user, *args, **kwargs) diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py index 3a68c1ba68db..d3398bf48915 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py @@ -6,13 +6,13 @@ import logging import beeline - -from tahoe_sites.api import get_uuid_by_organization +from uuid import UUID from django.utils import timezone -from django.db.models import Q, F from tiers.models import Tier +from ..tahoe_tiers.tier_info import TierInfo + log = logging.getLogger(__name__) @@ -24,19 +24,43 @@ def get_amc_tier_info(site_uuid): # pragma: no cover Hack: This queries the django-tier database in a rather hacky way. - WARNING: !! This function is _not_ covered with tests. Please edit with caution and test manually. !! + WARNING: !! This function is _not_ fully covered with tests. Please edit with caution and test on staging. !! """ try: + site_uuid_hex = UUID(str(site_uuid)).hex + # Query the AMC Postgres database directly - tier = Tier.objects.defer('organization').get(organization__edx_uuid=site_uuid) - return tier.get_tier_info() - except Tier.DoesNotExist: - # If the organization has no AMC-tier fail silently and log it in honeycomb. - # This either happens in the case of a Tahoe 2.0 site or a missing tier - # from AMC (although that shouldn't happen). - beeline.add_context_field("tiers.organization_without_tier", True) - return None - except Exception: + tiers = Tier.objects.raw( + """SELECT + t.id as id, + t.name AS name, + t.tier_expires_at AS tier_expires_at, + org.edx_uuid as edx_uuid, + t.tier_enforcement_exempt AS tier_enforcement_exempt + FROM tiers_tier as t + INNER JOIN organizations_organization as org on t.organization_id = org.id + WHERE org.edx_uuid = %s + LIMIT 1 + """, + [str(site_uuid_hex)] + ) + tiers_list = list(tiers) + + if not tiers_list: + # If the organization has no AMC-tier fail silently and log it in honeycomb. + # This either happens in the case of a Tahoe 2.0 site or a missing tier + # from AMC (although that shouldn't happen). + beeline.add_context_field("tiers.organization_without_tier", True) + return None + + tier = tiers[0] + return TierInfo( + tier=tier.name, + subscription_ends=tier.tier_expires_at, + always_active=tier.tier_enforcement_exempt, + ) + except Exception: # noqa + log.exception('Error with fetching the tier from AMC') beeline.add_context_field("tiers.exception_with_tier", True) log.exception("Organization has a problem with its Tier: {0}".format(site_uuid)) return None @@ -50,13 +74,22 @@ def get_active_tiers_uuids_from_amc_postgres(): # pragma: no cover Return a list of UUID objects. - WARNING: !! This function is _not_ covered with tests. Please edit with caution and test manually. !! + WARNING: !! This function is _not_ fully covered with tests. Please edit with caution and test on staging. !! """ # This queries the AMC Postgres database - active_tiers_uuids = Tier.objects.filter( - Q(tier_enforcement_exempt=True) | - Q(tier_expires_at__gte=timezone.now()) - ).annotate( - organization_edx_uuid=F('organization__edx_uuid') - ).values_list('organization_edx_uuid', flat=True) - return list(active_tiers_uuids) + tiers = Tier.objects.raw( + """ + SELECT + t.id as id, + org.edx_uuid as site_uuid + FROM tiers_tier as t + INNER JOIN organizations_organization as org on t.organization_id = org.id + WHERE t.tier_expires_at >= %s OR t.tier_enforcement_exempt + """, + [str(timezone.now())] + ) + + return [ + UUID(str(t.site_uuid)) + for t in tiers + ] diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py new file mode 100644 index 000000000000..5163fd73085f --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py @@ -0,0 +1,63 @@ +from uuid import UUID + +import pytest + +from tahoe_sites.zd_helpers import should_site_use_org_models +from ..legacy_amc_helpers import ( + get_amc_tier_info, + get_active_tiers_uuids_from_amc_postgres, +) + + +@pytest.mark.django_db +@pytest.mark.parametrize('uuid', [ + '6229db46-76e7-11ed-bb20-37f3f60d0442', UUID('b29e2394-7baf-11ed-8efb-23999d1cbf5f') +]) +def test_get_amc_tier_info_not_found(uuid): + assert not get_amc_tier_info(uuid), 'Non-existent tier info' + + +@pytest.mark.django_db +@pytest.mark.skipif( + condition=not should_site_use_org_models(), + reason='Needs AMC database compatible edx-organizations' +) +def test_get_amc_tier_info_found(): + from tiers.models import Tier + from organizations.tests.factories import OrganizationFactory + + organization = OrganizationFactory.create(edx_uuid='2f51e0e1-7cd4-4447-86fc-5de03e2cf3b1') + tier = Tier.objects.create(organization=organization) + assert tier.organization == organization + tier = get_amc_tier_info(organization.edx_uuid) + assert tier, 'Should find tier' + assert tier.tier == 'trial', 'Should be trial' + + +@pytest.mark.django_db +@pytest.mark.skipif( + condition=not should_site_use_org_models(), + reason='Needs AMC database compatible edx-organizations' +) +def test_active_tiers(): + from tiers.models import Tier + from organizations.tests.factories import OrganizationFactory + + active_org = OrganizationFactory.create(edx_uuid='2f51e0e1-7cd4-4447-86fc-5de03e2cf3b1') + Tier.objects.create(organization=active_org) + + inactive_org = OrganizationFactory.create(edx_uuid='9e29c034-76f1-11ed-a879-7702b938796e') + Tier.objects.create(organization=inactive_org, tier_expires_at='2017-01-01') + + exempted_org = OrganizationFactory.create(edx_uuid='496efb30-76f2-11ed-b521-37e754be4889') + Tier.objects.create(organization=exempted_org, tier_enforcement_exempt=True, tier_expires_at='2017-01-01') + + org_without_tier = OrganizationFactory.create(edx_uuid='e54d116e-76f1-11ed-b6c2-87cd9adfb48f') + + active_tier_uuids = get_active_tiers_uuids_from_amc_postgres() + active_tier_uuids = [str(site_uuid) for site_uuid in active_tier_uuids] + + assert active_org.edx_uuid in active_tier_uuids, 'Should only list orgs with active tiers' + assert exempted_org.edx_uuid in active_tier_uuids, 'Exempted orgs are considered active' + assert inactive_org.edx_uuid not in active_tier_uuids, 'Expired org tier' + assert org_without_tier.edx_uuid not in active_tier_uuids, 'Missing org tier should not appear here' diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py new file mode 100644 index 000000000000..c652e33b462e --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py @@ -0,0 +1,40 @@ +""" +Tests for the TierInfo helper class. +""" + +from datetime import timedelta +from django.utils.timezone import now +from ..tier_info import TierInfo + + +def tier_info_factory( + tier=TierInfo.TRIAL, + subscription_ends=now() + timedelta(days=30), + always_active=False, +): + return TierInfo( + tier=tier, + subscription_ends=subscription_ends, + always_active=always_active, + ) + + +def test_non_expired_tier(): + t = tier_info_factory() + assert not t.always_active + assert not t.has_subscription_ended() + + +def test_expired_tier(): + t = tier_info_factory(subscription_ends=(now() - timedelta(days=2))) + assert not t.always_active + assert t.has_subscription_ended() + + +def test_exemption(): + t = tier_info_factory( + always_active=True, + subscription_ends=(now() - timedelta(days=20)), + ) + assert t.always_active + assert not t.has_subscription_ended() diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py new file mode 100644 index 000000000000..23aa5f0ec4eb --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py @@ -0,0 +1,63 @@ +""" +Tier helper and calculation classes with no model dependency. + +Note: This is cloned from `django-tiers:tiers.tier_info.py` to prepare for removing/refactoring the dependency. +""" +from collections import namedtuple + +from django.utils import timezone +from django.utils.timesince import timeuntil + +TierTuple = namedtuple('TierTuple', ['id', 'name']) + + +class TierInfo: + """ + Tier info and calculator class. + + TODO: Move into the Site Configuration Client package. + """ + + TRIAL = TierTuple('trial', 'Trial') # Expires in 30 days + BASIC = TierTuple('basic', 'Basic') + PRO = TierTuple('pro', 'Professional') + PREMIUM = TierTuple('premium', 'Premium') + + TIERS = ( + TRIAL, + BASIC, + PRO, + PREMIUM, + ) + + def __init__(self, tier, subscription_ends, always_active): + self.tier = tier + self.subscription_ends = subscription_ends + self.always_active = always_active + + def has_subscription_ended(self, now=None): + """Helper function that checks whether a subscription has expired""" + if self.always_active: + return False + + if not now: + now = timezone.now() + + return now > self.subscription_ends + + def should_show_expiration_warning(self): + """Decide if expiration warning is needed.""" + if self.always_active: + return False + + return self.tier == self.TRIAL.id + + def time_til_expiration(self, now=None): + """Pretty prints time left til expiration""" + if self.always_active: + return False + + if not now: + now = timezone.now() + + return timeuntil(self.subscription_ends, now) diff --git a/openedx/core/djangoapps/appsembler/waffle.py b/openedx/core/djangoapps/appsembler/waffle.py new file mode 100644 index 000000000000..a67fa0fd82c4 --- /dev/null +++ b/openedx/core/djangoapps/appsembler/waffle.py @@ -0,0 +1,47 @@ +""" +Appsembler-specific Waffle setup for Open edX Django apps. + +Waffle namespaces, flags, that are for specific Appsembler Django apps +should go in those apps. This module should be used for Flags and Switches +used to override, rollout, or modify changes to non-Appsembler apps. +""" + +from openedx.core.djangoapps.waffle_utils import WaffleFlag, WaffleFlagNamespace + +# Namespace +WAFFLE_NAMESPACE = u'appsembler' + +# Flags +DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP = 'disable_tpa_pipeline_socialcore_create_user' + + +def waffle(): + """ + Returns the namespaced, cached, audited Waffle class for Appsembler. + """ + return WaffleFlagNamespace(name=WAFFLE_NAMESPACE, log_prefix=u'Appsembler: ') + + +def waffle_flags(): + """ + Returns the namespaced, cached, audited Waffle flags dictionary for Appsembler. + """ + namespace = waffle() + return { + DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP: WaffleFlag( + namespace, + DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP, + flag_undefined_default=False, + ), + } + + +def disable_tpa_create_user_step(request): + """ + Returns whether use of the create_user step in the third_party_auth pipeline is disabled or not. + It does not appear to be necessary because the hidden registration form POSTs to /user_authn/ + endpoint to create a user prior this step. The step adds complexity and may not be needed or + even cause issues. Using a Waffle Flag to be able to activate selectively for production + testing. + """ + return waffle().is_flag_active(DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP) diff --git a/openedx/core/djangoapps/site_configuration/models.py b/openedx/core/djangoapps/site_configuration/models.py index a07fa1d96066..ba2b81e431d7 100644 --- a/openedx/core/djangoapps/site_configuration/models.py +++ b/openedx/core/djangoapps/site_configuration/models.py @@ -103,7 +103,6 @@ def api_adapter(self): return self._api_adapter - @beeline.traced('site_config.get_value') def get_value(self, name, default=None): """ Return Configuration value for the key specified as name argument. @@ -117,7 +116,6 @@ def get_value(self, name, default=None): Returns: Configuration value for the given key or returns `None` if configuration is not enabled. """ - beeline.add_context_field('value_name', name) if self.enabled: if self.tahoe_config_modifier: name, default = self.tahoe_config_modifier.normalize_get_value_params(name, default) @@ -128,10 +126,8 @@ def get_value(self, name, default=None): try: if self.api_adapter: # Tahoe: Use `SiteConfigAdapter` if available. - beeline.add_context_field('value_source', 'site_config_service') return self.api_adapter.get_value_of_type(self.api_adapter.TYPE_SETTING, name, default) else: - beeline.add_context_field('value_source', 'django_model') return self.site_values.get(name, default) if self.site_values else default except AttributeError as error: logger.exception(u'Invalid JSON data. \n [%s]', error) diff --git a/openedx/core/djangoapps/user_authn/cookies.py b/openedx/core/djangoapps/user_authn/cookies.py index 159084b7ee04..76572284a256 100644 --- a/openedx/core/djangoapps/user_authn/cookies.py +++ b/openedx/core/djangoapps/user_authn/cookies.py @@ -73,10 +73,14 @@ def delete_logged_in_cookies(response): HttpResponse """ for cookie_name in ALL_LOGGED_IN_COOKIE_NAMES: - response.delete_cookie( + response.set_cookie( cookie_name, + '', + max_age=0, + expires='Thu, 01 Jan 1970 00:00:00 GMT', path='/', - domain=settings.SESSION_COOKIE_DOMAIN + domain=settings.SESSION_COOKIE_DOMAIN, + secure=True if settings.HTTPS == 'on' else False ) return response diff --git a/openedx/core/djangoapps/user_authn/views/logout.py b/openedx/core/djangoapps/user_authn/views/logout.py index decf10928355..b24d026775c0 100644 --- a/openedx/core/djangoapps/user_authn/views/logout.py +++ b/openedx/core/djangoapps/user_authn/views/logout.py @@ -2,6 +2,8 @@ import re +import bleach +from urllib.parse import urlparse import six.moves.urllib.parse as parse # pylint: disable=import-error from django.conf import settings @@ -15,7 +17,6 @@ from openedx.core.djangoapps.user_authn.utils import is_safe_login_or_logout_redirect from third_party_auth import pipeline as tpa_pipeline - from openedx.core.djangoapps.appsembler.tahoe_idp import helpers as tahoe_idp_helpers @@ -60,7 +61,16 @@ def target(self): # >> /courses/course-v1:ARTS+D1+2018_T/course/ # to handle this scenario we need to encode our URL using quote_plus and then unquote it again. if target_url: - target_url = parse.unquote(parse.quote_plus(target_url)) + target_url = bleach.clean( + parse.unquote(parse.quote_plus(target_url)) + ) + parsed_url = urlparse(target_url) + valid_url_pattern = re.compile(r'^(http|https)://', re.IGNORECASE) + + # Allow URLs starting with http or https, as well as relative URLs + if parsed_url.scheme not in ('http', 'https') and not valid_url_pattern.match(target_url) and not parsed_url.path.startswith('/'): + # If the target_url doesn't start with a valid protocol or is not a relative URL, either use a default URL or raise an error + target_url = self.default_target use_target_url = target_url and is_safe_login_or_logout_redirect( redirect_to=target_url, diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py index 2ba360ab8996..d71830ce2b82 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py @@ -8,6 +8,8 @@ import ddt import mock import six +import bleach +import urllib from django.conf import settings from django.test import TestCase from django.test.utils import override_settings @@ -194,3 +196,21 @@ def test_learner_portal_logout_having_idp_logout_url(self): 'show_tpa_logout_link': True, } self.assertDictContainsSubset(expected, response.context_data) + + @ddt.data( + ('%22%3E%3Cscript%3Ealert(%27xss%27)%3C/script%3E', 'edx.org'), + ) + @ddt.unpack + def test_logout_redirect_failure_with_xss_vulnerability(self, redirect_url, host): + """ + Verify that it will block the XSS attack on edX’s LMS logout page + """ + url = '{logout_path}?redirect_url={redirect_url}'.format( + logout_path=reverse('logout'), + redirect_url=redirect_url + ) + response = self.client.get(url, HTTP_HOST=host) + expected = { + 'target': '/', + } + self.assertDictContainsSubset(expected, response.context_data) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index 9813c6804459..99d255d34214 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -14,16 +14,16 @@ django-hijack-admin==2.1.10 honeycomb-beeline==2.12.1 # Patched upstream packages +https://github.com/mitodl/edx-sga/archive/refs/tags/v0.12.0.tar.gz https://github.com/edx-solutions/xblock-google-drive/archive/589d9f51f9b.tar.gz # v0.2.0 but the repo has no tags https://github.com/appsembler/edx-ora2/archive/2.7.6-appsembler.1.tar.gz -https://github.com/appsembler/edx-sga/archive/v0.11.0.appsembler2.tar.gz https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz # Tahoe plugins and customizations django-tiers==0.2.7 fusionauth-client==1.36.0 google-cloud-storage==1.32.0 -tahoe-idp==2.0.0 +tahoe-idp==2.6.0 tahoe-sites==1.3.2 tahoe-lti==0.3.0 site-configuration-client==0.2.3 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 17f3e0db2b80..775e61c4ed32 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -181,7 +181,7 @@ pillow==7.1.2 # via -r requirements/edx/base.in, edx-enterprise, edx pkgconfig==1.5.1 # via xmlsec polib==1.1.0 # via edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/paver.txt, edx-django-utils --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # via -r requirements/edx/github.in # via -r requirements/edx/base.in +py2neo-history==3.1.2 pycontracts==1.8.12 # via -r requirements/edx/base.in, edx-user-state-client pycountry==19.8.18 # via -r requirements/edx/base.in pycparser==2.20 # via -r requirements/edx/../edx-sandbox/shared.txt, cffi diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 2da53ef6e013..616984a8598e 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -218,7 +218,7 @@ pkgconfig==1.5.1 # via -r requirements/edx/testing.txt, xmlsec pluggy==0.13.1 # via -r requirements/edx/testing.txt, diff-cover, pytest, tox polib==1.1.0 # via -r requirements/edx/testing.txt, edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/testing.txt, edx-django-utils --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # via -r requirements/edx/github.in # via -r requirements/edx/testing.txt +py2neo-history==3.1.2 py==1.8.1 # via -r requirements/edx/testing.txt, pytest, tox pycodestyle==2.6.0 # via -r requirements/edx/testing.txt, flake8 pycontracts==1.8.12 # via -r requirements/edx/testing.txt, edx-user-state-client diff --git a/requirements/edx/github.in b/requirements/edx/github.in index a0ae6f9b0ff9..9a8b2966e725 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -59,7 +59,6 @@ git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2 git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # The latest 2.0.0 release doesn't yet support Django 2.2, this commit from master does -e git+https://github.com/jsocol/django-ratelimit.git@72edbe8949fbf6699848e5847645a1998f121d46#egg=ratelimit diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 879892db16fe..ef360465d765 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -209,7 +209,9 @@ pkgconfig==1.5.1 # via -r requirements/edx/base.txt, xmlsec pluggy==0.13.1 # via -r requirements/edx/coverage.txt, diff-cover, pytest, tox polib==1.1.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/base.txt, edx-django-utils --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # via -r requirements/edx/github.in # via -r requirements/edx/base.txt +# modified from https://github.com/openedx/edx-platform/pull/33453 +# for earlier pip version in use on Juniper release +py2neo-history==3.1.2 py==1.8.1 # via pytest, tox pycodestyle==2.6.0 # via -r requirements/edx/testing.in, flake8 pycontracts==1.8.12 # via -r requirements/edx/base.txt, edx-user-state-client diff --git a/test_root/uploads/.gitignore b/test_root/uploads/.gitignore deleted file mode 100644 index 8e2c34b24a81..000000000000 --- a/test_root/uploads/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.csv -*.jpg -*.png -*.txt -*.svg -!test_icon.png diff --git a/test_root/uploads/course_complete_badges/honor.png b/test_root/uploads/course_complete_badges/honor.png deleted file mode 100644 index 7dbe80ed655a..000000000000 Binary files a/test_root/uploads/course_complete_badges/honor.png and /dev/null differ diff --git a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_120.jpg b/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_120.jpg deleted file mode 100755 index 1d19fb8ee006..000000000000 Binary files a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_120.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_30.jpg b/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_30.jpg deleted file mode 100755 index 990c1c49761e..000000000000 Binary files a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_30.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_50.jpg b/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_50.jpg deleted file mode 100755 index bda27ebd622a..000000000000 Binary files a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_50.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_500.jpg b/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_500.jpg deleted file mode 100755 index 0abc74f30c94..000000000000 Binary files a/test_root/uploads/profile-images/ebe754f90bc88acf8ec6a1d27b87f743_500.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_120.jpg b/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_120.jpg deleted file mode 100755 index 1d19fb8ee006..000000000000 Binary files a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_120.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_30.jpg b/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_30.jpg deleted file mode 100755 index 990c1c49761e..000000000000 Binary files a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_30.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_50.jpg b/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_50.jpg deleted file mode 100755 index bda27ebd622a..000000000000 Binary files a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_50.jpg and /dev/null differ diff --git a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_500.jpg b/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_500.jpg deleted file mode 100755 index 0abc74f30c94..000000000000 Binary files a/test_root/uploads/profile-images/f0d065035a5c4d32df318fbc54138765_500.jpg and /dev/null differ diff --git a/test_root/uploads/test-icon.png b/test_root/uploads/test-icon.png deleted file mode 100644 index c1c8e813fdc5..000000000000 Binary files a/test_root/uploads/test-icon.png and /dev/null differ diff --git a/themes/red-theme/lms/static/css/README.txt b/themes/red-theme/lms/static/css/README.txt deleted file mode 100644 index 5af45a252efd..000000000000 --- a/themes/red-theme/lms/static/css/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -CSS files should go in this directory. If you are using a CSS preprocessor -like Sass, you should configure it to output CSS in this directory. Running -`paver compile_sass` should do the right thing in this case. diff --git a/themes/stanford-style/lms/static/css/README.txt b/themes/stanford-style/lms/static/css/README.txt deleted file mode 100644 index 5af45a252efd..000000000000 --- a/themes/stanford-style/lms/static/css/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -CSS files should go in this directory. If you are using a CSS preprocessor -like Sass, you should configure it to output CSS in this directory. Running -`paver compile_sass` should do the right thing in this case. diff --git a/tox.ini b/tox.ini index 6007a5ab87ff..b76bb0d699b5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = studio,lms-1,lms-2,mte,common,pep8 +envlist = studio,lms-1,lms-2,mte,legacy-amc-tests,common,db-migrations,pep8 # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages @@ -29,6 +29,7 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} PYTEST_ARGS={env:PYTEST_ARGS:} + TEST_ENABLE_TIERS_APP=true passenv = BOK_CHOY_CMS_PORT BOKCHOY_HEADLESS @@ -113,7 +114,6 @@ commands = [testenv:lms-1] commands = pytest {env:PYTEST_ARGS} \ - common/djangoapps/util/tests/test_db.py::MigrationTests \ lms/tests.py \ lms/djangoapps/certificates/tests/test_webview_appsembler_changes.py \ lms/djangoapps/course_api/ \ @@ -147,6 +147,24 @@ setenv = commands = pytest {env:PYTEST_ARGS} {posargs:openedx/core/djangoapps/appsembler/multi_tenant_emails} +[testenv:db-migrations] +commands = + pytest {env:PYTEST_ARGS} \ + common/djangoapps/util/tests/test_db.py::MigrationTests + +[testenv:legacy-amc-tests] +# Keep this until all AMC-related code is gone. +setenv = + PYTHONHASHSEED=0 + TOXENV={envname} + PYTEST_ARGS={env:PYTEST_ARGS:} + TEST_TAHOE_SITES_USE_ORGS_MODELS=true + TEST_ENABLE_TIERS_APP=true +commands = + pip install https://github.com/appsembler/edx-organizations/archive/5.2.0-appsembler14.tar.gz + pytest {env:PYTEST_ARGS} {posargs:openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py} + + [testenv:pytest] commands = {posargs} @@ -155,4 +173,4 @@ commands = deps = pycodestyle==2.3.1 commands = - pycodestyle . + pycodestyle {posargs:.}