From 2e968d418ceb8fb4f89598ade30eda73a510bf4a Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 23 Sep 2021 10:34:58 -0700 Subject: [PATCH 01/46] Configure task behavior to acknowledge after execution to trigger requeuing when interrupted --- .../contentcuration/utils/celery/app.py | 26 +++++++++++++++++++ .../contentcuration/utils/celery/tasks.py | 5 ++++ 2 files changed, 31 insertions(+) diff --git a/contentcuration/contentcuration/utils/celery/app.py b/contentcuration/contentcuration/utils/celery/app.py index 5653ee4f7a..6be59ca3f3 100644 --- a/contentcuration/contentcuration/utils/celery/app.py +++ b/contentcuration/contentcuration/utils/celery/app.py @@ -1,3 +1,6 @@ +import base64 +import json + from celery import Celery from contentcuration.utils.celery.tasks import CeleryTask @@ -18,3 +21,26 @@ def on_init(self): @property def AsyncResult(self): return self._result_cls + + def get_queued_tasks(self, queue_name="celery"): + """ + Returns the list of tasks in the queue. + + Use `app.control.inspect()` to get information about tasks no longer in the queue + + :param queue_name: The queue name, defaults to the default "celery" queue + :return: dict[] + """ + decoded_tasks = [] + with self.pool.acquire(block=True) as conn: + tasks = conn.default_channel.client.lrange(queue_name, 0, -1) + + for task in tasks: + try: + j = json.loads(task) + body = json.loads(base64.b64decode(j['body'])) + decoded_tasks.append(body) + except (TypeError, json.JSONDecodeError, AttributeError): + pass + + return decoded_tasks diff --git a/contentcuration/contentcuration/utils/celery/tasks.py b/contentcuration/contentcuration/utils/celery/tasks.py index f98ae4af5b..92ae0b68fe 100644 --- a/contentcuration/contentcuration/utils/celery/tasks.py +++ b/contentcuration/contentcuration/utils/celery/tasks.py @@ -76,6 +76,11 @@ def my_task(self): # custom task option track_progress = False + # ensure our tasks are restarted if they're interrupted + acks_late = True + acks_on_failure_or_timeout = True + reject_on_worker_lost = True + _progress_tracker = None def after_return(self, status, retval, task_id, args, kwargs, einfo): From f382b192b9a151275c69b2a1159b5dd5478afb1e Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 21 Oct 2021 15:26:11 -0700 Subject: [PATCH 02/46] Re-enable gossip and mingle --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f0035c334..b69f193a7f 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ dummyusers: cd contentcuration/ && python manage.py loaddata contentcuration/fixtures/admin_user_token.json prodceleryworkers: - cd contentcuration/ && celery -A contentcuration worker -l info --concurrency=3 --task-events --without-mingle --without-gossip + cd contentcuration/ && celery -A contentcuration worker -l info --concurrency=3 --task-events prodcelerydashboard: # connect to the celery dashboard by visiting http://localhost:5555 From 95657fd61ea7403c6d468a2257e7c56583a51c6b Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 11 Jan 2022 11:50:01 -0800 Subject: [PATCH 03/46] Don't use git:// protocol for unauthenticated github access. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cb5c91e38..8676d0ccac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: -- repo: git://github.com/pre-commit/pre-commit-hooks +- repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.2.1 hooks: - id: trailing-whitespace From 4b7d721e82c3e4722805b0f1de013f8244455d34 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 31 Aug 2021 08:41:31 -0700 Subject: [PATCH 04/46] Move minio to dev requirements, update urllib3 --- contentcuration/contentcuration/apps.py | 6 ++++-- .../contentcuration/management/commands/setup.py | 6 ++++-- .../contentcuration/utils/minio_utils.py | 16 ++++++++-------- .../contentcuration/utils/storage_common.py | 10 ++++++++++ requirements-dev.in | 1 + requirements.in | 1 - requirements.txt | 7 +------ 7 files changed, 28 insertions(+), 19 deletions(-) diff --git a/contentcuration/contentcuration/apps.py b/contentcuration/contentcuration/apps.py index e448580629..f42624ce9e 100644 --- a/contentcuration/contentcuration/apps.py +++ b/contentcuration/contentcuration/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from django.conf import settings -from contentcuration.utils.minio_utils import ensure_storage_bucket_public +from contentcuration.utils.storage_common import is_gcs_backend class ContentConfig(AppConfig): @@ -10,5 +10,7 @@ class ContentConfig(AppConfig): def ready(self): # see note in the celery_signals.py file for why we import here. import contentcuration.utils.celery.signals # noqa - if settings.AWS_AUTO_CREATE_BUCKET: + + if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend(): + from contentcuration.utils.minio_utils import ensure_storage_bucket_public ensure_storage_bucket_public() diff --git a/contentcuration/contentcuration/management/commands/setup.py b/contentcuration/contentcuration/management/commands/setup.py index b966f52208..c601870da7 100644 --- a/contentcuration/contentcuration/management/commands/setup.py +++ b/contentcuration/contentcuration/management/commands/setup.py @@ -23,7 +23,7 @@ from contentcuration.utils.db_tools import create_topic from contentcuration.utils.db_tools import create_user from contentcuration.utils.files import duplicate_file -from contentcuration.utils.minio_utils import ensure_storage_bucket_public +from contentcuration.utils.storage_common import is_gcs_backend logmodule.basicConfig() logging = logmodule.getLogger(__name__) @@ -55,7 +55,9 @@ def handle(self, *args, **options): sys.exit() # create the minio bucket - ensure_storage_bucket_public() + if not is_gcs_backend(): + from contentcuration.utils.minio_utils import ensure_storage_bucket_public + ensure_storage_bucket_public() # create the cache table try: diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index bb08e06f11..07d1bfe3cb 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -13,6 +13,8 @@ from minio.error import BucketAlreadyOwnedByYou from minio.error import ResponseError +from contentcuration.utils.storage_common import is_gcs_backend + logger = logging.getLogger(__name__) @@ -42,6 +44,12 @@ def stop_minio(p): def ensure_storage_bucket_public(bucket=None, will_sleep=True): + # GCS' S3 compatibility is broken, especially in bucket operations; + # skip bucket creation there and just bug Aron to create buckets with + # public-read access for you + if is_gcs_backend(): + logging.info("Skipping storage creation on googleapis") + return # If true, sleep for 5 seconds to wait for minio to start if will_sleep: @@ -53,14 +61,6 @@ def ensure_storage_bucket_public(bucket=None, will_sleep=True): bucketname = bucket host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc - - # GCS' S3 compatibility is broken, especially in bucket operations; - # skip bucket creation there and just bug Aron to create buckets with - # public-read access for you - if "storage.googleapis.com" in host: - logging.info("Skipping storage creation on googleapis") - return - c = minio.Minio( host, access_key=settings.AWS_ACCESS_KEY_ID, diff --git a/contentcuration/contentcuration/utils/storage_common.py b/contentcuration/contentcuration/utils/storage_common.py index b41b018511..9ce747fe1e 100644 --- a/contentcuration/contentcuration/utils/storage_common.py +++ b/contentcuration/contentcuration/utils/storage_common.py @@ -1,6 +1,7 @@ import mimetypes import os from datetime import timedelta +from urllib.parse import urlparse from django.conf import settings from django.core.files.storage import default_storage @@ -19,6 +20,15 @@ class UnknownStorageBackendError(Exception): pass +def is_gcs_backend(): + """ + Determines if storage is GCS backend, which if not we can assume it is minio + :return: A bool + """ + host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc + return "storage.googleapis.com" in host + + def determine_content_type(filename): """ Guesses the content type of a filename. Returns the mimetype of a file. diff --git a/requirements-dev.in b/requirements-dev.in index bb33682a2b..45eaec50aa 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -40,3 +40,4 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi tabulate==0.8.2 fonttools flower==0.9.4 +minio==3.0.3 diff --git a/requirements.in b/requirements.in index abae5cd8a8..3be1692c70 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,6 @@ newrelic>=2.86.3.70 celery<5 redis pycountry==17.5.14 -minio==3.0.3 pathlib progressbar2==3.38.0 python-postmark==0.5.0 diff --git a/requirements.txt b/requirements.txt index 19715970f9..6d29bcaec8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,6 @@ celery==4.4.7 # via -r requirements.in certifi==2020.12.5 # via - # minio # requests # sentry-sdk cffi==1.14.5 @@ -168,8 +167,6 @@ kombu==4.6.11 # via celery le-utils==0.1.31 # via -r requirements.in -minio==3.0.3 - # via -r requirements.in newrelic==6.2.0.156 # via -r requirements.in oauth2client==4.1.3 @@ -232,7 +229,6 @@ pytz==2021.1 # django # django-postmark # google-api-core - # minio raven==6.10.0 # via -r requirements.in redis==3.5.3 @@ -278,10 +274,9 @@ typing-extensions==3.10.0.0 # via asgiref uritemplate==3.0.1 # via google-api-python-client -urllib3==1.26.4 +urllib3==1.26.5 # via # botocore - # minio # requests # sentry-sdk vine==1.3.0 From 99c04d59dffc9b0405ed42bedb87b23e4a14c9d5 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 31 Aug 2021 08:46:46 -0700 Subject: [PATCH 05/46] Update dev requirements --- requirements-dev.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0d03c993d8..ef41dbe4db 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,6 +40,7 @@ certifi==2020.12.5 # via # -c requirements.txt # geventhttpclient + # minio # requests cfgv==3.3.0 # via pre-commit @@ -194,6 +195,8 @@ mccabe==0.6.1 # via # flake8 # pylint +minio==3.0.3 + # via -r requirements-dev.in mixer==6.1.3 # via -r requirements-dev.in mock==4.0.3 @@ -301,6 +304,7 @@ pytz==2021.1 # celery # django # flower + # minio pyyaml==5.4.1 # via # aspy.yaml @@ -369,9 +373,10 @@ uritemplate==3.0.1 # -c requirements.txt # coreapi # drf-yasg -urllib3==1.26.4 +urllib3==1.26.5 # via # -c requirements.txt + # minio # requests vine==1.3.0 # via From 45632d1c5e4516a34681caf43a3048bd13216e4f Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 18 Nov 2021 05:35:25 +0530 Subject: [PATCH 06/46] chore: minio 3.0.3 -> 7.1.1 --- .../contentcuration/utils/minio_utils.py | 61 ++++++++++++------- requirements-dev.in | 2 +- requirements-dev.txt | 7 ++- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index 07d1bfe3cb..559d09b417 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -1,20 +1,20 @@ from future import standard_library + standard_library.install_aliases() import atexit import logging import multiprocessing import subprocess import time +import json from urllib.parse import urlparse import minio from django.conf import settings -from minio import policy -from minio.error import BucketAlreadyOwnedByYou -from minio.error import ResponseError from contentcuration.utils.storage_common import is_gcs_backend + logger = logging.getLogger(__name__) @@ -65,20 +65,34 @@ def ensure_storage_bucket_public(bucket=None, will_sleep=True): host, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, - secure=False + secure=False, ) + READ_ONLY_POLICY = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": "arn:aws:s3:::{bucketname}".format(bucketname=bucketname), + }, + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::{bucketname}/*".format(bucketname=bucketname), + }, + ], + } + if not c.bucket_exists(bucketname): - try: - c.make_bucket(bucketname) - except BucketAlreadyOwnedByYou: - pass + c.make_bucket(bucketname) - try: - c.set_bucket_policy(bucketname, "", policy.Policy.READ_ONLY) - logger.debug("Successfully set the bucket policy to read only!") - except ResponseError as e: - logger.warning("Error setting bucket {} to readonly: {}".format(bucket, e)) + c.set_bucket_policy( + bucketname, + json.dumps(READ_ONLY_POLICY), + ) def ensure_bucket_deleted(bucket=None): @@ -94,21 +108,24 @@ def ensure_bucket_deleted(bucket=None): # skip bucket creation there and just bug Aron to create buckets with # public-read access for you if "storage.googleapis.com" in host: - logging.info("Skipping storage deletion on googleapis; that sounds like a production bucket!") + logging.info( + "Skipping storage deletion on googleapis; that sounds like a production bucket!" + ) return minio_client = minio.Minio( host, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, - secure=False + secure=False, ) if minio_client.bucket_exists(bucketname): - try: - # We need to delete all objects first, before we can actually delete the bucket. - objs = (o.object_name for o in minio_client.list_objects(bucketname, recursive=True)) - list(minio_client.remove_objects(bucketname, objs)) # evaluate the generator, or else remove_objects won't actually execute - minio_client.remove_bucket(bucketname) - except BucketAlreadyOwnedByYou: - pass + # We need to delete all objects first, before we can actually delete the bucket. + objs = ( + o.object_name for o in minio_client.list_objects(bucketname, recursive=True) + ) + list( + minio_client.remove_objects(bucketname, objs) + ) # evaluate the generator, or else remove_objects won't actually execute + minio_client.remove_bucket(bucketname) diff --git a/requirements-dev.in b/requirements-dev.in index 45eaec50aa..ddb1356b3a 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -40,4 +40,4 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi tabulate==0.8.2 fonttools flower==0.9.4 -minio==3.0.3 +minio==7.1.1 diff --git a/requirements-dev.txt b/requirements-dev.txt index ef41dbe4db..b9cf415e9d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -195,7 +195,7 @@ mccabe==0.6.1 # via # flake8 # pylint -minio==3.0.3 +minio==7.1.1 # via -r requirements-dev.in mixer==6.1.3 # via -r requirements-dev.in @@ -304,7 +304,6 @@ pytz==2021.1 # celery # django # flower - # minio pyyaml==5.4.1 # via # aspy.yaml @@ -361,7 +360,9 @@ traitlets==4.3.3 typed-ast==1.4.3 # via astroid typing-extensions==3.10.0.0 - # via asgiref + # via + # -c requirements.txt + # asgiref ujson==4.0.2 # via # python-jsonrpc-server From 9b341111dac2d2d4c3b13273aa6759b10f9773ee Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 18 Nov 2021 06:10:16 +0530 Subject: [PATCH 07/46] chore: remove ipdb --- requirements-dev.in | 1 - requirements-dev.txt | 29 ----------------------------- 2 files changed, 30 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index ddb1356b3a..c74804745c 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,5 +1,4 @@ -c requirements.txt -ipdb python-language-server django-concurrent-test-helper==0.7.0 django-debug-panel==0.8.3 diff --git a/requirements-dev.txt b/requirements-dev.txt index b9cf415e9d..6ff55ed095 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,8 +26,6 @@ autoflake==1.4 # via -r requirements-dev.in autopep8==1.4 # via -r requirements-dev.in -backcall==0.2.0 - # via ipython billiard==3.6.4.0 # via # -c requirements.txt @@ -73,10 +71,6 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi # via -r requirements-dev.in dataclasses==0.8 # via werkzeug -decorator==5.0.9 - # via - # ipython - # traitlets distlib==0.3.1 # via virtualenv django-concurrent-test-helper==0.7.0 @@ -154,12 +148,6 @@ inflection==0.5.1 # via drf-yasg iniconfig==1.1.1 # via pytest -ipdb==0.13.7 - # via -r requirements-dev.in -ipython-genutils==0.2.0 - # via traitlets -ipython==7.16.1 - # via ipdb isort==5.8.0 # via # -r requirements-dev.in @@ -172,7 +160,6 @@ itypes==1.2.0 jedi==0.17.2 # via # -r requirements-dev.in - # ipython # python-language-server # python-lsp-server jinja2==3.0.1 @@ -218,10 +205,6 @@ parso==0.7.1 # via jedi pep517==0.10.0 # via pip-tools -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython pip-tools==6.1.0 # via -r requirements-dev.in pluggy==0.13.1 @@ -231,12 +214,8 @@ pluggy==0.13.1 # python-lsp-server pre-commit==1.15.1 # via -r requirements-dev.in -prompt-toolkit==3.0.18 - # via ipython psutil==5.8.0 # via locust -ptyprocess==0.7.0 - # via pexpect py==1.10.0 # via pytest pycodestyle==2.3.1 @@ -247,8 +226,6 @@ pyflakes==1.5.0 # via # autoflake # flake8 -pygments==2.9.0 - # via ipython pyinstrument-cext==0.2.4 # via pyinstrument pyinstrument==3.4.2 @@ -332,7 +309,6 @@ six==1.16.0 # geventhttpclient # pre-commit # python-dateutil - # traitlets # virtualenv sqlparse==0.4.1 # via @@ -348,15 +324,12 @@ text-unidecode==1.2 toml==0.10.2 # via # coverage - # ipdb # pep517 # pre-commit # pylint # pytest tornado==6.1 # via flower -traitlets==4.3.3 - # via ipython typed-ast==1.4.3 # via astroid typing-extensions==3.10.0.0 @@ -388,8 +361,6 @@ virtualenv==20.4.6 # via pre-commit watchdog==2.1.2 # via pytest-watch -wcwidth==0.2.5 - # via prompt-toolkit werkzeug==2.0.1 # via # flask From bb6e2d7955ab0a5afb31d4fdfceb0266dcecb1e2 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Sat, 20 Nov 2021 22:29:17 +0530 Subject: [PATCH 08/46] tests: use the new remove object api & cleanup unused functions --- .../contentcuration/utils/minio_utils.py | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index 559d09b417..b7d0a7d62b 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -1,10 +1,8 @@ from future import standard_library standard_library.install_aliases() -import atexit + import logging -import multiprocessing -import subprocess import time import json from urllib.parse import urlparse @@ -18,31 +16,6 @@ logger = logging.getLogger(__name__) -def start_minio(): - """ - Start a minio subprocess, controlled by another thread. - - Returns the daemonized thread controlling the minio subprocess. - """ - minio_process = multiprocessing.Process(target=_start_minio) - minio_process.start() - atexit.register(lambda: stop_minio(minio_process)) - return minio_process - - -def _start_minio(): - logger.info("Starting minio") - - subprocess.Popen( - ["run_minio.py"], - stdin=subprocess.PIPE, - ) - - -def stop_minio(p): - p.terminate() - - def ensure_storage_bucket_public(bucket=None, will_sleep=True): # GCS' S3 compatibility is broken, especially in bucket operations; # skip bucket creation there and just bug Aron to create buckets with @@ -107,7 +80,7 @@ def ensure_bucket_deleted(bucket=None): # GCS' S3 compatibility is broken, especially in bucket operations; # skip bucket creation there and just bug Aron to create buckets with # public-read access for you - if "storage.googleapis.com" in host: + if is_gcs_backend(): logging.info( "Skipping storage deletion on googleapis; that sounds like a production bucket!" ) @@ -122,10 +95,11 @@ def ensure_bucket_deleted(bucket=None): if minio_client.bucket_exists(bucketname): # We need to delete all objects first, before we can actually delete the bucket. - objs = ( + objs_name = ( o.object_name for o in minio_client.list_objects(bucketname, recursive=True) ) - list( - minio_client.remove_objects(bucketname, objs) - ) # evaluate the generator, or else remove_objects won't actually execute + + for o in objs_name: + minio_client.remove_object(bucketname, o) + minio_client.remove_bucket(bucketname) From a47396f2133ff604a8081f5a3faab18be07ed100 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Wed, 9 Mar 2022 01:19:00 +0530 Subject: [PATCH 09/46] feat: welcome postgres 12! --- .github/workflows/pythontest.yml | 2 +- README.md | 2 +- docker-compose.yml | 2 +- docs/manual_setup.md | 14 ++++++-------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 8544989bf6..096aae7269 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -24,7 +24,7 @@ jobs: # Label used to access the service container postgres: # Docker Hub image - image: postgres + image: postgres:12.10 # Provide the password for postgres env: POSTGRES_USER: learningequality diff --git a/README.md b/README.md index 4bbb1f9a81..440b29d1bd 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ To confirm that the services are running, run `docker ps`, and you should see th CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e09c5c203b93 redis:4.0.9 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:6379->6379/tcp studio_vue-refactor_redis_1 6164371efb6b minio/minio "minio server /data" 51 seconds ago Up 49 seconds 0.0.0.0:9000->9000/tcp studio_vue-refactor_minio_1 -c86bbfa3a59e postgres:9.6 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:5432->5432/tcp studio_vue-refactor_postgres_1 +c86bbfa3a59e postgres:12.10 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:5432->5432/tcp studio_vue-refactor_postgres_1 ``` diff --git a/docker-compose.yml b/docker-compose.yml index cb1cc9decb..8b2bb69209 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: - minio_data:/data postgres: - image: postgres:9.6 + image: postgres:12.10 environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: learningequality diff --git a/docs/manual_setup.md b/docs/manual_setup.md index c61e0cc082..0db63b8a65 100644 --- a/docs/manual_setup.md +++ b/docs/manual_setup.md @@ -1,5 +1,3 @@ - - # Manually installing dependencies ## Install @@ -9,7 +7,7 @@ Rather than using Docker, it is also possible to manually install the dependenci - [python (2.7)](https://www.python.org/downloads/release/python-2713/) - [python-pip](https://pip.pypa.io/en/stable/installing/) - [nodejs (10.x)](https://nodejs.org/en/download/) - - [Postgres DB](https://www.postgresql.org/download/) + - [Postgres DB (12.10)](https://www.postgresql.org/download/) - [redis](https://redis.io/topics/quickstart) - [minio server](https://www.minio.io/downloads.html) - [nginx](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/) @@ -37,7 +35,7 @@ curl -sL https://deb.nodesource.com/setup_10.x | bash - # Install packages apt-get install -y python python-pip python-dev python-tk \ - postgresql-server-dev-all postgresql-contrib postgresql-client postgresql \ + postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12.10 \ ffmpeg nodejs libmagickwand-dev nginx redis-server wkhtmltopdf ``` @@ -46,9 +44,9 @@ apt-get install -y python python-pip python-dev python-tk \ You can install the corresponding packages using Homebrew: ```bash -brew install postgresql@9.6 redis node ffmpeg imagemagick@6 gs +brew install postgresql@12.10 redis node ffmpeg imagemagick@6 gs brew install minio/stable/minio -brew link --force postgresql@9.6 +brew link --force postgresql@12.10 brew link --force imagemagick@6 ``` @@ -62,13 +60,13 @@ Windows is no longer supported due to incompatibilities with some of the require ## Set up the database -Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. +Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12.10`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. Make sure postgres is running: ```bash service postgresql start -# alternatively: pg_ctl -D /usr/local/var/postgresql@9.6 start +# alternatively: pg_ctl -D /usr/local/var/postgresql@12.10 start ``` Start the client with: From 3ce3548ec159bb40dfd06f779e024eb47764662c Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Wed, 9 Mar 2022 02:12:16 +0530 Subject: [PATCH 10/46] Pin CI services version to same as docker-compose services --- .github/workflows/pythontest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 096aae7269..71f1663235 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -42,7 +42,7 @@ jobs: # Label used to access the service container redis: # Docker Hub image - image: redis + image: redis:4.0.9 # Set health checks to wait until redis has started options: >- --health-cmd "redis-cli ping" @@ -53,7 +53,7 @@ jobs: # Maps port 6379 on service container to the host - 6379:6379 minio: - image: bitnami/minio + image: minio/minio:RELEASE.2020-06-22T03-12-50Z env: MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080 MINIO_ACCESS_KEY: development From 35e2f95802ba7a8c86e06e8b89944518ec0e70ec Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:22:57 +0530 Subject: [PATCH 11/46] fix: pin postgres major & use minio/minio on CI --- .github/workflows/pythontest.yml | 31 ++++++++++++++++++++++--------- docker-compose.yml | 2 +- docs/manual_setup.md | 12 ++++++------ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 71f1663235..6c4f1f3aa7 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -15,16 +15,18 @@ jobs: with: github_token: ${{ github.token }} paths: '["**.py", "requirements.txt", "requirements-dev.txt", ".github/workflows/pythontest.yml"]' + unit_test: name: Python unit tests needs: pre_job if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-latest + services: # Label used to access the service container postgres: # Docker Hub image - image: postgres:12.10 + image: postgres:12 # Provide the password for postgres env: POSTGRES_USER: learningequality @@ -52,16 +54,22 @@ jobs: ports: # Maps port 6379 on service container to the host - 6379:6379 - minio: - image: minio/minio:RELEASE.2020-06-22T03-12-50Z - env: - MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080 - MINIO_ACCESS_KEY: development - MINIO_SECRET_KEY: development - ports: - - 9000:9000 steps: - uses: actions/checkout@v2 + - name: Set up minio + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/minio_data:/data \ + -v /tmp/minio_config:/root/.minio \ + minio/minio server /data + + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: @@ -79,6 +87,11 @@ jobs: pip install pip-tools pip-sync requirements.txt requirements-dev.txt - name: Test pytest + env: + AWS_BUCKET_NAME: testbucket + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_S3_ENDPOINT_URL: http://127.0.0.1:9000 run: | sh -c './contentcuration/manage.py makemigrations --check' pytest diff --git a/docker-compose.yml b/docker-compose.yml index 8b2bb69209..02732ba585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: - minio_data:/data postgres: - image: postgres:12.10 + image: postgres:12 environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: learningequality diff --git a/docs/manual_setup.md b/docs/manual_setup.md index 0db63b8a65..f0e3c77cf7 100644 --- a/docs/manual_setup.md +++ b/docs/manual_setup.md @@ -7,7 +7,7 @@ Rather than using Docker, it is also possible to manually install the dependenci - [python (2.7)](https://www.python.org/downloads/release/python-2713/) - [python-pip](https://pip.pypa.io/en/stable/installing/) - [nodejs (10.x)](https://nodejs.org/en/download/) - - [Postgres DB (12.10)](https://www.postgresql.org/download/) + - [Postgres DB (12.x)](https://www.postgresql.org/download/) - [redis](https://redis.io/topics/quickstart) - [minio server](https://www.minio.io/downloads.html) - [nginx](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/) @@ -35,7 +35,7 @@ curl -sL https://deb.nodesource.com/setup_10.x | bash - # Install packages apt-get install -y python python-pip python-dev python-tk \ - postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12.10 \ + postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12 \ ffmpeg nodejs libmagickwand-dev nginx redis-server wkhtmltopdf ``` @@ -44,9 +44,9 @@ apt-get install -y python python-pip python-dev python-tk \ You can install the corresponding packages using Homebrew: ```bash -brew install postgresql@12.10 redis node ffmpeg imagemagick@6 gs +brew install postgresql@12 redis node ffmpeg imagemagick@6 gs brew install minio/stable/minio -brew link --force postgresql@12.10 +brew link --force postgresql@12 brew link --force imagemagick@6 ``` @@ -60,13 +60,13 @@ Windows is no longer supported due to incompatibilities with some of the require ## Set up the database -Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12.10`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. +Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. Make sure postgres is running: ```bash service postgresql start -# alternatively: pg_ctl -D /usr/local/var/postgresql@12.10 start +# alternatively: pg_ctl -D /usr/local/var/postgresql@12 start ``` Start the client with: From 94478d924aee91b31b5c565ed138df9281eeefca Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:28:19 +0530 Subject: [PATCH 12/46] fix: unnecessary exports removed from minio CI --- .github/workflows/pythontest.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 6c4f1f3aa7..9b26f09ba8 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -64,11 +64,6 @@ jobs: -v /tmp/minio_data:/data \ -v /tmp/minio_config:/root/.minio \ minio/minio server /data - - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 From a963f973a27fab4b6e941b2d51662b5cfc1837a1 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:31:35 +0530 Subject: [PATCH 13/46] fix: CI --- .github/workflows/pythontest.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 9b26f09ba8..966dd9613f 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -58,13 +58,16 @@ jobs: - uses: actions/checkout@v2 - name: Set up minio run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/minio_data:/data \ - -v /tmp/minio_config:/root/.minio \ - minio/minio server /data - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/minio_data:/data \ + -v /tmp/minio_config:/root/.minio \ + minio/minio server /data + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: From 5b32ef9715c68a41e370cee15d4e8fa2798931c9 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:35:55 +0530 Subject: [PATCH 14/46] fix: Blaine's elegant solution! --- .github/workflows/pythontest.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 966dd9613f..7fdf6e178c 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -59,15 +59,11 @@ jobs: - name: Set up minio run: | docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ + -e "MINIO_ACCESS_KEY=development" \ + -e "MINIO_SECRET_KEY=development" \ -v /tmp/minio_data:/data \ -v /tmp/minio_config:/root/.minio \ minio/minio server /data - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: @@ -85,11 +81,6 @@ jobs: pip install pip-tools pip-sync requirements.txt requirements-dev.txt - name: Test pytest - env: - AWS_BUCKET_NAME: testbucket - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - AWS_S3_ENDPOINT_URL: http://127.0.0.1:9000 run: | sh -c './contentcuration/manage.py makemigrations --check' pytest From 66dca07d6b08b14af1186d93b205fcb026b5147d Mon Sep 17 00:00:00 2001 From: Aron Fyodor Asor <191955+aronasorman@users.noreply.github.com> Date: Thu, 24 Mar 2022 14:50:16 -0500 Subject: [PATCH 15/46] fix: use the jspdf https url instead of git:// (#3349) * fix: use the jspdf https url instead of git:// git:// started failing recently, and no docker build can happen * fixup! fix: use the jspdf https url instead of git:// Co-authored-by: Aron Asor --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a7d64edda6..89431656f0 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "html2canvas": "^1.0.0-rc.5", "i18n-iso-countries": "^5.1.0", "jquery": "^2.2.4", - "jspdf": "git://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e", + "jspdf": "https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e", "jszip": "^2.5.0", "jszip-utils": "0.0.2", "kolibri-tools": "^0.14.5-dev.4", diff --git a/yarn.lock b/yarn.lock index 6c3e36512c..8da1fa3deb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12320,9 +12320,9 @@ jsonparse@^1.2.0: resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= -"jspdf@git://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e": +"jspdf@https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e": version "2.1.1" - resolved "git://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e" + resolved "https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e" dependencies: atob "^2.1.2" btoa "^1.2.1" From b02d45cd4177c94c56929303c51c6976f9240c71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Sep 2021 23:28:09 +0000 Subject: [PATCH 16/46] Bump pillow from 8.2.0 to 8.3.2 Bumps [pillow](https://github.com/python-pillow/Pillow) from 8.2.0 to 8.3.2. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/8.2.0...8.3.2) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.in | 2 +- requirements.txt | 57 ++++++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/requirements.in b/requirements.in index 3be1692c70..d447ac5b0b 100644 --- a/requirements.in +++ b/requirements.in @@ -41,7 +41,7 @@ sentry-sdk raven django-bulk-update html5lib==1.1 -pillow==8.2.0 +pillow==8.3.2 python-dateutil>=2.8.1 jsonschema>=3.2.0 importlib-metadata==1.7.0 diff --git a/requirements.txt b/requirements.txt index 6d29bcaec8..7666f87d96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,20 @@ chardet==4.0.0 # via requests confusable-homoglyphs==3.2.0 # via django-registration +django==3.2.4 + # via + # -r requirements.in + # django-bulk-update + # django-db-readonly + # django-filter + # django-js-reverse + # django-model-utils + # django-mptt + # django-redis + # django-registration + # django-s3-storage + # djangorestframework + # jsonfield django-bulk-update==2.2.0 # via -r requirements.in django-cte==1.1.5 @@ -68,20 +82,6 @@ django-s3-storage==0.13.4 # via -r requirements.in django-webpack-loader==0.7.0 # via -r requirements.in -django==3.2.4 - # via - # -r requirements.in - # django-bulk-update - # django-db-readonly - # django-filter - # django-js-reverse - # django-model-utils - # django-mptt - # django-redis - # django-registration - # django-s3-storage - # djangorestframework - # jsonfield djangorestframework==3.12.4 # via -r requirements.in future==0.18.2 @@ -95,10 +95,6 @@ google-api-core[grpc]==1.27.0 # google-cloud-logging google-api-python-client==2.4.0 # via -r requirements.in -google-auth-httplib2==0.1.0 - # via google-api-python-client -google-auth-oauthlib==0.4.4 - # via gspread google-auth==1.30.0 # via # google-api-core @@ -108,6 +104,10 @@ google-auth==1.30.0 # google-cloud-core # google-cloud-storage # gspread +google-auth-httplib2==0.1.0 + # via google-api-python-client +google-auth-oauthlib==0.4.4 + # via gspread google-cloud-core==1.6.0 # via # -r requirements.in @@ -151,10 +151,7 @@ httplib2==0.19.1 idna==2.10 # via requests importlib-metadata==1.7.0 - # via - # -r requirements.in - # jsonschema - # kombu + # via -r requirements.in jmespath==0.10.0 # via # boto3 @@ -177,7 +174,7 @@ packaging==20.9 # via google-api-core pathlib==1.0.1 # via -r requirements.in -pillow==8.2.0 +pillow==8.3.2 # via -r requirements.in progressbar2==3.38.0 # via -r requirements.in @@ -194,15 +191,15 @@ protobuf==3.17.0 # proto-plus psycopg2-binary==2.8.6 # via -r requirements.in -pyasn1-modules==0.2.8 - # via - # google-auth - # oauth2client pyasn1==0.4.8 # via # oauth2client # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via + # google-auth + # oauth2client pycountry==17.5.14 # via # -r requirements.in @@ -235,8 +232,6 @@ redis==3.5.3 # via # -r requirements.in # django-redis -requests-oauthlib==1.3.0 - # via google-auth-oauthlib requests==2.25.1 # via # -r requirements.in @@ -244,6 +239,8 @@ requests==2.25.1 # google-cloud-storage # gspread # requests-oauthlib +requests-oauthlib==1.3.0 + # via google-auth-oauthlib rsa==4.7.2 # via # google-auth @@ -270,8 +267,6 @@ six==1.16.0 # python-utils sqlparse==0.4.1 # via django -typing-extensions==3.10.0.0 - # via asgiref uritemplate==3.0.1 # via google-api-python-client urllib3==1.26.5 From 5f7775e89e92bc92e5ee4af6ace4c5454fcd8cab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Sep 2021 17:37:46 +0000 Subject: [PATCH 17/46] Bump django from 3.2.4 to 3.2.5 Bumps [django](https://github.com/django/django) from 3.2.4 to 3.2.5. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.2.4...3.2.5) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 128 ++++++++++++++++++++++++------------------- requirements.in | 2 +- requirements.txt | 2 +- 3 files changed, 75 insertions(+), 57 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6ff55ed095..3fba16554f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,15 +8,13 @@ amqp==2.6.1 # via # -c requirements.txt # kombu -appdirs==1.4.4 - # via virtualenv asgiref==3.3.4 # via # -c requirements.txt # django aspy.yaml==1.3.0 # via pre-commit -astroid==2.5.6 +astroid==2.11.5 # via pylint attrs==19.3.0 # via @@ -30,6 +28,8 @@ billiard==3.6.4.0 # via # -c requirements.txt # celery +brotli==1.0.9 + # via geventhttpclient celery==4.4.7 # via # -c requirements.txt @@ -40,21 +40,21 @@ certifi==2020.12.5 # geventhttpclient # minio # requests -cfgv==3.3.0 +cfgv==3.3.1 # via pre-commit chardet==4.0.0 # via # -c requirements.txt # requests -click==8.0.0 +click==8.0.4 # via # flask # pip-tools -codecov==2.1.11 +codecov==2.1.12 # via -r requirements-dev.in colorama==0.4.4 # via pytest-watch -configargparse==1.4.1 +configargparse==1.5.3 # via locust coreapi==2.3.3 # via drf-yasg @@ -62,7 +62,7 @@ coreschema==0.0.4 # via # coreapi # drf-yasg -coverage[toml]==5.5 +coverage[toml]==6.2 # via # -r requirements-dev.in # codecov @@ -71,7 +71,9 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi # via -r requirements-dev.in dataclasses==0.8 # via werkzeug -distlib==0.3.1 +dill==0.3.4 + # via pylint +distlib==0.3.4 # via virtualenv django-concurrent-test-helper==0.7.0 # via -r requirements-dev.in @@ -81,7 +83,7 @@ django-debug-toolbar==1.9.1 # via # -r requirements-dev.in # django-debug-panel -django==3.2.4 +django==3.2.5 # via # -c requirements.txt # django-debug-toolbar @@ -99,31 +101,34 @@ faker==0.9.1 # via # -r requirements-dev.in # mixer -filelock==3.0.12 +filelock==3.4.1 # via virtualenv flake8==3.4.1 # via -r requirements-dev.in flask-basicauth==0.2.0 # via locust -flask==1.1.2 +flask-cors==3.0.10 + # via locust +flask==2.0.3 # via # flask-basicauth + # flask-cors # locust flower==0.9.4 # via -r requirements-dev.in -fonttools==4.24.0 +fonttools==4.27.1 # via -r requirements-dev.in -gevent==21.1.2 +gevent==21.12.0 # via # geventhttpclient # locust -geventhttpclient==1.4.4 +geventhttpclient==1.5.3 # via locust -greenlet==1.1.0 +greenlet==1.1.2 # via gevent humanize==0.5.1 # via flower -identify==2.2.4 +identify==2.4.4 # via pre-commit idna==2.10 # via @@ -132,13 +137,14 @@ idna==2.10 importlib-metadata==1.7.0 # via # -c requirements.txt + # click # kombu # pep517 # pluggy # pre-commit # pytest # virtualenv -importlib-resources==5.2.0 +importlib-resources==5.4.0 # via # pre-commit # virtualenv @@ -148,7 +154,7 @@ inflection==0.5.1 # via drf-yasg iniconfig==1.1.1 # via pytest -isort==5.8.0 +isort==5.10.1 # via # -r requirements-dev.in # pylint @@ -162,19 +168,20 @@ jedi==0.17.2 # -r requirements-dev.in # python-language-server # python-lsp-server -jinja2==3.0.1 +jinja2==3.0.3 # via # coreschema # flask + # locust json-rpc==1.13.0 # via -r requirements-dev.in kombu==4.6.11 # via # -c requirements.txt # celery -lazy-object-proxy==1.6.0 +lazy-object-proxy==1.7.1 # via astroid -locust==1.5.3 +locust==2.8.6 # via -r requirements-dev.in markupsafe==2.0.1 # via jinja2 @@ -190,7 +197,7 @@ mock==4.0.3 # via # -r requirements-dev.in # django-concurrent-test-helper -msgpack==1.0.2 +msgpack==1.0.4 # via locust nodeenv==1.6.0 # via @@ -203,20 +210,24 @@ packaging==20.9 # pytest parso==0.7.1 # via jedi -pep517==0.10.0 +pep517==0.12.0 # via pip-tools -pip-tools==6.1.0 +pip-tools==6.4.0 # via -r requirements-dev.in -pluggy==0.13.1 +platformdirs==2.4.0 + # via + # pylint + # virtualenv +pluggy==1.0.0 # via # pytest # python-language-server # python-lsp-server pre-commit==1.15.1 # via -r requirements-dev.in -psutil==5.8.0 +psutil==5.9.1 # via locust -py==1.10.0 +py==1.11.0 # via pytest pycodestyle==2.3.1 # via @@ -230,31 +241,31 @@ pyinstrument-cext==0.2.4 # via pyinstrument pyinstrument==3.4.2 # via -r requirements-dev.in -pylint==2.8.2 +pylint==2.13.9 # via -r requirements-dev.in pyls-isort==0.2.2 # via -r requirements-dev.in -pympler==0.9 +pympler==1.0.1 # via -r requirements-dev.in -pypandoc==1.5 +pypandoc==1.8.1 # via -r requirements-dev.in pyparsing==2.4.7 # via # -c requirements.txt # packaging -pytest-cov==2.12.0 +pytest-cov==3.0.0 # via -r requirements-dev.in -pytest-django==4.3.0 +pytest-django==4.5.2 # via -r requirements-dev.in pytest-logging==2015.11.4 # via -r requirements-dev.in -pytest-pythonpath==0.7.3 +pytest-pythonpath==0.7.4 # via -r requirements-dev.in -pytest-timeout==1.4.2 +pytest-timeout==2.1.0 # via -r requirements-dev.in pytest-watch==4.2.0 # via -r requirements-dev.in -pytest==6.2.4 +pytest==6.2.5 # via # -r requirements-dev.in # pytest-cov @@ -273,7 +284,7 @@ python-language-server==0.36.2 # via -r requirements-dev.in python-lsp-jsonrpc==1.0.0 # via python-lsp-server -python-lsp-server==1.0.1 +python-lsp-server==1.3.3 # via pyls-isort pytz==2021.1 # via @@ -281,11 +292,11 @@ pytz==2021.1 # celery # django # flower -pyyaml==5.4.1 +pyyaml==6.0 # via # aspy.yaml # pre-commit -pyzmq==22.0.3 +pyzmq==23.1.0 # via locust requests==2.25.1 # via @@ -293,11 +304,13 @@ requests==2.25.1 # codecov # coreapi # locust -rope==0.19.0 +rope==1.1.1 # via -r requirements-dev.in -ruamel.yaml.clib==0.2.2 +roundrobin==0.0.2 + # via locust +ruamel.yaml.clib==0.2.6 # via ruamel.yaml -ruamel.yaml==0.17.4 +ruamel.yaml==0.17.21 # via drf-yasg service-factory==0.1.6 # via -r requirements-dev.in @@ -306,6 +319,7 @@ six==1.16.0 # -c requirements.txt # django-concurrent-test-helper # faker + # flask-cors # geventhttpclient # pre-commit # python-dateutil @@ -322,21 +336,25 @@ tblib==1.7.0 text-unidecode==1.2 # via faker toml==0.10.2 + # via + # pre-commit + # pytest +tomli==1.2.3 # via # coverage # pep517 - # pre-commit # pylint - # pytest tornado==6.1 # via flower -typed-ast==1.4.3 +typed-ast==1.5.4 # via astroid -typing-extensions==3.10.0.0 +typing-extensions==4.1.1 # via - # -c requirements.txt # asgiref -ujson==4.0.2 + # astroid + # locust + # pylint +ujson==4.3.0 # via # python-jsonrpc-server # python-language-server @@ -357,19 +375,19 @@ vine==1.3.0 # -c requirements.txt # amqp # celery -virtualenv==20.4.6 +virtualenv==20.14.1 # via pre-commit -watchdog==2.1.2 +watchdog==2.1.8 # via pytest-watch -werkzeug==2.0.1 +werkzeug==2.0.3 # via # flask # locust -wheel==0.36.2 - # via pypandoc -wrapt==1.12.1 +wheel==0.37.1 + # via pip-tools +wrapt==1.14.1 # via astroid -yapf==0.31.0 +yapf==0.32.0 # via -r requirements-dev.in zipp==3.4.1 # via diff --git a/requirements.in b/requirements.in index d447ac5b0b..e83cf286cc 100644 --- a/requirements.in +++ b/requirements.in @@ -17,7 +17,7 @@ pycountry==17.5.14 pathlib progressbar2==3.38.0 python-postmark==0.5.0 -Django==3.2.4 +Django==3.2.5 django-webpack-loader==0.7.0 google-cloud-error-reporting google-cloud-storage diff --git a/requirements.txt b/requirements.txt index 7666f87d96..8241c95224 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ chardet==4.0.0 # via requests confusable-homoglyphs==3.2.0 # via django-registration -django==3.2.4 +django==3.2.5 # via # -r requirements.in # django-bulk-update From 76cdf6da34b4b7a0a094d4599026f3c0d56060da Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Wed, 2 Feb 2022 08:58:30 -0800 Subject: [PATCH 18/46] Update le-utils with release containing new completion criteria schema --- requirements-dev.txt | 3 +- requirements-docs.txt | 121 ++++++++++++++++++++++++++++++++---------- requirements.in | 2 +- requirements.txt | 61 +++++++++++---------- 4 files changed, 127 insertions(+), 60 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3fba16554f..49dd229158 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -348,8 +348,9 @@ tornado==6.1 # via flower typed-ast==1.5.4 # via astroid -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via + # -c requirements.txt # asgiref # astroid # locust diff --git a/requirements-docs.txt b/requirements-docs.txt index 221ebc84f4..56f0caf86d 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,35 +4,98 @@ # # pip-compile requirements-docs.in # -alabaster==0.7.12 # via sphinx -argh==0.26.2 # via sphinx-autobuild -babel==2.8.0 # via sphinx, sphinx-intl -certifi==2019.11.28 # via -c requirements.txt, requests -chardet==3.0.4 # via -c requirements.txt, requests -click==7.1.2 # via sphinx-intl -docutils==0.15.2 # via -c requirements.txt, sphinx -idna==2.8 # via -c requirements.txt, requests -imagesize==1.2.0 # via sphinx -jinja2==2.11.3 # via sphinx -livereload==2.6.3 # via sphinx-autobuild -markupsafe==1.1.1 # via jinja2 -pathtools==0.1.2 # via sphinx-autobuild, watchdog -port_for==0.3.1 # via sphinx-autobuild -pygments==2.7.4 # via sphinx -pytz==2019.3 # via -c requirements.txt, babel -pyyaml==5.4 # via sphinx-autobuild -requests==2.22.0 # via -c requirements.txt, sphinx -six==1.14.0 # via -c requirements.txt, livereload, sphinx -snowballstemmer==2.0.0 # via sphinx -sphinx-autobuild==0.7.1 # via -r requirements-docs.in -sphinx-intl==2.0.1 # via -r requirements-docs.in -sphinx-rtd-theme==0.5.0 # via -r requirements-docs.in -sphinx==1.6.4 # via -r requirements-docs.in, sphinx-intl, sphinx-rtd-theme -sphinxcontrib-serializinghtml==1.1.4 # via sphinxcontrib-websupport -sphinxcontrib-websupport==1.2.4 # via -r requirements-docs.in, sphinx -tornado==6.0.4 # via livereload, sphinx-autobuild -urllib3==1.25.8 # via -c requirements.txt, requests -watchdog==0.10.3 # via sphinx-autobuild +alabaster==0.7.12 + # via sphinx +argh==0.26.2 + # via sphinx-autobuild +babel==2.10.1 + # via + # sphinx + # sphinx-intl +certifi==2020.12.5 + # via + # -c requirements.txt + # requests +chardet==4.0.0 + # via + # -c requirements.txt + # requests +click==8.0.4 + # via sphinx-intl +docutils==0.17.1 + # via + # sphinx + # sphinx-rtd-theme +idna==2.10 + # via + # -c requirements.txt + # requests +imagesize==1.3.0 + # via sphinx +importlib-metadata==1.7.0 + # via + # -c requirements.txt + # click +jinja2==3.0.3 + # via sphinx +livereload==2.6.3 + # via sphinx-autobuild +markupsafe==2.0.1 + # via jinja2 +pathtools==0.1.2 + # via sphinx-autobuild +port_for==0.3.1 + # via sphinx-autobuild +pygments==2.12.0 + # via sphinx +pytz==2021.1 + # via + # -c requirements.txt + # babel +pyyaml==6.0 + # via sphinx-autobuild +requests==2.25.1 + # via + # -c requirements.txt + # sphinx +six==1.16.0 + # via + # -c requirements.txt + # livereload + # sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx-autobuild==0.7.1 + # via -r requirements-docs.in +sphinx-intl==2.0.1 + # via -r requirements-docs.in +sphinx-rtd-theme==1.0.0 + # via -r requirements-docs.in +sphinx==1.6.4 + # via + # -r requirements-docs.in + # sphinx-intl + # sphinx-rtd-theme +sphinxcontrib-serializinghtml==1.1.5 + # via sphinxcontrib-websupport +sphinxcontrib-websupport==1.2.4 + # via + # -r requirements-docs.in + # sphinx +tornado==6.1 + # via + # livereload + # sphinx-autobuild +urllib3==1.26.5 + # via + # -c requirements.txt + # requests +watchdog==2.1.8 + # via sphinx-autobuild +zipp==3.4.1 + # via + # -c requirements.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.in b/requirements.in index e83cf286cc..3f5d0f23a8 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ djangorestframework==3.12.4 psycopg2-binary==2.8.6 django-js-reverse==0.9.1 django-registration==3.1.2 -le-utils==0.1.31 +le-utils==0.1.39 gunicorn==19.6.0 django-postmark==0.1.6 jsonfield==3.1.0 diff --git a/requirements.txt b/requirements.txt index 8241c95224..fce8e1d401 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,20 +38,6 @@ chardet==4.0.0 # via requests confusable-homoglyphs==3.2.0 # via django-registration -django==3.2.5 - # via - # -r requirements.in - # django-bulk-update - # django-db-readonly - # django-filter - # django-js-reverse - # django-model-utils - # django-mptt - # django-redis - # django-registration - # django-s3-storage - # djangorestframework - # jsonfield django-bulk-update==2.2.0 # via -r requirements.in django-cte==1.1.5 @@ -82,6 +68,20 @@ django-s3-storage==0.13.4 # via -r requirements.in django-webpack-loader==0.7.0 # via -r requirements.in +django==3.2.5 + # via + # -r requirements.in + # django-bulk-update + # django-db-readonly + # django-filter + # django-js-reverse + # django-model-utils + # django-mptt + # django-redis + # django-registration + # django-s3-storage + # djangorestframework + # jsonfield djangorestframework==3.12.4 # via -r requirements.in future==0.18.2 @@ -95,6 +95,10 @@ google-api-core[grpc]==1.27.0 # google-cloud-logging google-api-python-client==2.4.0 # via -r requirements.in +google-auth-httplib2==0.1.0 + # via google-api-python-client +google-auth-oauthlib==0.4.4 + # via gspread google-auth==1.30.0 # via # google-api-core @@ -104,10 +108,6 @@ google-auth==1.30.0 # google-cloud-core # google-cloud-storage # gspread -google-auth-httplib2==0.1.0 - # via google-api-python-client -google-auth-oauthlib==0.4.4 - # via gspread google-cloud-core==1.6.0 # via # -r requirements.in @@ -151,7 +151,10 @@ httplib2==0.19.1 idna==2.10 # via requests importlib-metadata==1.7.0 - # via -r requirements.in + # via + # -r requirements.in + # jsonschema + # kombu jmespath==0.10.0 # via # boto3 @@ -162,7 +165,7 @@ jsonschema==3.2.0 # via -r requirements.in kombu==4.6.11 # via celery -le-utils==0.1.31 +le-utils==0.1.39 # via -r requirements.in newrelic==6.2.0.156 # via -r requirements.in @@ -191,19 +194,17 @@ protobuf==3.17.0 # proto-plus psycopg2-binary==2.8.6 # via -r requirements.in +pyasn1-modules==0.2.8 + # via + # google-auth + # oauth2client pyasn1==0.4.8 # via # oauth2client # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via - # google-auth - # oauth2client pycountry==17.5.14 - # via - # -r requirements.in - # le-utils + # via -r requirements.in pycparser==2.20 # via cffi pyparsing==2.4.7 @@ -232,6 +233,8 @@ redis==3.5.3 # via # -r requirements.in # django-redis +requests-oauthlib==1.3.0 + # via google-auth-oauthlib requests==2.25.1 # via # -r requirements.in @@ -239,8 +242,6 @@ requests==2.25.1 # google-cloud-storage # gspread # requests-oauthlib -requests-oauthlib==1.3.0 - # via google-auth-oauthlib rsa==4.7.2 # via # google-auth @@ -267,6 +268,8 @@ six==1.16.0 # python-utils sqlparse==0.4.1 # via django +typing-extensions==4.0.1 + # via asgiref uritemplate==3.0.1 # via google-api-python-client urllib3==1.26.5 From 5341ea4f7f92bdf43e936954f7be493c446a3ae0 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 29 Mar 2022 13:27:53 -0700 Subject: [PATCH 19/46] Update dependencies --- package.json | 3 +-- requirements.in | 2 +- requirements.txt | 2 +- yarn.lock | 17 +++++------------ 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 89431656f0..46276b3a80 100644 --- a/package.json +++ b/package.json @@ -80,8 +80,7 @@ "i18n-iso-countries": "^5.1.0", "jquery": "^2.2.4", "jspdf": "https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e", - "jszip": "^2.5.0", - "jszip-utils": "0.0.2", + "kolibri-constants": "^0.1.40", "kolibri-tools": "^0.14.5-dev.4", "lodash": "^4.17.21", "marked": "^0.3.5", diff --git a/requirements.in b/requirements.in index 3f5d0f23a8..8b2146970e 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ djangorestframework==3.12.4 psycopg2-binary==2.8.6 django-js-reverse==0.9.1 django-registration==3.1.2 -le-utils==0.1.39 +le-utils==0.1.40 gunicorn==19.6.0 django-postmark==0.1.6 jsonfield==3.1.0 diff --git a/requirements.txt b/requirements.txt index fce8e1d401..966cd183be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -165,7 +165,7 @@ jsonschema==3.2.0 # via -r requirements.in kombu==4.6.11 # via celery -le-utils==0.1.39 +le-utils==0.1.40 # via -r requirements.in newrelic==6.2.0.156 # via -r requirements.in diff --git a/yarn.lock b/yarn.lock index 8da1fa3deb..26dcc3b0c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12343,18 +12343,6 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" -jszip-utils@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/jszip-utils/-/jszip-utils-0.0.2.tgz#457d5cbca60a1c2e0706e9da2b544e8e7bc50bf8" - integrity sha1-RX1cvKYKHC4HBunaK1ROjnvFC/g= - -jszip@^2.5.0: - version "2.6.1" - resolved "https://registry.yarnpkg.com/jszip/-/jszip-2.6.1.tgz#b88f3a7b2e67a2a048152982c7a3756d9c4828f0" - integrity sha1-uI86ey5noqBIFSmCx6N1bZxIKPA= - dependencies: - pako "~1.0.2" - jszip@^3.4.0: version "3.5.0" resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.5.0.tgz#b4fd1f368245346658e781fec9675802489e15f6" @@ -12443,6 +12431,11 @@ knuth-shuffle-seeded@^1.0.6: dependencies: seed-random "~2.2.0" +kolibri-constants@^0.1.40: + version "0.1.40" + resolved "https://registry.yarnpkg.com/kolibri-constants/-/kolibri-constants-0.1.40.tgz#ea75e7ce9716a027638c4ac6db73867bc04d1858" + integrity sha512-264LtFuLoQ15GbZKxWnMcl+uDovzj75GWopnW40ixAVMERrmhSIGuSl1wID9IVOyT788xLQCPhcIqOP5LmDA9A== + "kolibri-design-system@git+https://github.com/learningequality/kolibri-design-system#v0.2.x": version "0.2.2-beta4" resolved "git+https://github.com/learningequality/kolibri-design-system#3288f31a56e72f257ab6cd0b9def97f323637e53" From 8e868d6a68d677e6f47ff2c65af28a609e5331c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 16:58:16 +0000 Subject: [PATCH 20/46] Bump django from 3.2.5 to 3.2.13 Bumps [django](https://github.com/django/django) from 3.2.5 to 3.2.13. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.2.5...3.2.13) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 5 ++-- requirements.in | 2 +- requirements.txt | 55 ++++++++++++++++++++------------------------ 3 files changed, 28 insertions(+), 34 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 49dd229158..d75192e42b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -83,7 +83,7 @@ django-debug-toolbar==1.9.1 # via # -r requirements-dev.in # django-debug-panel -django==3.2.5 +django==3.2.13 # via # -c requirements.txt # django-debug-toolbar @@ -348,9 +348,8 @@ tornado==6.1 # via flower typed-ast==1.5.4 # via astroid -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # -c requirements.txt # asgiref # astroid # locust diff --git a/requirements.in b/requirements.in index 8b2146970e..b94702b943 100644 --- a/requirements.in +++ b/requirements.in @@ -17,7 +17,7 @@ pycountry==17.5.14 pathlib progressbar2==3.38.0 python-postmark==0.5.0 -Django==3.2.5 +Django==3.2.13 django-webpack-loader==0.7.0 google-cloud-error-reporting google-cloud-storage diff --git a/requirements.txt b/requirements.txt index 966cd183be..168aed79a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,20 @@ chardet==4.0.0 # via requests confusable-homoglyphs==3.2.0 # via django-registration +django==3.2.13 + # via + # -r requirements.in + # django-bulk-update + # django-db-readonly + # django-filter + # django-js-reverse + # django-model-utils + # django-mptt + # django-redis + # django-registration + # django-s3-storage + # djangorestframework + # jsonfield django-bulk-update==2.2.0 # via -r requirements.in django-cte==1.1.5 @@ -68,20 +82,6 @@ django-s3-storage==0.13.4 # via -r requirements.in django-webpack-loader==0.7.0 # via -r requirements.in -django==3.2.5 - # via - # -r requirements.in - # django-bulk-update - # django-db-readonly - # django-filter - # django-js-reverse - # django-model-utils - # django-mptt - # django-redis - # django-registration - # django-s3-storage - # djangorestframework - # jsonfield djangorestframework==3.12.4 # via -r requirements.in future==0.18.2 @@ -95,10 +95,6 @@ google-api-core[grpc]==1.27.0 # google-cloud-logging google-api-python-client==2.4.0 # via -r requirements.in -google-auth-httplib2==0.1.0 - # via google-api-python-client -google-auth-oauthlib==0.4.4 - # via gspread google-auth==1.30.0 # via # google-api-core @@ -108,6 +104,10 @@ google-auth==1.30.0 # google-cloud-core # google-cloud-storage # gspread +google-auth-httplib2==0.1.0 + # via google-api-python-client +google-auth-oauthlib==0.4.4 + # via gspread google-cloud-core==1.6.0 # via # -r requirements.in @@ -151,10 +151,7 @@ httplib2==0.19.1 idna==2.10 # via requests importlib-metadata==1.7.0 - # via - # -r requirements.in - # jsonschema - # kombu + # via -r requirements.in jmespath==0.10.0 # via # boto3 @@ -194,15 +191,15 @@ protobuf==3.17.0 # proto-plus psycopg2-binary==2.8.6 # via -r requirements.in -pyasn1-modules==0.2.8 - # via - # google-auth - # oauth2client pyasn1==0.4.8 # via # oauth2client # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via + # google-auth + # oauth2client pycountry==17.5.14 # via -r requirements.in pycparser==2.20 @@ -233,8 +230,6 @@ redis==3.5.3 # via # -r requirements.in # django-redis -requests-oauthlib==1.3.0 - # via google-auth-oauthlib requests==2.25.1 # via # -r requirements.in @@ -242,6 +237,8 @@ requests==2.25.1 # google-cloud-storage # gspread # requests-oauthlib +requests-oauthlib==1.3.0 + # via google-auth-oauthlib rsa==4.7.2 # via # google-auth @@ -268,8 +265,6 @@ six==1.16.0 # python-utils sqlparse==0.4.1 # via django -typing-extensions==4.0.1 - # via asgiref uritemplate==3.0.1 # via google-api-python-client urllib3==1.26.5 From 1fa2d282ec650a57e260fd55d79868a036036558 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Wed, 7 Jul 2021 17:15:17 -0700 Subject: [PATCH 21/46] Delete unused Exercise model. --- contentcuration/contentcuration/admin.py | 5 +++-- .../migrations/0132_auto_20210708_0011.py | 16 ++++++++++++++++ contentcuration/contentcuration/models.py | 5 ----- 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 contentcuration/contentcuration/migrations/0132_auto_20210708_0011.py diff --git a/contentcuration/contentcuration/admin.py b/contentcuration/contentcuration/admin.py index 713714713d..37b76372c9 100644 --- a/contentcuration/contentcuration/admin.py +++ b/contentcuration/contentcuration/admin.py @@ -1,8 +1,9 @@ from django.contrib import admin -from contentcuration.models import Exercise, AssessmentItem, License, User +from contentcuration.models import AssessmentItem +from contentcuration.models import License +from contentcuration.models import User -admin.site.register(Exercise) admin.site.register(AssessmentItem) admin.site.register(License) diff --git a/contentcuration/contentcuration/migrations/0132_auto_20210708_0011.py b/contentcuration/contentcuration/migrations/0132_auto_20210708_0011.py new file mode 100644 index 0000000000..16c715d29e --- /dev/null +++ b/contentcuration/contentcuration/migrations/0132_auto_20210708_0011.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.4 on 2021-07-08 00:11 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0131_auto_20210707_2326'), + ] + + operations = [ + migrations.DeleteModel( + name='Exercise', + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index ea8fae573b..5487d46b46 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2163,11 +2163,6 @@ def save(self, *args, **kwargs): super(RelatedContentRelationship, self).save(*args, **kwargs) -class Exercise(models.Model): - contentnode = models.ForeignKey('ContentNode', related_name="exercise", null=True, on_delete=models.CASCADE) - mastery_model = models.CharField(max_length=200, default=exercises.DO_ALL, choices=exercises.MASTERY_MODELS) - - class Invitation(models.Model): """ Invitation to edit channel """ id = UUIDField(primary_key=True, default=uuid.uuid4) From 75486ce712581053fc8e83ccf889c05531bc955f Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Wed, 7 Jul 2021 16:28:03 -0700 Subject: [PATCH 22/46] Delete unused permissions, tasks, and models. --- .../management/commands/loadconstants.py | 7 +-- .../migrations/0131_auto_20210707_2326.py | 16 +++++ contentcuration/contentcuration/models.py | 32 +--------- .../contentcuration/permissions.py | 58 ------------------- contentcuration/contentcuration/settings.py | 1 - contentcuration/contentcuration/tasks.py | 16 ----- 6 files changed, 19 insertions(+), 111 deletions(-) create mode 100644 contentcuration/contentcuration/migrations/0131_auto_20210707_2326.py delete mode 100644 contentcuration/contentcuration/permissions.py diff --git a/contentcuration/contentcuration/management/commands/loadconstants.py b/contentcuration/contentcuration/management/commands/loadconstants.py index 46de83a0ce..d11bee7108 100644 --- a/contentcuration/contentcuration/management/commands/loadconstants.py +++ b/contentcuration/contentcuration/management/commands/loadconstants.py @@ -1,9 +1,6 @@ -from future import standard_library - -standard_library.install_aliases() -from builtins import str -from builtins import object import logging as logmodule +from builtins import object +from builtins import str from django.conf import settings from django.contrib.sites.models import Site diff --git a/contentcuration/contentcuration/migrations/0131_auto_20210707_2326.py b/contentcuration/contentcuration/migrations/0131_auto_20210707_2326.py new file mode 100644 index 0000000000..b27a9f14f0 --- /dev/null +++ b/contentcuration/contentcuration/migrations/0131_auto_20210707_2326.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.4 on 2021-07-07 23:26 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0130_auto_20210706_2005'), + ] + + operations = [ + migrations.DeleteModel( + name='ChannelResourceSize', + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 5487d46b46..2d35b33ab8 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -20,7 +20,6 @@ from django.core.files.storage import default_storage from django.core.files.storage import FileSystemStorage from django.core.mail import send_mail -from django.db import connection from django.db import IntegrityError from django.db import models from django.db.models import Count @@ -35,8 +34,8 @@ from django.db.models import Sum from django.db.models import UUIDField as DjangoUUIDField from django.db.models import Value -from django.db.models.expressions import RawSQL from django.db.models.expressions import ExpressionList +from django.db.models.expressions import RawSQL from django.db.models.functions import Cast from django.db.models.functions import Lower from django.db.models.indexes import IndexExpression @@ -547,35 +546,6 @@ def _save(self, name, content): return super(FileOnDiskStorage, self)._save(name, content) -class ChannelResourceSize(models.Model): - tree_id = models.IntegerField() - resource_size = models.IntegerField() - - pg_view_name = "contentcuration_channel_resource_sizes" - file_table = "contentcuration_file" - node_table = "contentcuration_contentnode" - - @classmethod - def initialize_view(cls): - sql = 'CREATE MATERIALIZED VIEW {view} AS ' \ - 'SELECT tree_id as id, tree_id, SUM("{file_table}"."file_size") AS ' \ - '"resource_size" FROM "{node}" LEFT OUTER JOIN "{file_table}" ON ' \ - '("{node}"."id" = "{file_table}"."contentnode_id") GROUP BY {node}.tree_id' \ - ' WITH DATA;'.format(view=cls.pg_view_name, file_table=cls.file_table, node=cls.node_table) - with connection.cursor() as cursor: - cursor.execute(sql) - - @classmethod - def refresh_view(cls): - sql = "REFRESH MATERIALIZED VIEW {}".format(cls.pg_view_name) - with connection.cursor() as cursor: - cursor.execute(sql) - - class Meta: - managed = False - db_table = "contentcuration_channel_resource_sizes" - - class SecretToken(models.Model): """Tokens for channels""" token = models.CharField(max_length=100, unique=True) diff --git a/contentcuration/contentcuration/permissions.py b/contentcuration/contentcuration/permissions.py deleted file mode 100644 index 2ced0097a7..0000000000 --- a/contentcuration/contentcuration/permissions.py +++ /dev/null @@ -1,58 +0,0 @@ -from django.core.exceptions import PermissionDenied -from past.builtins import basestring -from rest_framework import permissions - -from contentcuration.models import AssessmentItem -from contentcuration.models import Channel -from contentcuration.models import ChannelSet -from contentcuration.models import ContentNode -from contentcuration.models import ContentTag -from contentcuration.models import File -from contentcuration.models import Invitation -from contentcuration.models import User - - -def user_can_edit(user, channel): - if isinstance(channel, int) or isinstance(channel, basestring): - channel = Channel.objects.filter(pk=channel).first() - if not user.is_admin and channel and not channel.editors.filter(pk=user.pk).exists(): - raise PermissionDenied("Cannot edit content") - return True - - -def user_can_view(user, channel): - if isinstance(channel, int) or isinstance(channel, basestring): - channel = Channel.objects.filter(pk=channel).first() - if not user.is_admin and channel and not channel.editors.filter(pk=user.pk).exists() and not channel.viewers.filter(pk=user.pk).exists(): - raise PermissionDenied("Cannot view content") - return True - - -class CustomPermission(permissions.BasePermission): - - def has_object_permission(self, request, view, obj): # noqa: C901 - if request.method in permissions.SAFE_METHODS or request.user.is_admin: - return True - if isinstance(obj, User) and obj.pk == request.user.pk: - return True - if isinstance(obj, Invitation): - if obj.channel.pending_editors.filter(pk=obj.pk).exists() or \ - obj.channel.pending_editors.filter(pk=obj.pk).exists() or \ - user_can_view(request.user, obj.channel): - return True - elif isinstance(obj, ChannelSet): - return request.user.is_admin or obj.editors.filter(pk=request.user.pk).exists() - elif isinstance(obj, Channel): - if user_can_edit(request.user, obj): - return True - elif isinstance(obj, ContentNode): - if user_can_edit(request.user, obj.get_channel()): - return True - elif isinstance(obj, ContentTag): - if user_can_edit(request.user, obj.channel): - return True - elif isinstance(obj, File) or isinstance(obj, AssessmentItem): - if user_can_edit(request.user, obj.contentnode and obj.contentnode.get_channel()): - return True - - raise PermissionDenied("Cannot edit models without editing permissions") diff --git a/contentcuration/contentcuration/settings.py b/contentcuration/contentcuration/settings.py index 134b8d907b..1434ae9877 100644 --- a/contentcuration/contentcuration/settings.py +++ b/contentcuration/contentcuration/settings.py @@ -170,7 +170,6 @@ REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': ( 'rest_framework.permissions.IsAuthenticated', - 'contentcuration.permissions.CustomPermission', ), 'DEFAULT_AUTHENTICATION_CLASSES': ( 'rest_framework.authentication.SessionAuthentication', diff --git a/contentcuration/contentcuration/tasks.py b/contentcuration/contentcuration/tasks.py index e6dbe1530c..dbee8569f2 100644 --- a/contentcuration/contentcuration/tasks.py +++ b/contentcuration/contentcuration/tasks.py @@ -20,7 +20,6 @@ from contentcuration.models import ContentNode from contentcuration.models import Task from contentcuration.models import User -from contentcuration.utils.csv_writer import write_channel_csv_file from contentcuration.utils.csv_writer import write_user_csv from contentcuration.utils.nodes import calculate_resource_size from contentcuration.utils.nodes import generate_diff @@ -194,21 +193,6 @@ def sync_channel_task( ) -@app.task(name="generatechannelcsv_task") -def generatechannelcsv_task(channel_id, domain, user_id): - channel = Channel.objects.get(pk=channel_id) - user = User.objects.get(pk=user_id) - csv_path = write_channel_csv_file(channel, site=domain) - subject = render_to_string("export/csv_email_subject.txt", {"channel": channel}) - message = render_to_string( - "export/csv_email.txt", {"channel": channel, "user": user} - ) - - email = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL, [user.email]) - email.attach_file(csv_path) - email.send() - - class CustomEmailMessage(EmailMessage): """ jayoshih: There's an issue with the django postmark backend where From 2ba683a3f96535ac0a3f269c6e4a39a68a262b66 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 24 Feb 2022 16:41:09 -0800 Subject: [PATCH 23/46] Add metadata labels to contentnode model. --- .../migrations/0135_add_metadata_labels.py | 43 +++++++++++++++++++ contentcuration/contentcuration/models.py | 13 ++++++ 2 files changed, 56 insertions(+) create mode 100644 contentcuration/contentcuration/migrations/0135_add_metadata_labels.py diff --git a/contentcuration/contentcuration/migrations/0135_add_metadata_labels.py b/contentcuration/contentcuration/migrations/0135_add_metadata_labels.py new file mode 100644 index 0000000000..f1332bc008 --- /dev/null +++ b/contentcuration/contentcuration/migrations/0135_add_metadata_labels.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.5 on 2022-02-25 00:23 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0134_alter_contentkind_kind'), + ] + + operations = [ + migrations.AddField( + model_name='contentnode', + name='accessibility_labels', + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='contentnode', + name='categories', + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='contentnode', + name='grade_levels', + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='contentnode', + name='learner_needs', + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='contentnode', + name='learning_activities', + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='contentnode', + name='resource_types', + field=models.JSONField(blank=True, null=True), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 2d35b33ab8..f0fc0267f9 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -1146,6 +1146,19 @@ class ContentNode(MPTTModel, models.Model): role_visibility = models.CharField(max_length=50, choices=roles.choices, default=roles.LEARNER) freeze_authoring_data = models.BooleanField(default=False) + # Fields for metadata labels + # These fields use a map to store applied labels + # { + # "": true, + # "": true, + # } + grade_levels = models.JSONField(blank=True, null=True) + resource_types = models.JSONField(blank=True, null=True) + learning_activities = models.JSONField(blank=True, null=True) + accessibility_labels = models.JSONField(blank=True, null=True) + categories = models.JSONField(blank=True, null=True) + learner_needs = models.JSONField(blank=True, null=True) + objects = CustomContentNodeTreeManager() # Track all updates and ignore a blacklist of attributes From b051b447572dacad72519432766940ce3de217d9 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Wed, 7 Jul 2021 16:25:31 -0700 Subject: [PATCH 24/46] Remove unused views. --- contentcuration/contentcuration/dev_urls.py | 16 ---- contentcuration/contentcuration/urls.py | 2 - contentcuration/contentcuration/views/base.py | 75 ------------------- .../contentcuration/views/files.py | 42 ----------- 4 files changed, 135 deletions(-) delete mode 100644 contentcuration/contentcuration/views/files.py diff --git a/contentcuration/contentcuration/dev_urls.py b/contentcuration/contentcuration/dev_urls.py index a302e644d0..ec4369fd1b 100644 --- a/contentcuration/contentcuration/dev_urls.py +++ b/contentcuration/contentcuration/dev_urls.py @@ -8,7 +8,6 @@ from drf_yasg.views import get_schema_view from rest_framework import permissions -import contentcuration.views.files as file_views from .urls import urlpatterns @@ -48,21 +47,6 @@ def webpack_redirect_view(request): r"^redoc/$", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc" ), re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), - re_path( - r"^" + settings.STORAGE_URL[1:] + "(?P.*)$", - file_views.debug_serve_file, - name="debug_serve_file", - ), - re_path( - r"^" + settings.CONTENT_DATABASE_URL[1:] + "(?P.*)$", - file_views.debug_serve_content_database_file, - name="content_database_debug_serve_file", - ), - re_path( - r"^" + settings.CSV_URL[1:] + "(?P.*)$", - file_views.debug_serve_file, - name="csv_debug_serve_file", - ), ] if getattr(settings, "DEBUG_PANEL_ACTIVE", False): diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index 816f8e6884..973d8a34e3 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -75,7 +75,6 @@ def get_redirect_url(self, *args, **kwargs): re_path(r'^healthz$', views.health, name='health'), re_path(r'^stealthz$', views.stealth, name='stealth'), re_path(r'^api/search/', include('search.urls'), name='search'), - re_path(r'^api/download_channel_content_csv/(?P[^/]{32})$', views.download_channel_content_csv, name='download_channel_content_csv'), re_path(r'^api/probers/get_prober_channel', views.get_prober_channel, name='get_prober_channel'), re_path(r'^api/sync/$', sync, name="sync"), ] @@ -160,7 +159,6 @@ def get_redirect_url(self, *args, **kwargs): # Redirect deprecated staging URL to new URL re_path(r'^channels/(?P[^/]{32})/staging/$', StagingPageRedirectView.as_view(), name='staging_redirect'), re_path(r'^channels/(?P[^/]{32})/$', views.channel, name='channel'), - re_path(r'^accessible_channels/(?P[^/]{32})$', views.accessible_channels, name='accessible_channels'), re_path(r'^accounts/login/$', registration_views.login, name='login'), re_path(r'^accounts/logout/$', registration_views.logout, name='logout'), re_path(r'^accounts/request_activation_link/$', registration_views.request_activation_link, name='request_activation_link'), diff --git a/contentcuration/contentcuration/views/base.py b/contentcuration/contentcuration/views/base.py index 96554c13e2..78f774c1c7 100644 --- a/contentcuration/contentcuration/views/base.py +++ b/contentcuration/contentcuration/views/base.py @@ -1,19 +1,14 @@ import json from builtins import str - from urllib.parse import urlsplit from urllib.parse import urlunsplit from django.conf import settings from django.contrib.auth.decorators import login_required -from django.contrib.postgres.aggregates import ArrayAgg -from django.contrib.sites.shortcuts import get_current_site -from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.db.models import Count from django.db.models import IntegerField from django.db.models import OuterRef -from django.db.models import Q from django.db.models import Subquery from django.http import HttpResponse from django.http import HttpResponseForbidden @@ -28,7 +23,6 @@ from django.utils.translation import LANGUAGE_SESSION_KEY from django.views.decorators.http import require_POST from django.views.i18n import LANGUAGE_QUERY_PARAMETER -from le_utils.constants import content_kinds from rest_framework.authentication import BasicAuthentication from rest_framework.authentication import SessionAuthentication from rest_framework.authentication import TokenAuthentication @@ -37,7 +31,6 @@ from rest_framework.decorators import permission_classes from rest_framework.permissions import AllowAny from rest_framework.permissions import IsAuthenticated -from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from .json_dump import json_for_parse_from_data @@ -47,12 +40,10 @@ from contentcuration.models import Channel from contentcuration.models import ChannelSet from contentcuration.models import ContentKind -from contentcuration.models import ContentNode from contentcuration.models import DEFAULT_USER_PREFERENCES from contentcuration.models import Language from contentcuration.models import License from contentcuration.serializers import SimplifiedChannelProbeCheckSerializer -from contentcuration.tasks import generatechannelcsv_task from contentcuration.utils.i18n import SUPPORTED_LANGUAGES from contentcuration.utils.messages import get_messages from contentcuration.viewsets.channelset import PublicChannelSetSerializer @@ -127,17 +118,6 @@ def get_prober_channel(request): """ END HEALTH CHECKS """ -def get_or_set_cached_constants(constant, serializer): - cached_data = cache.get(constant.__name__) - if cached_data: - return cached_data - constant_objects = constant.objects.all() - constant_serializer = serializer(constant_objects, many=True) - constant_data = JSONRenderer().render(constant_serializer.data) - cache.set(constant.__name__, constant_data, None) - return constant_data - - @browser_is_supported @permission_classes((AllowAny,)) def channel_list(request): @@ -269,49 +249,6 @@ class SQCountDistinct(Subquery): output_field = IntegerField() -def map_channel_data(channel): - channel["id"] = channel.pop("main_tree__id") - channel["title"] = channel.pop("name") - channel["children"] = [child for child in channel["children"] if child] - return channel - - -@api_view(["GET"]) -@authentication_classes((TokenAuthentication, SessionAuthentication)) -@permission_classes((IsAuthenticated,)) -def accessible_channels(request, channel_id): - # Used for import modal - # Returns a list of objects with the following parameters: - # id, title, resource_count, children - channels = ( - Channel.objects.filter( - Q(deleted=False) - & (Q(public=True) | Q(editors=request.user) | Q(viewers=request.user)) - ) - .exclude(pk=channel_id) - .select_related("main_tree") - ) - channel_main_tree_nodes = ContentNode.objects.filter( - tree_id=OuterRef("main_tree__tree_id") - ) - # Add the unique count of distinct non-topic node content_ids - non_topic_content_ids = ( - channel_main_tree_nodes.exclude(kind_id=content_kinds.TOPIC) - .order_by("content_id") - .distinct("content_id") - .values_list("content_id", flat=True) - ) - channels = channels.annotate( - resource_count=SQCountDistinct(non_topic_content_ids, field="content_id"), - children=ArrayAgg("main_tree__children", distinct=True), - ) - channels_data = channels.values( - "name", "resource_count", "children", "main_tree__id" - ) - - return Response(map(map_channel_data, channels_data)) - - @api_view(['POST']) @authentication_classes((SessionAuthentication,)) @permission_classes((IsAuthenticated,)) @@ -331,18 +268,6 @@ def activate_channel_endpoint(request): return HttpResponse(json.dumps({"success": True, "changes": changes})) -@authentication_classes( - (SessionAuthentication, BasicAuthentication, TokenAuthentication) -) -@permission_classes((IsAuthenticated,)) -def download_channel_content_csv(request, channel_id): - """ Writes list of channels to csv, which is then emailed """ - site = get_current_site(request) - generatechannelcsv_task.delay(channel_id, site.domain, request.user.id) - - return HttpResponse({"success": True}) - - # Taken from kolibri.core.views which was # modified from django.views.i18n @require_POST diff --git a/contentcuration/contentcuration/views/files.py b/contentcuration/contentcuration/views/files.py deleted file mode 100644 index ce70578cf1..0000000000 --- a/contentcuration/contentcuration/views/files.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -from wsgiref.util import FileWrapper - -from django.conf import settings -from django.core.files.storage import default_storage -from django.http import Http404 -from django.http import HttpResponse -from rest_framework.authentication import SessionAuthentication -from rest_framework.authentication import TokenAuthentication -from rest_framework.decorators import authentication_classes -from rest_framework.decorators import permission_classes -from rest_framework.permissions import IsAuthenticated - -from contentcuration.models import generate_object_storage_name - - -@authentication_classes((TokenAuthentication, SessionAuthentication)) -@permission_classes((IsAuthenticated,)) -def debug_serve_file(request, path): - # There's a problem with loading exercise images, so use this endpoint - # to serve the image files to the /content/storage url - filename = os.path.basename(path) - checksum, _ext = os.path.splitext(filename) - filepath = generate_object_storage_name(checksum, filename) - - if not default_storage.exists(filepath): - raise Http404("The object requested does not exist.") - with default_storage.open(filepath, "rb") as fobj: - response = HttpResponse( - FileWrapper(fobj), content_type="application/octet-stream" - ) - return response - - -def debug_serve_content_database_file(request, path): - filename = os.path.basename(path) - path = "/".join([settings.DB_ROOT, filename]) - if not default_storage.exists(path): - raise Http404("The object requested does not exist.") - with default_storage.open(path, "rb") as f: - response = HttpResponse(FileWrapper(f), content_type="application/octet-stream") - return response From d962552be3408eb66b553fc8112d8b99b33e1caf Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 27 Jan 2022 11:38:55 -0800 Subject: [PATCH 25/46] Add duration to File model --- .../migrations/0133_auto_20220124_2149.py | 27 +++++++++++++++++++ contentcuration/contentcuration/models.py | 10 +++++-- .../contentcuration/utils/publish.py | 7 +++++ .../contentcuration/viewsets/file.py | 3 +++ .../migrations/0016_alter_contentnode_kind.py | 18 +++++++++++++ .../migrations/0017_contentnode_duration.py | 18 +++++++++++++ contentcuration/kolibri_content/models.py | 3 +++ 7 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py create mode 100644 contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py create mode 100644 contentcuration/kolibri_content/migrations/0017_contentnode_duration.py diff --git a/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py b/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py new file mode 100644 index 0000000000..8a5abe653c --- /dev/null +++ b/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.5 on 2022-01-24 21:49 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0132_auto_20210708_0011'), + ] + + operations = [ + migrations.AddField( + model_name='file', + name='duration', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='contentkind', + name='kind', + field=models.CharField(choices=[('topic', 'Topic'), ('video', 'Video'), ('audio', 'Audio'), ('exercise', 'Exercise'), ('document', 'Document'), ('html5', 'HTML5 App'), ('slideshow', 'Slideshow'), ('h5p', 'H5P'), ('zim', 'Zim'), ('quiz', 'Quiz')], max_length=200, primary_key=True, serialize=False), + ), + migrations.AddConstraint( + model_name='file', + constraint=models.CheckConstraint(check=models.Q(models.Q(('duration__gt', 0), ('preset__in', ['audio', 'high_res_video', 'low_res_video'])), ('duration__isnull', True), _connector='OR'), name='file_media_duration_int'), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index f0fc0267f9..f40ce07d99 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -1935,6 +1935,8 @@ class StagedFile(models.Model): FILE_DISTINCT_INDEX_NAME = "file_checksum_file_size_idx" FILE_MODIFIED_DESC_INDEX_NAME = "file_modified_desc_idx" +FILE_DURATION_CONSTRAINT = "file_media_duration_int" +MEDIA_PRESETS = [format_presets.AUDIO, format_presets.VIDEO_HIGH_RES, format_presets.VIDEO_LOW_RES] class File(models.Model): @@ -1958,6 +1960,7 @@ class File(models.Model): uploaded_by = models.ForeignKey(User, related_name='files', blank=True, null=True, on_delete=models.SET_NULL) modified = models.DateTimeField(auto_now=True, verbose_name="modified", null=True) + duration = models.IntegerField(blank=True, null=True) objects = CustomManager() @@ -2069,6 +2072,9 @@ class Meta: models.Index(fields=['checksum', 'file_size'], name=FILE_DISTINCT_INDEX_NAME), models.Index(fields=["-modified"], name=FILE_MODIFIED_DESC_INDEX_NAME), ] + constraints = [ + models.CheckConstraint(check=(Q(preset__in=MEDIA_PRESETS, duration__gt=0) | Q(duration__isnull=True)), name=FILE_DURATION_CONSTRAINT) + ] @receiver(models.signals.post_delete, sender=File) @@ -2108,7 +2114,7 @@ def clean(self, *args, **kwargs): raise IntegrityError('Cannot self reference as prerequisite.') # immediate cyclic exception if PrerequisiteContentRelationship.objects.using(self._state.db) \ - .filter(target_node=self.prerequisite, prerequisite=self.target_node): + .filter(target_node=self.prerequisite, prerequisite=self.target_node): raise IntegrityError( 'Note: Prerequisite relationship is directional! %s and %s cannot be prerequisite of each other!' % (self.target_node, self.prerequisite)) @@ -2141,7 +2147,7 @@ def save(self, *args, **kwargs): raise IntegrityError('Cannot self reference as related.') # handle immediate cyclic if RelatedContentRelationship.objects.using(self._state.db) \ - .filter(contentnode_1=self.contentnode_2, contentnode_2=self.contentnode_1): + .filter(contentnode_1=self.contentnode_2, contentnode_2=self.contentnode_1): return # silently cancel the save super(RelatedContentRelationship, self).save(*args, **kwargs) diff --git a/contentcuration/contentcuration/utils/publish.py b/contentcuration/contentcuration/utils/publish.py index 70232a55ac..39c1ca39ef 100644 --- a/contentcuration/contentcuration/utils/publish.py +++ b/contentcuration/contentcuration/utils/publish.py @@ -20,6 +20,7 @@ from django.core.management import call_command from django.db import transaction from django.db.models import Count +from django.db.models import Max from django.db.models import Q from django.db.models import Sum from django.db.utils import IntegrityError @@ -225,6 +226,11 @@ def create_bare_contentnode(ccnode, default_language, channel_id, channel_name): if ccnode.language or default_language: language, _new = get_or_create_language(ccnode.language or default_language) + duration = None + if ccnode.kind_id in [content_kinds.AUDIO, content_kinds.VIDEO]: + # aggregate duration from associated files, choosing maximum if there are multiple, like hi and lo res videos + duration = ccnode.files.aggregate(duration=Max("duration")).get("duration") + options = {} if ccnode.extra_fields and 'options' in ccnode.extra_fields: options = ccnode.extra_fields['options'] @@ -247,6 +253,7 @@ def create_bare_contentnode(ccnode, default_language, channel_id, channel_name): 'license_name': kolibri_license.license_name if kolibri_license is not None else None, 'license_description': kolibri_license.license_description if kolibri_license is not None else None, 'coach_content': ccnode.role_visibility == roles.COACH, + 'duration': duration, 'options': json.dumps(options) } ) diff --git a/contentcuration/contentcuration/viewsets/file.py b/contentcuration/contentcuration/viewsets/file.py index 6c65582a56..cd04538815 100644 --- a/contentcuration/contentcuration/viewsets/file.py +++ b/contentcuration/contentcuration/viewsets/file.py @@ -71,6 +71,7 @@ class Meta: "contentnode", "assessment_item", "preset", + "duration" ) list_serializer_class = BulkListSerializer @@ -98,6 +99,7 @@ class FileViewSet(BulkDeleteMixin, BulkUpdateMixin, ReadOnlyValuesViewset): "language_id", "original_filename", "uploaded_by", + "duration" ) field_map = { @@ -164,6 +166,7 @@ def upload_url(self, request): file_format_id=file_format, preset_id=preset, uploaded_by=request.user, + duration=request.data.get("duration"), ) # Avoid using our file_on_disk attribute for checks diff --git a/contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py b/contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py new file mode 100644 index 0000000000..669c7d3245 --- /dev/null +++ b/contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.5 on 2022-01-24 21:36 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('content', '0015_auto_20210707_1606'), + ] + + operations = [ + migrations.AlterField( + model_name='contentnode', + name='kind', + field=models.CharField(blank=True, choices=[('topic', 'Topic'), ('video', 'Video'), ('audio', 'Audio'), ('exercise', 'Exercise'), ('document', 'Document'), ('html5', 'HTML5 App'), ('slideshow', 'Slideshow'), ('h5p', 'H5P'), ('zim', 'Zim'), ('quiz', 'Quiz')], max_length=200), + ), + ] diff --git a/contentcuration/kolibri_content/migrations/0017_contentnode_duration.py b/contentcuration/kolibri_content/migrations/0017_contentnode_duration.py new file mode 100644 index 0000000000..947fab093f --- /dev/null +++ b/contentcuration/kolibri_content/migrations/0017_contentnode_duration.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.5 on 2022-01-27 18:56 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('content', '0016_alter_contentnode_kind'), + ] + + operations = [ + migrations.AddField( + model_name='contentnode', + name='duration', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/contentcuration/kolibri_content/models.py b/contentcuration/kolibri_content/models.py index 1ee710a686..a8c9c8fc8f 100644 --- a/contentcuration/kolibri_content/models.py +++ b/contentcuration/kolibri_content/models.py @@ -122,6 +122,9 @@ class ContentNode(MPTTModel): # A JSON Dictionary of properties to configure loading, rendering, etc. the file options = models.TextField(default="{}") + # If media, the duration in seconds + duration = models.IntegerField(null=True, blank=True) + class Meta: ordering = ("lft",) index_together = [ From 5d6711a332c482026258ebb04d1b8c63970c135a Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 7 Jun 2022 14:44:51 -0700 Subject: [PATCH 26/46] Do a direct lift from unstable to ensure exact migrations. --- .../migrations/0133_auto_20220124_2149.py | 5 --- .../migrations/0134_alter_contentkind_kind.py | 34 +++++++++++++++++++ .../migrations/0016_alter_contentnode_kind.py | 18 ---------- ...ration.py => 0016_contentnode_duration.py} | 2 +- .../migrations/0017_alter_contentnode_kind.py | 33 ++++++++++++++++++ 5 files changed, 68 insertions(+), 24 deletions(-) create mode 100644 contentcuration/contentcuration/migrations/0134_alter_contentkind_kind.py delete mode 100644 contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py rename contentcuration/kolibri_content/migrations/{0017_contentnode_duration.py => 0016_contentnode_duration.py} (87%) create mode 100644 contentcuration/kolibri_content/migrations/0017_alter_contentnode_kind.py diff --git a/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py b/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py index 8a5abe653c..ef0fec8d19 100644 --- a/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py +++ b/contentcuration/contentcuration/migrations/0133_auto_20220124_2149.py @@ -15,11 +15,6 @@ class Migration(migrations.Migration): name='duration', field=models.IntegerField(blank=True, null=True), ), - migrations.AlterField( - model_name='contentkind', - name='kind', - field=models.CharField(choices=[('topic', 'Topic'), ('video', 'Video'), ('audio', 'Audio'), ('exercise', 'Exercise'), ('document', 'Document'), ('html5', 'HTML5 App'), ('slideshow', 'Slideshow'), ('h5p', 'H5P'), ('zim', 'Zim'), ('quiz', 'Quiz')], max_length=200, primary_key=True, serialize=False), - ), migrations.AddConstraint( model_name='file', constraint=models.CheckConstraint(check=models.Q(models.Q(('duration__gt', 0), ('preset__in', ['audio', 'high_res_video', 'low_res_video'])), ('duration__isnull', True), _connector='OR'), name='file_media_duration_int'), diff --git a/contentcuration/contentcuration/migrations/0134_alter_contentkind_kind.py b/contentcuration/contentcuration/migrations/0134_alter_contentkind_kind.py new file mode 100644 index 0000000000..6a1795817b --- /dev/null +++ b/contentcuration/contentcuration/migrations/0134_alter_contentkind_kind.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.5 on 2022-02-23 18:42 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contentcuration", "0133_auto_20220124_2149"), + ] + + operations = [ + migrations.AlterField( + model_name="contentkind", + name="kind", + field=models.CharField( + choices=[ + ("topic", "Topic"), + ("video", "Video"), + ("audio", "Audio"), + ("exercise", "Exercise"), + ("document", "Document"), + ("html5", "HTML5 App"), + ("slideshow", "Slideshow"), + ("h5p", "H5P"), + ("zim", "Zim"), + ("quiz", "Quiz"), + ], + max_length=200, + primary_key=True, + serialize=False, + ), + ), + ] diff --git a/contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py b/contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py deleted file mode 100644 index 669c7d3245..0000000000 --- a/contentcuration/kolibri_content/migrations/0016_alter_contentnode_kind.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.5 on 2022-01-24 21:36 -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ('content', '0015_auto_20210707_1606'), - ] - - operations = [ - migrations.AlterField( - model_name='contentnode', - name='kind', - field=models.CharField(blank=True, choices=[('topic', 'Topic'), ('video', 'Video'), ('audio', 'Audio'), ('exercise', 'Exercise'), ('document', 'Document'), ('html5', 'HTML5 App'), ('slideshow', 'Slideshow'), ('h5p', 'H5P'), ('zim', 'Zim'), ('quiz', 'Quiz')], max_length=200), - ), - ] diff --git a/contentcuration/kolibri_content/migrations/0017_contentnode_duration.py b/contentcuration/kolibri_content/migrations/0016_contentnode_duration.py similarity index 87% rename from contentcuration/kolibri_content/migrations/0017_contentnode_duration.py rename to contentcuration/kolibri_content/migrations/0016_contentnode_duration.py index 947fab093f..7c442f6e05 100644 --- a/contentcuration/kolibri_content/migrations/0017_contentnode_duration.py +++ b/contentcuration/kolibri_content/migrations/0016_contentnode_duration.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('content', '0016_alter_contentnode_kind'), + ('content', '0015_auto_20210707_1606'), ] operations = [ diff --git a/contentcuration/kolibri_content/migrations/0017_alter_contentnode_kind.py b/contentcuration/kolibri_content/migrations/0017_alter_contentnode_kind.py new file mode 100644 index 0000000000..2533c9e416 --- /dev/null +++ b/contentcuration/kolibri_content/migrations/0017_alter_contentnode_kind.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.5 on 2022-02-23 18:42 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("content", "0016_contentnode_duration"), + ] + + operations = [ + migrations.AlterField( + model_name="contentnode", + name="kind", + field=models.CharField( + blank=True, + choices=[ + ("topic", "Topic"), + ("video", "Video"), + ("audio", "Audio"), + ("exercise", "Exercise"), + ("document", "Document"), + ("html5", "HTML5 App"), + ("slideshow", "Slideshow"), + ("h5p", "H5P"), + ("zim", "Zim"), + ("quiz", "Quiz"), + ], + max_length=200, + ), + ), + ] From eb07b7591d1bbd84cdc0aabde2384bbd2bf396d8 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 7 Jun 2022 15:31:21 -0700 Subject: [PATCH 27/46] Set default learning activity on creation if not already set. --- .../contentcuration/constants/contentnode.py | 13 +++++++++++++ contentcuration/contentcuration/models.py | 9 +++++++++ 2 files changed, 22 insertions(+) create mode 100644 contentcuration/contentcuration/constants/contentnode.py diff --git a/contentcuration/contentcuration/constants/contentnode.py b/contentcuration/contentcuration/constants/contentnode.py new file mode 100644 index 0000000000..c711b26d5e --- /dev/null +++ b/contentcuration/contentcuration/constants/contentnode.py @@ -0,0 +1,13 @@ +from le_utils.constants import content_kinds +from le_utils.constants.labels import learning_activities + + +kind_activity_map = { + content_kinds.EXERCISE: learning_activities.PRACTICE, + content_kinds.VIDEO: learning_activities.WATCH, + content_kinds.AUDIO: learning_activities.LISTEN, + content_kinds.DOCUMENT: learning_activities.READ, + content_kinds.HTML5: learning_activities.EXPLORE, + content_kinds.H5P: learning_activities.EXPLORE, + content_kinds.SLIDESHOW: learning_activities.READ, +} diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index f40ce07d99..85827078b4 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -60,6 +60,7 @@ from postmark.core import PMMailUnauthorizedException from rest_framework.authtoken.models import Token +from contentcuration.constants.contentnode import kind_activity_map from contentcuration.db.models.expressions import Array from contentcuration.db.models.functions import ArrayRemove from contentcuration.db.models.functions import Unnest @@ -1657,6 +1658,7 @@ def recalculate_editors_storage(self): def on_create(self): self.changed = True self.recalculate_editors_storage() + self.set_default_learning_activity() def on_update(self): self.changed = self.changed or self.has_changes() @@ -1669,6 +1671,13 @@ def move_to(self, target, *args, **kwargs): if target.channel_trash.exists() or parent_was_trashtree: self.recalculate_editors_storage() + def set_default_learning_activity(self): + if self.learning_activities is None: + if self.kind in kind_activity_map: + self.learning_activities = { + kind_activity_map[self.kind]: True + } + def save(self, skip_lock=False, *args, **kwargs): if self._state.adding: self.on_create() From 1764838167106018f09d4ce29094d48ea4a5200c Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 7 Jun 2022 15:45:23 -0700 Subject: [PATCH 28/46] Add management command to update default learning activities. --- .../set_default_learning_activities.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 contentcuration/contentcuration/management/commands/set_default_learning_activities.py diff --git a/contentcuration/contentcuration/management/commands/set_default_learning_activities.py b/contentcuration/contentcuration/management/commands/set_default_learning_activities.py new file mode 100644 index 0000000000..9d958227e0 --- /dev/null +++ b/contentcuration/contentcuration/management/commands/set_default_learning_activities.py @@ -0,0 +1,37 @@ +import logging as logmodule +import time + +from django.core.management.base import BaseCommand + +from contentcuration.constants.contentnode import kind_activity_map +from contentcuration.models import ContentNode + +logmodule.basicConfig(level=logmodule.INFO) +logging = logmodule.getLogger('command') + + +CHUNKSIZE = 10000 + + +class Command(BaseCommand): + + def handle(self, *args, **options): + start = time.time() + + for kind, activity in kind_activity_map.items(): + kind_start = time.time() + map_to_set = { + activity: True + } + + null_learning_activities = ContentNode.objects.filter(kind=kind, learning_activities__isnull=True).values_list("id", flat=True) + + logging.info("Setting default learning activities for kind: {}".format(kind)) + + while null_learning_activities.exists(): + updated_count = ContentNode.objects.filter(id__in=null_learning_activities[0:CHUNKSIZE]).update(learning_activities=map_to_set) + logging.info("Updated {} content nodes of kind {} with learning activity {}".format(updated_count, kind, activity)) + + logging.info("Finished setting default learning activities for kind: {} in {} seconds".format(kind, time.time() - kind_start)) + + logging.info('Finished setting all null learning activities in {} seconds'.format(time.time() - start)) From 30c025c756dbb64ccfe9ea299c0eb582d4905729 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 7 Jun 2022 15:52:37 -0700 Subject: [PATCH 29/46] Add ffmpeg back into Dockerfile dependencies. --- docker/Dockerfile.demo | 2 +- docker/Dockerfile.dev | 2 +- k8s/images/app/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.demo b/docker/Dockerfile.demo index d6f3a356db..5a6bf72eb5 100644 --- a/docker/Dockerfile.demo +++ b/docker/Dockerfile.demo @@ -11,7 +11,7 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - ENV DEBIAN_FRONTEND noninteractive # Default Python file.open file encoding to UTF-8 instead of ASCII, workaround for le-utils setup.py issue ENV LANG C.UTF-8 -RUN apt-get update && apt-get -y install nodejs python-minimal python3.6 python3-pip python3-dev gcc libpq-dev make git curl libjpeg-dev +RUN apt-get update && apt-get -y install nodejs python-minimal python3.6 python3-pip python3-dev gcc libpq-dev make git curl libjpeg-dev ffmpeg # Ensure that python is using python3 # copying approach from official python images diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index b06daf4800..210866232c 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -17,7 +17,7 @@ RUN apt-get update --fix-missing && \ apt-get -y install \ curl fish man \ python-minimal python3.6 python3-pip python3-dev \ - gcc libpq-dev make git gettext libjpeg-dev + gcc libpq-dev make git gettext libjpeg-dev ffmpeg # Ensure that python is using python3 # copying approach from official python images diff --git a/k8s/images/app/Dockerfile b/k8s/images/app/Dockerfile index 0c2a29e6af..496a84b8c4 100644 --- a/k8s/images/app/Dockerfile +++ b/k8s/images/app/Dockerfile @@ -11,7 +11,7 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - ENV DEBIAN_FRONTEND noninteractive # Default Python file.open file encoding to UTF-8 instead of ASCII, workaround for le-utils setup.py issue ENV LANG C.UTF-8 -RUN apt-get update && apt-get -y install nodejs python-minimal python3.6 python3-pip python3-dev gcc libpq-dev make git curl libjpeg-dev +RUN apt-get update && apt-get -y install nodejs python-minimal python3.6 python3-pip python3-dev gcc libpq-dev make git curl libjpeg-dev ffmpeg # Ensure that python is using python3 # copying approach from official python images From 1b3b38ee5dd50d5c785e3087333edc77eada1f33 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 7 Jun 2022 16:46:59 -0700 Subject: [PATCH 30/46] Management command to set duration on file objects. --- .../management/commands/set_file_duration.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 contentcuration/contentcuration/management/commands/set_file_duration.py diff --git a/contentcuration/contentcuration/management/commands/set_file_duration.py b/contentcuration/contentcuration/management/commands/set_file_duration.py new file mode 100644 index 0000000000..1b9ea693eb --- /dev/null +++ b/contentcuration/contentcuration/management/commands/set_file_duration.py @@ -0,0 +1,78 @@ +import logging as logmodule +import os +import shutil +import subprocess +import tempfile +import time + +from django.core.management.base import BaseCommand + +from contentcuration.models import File +from contentcuration.models import MEDIA_PRESETS + +logmodule.basicConfig(level=logmodule.INFO) +logging = logmodule.getLogger('command') + + +CHUNKSIZE = 10000 + + +def extract_duration_of_media(fpath_in): + result = subprocess.check_output( + [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + "-loglevel", + "panic", + str(fpath_in), + ] + ) + return int(float(result.decode("utf-8").strip())) + + +class Command(BaseCommand): + + def handle(self, *args, **options): + start = time.time() + + logging.info("Setting default duration for media presets: {}".format(MEDIA_PRESETS)) + + excluded_files = set() + + null_duration = File.objects.filter(preset_id__in=MEDIA_PRESETS, duration__isnull=True) + null_duration_count = null_duration.count() + updated_count = 0 + + i = 0 + + while i < null_duration_count: + for file in null_duration[i:i + CHUNKSIZE]: + if file.file_on_disk.name in excluded_files: + continue + file.refresh_from_db() + if file.duration is not None: + continue + try: + with file.file_on_disk.open() as f: + fobj = tempfile.NamedTemporaryFile(delete=False) + shutil.copyfileobj(f, fobj) + fobj.close() + duration = extract_duration_of_media(fobj.name) + os.unlink(fobj.name) + if duration: + updated_count += File.objects.filter(checksum=file.checksum, preset_id__in=MEDIA_PRESETS).update(duration=duration) + except FileNotFoundError: + logging.warning("File {} not found".format(file)) + excluded_files.add(file.file_on_disk.name) + except subprocess.CalledProcessError: + logging.warning("File {} could not be read for duration".format(file)) + excluded_files.add(file.file_on_disk.name) + + i += CHUNKSIZE + + logging.info('Finished setting all null duration for {} files in {} seconds'.format(updated_count, time.time() - start)) From a8b8bb5999691e60935bd7f5b1bc2e3a9eee32de Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 5 May 2022 13:40:39 -0700 Subject: [PATCH 31/46] Try different structure for stalling query --- contentcuration/contentcuration/models.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 85827078b4..1053cac90f 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -24,6 +24,7 @@ from django.db import models from django.db.models import Count from django.db.models import Exists +from django.db.models import F from django.db.models import Index from django.db.models import IntegerField from django.db.models import JSONField @@ -262,12 +263,15 @@ def get_available_space(self, active_files=None): def get_user_active_trees(self): return self.editable_channels.exclude(deleted=True)\ - .values_list('main_tree__tree_id', flat=True) + .values(tree_id=F("main_tree__tree_id")) def get_user_active_files(self): - active_trees = self.get_user_active_trees() - return self.files.filter(contentnode__tree_id__in=active_trees)\ - .values('checksum').distinct() + cte = With(self.get_user_active_trees().distinct()) + + return cte.join(self.files.get_queryset(), contentnode__tree_id=cte.col.tree_id)\ + .with_cte(cte)\ + .values('checksum')\ + .distinct() def get_space_used(self, active_files=None): active_files = active_files or self.get_user_active_files() From 5c9fb871b8e59d04c561ae380c7e9a7f6477f32f Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 9 Jun 2022 11:00:29 -0700 Subject: [PATCH 32/46] Pipe file to ffprobe to avoid temporary file. --- .../management/commands/set_file_duration.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/contentcuration/contentcuration/management/commands/set_file_duration.py b/contentcuration/contentcuration/management/commands/set_file_duration.py index 1b9ea693eb..f93a6097c1 100644 --- a/contentcuration/contentcuration/management/commands/set_file_duration.py +++ b/contentcuration/contentcuration/management/commands/set_file_duration.py @@ -1,8 +1,5 @@ import logging as logmodule -import os -import shutil import subprocess -import tempfile import time from django.core.management.base import BaseCommand @@ -17,7 +14,7 @@ CHUNKSIZE = 10000 -def extract_duration_of_media(fpath_in): +def extract_duration_of_media(f_in): result = subprocess.check_output( [ "ffprobe", @@ -29,8 +26,9 @@ def extract_duration_of_media(fpath_in): "default=noprint_wrappers=1:nokey=1", "-loglevel", "panic", - str(fpath_in), - ] + "-" + ], + stdin=f_in, ) return int(float(result.decode("utf-8").strip())) @@ -59,11 +57,7 @@ def handle(self, *args, **options): continue try: with file.file_on_disk.open() as f: - fobj = tempfile.NamedTemporaryFile(delete=False) - shutil.copyfileobj(f, fobj) - fobj.close() - duration = extract_duration_of_media(fobj.name) - os.unlink(fobj.name) + duration = extract_duration_of_media(f) if duration: updated_count += File.objects.filter(checksum=file.checksum, preset_id__in=MEDIA_PRESETS).update(duration=duration) except FileNotFoundError: From e8b3908da4a4a36a7741bfb8c9811154ad2cb9e3 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 23 Jun 2022 15:00:27 -0700 Subject: [PATCH 33/46] Add makefile target for set_file_duration command --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index b69f193a7f..7343eddc6c 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,9 @@ docs: clean-docs setup: python contentcuration/manage.py setup +filedurations: + python contentcuration/manage.py set_file_duration + export COMPOSE_PROJECT_NAME=studio_$(shell git rev-parse --abbrev-ref HEAD) purge-postgres: From 74896cae72ada635638784e3dc8cc6c7a585cf47 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 28 Jun 2022 15:38:40 -0700 Subject: [PATCH 34/46] Add admin trap door for space checks. --- contentcuration/contentcuration/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 1053cac90f..1f3e5a2fa7 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -226,6 +226,9 @@ def can_edit(self, channel_id): return Channel.filter_edit_queryset(Channel.objects.all(), self).filter(pk=channel_id).exists() def check_space(self, size, checksum): + if self.is_admin: + return True + active_files = self.get_user_active_files() if active_files.filter(checksum=checksum).exists(): return True From 6995c3065f6567b789455084e8393c35efbb237a Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 28 Jun 2022 15:47:46 -0700 Subject: [PATCH 35/46] Rollback late acknowledgement until we upgrade celery --- contentcuration/contentcuration/utils/celery/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/utils/celery/tasks.py b/contentcuration/contentcuration/utils/celery/tasks.py index 92ae0b68fe..b77ddb6609 100644 --- a/contentcuration/contentcuration/utils/celery/tasks.py +++ b/contentcuration/contentcuration/utils/celery/tasks.py @@ -77,8 +77,8 @@ def my_task(self): track_progress = False # ensure our tasks are restarted if they're interrupted - acks_late = True - acks_on_failure_or_timeout = True + acks_late = False + acks_on_failure_or_timeout = False reject_on_worker_lost = True _progress_tracker = None From 59773a350ec1de6d389f785611922517fd7eb630 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 28 Jun 2022 15:48:18 -0700 Subject: [PATCH 36/46] Increase visibility timeout from 1 hr to 4 hrs --- contentcuration/contentcuration/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contentcuration/contentcuration/settings.py b/contentcuration/contentcuration/settings.py index 1434ae9877..cf3684af57 100644 --- a/contentcuration/contentcuration/settings.py +++ b/contentcuration/contentcuration/settings.py @@ -342,6 +342,8 @@ def gettext(s): # CELERY CONFIGURATIONS CELERY_BROKER_URL = REDIS_URL +# with a redis broker, tasks will be re-sent if not completed within the duration of this timeout +CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 4 * 3600} CELERY_RESULT_BACKEND = REDIS_URL CELERY_REDIS_DB = os.getenv("CELERY_REDIS_DB") or "0" CELERY_BROKER_URL = "{url}{db}".format( From 0aca5e6b49375d5717bb97ec02e3bb5cca920463 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 5 Jul 2022 16:43:53 -0700 Subject: [PATCH 37/46] Add stream parsing fallback for duration setting. --- .../management/commands/set_file_duration.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/management/commands/set_file_duration.py b/contentcuration/contentcuration/management/commands/set_file_duration.py index f93a6097c1..0c607d88ea 100644 --- a/contentcuration/contentcuration/management/commands/set_file_duration.py +++ b/contentcuration/contentcuration/management/commands/set_file_duration.py @@ -15,6 +15,10 @@ def extract_duration_of_media(f_in): + """ + For more details on these commands, refer to the ffmpeg Wiki: + https://trac.ffmpeg.org/wiki/FFprobeTips#Formatcontainerduration + """ result = subprocess.check_output( [ "ffprobe", @@ -30,7 +34,42 @@ def extract_duration_of_media(f_in): ], stdin=f_in, ) - return int(float(result.decode("utf-8").strip())) + result = result.decode("utf-8").strip() + try: + # return int(float(result)) + raise ValueError + except ValueError: + # This can happen if ffprobe returns N/A for the duration + # So instead we try to stream the entire file to get the value + f_in.seek(0) + result = subprocess.run( + [ + "ffmpeg", + "-i", + "pipe:", + "-f", + "null", + "-", + ], + stdin=f_in, + stderr=subprocess.PIPE + ) + second_last_line = result.stderr.decode("utf-8").strip().splitlines()[-2] + time_code = second_last_line.split(" time=")[1].split(" ")[0] + hours, minutes, seconds = time_code.split(":") + try: + hours = int(hours) + except ValueError: + hours = 0 + try: + minutes = int(minutes) + except ValueError: + minutes = 0 + try: + seconds = int(float(seconds)) + except ValueError: + seconds = 0 + return (hours * 60 + minutes) * 60 + seconds class Command(BaseCommand): From de3acc50b9cf10f2be46c96790b03d242e301e10 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 5 Jul 2022 17:20:31 -0700 Subject: [PATCH 38/46] Exclude dependency presets from listings. --- .../frontend/channelEdit/views/files/FileUpload.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/FileUpload.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/FileUpload.vue index 9de99c758f..7cb7528aa5 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/FileUpload.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/FileUpload.vue @@ -108,7 +108,10 @@ return this.getContentNodeFiles(this.nodeId); }, presets() { - return FormatPresetsList.filter(p => p.kind_id === this.node.kind); + // Explicitly exclude any 'dependency' presets for now + return FormatPresetsList.filter( + p => p.kind_id === this.node.kind && !p.id.includes('dependency') + ); }, fileCount() { return this.primaryFileMapping.filter(item => item.file && !item.file.error).length; From 246e6553cffecf5f011a181ad72d9d93f079ff48 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 5 Jul 2022 17:24:08 -0700 Subject: [PATCH 39/46] Add makefile target for set_default_learning_activities --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 7343eddc6c..383bd3ad32 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,9 @@ setup: filedurations: python contentcuration/manage.py set_file_duration +learningactivities: + python contentcuration/manage.py set_default_learning_activities + export COMPOSE_PROJECT_NAME=studio_$(shell git rev-parse --abbrev-ref HEAD) purge-postgres: From 7ca38560760aadd9249bcd15a532e632fdb5a3bf Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Wed, 6 Jul 2022 10:03:42 -0700 Subject: [PATCH 40/46] Remove debugging code. --- .../contentcuration/management/commands/set_file_duration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/management/commands/set_file_duration.py b/contentcuration/contentcuration/management/commands/set_file_duration.py index 0c607d88ea..6fad0f8b73 100644 --- a/contentcuration/contentcuration/management/commands/set_file_duration.py +++ b/contentcuration/contentcuration/management/commands/set_file_duration.py @@ -36,8 +36,7 @@ def extract_duration_of_media(f_in): ) result = result.decode("utf-8").strip() try: - # return int(float(result)) - raise ValueError + return int(float(result)) except ValueError: # This can happen if ffprobe returns N/A for the duration # So instead we try to stream the entire file to get the value From 9dd686d9a3e22675c6d22ddb6387d3945d141f20 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Wed, 6 Jul 2022 10:17:46 -0700 Subject: [PATCH 41/46] force file format from extension for ffprobe call. --- .../management/commands/set_file_duration.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/management/commands/set_file_duration.py b/contentcuration/contentcuration/management/commands/set_file_duration.py index 6fad0f8b73..fd09aaf51d 100644 --- a/contentcuration/contentcuration/management/commands/set_file_duration.py +++ b/contentcuration/contentcuration/management/commands/set_file_duration.py @@ -14,7 +14,7 @@ CHUNKSIZE = 10000 -def extract_duration_of_media(f_in): +def extract_duration_of_media(f_in, extension): """ For more details on these commands, refer to the ffmpeg Wiki: https://trac.ffmpeg.org/wiki/FFprobeTips#Formatcontainerduration @@ -30,6 +30,8 @@ def extract_duration_of_media(f_in): "default=noprint_wrappers=1:nokey=1", "-loglevel", "panic", + "-f", + extension, "-" ], stdin=f_in, @@ -95,7 +97,7 @@ def handle(self, *args, **options): continue try: with file.file_on_disk.open() as f: - duration = extract_duration_of_media(f) + duration = extract_duration_of_media(f, file.file_format.extension) if duration: updated_count += File.objects.filter(checksum=file.checksum, preset_id__in=MEDIA_PRESETS).update(duration=duration) except FileNotFoundError: From a21065aa3e6dd7d5c19aa47a11e545fc033c9f00 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Wed, 6 Jul 2022 11:36:02 -0700 Subject: [PATCH 42/46] patch QJoin class to resolve Django 3+ interaction issue --- contentcuration/contentcuration/apps.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/contentcuration/contentcuration/apps.py b/contentcuration/contentcuration/apps.py index f42624ce9e..e45938e043 100644 --- a/contentcuration/contentcuration/apps.py +++ b/contentcuration/contentcuration/apps.py @@ -14,3 +14,22 @@ def ready(self): if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend(): from contentcuration.utils.minio_utils import ensure_storage_bucket_public ensure_storage_bucket_public() + + self._patch_django_cte_qjoin() + + def _patch_django_cte_qjoin(self): + """ + TODO Remove after the following prs/issues are resolved: + https://github.com/learningequality/studio/pull/3442 + https://github.com/dimagi/django-cte/pull/60 + + @see fix: https://github.com/dimagi/django-cte/pull/50/files + """ + from django_cte.join import QJoin + + class join_field: + class related_model: + class _meta: + local_concrete_fields = () + + QJoin.join_field = join_field From 3a8fc1c04bdd33ffb99e21da0a3a2689c0bd5474 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 5 Jul 2022 15:36:36 -0700 Subject: [PATCH 43/46] Update django cte to resolve API key issue --- requirements-dev.txt | 1 + requirements.in | 2 +- requirements.txt | 57 ++++++++++++++++++++++++-------------------- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d75192e42b..ae9b4a0607 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -350,6 +350,7 @@ typed-ast==1.5.4 # via astroid typing-extensions==4.1.1 # via + # -c requirements.txt # asgiref # astroid # locust diff --git a/requirements.in b/requirements.in index b94702b943..6b3b028750 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ attrs==19.3.0 -django-cte==1.1.5 +django-cte==1.2.0 django-mptt==0.11.0 django-filter==2.4.0 djangorestframework==3.12.4 diff --git a/requirements.txt b/requirements.txt index 168aed79a9..856c1a19f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,23 +38,9 @@ chardet==4.0.0 # via requests confusable-homoglyphs==3.2.0 # via django-registration -django==3.2.13 - # via - # -r requirements.in - # django-bulk-update - # django-db-readonly - # django-filter - # django-js-reverse - # django-model-utils - # django-mptt - # django-redis - # django-registration - # django-s3-storage - # djangorestframework - # jsonfield django-bulk-update==2.2.0 # via -r requirements.in -django-cte==1.1.5 +django-cte==1.2.0 # via -r requirements.in django-db-readonly==0.7.0 # via -r requirements.in @@ -82,6 +68,20 @@ django-s3-storage==0.13.4 # via -r requirements.in django-webpack-loader==0.7.0 # via -r requirements.in +django==3.2.13 + # via + # -r requirements.in + # django-bulk-update + # django-db-readonly + # django-filter + # django-js-reverse + # django-model-utils + # django-mptt + # django-redis + # django-registration + # django-s3-storage + # djangorestframework + # jsonfield djangorestframework==3.12.4 # via -r requirements.in future==0.18.2 @@ -95,6 +95,10 @@ google-api-core[grpc]==1.27.0 # google-cloud-logging google-api-python-client==2.4.0 # via -r requirements.in +google-auth-httplib2==0.1.0 + # via google-api-python-client +google-auth-oauthlib==0.4.4 + # via gspread google-auth==1.30.0 # via # google-api-core @@ -104,10 +108,6 @@ google-auth==1.30.0 # google-cloud-core # google-cloud-storage # gspread -google-auth-httplib2==0.1.0 - # via google-api-python-client -google-auth-oauthlib==0.4.4 - # via gspread google-cloud-core==1.6.0 # via # -r requirements.in @@ -151,7 +151,10 @@ httplib2==0.19.1 idna==2.10 # via requests importlib-metadata==1.7.0 - # via -r requirements.in + # via + # -r requirements.in + # jsonschema + # kombu jmespath==0.10.0 # via # boto3 @@ -191,15 +194,15 @@ protobuf==3.17.0 # proto-plus psycopg2-binary==2.8.6 # via -r requirements.in +pyasn1-modules==0.2.8 + # via + # google-auth + # oauth2client pyasn1==0.4.8 # via # oauth2client # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via - # google-auth - # oauth2client pycountry==17.5.14 # via -r requirements.in pycparser==2.20 @@ -230,6 +233,8 @@ redis==3.5.3 # via # -r requirements.in # django-redis +requests-oauthlib==1.3.0 + # via google-auth-oauthlib requests==2.25.1 # via # -r requirements.in @@ -237,8 +242,6 @@ requests==2.25.1 # google-cloud-storage # gspread # requests-oauthlib -requests-oauthlib==1.3.0 - # via google-auth-oauthlib rsa==4.7.2 # via # google-auth @@ -265,6 +268,8 @@ six==1.16.0 # python-utils sqlparse==0.4.1 # via django +typing-extensions==4.1.1 + # via asgiref uritemplate==3.0.1 # via google-api-python-client urllib3==1.26.5 From 78428526e797a46e7c2de4b62273a3307cc6fd20 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 5 Jul 2022 15:59:28 -0700 Subject: [PATCH 44/46] Remove runserver --- .../management/commands/runserver.py | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 contentcuration/contentcuration/management/commands/runserver.py diff --git a/contentcuration/contentcuration/management/commands/runserver.py b/contentcuration/contentcuration/management/commands/runserver.py deleted file mode 100644 index be915118c9..0000000000 --- a/contentcuration/contentcuration/management/commands/runserver.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import print_function - -import subprocess - -from django.contrib.staticfiles.management.commands.runserver import \ - Command as RunserverCommand - -from contentcuration.utils.minio_utils import start_minio - - -class Command(RunserverCommand): - """ - Subclass the RunserverCommand from Staticfiles to run webpack. - """ - - def __init__(self, *args, **kwargs): - super(Command, self).__init__(*args, **kwargs) - - def handle(self, *args, **options): - return super(Command, self).handle(*args, **options) - - def start_minio(self): - self.stdout.write("Starting minio") - - self.minio_process = subprocess.Popen( - ["run_minio.py"], - stdin=subprocess.PIPE, - ) From a11789e3df34a87bc55953c6f45cb6eafe5b1ba4 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 7 Jul 2022 07:39:12 -0700 Subject: [PATCH 45/46] Upgrade django-cte again after upstream fix and release --- requirements-dev.txt | 50 ++++++++++++++++++++--------------------- requirements-docs.txt | 12 +++++----- requirements.in | 2 +- requirements.txt | 52 +++++++++++++++++++++---------------------- 4 files changed, 58 insertions(+), 58 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ae9b4a0607..6371406673 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.6 # To update, run: # # pip-compile requirements-dev.in @@ -67,7 +67,7 @@ coverage[toml]==6.2 # -r requirements-dev.in # codecov # pytest-cov -git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customizable-django-profiler +customizable-django-profiler @ git+https://github.com/someshchaturvedi/customizable-django-profiler.git # via -r requirements-dev.in dataclasses==0.8 # via werkzeug @@ -75,6 +75,12 @@ dill==0.3.4 # via pylint distlib==0.3.4 # via virtualenv +django==3.2.13 + # via + # -c requirements.txt + # django-debug-toolbar + # djangorestframework + # drf-yasg django-concurrent-test-helper==0.7.0 # via -r requirements-dev.in django-debug-panel==0.8.3 @@ -83,12 +89,6 @@ django-debug-toolbar==1.9.1 # via # -r requirements-dev.in # django-debug-panel -django==3.2.13 - # via - # -c requirements.txt - # django-debug-toolbar - # djangorestframework - # drf-yasg djangorestframework==3.12.4 # via # -c requirements.txt @@ -105,15 +105,15 @@ filelock==3.4.1 # via virtualenv flake8==3.4.1 # via -r requirements-dev.in -flask-basicauth==0.2.0 - # via locust -flask-cors==3.0.10 - # via locust flask==2.0.3 # via # flask-basicauth # flask-cors # locust +flask-basicauth==0.2.0 + # via locust +flask-cors==3.0.10 + # via locust flower==0.9.4 # via -r requirements-dev.in fonttools==4.27.1 @@ -237,10 +237,10 @@ pyflakes==1.5.0 # via # autoflake # flake8 -pyinstrument-cext==0.2.4 - # via pyinstrument pyinstrument==3.4.2 # via -r requirements-dev.in +pyinstrument-cext==0.2.4 + # via pyinstrument pylint==2.13.9 # via -r requirements-dev.in pyls-isort==0.2.2 @@ -253,6 +253,15 @@ pyparsing==2.4.7 # via # -c requirements.txt # packaging +pytest==6.2.5 + # via + # -r requirements-dev.in + # pytest-cov + # pytest-django + # pytest-logging + # pytest-pythonpath + # pytest-timeout + # pytest-watch pytest-cov==3.0.0 # via -r requirements-dev.in pytest-django==4.5.2 @@ -265,15 +274,6 @@ pytest-timeout==2.1.0 # via -r requirements-dev.in pytest-watch==4.2.0 # via -r requirements-dev.in -pytest==6.2.5 - # via - # -r requirements-dev.in - # pytest-cov - # pytest-django - # pytest-logging - # pytest-pythonpath - # pytest-timeout - # pytest-watch python-dateutil==2.8.1 # via # -c requirements.txt @@ -308,10 +308,10 @@ rope==1.1.1 # via -r requirements-dev.in roundrobin==0.0.2 # via locust -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml ruamel.yaml==0.17.21 # via drf-yasg +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml service-factory==0.1.6 # via -r requirements-dev.in six==1.16.0 diff --git a/requirements-docs.txt b/requirements-docs.txt index 56f0caf86d..5a5073aefd 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.6 # To update, run: # # pip-compile requirements-docs.in @@ -65,17 +65,17 @@ six==1.16.0 # sphinx snowballstemmer==2.2.0 # via sphinx +sphinx==1.6.4 + # via + # -r requirements-docs.in + # sphinx-intl + # sphinx-rtd-theme sphinx-autobuild==0.7.1 # via -r requirements-docs.in sphinx-intl==2.0.1 # via -r requirements-docs.in sphinx-rtd-theme==1.0.0 # via -r requirements-docs.in -sphinx==1.6.4 - # via - # -r requirements-docs.in - # sphinx-intl - # sphinx-rtd-theme sphinxcontrib-serializinghtml==1.1.5 # via sphinxcontrib-websupport sphinxcontrib-websupport==1.2.4 diff --git a/requirements.in b/requirements.in index 6b3b028750..041c79f766 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ attrs==19.3.0 -django-cte==1.2.0 +django-cte==1.2.1 django-mptt==0.11.0 django-filter==2.4.0 djangorestframework==3.12.4 diff --git a/requirements.txt b/requirements.txt index 856c1a19f9..8764485411 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.6 # To update, run: # # pip-compile requirements.in @@ -38,9 +38,23 @@ chardet==4.0.0 # via requests confusable-homoglyphs==3.2.0 # via django-registration +django==3.2.13 + # via + # -r requirements.in + # django-bulk-update + # django-db-readonly + # django-filter + # django-js-reverse + # django-model-utils + # django-mptt + # django-redis + # django-registration + # django-s3-storage + # djangorestframework + # jsonfield django-bulk-update==2.2.0 # via -r requirements.in -django-cte==1.2.0 +django-cte==1.2.1 # via -r requirements.in django-db-readonly==0.7.0 # via -r requirements.in @@ -68,20 +82,6 @@ django-s3-storage==0.13.4 # via -r requirements.in django-webpack-loader==0.7.0 # via -r requirements.in -django==3.2.13 - # via - # -r requirements.in - # django-bulk-update - # django-db-readonly - # django-filter - # django-js-reverse - # django-model-utils - # django-mptt - # django-redis - # django-registration - # django-s3-storage - # djangorestframework - # jsonfield djangorestframework==3.12.4 # via -r requirements.in future==0.18.2 @@ -95,10 +95,6 @@ google-api-core[grpc]==1.27.0 # google-cloud-logging google-api-python-client==2.4.0 # via -r requirements.in -google-auth-httplib2==0.1.0 - # via google-api-python-client -google-auth-oauthlib==0.4.4 - # via gspread google-auth==1.30.0 # via # google-api-core @@ -108,6 +104,10 @@ google-auth==1.30.0 # google-cloud-core # google-cloud-storage # gspread +google-auth-httplib2==0.1.0 + # via google-api-python-client +google-auth-oauthlib==0.4.4 + # via gspread google-cloud-core==1.6.0 # via # -r requirements.in @@ -194,15 +194,15 @@ protobuf==3.17.0 # proto-plus psycopg2-binary==2.8.6 # via -r requirements.in -pyasn1-modules==0.2.8 - # via - # google-auth - # oauth2client pyasn1==0.4.8 # via # oauth2client # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via + # google-auth + # oauth2client pycountry==17.5.14 # via -r requirements.in pycparser==2.20 @@ -233,8 +233,6 @@ redis==3.5.3 # via # -r requirements.in # django-redis -requests-oauthlib==1.3.0 - # via google-auth-oauthlib requests==2.25.1 # via # -r requirements.in @@ -242,6 +240,8 @@ requests==2.25.1 # google-cloud-storage # gspread # requests-oauthlib +requests-oauthlib==1.3.0 + # via google-auth-oauthlib rsa==4.7.2 # via # google-auth From 487c052496d447d45fd619a4c94c6153903e8539 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 7 Jul 2022 07:40:16 -0700 Subject: [PATCH 46/46] Remove django-cte patch --- contentcuration/contentcuration/apps.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/contentcuration/contentcuration/apps.py b/contentcuration/contentcuration/apps.py index e45938e043..f42624ce9e 100644 --- a/contentcuration/contentcuration/apps.py +++ b/contentcuration/contentcuration/apps.py @@ -14,22 +14,3 @@ def ready(self): if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend(): from contentcuration.utils.minio_utils import ensure_storage_bucket_public ensure_storage_bucket_public() - - self._patch_django_cte_qjoin() - - def _patch_django_cte_qjoin(self): - """ - TODO Remove after the following prs/issues are resolved: - https://github.com/learningequality/studio/pull/3442 - https://github.com/dimagi/django-cte/pull/60 - - @see fix: https://github.com/dimagi/django-cte/pull/50/files - """ - from django_cte.join import QJoin - - class join_field: - class related_model: - class _meta: - local_concrete_fields = () - - QJoin.join_field = join_field