diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index 48d8bc779..b26619ce8 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -57,7 +57,7 @@ jobs: DJANGO_DANDI_INSTANCE_IDENTIFIER: "RRID:SCR_026700" DJANGO_DANDI_DOI_API_PREFIX: "10.82754" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags fetch-tags: true diff --git a/.github/workflows/backend-production-deploy.yml b/.github/workflows/backend-production-deploy.yml index 95e422a07..effe0d562 100644 --- a/.github/workflows/backend-production-deploy.yml +++ b/.github/workflows/backend-production-deploy.yml @@ -27,7 +27,7 @@ jobs: # name: Update release branch # runs-on: ubuntu-22.04 # steps: - # - uses: actions/checkout@v5 + # - uses: actions/checkout@v6 # with: # fetch-depth: 0 # fetch history for all branches and tags # token: ${{ secrets.GH_TOKEN }} # TODO: switch to GITHUB_TOKEN @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-22.04 #needs: reset-release-branch steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags #ref: release diff --git a/.github/workflows/backend-staging-deploy.yml b/.github/workflows/backend-staging-deploy.yml index 597f00757..741c27a0a 100644 --- a/.github/workflows/backend-staging-deploy.yml +++ b/.github/workflows/backend-staging-deploy.yml @@ -23,7 +23,7 @@ jobs: name: Deploy to Heroku runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags diff --git a/.github/workflows/cli-integration.yml b/.github/workflows/cli-integration.yml index bafaa4a00..a64487547 100644 --- a/.github/workflows/cli-integration.yml +++ b/.github/workflows/cli-integration.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Check out this repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags @@ -34,7 +34,7 @@ jobs: docker image save -o dandiarchive-api.tgz dandiarchive/dandiarchive-api - name: Upload Docker image tarball - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: dandiarchive-api.tgz path: dandiarchive-api.tgz @@ -54,7 +54,7 @@ jobs: DANDI_TESTS_PULL_DOCKER_COMPOSE: 0 steps: - name: Download Docker image tarball - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: dandiarchive-api.tgz diff --git a/.github/workflows/frontend-ci.yml b/.github/workflows/frontend-ci.yml index 036b062e2..38ca6cb61 100644 --- a/.github/workflows/frontend-ci.yml +++ b/.github/workflows/frontend-ci.yml @@ -13,7 +13,7 @@ jobs: working-directory: web runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags @@ -87,7 +87,7 @@ jobs: VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags @@ -105,6 +105,9 @@ jobs: - name: Apply migrations to API server run: ./manage.py migrate + - name: Create any cache tables + run: ./manage.py createcachetable + - name: Install test data run: ./manage.py loaddata playwright @@ -130,7 +133,7 @@ jobs: # run the tests cd e2e && npx playwright test - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 if: always() with: name: playwright-report diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0cb733817..ba612620b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')" runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 # fetch history for all branches and tags token: ${{ secrets.GH_TOKEN }} # TODO: switch to GITHUB_TOKEN diff --git a/CHANGELOG.md b/CHANGELOG.md index e339a7ba4..8d0263b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,51 @@ +# v0.21.2 (Tue Jan 13 2026) + +#### 🐛 Bug Fix + +- Disable throttling of unauthenticated users [#2694](https://github.com/dandi/dandi-archive/pull/2694) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 1 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + +# v0.21.1 (Tue Jan 13 2026) + +#### 🐛 Bug Fix + +- Improve performance of asset list endpoint when exact path is provided [#2691](https://github.com/dandi/dandi-archive/pull/2691) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 1 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + +# v0.21.0 (Thu Jan 08 2026) + +#### 🚀 Enhancement + +- Add basic rate limit implementation [#2665](https://github.com/dandi/dandi-archive/pull/2665) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### 🐛 Bug Fix + +- Remove user_migration scripts [#2595](https://github.com/dandi/dandi-archive/pull/2595) ([@brianhelba](https://github.com/brianhelba)) + +#### 🔩 Dependency Updates + +- [gh-actions](deps): Bump actions/upload-artifact from 5 to 6 [#2674](https://github.com/dandi/dandi-archive/pull/2674) ([@dependabot[bot]](https://github.com/dependabot[bot])) +- [gh-actions](deps): Bump actions/download-artifact from 6 to 7 [#2675](https://github.com/dandi/dandi-archive/pull/2675) ([@dependabot[bot]](https://github.com/dependabot[bot])) +- [gh-actions](deps): Bump actions/checkout from 5 to 6 [#2651](https://github.com/dandi/dandi-archive/pull/2651) ([@dependabot[bot]](https://github.com/dependabot[bot])) + +#### Authors: 3 + +- [@dependabot[bot]](https://github.com/dependabot[bot]) +- Brian Helba ([@brianhelba](https://github.com/brianhelba)) +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + # v0.20.0 (Wed Dec 17 2025) ### Release Notes diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8645f271c..65349f538 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,6 +19,7 @@ This is the simplest configuration for developers to start with. 1. From VSCode, use `Ctrl-Shift-p` and run the command `Dev Containers: Reopen in Container`. 1. From the VSCode built-in terminal, run `./manage.py migrate`. +1. Run `docker compose run --rm django ./manage.py createcachetable` 1. From the VSCode built-in terminal, run `./manage.py createsuperuser --email $(git config user.email)` and follow the prompts. 1. From the VSCode built-in terminal, run `./manage.py create_dev_dandiset --owner $(git config user.email)` to create a dummy dandiset to start working with. @@ -37,6 +38,7 @@ This configuration also uses containers, but with Docker Compose instead of VSco ### Initial Setup 1. Install [Docker Compose](https://docs.docker.com/compose/install/) 1. Run `docker compose run --rm django ./manage.py migrate` +1. Run `docker compose run --rm django ./manage.py createcachetable` 1. Run `docker compose run --rm django ./manage.py createsuperuser --email $(git config user.email)` and follow the prompts to create your own user. This sets your username to your git email to ensure parity with how GitHub logins work. You can also replace the command substitution expression with a literal email address, or omit the `--email` option entirely to run the command in interactive mode. @@ -67,6 +69,7 @@ but allows developers to run Python code on their native system. 1. [Install `uv`](https://docs.astral.sh/uv/getting-started/installation/) 1. Run `export UV_ENV_FILE=./dev/.env.docker-compose-native` 1. Run `./manage.py migrate` +1. Run `docker compose run --rm django ./manage.py createcachetable` 1. Run `./manage.py createsuperuser --email $(git config user.email)` and follow the prompts. 1. Run `./manage.py create_dev_dandiset --owner $(git config user.email)` to create a dummy dandiset to start working with. diff --git a/dandiapi/api/__init__.py b/dandiapi/api/__init__.py index 0fa5df86a..e69de29bb 100644 --- a/dandiapi/api/__init__.py +++ b/dandiapi/api/__init__.py @@ -1,4 +0,0 @@ -from __future__ import annotations - -# TODO: remove this after migration is complete -import dandiapi.api.user_migration # noqa: F401 diff --git a/dandiapi/api/management/commands/depose_placeholder.py b/dandiapi/api/management/commands/depose_placeholder.py deleted file mode 100644 index 651916218..000000000 --- a/dandiapi/api/management/commands/depose_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations - -from django.contrib.auth.models import User -import djclick as click - -from dandiapi.api.user_migration import copy_ownership - - -@click.command() -@click.argument('placeholder_email') -@click.argument('github_email') -def depose_placeholder(*, placeholder_email: str, github_email: str): - placeholder_user = User.objects.get(email=placeholder_email) - github_user = User.objects.get(email=github_email) - - click.echo(f'Replacing {placeholder_email} with {github_email}') - copy_ownership(placeholder_user, github_user) - - click.echo(f'Deleting {placeholder_email}') - placeholder_user.delete() diff --git a/dandiapi/api/management/commands/depose_placeholders.py b/dandiapi/api/management/commands/depose_placeholders.py deleted file mode 100644 index 1fd7d5cfd..000000000 --- a/dandiapi/api/management/commands/depose_placeholders.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import annotations - -from django.contrib.auth.models import User -import djclick as click - -from dandiapi.api.user_migration import depose_placeholder - - -@click.command() -def depose_placeholders(): - for user in User.objects.all(): - depose_placeholder(user) diff --git a/dandiapi/api/management/commands/list_placeholders.py b/dandiapi/api/management/commands/list_placeholders.py deleted file mode 100644 index 86f462af7..000000000 --- a/dandiapi/api/management/commands/list_placeholders.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from django.contrib.auth.models import User -import djclick as click - - -@click.command() -def list_placeholders(): - """ - Return a list of all user emails who still have a placeholder in Django. - - This may be useful to contact all users who have yet to log in. - """ - placeholder_emails = User.objects.filter(email__startswith='placeholder_').values_list( - 'email', flat=True - ) - emails = [email[12:] for email in placeholder_emails] - click.echo(' '.join(emails)) diff --git a/dandiapi/api/migrations/0030_alter_asset_path.py b/dandiapi/api/migrations/0030_alter_asset_path.py new file mode 100644 index 000000000..4f4fc68cb --- /dev/null +++ b/dandiapi/api/migrations/0030_alter_asset_path.py @@ -0,0 +1,25 @@ +# Generated by Django 5.2.7 on 2026-01-12 21:55 +from __future__ import annotations + +from django.db import migrations, models + +import dandiapi.api.models.asset + + +class Migration(migrations.Migration): + dependencies = [ + ('api', '0029_merge'), + ] + + operations = [ + migrations.AlterField( + model_name='asset', + name='path', + field=models.CharField( + db_collation='C', + db_index=True, + max_length=512, + validators=[dandiapi.api.models.asset.validate_asset_path], + ), + ), + ] diff --git a/dandiapi/api/models/asset.py b/dandiapi/api/models/asset.py index 53dd2931e..ef3756f94 100644 --- a/dandiapi/api/models/asset.py +++ b/dandiapi/api/models/asset.py @@ -104,7 +104,9 @@ class Asset(PublishableMetadataMixin, TimeStampedModel): UUID_REGEX = r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}' asset_id = models.UUIDField(unique=True, default=uuid.uuid4) - path = models.CharField(max_length=512, validators=[validate_asset_path], db_collation='C') + path = models.CharField( + max_length=512, validators=[validate_asset_path], db_collation='C', db_index=True + ) blob = models.ForeignKey( AssetBlob, related_name='assets', on_delete=models.CASCADE, null=True, blank=True ) diff --git a/dandiapi/api/tests/test_info.py b/dandiapi/api/tests/test_info.py index 2c02b0870..da82b8b2b 100644 --- a/dandiapi/api/tests/test_info.py +++ b/dandiapi/api/tests/test_info.py @@ -1,8 +1,10 @@ from __future__ import annotations from dandischema.conf import get_instance_config +import pytest +@pytest.mark.django_db def test_rest_info_instance_config_include_none(api_client): resp = api_client.get('/api/info/') assert resp.status_code == 200 diff --git a/dandiapi/api/tests/test_schema.py b/dandiapi/api/tests/test_schema.py index 370b835f3..29362a041 100644 --- a/dandiapi/api/tests/test_schema.py +++ b/dandiapi/api/tests/test_schema.py @@ -14,6 +14,7 @@ PublishedAsset, ], ) +@pytest.mark.django_db def test_schema_latest(api_client, model: CommonModel): """Test that the schema endpoints return valid schemas.""" resp = api_client.get('/api/schemas/', {'model': model.__name__}) @@ -30,6 +31,7 @@ def test_schema_latest(api_client, model: CommonModel): assert schema == expected_schema +@pytest.mark.django_db def test_schema_unsupported_model(api_client): """Test that the schema endpoint returns an error when passed invalid choice.""" resp = api_client.get('/api/schemas/', {'model': 'NotAValidModel'}) diff --git a/dandiapi/api/throttling.py b/dandiapi/api/throttling.py new file mode 100644 index 000000000..889fe8287 --- /dev/null +++ b/dandiapi/api/throttling.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from rest_framework.throttling import UserRateThrottle + + +# This is not currently used, but if we ever choose to rate limit logged-in users, +# this is how we can accomplish that, without applying it to admins. +class DandiUserRateThrottle(UserRateThrottle): + def get_cache_key(self, request, view): + # Don't rate limit admin users + if request.user and (request.user.is_staff or request.user.is_superuser): + return None + + return super().get_cache_key(request, view) diff --git a/dandiapi/api/user_migration.py b/dandiapi/api/user_migration.py deleted file mode 100644 index 51c804c16..000000000 --- a/dandiapi/api/user_migration.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import logging - -from allauth.account.signals import user_logged_in -from django.dispatch import receiver - -logger = logging.getLogger(__name__) - - -def copy_ownership(placeholder_user, user): - """Copy dandiset ownership from a placeholder user to the real user.""" - from guardian.shortcuts import assign_perm, get_objects_for_user, remove_perm - - from dandiapi.api.models import Dandiset - - owned_dandisets = get_objects_for_user(placeholder_user, 'owner', Dandiset) - logger.info('%s owns %s', placeholder_user, owned_dandisets) - for dandiset in owned_dandisets: - logger.info('Moving ownership on %s', dandiset.identifier) - assign_perm('owner', user, dandiset) - remove_perm('owner', placeholder_user, dandiset) - - -def depose_placeholder(user): - """Replace a placeholder user with a real user, if a placeholder exists.""" - from django.contrib.auth.models import User - - placeholder_email = 'placeholder_' + user.email - try: - placeholder_user = User.objects.get(email=placeholder_email) - except User.DoesNotExist: - # No placeholder user, nothing to do - return - - logger.info('Replacing %s with %s', placeholder_user, user) - copy_ownership(placeholder_user, user) - # The placeholder user has no further purpose, delete it - logger.info('Deleting %s', placeholder_user) - placeholder_user.delete() - - -@receiver(user_logged_in) -def user_log_in_listener(*, sender, user, **kwargs): - """Attempt replace a placeholder user every time a real user logs in.""" - depose_placeholder(user) diff --git a/dandiapi/api/views/asset.py b/dandiapi/api/views/asset.py index 34daed73d..de3a2350f 100644 --- a/dandiapi/api/views/asset.py +++ b/dandiapi/api/views/asset.py @@ -423,9 +423,20 @@ def list(self, request, *args, **kwargs): version=self.kwargs['versions__version'], ) + # Use custom pagination class to reduce unnecessary counts of assets + paginator = LazyPagination() + # Apply filtering from included filter class first asset_queryset = self.filter_queryset(version.assets.all()) + # Check if the path query arg is pointing at a direct path. + # If that's the case, just retrieve the single asset. + path = self.request.query_params.get('path') + if path: + assets = Asset.objects.filter(path=path, versions=version) + if assets.exists(): + asset_queryset = assets + # Filter query to only zarr assets, if requested zarr_only = serializer.validated_data['zarr'] if zarr_only: @@ -442,8 +453,6 @@ def list(self, request, *args, **kwargs): asset_queryset = asset_queryset.filter(path__iregex=glob_pattern.replace('\\*', '.*')) # Retrieve just the first N asset IDs, and use them for pagination - # Use custom pagination class to reduce unnecessary counts of assets - paginator = LazyPagination() qs = asset_queryset.values_list('id', flat=True) page_of_asset_ids = paginator.paginate_queryset(qs, request=self.request, view=self) diff --git a/dandiapi/settings/base.py b/dandiapi/settings/base.py index 5eae2f8fd..f257a8a78 100644 --- a/dandiapi/settings/base.py +++ b/dandiapi/settings/base.py @@ -3,6 +3,7 @@ from datetime import timedelta import logging from pathlib import Path +import sys from typing import TYPE_CHECKING, cast from urllib.parse import urlunparse @@ -147,6 +148,26 @@ REST_FRAMEWORK['DEFAULT_PAGINATION_CLASS'] = 'dandiapi.api.views.pagination.DandiPagination' REST_FRAMEWORK['EXCEPTION_HANDLER'] = 'dandiapi.drf_utils.rewrap_django_core_exceptions' +# Throttling configuration +# TODO: Uncomment once throttling is re-enabled +# REST_FRAMEWORK['DEFAULT_THROTTLE_CLASSES'] = [ +# 'rest_framework.throttling.AnonRateThrottle', +# ] + +# By default, set request rate limit to a very high number, effectively disabling it. +# This is done to preserve the rate limiting behavior between dev and prod, +# without actually impeding developer experience. +REST_FRAMEWORK['DEFAULT_THROTTLE_RATES'] = { + 'anon': f'{sys.maxsize}/minute', +} + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'dandi_cache_table', + } +} + REST_FRAMEWORK_EXTENSIONS = {'DEFAULT_PARENT_LOOKUP_KWARG_NAME_PREFIX': ''} # Clearing out the stock `SWAGGER_SETTINGS` variable causes a Django login diff --git a/dandiapi/settings/production.py b/dandiapi/settings/production.py index 523ec225e..a45e507a5 100644 --- a/dandiapi/settings/production.py +++ b/dandiapi/settings/production.py @@ -44,6 +44,12 @@ }, } +# In production, enable rate limiting for unauthenticated users +REST_FRAMEWORK['DEFAULT_THROTTLE_RATES'] = { + 'anon': '500/minute', +} + + DANDI_DEV_EMAIL: str = env.str('DJANGO_DANDI_DEV_EMAIL') DANDI_ADMIN_EMAIL: str = env.str('DJANGO_DANDI_ADMIN_EMAIL') diff --git a/web/src/components/FileBrowser/FileUploadInstructions.vue b/web/src/components/FileBrowser/FileUploadInstructions.vue index 2d69117ac..de343e162 100644 --- a/web/src/components/FileBrowser/FileUploadInstructions.vue +++ b/web/src/components/FileBrowser/FileUploadInstructions.vue @@ -26,7 +26,7 @@
> cd {{ dandisetIdentifier }}
> dandi organize <source_folder> -f dry
> dandi organize <source_folder>
-
> dandi upload
+
> dandi upload -i {{ instanceName }}
@@ -48,13 +48,21 @@