From 4f4bdc7b6bfc3c5b22b1456366c5ec87d24a6688 Mon Sep 17 00:00:00 2001 From: Michael Nahkies Date: Sun, 15 Nov 2020 09:55:53 +0000 Subject: [PATCH 001/170] chore: fix install instructions in readme, .gitignore venv / secret.txt (#68) --- .gitignore | 3 ++- README.md | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7f220af..2c3f6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,10 @@ Session.vim /.coverage /tmp /media +/.idea __pycache__ .*.swp - /etebase_server_settings.py +/secret.txt diff --git a/README.md b/README.md index dc05b31..9940271 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,10 @@ Then just clone the git repo and set up this app: git clone https://github.com/etesync/server.git etebase cd etebase -git checkout etebase # Set up the environment and deps -virtualenv -p python3 venv # If doesn't work, try: virtualenv3 venv -source venv/bin/activate +virtualenv -p python3 .venv # If doesn't work, try: virtualenv3 .venv +source .venv/bin/activate pip install -r requirements.txt ``` From 13a46cb1dbf44326b3cd8c22a37b170e5894424c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 19 Nov 2020 10:12:12 +0200 Subject: [PATCH 002/170] Myauth: add missing migration. Fixes #70 --- myauth/migrations/0003_auto_20201119_0810.py | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 myauth/migrations/0003_auto_20201119_0810.py diff --git a/myauth/migrations/0003_auto_20201119_0810.py b/myauth/migrations/0003_auto_20201119_0810.py new file mode 100644 index 0000000..cfd2ec3 --- /dev/null +++ b/myauth/migrations/0003_auto_20201119_0810.py @@ -0,0 +1,37 @@ +# Generated by Django 3.1.1 on 2020-11-19 08:10 + +from django.db import migrations, models +import myauth.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("myauth", "0002_auto_20200515_0801"), + ] + + operations = [ + migrations.AlterModelManagers( + name="user", + managers=[ + ("objects", myauth.models.UserManager()), + ], + ), + migrations.AlterField( + model_name="user", + name="first_name", + field=models.CharField(blank=True, max_length=150, verbose_name="first name"), + ), + migrations.AlterField( + model_name="user", + name="username", + field=models.CharField( + error_messages={"unique": "A user with that username already exists."}, + help_text="Required. 150 characters or fewer. Letters, digits and ./-/_ only.", + max_length=150, + unique=True, + validators=[myauth.models.UnicodeUsernameValidator()], + verbose_name="username", + ), + ), + ] From 1c8f7cdc609065c9e1700c8f5f159f65f4558cec Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 19 Nov 2020 10:12:42 +0200 Subject: [PATCH 003/170] Update changelog. --- ChangeLog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index d3adef0..3de4eef 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,8 @@ # Changelog +## Version 0.5.3 +* Add missing migration + ## Version 0.5.2 * Fix issues with host verification failing with a custom port * Add env variable to change configuration file path. From 49da4ea6660a3bbcc0afa3764ca8f083f75afa84 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 19 Nov 2020 14:42:57 +0200 Subject: [PATCH 004/170] README: document MEDIA_ROOT and add a section about backups --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9940271..f347133 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,14 @@ To use the easy configuration file rename it to `etebase-server.ini` and place i There is also a [wikipage](https://github.com/etesync/server/wiki/Basic-Setup-Etebase-(EteSync-v2)) detailing this basic setup. Some particular settings that should be edited are: - * [`ALLOWED_HOSTS`](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-ALLOWED_HOSTS) + * [`ALLOWED_HOSTS`](https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-ALLOWED_HOSTS) -- this is the list of host/domain names or addresses on which the app will be served - * [`DEBUG`](https://docs.djangoproject.com/en/1.11/ref/settings/#debug) + * [`DEBUG`](https://docs.djangoproject.com/en/dev/ref/settings/#debug) -- handy for debugging, set to `False` for production - * [`SECRET_KEY`](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY) + * [`MEDIA_ROOT`](https://docs.djangoproject.com/en/dev/ref/settings/#media-root) + -- the path to the directory that will hold user data. + * [`SECRET_KEY`](https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY) -- an ephemeral secret used for various cryptographic signing and token generation purposes. See below for how default configuration of `SECRET_KEY` works for this project. @@ -76,6 +78,12 @@ Etebase is based on Django so you should refer to one of the following The webserver should also be configured to serve Etebase using TLS. A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-Etebase) as well. +# Data locations and backups + +The server stores user data in two different locations that need to be backed up: +1. The database - how to backup depends on which database you use. +2. The `MEDIA_ROOT` - the path where user data is stored. + # Usage Create yourself an admin user: From d893d35c6f792b06888cdf990937e83f01eb14e5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 20 Nov 2020 18:11:35 +0200 Subject: [PATCH 005/170] Fix the host checks to only check against hostname. Fixes https://github.com/etesync/etesync-web/issues/183 As discussed in #66 Continuation of 843b59a0ac1e2076a514b9c52fc5c6941b27dff0. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 7dd7526..f3abcbc 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -708,7 +708,7 @@ class AuthenticationViewSet(viewsets.ViewSet): elif challenge_data["userId"] != user.id: content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif not settings.DEBUG and host.split(":", 1)[0] != request.get_host(): + elif not settings.DEBUG and host.split(':', 1)[0] != request.get_host().split(':', 1)[0]: detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) content = {"code": "wrong_host", "detail": detail} return Response(content, status=status.HTTP_400_BAD_REQUEST) From 5792cd5418283d2ad590e8ef1bcdbbe601ed7365 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 27 Nov 2020 10:29:24 +0200 Subject: [PATCH 006/170] README: add a TL;DR for licensing Added it following questions via email. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f347133..12f7747 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,16 @@ Here are the update steps: 4. Run the migration tool to migrate all of your data. 5. Add your new EteSync 2.0 accounts to all of your devices. +# License + +Etebase is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation. See the [LICENSE](./LICENSE) for more information. + +A quick summary can be found [on tldrlegal](https://tldrlegal.com/license/gnu-affero-general-public-license-v3-(agpl-3.0)). Though in even simpler terms (not part of the license, and not legal advice): you can use it in however way you want, including self-hosting and commercial offerings as long as you release the code to any modifications you have made to the server software (clients are not affected). + +## Commercial licensing + +For commercial licensing options, contact license@etebase.com + # Supporting Etebase Please consider registering an account even if you self-host in order to support the development of Etebase, or visit the [contribution](https://www.etesync.com/contribute/) for more information on how to support the service. From bb4a8c998e147c5c55e6c75a15cd8acac70820fb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 29 Nov 2020 17:02:49 +0200 Subject: [PATCH 007/170] README: updateinformation about passing the Host header As mentioned in https://github.com/etesync/server/issues/75#issuecomment-735370709 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 12f7747..cfb0be9 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ There is also a [wikipage](https://github.com/etesync/server/wiki/Basic-Setup-Et Some particular settings that should be edited are: * [`ALLOWED_HOSTS`](https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-ALLOWED_HOSTS) -- this is the list of host/domain names or addresses on which the app -will be served +will be served. For example: `etebase.example.com` * [`DEBUG`](https://docs.djangoproject.com/en/dev/ref/settings/#debug) -- handy for debugging, set to `False` for production * [`MEDIA_ROOT`](https://docs.djangoproject.com/en/dev/ref/settings/#media-root) @@ -78,6 +78,8 @@ Etebase is based on Django so you should refer to one of the following The webserver should also be configured to serve Etebase using TLS. A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-Etebase) as well. +The Etebase server needs to be aware of the URL it's been served as, so make sure to forward the `Host` header to the server if using a reverse proxy. For example, you would need to use the following directive in nginx: `proxy_set_header Host $host;`. + # Data locations and backups The server stores user data in two different locations that need to be backed up: From 7513b058ba51ec911c4ba6a0f7851889b461b055 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 1 Dec 2020 10:08:04 +0200 Subject: [PATCH 008/170] FUNDING.yml: add funding links --- .github/FUNDING.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..3ece078 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +github: etesync +custom: https://www.etesync.com/contribute/#donate From c790b5f4899fbb6f17567e6ed9053b741f227300 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 1 Dec 2020 12:45:23 +0200 Subject: [PATCH 009/170] Reformat some files using black. --- django_etebase/exceptions.py | 5 +++- django_etebase/migrations/0001_initial.py | 20 ++++++++++++---- .../migrations/0003_collectioninvitation.py | 4 +++- .../0004_collectioninvitation_version.py | 4 +++- .../migrations/0005_auto_20200526_1021.py | 6 ++++- .../0013_collectionmemberremoved.py | 4 +++- .../migrations/0014_auto_20200602_1558.py | 6 ++++- .../0015_collectionitemrevision_salt.py | 4 +++- .../migrations/0016_auto_20200623_0820.py | 15 +++++++++--- ...0020_remove_collectionitemrevision_salt.py | 5 +++- .../migrations/0022_auto_20200804_1059.py | 5 +++- .../migrations/0025_auto_20200804_1216.py | 10 ++++++-- .../migrations/0026_auto_20200907_0752.py | 12 ++++++++-- .../migrations/0029_auto_20200907_0801.py | 10 ++++++-- django_etebase/models.py | 16 ++++++++++--- django_etebase/serializers.py | 24 +++++++++++++++---- django_etebase/views.py | 7 ++++-- 17 files changed, 125 insertions(+), 32 deletions(-) diff --git a/django_etebase/exceptions.py b/django_etebase/exceptions.py index f3aa08a..437a71c 100644 --- a/django_etebase/exceptions.py +++ b/django_etebase/exceptions.py @@ -4,6 +4,9 @@ from rest_framework import serializers, status class EtebaseValidationError(serializers.ValidationError): def __init__(self, code, detail, status_code=status.HTTP_400_BAD_REQUEST): super().__init__( - {"code": code, "detail": detail,} + { + "code": code, + "detail": detail, + } ) self.status_code = status_code diff --git a/django_etebase/migrations/0001_initial.py b/django_etebase/migrations/0001_initial.py index 86f0fa6..42ee022 100644 --- a/django_etebase/migrations/0001_initial.py +++ b/django_etebase/migrations/0001_initial.py @@ -33,7 +33,9 @@ class Migration(migrations.Migration): ("version", models.PositiveSmallIntegerField()), ("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], - options={"unique_together": {("uid", "owner")},}, + options={ + "unique_together": {("uid", "owner")}, + }, ), migrations.CreateModel( name="CollectionItem", @@ -61,7 +63,9 @@ class Migration(migrations.Migration): ), ), ], - options={"unique_together": {("uid", "collection")},}, + options={ + "unique_together": {("uid", "collection")}, + }, ), migrations.CreateModel( name="CollectionItemChunk", @@ -122,7 +126,9 @@ class Migration(migrations.Migration): ), ), ], - options={"unique_together": {("item", "current")},}, + options={ + "unique_together": {("item", "current")}, + }, ), migrations.CreateModel( name="RevisionChunkRelation", @@ -145,7 +151,9 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ("id",),}, + options={ + "ordering": ("id",), + }, ), migrations.CreateModel( name="CollectionMember", @@ -170,6 +178,8 @@ class Migration(migrations.Migration): ), ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], - options={"unique_together": {("user", "collection")},}, + options={ + "unique_together": {("user", "collection")}, + }, ), ] diff --git a/django_etebase/migrations/0003_collectioninvitation.py b/django_etebase/migrations/0003_collectioninvitation.py index 1b416ab..4aa05c9 100644 --- a/django_etebase/migrations/0003_collectioninvitation.py +++ b/django_etebase/migrations/0003_collectioninvitation.py @@ -54,6 +54,8 @@ class Migration(migrations.Migration): ), ), ], - options={"unique_together": {("user", "fromMember")},}, + options={ + "unique_together": {("user", "fromMember")}, + }, ), ] diff --git a/django_etebase/migrations/0004_collectioninvitation_version.py b/django_etebase/migrations/0004_collectioninvitation_version.py index 29ae3f1..40a290e 100644 --- a/django_etebase/migrations/0004_collectioninvitation_version.py +++ b/django_etebase/migrations/0004_collectioninvitation_version.py @@ -11,6 +11,8 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( - model_name="collectioninvitation", name="version", field=models.PositiveSmallIntegerField(default=1), + model_name="collectioninvitation", + name="version", + field=models.PositiveSmallIntegerField(default=1), ), ] diff --git a/django_etebase/migrations/0005_auto_20200526_1021.py b/django_etebase/migrations/0005_auto_20200526_1021.py index 3775277..9855ff1 100644 --- a/django_etebase/migrations/0005_auto_20200526_1021.py +++ b/django_etebase/migrations/0005_auto_20200526_1021.py @@ -10,5 +10,9 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameField(model_name="userinfo", old_name="pubkey", new_name="loginPubkey",), + migrations.RenameField( + model_name="userinfo", + old_name="pubkey", + new_name="loginPubkey", + ), ] diff --git a/django_etebase/migrations/0013_collectionmemberremoved.py b/django_etebase/migrations/0013_collectionmemberremoved.py index 4481e80..d8c57ea 100644 --- a/django_etebase/migrations/0013_collectionmemberremoved.py +++ b/django_etebase/migrations/0013_collectionmemberremoved.py @@ -33,6 +33,8 @@ class Migration(migrations.Migration): ), ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], - options={"unique_together": {("user", "collection")},}, + options={ + "unique_together": {("user", "collection")}, + }, ), ] diff --git a/django_etebase/migrations/0014_auto_20200602_1558.py b/django_etebase/migrations/0014_auto_20200602_1558.py index 42bed52..ec0de46 100644 --- a/django_etebase/migrations/0014_auto_20200602_1558.py +++ b/django_etebase/migrations/0014_auto_20200602_1558.py @@ -10,5 +10,9 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameField(model_name="userinfo", old_name="encryptedSeckey", new_name="encryptedContent",), + migrations.RenameField( + model_name="userinfo", + old_name="encryptedSeckey", + new_name="encryptedContent", + ), ] diff --git a/django_etebase/migrations/0015_collectionitemrevision_salt.py b/django_etebase/migrations/0015_collectionitemrevision_salt.py index c4dc3e9..3a13baa 100644 --- a/django_etebase/migrations/0015_collectionitemrevision_salt.py +++ b/django_etebase/migrations/0015_collectionitemrevision_salt.py @@ -11,6 +11,8 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( - model_name="collectionitemrevision", name="salt", field=models.BinaryField(default=b"", editable=True), + model_name="collectionitemrevision", + name="salt", + field=models.BinaryField(default=b"", editable=True), ), ] diff --git a/django_etebase/migrations/0016_auto_20200623_0820.py b/django_etebase/migrations/0016_auto_20200623_0820.py index a273b0d..14871a2 100644 --- a/django_etebase/migrations/0016_auto_20200623_0820.py +++ b/django_etebase/migrations/0016_auto_20200623_0820.py @@ -21,7 +21,16 @@ class Migration(migrations.Migration): to="django_etebase.CollectionItem", ), ), - migrations.AlterUniqueTogether(name="collection", unique_together=set(),), - migrations.RemoveField(model_name="collection", name="uid",), - migrations.RemoveField(model_name="collection", name="version",), + migrations.AlterUniqueTogether( + name="collection", + unique_together=set(), + ), + migrations.RemoveField( + model_name="collection", + name="uid", + ), + migrations.RemoveField( + model_name="collection", + name="version", + ), ] diff --git a/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py b/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py index 21d0337..1b69a5c 100644 --- a/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py +++ b/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py @@ -10,5 +10,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RemoveField(model_name="collectionitemrevision", name="salt",), + migrations.RemoveField( + model_name="collectionitemrevision", + name="salt", + ), ] diff --git a/django_etebase/migrations/0022_auto_20200804_1059.py b/django_etebase/migrations/0022_auto_20200804_1059.py index 60af33f..bc4bad9 100644 --- a/django_etebase/migrations/0022_auto_20200804_1059.py +++ b/django_etebase/migrations/0022_auto_20200804_1059.py @@ -10,5 +10,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterUniqueTogether(name="collectionitemchunk", unique_together={("item", "uid")},), + migrations.AlterUniqueTogether( + name="collectionitemchunk", + unique_together={("item", "uid")}, + ), ] diff --git a/django_etebase/migrations/0025_auto_20200804_1216.py b/django_etebase/migrations/0025_auto_20200804_1216.py index 91bf4c8..995d275 100644 --- a/django_etebase/migrations/0025_auto_20200804_1216.py +++ b/django_etebase/migrations/0025_auto_20200804_1216.py @@ -18,6 +18,12 @@ class Migration(migrations.Migration): on_delete=django.db.models.deletion.CASCADE, related_name="chunks", to="django_etebase.Collection" ), ), - migrations.AlterUniqueTogether(name="collectionitemchunk", unique_together={("collection", "uid")},), - migrations.RemoveField(model_name="collectionitemchunk", name="item",), + migrations.AlterUniqueTogether( + name="collectionitemchunk", + unique_together={("collection", "uid")}, + ), + migrations.RemoveField( + model_name="collectionitemchunk", + name="item", + ), ] diff --git a/django_etebase/migrations/0026_auto_20200907_0752.py b/django_etebase/migrations/0026_auto_20200907_0752.py index 3283654..1c90610 100644 --- a/django_etebase/migrations/0026_auto_20200907_0752.py +++ b/django_etebase/migrations/0026_auto_20200907_0752.py @@ -10,6 +10,14 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameField(model_name="collectioninvitation", old_name="accessLevel", new_name="accessLevelOld",), - migrations.RenameField(model_name="collectionmember", old_name="accessLevel", new_name="accessLevelOld",), + migrations.RenameField( + model_name="collectioninvitation", + old_name="accessLevel", + new_name="accessLevelOld", + ), + migrations.RenameField( + model_name="collectionmember", + old_name="accessLevel", + new_name="accessLevelOld", + ), ] diff --git a/django_etebase/migrations/0029_auto_20200907_0801.py b/django_etebase/migrations/0029_auto_20200907_0801.py index f3bfe61..1e00ffe 100644 --- a/django_etebase/migrations/0029_auto_20200907_0801.py +++ b/django_etebase/migrations/0029_auto_20200907_0801.py @@ -10,6 +10,12 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RemoveField(model_name="collectioninvitation", name="accessLevelOld",), - migrations.RemoveField(model_name="collectionmember", name="accessLevelOld",), + migrations.RemoveField( + model_name="collectioninvitation", + name="accessLevelOld", + ), + migrations.RemoveField( + model_name="collectionmember", + name="accessLevelOld", + ), ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 691947d..00bbf71 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -182,7 +182,10 @@ class CollectionMember(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) encryptionKey = models.BinaryField(editable=True, blank=False, null=False) collectionType = models.ForeignKey(CollectionType, on_delete=models.PROTECT, null=True) - accessLevel = models.IntegerField(choices=AccessLevels.choices, default=AccessLevels.READ_ONLY,) + accessLevel = models.IntegerField( + choices=AccessLevels.choices, + default=AccessLevels.READ_ONLY, + ) class Meta: unique_together = ("user", "collection") @@ -193,7 +196,11 @@ class CollectionMember(models.Model): def revoke(self): with transaction.atomic(): CollectionMemberRemoved.objects.update_or_create( - collection=self.collection, user=self.user, defaults={"stoken": Stoken.objects.create(),}, + collection=self.collection, + user=self.user, + defaults={ + "stoken": Stoken.objects.create(), + }, ) self.delete() @@ -220,7 +227,10 @@ class CollectionInvitation(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="incoming_invitations", on_delete=models.CASCADE) signedEncryptionKey = models.BinaryField(editable=False, blank=False, null=False) - accessLevel = models.IntegerField(choices=AccessLevels.choices, default=AccessLevels.READ_ONLY,) + accessLevel = models.IntegerField( + choices=AccessLevels.choices, + default=AccessLevels.READ_ONLY, + ) class Meta: unique_together = ("user", "fromMember") diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index ef3b296..b0769ef 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -156,7 +156,11 @@ class BetterErrorsMixin: else: message = str(error) ret.append( - {"field": field_name, "code": error.code, "detail": message,} + { + "field": field_name, + "code": error.code, + "detail": message, + } ) return ret @@ -169,7 +173,11 @@ class BetterErrorsMixin: raise EtebaseValidationError(err.code, err.message) raise serializers.ValidationError( - {"code": "field_errors", "detail": "Field validations failed.", "errors": errors,} + { + "code": "field_errors", + "detail": "Field validations failed.", + "errors": errors, + } ) @@ -343,7 +351,11 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField(source="user", read_only=True, style={"base_template": "input.html"},) + username = UserSlugRelatedField( + source="user", + read_only=True, + style={"base_template": "input.html"}, + ) class Meta: model = models.CollectionMember @@ -365,7 +377,11 @@ class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer) class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField(source="user", queryset=User.objects, style={"base_template": "input.html"},) + username = UserSlugRelatedField( + source="user", + queryset=User.objects, + style={"base_template": "input.html"}, + ) collection = serializers.CharField(source="collection.uid") fromUsername = BinaryBase64Field(source="fromMember.user.username", read_only=True) fromPubkey = BinaryBase64Field(source="fromMember.user.userinfo.pubkey", read_only=True) diff --git a/django_etebase/views.py b/django_etebase/views.py index f3abcbc..8c05948 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -413,7 +413,10 @@ class CollectionItemViewSet(BaseViewSet): return Response(ret, status=status.HTTP_200_OK) return Response( - {"items": serializer.errors, "deps": deps_serializer.errors if deps is not None else [],}, + { + "items": serializer.errors, + "deps": deps_serializer.errors if deps is not None else [], + }, status=status.HTTP_409_CONFLICT, ) @@ -708,7 +711,7 @@ class AuthenticationViewSet(viewsets.ViewSet): elif challenge_data["userId"] != user.id: content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif not settings.DEBUG and host.split(':', 1)[0] != request.get_host().split(':', 1)[0]: + elif not settings.DEBUG and host.split(":", 1)[0] != request.get_host().split(":", 1)[0]: detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) content = {"code": "wrong_host", "detail": detail} return Response(content, status=status.HTTP_400_BAD_REQUEST) From bb070639a3abfcd2ad39466833cc71abf8db4599 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 4 Dec 2020 18:19:54 +0200 Subject: [PATCH 010/170] Collection: fix the slow performance when calculating stoken. We were running a very expensive query instead of the much simpler and more efficient alternative we just introduced. --- django_etebase/models.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index 00bbf71..b8ead94 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -17,7 +17,7 @@ from pathlib import Path from django.db import models, transaction from django.conf import settings from django.core.validators import RegexValidator -from django.db.models import Q +from django.db.models import Max from django.utils.functional import cached_property from django.utils.crypto import get_random_string @@ -56,18 +56,13 @@ class Collection(models.Model): @cached_property def stoken(self): - stoken = ( - Stoken.objects.filter( - Q(collectionitemrevision__item__collection=self) | Q(collectionmember__collection=self) - ) - .order_by("id") - .last() - ) - - if stoken is None: + stoken1 = self.items.aggregate(stoken=Max("revisions__stoken"))["stoken"] or 0 + stoken2 = self.members.aggregate(stoken=Max("stoken"))["stoken"] or 0 + stoken_id = max(stoken1, stoken2) + if stoken_id == 0: raise Exception("stoken is None. Should never happen") - return stoken.uid + return Stoken.objects.get(id=stoken_id).uid def validate_unique(self, exclude=None): super().validate_unique(exclude=exclude) From 4ce96e043e55070e7a0eed98592c545dc435c2d4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 4 Dec 2020 18:55:22 +0200 Subject: [PATCH 011/170] Collection: further improve stoken performance. We merged the two queries into one and we made it like in the view, so we can now merge the two instead of having two implementations. --- django_etebase/models.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index b8ead94..d1095fb 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -17,7 +17,8 @@ from pathlib import Path from django.db import models, transaction from django.conf import settings from django.core.validators import RegexValidator -from django.db.models import Max +from django.db.models import Max, Value as V +from django.db.models.functions import Coalesce, Greatest from django.utils.functional import cached_property from django.utils.crypto import get_random_string @@ -39,6 +40,8 @@ class Collection(models.Model): main_item = models.OneToOneField("CollectionItem", related_name="parent", null=True, on_delete=models.SET_NULL) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + stoken_id_fields = ["items__revisions__stoken", "members__stoken"] + def __str__(self): return self.uid @@ -56,9 +59,15 @@ class Collection(models.Model): @cached_property def stoken(self): - stoken1 = self.items.aggregate(stoken=Max("revisions__stoken"))["stoken"] or 0 - stoken2 = self.members.aggregate(stoken=Max("stoken"))["stoken"] or 0 - stoken_id = max(stoken1, stoken2) + aggr_fields = [Coalesce(Max(field), V(0)) for field in self.stoken_id_fields] + max_stoken = Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] + stoken_id = ( + self.__class__.objects.filter(main_item=self.main_item) + .annotate(max_stoken=max_stoken) + .values("max_stoken") + .first()["max_stoken"] + ) + if stoken_id == 0: raise Exception("stoken is None. Should never happen") From 2d0bcbdc2014ed6be428082bbb37e6a822a8a1a8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 4 Dec 2020 19:00:14 +0200 Subject: [PATCH 012/170] Stoken annotation: move it all to one place to reduce duplication. --- django_etebase/models.py | 15 +++++++++++---- django_etebase/views.py | 15 ++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index d1095fb..af4d022 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -31,6 +31,11 @@ from .exceptions import EtebaseValidationError UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID") +def stoken_annotation_builder(stoken_id_fields): + aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields] + return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] + + class CollectionType(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) uid = models.BinaryField(editable=True, blank=False, null=False, db_index=True, unique=True) @@ -40,7 +45,7 @@ class Collection(models.Model): main_item = models.OneToOneField("CollectionItem", related_name="parent", null=True, on_delete=models.SET_NULL) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - stoken_id_fields = ["items__revisions__stoken", "members__stoken"] + stoken_annotation = stoken_annotation_builder(["items__revisions__stoken", "members__stoken"]) def __str__(self): return self.uid @@ -59,11 +64,9 @@ class Collection(models.Model): @cached_property def stoken(self): - aggr_fields = [Coalesce(Max(field), V(0)) for field in self.stoken_id_fields] - max_stoken = Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] stoken_id = ( self.__class__.objects.filter(main_item=self.main_item) - .annotate(max_stoken=max_stoken) + .annotate(max_stoken=self.stoken_annotation) .values("max_stoken") .first()["max_stoken"] ) @@ -94,6 +97,8 @@ class CollectionItem(models.Model): version = models.PositiveSmallIntegerField() encryptionKey = models.BinaryField(editable=True, blank=False, null=True) + stoken_annotation = stoken_annotation_builder(["revisions__stoken"]) + class Meta: unique_together = ("uid", "collection") @@ -191,6 +196,8 @@ class CollectionMember(models.Model): default=AccessLevels.READ_ONLY, ) + stoken_annotation = stoken_annotation_builder(["stoken"]) + class Meta: unique_together = ("user", "collection") diff --git a/django_etebase/views.py b/django_etebase/views.py index 8c05948..d1266c6 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -18,8 +18,7 @@ from django.conf import settings from django.contrib.auth import get_user_model, user_logged_in, user_logged_out from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError -from django.db.models import Max, Value as V, Q -from django.db.models.functions import Coalesce, Greatest +from django.db.models import Q from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -94,7 +93,7 @@ class BaseViewSet(viewsets.ModelViewSet): permission_classes = tuple(app_settings.API_PERMISSIONS) renderer_classes = [JSONRenderer, MessagePackRenderer] + ([BrowsableAPIRenderer] if settings.DEBUG else []) parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] - stoken_id_fields = None + stoken_annotation = None def get_serializer_class(self): serializer_class = self.serializer_class @@ -125,9 +124,7 @@ class BaseViewSet(viewsets.ModelViewSet): def filter_by_stoken(self, request, queryset): stoken_rev = self.get_stoken_obj(request) - aggr_fields = [Coalesce(Max(field), V(0)) for field in self.stoken_id_fields] - max_stoken = Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] - queryset = queryset.annotate(max_stoken=max_stoken).order_by("max_stoken") + queryset = queryset.annotate(max_stoken=self.stoken_annotation).order_by("max_stoken") if stoken_rev is not None: queryset = queryset.filter(max_stoken__gt=stoken_rev.id) @@ -179,7 +176,7 @@ class CollectionViewSet(BaseViewSet): serializer_class = CollectionSerializer lookup_field = "main_item__uid" lookup_url_kwarg = "uid" - stoken_id_fields = ["items__revisions__stoken__id", "members__stoken__id"] + stoken_annotation = Collection.stoken_annotation def get_queryset(self, queryset=None): if queryset is None: @@ -262,7 +259,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = "uid" - stoken_id_fields = ["revisions__stoken__id"] + stoken_annotation = CollectionItem.stoken_annotation def get_queryset(self): collection_uid = self.kwargs["collection_uid"] @@ -482,7 +479,7 @@ class CollectionMemberViewSet(BaseViewSet): serializer_class = CollectionMemberSerializer lookup_field = f"user__{User.USERNAME_FIELD}__iexact" lookup_url_kwarg = "username" - stoken_id_fields = ["stoken__id"] + stoken_annotation = CollectionMember.stoken_annotation # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access # (if we want to transfer, we need to do that specifically) From 057b908565072bf9b1003410c0080f42294d446a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 4 Dec 2020 19:15:10 +0200 Subject: [PATCH 013/170] Update changelog. --- ChangeLog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index 3de4eef..b47be77 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,9 @@ # Changelog +## Version 0.6.0 +* Fix stoken calculation performance - was VERY slow in some rare cases +* Fix issues with host verification failing with a custom port - part 2 + ## Version 0.5.3 * Add missing migration From baa42d040d1012267107866ad4286292063e6991 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 14 Dec 2020 13:30:30 +0200 Subject: [PATCH 014/170] Collection: also save the collection UID on the model itself. This enables us to have db-constraints for making sure that UIDs are unique, as well as being more efficient for lookups (which are very common). The UID should always be the same as the main_item.uid, though that's easily enforced as neither of them is allowed to change. --- .../migrations/0033_collection_uid.py | 19 ++++++++++++++++ .../migrations/0034_auto_20201214_1124.py | 22 +++++++++++++++++++ .../migrations/0035_auto_20201214_1126.py | 19 ++++++++++++++++ .../migrations/0036_auto_20201214_1128.py | 19 ++++++++++++++++ django_etebase/models.py | 20 ++--------------- django_etebase/serializers.py | 4 ++-- django_etebase/views.py | 20 ++++++++--------- 7 files changed, 93 insertions(+), 30 deletions(-) create mode 100644 django_etebase/migrations/0033_collection_uid.py create mode 100644 django_etebase/migrations/0034_auto_20201214_1124.py create mode 100644 django_etebase/migrations/0035_auto_20201214_1126.py create mode 100644 django_etebase/migrations/0036_auto_20201214_1128.py diff --git a/django_etebase/migrations/0033_collection_uid.py b/django_etebase/migrations/0033_collection_uid.py new file mode 100644 index 0000000..e4a829e --- /dev/null +++ b/django_etebase/migrations/0033_collection_uid.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.1 on 2020-12-14 11:21 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0032_auto_20201013_1409'), + ] + + operations = [ + migrations.AddField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=43, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + ] diff --git a/django_etebase/migrations/0034_auto_20201214_1124.py b/django_etebase/migrations/0034_auto_20201214_1124.py new file mode 100644 index 0000000..b05060b --- /dev/null +++ b/django_etebase/migrations/0034_auto_20201214_1124.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.1 on 2020-12-14 11:24 + +from django.db import migrations + + +def update_collection_uid(apps, schema_editor): + Collection = apps.get_model("django_etebase", "Collection") + + for collection in Collection.objects.all(): + collection.uid = collection.main_item.uid + collection.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_etebase", "0033_collection_uid"), + ] + + operations = [ + migrations.RunPython(update_collection_uid), + ] diff --git a/django_etebase/migrations/0035_auto_20201214_1126.py b/django_etebase/migrations/0035_auto_20201214_1126.py new file mode 100644 index 0000000..489c5e0 --- /dev/null +++ b/django_etebase/migrations/0035_auto_20201214_1126.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.1 on 2020-12-14 11:26 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0034_auto_20201214_1124'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + ] diff --git a/django_etebase/migrations/0036_auto_20201214_1128.py b/django_etebase/migrations/0036_auto_20201214_1128.py new file mode 100644 index 0000000..241adf1 --- /dev/null +++ b/django_etebase/migrations/0036_auto_20201214_1128.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.1 on 2020-12-14 11:28 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0035_auto_20201214_1126'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index af4d022..11b6506 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -43,6 +43,8 @@ class CollectionType(models.Model): class Collection(models.Model): main_item = models.OneToOneField("CollectionItem", related_name="parent", null=True, on_delete=models.SET_NULL) + # The same as main_item.uid, we just also save it here so we have DB constraints for uniqueness (and efficiency) + uid = models.CharField(db_index=True, unique=True, blank=False, max_length=43, validators=[UidValidator]) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) stoken_annotation = stoken_annotation_builder(["items__revisions__stoken", "members__stoken"]) @@ -50,10 +52,6 @@ class Collection(models.Model): def __str__(self): return self.uid - @cached_property - def uid(self): - return self.main_item.uid - @property def content(self): return self.main_item.content @@ -76,20 +74,6 @@ class Collection(models.Model): return Stoken.objects.get(id=stoken_id).uid - def validate_unique(self, exclude=None): - super().validate_unique(exclude=exclude) - if exclude is None or "main_item" in exclude: - return - - if ( - self.__class__.objects.filter(owner=self.owner, main_item__uid=self.main_item.uid) - .exclude(id=self.id) - .exists() - ): - raise EtebaseValidationError( - "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT - ) - class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, max_length=43, validators=[UidValidator]) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index b0769ef..9628986 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -313,13 +313,13 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): user = validated_data.get("owner") main_item_data = validated_data.pop("main_item") + uid = main_item_data.get("uid") etag = main_item_data.pop("etag") revision_data = main_item_data.pop("content") - instance = self.__class__.Meta.model(**validated_data) + instance = self.__class__.Meta.model(uid=uid, **validated_data) with transaction.atomic(): - _ = self.__class__.Meta.model.objects.select_for_update().filter(owner=user) if etag is not None: raise EtebaseValidationError("bad_etag", "etag is not null") diff --git a/django_etebase/views.py b/django_etebase/views.py index d1266c6..44c3383 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -174,7 +174,7 @@ class CollectionViewSet(BaseViewSet): permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly,) queryset = Collection.objects.all() serializer_class = CollectionSerializer - lookup_field = "main_item__uid" + lookup_field = "uid" lookup_url_kwarg = "uid" stoken_annotation = Collection.stoken_annotation @@ -246,7 +246,7 @@ class CollectionViewSet(BaseViewSet): # can point to the most recent collection change rather than most recent removed membership. remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) - remed = remed_qs.values_list("collection__main_item__uid", flat=True) + remed = remed_qs.values_list("collection__uid", flat=True) if len(remed) > 0: ret["removedMemberships"] = [{"uid": x} for x in remed] @@ -264,7 +264,7 @@ class CollectionItemViewSet(BaseViewSet): def get_queryset(self): collection_uid = self.kwargs["collection_uid"] try: - collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) except Collection.DoesNotExist: raise Http404("Collection does not exist") # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') @@ -312,7 +312,7 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=True, methods=["GET"]) def revision(self, request, collection_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) item = get_object_or_404(col.items, uid=uid) limit = int(request.GET.get("limit", 50)) @@ -386,7 +386,7 @@ class CollectionItemViewSet(BaseViewSet): with transaction.atomic(): # We need this for locking on the collection object collection_object = get_object_or_404( self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection - main_item__uid=collection_uid, + uid=collection_uid, ) if stoken is not None and stoken != collection_object.stoken: @@ -435,7 +435,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return queryset.filter(members__user=user) def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) data = { @@ -459,7 +459,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): import os from django.views.static import serve - col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) chunk = get_object_or_404(col.chunks, uid=uid) @@ -487,7 +487,7 @@ class CollectionMemberViewSet(BaseViewSet): def get_queryset(self, queryset=None): collection_uid = self.kwargs["collection_uid"] try: - collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) except Collection.DoesNotExist: raise Http404("Collection does not exist") @@ -525,7 +525,7 @@ class CollectionMemberViewSet(BaseViewSet): @action_decorator(detail=False, methods=["POST"], permission_classes=our_base_permission_classes) def leave(self, request, collection_uid=None, *args, **kwargs): collection_uid = self.kwargs["collection_uid"] - col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) member = col.members.get(user=request.user) self.perform_destroy(member) @@ -584,7 +584,7 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): collection_uid = serializer.validated_data.get("collection", {}).get("uid") try: - collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) except Collection.DoesNotExist: raise Http404("Collection does not exist") From 0407320ad40d6419945751d3a08b3fcde0bc8d88 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 14 Dec 2020 13:48:05 +0200 Subject: [PATCH 015/170] Update changelog. --- ChangeLog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index b47be77..e3c8232 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,8 @@ # Changelog +## Version 0.6.1 +* Collection: save the UID on the model to use the db for enforcing uniqueness + ## Version 0.6.0 * Fix stoken calculation performance - was VERY slow in some rare cases * Fix issues with host verification failing with a custom port - part 2 From 75712614791c86759d7b71ee2d75a49fb6802e5a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 14 Dec 2020 16:03:11 +0200 Subject: [PATCH 016/170] Remove unused imports. --- django_etebase/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index 11b6506..096371d 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -22,10 +22,7 @@ from django.db.models.functions import Coalesce, Greatest from django.utils.functional import cached_property from django.utils.crypto import get_random_string -from rest_framework import status - from . import app_settings -from .exceptions import EtebaseValidationError UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID") From 070abfcdd8b3ea1973ae38291eead680455622db Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 14 Dec 2020 16:12:34 +0200 Subject: [PATCH 017/170] Format using black. --- etebase_server/settings.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 9baf8d3..0e78c9b 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -96,10 +96,18 @@ WSGI_APPLICATION = "etebase_server.wsgi.application" # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ - {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",}, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, ] From a19a982b1c09a790df8afa131f64a9e44d488fbe Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 14 Dec 2020 16:00:48 +0200 Subject: [PATCH 018/170] Sendfile: add a sendfile module based on django-sendfile2 --- django_etebase/sendfile/LICENSE | 28 ++++++ django_etebase/sendfile/README.md | 3 + django_etebase/sendfile/__init__.py | 1 + django_etebase/sendfile/backends/__init__.py | 0 .../sendfile/backends/development.py | 17 ++++ django_etebase/sendfile/backends/mod_wsgi.py | 17 ++++ django_etebase/sendfile/backends/nginx.py | 12 +++ django_etebase/sendfile/backends/simple.py | 60 +++++++++++++ django_etebase/sendfile/backends/xsendfile.py | 9 ++ django_etebase/sendfile/utils.py | 85 +++++++++++++++++++ 10 files changed, 232 insertions(+) create mode 100644 django_etebase/sendfile/LICENSE create mode 100644 django_etebase/sendfile/README.md create mode 100644 django_etebase/sendfile/__init__.py create mode 100644 django_etebase/sendfile/backends/__init__.py create mode 100644 django_etebase/sendfile/backends/development.py create mode 100644 django_etebase/sendfile/backends/mod_wsgi.py create mode 100644 django_etebase/sendfile/backends/nginx.py create mode 100644 django_etebase/sendfile/backends/simple.py create mode 100644 django_etebase/sendfile/backends/xsendfile.py create mode 100644 django_etebase/sendfile/utils.py diff --git a/django_etebase/sendfile/LICENSE b/django_etebase/sendfile/LICENSE new file mode 100644 index 0000000..4b733c8 --- /dev/null +++ b/django_etebase/sendfile/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2011, Sensible Development. +Copyright (c) 2019, Matt Molyneaux +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Django Send File nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/django_etebase/sendfile/README.md b/django_etebase/sendfile/README.md new file mode 100644 index 0000000..aab5091 --- /dev/null +++ b/django_etebase/sendfile/README.md @@ -0,0 +1,3 @@ +Heavily inspired + code borrowed from: https://github.com/moggers87/django-sendfile2/ + +We just simplified and inlined it because we don't want another external dependency for distribution packagers to package, as well as need a much simpler version. diff --git a/django_etebase/sendfile/__init__.py b/django_etebase/sendfile/__init__.py new file mode 100644 index 0000000..4949aa5 --- /dev/null +++ b/django_etebase/sendfile/__init__.py @@ -0,0 +1 @@ +from .utils import sendfile # noqa diff --git a/django_etebase/sendfile/backends/__init__.py b/django_etebase/sendfile/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/sendfile/backends/development.py b/django_etebase/sendfile/backends/development.py new file mode 100644 index 0000000..d321932 --- /dev/null +++ b/django_etebase/sendfile/backends/development.py @@ -0,0 +1,17 @@ +import os.path + +from django.views.static import serve + + +def sendfile(request, filename, **kwargs): + """ + Send file using Django dev static file server. + + .. warning:: + + Do not use in production. This is only to be used when developing and + is provided for convenience only + """ + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + return serve(request, basename, dirname) diff --git a/django_etebase/sendfile/backends/mod_wsgi.py b/django_etebase/sendfile/backends/mod_wsgi.py new file mode 100644 index 0000000..07ba3f1 --- /dev/null +++ b/django_etebase/sendfile/backends/mod_wsgi.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +from django.http import HttpResponse + +from ..utils import _convert_file_to_url + + +def sendfile(request, filename, **kwargs): + response = HttpResponse() + response['Location'] = _convert_file_to_url(filename) + # need to destroy get_host() to stop django + # rewriting our location to include http, so that + # mod_wsgi is able to do the internal redirect + request.get_host = lambda: '' + request.build_absolute_uri = lambda location: location + + return response diff --git a/django_etebase/sendfile/backends/nginx.py b/django_etebase/sendfile/backends/nginx.py new file mode 100644 index 0000000..8764309 --- /dev/null +++ b/django_etebase/sendfile/backends/nginx.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import + +from django.http import HttpResponse + +from ..utils import _convert_file_to_url + + +def sendfile(request, filename, **kwargs): + response = HttpResponse() + response['X-Accel-Redirect'] = _convert_file_to_url(filename) + + return response diff --git a/django_etebase/sendfile/backends/simple.py b/django_etebase/sendfile/backends/simple.py new file mode 100644 index 0000000..0549b20 --- /dev/null +++ b/django_etebase/sendfile/backends/simple.py @@ -0,0 +1,60 @@ +from email.utils import mktime_tz, parsedate_tz +import re + +from django.core.files.base import File +from django.http import HttpResponse, HttpResponseNotModified +from django.utils.http import http_date + + +def sendfile(request, filepath, **kwargs): + '''Use the SENDFILE_ROOT value composed with the path arrived as argument + to build an absolute path with which resolve and return the file contents. + + If the path points to a file out of the root directory (should cover both + situations with '..' and symlinks) then a 404 is raised. + ''' + statobj = filepath.stat() + + # Respect the If-Modified-Since header. + if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), + statobj.st_mtime, statobj.st_size): + return HttpResponseNotModified() + + with File(filepath.open('rb')) as f: + response = HttpResponse(f.chunks()) + + response["Last-Modified"] = http_date(statobj.st_mtime) + return response + + +def was_modified_since(header=None, mtime=0, size=0): + """ + Was something modified since the user last downloaded it? + + header + This is the value of the If-Modified-Since header. If this is None, + I'll just return True. + + mtime + This is the modification time of the item we're talking about. + + size + This is the size of the item we're talking about. + """ + try: + if header is None: + raise ValueError + matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, + re.IGNORECASE) + header_date = parsedate_tz(matches.group(1)) + if header_date is None: + raise ValueError + header_mtime = mktime_tz(header_date) + header_len = matches.group(3) + if header_len and int(header_len) != size: + raise ValueError + if mtime > header_mtime: + raise ValueError + except (AttributeError, ValueError, OverflowError): + return True + return False diff --git a/django_etebase/sendfile/backends/xsendfile.py b/django_etebase/sendfile/backends/xsendfile.py new file mode 100644 index 0000000..74993ee --- /dev/null +++ b/django_etebase/sendfile/backends/xsendfile.py @@ -0,0 +1,9 @@ +from django.http import HttpResponse + + +def sendfile(request, filename, **kwargs): + filename = str(filename) + response = HttpResponse() + response['X-Sendfile'] = filename + + return response diff --git a/django_etebase/sendfile/utils.py b/django_etebase/sendfile/utils.py new file mode 100644 index 0000000..97c06d7 --- /dev/null +++ b/django_etebase/sendfile/utils.py @@ -0,0 +1,85 @@ +from functools import lru_cache +from importlib import import_module +from pathlib import Path, PurePath +from urllib.parse import quote +import logging + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.http import Http404 + +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=None) +def _get_sendfile(): + backend = getattr(settings, "SENDFILE_BACKEND", None) + if not backend: + raise ImproperlyConfigured("You must specify a value for SENDFILE_BACKEND") + module = import_module(backend) + return module.sendfile + + +def _convert_file_to_url(path): + try: + url_root = PurePath(getattr(settings, "SENDFILE_URL", None)) + except TypeError: + return path + + path_root = PurePath(settings.SENDFILE_ROOT) + path_obj = PurePath(path) + + relpath = path_obj.relative_to(path_root) + # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an + # already instantiated Path object + url = relpath._flavour.pathmod.normpath(str(url_root / relpath)) + + return quote(str(url)) + + +def _sanitize_path(filepath): + try: + path_root = Path(getattr(settings, "SENDFILE_ROOT", None)) + except TypeError: + raise ImproperlyConfigured("You must specify a value for SENDFILE_ROOT") + + filepath_obj = Path(filepath) + + # get absolute path + # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an + # already instantiated Path object + filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj))) + + # if filepath_abs is not relative to path_root, relative_to throws an error + try: + filepath_abs.relative_to(path_root) + except ValueError: + raise Http404("{} wrt {} is impossible".format(filepath_abs, path_root)) + + return filepath_abs + + +def sendfile(request, filename, mimetype="application/octet-stream", encoding=None): + """ + Create a response to send file using backend configured in ``SENDFILE_BACKEND`` + + ``filename`` is the absolute path to the file to send. + """ + filepath_obj = _sanitize_path(filename) + logger.debug( + "filename '%s' requested \"\ + \"-> filepath '%s' obtained", + filename, + filepath_obj, + ) + _sendfile = _get_sendfile() + + if not filepath_obj.exists(): + raise Http404('"%s" does not exist' % filepath_obj) + + response = _sendfile(request, filepath_obj, mimetype=mimetype) + + response["Content-length"] = filepath_obj.stat().st_size + response["Content-Type"] = mimetype + + return response From 9559a0fd35bf0c95757a1ecadebf6418a6536515 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 14 Dec 2020 16:40:08 +0200 Subject: [PATCH 019/170] Chunk download: use the new sendfile to serve files. --- django_etebase/views.py | 11 ++--------- etebase_server/settings.py | 4 ++++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 44c3383..1de5ed7 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -36,6 +36,7 @@ import nacl.signing import nacl.secret import nacl.hash +from .sendfile import sendfile from .token_auth.models import AuthToken from .drf_msgpack.parsers import MessagePackParser @@ -456,19 +457,11 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): @action_decorator(detail=True, methods=["GET"]) def download(self, request, collection_uid=None, collection_item_uid=None, uid=None, *args, **kwargs): - import os - from django.views.static import serve - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) chunk = get_object_or_404(col.chunks, uid=uid) filename = chunk.chunkFile.path - dirname = os.path.dirname(filename) - basename = os.path.basename(filename) - - # FIXME: DO NOT USE! Use django-send file or etc instead. - return serve(request, basename, dirname) + return sendfile(request, filename) class CollectionMemberViewSet(BaseViewSet): diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 0e78c9b..325dca9 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -173,6 +173,10 @@ ETEBASE_API_AUTHENTICATORS = ( ) ETEBASE_CREATE_USER_FUNC = "django_etebase.utils.create_user_blocked" +# Efficient file streaming (for large files) +SENDFILE_BACKEND = "django_etebase.sendfile.backends.simple" +SENDFILE_ROOT = MEDIA_URL + # Make an `etebase_server_settings` module available to override settings. try: from etebase_server_settings import * From cd86c060b5242d82a3b2a62b0ad93bd5f42eb1c5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 15 Dec 2020 10:14:15 +0200 Subject: [PATCH 020/170] Collection: fix UID validation to return a Conflict error. --- django_etebase/serializers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 9628986..223e48f 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -323,12 +323,17 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): if etag is not None: raise EtebaseValidationError("bad_etag", "etag is not null") + try: + instance.validate_unique() + except django_exceptions.ValidationError: + raise EtebaseValidationError( + "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT + ) instance.save() + main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) instance.main_item = main_item - - instance.full_clean() instance.save() process_revisions_for_item(main_item, revision_data) From 0585d6ee927ad5aa4a4baea3dac786db1a49b647 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 15 Dec 2020 13:58:59 +0200 Subject: [PATCH 021/170] Chunk file uploader: accept all kinds of media types. This restriction was unnecessary and annoying to deal with in clients. --- django_etebase/parsers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django_etebase/parsers.py b/django_etebase/parsers.py index c7fe58c..ed1e713 100644 --- a/django_etebase/parsers.py +++ b/django_etebase/parsers.py @@ -6,8 +6,6 @@ class ChunkUploadParser(FileUploadParser): Parser for chunk upload data. """ - media_type = "application/octet-stream" - def get_filename(self, stream, media_type, parser_context): """ Detects the uploaded file name. From 7ae172e44e71e7e0b541ba854dc4ac037523e949 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 16 Dec 2020 09:24:53 +0200 Subject: [PATCH 022/170] README: update contributors from github sponsors --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cfb0be9..3e0bd53 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,13 @@ A quick summary can be found [on tldrlegal](https://tldrlegal.com/license/gnu-af For commercial licensing options, contact license@etebase.com -# Supporting Etebase +# Financially Supporting Etebase Please consider registering an account even if you self-host in order to support the development of Etebase, or visit the [contribution](https://www.etesync.com/contribute/) for more information on how to support the service. + +Become a financial contributor and help us sustain our community! + +## Contributors ($10 / month) + +[![ilovept](https://github.com/ilovept.png?size=40)](https://github.com/ilovept) +[![ryanleesipes](https://github.com/ryanleesipes.png?size=40)](https://github.com/ryanleesipes) From 3b4ba75930822d4cfd17cabc6b0d71b92c940150 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 16 Dec 2020 15:04:00 +0200 Subject: [PATCH 023/170] Chunk serialization: support not passing chunk content if exists. --- django_etebase/serializers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 223e48f..eebae91 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -37,15 +37,14 @@ def process_revisions_for_item(item, revision_data): for chunk in chunks: uid = chunk[0] chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() - if len(chunk) > 1: - content = chunk[1] - # If the chunk already exists we assume it's fine. Otherwise, we upload it. - if chunk_obj is None: + content = chunk[1] if len(chunk) > 1 else None + # If the chunk already exists we assume it's fine. Otherwise, we upload it. + if chunk_obj is None: + if content is not None: chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) chunk_obj.save() - else: - if chunk_obj is None: + else: raise EtebaseValidationError("chunk_no_content", "Tried to create a new chunk without content") chunks_objs.append(chunk_obj) @@ -122,9 +121,10 @@ class ChunksField(serializers.RelatedField): return (obj.uid,) def to_internal_value(self, data): - if data[0] is None or data[1] is None: + content = data[1] if len(data) > 1 else None + if data[0] is None: raise EtebaseValidationError("no_null", "null is not allowed") - return (data[0], b64decode_or_bytes(data[1])) + return (data[0], b64decode_or_bytes(content) if content is not None else None) class BetterErrorsMixin: From 3fcea20d68f7a820386f1e6c95ce8528a760c4f4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 22 Dec 2020 12:46:42 +0200 Subject: [PATCH 024/170] Serializers: fully clean an object on signup. --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index eebae91..9408b78 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -518,7 +518,7 @@ class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): # Create the user and save the casing the user chose as the first name try: instance = create_user(**user_data, password=None, first_name=user_data["username"], view=view) - instance.clean_fields() + instance.full_clean() except EtebaseValidationError as e: raise e except django_exceptions.ValidationError as e: From 1cb84cfa6d8d10876ea4e5207963728dedcb77e9 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 22 Dec 2020 13:15:12 +0200 Subject: [PATCH 025/170] Serializers: cleanup how we handle validation errors. --- django_etebase/serializers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 9408b78..24d78a4 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -149,10 +149,8 @@ class BetterErrorsMixin: ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) else: for error in errors: - if hasattr(error, "detail"): - message = error.detail[0] - elif hasattr(error, "message"): - message = error.message + if error.messages: + message = error.messages[0] else: message = str(error) ret.append( From 7eb08d29462d6e663d5af57795fe1ed3f7b21652 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 22 Dec 2020 21:06:41 +0200 Subject: [PATCH 026/170] Collection invitation: fix the wrong field type --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 24d78a4..1e5c78e 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -386,7 +386,7 @@ class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSeriali style={"base_template": "input.html"}, ) collection = serializers.CharField(source="collection.uid") - fromUsername = BinaryBase64Field(source="fromMember.user.username", read_only=True) + fromUsername = serializers.CharField(source="fromMember.user.username", read_only=True) fromPubkey = BinaryBase64Field(source="fromMember.user.userinfo.pubkey", read_only=True) signedEncryptionKey = BinaryBase64Field() From 2de51b978abfac206f05624faacbbfa414cef32c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 24 Dec 2020 16:14:52 +0200 Subject: [PATCH 027/170] Serializers: fix an issue with handling validation errors. --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 1e5c78e..32a2757 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -149,7 +149,7 @@ class BetterErrorsMixin: ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) else: for error in errors: - if error.messages: + if getattr(error, "messages", None): message = error.messages[0] else: message = str(error) From 5a6c8a1d05a885dd4a85b2ebde1b49690edbd0e9 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 10:53:01 +0200 Subject: [PATCH 028/170] Gracefully handle uploading the same revision This is needed so that immediately re-played requests don't fail. Consider for example the following scenario: a client makes a batch request that registers correctly on the server, but fails to return (e.g. a networking error after the request has been processed). The client would think that the request failed, but the server will already have the up to date information. This commit just returns a successful status if this request is sent again (by the client retrying the request) instead of returning a conflict. This however doesn't handle the case of a request failing, a modification being made by another client, and then the request being retried. This case will stay fail. --- django_etebase/serializers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 32a2757..06bc8ad 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -239,6 +239,11 @@ class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): # We don't have to use select_for_update here because the unique constraint on current guards against # the race condition. But it's a good idea because it'll lock and wait rather than fail. current_revision = instance.revisions.filter(current=True).select_for_update().first() + + # If we are just re-uploading the same revision, consider it a succes and return. + if current_revision.uid == revision_data.get("uid"): + return instance + current_revision.current = None current_revision.save() From c2eb4fd30cd3dceca05514788a0c450045efc391 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:03:07 +0200 Subject: [PATCH 029/170] Pass generic context to callbacks instead of the whole view --- django_etebase/serializers.py | 13 +++++++++---- django_etebase/utils.py | 17 +++++++++++++---- django_etebase/views.py | 8 ++++---- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 06bc8ad..dce2fe6 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -20,7 +20,7 @@ from django.contrib.auth import get_user_model from django.db import IntegrityError, transaction from rest_framework import serializers, status from . import models -from .utils import get_user_queryset, create_user +from .utils import get_user_queryset, create_user, CallbackContext from .exceptions import EtebaseValidationError @@ -102,7 +102,7 @@ class CollectionTypeField(BinaryBase64Field): class UserSlugRelatedField(serializers.SlugRelatedField): def get_queryset(self): view = self.context.get("view", None) - return get_user_queryset(super().get_queryset(), view) + return get_user_queryset(super().get_queryset(), context=CallbackContext(view.kwargs)) def __init__(self, **kwargs): super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) @@ -515,12 +515,17 @@ class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): with transaction.atomic(): try: view = self.context.get("view", None) - user_queryset = get_user_queryset(User.objects.all(), view) + user_queryset = get_user_queryset(User.objects.all(), context=CallbackContext(view.kwargs)) instance = user_queryset.get(**{User.USERNAME_FIELD: user_data["username"].lower()}) except User.DoesNotExist: # Create the user and save the casing the user chose as the first name try: - instance = create_user(**user_data, password=None, first_name=user_data["username"], view=view) + instance = create_user( + **user_data, + password=None, + first_name=user_data["username"], + context=CallbackContext(view.kwargs) + ) instance.full_clean() except EtebaseValidationError as e: raise e diff --git a/django_etebase/utils.py b/django_etebase/utils.py index e496a77..1c8654b 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -1,3 +1,6 @@ +import typing as t +from dataclasses import dataclass + from django.contrib.auth import get_user_model from django.core.exceptions import PermissionDenied @@ -7,18 +10,24 @@ from . import app_settings User = get_user_model() -def get_user_queryset(queryset, view): +@dataclass +class CallbackContext: + """Class for passing extra context to callbacks""" + + url_kwargs: t.Dict[str, t.Any] + + +def get_user_queryset(queryset, context: CallbackContext): custom_func = app_settings.GET_USER_QUERYSET_FUNC if custom_func is not None: - return custom_func(queryset, view) + return custom_func(queryset, context) return queryset -def create_user(*args, **kwargs): +def create_user(context: CallbackContext, *args, **kwargs): custom_func = app_settings.CREATE_USER_FUNC if custom_func is not None: return custom_func(*args, **kwargs) - _ = kwargs.pop("view") return User.objects.create_user(*args, **kwargs) diff --git a/django_etebase/views.py b/django_etebase/views.py index 1de5ed7..5a03aa4 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -73,7 +73,7 @@ from .serializers import ( UserInfoPubkeySerializer, UserSerializer, ) -from .utils import get_user_queryset +from .utils import get_user_queryset, CallbackContext from .exceptions import EtebaseValidationError from .parsers import ChunkUploadParser from .signals import user_signed_up @@ -598,7 +598,7 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): def fetch_user_profile(self, request, *args, **kwargs): username = request.GET.get("username") kwargs = {User.USERNAME_FIELD: username.lower()} - user = get_object_or_404(get_user_queryset(User.objects.all(), self), **kwargs) + user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)), **kwargs) user_info = get_object_or_404(UserInfo.objects.all(), owner=user) serializer = UserInfoPubkeySerializer(user_info) return Response(serializer.data) @@ -642,7 +642,7 @@ class AuthenticationViewSet(viewsets.ViewSet): ) def get_queryset(self): - return get_user_queryset(User.objects.all(), self) + return get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) def get_serializer_context(self): return {"request": self.request, "format": self.format_kwarg, "view": self} @@ -837,7 +837,7 @@ class TestAuthenticationViewSet(viewsets.ViewSet): return HttpResponseBadRequest("Only allowed in debug mode.") with transaction.atomic(): - user_queryset = get_user_queryset(User.objects.all(), self) + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) user = get_object_or_404(user_queryset, username=request.data.get("user").get("username")) # Only allow test users for extra safety From 70619fc1c7b13f0311170d6087b5b42ae876cb49 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:14:42 +0200 Subject: [PATCH 030/170] Fix unbound variable warning. --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index dce2fe6..26ac5a7 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -513,8 +513,8 @@ class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): user_data = validated_data.pop("user") with transaction.atomic(): + view = self.context.get("view", None) try: - view = self.context.get("view", None) user_queryset = get_user_queryset(User.objects.all(), context=CallbackContext(view.kwargs)) instance = user_queryset.get(**{User.USERNAME_FIELD: user_data["username"].lower()}) except User.DoesNotExist: From c1534f6587dfc25569fb270b8b7b43984da96f3e Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Wed, 23 Dec 2020 16:29:08 -0500 Subject: [PATCH 031/170] first commit --- etebase_fastapi/__init__.py | 0 etebase_fastapi/app.py | 29 ++++ etebase_fastapi/authentication.py | 251 ++++++++++++++++++++++++++++++ etebase_fastapi/collections.py | 0 etebase_fastapi/execptions.py | 42 +++++ etebase_fastapi/msgpack.py | 63 ++++++++ requirements.in/base.txt | 2 + requirements.txt | 30 ++-- 8 files changed, 405 insertions(+), 12 deletions(-) create mode 100644 etebase_fastapi/__init__.py create mode 100644 etebase_fastapi/app.py create mode 100644 etebase_fastapi/authentication.py create mode 100644 etebase_fastapi/collections.py create mode 100644 etebase_fastapi/execptions.py create mode 100644 etebase_fastapi/msgpack.py diff --git a/etebase_fastapi/__init__.py b/etebase_fastapi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py new file mode 100644 index 0000000..0ee7aae --- /dev/null +++ b/etebase_fastapi/app.py @@ -0,0 +1,29 @@ +import os + +from django.core.wsgi import get_wsgi_application +from fastapi.middleware.cors import CORSMiddleware + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") +application = get_wsgi_application() +from fastapi import FastAPI, Request + +from .execptions import CustomHttpException +from .authentication import authentication_router +from .msgpack import MsgpackResponse + +app = FastAPI() +app.include_router(authentication_router, prefix="/api/v1/authentication") +app.add_middleware( + CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] +) + + +@app.exception_handler(CustomHttpException) +async def custom_exception_handler(request: Request, exc: CustomHttpException): + return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8080) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py new file mode 100644 index 0000000..b1a3272 --- /dev/null +++ b/etebase_fastapi/authentication.py @@ -0,0 +1,251 @@ +import dataclasses +import typing as t +from datetime import datetime +from functools import cached_property + +import nacl +import nacl.encoding +import nacl.hash +import nacl.secret +import nacl.signing +from asgiref.sync import sync_to_async +from django.conf import settings +from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.utils import timezone +from fastapi import APIRouter, Depends, status, Request, Response +from fastapi.security import APIKeyHeader +from pydantic import BaseModel + +from django_etebase import app_settings +from django_etebase.models import UserInfo +from django_etebase.serializers import UserSerializer +from django_etebase.token_auth.models import AuthToken +from django_etebase.token_auth.models import get_default_expiry +from django_etebase.views import msgpack_encode, msgpack_decode +from .execptions import AuthenticationFailed +from .msgpack import MsgpackResponse, MsgpackRoute + +User = get_user_model() +token_scheme = APIKeyHeader(name="Authorization") +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 +authentication_router = APIRouter(route_class=MsgpackRoute) + + +@dataclasses.dataclass(frozen=True) +class AuthData: + user: User + token: AuthToken + + +class LoginChallengeData(BaseModel): + username: str + + +class LoginResponse(BaseModel): + username: str + challenge: bytes + host: str + action: t.Literal["login", "changePassword"] + + +class Authentication(BaseModel): + response: bytes + signature: bytes + + +class Login(Authentication): + @cached_property + def response_data(self) -> LoginResponse: + return LoginResponse(**msgpack_decode(self.response)) + + +class ChangePasswordResponse(LoginResponse): + loginPubkey: bytes + encryptedContent: bytes + + +class ChangePassword(Authentication): + @cached_property + def response_data(self) -> ChangePasswordResponse: + return ChangePasswordResponse(**msgpack_decode(self.response)) + + +def __renew_token(auth_token: AuthToken): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=("expiry",)) + + +@sync_to_async +def __get_authenticated_user(api_token: str): + api_token = api_token.split()[1] + try: + token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) + except AuthToken.DoesNotExist: + raise AuthenticationFailed(detail="Invalid token.") + if not token.user.is_active: + raise AuthenticationFailed(detail="User inactive or deleted.") + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise AuthenticationFailed(detail="Invalid token.") + + if AUTO_REFRESH: + __renew_token(token) + + return token.user, token + + +async def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: + user, token = await __get_authenticated_user(api_token) + return AuthData(user, token) + + +async def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: + user, token = await __get_authenticated_user(api_token) + return user + + +@sync_to_async +def __get_login_user(username: str) -> User: + kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} + try: + user = User.objects.get(**kwargs) + if not hasattr(user, "userinfo"): + raise AuthenticationFailed(code="user_not_init", detail="User not properly init") + return user + except User.DoesNotExist: + raise AuthenticationFailed(code="user_not_found", detail="User not found") + + +async def get_login_user(challenge: LoginChallengeData) -> User: + user = await __get_login_user(challenge.username) + return user + + +def get_encryption_key(salt): + key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) + return nacl.hash.blake2b( + b"", + key=key, + salt=salt[: nacl.hash.BLAKE2B_SALTBYTES], + person=b"etebase-auth", + encoder=nacl.encoding.RawEncoder, + ) + + +@sync_to_async +def save_changed_password(data: ChangePassword, user: User): + response_data = data.response_data + user_info: UserInfo = user.userinfo + user_info.loginPubkey = response_data.loginPubkey + user_info.encryptedContent = response_data.encryptedContent + user_info.save() + + +@sync_to_async +def login_response_data(user: User): + return { + "token": AuthToken.objects.create(user=user).key, + "user": UserSerializer(user).data, + } + + +@sync_to_async +def send_user_logged_in_async(user: User, request: Request): + user_logged_in.send(sender=user.__class__, request=request, user=user) + + +@sync_to_async +def send_user_logged_out_async(user: User, request: Request): + user_logged_out.send(sender=user.__class__, request=request, user=user) + + +@sync_to_async +def validate_login_request( + validated_data: LoginResponse, + challenge_sent_to_user: Authentication, + user: User, + expected_action: str, + host_from_request: str, +) -> t.Optional[MsgpackResponse]: + + enc_key = get_encryption_key(bytes(user.userinfo.salt)) + box = nacl.secret.SecretBox(enc_key) + challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) + now = int(datetime.now().timestamp()) + if validated_data.action != expected_action: + content = { + "code": "wrong_action", + "detail": 'Expected "{}" but got something else'.format(challenge_sent_to_user.response), + } + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: + content = {"code": "challenge_expired", "detail": "Login challenge has expired"} + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + elif challenge_data["userId"] != user.id: + content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: + detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(validated_data.host, host_from_request) + content = {"code": "wrong_host", "detail": detail} + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + + verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) + + try: + verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) + except nacl.exceptions.BadSignatureError: + return MsgpackResponse( + {"code": "login_bad_signature", "detail": "Wrong password for user."}, + status_code=status.HTTP_401_UNAUTHORIZED, + ) + + return None + + +@authentication_router.post("/login_challenge/") +async def login_challenge(user: User = Depends(get_login_user)): + enc_key = get_encryption_key(user.userinfo.salt) + box = nacl.secret.SecretBox(enc_key) + challenge_data = { + "timestamp": int(datetime.now().timestamp()), + "userId": user.id, + } + challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) + return MsgpackResponse({"salt": user.userinfo.salt, "version": user.userinfo.version, "challenge": challenge}) + + +@authentication_router.post("/login/") +async def login(data: Login, request: Request): + user = await get_login_user(LoginChallengeData(username=data.response_data.username)) + host = request.headers.get("Host") + bad_login_response = await validate_login_request(data.response_data, data, user, "login", host) + if bad_login_response is not None: + return bad_login_response + data = await login_response_data(user) + await send_user_logged_in_async(user, request) + return MsgpackResponse(data, status_code=status.HTTP_200_OK) + + +@authentication_router.post("/logout/") +async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): + await sync_to_async(auth_data.token.delete)() + await send_user_logged_out_async(auth_data.user, request) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@authentication_router.post("/change_password/") +async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): + host = request.headers.get("Host") + bad_login_response = await validate_login_request(data.response_data, data, user, "changePassword", host) + if bad_login_response is not None: + return bad_login_response + await save_changed_password(data, user) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/etebase_fastapi/collections.py b/etebase_fastapi/collections.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/execptions.py new file mode 100644 index 0000000..8808f5d --- /dev/null +++ b/etebase_fastapi/execptions.py @@ -0,0 +1,42 @@ +from fastapi import status + + +class CustomHttpException(Exception): + def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + self.status_code = status_code + self.code = code + self.detail = detail + + @property + def as_dict(self) -> dict: + return {"code": self.code, "detail": self.detail} + + +class AuthenticationFailed(CustomHttpException): + def __init__( + self, + code="authentication_failed", + detail: str = "Incorrect authentication credentials.", + status_code: int = status.HTTP_401_UNAUTHORIZED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class NotAuthenticated(CustomHttpException): + def __init__( + self, + code="not_authenticated", + detail: str = "Authentication credentials were not provided.", + status_code: int = status.HTTP_401_UNAUTHORIZED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class PermissionDenied(CustomHttpException): + def __init__( + self, + code="permission_denied", + detail: str = "You do not have permission to perform this action.", + status_code: int = status.HTTP_403_FORBIDDEN, + ): + super().__init__(code=code, detail=detail, status_code=status_code) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py new file mode 100644 index 0000000..53e18cb --- /dev/null +++ b/etebase_fastapi/msgpack.py @@ -0,0 +1,63 @@ +import typing as t +import msgpack +from fastapi.routing import APIRoute, get_request_handler +from starlette.requests import Request +from starlette.responses import Response + + +class MsgpackRequest(Request): + media_type = "application/msgpack" + + async def json(self) -> bytes: + if not hasattr(self, "_json"): + body = await super().body() + self._json = msgpack.unpackb(body, raw=False) + return self._json + + +class MsgpackResponse(Response): + media_type = "application/msgpack" + + def render(self, content: t.Any) -> bytes: + return msgpack.packb(content, use_bin_type=True) + + +class MsgpackRoute(APIRoute): + # keep track of content-type -> request classes + REQUESTS_CLASSES = {MsgpackRequest.media_type: MsgpackRequest} + # keep track of content-type -> response classes + ROUTES_HANDLERS_CLASSES = {MsgpackResponse.media_type: MsgpackResponse} + + def _get_media_type_route_handler(self, media_type): + return get_request_handler( + dependant=self.dependant, + body_field=self.body_field, + status_code=self.status_code, + # use custom response class or fallback on default self.response_class + response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class), + response_field=self.secure_cloned_response_field, + response_model_include=self.response_model_include, + response_model_exclude=self.response_model_exclude, + response_model_by_alias=self.response_model_by_alias, + response_model_exclude_unset=self.response_model_exclude_unset, + response_model_exclude_defaults=self.response_model_exclude_defaults, + response_model_exclude_none=self.response_model_exclude_none, + dependency_overrides_provider=self.dependency_overrides_provider, + ) + + def get_route_handler(self) -> t.Callable: + async def custom_route_handler(request: Request) -> Response: + + content_type = request.headers.get("Content-Type") + try: + request_cls = self.REQUESTS_CLASSES[content_type] + request = request_cls(request.scope, request.receive) + except KeyError: + # nothing registered to handle content_type, process given requests as-is + pass + + accept = request.headers.get("Accept") + route_handler = self._get_media_type_route_handler(accept) + return await route_handler(request) + + return custom_route_handler diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 7d5bf7e..ca8dd94 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -5,3 +5,5 @@ drf-nested-routers msgpack psycopg2-binary pynacl +fastapi +uvicorn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f6c8ed4..3d19eaf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,16 +4,22 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # -asgiref==3.2.10 # via django -cffi==1.14.0 # via pynacl -django-cors-headers==3.2.1 # via -r requirements.in/base.txt -django==3.1.1 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers -djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers -drf-nested-routers==0.91 # via -r requirements.in/base.txt -msgpack==1.0.0 # via -r requirements.in/base.txt -psycopg2-binary==2.8.4 # via -r requirements.in/base.txt +asgiref==3.3.1 # via django +cffi==1.14.4 # via pynacl +click==7.1.2 # via uvicorn +django-cors-headers==3.6.0 # via -r requirements.in/base.txt +django==3.1.4 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers +djangorestframework==3.12.2 # via -r requirements.in/base.txt, drf-nested-routers +drf-nested-routers==0.92.5 # via -r requirements.in/base.txt +fastapi==0.63.0 # via -r requirements.in/base.txt +h11==0.11.0 # via uvicorn +msgpack==1.0.2 # via -r requirements.in/base.txt +psycopg2-binary==2.8.6 # via -r requirements.in/base.txt pycparser==2.20 # via cffi -pynacl==1.3.0 # via -r requirements.in/base.txt -pytz==2019.3 # via django -six==1.14.0 # via pynacl -sqlparse==0.3.0 # via django +pydantic==1.7.3 # via fastapi +pynacl==1.4.0 # via -r requirements.in/base.txt +pytz==2020.4 # via django +six==1.15.0 # via pynacl +sqlparse==0.4.1 # via django +starlette==0.13.6 # via fastapi +uvicorn==0.13.2 # via -r requirements.in/base.txt From 25cb4fec0cbe16d062123460d3ca2ed2669b3738 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:10:43 +0200 Subject: [PATCH 032/170] msgpack.py: allow pydantic BaseModel in content --- etebase_fastapi/msgpack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 53e18cb..399f3d0 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -1,6 +1,7 @@ import typing as t import msgpack from fastapi.routing import APIRoute, get_request_handler +from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response @@ -19,6 +20,8 @@ class MsgpackResponse(Response): media_type = "application/msgpack" def render(self, content: t.Any) -> bytes: + if isinstance(content, BaseModel): + content = content.dict() return msgpack.packb(content, use_bin_type=True) From 16a99f02ea9465fb5d5690820b38c4055cbc66ea Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:12:22 +0200 Subject: [PATCH 033/170] snapshot --- etebase_fastapi/app.py | 8 +++- etebase_fastapi/authentication.py | 3 ++ etebase_fastapi/collection.py | 72 +++++++++++++++++++++++++++++++ etebase_fastapi/collections.py | 0 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 etebase_fastapi/collection.py delete mode 100644 etebase_fastapi/collections.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 0ee7aae..acfb42f 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -9,10 +9,14 @@ from fastapi import FastAPI, Request from .execptions import CustomHttpException from .authentication import authentication_router +from .collection import collection_router from .msgpack import MsgpackResponse app = FastAPI() -app.include_router(authentication_router, prefix="/api/v1/authentication") +VERSION = "v1" +BASE_PATH = f"/api/{VERSION}" +app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") +app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) @@ -26,4 +30,4 @@ async def custom_exception_handler(request: Request, exc: CustomHttpException): if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8080) + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index b1a3272..697f3f4 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -50,6 +50,9 @@ class LoginResponse(BaseModel): class Authentication(BaseModel): + class Config: + keep_untouched = (cached_property,) + response: bytes signature: bytes diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py new file mode 100644 index 0000000..ec0125d --- /dev/null +++ b/etebase_fastapi/collection.py @@ -0,0 +1,72 @@ +import typing as t + +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.db.models import QuerySet +from fastapi import APIRouter, Depends +from pydantic import BaseModel +from asgiref.sync import sync_to_async + +from django_etebase.models import Collection, Stoken, AccessLevels, CollectionMember +from .authentication import get_authenticated_user +from .msgpack import MsgpackRoute, MsgpackResponse + +User = get_user_model() +collection_router = APIRouter(route_class=MsgpackRoute) +default_queryset = Collection.objects.all() + + +class ListMulti(BaseModel): + collectionTypes: t.List[bytes] + + +class CollectionItemOut(BaseModel): + uid: str + + +class CollectionOut(BaseModel): + collectionKey: bytes + collectionType: bytes + accessLevel: AccessLevels + stoken: str + item: CollectionItemOut + + @classmethod + def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User) -> "CollectionOut": + member: CollectionMember = obj.members.get(user=user) + collection_type = member.collectionType + return cls( + collectionType=collection_type and collection_type.uid, + collectionKey=member.encryptionKey, + accessLevel=member.accessLevel, + stoken=obj.stoken, + item=CollectionItemOut(uid=obj.main_item.uid), + ) + + +class ListResponse(BaseModel): + data: t.List[CollectionOut] + stoken: t.Optional[str] + done: bool + + +@sync_to_async +def list_common(queryset: QuerySet, stoken: t.Optional[str], user: User) -> MsgpackResponse: + data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user) for item in queryset] + ret = ListResponse(data=data, stoken=stoken, done=True) + return MsgpackResponse(content=ret) + + +def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: + return queryset.filter(members__user=user) + + +@collection_router.post("/list_multi/") +async def list_multi(limit: int, data: ListMulti, user: User = Depends(get_authenticated_user)): + queryset = get_collection_queryset(user, default_queryset) + # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") + queryset = queryset.filter( + Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) + ) + response = await list_common(queryset, None, user) + return response diff --git a/etebase_fastapi/collections.py b/etebase_fastapi/collections.py deleted file mode 100644 index e69de29..0000000 From f70e2d80a64d33ad485dbb58f0cfa113cdc72f4b Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:52:43 +0200 Subject: [PATCH 034/170] stoken_handler.py --- etebase_fastapi/stoken_handler.py | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 etebase_fastapi/stoken_handler.py diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py new file mode 100644 index 0000000..c840f0e --- /dev/null +++ b/etebase_fastapi/stoken_handler.py @@ -0,0 +1,61 @@ +import typing as t + +from django.db.models import QuerySet +from fastapi import status + +from django_etebase.exceptions import EtebaseValidationError +from django_etebase.models import Stoken + +# TODO missing stoken_annotation type +StokenAnnotation = t.Any + + +def get_stoken_obj(stoken: t.Optional[str]): + if stoken is not None: + try: + return Stoken.objects.get(uid=stoken) + except Stoken.DoesNotExist: + raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) + + return None + + +def filter_by_stoken( + stoken: t.Optional[str], queryset: QuerySet, stoken_annotation: StokenAnnotation +) -> t.Tuple[QuerySet, t.Optional[str]]: + stoken_rev = get_stoken_obj(stoken) + + queryset = queryset.annotate(max_stoken=stoken_annotation).order_by("max_stoken") + + if stoken_rev is not None: + queryset = queryset.filter(max_stoken__gt=stoken_rev.id) + + return queryset, stoken_rev + + +def get_queryset_stoken(queryset: list) -> t.Optional[Stoken]: + maxid = -1 + for row in queryset: + rowmaxid = getattr(row, "max_stoken") or -1 + maxid = max(maxid, rowmaxid) + new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) + + return new_stoken or None + + +def filter_by_stoken_and_limit( + stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation +) -> t.Tuple[list, t.Optional[Stoken], bool]: + + queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + new_stoken_obj = get_queryset_stoken(result) or stoken_rev + + return result, new_stoken_obj, done From 7d864594802b5f7f0d8e23ab383f37e68a79df79 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:53:11 +0200 Subject: [PATCH 035/170] collection.pyL list_multi --- etebase_fastapi/collection.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ec0125d..7c9aca8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -10,6 +10,7 @@ from asgiref.sync import sync_to_async from django_etebase.models import Collection, Stoken, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .msgpack import MsgpackRoute, MsgpackResponse +from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_queryset_stoken User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -51,9 +52,11 @@ class ListResponse(BaseModel): @sync_to_async -def list_common(queryset: QuerySet, stoken: t.Optional[str], user: User) -> MsgpackResponse: +def list_common(queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int) -> MsgpackResponse: + result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) + new_stoken = new_stoken_obj and new_stoken_obj.uid data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user) for item in queryset] - ret = ListResponse(data=data, stoken=stoken, done=True) + ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -62,11 +65,13 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: @collection_router.post("/list_multi/") -async def list_multi(limit: int, data: ListMulti, user: User = Depends(get_authenticated_user)): +async def list_multi( + data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, user: User = Depends(get_authenticated_user) +): queryset = get_collection_queryset(user, default_queryset) # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - response = await list_common(queryset, None, user) + response = await list_common(queryset, user, stoken, limit) return response From c90e92b0f00aea03db62edd803925cfb8ceb4ea5 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 12:36:06 +0200 Subject: [PATCH 036/170] snapshot --- etebase_fastapi/authentication.py | 52 +++++++++++++++++++++++++++++++ etebase_fastapi/execptions.py | 46 +++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 697f3f4..3ae6c4b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -2,6 +2,7 @@ import dataclasses import typing as t from datetime import datetime from functools import cached_property +from django.core import exceptions as django_exceptions import nacl import nacl.encoding @@ -11,16 +12,19 @@ import nacl.signing from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.db import transaction from django.utils import timezone from fastapi import APIRouter, Depends, status, Request, Response from fastapi.security import APIKeyHeader from pydantic import BaseModel from django_etebase import app_settings +from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo from django_etebase.serializers import UserSerializer from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry +from django_etebase.utils import create_user from django_etebase.views import msgpack_encode, msgpack_decode from .execptions import AuthenticationFailed from .msgpack import MsgpackResponse, MsgpackRoute @@ -74,6 +78,19 @@ class ChangePassword(Authentication): return ChangePasswordResponse(**msgpack_decode(self.response)) +class UserSignup(BaseModel): + username: str + email: str + + +class SignupIn(BaseModel): + user: UserSignup + salt: bytes + loginPubkey: bytes + pubkey: bytes + encryptedContent: bytes + + def __renew_token(auth_token: AuthToken): current_expiry = auth_token.expiry new_expiry = get_default_expiry() @@ -252,3 +269,38 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return bad_login_response await save_changed_password(data, user) return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@sync_to_async +def signup_save(data: SignupIn): + user_data = data.user + with transaction.atomic(): + try: + # XXX-TOM + # view = self.context.get("view", None) + # user_queryset = get_user_queryset(User.objects.all(), view) + user_queryset = User.objects.all() + instance = user_queryset.get(**{User.USERNAME_FIELD: user_data.username.lower()}) + except User.DoesNotExist: + # Create the user and save the casing the user chose as the first name + try: + # XXX-TOM + instance = create_user(**user_data.dict(), password=None, first_name=user_data.username, view=None) + instance.full_clean() + except EtebaseValidationError as e: + raise e + except django_exceptions.ValidationError as e: + self.transform_validation_error("user", e) + except Exception as e: + raise EtebaseValidationError("generic", str(e)) + + if hasattr(instance, "userinfo"): + raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + + models.UserInfo.objects.create(**validated_data, owner=instance) + return instance + + +@authentication_router.post("/signup/") +async def signup(data: SignupIn): + pass diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/execptions.py index 8808f5d..2b35634 100644 --- a/etebase_fastapi/execptions.py +++ b/etebase_fastapi/execptions.py @@ -1,5 +1,7 @@ from fastapi import status +from django_etebase.exceptions import EtebaseValidationError + class CustomHttpException(Exception): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): @@ -40,3 +42,47 @@ class PermissionDenied(CustomHttpException): status_code: int = status.HTTP_403_FORBIDDEN, ): super().__init__(code=code, detail=detail, status_code=status_code) + + +class ValidationError(CustomHttpException): + def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + super().__init__(code=code, detail=detail, status_code=status_code) + + +def flatten_errors(field_name, errors): + ret = [] + if isinstance(errors, dict): + for error_key in errors: + error = errors[error_key] + ret.extend(flatten_errors("{}.{}".format(field_name, error_key), error)) + else: + for error in errors: + if error.messages: + message = error.messages[0] + else: + message = str(error) + ret.append( + { + "field": field_name, + "code": error.code, + "detail": message, + } + ) + return ret + + +def transform_validation_error(prefix, err): + if hasattr(err, "error_dict"): + errors = flatten_errors(prefix, err.error_dict) + elif not hasattr(err, "message"): + errors = flatten_errors(prefix, err.error_list) + else: + raise EtebaseValidationError(err.code, err.message) + raise ValidationError(code="field_errors", detail="Field validations failed.") + raise serializers.ValidationError( + { + "code": "field_errors", + "detail": "Field validations failed.", + "errors": errors, + } + ) From 72d4a725f5c969bff271b34239363a7c3ee909e3 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 13:21:20 +0200 Subject: [PATCH 037/170] validation errors --- etebase_fastapi/authentication.py | 24 ++++++++++----- etebase_fastapi/execptions.py | 51 ++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 3ae6c4b..287b46e 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -18,15 +18,16 @@ from fastapi import APIRouter, Depends, status, Request, Response from fastapi.security import APIKeyHeader from pydantic import BaseModel -from django_etebase import app_settings +from django_etebase import app_settings, models from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo from django_etebase.serializers import UserSerializer +from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user from django_etebase.views import msgpack_encode, msgpack_decode -from .execptions import AuthenticationFailed +from .execptions import AuthenticationFailed, transform_validation_error, ValidationError from .msgpack import MsgpackResponse, MsgpackRoute User = get_user_model() @@ -272,7 +273,7 @@ async def change_password(data: ChangePassword, request: Request, user: User = D @sync_to_async -def signup_save(data: SignupIn): +def signup_save(data: SignupIn) -> User: user_data = data.user with transaction.atomic(): try: @@ -290,17 +291,26 @@ def signup_save(data: SignupIn): except EtebaseValidationError as e: raise e except django_exceptions.ValidationError as e: - self.transform_validation_error("user", e) + transform_validation_error("user", e) except Exception as e: raise EtebaseValidationError("generic", str(e)) if hasattr(instance, "userinfo"): - raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + raise ValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) - models.UserInfo.objects.create(**validated_data, owner=instance) + models.UserInfo.objects.create(**data.dict(exclude={"user"}), owner=instance) return instance +@sync_to_async +def send_user_signed_up_async(user: User, request): + user_signed_up.send(sender=user.__class__, request=request, user=user) + + @authentication_router.post("/signup/") async def signup(data: SignupIn): - pass + user = await signup_save(data) + # XXX-TOM + data = await login_response_data(user) + await send_user_signed_up_async(user, None) + return MsgpackResponse(content=data, status_code=status.HTTP_201_CREATED) diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/execptions.py index 2b35634..fa76c45 100644 --- a/etebase_fastapi/execptions.py +++ b/etebase_fastapi/execptions.py @@ -1,8 +1,23 @@ from fastapi import status +import typing as t + +from pydantic import BaseModel from django_etebase.exceptions import EtebaseValidationError +class ValidationErrorField(BaseModel): + field: str + code: str + detail: str + + +class ValidationErrorOut(BaseModel): + code: str + detail: str + errors: t.Optional[t.List[ValidationErrorField]] + + class CustomHttpException(Exception): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): self.status_code = status_code @@ -44,12 +59,27 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) +from django_etebase.exceptions import EtebaseValidationError + + class ValidationError(CustomHttpException): - def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + def __init__( + self, + code: str, + detail: str, + status_code: int = status.HTTP_400_BAD_REQUEST, + field: t.Optional[str] = None, + errors: t.Optional[t.List["ValidationError"]] = None, + ): + self.errors = errors super().__init__(code=code, detail=detail, status_code=status_code) + @property + def as_dict(self) -> dict: + return ValidationErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() -def flatten_errors(field_name, errors): + +def flatten_errors(field_name, errors) -> t.List[ValidationError]: ret = [] if isinstance(errors, dict): for error_key in errors: @@ -61,13 +91,7 @@ def flatten_errors(field_name, errors): message = error.messages[0] else: message = str(error) - ret.append( - { - "field": field_name, - "code": error.code, - "detail": message, - } - ) + ret.append(dict(code=error.code, detail=message, field=field_name)) return ret @@ -78,11 +102,4 @@ def transform_validation_error(prefix, err): errors = flatten_errors(prefix, err.error_list) else: raise EtebaseValidationError(err.code, err.message) - raise ValidationError(code="field_errors", detail="Field validations failed.") - raise serializers.ValidationError( - { - "code": "field_errors", - "detail": "Field validations failed.", - "errors": errors, - } - ) + raise ValidationError(code="field_errors", detail="Field validations failed.", errors=errors) From 2e5dd586574600810dae67501372d254ec917722 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 14:06:35 +0200 Subject: [PATCH 038/170] snapshot --- etebase_fastapi/app.py | 6 +++++ etebase_fastapi/authentication.py | 4 +-- etebase_fastapi/test_reset_view.py | 39 ++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 etebase_fastapi/test_reset_view.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index acfb42f..449059a 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -3,6 +3,8 @@ import os from django.core.wsgi import get_wsgi_application from fastapi.middleware.cors import CORSMiddleware +from django.conf import settings + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() from fastapi import FastAPI, Request @@ -17,6 +19,10 @@ VERSION = "v1" BASE_PATH = f"/api/{VERSION}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") +if settings.DEBUG: + from .test_reset_view import test_reset_view_router + + app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 287b46e..9c770d4 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -217,7 +217,6 @@ def validate_login_request( detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(validated_data.host, host_from_request) content = {"code": "wrong_host", "detail": detail} return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) - verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) try: @@ -272,7 +271,6 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return Response(status_code=status.HTTP_204_NO_CONTENT) -@sync_to_async def signup_save(data: SignupIn) -> User: user_data = data.user with transaction.atomic(): @@ -309,7 +307,7 @@ def send_user_signed_up_async(user: User, request): @authentication_router.post("/signup/") async def signup(data: SignupIn): - user = await signup_save(data) + user = await sync_to_async(signup_save)(data) # XXX-TOM data = await login_response_data(user) await send_user_signed_up_async(user, None) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py new file mode 100644 index 0000000..ee6a1c3 --- /dev/null +++ b/etebase_fastapi/test_reset_view.py @@ -0,0 +1,39 @@ +from django.conf import settings +from django.contrib.auth import get_user_model +from django.db import transaction +from django.shortcuts import get_object_or_404 +from fastapi import APIRouter, Response, status, Depends + +from django_etebase.utils import get_user_queryset +from etebase_fastapi.authentication import get_authenticated_user, SignupIn, signup_save +from etebase_fastapi.msgpack import MsgpackRoute + +test_reset_view_router = APIRouter(route_class=MsgpackRoute) +User = get_user_model() + + +@test_reset_view_router.post("/reset/") +def reset(data: SignupIn): + # Only run when in DEBUG mode! It's only used for tests + if not settings.DEBUG: + return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) + + with transaction.atomic(): + # XXX-TOM + user_queryset = get_user_queryset(User.objects.all(), None) + user = get_object_or_404(user_queryset, username=data.user.username) + # Only allow test users for extra safety + if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): + return Response("Endpoint not allowed for user.", status_code=status.HTTP_400_BAD_REQUEST) + + if hasattr(user, "userinfo"): + user.userinfo.delete() + signup_save(data) + # Delete all of the journal data for this user for a clear test env + user.collection_set.all().delete() + user.collectionmember_set.all().delete() + user.incoming_invitations.all().delete() + + # FIXME: also delete chunk files!!! + + return Response(status_code=status.HTTP_204_NO_CONTENT) From a0d1d23d2d6a1ba11a84274bc96667f3d21625cf Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 17:22:14 +0200 Subject: [PATCH 039/170] imports --- etebase_fastapi/test_reset_view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index ee6a1c3..ea7d8d9 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -2,10 +2,10 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 -from fastapi import APIRouter, Response, status, Depends +from fastapi import APIRouter, Response, status from django_etebase.utils import get_user_queryset -from etebase_fastapi.authentication import get_authenticated_user, SignupIn, signup_save +from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute test_reset_view_router = APIRouter(route_class=MsgpackRoute) From 31e0e0b8323060e8265538ad96666d63724ae1f4 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 17:23:44 +0200 Subject: [PATCH 040/170] change response content to pydantic models and error handling --- etebase_fastapi/authentication.py | 115 +++++++++++++----------------- 1 file changed, 51 insertions(+), 64 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 9c770d4..9ecee4c 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -2,7 +2,6 @@ import dataclasses import typing as t from datetime import datetime from functools import cached_property -from django.core import exceptions as django_exceptions import nacl import nacl.encoding @@ -12,6 +11,7 @@ import nacl.signing from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.core import exceptions as django_exceptions from django.db import transaction from django.utils import timezone from fastapi import APIRouter, Depends, status, Request, Response @@ -21,7 +21,6 @@ from pydantic import BaseModel from django_etebase import app_settings, models from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo -from django_etebase.serializers import UserSerializer from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry @@ -43,10 +42,16 @@ class AuthData: token: AuthToken -class LoginChallengeData(BaseModel): +class LoginChallengeIn(BaseModel): username: str +class LoginChallengeOut(BaseModel): + salt: bytes + challenge: bytes + version: int + + class LoginResponse(BaseModel): username: str challenge: bytes @@ -54,6 +59,26 @@ class LoginResponse(BaseModel): action: t.Literal["login", "changePassword"] +class UserOut(BaseModel): + pubkey: bytes + encryptedContent: bytes + + @classmethod + def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": + return cls(pubkey=obj.userinfo.pubkey, encryptedContent=obj.userinfo.encryptedContent) + + +class LoginOut(BaseModel): + token: str + user: UserOut + + @classmethod + def from_orm(cls: t.Type["LoginOut"], obj: User) -> "LoginOut": + token = AuthToken.objects.create(user=obj).key + user = UserOut.from_orm(obj) + return cls(token=token, user=user) + + class Authentication(BaseModel): class Config: keep_untouched = (cached_property,) @@ -145,7 +170,7 @@ def __get_login_user(username: str) -> User: raise AuthenticationFailed(code="user_not_found", detail="User not found") -async def get_login_user(challenge: LoginChallengeData) -> User: +async def get_login_user(challenge: LoginChallengeIn) -> User: user = await __get_login_user(challenge.username) return user @@ -161,7 +186,6 @@ def get_encryption_key(salt): ) -@sync_to_async def save_changed_password(data: ChangePassword, user: User): response_data = data.response_data user_info: UserInfo = user.userinfo @@ -170,24 +194,6 @@ def save_changed_password(data: ChangePassword, user: User): user_info.save() -@sync_to_async -def login_response_data(user: User): - return { - "token": AuthToken.objects.create(user=user).key, - "user": UserSerializer(user).data, - } - - -@sync_to_async -def send_user_logged_in_async(user: User, request: Request): - user_logged_in.send(sender=user.__class__, request=request, user=user) - - -@sync_to_async -def send_user_logged_out_async(user: User, request: Request): - user_logged_out.send(sender=user.__class__, request=request, user=user) - - @sync_to_async def validate_login_request( validated_data: LoginResponse, @@ -195,39 +201,26 @@ def validate_login_request( user: User, expected_action: str, host_from_request: str, -) -> t.Optional[MsgpackResponse]: - +): enc_key = get_encryption_key(bytes(user.userinfo.salt)) box = nacl.secret.SecretBox(enc_key) challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) now = int(datetime.now().timestamp()) if validated_data.action != expected_action: - content = { - "code": "wrong_action", - "detail": 'Expected "{}" but got something else'.format(challenge_sent_to_user.response), - } - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - content = {"code": "challenge_expired", "detail": "Login challenge has expired"} - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError("challenge_expired", "Login challenge has expired") elif challenge_data["userId"] != user.id: - content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError("wrong_user", "This challenge is for the wrong user") elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: - detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(validated_data.host, host_from_request) - content = {"code": "wrong_host", "detail": detail} - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError( + "wrong_host", f'Found wrong host name. Got: "{validated_data.host}" expected: "{host_from_request}"' + ) verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) - try: verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) except nacl.exceptions.BadSignatureError: - return MsgpackResponse( - {"code": "login_bad_signature", "detail": "Wrong password for user."}, - status_code=status.HTTP_401_UNAUTHORIZED, - ) - - return None + raise ValidationError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) @authentication_router.post("/login_challenge/") @@ -239,35 +232,34 @@ async def login_challenge(user: User = Depends(get_login_user)): "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return MsgpackResponse({"salt": user.userinfo.salt, "version": user.userinfo.version, "challenge": challenge}) + return MsgpackResponse( + LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) + ) @authentication_router.post("/login/") async def login(data: Login, request: Request): - user = await get_login_user(LoginChallengeData(username=data.response_data.username)) + user = await get_login_user(LoginChallengeIn(username=data.response_data.username)) host = request.headers.get("Host") - bad_login_response = await validate_login_request(data.response_data, data, user, "login", host) - if bad_login_response is not None: - return bad_login_response - data = await login_response_data(user) - await send_user_logged_in_async(user, request) - return MsgpackResponse(data, status_code=status.HTTP_200_OK) + await validate_login_request(data.response_data, data, user, "login", host) + data = await sync_to_async(LoginOut.from_orm)(user) + await sync_to_async(user_logged_in.send)(sender=user.__class__, request=None, user=user) + return MsgpackResponse(content=data, status_code=status.HTTP_200_OK) @authentication_router.post("/logout/") async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): await sync_to_async(auth_data.token.delete)() - await send_user_logged_out_async(auth_data.user, request) + # XXX-TOM + await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) return Response(status_code=status.HTTP_204_NO_CONTENT) @authentication_router.post("/change_password/") async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): host = request.headers.get("Host") - bad_login_response = await validate_login_request(data.response_data, data, user, "changePassword", host) - if bad_login_response is not None: - return bad_login_response - await save_changed_password(data, user) + await validate_login_request(data.response_data, data, user, "changePassword", host) + await sync_to_async(save_changed_password)(data, user) return Response(status_code=status.HTTP_204_NO_CONTENT) @@ -300,15 +292,10 @@ def signup_save(data: SignupIn) -> User: return instance -@sync_to_async -def send_user_signed_up_async(user: User, request): - user_signed_up.send(sender=user.__class__, request=request, user=user) - - @authentication_router.post("/signup/") async def signup(data: SignupIn): user = await sync_to_async(signup_save)(data) # XXX-TOM - data = await login_response_data(user) - await send_user_signed_up_async(user, None) + data = await sync_to_async(LoginOut.from_orm)(user) + await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) return MsgpackResponse(content=data, status_code=status.HTTP_201_CREATED) From 4bd826b3bed6fbcbd2b5029094c38e548b38583d Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 19:08:22 +0200 Subject: [PATCH 041/170] remove uvicorn run --- etebase_fastapi/app.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 449059a..fac2a31 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -31,9 +31,3 @@ app.add_middleware( @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) - - -if __name__ == "__main__": - import uvicorn - - uvicorn.run(app, host="0.0.0.0", port=8000) From be7b830804bbb693ce9abd4e0c095f29ad10a4b9 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 19:23:46 +0200 Subject: [PATCH 042/170] collection.py: create --- etebase_fastapi/collection.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 7c9aca8..ba462c9 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,16 +1,16 @@ import typing as t +from asgiref.sync import sync_to_async from django.contrib.auth import get_user_model from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, status from pydantic import BaseModel -from asgiref.sync import sync_to_async -from django_etebase.models import Collection, Stoken, AccessLevels, CollectionMember +from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_queryset_stoken +from .stoken_handler import filter_by_stoken_and_limit User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -75,3 +75,29 @@ async def list_multi( ) response = await list_common(queryset, user, stoken, limit) return response + + +class CollectionItemContent(BaseModel): + uid: str + meta: bytes + deleted: bool + chunks: t.List[t.List[t.Union[str, bytes]]] + + +class Item(BaseModel): + uid: str + version: int + etag: t.Optional[str] + content: CollectionItemContent + + +class CollectionItemIn(BaseModel): + collectionType: bytes + collectionKey: bytes + item: Item + + +@collection_router.post("/") +def create(data: CollectionItemIn): + # FIXME save actual item + return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) From daac0c163b8b6943d89bd010b68d1c2fdae3ca9d Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 12:39:20 +0200 Subject: [PATCH 043/170] collection.py: save to db --- etebase_fastapi/collection.py | 75 ++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ba462c9..1d876e1 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -2,13 +2,18 @@ import typing as t from asgiref.sync import sync_to_async from django.contrib.auth import get_user_model +from django.core import exceptions as django_exceptions +from django.core.files.base import ContentFile +from django.db import transaction from django.db.models import Q from django.db.models import QuerySet from fastapi import APIRouter, Depends, status from pydantic import BaseModel +from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user +from .execptions import ValidationError from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit @@ -88,6 +93,7 @@ class Item(BaseModel): uid: str version: int etag: t.Optional[str] + encryptionKey: t.Optional[bytes] content: CollectionItemContent @@ -97,7 +103,72 @@ class CollectionItemIn(BaseModel): item: Item +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemContent): + chunks_objs = [] + + revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) + revision.validate_unique() # Verify there aren't any validation issues + + for chunk in revision_data.chunks: + uid = chunk[0] + chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() + content = chunk[1] if len(chunk) > 1 else None + # If the chunk already exists we assume it's fine. Otherwise, we upload it. + if chunk_obj is None: + if content is not None: + chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) + chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) + chunk_obj.save() + else: + raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") + + chunks_objs.append(chunk_obj) + + stoken = models.Stoken.objects.create() + revision.stoken = stoken + revision.save() + + for chunk in chunks_objs: + models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) + return revision + + +def _create(data: CollectionItemIn, user: User): + with transaction.atomic(): + if data.item.etag is not None: + raise ValidationError("bad_etag", "etag is not null") + instance = models.Collection(uid=data.item.uid, owner=user) + try: + instance.validate_unique() + except django_exceptions.ValidationError: + raise ValidationError( + "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT + ) + instance.save() + + main_item = models.CollectionItem.objects.create( + uid=data.item.uid, version=data.item.version, encryptionKey=data.item.encryptionKey, collection=instance + ) + + instance.main_item = main_item + instance.save() + + # TODO + process_revisions_for_item(main_item, data.item.content) + + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user) + + models.CollectionMember( + collection=instance, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=models.AccessLevels.ADMIN, + encryptionKey=data.collectionKey, + collectionType=collection_type_obj, + ).save() + + @collection_router.post("/") -def create(data: CollectionItemIn): - # FIXME save actual item +async def create(data: CollectionItemIn, user: User = Depends(get_authenticated_user)): + await sync_to_async(_create)(data, user) return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) From 8d09e40b3b63e5917a345262e4a28cd728d38a05 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 18:01:55 +0200 Subject: [PATCH 044/170] rename --- etebase_fastapi/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 1d876e1..b4246ca 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -97,7 +97,7 @@ class Item(BaseModel): content: CollectionItemContent -class CollectionItemIn(BaseModel): +class CollectionIn(BaseModel): collectionType: bytes collectionKey: bytes item: Item @@ -133,7 +133,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle return revision -def _create(data: CollectionItemIn, user: User): +def _create(data: CollectionIn, user: User): with transaction.atomic(): if data.item.etag is not None: raise ValidationError("bad_etag", "etag is not null") @@ -169,6 +169,6 @@ def _create(data: CollectionItemIn, user: User): @collection_router.post("/") -async def create(data: CollectionItemIn, user: User = Depends(get_authenticated_user)): +async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) From 1e60938430884fb8a8cf9d0d1710e92894d81bb4 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 18:02:29 +0200 Subject: [PATCH 045/170] rename --- etebase_fastapi/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index b4246ca..9e64072 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -82,7 +82,7 @@ async def list_multi( return response -class CollectionItemContent(BaseModel): +class CollectionItemRevision(BaseModel): uid: str meta: bytes deleted: bool @@ -94,7 +94,7 @@ class Item(BaseModel): version: int etag: t.Optional[str] encryptionKey: t.Optional[bytes] - content: CollectionItemContent + content: CollectionItemRevision class CollectionIn(BaseModel): @@ -103,7 +103,7 @@ class CollectionIn(BaseModel): item: Item -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemContent): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) From 94161943ca019d1e887ce8665e1c0e82df941eee Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 18:09:46 +0200 Subject: [PATCH 046/170] chunks type hint --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 9e64072..77abe0f 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -86,7 +86,7 @@ class CollectionItemRevision(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.List[t.Union[str, bytes]]] + chunks: t.List[t.Tuple[str, t.Optional[bytes]]] class Item(BaseModel): From 53662451a34e413457beb806564bcd98ac33e743 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sun, 27 Dec 2020 11:36:18 +0200 Subject: [PATCH 047/170] collection.py: get by uid and fixed create --- etebase_fastapi/collection.py | 115 ++++++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 77abe0f..0ceb988 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,8 +7,8 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status -from pydantic import BaseModel +from fastapi import APIRouter, Depends, status, Query +from pydantic import BaseModel, Field from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember @@ -22,12 +22,57 @@ collection_router = APIRouter(route_class=MsgpackRoute) default_queryset = Collection.objects.all() +Prefetch = t.Literal["auto", "medium"] +PrefetchQuery = Query(default="auto") + + class ListMulti(BaseModel): collectionTypes: t.List[bytes] +class CollectionItemRevisionOut(BaseModel): + uid: str + meta: bytes + deleted: bool + chunks: t.List[t.Tuple[str, t.Optional[bytes]]] + + class Config: + orm_mode = True + + @classmethod + def from_orm_user( + cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, prefetch: Prefetch + ) -> "CollectionItemRevisionOut": + chunk_obj = obj.chunks_relation.get().chunk + if prefetch == "auto": + with open(chunk_obj.chunkFile.path, "rb") as f: + chunks = chunk_obj.uid, f.read() + else: + chunks = (chunk_obj.uid,) + return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=[chunks]) + + class CollectionItemOut(BaseModel): uid: str + version: int + encryptionKey: t.Optional[bytes] + etag: t.Optional[str] + content: CollectionItemRevisionOut + + class Config: + orm_mode = True + + @classmethod + def from_orm_user( + cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, prefetch: Prefetch + ) -> "CollectionItemOut": + return cls( + uid=obj.uid, + version=obj.version, + encryptionKey=obj.encryptionKey, + etag=obj.etag, + content=CollectionItemRevisionOut.from_orm_user(obj.content, prefetch), + ) class CollectionOut(BaseModel): @@ -38,16 +83,17 @@ class CollectionOut(BaseModel): item: CollectionItemOut @classmethod - def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User) -> "CollectionOut": + def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User, prefetch: Prefetch) -> "CollectionOut": member: CollectionMember = obj.members.get(user=user) collection_type = member.collectionType - return cls( + ret = cls( collectionType=collection_type and collection_type.uid, collectionKey=member.encryptionKey, accessLevel=member.accessLevel, stoken=obj.stoken, - item=CollectionItemOut(uid=obj.main_item.uid), + item=CollectionItemOut.from_orm_user(obj.main_item, prefetch), ) + return ret class ListResponse(BaseModel): @@ -56,11 +102,26 @@ class ListResponse(BaseModel): done: bool +class ItemIn(BaseModel): + uid: str + version: int + etag: t.Optional[str] + content: CollectionItemRevisionOut + + +class CollectionIn(BaseModel): + collectionType: bytes + collectionKey: bytes + item: ItemIn + + @sync_to_async -def list_common(queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int) -> MsgpackResponse: +def list_common( + queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int, prefetch: Prefetch +) -> MsgpackResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid - data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user) for item in queryset] + data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user, prefetch) for item in queryset] ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -71,39 +132,22 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: @collection_router.post("/list_multi/") async def list_multi( - data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, user: User = Depends(get_authenticated_user) + data: ListMulti, + stoken: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, ): queryset = get_collection_queryset(user, default_queryset) # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - response = await list_common(queryset, user, stoken, limit) + response = await list_common(queryset, user, stoken, limit, prefetch) return response -class CollectionItemRevision(BaseModel): - uid: str - meta: bytes - deleted: bool - chunks: t.List[t.Tuple[str, t.Optional[bytes]]] - - -class Item(BaseModel): - uid: str - version: int - etag: t.Optional[str] - encryptionKey: t.Optional[bytes] - content: CollectionItemRevision - - -class CollectionIn(BaseModel): - collectionType: bytes - collectionKey: bytes - item: Item - - -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionOut): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) @@ -147,7 +191,7 @@ def _create(data: CollectionIn, user: User): instance.save() main_item = models.CollectionItem.objects.create( - uid=data.item.uid, version=data.item.version, encryptionKey=data.item.encryptionKey, collection=instance + uid=data.item.uid, version=data.item.version, collection=instance ) instance.main_item = main_item @@ -172,3 +216,10 @@ def _create(data: CollectionIn, user: User): async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) + + +@collection_router.get("/{uid}/") +def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): + obj = get_collection_queryset(user, default_queryset).get(uid=uid) + ret = CollectionOut.from_orm_user(obj, user, prefetch) + return MsgpackResponse(ret) From b3c177faa63c47e99411cd56802e75ee84304bad Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sun, 27 Dec 2020 14:23:19 +0200 Subject: [PATCH 048/170] from_orm_context --- etebase_fastapi/collection.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 0ceb988..2b0b876 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,3 +1,4 @@ +import dataclasses import typing as t from asgiref.sync import sync_to_async @@ -26,6 +27,12 @@ Prefetch = t.Literal["auto", "medium"] PrefetchQuery = Query(default="auto") +@dataclasses.dataclass +class Context: + user: t.Optional[User] + prefetch: t.Optional[Prefetch] + + class ListMulti(BaseModel): collectionTypes: t.List[bytes] @@ -40,11 +47,11 @@ class CollectionItemRevisionOut(BaseModel): orm_mode = True @classmethod - def from_orm_user( - cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, prefetch: Prefetch + def from_orm_context( + cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, context: Context ) -> "CollectionItemRevisionOut": chunk_obj = obj.chunks_relation.get().chunk - if prefetch == "auto": + if context.prefetch == "auto": with open(chunk_obj.chunkFile.path, "rb") as f: chunks = chunk_obj.uid, f.read() else: @@ -63,15 +70,15 @@ class CollectionItemOut(BaseModel): orm_mode = True @classmethod - def from_orm_user( - cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, prefetch: Prefetch + def from_orm_context( + cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, context: Context ) -> "CollectionItemOut": return cls( uid=obj.uid, version=obj.version, encryptionKey=obj.encryptionKey, etag=obj.etag, - content=CollectionItemRevisionOut.from_orm_user(obj.content, prefetch), + content=CollectionItemRevisionOut.from_orm_context(obj.content, context), ) @@ -83,15 +90,15 @@ class CollectionOut(BaseModel): item: CollectionItemOut @classmethod - def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User, prefetch: Prefetch) -> "CollectionOut": - member: CollectionMember = obj.members.get(user=user) + def from_orm_context(cls: t.Type["CollectionOut"], obj: Collection, context: Context) -> "CollectionOut": + member: CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( collectionType=collection_type and collection_type.uid, collectionKey=member.encryptionKey, accessLevel=member.accessLevel, stoken=obj.stoken, - item=CollectionItemOut.from_orm_user(obj.main_item, prefetch), + item=CollectionItemOut.from_orm_context(obj.main_item, context), ) return ret @@ -121,7 +128,8 @@ def list_common( ) -> MsgpackResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid - data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user, prefetch) for item in queryset] + context = Context(user, prefetch) + data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in queryset] ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -221,5 +229,5 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user @collection_router.get("/{uid}/") def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): obj = get_collection_queryset(user, default_queryset).get(uid=uid) - ret = CollectionOut.from_orm_user(obj, user, prefetch) + ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) From 6e4f8f9917f07b4727adb7e9eb64634ac21b586a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 14:36:16 +0200 Subject: [PATCH 049/170] Fix list_multi to return the filtered queryset. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 2b0b876..ff31e27 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -129,7 +129,7 @@ def list_common( result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) - data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in queryset] + data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) From a0aaf79f42450343dee7c095a9269dabd9a1d39d Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sun, 27 Dec 2020 15:53:31 +0200 Subject: [PATCH 050/170] item_batch and item_transaction boilerplate only --- etebase_fastapi/collection.py | 52 +++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ff31e27..093ea8a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -8,7 +8,7 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Query +from fastapi import APIRouter, Depends, status, Query, Request from pydantic import BaseModel, Field from django_etebase import models @@ -20,7 +20,7 @@ from .stoken_handler import filter_by_stoken_and_limit User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) -default_queryset = Collection.objects.all() +default_queryset: QuerySet = Collection.objects.all() Prefetch = t.Literal["auto", "medium"] @@ -122,6 +122,19 @@ class CollectionIn(BaseModel): item: ItemIn +class ItemDepIn(BaseModel): + etag: str + uid: str + + class Config: + orm_mode = True + + +class ItemBatchIn(BaseModel): + items: t.List[ItemIn] + deps: t.Optional[ItemDepIn] + + @sync_to_async def list_common( queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int, prefetch: Prefetch @@ -155,6 +168,14 @@ async def list_multi( return response +@collection_router.post("/list/") +async def collection_list( + req: Request, + user: User = Depends(get_authenticated_user), +): + pass + + def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionOut): chunks_objs = [] @@ -231,3 +252,30 @@ def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefe obj = get_collection_queryset(user, default_queryset).get(uid=uid) ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) + + +def item_bulk_common(data: ItemBatchIn, user: User, stoken: str, uid: str, validate_etag: bool): + queryset = get_collection_queryset(user, default_queryset) + with transaction.atomic(): # We need this for locking the collection object + collection_object = queryset.select_for_update().get(uid=uid) + if stoken is not None and stoken != collection_object.stoken: + raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + + + +def item_create(): + pass # + + +@collection_router.post("/{uid}/item/transaction/") +def item_transaction( + uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) +): + item_bulk_common(data, user, stoken, uid, validate_etag=True) + + +@collection_router.post("/{uid}/item/batch/") +def item_batch( + uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) +): + item_bulk_common(data, user, stoken, uid, validate_etag=False) From 6f543751a6f511835764cf14a98b016f47cede96 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:25:16 +0200 Subject: [PATCH 051/170] Fix and improve typing. --- etebase_fastapi/stoken_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index c840f0e..9ea1500 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -10,7 +10,7 @@ from django_etebase.models import Stoken StokenAnnotation = t.Any -def get_stoken_obj(stoken: t.Optional[str]): +def get_stoken_obj(stoken: t.Optional[str]) -> t.Optional[Stoken]: if stoken is not None: try: return Stoken.objects.get(uid=stoken) @@ -22,7 +22,7 @@ def get_stoken_obj(stoken: t.Optional[str]): def filter_by_stoken( stoken: t.Optional[str], queryset: QuerySet, stoken_annotation: StokenAnnotation -) -> t.Tuple[QuerySet, t.Optional[str]]: +) -> t.Tuple[QuerySet, t.Optional[Stoken]]: stoken_rev = get_stoken_obj(stoken) queryset = queryset.annotate(max_stoken=stoken_annotation).order_by("max_stoken") From df855897f8a4d9fb1d1501d87b7ae3a585aa7256 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:26:36 +0200 Subject: [PATCH 052/170] Fix type error. --- etebase_fastapi/stoken_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index 9ea1500..fb89651 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -38,7 +38,7 @@ def get_queryset_stoken(queryset: list) -> t.Optional[Stoken]: for row in queryset: rowmaxid = getattr(row, "max_stoken") or -1 maxid = max(maxid, rowmaxid) - new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) + new_stoken = Stoken.objects.get(id=maxid) if (maxid >= 0) else None return new_stoken or None From 9d213350e74072aca53167de278692a039d7ad55 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:45:29 +0200 Subject: [PATCH 053/170] exceptions.py: fix typo in filename. --- etebase_fastapi/app.py | 2 +- etebase_fastapi/authentication.py | 2 +- etebase_fastapi/collection.py | 2 +- etebase_fastapi/{execptions.py => exceptions.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename etebase_fastapi/{execptions.py => exceptions.py} (100%) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index fac2a31..6fbdd04 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -9,7 +9,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() from fastapi import FastAPI, Request -from .execptions import CustomHttpException +from .exceptions import CustomHttpException from .authentication import authentication_router from .collection import collection_router from .msgpack import MsgpackResponse diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 9ecee4c..51ba80d 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -26,7 +26,7 @@ from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user from django_etebase.views import msgpack_encode, msgpack_decode -from .execptions import AuthenticationFailed, transform_validation_error, ValidationError +from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError from .msgpack import MsgpackResponse, MsgpackRoute User = get_user_model() diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 093ea8a..b4e9dae 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, Field from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user -from .execptions import ValidationError +from .exceptions import ValidationError from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/exceptions.py similarity index 100% rename from etebase_fastapi/execptions.py rename to etebase_fastapi/exceptions.py From 249c3dc2be7dfe2015a19e5758bb9a60e58e6b9f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:04:33 +0200 Subject: [PATCH 054/170] Cleanup models to have common parents. --- etebase_fastapi/collection.py | 49 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index b4e9dae..f687127 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -9,7 +9,7 @@ from django.db import transaction from django.db.models import Q from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Query, Request -from pydantic import BaseModel, Field +from pydantic import BaseModel from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember @@ -37,7 +37,7 @@ class ListMulti(BaseModel): collectionTypes: t.List[bytes] -class CollectionItemRevisionOut(BaseModel): +class CollectionItemRevision(BaseModel): uid: str meta: bytes deleted: bool @@ -48,8 +48,8 @@ class CollectionItemRevisionOut(BaseModel): @classmethod def from_orm_context( - cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, context: Context - ) -> "CollectionItemRevisionOut": + cls: t.Type["CollectionItemRevision"], obj: models.CollectionItemRevision, context: Context + ) -> "CollectionItemRevision": chunk_obj = obj.chunks_relation.get().chunk if context.prefetch == "auto": with open(chunk_obj.chunkFile.path, "rb") as f: @@ -59,13 +59,14 @@ class CollectionItemRevisionOut(BaseModel): return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=[chunks]) -class CollectionItemOut(BaseModel): +class CollectionItemCommon(BaseModel): uid: str version: int encryptionKey: t.Optional[bytes] - etag: t.Optional[str] - content: CollectionItemRevisionOut + content: CollectionItemRevision + +class CollectionItemOut(CollectionItemCommon): class Config: orm_mode = True @@ -82,9 +83,16 @@ class CollectionItemOut(BaseModel): ) -class CollectionOut(BaseModel): - collectionKey: bytes +class CollectionItemIn(CollectionItemCommon): + etag: t.Optional[str] + + +class CollectionCommon(BaseModel): collectionType: bytes + collectionKey: bytes + + +class CollectionOut(CollectionCommon): accessLevel: AccessLevels stoken: str item: CollectionItemOut @@ -103,35 +111,23 @@ class CollectionOut(BaseModel): return ret +class CollectionIn(CollectionCommon): + item: CollectionItemIn + + class ListResponse(BaseModel): data: t.List[CollectionOut] stoken: t.Optional[str] done: bool -class ItemIn(BaseModel): - uid: str - version: int - etag: t.Optional[str] - content: CollectionItemRevisionOut - - -class CollectionIn(BaseModel): - collectionType: bytes - collectionKey: bytes - item: ItemIn - - class ItemDepIn(BaseModel): etag: str uid: str - class Config: - orm_mode = True - class ItemBatchIn(BaseModel): - items: t.List[ItemIn] + items: t.List[CollectionItemIn] deps: t.Optional[ItemDepIn] @@ -262,7 +258,6 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: str, uid: str, valid raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) - def item_create(): pass # From b2fe30ac26f9d6380164baa72faa720c493de354 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:33:34 +0200 Subject: [PATCH 055/170] Implement item_create, batch and transaction. --- etebase_fastapi/collection.py | 71 ++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index f687127..5656fb6 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,7 +14,7 @@ from pydantic import BaseModel from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user -from .exceptions import ValidationError +from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit @@ -79,7 +79,7 @@ class CollectionItemOut(CollectionItemCommon): version=obj.version, encryptionKey=obj.encryptionKey, etag=obj.etag, - content=CollectionItemRevisionOut.from_orm_context(obj.content, context), + content=CollectionItemRevision.from_orm_context(obj.content, context), ) @@ -125,11 +125,26 @@ class ItemDepIn(BaseModel): etag: str uid: str + def validate_db(self): + item = models.CollectionItem.objects.get(uid=self.uid) + etag = self.etag + if item.etag != etag: + raise ValidationError( + "wrong_etag", + "Wrong etag. Expected {} got {}".format(item.etag, etag), + status_code=status.HTTP_409_CONFLICT, + ) + class ItemBatchIn(BaseModel): items: t.List[CollectionItemIn] deps: t.Optional[ItemDepIn] + def validate_db(self): + if self.deps is not None: + for key, _value in self.deps: + getattr(self.deps, key).validate_db() + @sync_to_async def list_common( @@ -172,7 +187,7 @@ async def collection_list( pass -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionOut): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) @@ -250,16 +265,60 @@ def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefe return MsgpackResponse(ret) -def item_bulk_common(data: ItemBatchIn, user: User, stoken: str, uid: str, validate_etag: bool): +def item_create(item_model: CollectionItemIn, validate_etag: bool): + """Function that's called when this serializer creates an item""" + etag = item_model.etag + revision_data = item_model.content + uid = item_model.uid + + Model = models.CollectionItem + + with transaction.atomic(): + instance, created = Model.objects.get_or_create( + uid=uid, defaults=item_model.dict(exclude={"uid", "etag", "content"}) + ) + cur_etag = instance.etag if not created else None + + # If we are trying to update an up to date item, abort early and consider it a success + if cur_etag == revision_data.uid: + return instance + + if validate_etag and cur_etag != etag: + raise ValidationError( + "wrong_etag", + "Wrong etag. Expected {} got {}".format(cur_etag, etag), + status_code=status.HTTP_409_CONFLICT, + ) + + if not created: + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = instance.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() + + try: + process_revisions_for_item(instance, revision_data) + except django_exceptions.ValidationError as e: + transform_validation_error("content", e) + + return instance + + +def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user, default_queryset) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) + if stoken is not None and stoken != collection_object.stoken: raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + # XXX-TOM: make sure we return compatible errors + data.validate_db() + for item in data.items: + item_create(item, validate_etag) -def item_create(): - pass # + return MsgpackResponse({}) @collection_router.post("/{uid}/item/transaction/") From aa483709c33952b91cc9d75b47ef1c68b197b22e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:39:20 +0200 Subject: [PATCH 056/170] Fix item creation. --- etebase_fastapi/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 5656fb6..8475f25 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -265,7 +265,7 @@ def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefe return MsgpackResponse(ret) -def item_create(item_model: CollectionItemIn, validate_etag: bool): +def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): """Function that's called when this serializer creates an item""" etag = item_model.etag revision_data = item_model.content @@ -275,7 +275,7 @@ def item_create(item_model: CollectionItemIn, validate_etag: bool): with transaction.atomic(): instance, created = Model.objects.get_or_create( - uid=uid, defaults=item_model.dict(exclude={"uid", "etag", "content"}) + uid=uid, collection=collection, defaults=item_model.dict(exclude={"uid", "etag", "content"}) ) cur_etag = instance.etag if not created else None @@ -316,7 +316,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid # XXX-TOM: make sure we return compatible errors data.validate_db() for item in data.items: - item_create(item, validate_etag) + item_create(item, collection_object, validate_etag) return MsgpackResponse({}) From 8afca6ca96c3f15807651be64b476374adeeee98 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:44:18 +0200 Subject: [PATCH 057/170] kwarg items: use the same naming as django_etebase. --- etebase_fastapi/collection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 8475f25..102a7e0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -321,15 +321,15 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@collection_router.post("/{uid}/item/transaction/") +@collection_router.post("/{collection_uid}/item/transaction/") def item_transaction( - uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, uid, validate_etag=True) + item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@collection_router.post("/{uid}/item/batch/") +@collection_router.post("/{collection_uid}/item/batch/") def item_batch( - uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, uid, validate_etag=False) + item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) From 92f6ccbc28f99494a464f717c769b98ba56ec8c1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:02:36 +0200 Subject: [PATCH 058/170] Implement item_list and item_get. --- etebase_fastapi/collection.py | 83 ++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 102a7e0..d0a9e61 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -11,16 +11,18 @@ from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Query, Request from pydantic import BaseModel +# FIXME: it's not good that some things are imported, and some are used from the model including all of the name clashes from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit +from .stoken_handler import filter_by_stoken_and_limit, StokenAnnotation User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = Collection.objects.all() +default_item_queryset: QuerySet = models.CollectionItem.objects.all() Prefetch = t.Literal["auto", "medium"] @@ -115,12 +117,18 @@ class CollectionIn(CollectionCommon): item: CollectionItemIn -class ListResponse(BaseModel): +class CollectionListResponse(BaseModel): data: t.List[CollectionOut] stoken: t.Optional[str] done: bool +class CollectionItemListResponse(BaseModel): + data: t.List[CollectionItemOut] + stoken: t.Optional[str] + done: bool + + class ItemDepIn(BaseModel): etag: str uid: str @@ -147,14 +155,18 @@ class ItemBatchIn(BaseModel): @sync_to_async -def list_common( - queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int, prefetch: Prefetch +def collection_list_common( + queryset: QuerySet, + user: User, + stoken: t.Optional[str], + limit: int, + prefetch: Prefetch, ) -> MsgpackResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] - ret = ListResponse(data=data, stoken=new_stoken, done=done) + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -162,6 +174,19 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: return queryset.filter(members__user=user) +def get_item_queryset( + user: User, collection_uid: str, queryset: QuerySet = default_item_queryset +) -> t.Tuple[models.Collection, QuerySet]: + try: + collection = get_collection_queryset(user, Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise ValidationError("does_not_exist", "Collection does not exist", status_code=status.HTTP_404_NOT_FOUND) + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) + + return collection, queryset + + @collection_router.post("/list_multi/") async def list_multi( data: ListMulti, @@ -175,7 +200,8 @@ async def list_multi( queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - response = await list_common(queryset, user, stoken, limit, prefetch) + # XXX-TOM: missing removedMemeberships + response = await collection_list_common(queryset, user, stoken, limit, prefetch) return response @@ -305,6 +331,51 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance +@collection_router.get("/{collection_uid}/item/{uid}/") +def item_get( + collection_uid: str, uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery +): + _, queryset = get_item_queryset(user, collection_uid) + obj = queryset.get(uid=uid) + ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) + return MsgpackResponse(ret) + + +@sync_to_async +def item_list_common( + queryset: QuerySet, + user: User, + stoken: t.Optional[str], + limit: int, + prefetch: Prefetch, +) -> MsgpackResponse: + result, new_stoken_obj, done = filter_by_stoken_and_limit( + stoken, limit, queryset, models.CollectionItem.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + context = Context(user, prefetch) + data: t.List[CollectionItemOut] = [CollectionItemOut.from_orm_context(item, context) for item in result] + ret = CollectionItemListResponse(data=data, stoken=new_stoken, done=done) + return MsgpackResponse(content=ret) + + +@collection_router.get("/{collection_uid}/item/") +async def item_list( + collection_uid: str, + stoken: t.Optional[str] = None, + limit: int = 50, + prefetch: Prefetch = PrefetchQuery, + withCollection: bool = False, + user: User = Depends(get_authenticated_user), +): + _, queryset = await sync_to_async(get_item_queryset)(user, collection_uid) + if not withCollection: + queryset = queryset.filter(parent__isnull=True) + + response = await item_list_common(queryset, user, stoken, limit, prefetch) + return response + + def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user, default_queryset) with transaction.atomic(): # We need this for locking the collection object From 611c0f3b0a89088aa36afb05e4873281eb971852 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:03:17 +0200 Subject: [PATCH 059/170] Conform to naming conventions. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index d0a9e61..91541ce 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -285,7 +285,7 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user @collection_router.get("/{uid}/") -def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): +def collection_get(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): obj = get_collection_queryset(user, default_queryset).get(uid=uid) ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) From e5dbfb57460439cdd28b0d8c9656599ecbe99f03 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:07:07 +0200 Subject: [PATCH 060/170] Make the import of models more consistent. --- etebase_fastapi/collection.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 91541ce..6e567fd 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -11,17 +11,15 @@ from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Query, Request from pydantic import BaseModel -# FIXME: it's not good that some things are imported, and some are used from the model including all of the name clashes from django_etebase import models -from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit, StokenAnnotation +from .stoken_handler import filter_by_stoken_and_limit User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) -default_queryset: QuerySet = Collection.objects.all() +default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() @@ -39,7 +37,7 @@ class ListMulti(BaseModel): collectionTypes: t.List[bytes] -class CollectionItemRevision(BaseModel): +class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool @@ -50,8 +48,8 @@ class CollectionItemRevision(BaseModel): @classmethod def from_orm_context( - cls: t.Type["CollectionItemRevision"], obj: models.CollectionItemRevision, context: Context - ) -> "CollectionItemRevision": + cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context + ) -> "CollectionItemRevisionInOut": chunk_obj = obj.chunks_relation.get().chunk if context.prefetch == "auto": with open(chunk_obj.chunkFile.path, "rb") as f: @@ -65,7 +63,7 @@ class CollectionItemCommon(BaseModel): uid: str version: int encryptionKey: t.Optional[bytes] - content: CollectionItemRevision + content: CollectionItemRevisionInOut class CollectionItemOut(CollectionItemCommon): @@ -81,7 +79,7 @@ class CollectionItemOut(CollectionItemCommon): version=obj.version, encryptionKey=obj.encryptionKey, etag=obj.etag, - content=CollectionItemRevision.from_orm_context(obj.content, context), + content=CollectionItemRevisionInOut.from_orm_context(obj.content, context), ) @@ -95,12 +93,12 @@ class CollectionCommon(BaseModel): class CollectionOut(CollectionCommon): - accessLevel: AccessLevels + accessLevel: models.AccessLevels stoken: str item: CollectionItemOut @classmethod - def from_orm_context(cls: t.Type["CollectionOut"], obj: Collection, context: Context) -> "CollectionOut": + def from_orm_context(cls: t.Type["CollectionOut"], obj: models.Collection, context: Context) -> "CollectionOut": member: CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( @@ -162,7 +160,7 @@ def collection_list_common( limit: int, prefetch: Prefetch, ) -> MsgpackResponse: - result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) + result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, models.Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] @@ -178,8 +176,8 @@ def get_item_queryset( user: User, collection_uid: str, queryset: QuerySet = default_item_queryset ) -> t.Tuple[models.Collection, QuerySet]: try: - collection = get_collection_queryset(user, Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: + collection = get_collection_queryset(user, models.Collection.objects).get(uid=collection_uid) + except models.Collection.DoesNotExist: raise ValidationError("does_not_exist", "Collection does not exist", status_code=status.HTTP_404_NOT_FOUND) # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) @@ -213,7 +211,7 @@ async def collection_list( pass -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) From 407ce0b7a5222900ff8d2d2c84b8e38726afcfc4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:10:59 +0200 Subject: [PATCH 061/170] Fixed collection_list_common. --- etebase_fastapi/collection.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6e567fd..0f019fa 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -15,7 +15,7 @@ from django_etebase import models from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit +from .stoken_handler import filter_by_stoken_and_limit, get_stoken_obj User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -115,11 +115,17 @@ class CollectionIn(CollectionCommon): item: CollectionItemIn +class RemovedMembershipOut(BaseModel): + uid: str + + class CollectionListResponse(BaseModel): data: t.List[CollectionOut] stoken: t.Optional[str] done: bool + removedMemberships: t.Optional[RemovedMembershipOut] + class CollectionItemListResponse(BaseModel): data: t.List[CollectionItemOut] @@ -164,7 +170,22 @@ def collection_list_common( new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] - ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) + + stoken_obj = get_stoken_obj(stoken) + removedMemberships = None + if stoken_obj is not None: + # FIXME: honour limit? (the limit should be combined for data and this because of stoken) + remed_qs = models.CollectionMemberRemoved.objects.filter(user=user, stoken__id__gt=stoken_obj.id) + if not done and new_stoken_obj is not None: + # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken + # can point to the most recent collection change rather than most recent removed membership. + remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) + + remed = remed_qs.values_list("collection__uid", flat=True) + if len(remed) > 0: + removedMemberships = [{"uid": x} for x in remed] + + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done, removedMemberships=removedMemberships) return MsgpackResponse(content=ret) @@ -194,13 +215,13 @@ async def list_multi( prefetch: Prefetch = PrefetchQuery, ): queryset = get_collection_queryset(user, default_queryset) + # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - # XXX-TOM: missing removedMemeberships - response = await collection_list_common(queryset, user, stoken, limit, prefetch) - return response + + return await collection_list_common(queryset, user, stoken, limit, prefetch) @collection_router.post("/list/") From 7ad98b8d28a70de1074d15f17eadc201720283a2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:30:17 +0200 Subject: [PATCH 062/170] Implement is_etebase. --- etebase_fastapi/authentication.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 51ba80d..e355aa4 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -223,6 +223,11 @@ def validate_login_request( raise ValidationError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) +@authentication_router.get("/is_etebase/") +async def is_etebase(): + return MsgpackResponse({}) + + @authentication_router.post("/login_challenge/") async def login_challenge(user: User = Depends(get_login_user)): enc_key = get_encryption_key(user.userinfo.salt) From c6c52cfe1100ef401d4941f71f7b28601473f579 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:33:01 +0200 Subject: [PATCH 063/170] Implement collection list. --- etebase_fastapi/collection.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 0f019fa..56e2180 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -166,7 +166,9 @@ def collection_list_common( limit: int, prefetch: Prefetch, ) -> MsgpackResponse: - result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, models.Collection.stoken_annotation) + result, new_stoken_obj, done = filter_by_stoken_and_limit( + stoken, limit, queryset, models.Collection.stoken_annotation + ) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] @@ -227,9 +229,13 @@ async def list_multi( @collection_router.post("/list/") async def collection_list( req: Request, + stoken: t.Optional[str] = None, + limit: int = 50, + prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), ): - pass + queryset = get_collection_queryset(user, default_queryset) + return await collection_list_common(queryset, user, stoken, limit, prefetch) def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): From a9bc08a98d59e60a929476f21bb17551bd65648b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:39:47 +0200 Subject: [PATCH 064/170] Item batch/transaction: fix return data. --- etebase_fastapi/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 56e2180..3a9ca8b 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -421,11 +421,11 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) @collection_router.post("/{collection_uid}/item/batch/") def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) From a3ae769a2ca3d4a9e3d1f47c67df3350dfd432c7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:58:15 +0200 Subject: [PATCH 065/170] Implement fetch_updates. --- etebase_fastapi/collection.py | 42 ++++++++++++++++++++++++++++++- etebase_fastapi/stoken_handler.py | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3a9ca8b..7679ed6 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -15,7 +15,7 @@ from django_etebase import models from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit, get_stoken_obj +from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -133,6 +133,11 @@ class CollectionItemListResponse(BaseModel): done: bool +class CollectionItemBulkGetIn(BaseModel): + uid: str + etag: t.Optional[str] + + class ItemDepIn(BaseModel): etag: str uid: str @@ -417,6 +422,41 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) +@collection_router.post("/{collection_uid}/item/fetch_updates/") +def fetch_updates( + collection_uid: str, + data: t.List[CollectionItemBulkGetIn], + stoken: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_item_queryset(user, collection_uid) + # FIXME: make configurable? + item_limit = 200 + + if len(data) > item_limit: + raise ValidationError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) + + queryset, stoken_rev = filter_by_stoken(stoken, queryset, models.CollectionItem.stoken_annotation) + + uids, etags = zip(*[(item.uid, item.etag) for item in data]) + revs = models.CollectionItemRevision.objects.filter(uid__in=etags, current=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) + + new_stoken_obj = get_queryset_stoken(queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + stoken = stoken_rev and getattr(stoken_rev, "uid", None) + new_stoken = new_stoken or stoken + + context = Context(user, prefetch) + ret = CollectionItemListResponse( + data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], + stoken=new_stoken, + done=True, # we always return all the items, so it's always done + ) + return MsgpackResponse(ret) + + @collection_router.post("/{collection_uid}/item/transaction/") def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index fb89651..a976830 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -33,7 +33,7 @@ def filter_by_stoken( return queryset, stoken_rev -def get_queryset_stoken(queryset: list) -> t.Optional[Stoken]: +def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]: maxid = -1 for row in queryset: rowmaxid = getattr(row, "max_stoken") or -1 From e7721e8fe52fce7667017acafee7a2c0bd8a7143 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:12:16 +0200 Subject: [PATCH 066/170] Fix chunk handling. --- etebase_fastapi/collection.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 7679ed6..229b8c4 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -50,13 +50,15 @@ class CollectionItemRevisionInOut(BaseModel): def from_orm_context( cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context ) -> "CollectionItemRevisionInOut": - chunk_obj = obj.chunks_relation.get().chunk - if context.prefetch == "auto": - with open(chunk_obj.chunkFile.path, "rb") as f: - chunks = chunk_obj.uid, f.read() - else: - chunks = (chunk_obj.uid,) - return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=[chunks]) + chunks = [] + for chunk_relation in obj.chunks_relation.all(): + chunk_obj = chunk_relation.chunk + if context.prefetch == "auto": + with open(chunk_obj.chunkFile.path, "rb") as f: + chunks.append((chunk_obj.uid, f.read())) + else: + chunks.append((chunk_obj.uid,)) + return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=chunks) class CollectionItemCommon(BaseModel): From e686f016521826c8764d8851bd594cef94848a6a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:34:23 +0200 Subject: [PATCH 067/170] Utils: add a utility for getting objects or 404ing. --- etebase_fastapi/collection.py | 6 ++---- etebase_fastapi/utils.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 etebase_fastapi/utils.py diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 229b8c4..2f8d1ed 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -16,6 +16,7 @@ from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken +from .utils import get_object_or_404 User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -205,10 +206,7 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: def get_item_queryset( user: User, collection_uid: str, queryset: QuerySet = default_item_queryset ) -> t.Tuple[models.Collection, QuerySet]: - try: - collection = get_collection_queryset(user, models.Collection.objects).get(uid=collection_uid) - except models.Collection.DoesNotExist: - raise ValidationError("does_not_exist", "Collection does not exist", status_code=status.HTTP_404_NOT_FOUND) + collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py new file mode 100644 index 0000000..d7f8c09 --- /dev/null +++ b/etebase_fastapi/utils.py @@ -0,0 +1,13 @@ +from fastapi import status + +from django.db.models import QuerySet +from django.core.exceptions import ObjectDoesNotExist + +from .exceptions import ValidationError + + +def get_object_or_404(queryset: QuerySet, **kwargs): + try: + return queryset.get(**kwargs) + except ObjectDoesNotExist as e: + raise ValidationError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) From 533b2787bb1ac716a9ba4c670d675432b84b3cec Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:34:40 +0200 Subject: [PATCH 068/170] Implement item revisions. --- etebase_fastapi/collection.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 2f8d1ed..8eabf9a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -136,6 +136,12 @@ class CollectionItemListResponse(BaseModel): done: bool +class CollectionItemRevisionListResponse(BaseModel): + data: t.List[CollectionItemRevisionInOut] + iterator: t.Optional[str] + done: bool + + class CollectionItemBulkGetIn(BaseModel): uid: str etag: t.Optional[str] @@ -422,6 +428,43 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) +@collection_router.get("/{collection_uid}/item/{uid}/revision/") +def item_revisions( + collection_uid: str, + uid: str, + limit: int = 50, + iterator: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), +): + _, items = get_item_queryset(user, collection_uid) + item = get_object_or_404(items, uid=uid) + + queryset = item.revisions.order_by("-id") + + if iterator is not None: + iterator_obj = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__lt=iterator_obj.id) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + context = Context(user, prefetch) + ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] + iterator = ret_data[-1].uid if len(result) > 0 else None + + ret = CollectionItemRevisionListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) + return MsgpackResponse(ret) + + @collection_router.post("/{collection_uid}/item/fetch_updates/") def fetch_updates( collection_uid: str, From 629a84f43243e1ee905ed985458cdd98c0192774 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:38:18 +0200 Subject: [PATCH 069/170] app.py: cleanup a bit. --- etebase_fastapi/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 6fbdd04..81772b4 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -3,10 +3,12 @@ import os from django.core.wsgi import get_wsgi_application from fastapi.middleware.cors import CORSMiddleware -from django.conf import settings - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() + +from django.conf import settings + +# Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request from .exceptions import CustomHttpException From 13d4121fc275a94c6d2a53d9d894e6754c24f740 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:54:06 +0200 Subject: [PATCH 070/170] Move utility functions to utils. --- etebase_fastapi/collection.py | 15 ++------------- etebase_fastapi/utils.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 8eabf9a..25883f8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,4 +1,3 @@ -import dataclasses import typing as t from asgiref.sync import sync_to_async @@ -8,7 +7,7 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Query, Request +from fastapi import APIRouter, Depends, status, Request from pydantic import BaseModel from django_etebase import models @@ -16,7 +15,7 @@ from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404 +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -24,16 +23,6 @@ default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() -Prefetch = t.Literal["auto", "medium"] -PrefetchQuery = Query(default="auto") - - -@dataclasses.dataclass -class Context: - user: t.Optional[User] - prefetch: t.Optional[Prefetch] - - class ListMulti(BaseModel): collectionTypes: t.List[bytes] diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index d7f8c09..d9bef73 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,10 +1,25 @@ -from fastapi import status +import dataclasses +import typing as t + +from fastapi import status, Query from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth import get_user_model from .exceptions import ValidationError +User = get_user_model() + +Prefetch = t.Literal["auto", "medium"] +PrefetchQuery = Query(default="auto") + + +@dataclasses.dataclass +class Context: + user: t.Optional[User] + prefetch: t.Optional[Prefetch] + def get_object_or_404(queryset: QuerySet, **kwargs): try: From ec8c69b3f3ff0c12a31dc941722c86d7ace6d36e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 20:36:11 +0200 Subject: [PATCH 071/170] Fix a few FIXMEs. --- etebase_fastapi/authentication.py | 21 +++++++++++---------- etebase_fastapi/test_reset_view.py | 6 +++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index e355aa4..a8fbbfe 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -24,7 +24,7 @@ from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry -from django_etebase.utils import create_user +from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError from .msgpack import MsgpackResponse, MsgpackRoute @@ -268,20 +268,21 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return Response(status_code=status.HTTP_204_NO_CONTENT) -def signup_save(data: SignupIn) -> User: +def signup_save(data: SignupIn, request: Request) -> User: user_data = data.user with transaction.atomic(): try: - # XXX-TOM - # view = self.context.get("view", None) - # user_queryset = get_user_queryset(User.objects.all(), view) - user_queryset = User.objects.all() + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) instance = user_queryset.get(**{User.USERNAME_FIELD: user_data.username.lower()}) except User.DoesNotExist: # Create the user and save the casing the user chose as the first name try: - # XXX-TOM - instance = create_user(**user_data.dict(), password=None, first_name=user_data.username, view=None) + instance = create_user( + **user_data.dict(), + password=None, + first_name=user_data.username, + context=CallbackContext(request.path_params), + ) instance.full_clean() except EtebaseValidationError as e: raise e @@ -298,8 +299,8 @@ def signup_save(data: SignupIn) -> User: @authentication_router.post("/signup/") -async def signup(data: SignupIn): - user = await sync_to_async(signup_save)(data) +async def signup(data: SignupIn, request: Request): + user = await sync_to_async(signup_save)(data, request) # XXX-TOM data = await sync_to_async(LoginOut.from_orm)(user) await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index ea7d8d9..435a56e 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 -from fastapi import APIRouter, Response, status +from fastapi import APIRouter, Request, Response, status from django_etebase.utils import get_user_queryset from etebase_fastapi.authentication import SignupIn, signup_save @@ -13,7 +13,7 @@ User = get_user_model() @test_reset_view_router.post("/reset/") -def reset(data: SignupIn): +def reset(data: SignupIn, request: Request): # Only run when in DEBUG mode! It's only used for tests if not settings.DEBUG: return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) @@ -28,7 +28,7 @@ def reset(data: SignupIn): if hasattr(user, "userinfo"): user.userinfo.delete() - signup_save(data) + signup_save(data, request) # Delete all of the journal data for this user for a clear test env user.collection_set.all().delete() user.collectionmember_set.all().delete() From 7f90edc5114b50fa2ad414a4f2abc18f263fbdf4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:01:14 +0200 Subject: [PATCH 072/170] MsgPack: handle no content. --- etebase_fastapi/msgpack.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 399f3d0..edffd7e 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -19,7 +19,10 @@ class MsgpackRequest(Request): class MsgpackResponse(Response): media_type = "application/msgpack" - def render(self, content: t.Any) -> bytes: + def render(self, content: t.Optional[t.Any]) -> t.Optional[bytes]: + if content is None: + return b"" + if isinstance(content, BaseModel): content = content.dict() return msgpack.packb(content, use_bin_type=True) From b70f2b74705a1ce91ad28c10f643e626b5df0459 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:08:00 +0200 Subject: [PATCH 073/170] Invitations: implement invitations endpoints. --- etebase_fastapi/app.py | 3 + etebase_fastapi/invitation.py | 229 ++++++++++++++++++++++++++++++++++ etebase_fastapi/utils.py | 7 ++ 3 files changed, 239 insertions(+) create mode 100644 etebase_fastapi/invitation.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 81772b4..ff50ce5 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -14,6 +14,7 @@ from fastapi import FastAPI, Request from .exceptions import CustomHttpException from .authentication import authentication_router from .collection import collection_router +from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse app = FastAPI() @@ -21,6 +22,8 @@ VERSION = "v1" BASE_PATH = f"/api/{VERSION}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") +app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") +app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") if settings.DEBUG: from .test_reset_view import test_reset_view_router diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py new file mode 100644 index 0000000..077dcfd --- /dev/null +++ b/etebase_fastapi/invitation.py @@ -0,0 +1,229 @@ +import typing as t + +from django.contrib.auth import get_user_model +from django.db import transaction, IntegrityError +from django.db.models import QuerySet +from fastapi import APIRouter, Depends, status, Request +from pydantic import BaseModel + +from django_etebase import models +from django_etebase.utils import get_user_queryset, CallbackContext +from .authentication import get_authenticated_user +from .exceptions import ValidationError, PermissionDenied +from .msgpack import MsgpackRoute, MsgpackResponse +from .utils import get_object_or_404, Context, is_collection_admin + +User = get_user_model() +invitation_incoming_router = APIRouter(route_class=MsgpackRoute) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) +default_queryset: QuerySet = models.CollectionInvitation.objects.all() + + +class UserInfoOut(BaseModel): + pubkey: bytes + + class Config: + orm_mode = True + + +class CollectionInvitationAcceptIn(BaseModel): + collectionType: bytes + encryptionKey: bytes + + +class CollectionInvitationCommon(BaseModel): + uid: str + version: int + accessLevel: models.AccessLevels + username: str + collection: str + signedEncryptionKey: bytes + + +class CollectionInvitationIn(CollectionInvitationCommon): + def validate_db(self, context: Context): + if context.user.username == self.username.lower(): + raise ValidationError("no_self_invite", "Inviting yourself is not allowed") + + +class CollectionInvitationOut(CollectionInvitationCommon): + fromUsername: str + fromPubkey: bytes + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut": + return cls( + uid=obj.uid, + version=obj.version, + accessLevel=obj.accessLevel, + username=obj.user.username, + collection=obj.collection.uid, + fromUsername=obj.fromMember.user.username, + fromPubkey=obj.fromMember.user.userinfo.pubkey, + signedEncryptionKey=obj.signedEncryptionKey, + ) + + +class InvitationListResponse(BaseModel): + data: t.List[CollectionInvitationOut] + iterator: t.Optional[str] + done: bool + + +def get_incoming_queryset(user: User, queryset=default_queryset): + return queryset.filter(user=user) + + +def get_outgoing_queryset(user: User, queryset=default_queryset): + return queryset.filter(fromMember__user=user) + + +def list_common( + queryset: QuerySet, + iterator: t.Optional[str], + limit: int, +) -> MsgpackResponse: + queryset = queryset.order_by("id") + + if iterator is not None: + iterator_obj = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__gt=iterator_obj.id) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + ret_data = result + iterator = ret_data[-1].uid if len(result) > 0 else None + + ret = InvitationListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) + return MsgpackResponse(ret) + + +@invitation_incoming_router.get("/", response_model=InvitationListResponse) +def incoming_list( + iterator: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), +): + return list_common(get_incoming_queryset(user), iterator, limit) + + +@invitation_incoming_router.get("/{invitation_uid}/", response_model=CollectionInvitationOut) +def incoming_get( + invitation_uid: str, + user: User = Depends(get_authenticated_user), +): + queryset = get_incoming_queryset(user) + obj = get_object_or_404(queryset, uid=invitation_uid) + ret = CollectionInvitationOut.from_orm(obj) + return MsgpackResponse(ret) + + +@invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +def incoming_delete( + invitation_uid: str, + user: User = Depends(get_authenticated_user), +): + queryset = get_incoming_queryset(user) + obj = get_object_or_404(queryset, uid=invitation_uid) + obj.delete() + + +@invitation_incoming_router.post("/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED) +def incoming_accept( + invitation_uid: str, + data: CollectionInvitationAcceptIn, + user: User = Depends(get_authenticated_user), +): + queryset = get_incoming_queryset(user) + invitation = get_object_or_404(queryset, uid=invitation_uid) + + with transaction.atomic(): + user = invitation.user + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user) + + models.CollectionMember.objects.create( + collection=invitation.collection, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=invitation.accessLevel, + encryptionKey=data.encryptionKey, + collectionType=collection_type_obj, + ) + + models.CollectionMemberRemoved.objects.filter(user=invitation.user, collection=invitation.collection).delete() + + invitation.delete() + + +@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED) +def outgoing_create( + data: CollectionInvitationIn, + request: Request, + user: User = Depends(get_authenticated_user), +): + collection = get_object_or_404(models.Collection.objects, uid=data.collection) + to_user = get_object_or_404( + get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), username=data.username + ) + + context = Context(user, None) + data.validate_db(context) + + if not is_collection_admin(collection, user): + raise PermissionDenied("admin_access_required", "User is not an admin of this collection") + + member = collection.members.get(user=user) + + with transaction.atomic(): + try: + ret = models.CollectionInvitation.objects.create( + **data.dict(exclude={"collection", "username"}), user=to_user, fromMember=member + ) + except IntegrityError: + raise ValidationError("invitation_exists", "Invitation already exists") + + return MsgpackResponse(CollectionInvitationOut.from_orm(ret), status_code=status.HTTP_201_CREATED) + + +@invitation_outgoing_router.get("/", response_model=InvitationListResponse) +def outgoing_list( + iterator: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), +): + return list_common(get_outgoing_queryset(user), iterator, limit) + + +@invitation_outgoing_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +def outgoing_delete( + invitation_uid: str, + user: User = Depends(get_authenticated_user), +): + queryset = get_outgoing_queryset(user) + obj = get_object_or_404(queryset, uid=invitation_uid) + obj.delete() + + +@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut) +def outgoing_fetch_user_profile( + username: str, + request: Request, + user: User = Depends(get_authenticated_user), +): + kwargs = {User.USERNAME_FIELD: username.lower()} + user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) + user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) + ret = UserInfoOut.from_orm(user_info) + return MsgpackResponse(ret) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index d9bef73..150afe8 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -7,6 +7,8 @@ from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model +from django_etebase.models import AccessLevels + from .exceptions import ValidationError User = get_user_model() @@ -26,3 +28,8 @@ def get_object_or_404(queryset: QuerySet, **kwargs): return queryset.get(**kwargs) except ObjectDoesNotExist as e: raise ValidationError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) + + +def is_collection_admin(collection, user): + member = collection.members.filter(user=user).first() + return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) From b5a750d6d09ee98fe3f73c9b6766458711a793a1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:41:31 +0200 Subject: [PATCH 074/170] Collection: fix removed memberships. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 25883f8..0af6056 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -116,7 +116,7 @@ class CollectionListResponse(BaseModel): stoken: t.Optional[str] done: bool - removedMemberships: t.Optional[RemovedMembershipOut] + removedMemberships: t.Optional[t.List[RemovedMembershipOut]] class CollectionItemListResponse(BaseModel): From 36e6d3df24628cba764559add4127ab99e57c6f6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:32:48 +0200 Subject: [PATCH 075/170] Members: add member endpoints. --- etebase_fastapi/app.py | 1 + etebase_fastapi/member.py | 83 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 etebase_fastapi/member.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index ff50ce5..755340c 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -14,6 +14,7 @@ from fastapi import FastAPI, Request from .exceptions import CustomHttpException from .authentication import authentication_router from .collection import collection_router +from . import member # noqa from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py new file mode 100644 index 0000000..36aa5ce --- /dev/null +++ b/etebase_fastapi/member.py @@ -0,0 +1,83 @@ +import typing as t + +from django.contrib.auth import get_user_model +from django.db.models import QuerySet +from fastapi import Depends, status +from pydantic import BaseModel + +from django_etebase import models +from .authentication import get_authenticated_user +from .msgpack import MsgpackResponse +from .utils import get_object_or_404 +from .stoken_handler import filter_by_stoken_and_limit + +from .collection import collection_router, get_collection_queryset + +User = get_user_model() +default_queryset: QuerySet = models.CollectionMember.objects.all() + + +def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> t.Tuple[models.Collection, QuerySet]: + collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) + return collection, queryset.filter(collection=collection) + + +class CollectionMemberOut(BaseModel): + username: str + accessLevel: models.AccessLevels + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut": + return cls(username=obj.user.username, accessLevel=obj.accessLevel) + + +class MemberListResponse(BaseModel): + data: t.List[CollectionMemberOut] + iterator: t.Optional[str] + done: bool + + +@collection_router.get("/{collection_uid}/member/", response_model=MemberListResponse) +def member_list( + collection_uid: str, + iterator: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_queryset(user, collection_uid) + queryset = queryset.order_by("id") + result, new_stoken_obj, done = filter_by_stoken_and_limit( + iterator, limit, queryset, models.CollectionMember.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + + ret = MemberListResponse( + data=[CollectionMemberOut.from_orm(item) for item in result], + iterator=new_stoken, + done=done, + ) + return MsgpackResponse(ret) + + +@collection_router.delete("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +def member_delete( + collection_uid: str, + username: str, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_queryset(user, collection_uid) + obj = get_object_or_404(queryset, user__username__iexact=username) + obj.revoke() + + +@collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) +def member_leave( + collection_uid: str, + user: User = Depends(get_authenticated_user), +): + collection, _ = get_queryset(user, collection_uid) + obj = get_object_or_404(collection.members, user=user) + obj.revoke() From e8bd8927a01617a5a8841071de30aa0120ad4e4f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:47:30 +0200 Subject: [PATCH 076/170] Implement modifying access level. --- etebase_fastapi/member.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 36aa5ce..2eeb365 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -1,6 +1,7 @@ import typing as t from django.contrib.auth import get_user_model +from django.db import transaction from django.db.models import QuerySet from fastapi import Depends, status from pydantic import BaseModel @@ -22,6 +23,10 @@ def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> return collection, queryset.filter(collection=collection) +class CollectionMemberModifyAccessLevelIn(BaseModel): + accessLevel: models.AccessLevels + + class CollectionMemberOut(BaseModel): username: str accessLevel: models.AccessLevels @@ -73,6 +78,24 @@ def member_delete( obj.revoke() +@collection_router.patch("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +def member_patch( + collection_uid: str, + username: str, + data: CollectionMemberModifyAccessLevelIn, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_queryset(user, collection_uid) + instance = get_object_or_404(queryset, user__username__iexact=username) + + with transaction.atomic(): + # We only allow updating accessLevel + if instance.accessLevel != data.accessLevel: + instance.stoken = models.Stoken.objects.create() + instance.accessLevel = data.accessLevel + instance.save() + + @collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) def member_leave( collection_uid: str, From fa0cd01a59095bb781663876f334523ed6f40b07 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:50:34 +0200 Subject: [PATCH 077/170] Authentication: implement part of get_dashboard_url. --- etebase_fastapi/authentication.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index a8fbbfe..13af2dd 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -268,6 +268,19 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return Response(status_code=status.HTTP_204_NO_CONTENT) +@authentication_router.post("/dashboard_url/") +def dashboard_url(user: User = Depends(get_authenticated_user)): + # XXX-TOM + get_dashboard_url = app_settings.DASHBOARD_URL_FUNC + if get_dashboard_url is None: + raise ValidationError("not_supported", "This server doesn't have a user dashboard.") + + ret = { + "url": get_dashboard_url(request, *args, **kwargs), + } + return MsgpackResponse(ret) + + def signup_save(data: SignupIn, request: Request) -> User: user_data = data.user with transaction.atomic(): From 403d975934072ff62ed0d147d433578b75a104e8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:58:58 +0200 Subject: [PATCH 078/170] Collection: fix dep handling. --- etebase_fastapi/collection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 0af6056..5e987d8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -137,8 +137,8 @@ class CollectionItemBulkGetIn(BaseModel): class ItemDepIn(BaseModel): - etag: str uid: str + etag: str def validate_db(self): item = models.CollectionItem.objects.get(uid=self.uid) @@ -153,12 +153,12 @@ class ItemDepIn(BaseModel): class ItemBatchIn(BaseModel): items: t.List[CollectionItemIn] - deps: t.Optional[ItemDepIn] + deps: t.Optional[t.List[ItemDepIn]] def validate_db(self): if self.deps is not None: - for key, _value in self.deps: - getattr(self.deps, key).validate_db() + for dep in self.deps: + dep.validate_db() @sync_to_async From 8160a333840aa0fe1450808395346f5d62f9793d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 22:13:36 +0200 Subject: [PATCH 079/170] Get collection queryset: remove param. --- etebase_fastapi/collection.py | 14 +++++++------- etebase_fastapi/member.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 5e987d8..1fc6f0a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -194,14 +194,14 @@ def collection_list_common( return MsgpackResponse(content=ret) -def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: - return queryset.filter(members__user=user) +def get_collection_queryset(user: User) -> QuerySet: + return default_queryset.filter(members__user=user) def get_item_queryset( user: User, collection_uid: str, queryset: QuerySet = default_item_queryset ) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) + collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) @@ -216,7 +216,7 @@ async def list_multi( user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - queryset = get_collection_queryset(user, default_queryset) + queryset = get_collection_queryset(user) # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( @@ -234,7 +234,7 @@ async def collection_list( prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), ): - queryset = get_collection_queryset(user, default_queryset) + queryset = get_collection_queryset(user) return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -311,7 +311,7 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user @collection_router.get("/{uid}/") def collection_get(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): - obj = get_collection_queryset(user, default_queryset).get(uid=uid) + obj = get_collection_queryset(user).get(uid=uid) ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -402,7 +402,7 @@ async def item_list( def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): - queryset = get_collection_queryset(user, default_queryset) + queryset = get_collection_queryset(user) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 2eeb365..534cad1 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -19,7 +19,7 @@ default_queryset: QuerySet = models.CollectionMember.objects.all() def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) + collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) return collection, queryset.filter(collection=collection) From df19887af7df5a78b394ec8d8182bc9cf82bfc4c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 22:27:33 +0200 Subject: [PATCH 080/170] Use dependency injection for getting collection/item queryset. --- etebase_fastapi/collection.py | 46 +++++++++++++++++------------------ etebase_fastapi/invitation.py | 28 +++++++++------------ etebase_fastapi/member.py | 34 +++++++++----------------- 3 files changed, 46 insertions(+), 62 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 1fc6f0a..196bb1d 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -194,18 +194,19 @@ def collection_list_common( return MsgpackResponse(content=ret) -def get_collection_queryset(user: User) -> QuerySet: +def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: return default_queryset.filter(members__user=user) -def get_item_queryset( - user: User, collection_uid: str, queryset: QuerySet = default_item_queryset -) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) - # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) +def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: + return get_object_or_404(queryset, uid=collection_uid) - return collection, queryset + +def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) + + return queryset @collection_router.post("/list_multi/") @@ -213,11 +214,10 @@ async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, + queryset: QuerySet = Depends(get_collection_queryset), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - queryset = get_collection_queryset(user) - # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) @@ -228,13 +228,12 @@ async def list_multi( @collection_router.post("/list/") async def collection_list( - req: Request, stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_collection_queryset), ): - queryset = get_collection_queryset(user) return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -309,9 +308,12 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) -@collection_router.get("/{uid}/") -def collection_get(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): - obj = get_collection_queryset(user).get(uid=uid) +@collection_router.get("/{collection_uid}/") +def collection_get( + obj: models.Collection = Depends(get_collection), + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery + ): ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -358,9 +360,10 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val @collection_router.get("/{collection_uid}/item/{uid}/") def item_get( - collection_uid: str, uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery + uid: str, + queryset: QuerySet = Depends(get_item_queryset), + user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - _, queryset = get_item_queryset(user, collection_uid) obj = queryset.get(uid=uid) ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -386,14 +389,13 @@ def item_list_common( @collection_router.get("/{collection_uid}/item/") async def item_list( - collection_uid: str, + queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, withCollection: bool = False, user: User = Depends(get_authenticated_user), ): - _, queryset = await sync_to_async(get_item_queryset)(user, collection_uid) if not withCollection: queryset = queryset.filter(parent__isnull=True) @@ -419,14 +421,13 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid @collection_router.get("/{collection_uid}/item/{uid}/revision/") def item_revisions( - collection_uid: str, uid: str, limit: int = 50, iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), + items: QuerySet = Depends(get_item_queryset), ): - _, items = get_item_queryset(user, collection_uid) item = get_object_or_404(items, uid=uid) queryset = item.revisions.order_by("-id") @@ -456,13 +457,12 @@ def item_revisions( @collection_router.post("/{collection_uid}/item/fetch_updates/") def fetch_updates( - collection_uid: str, data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_item_queryset), ): - _, queryset = get_item_queryset(user, collection_uid) # FIXME: make configurable? item_limit = 200 diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 077dcfd..cbf0554 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -73,12 +73,12 @@ class InvitationListResponse(BaseModel): done: bool -def get_incoming_queryset(user: User, queryset=default_queryset): - return queryset.filter(user=user) +def get_incoming_queryset(user: User = Depends(get_authenticated_user)): + return default_queryset.filter(user=user) -def get_outgoing_queryset(user: User, queryset=default_queryset): - return queryset.filter(fromMember__user=user) +def get_outgoing_queryset(user: User = Depends(get_authenticated_user)): + return default_queryset.filter(fromMember__user=user) def list_common( @@ -114,17 +114,16 @@ def list_common( def incoming_list( iterator: t.Optional[str] = None, limit: int = 50, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - return list_common(get_incoming_queryset(user), iterator, limit) + return list_common(queryset, iterator, limit) @invitation_incoming_router.get("/{invitation_uid}/", response_model=CollectionInvitationOut) def incoming_get( invitation_uid: str, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - queryset = get_incoming_queryset(user) obj = get_object_or_404(queryset, uid=invitation_uid) ret = CollectionInvitationOut.from_orm(obj) return MsgpackResponse(ret) @@ -133,9 +132,8 @@ def incoming_get( @invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) def incoming_delete( invitation_uid: str, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - queryset = get_incoming_queryset(user) obj = get_object_or_404(queryset, uid=invitation_uid) obj.delete() @@ -144,9 +142,8 @@ def incoming_delete( def incoming_accept( invitation_uid: str, data: CollectionInvitationAcceptIn, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - queryset = get_incoming_queryset(user) invitation = get_object_or_404(queryset, uid=invitation_uid) with transaction.atomic(): @@ -201,17 +198,16 @@ def outgoing_create( def outgoing_list( iterator: t.Optional[str] = None, limit: int = 50, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_outgoing_queryset), ): - return list_common(get_outgoing_queryset(user), iterator, limit) + return list_common(queryset, iterator, limit) @invitation_outgoing_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) def outgoing_delete( invitation_uid: str, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_outgoing_queryset), ): - queryset = get_outgoing_queryset(user) obj = get_object_or_404(queryset, uid=invitation_uid) obj.delete() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 534cad1..a491490 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -12,15 +12,18 @@ from .msgpack import MsgpackResponse from .utils import get_object_or_404 from .stoken_handler import filter_by_stoken_and_limit -from .collection import collection_router, get_collection_queryset +from .collection import collection_router, get_collection User = get_user_model() default_queryset: QuerySet = models.CollectionMember.objects.all() -def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) - return collection, queryset.filter(collection=collection) +def get_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + return default_queryset.filter(collection=collection) + + +def get_member(username: str, queryset: QuerySet = Depends(get_queryset)) -> QuerySet: + return get_object_or_404(queryset, user__username__iexact=username) class CollectionMemberModifyAccessLevelIn(BaseModel): @@ -47,12 +50,10 @@ class MemberListResponse(BaseModel): @collection_router.get("/{collection_uid}/member/", response_model=MemberListResponse) def member_list( - collection_uid: str, iterator: t.Optional[str] = None, limit: int = 50, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_queryset), ): - _, queryset = get_queryset(user, collection_uid) queryset = queryset.order_by("id") result, new_stoken_obj, done = filter_by_stoken_and_limit( iterator, limit, queryset, models.CollectionMember.stoken_annotation @@ -69,25 +70,16 @@ def member_list( @collection_router.delete("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) def member_delete( - collection_uid: str, - username: str, - user: User = Depends(get_authenticated_user), + obj: models.CollectionMember = Depends(get_member), ): - _, queryset = get_queryset(user, collection_uid) - obj = get_object_or_404(queryset, user__username__iexact=username) obj.revoke() @collection_router.patch("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) def member_patch( - collection_uid: str, - username: str, data: CollectionMemberModifyAccessLevelIn, - user: User = Depends(get_authenticated_user), + instance: models.CollectionMember = Depends(get_member), ): - _, queryset = get_queryset(user, collection_uid) - instance = get_object_or_404(queryset, user__username__iexact=username) - with transaction.atomic(): # We only allow updating accessLevel if instance.accessLevel != data.accessLevel: @@ -97,10 +89,6 @@ def member_patch( @collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) -def member_leave( - collection_uid: str, - user: User = Depends(get_authenticated_user), -): - collection, _ = get_queryset(user, collection_uid) +def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): obj = get_object_or_404(collection.members, user=user) obj.revoke() From c7b8b0373a171e114213fa0f5337ac571ea19e3c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 22:56:23 +0200 Subject: [PATCH 081/170] Add permissions. --- etebase_fastapi/collection.py | 28 ++++++++++++++++++++++++---- etebase_fastapi/member.py | 23 ++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 196bb1d..13c53be 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -12,10 +12,10 @@ from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user -from .exceptions import ValidationError, transform_validation_error +from .exceptions import ValidationError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -209,6 +209,26 @@ def get_item_queryset(collection: models.Collection = Depends(get_collection)) - return queryset +# permissions + + +def verify_collection_admin( + collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) +): + if not is_collection_admin(collection, user): + raise PermissionDenied("admin_access_required", "Only collection admins can perform this operation.") + + +def has_write_access( + collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) +): + member = collection.members.get(user=user) + if member.accessLevel == models.AccessLevels.READ_ONLY: + raise PermissionDenied("no_write_access", "You need write access to write to this collection") + + +# paths + @collection_router.post("/list_multi/") async def list_multi( data: ListMulti, @@ -489,14 +509,14 @@ def fetch_updates( return MsgpackResponse(ret) -@collection_router.post("/{collection_uid}/item/transaction/") +@collection_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@collection_router.post("/{collection_uid}/item/batch/") +@collection_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index a491490..f3c77e5 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -12,7 +12,7 @@ from .msgpack import MsgpackResponse from .utils import get_object_or_404 from .stoken_handler import filter_by_stoken_and_limit -from .collection import collection_router, get_collection +from .collection import collection_router, get_collection, verify_collection_admin User = get_user_model() default_queryset: QuerySet = models.CollectionMember.objects.all() @@ -48,7 +48,9 @@ class MemberListResponse(BaseModel): done: bool -@collection_router.get("/{collection_uid}/member/", response_model=MemberListResponse) +@collection_router.get( + "/{collection_uid}/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)] +) def member_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -68,14 +70,22 @@ def member_list( return MsgpackResponse(ret) -@collection_router.delete("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +@collection_router.delete( + "/{collection_uid}/member/{username}/", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(verify_collection_admin)], +) def member_delete( obj: models.CollectionMember = Depends(get_member), ): obj.revoke() -@collection_router.patch("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +@collection_router.patch( + "/{collection_uid}/member/{username}/", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(verify_collection_admin)], +) def member_patch( data: CollectionMemberModifyAccessLevelIn, instance: models.CollectionMember = Depends(get_member), @@ -88,7 +98,10 @@ def member_patch( instance.save() -@collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) +@collection_router.post( + "/{collection_uid}/member/leave/", + status_code=status.HTTP_204_NO_CONTENT, +) def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): obj = get_object_or_404(collection.members, user=user) obj.revoke() From 1c8684ee9280856d20d48ca742018dc34ac79995 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 23:06:25 +0200 Subject: [PATCH 082/170] Fix a FIXME. --- etebase_fastapi/test_reset_view.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index 435a56e..73a803e 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -4,7 +4,7 @@ from django.db import transaction from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, Response, status -from django_etebase.utils import get_user_queryset +from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute @@ -19,8 +19,7 @@ def reset(data: SignupIn, request: Request): return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) with transaction.atomic(): - # XXX-TOM - user_queryset = get_user_queryset(User.objects.all(), None) + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) user = get_object_or_404(user_queryset, username=data.user.username) # Only allow test users for extra safety if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): From d63c34693f4a88717353f7e708f8fed58bfe1b48 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 23:11:12 +0200 Subject: [PATCH 083/170] Change all item_uids to be called item_uids. --- etebase_fastapi/collection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 13c53be..33f1d64 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -378,13 +378,13 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@collection_router.get("/{collection_uid}/item/{uid}/") +@collection_router.get("/{collection_uid}/item/{item_uid}/") def item_get( - uid: str, + item_uid: str, queryset: QuerySet = Depends(get_item_queryset), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - obj = queryset.get(uid=uid) + obj = queryset.get(uid=item_uid) ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -439,16 +439,16 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@collection_router.get("/{collection_uid}/item/{uid}/revision/") +@collection_router.get("/{collection_uid}/item/{item_uid}/revision/") def item_revisions( - uid: str, + item_uid: str, limit: int = 50, iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), items: QuerySet = Depends(get_item_queryset), ): - item = get_object_or_404(items, uid=uid) + item = get_object_or_404(items, uid=item_uid) queryset = item.revisions.order_by("-id") From 15988235f27673398416b244fd40aa39009a89f1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 08:42:48 +0200 Subject: [PATCH 084/170] Exclude unset fields so fix removedMemberships return value. --- etebase_fastapi/collection.py | 6 +++--- etebase_fastapi/msgpack.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 33f1d64..1c5ca14 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -176,8 +176,9 @@ def collection_list_common( context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) + stoken_obj = get_stoken_obj(stoken) - removedMemberships = None if stoken_obj is not None: # FIXME: honour limit? (the limit should be combined for data and this because of stoken) remed_qs = models.CollectionMemberRemoved.objects.filter(user=user, stoken__id__gt=stoken_obj.id) @@ -188,9 +189,8 @@ def collection_list_common( remed = remed_qs.values_list("collection__uid", flat=True) if len(remed) > 0: - removedMemberships = [{"uid": x} for x in remed] + ret.removedMemberships = [{"uid": x} for x in remed] - ret = CollectionListResponse(data=data, stoken=new_stoken, done=done, removedMemberships=removedMemberships) return MsgpackResponse(content=ret) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index edffd7e..0c5cc30 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -24,7 +24,7 @@ class MsgpackResponse(Response): return b"" if isinstance(content, BaseModel): - content = content.dict() + content = content.dict(exclude_unset=True) return msgpack.packb(content, use_bin_type=True) From 9f26ecf27682b3a0cbea85c10f23bfccdcdacc73 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:04:45 +0200 Subject: [PATCH 085/170] API: add documentation tags. --- etebase_fastapi/app.py | 6 ++++-- etebase_fastapi/authentication.py | 2 +- etebase_fastapi/collection.py | 17 +++++++++-------- etebase_fastapi/invitation.py | 4 ++-- etebase_fastapi/member.py | 15 ++++++++------- etebase_fastapi/test_reset_view.py | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 755340c..ffea2fb 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -13,8 +13,8 @@ from fastapi import FastAPI, Request from .exceptions import CustomHttpException from .authentication import authentication_router -from .collection import collection_router -from . import member # noqa +from .collection import collection_router, item_router +from .member import member_router from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse @@ -23,6 +23,8 @@ VERSION = "v1" BASE_PATH = f"/api/{VERSION}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") +app.include_router(item_router, prefix=f"{BASE_PATH}/collection") +app.include_router(member_router, prefix=f"{BASE_PATH}/collection") app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") if settings.DEBUG: diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 13af2dd..f04753b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -33,7 +33,7 @@ User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") AUTO_REFRESH = True MIN_REFRESH_INTERVAL = 60 -authentication_router = APIRouter(route_class=MsgpackRoute) +authentication_router = APIRouter(route_class=MsgpackRoute, tags=["authentication"]) @dataclasses.dataclass(frozen=True) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 1c5ca14..ead113a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,7 +7,7 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, status from pydantic import BaseModel from django_etebase import models @@ -18,7 +18,8 @@ from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_st from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin User = get_user_model() -collection_router = APIRouter(route_class=MsgpackRoute) +collection_router = APIRouter(route_class=MsgpackRoute, tags=["collection"]) +item_router = APIRouter(route_class=MsgpackRoute, tags=["item"]) default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() @@ -378,7 +379,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@collection_router.get("/{collection_uid}/item/{item_uid}/") +@item_router.get("/{collection_uid}/item/{item_uid}/") def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), @@ -407,7 +408,7 @@ def item_list_common( return MsgpackResponse(content=ret) -@collection_router.get("/{collection_uid}/item/") +@item_router.get("/{collection_uid}/item/") async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -439,7 +440,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@collection_router.get("/{collection_uid}/item/{item_uid}/revision/") +@item_router.get("/{collection_uid}/item/{item_uid}/revision/") def item_revisions( item_uid: str, limit: int = 50, @@ -475,7 +476,7 @@ def item_revisions( return MsgpackResponse(ret) -@collection_router.post("/{collection_uid}/item/fetch_updates/") +@item_router.post("/{collection_uid}/item/fetch_updates/") def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -509,14 +510,14 @@ def fetch_updates( return MsgpackResponse(ret) -@collection_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) +@item_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@collection_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) +@item_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index cbf0554..9b166ee 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -14,8 +14,8 @@ from .msgpack import MsgpackRoute, MsgpackResponse from .utils import get_object_or_404, Context, is_collection_admin User = get_user_model() -invitation_incoming_router = APIRouter(route_class=MsgpackRoute) -invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) +invitation_incoming_router = APIRouter(route_class=MsgpackRoute, tags=["incoming invitation"]) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, tags=["outgoing invitation"]) default_queryset: QuerySet = models.CollectionInvitation.objects.all() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index f3c77e5..af349d1 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -3,18 +3,19 @@ import typing as t from django.contrib.auth import get_user_model from django.db import transaction from django.db.models import QuerySet -from fastapi import Depends, status +from fastapi import APIRouter, Depends, status from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user -from .msgpack import MsgpackResponse +from .msgpack import MsgpackRoute, MsgpackResponse from .utils import get_object_or_404 from .stoken_handler import filter_by_stoken_and_limit -from .collection import collection_router, get_collection, verify_collection_admin +from .collection import get_collection, verify_collection_admin User = get_user_model() +member_router = APIRouter(route_class=MsgpackRoute, tags=["member"]) default_queryset: QuerySet = models.CollectionMember.objects.all() @@ -48,7 +49,7 @@ class MemberListResponse(BaseModel): done: bool -@collection_router.get( +@member_router.get( "/{collection_uid}/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)] ) def member_list( @@ -70,7 +71,7 @@ def member_list( return MsgpackResponse(ret) -@collection_router.delete( +@member_router.delete( "/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], @@ -81,7 +82,7 @@ def member_delete( obj.revoke() -@collection_router.patch( +@member_router.patch( "/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], @@ -98,7 +99,7 @@ def member_patch( instance.save() -@collection_router.post( +@member_router.post( "/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT, ) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index 73a803e..f21fd84 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -8,7 +8,7 @@ from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute -test_reset_view_router = APIRouter(route_class=MsgpackRoute) +test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) User = get_user_model() From ee4e7cf498657ee78d3dbac6f8078085b2abc64d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:11:35 +0200 Subject: [PATCH 086/170] Unify the nested router prefix. --- etebase_fastapi/app.py | 5 +++-- etebase_fastapi/collection.py | 12 ++++++------ etebase_fastapi/member.py | 10 ++++------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index ffea2fb..a8b12d0 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -21,10 +21,11 @@ from .msgpack import MsgpackResponse app = FastAPI() VERSION = "v1" BASE_PATH = f"/api/{VERSION}" +COLLECTION_UID_MARKER = "{collection_uid}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") -app.include_router(item_router, prefix=f"{BASE_PATH}/collection") -app.include_router(member_router, prefix=f"{BASE_PATH}/collection") +app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") +app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") if settings.DEBUG: diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ead113a..c0efed1 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -379,7 +379,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@item_router.get("/{collection_uid}/item/{item_uid}/") +@item_router.get("/item/{item_uid}/") def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), @@ -408,7 +408,7 @@ def item_list_common( return MsgpackResponse(content=ret) -@item_router.get("/{collection_uid}/item/") +@item_router.get("/item/") async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -440,7 +440,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@item_router.get("/{collection_uid}/item/{item_uid}/revision/") +@item_router.get("/item/{item_uid}/revision/") def item_revisions( item_uid: str, limit: int = 50, @@ -476,7 +476,7 @@ def item_revisions( return MsgpackResponse(ret) -@item_router.post("/{collection_uid}/item/fetch_updates/") +@item_router.post("/item/fetch_updates/") def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -510,14 +510,14 @@ def fetch_updates( return MsgpackResponse(ret) -@item_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access)]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@item_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/batch/", dependencies=[Depends(has_write_access)]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index af349d1..2c9b631 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -49,9 +49,7 @@ class MemberListResponse(BaseModel): done: bool -@member_router.get( - "/{collection_uid}/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)] -) +@member_router.get("/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)]) def member_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -72,7 +70,7 @@ def member_list( @member_router.delete( - "/{collection_uid}/member/{username}/", + "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], ) @@ -83,7 +81,7 @@ def member_delete( @member_router.patch( - "/{collection_uid}/member/{username}/", + "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], ) @@ -100,7 +98,7 @@ def member_patch( @member_router.post( - "/{collection_uid}/member/leave/", + "/member/leave/", status_code=status.HTTP_204_NO_CONTENT, ) def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): From 80d69a566325755107d810c6fb6c67147f32c9cc Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:25:28 +0200 Subject: [PATCH 087/170] Fix collection list and how we return API responses. --- etebase_fastapi/authentication.py | 28 +++++++++---------- etebase_fastapi/collection.py | 45 +++++++++++++------------------ etebase_fastapi/invitation.py | 18 +++++-------- etebase_fastapi/member.py | 8 +++--- etebase_fastapi/msgpack.py | 2 +- etebase_fastapi/utils.py | 8 ++++++ 6 files changed, 49 insertions(+), 60 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index f04753b..13a8884 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -16,7 +16,6 @@ from django.db import transaction from django.utils import timezone from fastapi import APIRouter, Depends, status, Request, Response from fastapi.security import APIKeyHeader -from pydantic import BaseModel from django_etebase import app_settings, models from django_etebase.exceptions import EtebaseValidationError @@ -27,7 +26,8 @@ from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError -from .msgpack import MsgpackResponse, MsgpackRoute +from .msgpack import MsgpackRoute +from .utils import BaseModel User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -225,10 +225,10 @@ def validate_login_request( @authentication_router.get("/is_etebase/") async def is_etebase(): - return MsgpackResponse({}) + pass -@authentication_router.post("/login_challenge/") +@authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) async def login_challenge(user: User = Depends(get_login_user)): enc_key = get_encryption_key(user.userinfo.salt) box = nacl.secret.SecretBox(enc_key) @@ -237,35 +237,31 @@ async def login_challenge(user: User = Depends(get_login_user)): "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return MsgpackResponse( - LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) - ) + return LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) -@authentication_router.post("/login/") +@authentication_router.post("/login/", response_model=LoginOut) async def login(data: Login, request: Request): user = await get_login_user(LoginChallengeIn(username=data.response_data.username)) host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "login", host) data = await sync_to_async(LoginOut.from_orm)(user) await sync_to_async(user_logged_in.send)(sender=user.__class__, request=None, user=user) - return MsgpackResponse(content=data, status_code=status.HTTP_200_OK) + return data -@authentication_router.post("/logout/") +@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT) async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): await sync_to_async(auth_data.token.delete)() # XXX-TOM await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) - return Response(status_code=status.HTTP_204_NO_CONTENT) -@authentication_router.post("/change_password/") +@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT) async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "changePassword", host) await sync_to_async(save_changed_password)(data, user) - return Response(status_code=status.HTTP_204_NO_CONTENT) @authentication_router.post("/dashboard_url/") @@ -278,7 +274,7 @@ def dashboard_url(user: User = Depends(get_authenticated_user)): ret = { "url": get_dashboard_url(request, *args, **kwargs), } - return MsgpackResponse(ret) + return ret def signup_save(data: SignupIn, request: Request) -> User: @@ -311,10 +307,10 @@ def signup_save(data: SignupIn, request: Request) -> User: return instance -@authentication_router.post("/signup/") +@authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) async def signup(data: SignupIn, request: Request): user = await sync_to_async(signup_save)(data, request) # XXX-TOM data = await sync_to_async(LoginOut.from_orm)(user) await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) - return MsgpackResponse(content=data, status_code=status.HTTP_201_CREATED) + return data diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index c0efed1..993d144 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -8,14 +8,13 @@ from django.db import transaction from django.db.models import Q from django.db.models import QuerySet from fastapi import APIRouter, Depends, status -from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error, PermissionDenied -from .msgpack import MsgpackRoute, MsgpackResponse +from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, tags=["collection"]) @@ -169,7 +168,7 @@ def collection_list_common( stoken: t.Optional[str], limit: int, prefetch: Prefetch, -) -> MsgpackResponse: +) -> CollectionListResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit( stoken, limit, queryset, models.Collection.stoken_annotation ) @@ -192,7 +191,7 @@ def collection_list_common( if len(remed) > 0: ret.removedMemberships = [{"uid": x} for x in remed] - return MsgpackResponse(content=ret) + return ret def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: @@ -230,7 +229,7 @@ def has_write_access( # paths -@collection_router.post("/list_multi/") +@collection_router.post("/list_multi/", response_model=CollectionListResponse, response_model_exclude_unset=True) async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, @@ -247,7 +246,7 @@ async def list_multi( return await collection_list_common(queryset, user, stoken, limit, prefetch) -@collection_router.post("/list/") +@collection_router.get("/", response_model=CollectionListResponse) async def collection_list( stoken: t.Optional[str] = None, limit: int = 50, @@ -323,20 +322,18 @@ def _create(data: CollectionIn, user: User): ).save() -@collection_router.post("/") +@collection_router.post("/", status_code=status.HTTP_201_CREATED) async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) - return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) -@collection_router.get("/{collection_uid}/") +@collection_router.get("/{collection_uid}/", response_model=CollectionOut) def collection_get( obj: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery ): - ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) - return MsgpackResponse(ret) + return CollectionOut.from_orm_context(obj, Context(user, prefetch)) def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): @@ -379,15 +376,14 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@item_router.get("/item/{item_uid}/") +@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut) def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) - ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) - return MsgpackResponse(ret) + return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) @sync_to_async @@ -397,18 +393,17 @@ def item_list_common( stoken: t.Optional[str], limit: int, prefetch: Prefetch, -) -> MsgpackResponse: +) -> CollectionItemListResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit( stoken, limit, queryset, models.CollectionItem.stoken_annotation ) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionItemOut] = [CollectionItemOut.from_orm_context(item, context) for item in result] - ret = CollectionItemListResponse(data=data, stoken=new_stoken, done=done) - return MsgpackResponse(content=ret) + return CollectionItemListResponse(data=data, stoken=new_stoken, done=done) -@item_router.get("/item/") +@item_router.get("/item/", response_model=CollectionItemListResponse) async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -437,10 +432,10 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid for item in data.items: item_create(item, collection_object, validate_etag) - return MsgpackResponse({}) + return None -@item_router.get("/item/{item_uid}/revision/") +@item_router.get("/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse) def item_revisions( item_uid: str, limit: int = 50, @@ -468,15 +463,14 @@ def item_revisions( ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] iterator = ret_data[-1].uid if len(result) > 0 else None - ret = CollectionItemRevisionListResponse( + return CollectionItemRevisionListResponse( data=ret_data, iterator=iterator, done=done, ) - return MsgpackResponse(ret) -@item_router.post("/item/fetch_updates/") +@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse) def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -502,12 +496,11 @@ def fetch_updates( new_stoken = new_stoken or stoken context = Context(user, prefetch) - ret = CollectionItemListResponse( + return CollectionItemListResponse( data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], stoken=new_stoken, done=True, # we always return all the items, so it's always done ) - return MsgpackResponse(ret) @item_router.post("/item/transaction/", dependencies=[Depends(has_write_access)]) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 9b166ee..5c2c338 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -4,14 +4,13 @@ from django.contrib.auth import get_user_model from django.db import transaction, IntegrityError from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Request -from pydantic import BaseModel from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user from .exceptions import ValidationError, PermissionDenied -from .msgpack import MsgpackRoute, MsgpackResponse -from .utils import get_object_or_404, Context, is_collection_admin +from .msgpack import MsgpackRoute +from .utils import get_object_or_404, Context, is_collection_admin, BaseModel User = get_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, tags=["incoming invitation"]) @@ -85,7 +84,7 @@ def list_common( queryset: QuerySet, iterator: t.Optional[str], limit: int, -) -> MsgpackResponse: +) -> InvitationListResponse: queryset = queryset.order_by("id") if iterator is not None: @@ -102,12 +101,11 @@ def list_common( ret_data = result iterator = ret_data[-1].uid if len(result) > 0 else None - ret = InvitationListResponse( + return InvitationListResponse( data=ret_data, iterator=iterator, done=done, ) - return MsgpackResponse(ret) @invitation_incoming_router.get("/", response_model=InvitationListResponse) @@ -125,8 +123,7 @@ def incoming_get( queryset: QuerySet = Depends(get_incoming_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) - ret = CollectionInvitationOut.from_orm(obj) - return MsgpackResponse(ret) + return CollectionInvitationOut.from_orm(obj) @invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) @@ -191,8 +188,6 @@ def outgoing_create( except IntegrityError: raise ValidationError("invitation_exists", "Invitation already exists") - return MsgpackResponse(CollectionInvitationOut.from_orm(ret), status_code=status.HTTP_201_CREATED) - @invitation_outgoing_router.get("/", response_model=InvitationListResponse) def outgoing_list( @@ -221,5 +216,4 @@ def outgoing_fetch_user_profile( kwargs = {User.USERNAME_FIELD: username.lower()} user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) - ret = UserInfoOut.from_orm(user_info) - return MsgpackResponse(ret) + return UserInfoOut.from_orm(user_info) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 2c9b631..749092c 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -4,12 +4,11 @@ from django.contrib.auth import get_user_model from django.db import transaction from django.db.models import QuerySet from fastapi import APIRouter, Depends, status -from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user -from .msgpack import MsgpackRoute, MsgpackResponse -from .utils import get_object_or_404 +from .msgpack import MsgpackRoute +from .utils import get_object_or_404, BaseModel from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin @@ -61,12 +60,11 @@ def member_list( ) new_stoken = new_stoken_obj and new_stoken_obj.uid - ret = MemberListResponse( + return MemberListResponse( data=[CollectionMemberOut.from_orm(item) for item in result], iterator=new_stoken, done=done, ) - return MsgpackResponse(ret) @member_router.delete( diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 0c5cc30..edffd7e 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -24,7 +24,7 @@ class MsgpackResponse(Response): return b"" if isinstance(content, BaseModel): - content = content.dict(exclude_unset=True) + content = content.dict() return msgpack.packb(content, use_bin_type=True) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 150afe8..7168f87 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -2,6 +2,7 @@ import dataclasses import typing as t from fastapi import status, Query +from pydantic import BaseModel as PyBaseModel from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist @@ -17,6 +18,13 @@ Prefetch = t.Literal["auto", "medium"] PrefetchQuery = Query(default="auto") +class BaseModel(PyBaseModel): + class Config: + json_encoders = { + bytes: lambda x: x, + } + + @dataclasses.dataclass class Context: user: t.Optional[User] From 6517fc5db2dac325028bdb94f937333e9fd42b25 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:35:27 +0200 Subject: [PATCH 088/170] More route tags to a central place. --- etebase_fastapi/app.py | 12 ++++++------ etebase_fastapi/authentication.py | 2 +- etebase_fastapi/collection.py | 4 ++-- etebase_fastapi/invitation.py | 4 ++-- etebase_fastapi/member.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index a8b12d0..2bbfc2a 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -22,12 +22,12 @@ app = FastAPI() VERSION = "v1" BASE_PATH = f"/api/{VERSION}" COLLECTION_UID_MARKER = "{collection_uid}" -app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") -app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") -app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") -app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") -app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") -app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") +app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) +app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) +app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) +app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) +app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) +app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) if settings.DEBUG: from .test_reset_view import test_reset_view_router diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 13a8884..742f101 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -33,7 +33,7 @@ User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") AUTO_REFRESH = True MIN_REFRESH_INTERVAL = 60 -authentication_router = APIRouter(route_class=MsgpackRoute, tags=["authentication"]) +authentication_router = APIRouter(route_class=MsgpackRoute) @dataclasses.dataclass(frozen=True) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 993d144..8b69708 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -17,8 +17,8 @@ from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_st from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel User = get_user_model() -collection_router = APIRouter(route_class=MsgpackRoute, tags=["collection"]) -item_router = APIRouter(route_class=MsgpackRoute, tags=["item"]) +collection_router = APIRouter(route_class=MsgpackRoute) +item_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 5c2c338..ab0bf01 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -13,8 +13,8 @@ from .msgpack import MsgpackRoute from .utils import get_object_or_404, Context, is_collection_admin, BaseModel User = get_user_model() -invitation_incoming_router = APIRouter(route_class=MsgpackRoute, tags=["incoming invitation"]) -invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, tags=["outgoing invitation"]) +invitation_incoming_router = APIRouter(route_class=MsgpackRoute) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = models.CollectionInvitation.objects.all() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 749092c..26cfcff 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -14,7 +14,7 @@ from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin User = get_user_model() -member_router = APIRouter(route_class=MsgpackRoute, tags=["member"]) +member_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = models.CollectionMember.objects.all() From 34c548acda625c4dbfad57e4d6cd780d4d500250 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:47:37 +0200 Subject: [PATCH 089/170] Remove extra import. --- etebase_fastapi/exceptions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index fa76c45..b7bb0e9 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -59,9 +59,6 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) -from django_etebase.exceptions import EtebaseValidationError - - class ValidationError(CustomHttpException): def __init__( self, From a75d5479faef37c8d046ae0d5ebebb662d9d3645 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:51:27 +0200 Subject: [PATCH 090/170] Rename ValidationError to HttpError. --- etebase_fastapi/authentication.py | 16 ++++++++-------- etebase_fastapi/collection.py | 16 ++++++++-------- etebase_fastapi/exceptions.py | 21 +++++++++------------ etebase_fastapi/invitation.py | 6 +++--- etebase_fastapi/utils.py | 4 ++-- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 742f101..5650652 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -25,7 +25,7 @@ from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode -from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError +from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute from .utils import BaseModel @@ -207,20 +207,20 @@ def validate_login_request( challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) now = int(datetime.now().timestamp()) if validated_data.action != expected_action: - raise ValidationError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') + raise HttpError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - raise ValidationError("challenge_expired", "Login challenge has expired") + raise HttpError("challenge_expired", "Login challenge has expired") elif challenge_data["userId"] != user.id: - raise ValidationError("wrong_user", "This challenge is for the wrong user") + raise HttpError("wrong_user", "This challenge is for the wrong user") elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: - raise ValidationError( + raise HttpError( "wrong_host", f'Found wrong host name. Got: "{validated_data.host}" expected: "{host_from_request}"' ) verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) try: verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) except nacl.exceptions.BadSignatureError: - raise ValidationError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) + raise HttpError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) @authentication_router.get("/is_etebase/") @@ -269,7 +269,7 @@ def dashboard_url(user: User = Depends(get_authenticated_user)): # XXX-TOM get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: - raise ValidationError("not_supported", "This server doesn't have a user dashboard.") + raise HttpError("not_supported", "This server doesn't have a user dashboard.") ret = { "url": get_dashboard_url(request, *args, **kwargs), @@ -301,7 +301,7 @@ def signup_save(data: SignupIn, request: Request) -> User: raise EtebaseValidationError("generic", str(e)) if hasattr(instance, "userinfo"): - raise ValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + raise HttpError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) models.UserInfo.objects.create(**data.dict(exclude={"user"}), owner=instance) return instance diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 8b69708..7f01682 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user -from .exceptions import ValidationError, transform_validation_error, PermissionDenied +from .exceptions import HttpError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel @@ -144,7 +144,7 @@ class ItemDepIn(BaseModel): item = models.CollectionItem.objects.get(uid=self.uid) etag = self.etag if item.etag != etag: - raise ValidationError( + raise HttpError( "wrong_etag", "Wrong etag. Expected {} got {}".format(item.etag, etag), status_code=status.HTTP_409_CONFLICT, @@ -274,7 +274,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) chunk_obj.save() else: - raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") + raise HttpError("chunk_no_content", "Tried to create a new chunk without content") chunks_objs.append(chunk_obj) @@ -290,12 +290,12 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle def _create(data: CollectionIn, user: User): with transaction.atomic(): if data.item.etag is not None: - raise ValidationError("bad_etag", "etag is not null") + raise HttpError("bad_etag", "etag is not null") instance = models.Collection(uid=data.item.uid, owner=user) try: instance.validate_unique() except django_exceptions.ValidationError: - raise ValidationError( + raise HttpError( "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT ) instance.save() @@ -355,7 +355,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance if validate_etag and cur_etag != etag: - raise ValidationError( + raise HttpError( "wrong_etag", "Wrong etag. Expected {} got {}".format(cur_etag, etag), status_code=status.HTTP_409_CONFLICT, @@ -425,7 +425,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid collection_object = queryset.select_for_update().get(uid=uid) if stoken is not None and stoken != collection_object.stoken: - raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + raise HttpError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) # XXX-TOM: make sure we return compatible errors data.validate_db() @@ -482,7 +482,7 @@ def fetch_updates( item_limit = 200 if len(data) > item_limit: - raise ValidationError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) queryset, stoken_rev = filter_by_stoken(stoken, queryset, models.CollectionItem.stoken_annotation) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index b7bb0e9..2c1757c 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -3,19 +3,17 @@ import typing as t from pydantic import BaseModel -from django_etebase.exceptions import EtebaseValidationError - -class ValidationErrorField(BaseModel): +class HttpErrorField(BaseModel): field: str code: str detail: str -class ValidationErrorOut(BaseModel): +class HttpErrorOut(BaseModel): code: str detail: str - errors: t.Optional[t.List[ValidationErrorField]] + errors: t.Optional[t.List[HttpErrorField]] class CustomHttpException(Exception): @@ -59,24 +57,23 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) -class ValidationError(CustomHttpException): +class HttpError(CustomHttpException): def __init__( self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST, - field: t.Optional[str] = None, - errors: t.Optional[t.List["ValidationError"]] = None, + errors: t.Optional[t.List["HttpError"]] = None, ): self.errors = errors super().__init__(code=code, detail=detail, status_code=status_code) @property def as_dict(self) -> dict: - return ValidationErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() + return HttpErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() -def flatten_errors(field_name, errors) -> t.List[ValidationError]: +def flatten_errors(field_name, errors) -> t.List[HttpError]: ret = [] if isinstance(errors, dict): for error_key in errors: @@ -98,5 +95,5 @@ def transform_validation_error(prefix, err): elif not hasattr(err, "message"): errors = flatten_errors(prefix, err.error_list) else: - raise EtebaseValidationError(err.code, err.message) - raise ValidationError(code="field_errors", detail="Field validations failed.", errors=errors) + raise HttpError(err.code, err.message) + raise HttpError(code="field_errors", detail="Field validations failed.", errors=errors) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index ab0bf01..38b74d8 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, status, Request from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user -from .exceptions import ValidationError, PermissionDenied +from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute from .utils import get_object_or_404, Context, is_collection_admin, BaseModel @@ -42,7 +42,7 @@ class CollectionInvitationCommon(BaseModel): class CollectionInvitationIn(CollectionInvitationCommon): def validate_db(self, context: Context): if context.user.username == self.username.lower(): - raise ValidationError("no_self_invite", "Inviting yourself is not allowed") + raise HttpError("no_self_invite", "Inviting yourself is not allowed") class CollectionInvitationOut(CollectionInvitationCommon): @@ -186,7 +186,7 @@ def outgoing_create( **data.dict(exclude={"collection", "username"}), user=to_user, fromMember=member ) except IntegrityError: - raise ValidationError("invitation_exists", "Invitation already exists") + raise HttpError("invitation_exists", "Invitation already exists") @invitation_outgoing_router.get("/", response_model=InvitationListResponse) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 7168f87..6ea9513 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -10,7 +10,7 @@ from django.contrib.auth import get_user_model from django_etebase.models import AccessLevels -from .exceptions import ValidationError +from .exceptions import HttpError User = get_user_model() @@ -35,7 +35,7 @@ def get_object_or_404(queryset: QuerySet, **kwargs): try: return queryset.get(**kwargs) except ObjectDoesNotExist as e: - raise ValidationError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) + raise HttpError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) def is_collection_admin(collection, user): From 4b4be14d32330a6df16b51eb2c93d484402d157a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:00:35 +0200 Subject: [PATCH 091/170] Add more responses to the API. --- etebase_fastapi/authentication.py | 10 +++++----- etebase_fastapi/collection.py | 6 +++--- etebase_fastapi/invitation.py | 6 +++--- etebase_fastapi/member.py | 4 ++-- etebase_fastapi/utils.py | 6 +++++- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 5650652..902b79b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -14,7 +14,7 @@ from django.contrib.auth import get_user_model, user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction from django.utils import timezone -from fastapi import APIRouter, Depends, status, Request, Response +from fastapi import APIRouter, Depends, status, Request from fastapi.security import APIKeyHeader from django_etebase import app_settings, models @@ -27,7 +27,7 @@ from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute -from .utils import BaseModel +from .utils import BaseModel, permission_responses User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -250,21 +250,21 @@ async def login(data: Login, request: Request): return data -@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT) +@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): await sync_to_async(auth_data.token.delete)() # XXX-TOM await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) -@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT) +@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "changePassword", host) await sync_to_async(save_changed_password)(data, user) -@authentication_router.post("/dashboard_url/") +@authentication_router.post("/dashboard_url/", responses=permission_responses) def dashboard_url(user: User = Depends(get_authenticated_user)): # XXX-TOM get_dashboard_url = app_settings.DASHBOARD_URL_FUNC diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 7f01682..fad49aa 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,11 +14,11 @@ from .authentication import get_authenticated_user from .exceptions import HttpError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel, permission_responses User = get_user_model() -collection_router = APIRouter(route_class=MsgpackRoute) -item_router = APIRouter(route_class=MsgpackRoute) +collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 38b74d8..1d8df94 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -10,11 +10,11 @@ from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute -from .utils import get_object_or_404, Context, is_collection_admin, BaseModel +from .utils import get_object_or_404, Context, is_collection_admin, BaseModel, permission_responses User = get_user_model() -invitation_incoming_router = APIRouter(route_class=MsgpackRoute) -invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) +invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionInvitation.objects.all() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 26cfcff..8ffed9d 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -8,13 +8,13 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user from .msgpack import MsgpackRoute -from .utils import get_object_or_404, BaseModel +from .utils import get_object_or_404, BaseModel, permission_responses from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin User = get_user_model() -member_router = APIRouter(route_class=MsgpackRoute) +member_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionMember.objects.all() diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 6ea9513..487f03a 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -10,7 +10,7 @@ from django.contrib.auth import get_user_model from django_etebase.models import AccessLevels -from .exceptions import HttpError +from .exceptions import HttpError, HttpErrorOut User = get_user_model() @@ -41,3 +41,7 @@ def get_object_or_404(queryset: QuerySet, **kwargs): def is_collection_admin(collection, user): member = collection.members.filter(user=user).first() return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + + +response_model_dict = {"model": HttpErrorOut} +permission_responses = {403: response_model_dict} From b39f7951e292588092365964f7c2fc3f317c54d3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:18:35 +0200 Subject: [PATCH 092/170] chunk first-type. --- etebase_fastapi/collection.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index fad49aa..20d79e0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -515,3 +515,31 @@ def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) + + +# Chunks + + +@item_router.put("/item/{item_uid}/chunk/{chunk_uid}/", dependencies=[Depends(has_write_access)], status_code=status.HTTP_201_CREATED) +def chunk_update( + limit: int = 50, + iterator: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), + collection: models.Collection = Depends(get_collection), +): + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) + + data = { + "uid": chunk_uid, + "chunkFile": request.data["file"], + } + + serializer = self.get_serializer_class()(data=data) + serializer.is_valid(raise_exception=True) + try: + serializer.save(collection=col) + except IntegrityError: + return Response( + {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT + ) From 959dc9b576ddeeb69de6f367fd381d217a1bccb7 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Mon, 28 Dec 2020 10:27:49 +0200 Subject: [PATCH 093/170] minor fix --- etebase_fastapi/collection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 20d79e0..6757ac3 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -70,7 +70,6 @@ class CollectionItemOut(CollectionItemCommon): uid=obj.uid, version=obj.version, encryptionKey=obj.encryptionKey, - etag=obj.etag, content=CollectionItemRevisionInOut.from_orm_context(obj.content, context), ) @@ -91,7 +90,7 @@ class CollectionOut(CollectionCommon): @classmethod def from_orm_context(cls: t.Type["CollectionOut"], obj: models.Collection, context: Context) -> "CollectionOut": - member: CollectionMember = obj.members.get(user=context.user) + member: models.CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( collectionType=collection_type and collection_type.uid, From 1a09393dcb425250fc92c73b3584884f4f96e6c8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:29:47 +0200 Subject: [PATCH 094/170] Also add 401 to permission responses. --- etebase_fastapi/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 487f03a..2ee3700 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -44,4 +44,4 @@ def is_collection_admin(collection, user): response_model_dict = {"model": HttpErrorOut} -permission_responses = {403: response_model_dict} +permission_responses = {401: response_model_dict, 403: response_model_dict} From 37f5a4509f700169cdd60e5f47ccc69034fc9302 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:41:22 +0200 Subject: [PATCH 095/170] Improve chunks type. --- etebase_fastapi/collection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6757ac3..3af5af0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -31,7 +31,10 @@ class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.Tuple[str, t.Optional[bytes]]] + chunks: t.List[t.Union[ + t.Tuple[str], + t.Tuple[str, bytes], + ]] class Config: orm_mode = True From cf7690a60f23bbaee363ad15f0415384bf6f037e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:45:34 +0200 Subject: [PATCH 096/170] Remove usages of EtebaseValidationError. --- etebase_fastapi/authentication.py | 2 +- etebase_fastapi/stoken_handler.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 902b79b..a13cc51 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -298,7 +298,7 @@ def signup_save(data: SignupIn, request: Request) -> User: except django_exceptions.ValidationError as e: transform_validation_error("user", e) except Exception as e: - raise EtebaseValidationError("generic", str(e)) + raise HttpError("generic", str(e)) if hasattr(instance, "userinfo"): raise HttpError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index a976830..76d348a 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -3,9 +3,10 @@ import typing as t from django.db.models import QuerySet from fastapi import status -from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import Stoken +from .exceptions import HttpError + # TODO missing stoken_annotation type StokenAnnotation = t.Any @@ -15,7 +16,7 @@ def get_stoken_obj(stoken: t.Optional[str]) -> t.Optional[Stoken]: try: return Stoken.objects.get(uid=stoken) except Stoken.DoesNotExist: - raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) return None From 38884fead8cf91783e0213a948459c98973bb7db Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:47:07 +0200 Subject: [PATCH 097/170] Revert "Improve chunks type." This reverts commit 37f5a4509f700169cdd60e5f47ccc69034fc9302. --- etebase_fastapi/collection.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3af5af0..6757ac3 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -31,10 +31,7 @@ class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.Union[ - t.Tuple[str], - t.Tuple[str, bytes], - ]] + chunks: t.List[t.Tuple[str, t.Optional[bytes]]] class Config: orm_mode = True From ad2205e59616c90256afbb908ed95afaa2c24482 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:57:40 +0200 Subject: [PATCH 098/170] Add trusted host middleware. --- etebase_fastapi/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 2bbfc2a..75cb099 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -1,7 +1,6 @@ import os from django.core.wsgi import get_wsgi_application -from fastapi.middleware.cors import CORSMiddleware os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() @@ -10,6 +9,8 @@ from django.conf import settings # Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.trustedhost import TrustedHostMiddleware from .exceptions import CustomHttpException from .authentication import authentication_router @@ -35,6 +36,7 @@ if settings.DEBUG: app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) +app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) @app.exception_handler(CustomHttpException) From 08821c5e332faef831a4ba18cf6dc5645da4d583 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:39:16 +0200 Subject: [PATCH 099/170] Update changelog. --- ChangeLog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index e3c8232..a74a8af 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,13 @@ # Changelog +## Version 0.7.0 +* Chunks: improve the chunk download endpoint to use sendfile extensions +* Chunks: support not passing chunk content if exists +* Chunks: fix chunk uploading media type to accept everything +* Gracefull handle uploading the same revision +* Pass generic context to callbacks instead of the whole view +* Fix handling of some validation errors + ## Version 0.6.1 * Collection: save the UID on the model to use the db for enforcing uniqueness From 295ae6f3d34b6a6732a7789081a59c45a7de4733 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:39:16 +0200 Subject: [PATCH 100/170] Update changelog. --- ChangeLog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index e3c8232..a74a8af 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,13 @@ # Changelog +## Version 0.7.0 +* Chunks: improve the chunk download endpoint to use sendfile extensions +* Chunks: support not passing chunk content if exists +* Chunks: fix chunk uploading media type to accept everything +* Gracefull handle uploading the same revision +* Pass generic context to callbacks instead of the whole view +* Fix handling of some validation errors + ## Version 0.6.1 * Collection: save the UID on the model to use the db for enforcing uniqueness From 63afcc0830170834e59b4db17301743d3c636749 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 11:44:17 +0200 Subject: [PATCH 101/170] Mount the django application. --- etebase_fastapi/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 75cb099..6dffef4 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -9,6 +9,7 @@ from django.conf import settings # Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request +from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware @@ -37,6 +38,7 @@ app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) +app.mount("/", WSGIMiddleware(application)) @app.exception_handler(CustomHttpException) From 6c05a7898a14bd81f42ac0fd7c6760785ecb1d78 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 11:49:20 +0200 Subject: [PATCH 102/170] Add functions to split read and write permissions. --- django_etebase/app_settings.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 7c93f5f..c1e8dc9 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -33,14 +33,36 @@ class AppSettings: @cached_property def API_PERMISSIONS(self): # pylint: disable=invalid-name + """ + Deprecated. Do not use. + """ perms = self._setting("API_PERMISSIONS", ("rest_framework.permissions.IsAuthenticated",)) ret = [] for perm in perms: ret.append(self.import_from_str(perm)) return ret + @cached_property + def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS_READ", tuple()) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + + @cached_property + def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS_WRITE", tuple()) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + @cached_property def API_AUTHENTICATORS(self): # pylint: disable=invalid-name + """ + Deprecated. Do not use. + """ perms = self._setting( "API_AUTHENTICATORS", ( From b081d0129fafb02434fe94be69e3216f3ad74ac7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 12:12:00 +0200 Subject: [PATCH 103/170] Add support for read/write permissions. --- etebase_fastapi/collection.py | 61 +++++++++++++++++++++++------------ etebase_fastapi/invitation.py | 34 +++++++++++++------ etebase_fastapi/member.py | 15 ++++----- etebase_fastapi/utils.py | 7 +++- etebase_server/settings.py | 5 --- 5 files changed, 79 insertions(+), 43 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6757ac3..e6c10c3 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,7 +14,17 @@ from .authentication import get_authenticated_user from .exceptions import HttpError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel, permission_responses +from .utils import ( + get_object_or_404, + Context, + Prefetch, + PrefetchQuery, + is_collection_admin, + BaseModel, + permission_responses, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, +) User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -228,7 +238,13 @@ def has_write_access( # paths -@collection_router.post("/list_multi/", response_model=CollectionListResponse, response_model_exclude_unset=True) + +@collection_router.post( + "/list_multi/", + response_model=CollectionListResponse, + response_model_exclude_unset=True, + dependencies=PERMISSIONS_READ, +) async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, @@ -245,7 +261,7 @@ async def list_multi( return await collection_list_common(queryset, user, stoken, limit, prefetch) -@collection_router.get("/", response_model=CollectionListResponse) +@collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ) async def collection_list( stoken: t.Optional[str] = None, limit: int = 50, @@ -321,17 +337,17 @@ def _create(data: CollectionIn, user: User): ).save() -@collection_router.post("/", status_code=status.HTTP_201_CREATED) +@collection_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) -@collection_router.get("/{collection_uid}/", response_model=CollectionOut) +@collection_router.get("/{collection_uid}/", response_model=CollectionOut, dependencies=PERMISSIONS_READ) def collection_get( - obj: models.Collection = Depends(get_collection), - user: User = Depends(get_authenticated_user), - prefetch: Prefetch = PrefetchQuery - ): + obj: models.Collection = Depends(get_collection), + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, +): return CollectionOut.from_orm_context(obj, Context(user, prefetch)) @@ -375,11 +391,12 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut) +@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut, dependencies=PERMISSIONS_READ) def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), - user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) @@ -402,7 +419,7 @@ def item_list_common( return CollectionItemListResponse(data=data, stoken=new_stoken, done=done) -@item_router.get("/item/", response_model=CollectionItemListResponse) +@item_router.get("/item/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -434,7 +451,9 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return None -@item_router.get("/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse) +@item_router.get( + "/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse, dependencies=PERMISSIONS_READ +) def item_revisions( item_uid: str, limit: int = 50, @@ -469,7 +488,7 @@ def item_revisions( ) -@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse) +@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -502,14 +521,14 @@ def fetch_updates( ) -@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@item_router.post("/item/batch/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): @@ -519,7 +538,11 @@ def item_batch( # Chunks -@item_router.put("/item/{item_uid}/chunk/{chunk_uid}/", dependencies=[Depends(has_write_access)], status_code=status.HTTP_201_CREATED) +@item_router.put( + "/item/{item_uid}/chunk/{chunk_uid}/", + dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE], + status_code=status.HTTP_201_CREATED, +) def chunk_update( limit: int = 50, iterator: t.Optional[str] = None, @@ -539,6 +562,4 @@ def chunk_update( try: serializer.save(collection=col) except IntegrityError: - return Response( - {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT - ) + return Response({"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 1d8df94..39460a9 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -10,7 +10,15 @@ from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute -from .utils import get_object_or_404, Context, is_collection_admin, BaseModel, permission_responses +from .utils import ( + get_object_or_404, + Context, + is_collection_admin, + BaseModel, + permission_responses, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, +) User = get_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -108,7 +116,7 @@ def list_common( ) -@invitation_incoming_router.get("/", response_model=InvitationListResponse) +@invitation_incoming_router.get("/", response_model=InvitationListResponse, dependencies=PERMISSIONS_READ) def incoming_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -117,7 +125,9 @@ def incoming_list( return list_common(queryset, iterator, limit) -@invitation_incoming_router.get("/{invitation_uid}/", response_model=CollectionInvitationOut) +@invitation_incoming_router.get( + "/{invitation_uid}/", response_model=CollectionInvitationOut, dependencies=PERMISSIONS_READ +) def incoming_get( invitation_uid: str, queryset: QuerySet = Depends(get_incoming_queryset), @@ -126,7 +136,9 @@ def incoming_get( return CollectionInvitationOut.from_orm(obj) -@invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +@invitation_incoming_router.delete( + "/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READWRITE +) def incoming_delete( invitation_uid: str, queryset: QuerySet = Depends(get_incoming_queryset), @@ -135,7 +147,9 @@ def incoming_delete( obj.delete() -@invitation_incoming_router.post("/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED) +@invitation_incoming_router.post( + "/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE +) def incoming_accept( invitation_uid: str, data: CollectionInvitationAcceptIn, @@ -161,7 +175,7 @@ def incoming_accept( invitation.delete() -@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED) +@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) def outgoing_create( data: CollectionInvitationIn, request: Request, @@ -189,7 +203,7 @@ def outgoing_create( raise HttpError("invitation_exists", "Invitation already exists") -@invitation_outgoing_router.get("/", response_model=InvitationListResponse) +@invitation_outgoing_router.get("/", response_model=InvitationListResponse, dependencies=PERMISSIONS_READ) def outgoing_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -198,7 +212,9 @@ def outgoing_list( return list_common(queryset, iterator, limit) -@invitation_outgoing_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +@invitation_outgoing_router.delete( + "/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READWRITE +) def outgoing_delete( invitation_uid: str, queryset: QuerySet = Depends(get_outgoing_queryset), @@ -207,7 +223,7 @@ def outgoing_delete( obj.delete() -@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut) +@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut, dependencies=PERMISSIONS_READ) def outgoing_fetch_user_profile( username: str, request: Request, diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 8ffed9d..725d44b 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user from .msgpack import MsgpackRoute -from .utils import get_object_or_404, BaseModel, permission_responses +from .utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin @@ -48,7 +48,9 @@ class MemberListResponse(BaseModel): done: bool -@member_router.get("/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)]) +@member_router.get( + "/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READ] +) def member_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -70,7 +72,7 @@ def member_list( @member_router.delete( "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, - dependencies=[Depends(verify_collection_admin)], + dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READWRITE], ) def member_delete( obj: models.CollectionMember = Depends(get_member), @@ -81,7 +83,7 @@ def member_delete( @member_router.patch( "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, - dependencies=[Depends(verify_collection_admin)], + dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READWRITE], ) def member_patch( data: CollectionMemberModifyAccessLevelIn, @@ -95,10 +97,7 @@ def member_patch( instance.save() -@member_router.post( - "/member/leave/", - status_code=status.HTTP_204_NO_CONTENT, -) +@member_router.post("/member/leave/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READ) def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): obj = get_object_or_404(collection.members, user=user) obj.revoke() diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 2ee3700..165163a 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,13 +1,14 @@ import dataclasses import typing as t -from fastapi import status, Query +from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model +from django_etebase import app_settings from django_etebase.models import AccessLevels from .exceptions import HttpError, HttpErrorOut @@ -43,5 +44,9 @@ def is_collection_admin(collection, user): return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) +PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] +PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] + + response_model_dict = {"model": HttpErrorOut} permission_responses = {401: response_model_dict, 403: response_model_dict} diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 325dca9..46ad3c9 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -166,11 +166,6 @@ if any(os.path.isfile(x) for x in config_locations): if "database" in config: DATABASES = {"default": {x.upper(): y for x, y in config.items("database")}} -ETEBASE_API_PERMISSIONS = ("rest_framework.permissions.IsAuthenticated",) -ETEBASE_API_AUTHENTICATORS = ( - "django_etebase.token_auth.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", -) ETEBASE_CREATE_USER_FUNC = "django_etebase.utils.create_user_blocked" # Efficient file streaming (for large files) From 0fa2f2da3b0c1064c27b5e2851c147843c75a450 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 13:26:12 +0200 Subject: [PATCH 104/170] Make the fastapi application the main asgi one. --- etebase_fastapi/{app.py => main.py} | 13 +++---------- etebase_server/asgi.py | 21 +++++++++++---------- etebase_server/urls.py | 15 ++++++++++++++- etebase_server/wsgi.py | 16 ---------------- 4 files changed, 28 insertions(+), 37 deletions(-) rename etebase_fastapi/{app.py => main.py} (85%) delete mode 100644 etebase_server/wsgi.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/main.py similarity index 85% rename from etebase_fastapi/app.py rename to etebase_fastapi/main.py index 6dffef4..2c10854 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/main.py @@ -1,15 +1,7 @@ -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") -application = get_wsgi_application() - from django.conf import settings # Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request -from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware @@ -30,15 +22,16 @@ app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_ app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) + if settings.DEBUG: - from .test_reset_view import test_reset_view_router + from etebase_fastapi.test_reset_view import test_reset_view_router app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") + app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) -app.mount("/", WSGIMiddleware(application)) @app.exception_handler(CustomHttpException) diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py index 0bf63ec..92fad1c 100644 --- a/etebase_server/asgi.py +++ b/etebase_server/asgi.py @@ -1,16 +1,17 @@ -""" -ASGI config for etebase_server project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ -""" - import os from django.core.asgi import get_asgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") +django_application = get_asgi_application() -application = get_asgi_application() + +def create_application(): + from etebase_fastapi.main import app + + app.mount("/", django_application) + + return app + + +application = create_application() diff --git a/etebase_server/urls.py b/etebase_server/urls.py index f285977..443763d 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,8 +1,12 @@ +import os + from django.conf import settings from django.conf.urls import include, url from django.contrib import admin -from django.urls import path +from django.urls import path, re_path from django.views.generic import TemplateView +from django.views.static import serve +from django.contrib.staticfiles import finders urlpatterns = [ url(r"^api/", include("django_etebase.urls")), @@ -14,3 +18,12 @@ if settings.DEBUG: urlpatterns += [ url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), ] + + def serve_static(request, path): + filename = finders.find(path) + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + + return serve(request, basename, dirname) + + urlpatterns += [re_path(r"^static/(?P.*)$", serve_static)] diff --git a/etebase_server/wsgi.py b/etebase_server/wsgi.py deleted file mode 100644 index 908f88c..0000000 --- a/etebase_server/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for etebase_server project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") - -application = get_wsgi_application() From 4ceb42780ec3d8c475601e2e0b360fb50ceca21b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 13:56:53 +0200 Subject: [PATCH 105/170] Remove unused django_etebase code. --- README.md | 2 +- django_etebase/admin.py | 3 - django_etebase/drf_msgpack/__init__.py | 0 django_etebase/drf_msgpack/apps.py | 5 - .../drf_msgpack/migrations/__init__.py | 0 django_etebase/drf_msgpack/parsers.py | 14 - django_etebase/drf_msgpack/renderers.py | 15 - django_etebase/drf_msgpack/views.py | 3 - django_etebase/exceptions.py | 12 - django_etebase/parsers.py | 14 - django_etebase/permissions.py | 93 -- django_etebase/renderers.py | 19 - django_etebase/serializers.py | 598 ------------ django_etebase/tests.py | 3 - django_etebase/token_auth/admin.py | 0 django_etebase/token_auth/authentication.py | 46 - django_etebase/urls.py | 30 - django_etebase/views.py | 861 ------------------ etebase_fastapi/authentication.py | 6 +- etebase_fastapi/utils.py | 9 + etebase_server/settings.py | 6 - etebase_server/urls.py | 6 +- requirements.in/base.txt | 6 +- requirements.txt | 6 +- 24 files changed, 15 insertions(+), 1742 deletions(-) delete mode 100644 django_etebase/admin.py delete mode 100644 django_etebase/drf_msgpack/__init__.py delete mode 100644 django_etebase/drf_msgpack/apps.py delete mode 100644 django_etebase/drf_msgpack/migrations/__init__.py delete mode 100644 django_etebase/drf_msgpack/parsers.py delete mode 100644 django_etebase/drf_msgpack/renderers.py delete mode 100644 django_etebase/drf_msgpack/views.py delete mode 100644 django_etebase/exceptions.py delete mode 100644 django_etebase/parsers.py delete mode 100644 django_etebase/permissions.py delete mode 100644 django_etebase/renderers.py delete mode 100644 django_etebase/serializers.py delete mode 100644 django_etebase/tests.py delete mode 100644 django_etebase/token_auth/admin.py delete mode 100644 django_etebase/token_auth/authentication.py delete mode 100644 django_etebase/urls.py delete mode 100644 django_etebase/views.py diff --git a/README.md b/README.md index 3e0bd53..1787a2f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Now you can initialise our django app. And you are done! You can now run the debug server just to see everything works as expected by running: ``` -./manage.py runserver 0.0.0.0:8000 +uvicorn etebase_server.asgi:application --port 8000 ``` Using the debug server in production is not recommended, so please read the following section for a proper deployment. diff --git a/django_etebase/admin.py b/django_etebase/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/django_etebase/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/django_etebase/drf_msgpack/__init__.py b/django_etebase/drf_msgpack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/drf_msgpack/apps.py b/django_etebase/drf_msgpack/apps.py deleted file mode 100644 index 22ea2c1..0000000 --- a/django_etebase/drf_msgpack/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class DrfMsgpackConfig(AppConfig): - name = "drf_msgpack" diff --git a/django_etebase/drf_msgpack/migrations/__init__.py b/django_etebase/drf_msgpack/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/drf_msgpack/parsers.py b/django_etebase/drf_msgpack/parsers.py deleted file mode 100644 index 0504a76..0000000 --- a/django_etebase/drf_msgpack/parsers.py +++ /dev/null @@ -1,14 +0,0 @@ -import msgpack - -from rest_framework.parsers import BaseParser -from rest_framework.exceptions import ParseError - - -class MessagePackParser(BaseParser): - media_type = "application/msgpack" - - def parse(self, stream, media_type=None, parser_context=None): - try: - return msgpack.unpackb(stream.read(), raw=False) - except Exception as exc: - raise ParseError("MessagePack parse error - %s" % str(exc)) diff --git a/django_etebase/drf_msgpack/renderers.py b/django_etebase/drf_msgpack/renderers.py deleted file mode 100644 index 35a4afa..0000000 --- a/django_etebase/drf_msgpack/renderers.py +++ /dev/null @@ -1,15 +0,0 @@ -import msgpack - -from rest_framework.renderers import BaseRenderer - - -class MessagePackRenderer(BaseRenderer): - media_type = "application/msgpack" - format = "msgpack" - render_style = "binary" - charset = None - - def render(self, data, media_type=None, renderer_context=None): - if data is None: - return b"" - return msgpack.packb(data, use_bin_type=True) diff --git a/django_etebase/drf_msgpack/views.py b/django_etebase/drf_msgpack/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/django_etebase/drf_msgpack/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/django_etebase/exceptions.py b/django_etebase/exceptions.py deleted file mode 100644 index 437a71c..0000000 --- a/django_etebase/exceptions.py +++ /dev/null @@ -1,12 +0,0 @@ -from rest_framework import serializers, status - - -class EtebaseValidationError(serializers.ValidationError): - def __init__(self, code, detail, status_code=status.HTTP_400_BAD_REQUEST): - super().__init__( - { - "code": code, - "detail": detail, - } - ) - self.status_code = status_code diff --git a/django_etebase/parsers.py b/django_etebase/parsers.py deleted file mode 100644 index ed1e713..0000000 --- a/django_etebase/parsers.py +++ /dev/null @@ -1,14 +0,0 @@ -from rest_framework.parsers import FileUploadParser - - -class ChunkUploadParser(FileUploadParser): - """ - Parser for chunk upload data. - """ - - def get_filename(self, stream, media_type, parser_context): - """ - Detects the uploaded file name. - """ - view = parser_context["view"] - return parser_context["kwargs"][view.lookup_field] diff --git a/django_etebase/permissions.py b/django_etebase/permissions.py deleted file mode 100644 index 3c77d06..0000000 --- a/django_etebase/permissions.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from rest_framework import permissions -from django_etebase.models import Collection, AccessLevels - - -def is_collection_admin(collection, user): - member = collection.members.filter(user=user).first() - return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) - - -class IsCollectionAdmin(permissions.BasePermission): - """ - Custom permission to only allow owners of a collection to view it - """ - - message = { - "detail": "Only collection admins can perform this operation.", - "code": "admin_access_required", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs["collection_uid"] - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - return is_collection_admin(collection, request.user) - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True - - -class IsCollectionAdminOrReadOnly(permissions.BasePermission): - """ - Custom permission to only allow owners of a collection to edit it - """ - - message = { - "detail": "Only collection admins can edit collections.", - "code": "admin_access_required", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs.get("collection_uid", None) - - # Allow creating new collections - if collection_uid is None: - return True - - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - if request.method in permissions.SAFE_METHODS: - return True - - return is_collection_admin(collection, request.user) - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True - - -class HasWriteAccessOrReadOnly(permissions.BasePermission): - """ - Custom permission to restrict write - """ - - message = { - "detail": "You need write access to write to this collection", - "code": "no_write_access", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs["collection_uid"] - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - if request.method in permissions.SAFE_METHODS: - return True - else: - member = collection.members.get(user=request.user) - return member.accessLevel != AccessLevels.READ_ONLY - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True diff --git a/django_etebase/renderers.py b/django_etebase/renderers.py deleted file mode 100644 index 0d359d3..0000000 --- a/django_etebase/renderers.py +++ /dev/null @@ -1,19 +0,0 @@ -from rest_framework.utils.encoders import JSONEncoder as DRFJSONEncoder -from rest_framework.renderers import JSONRenderer as DRFJSONRenderer - -from .serializers import b64encode - - -class JSONEncoder(DRFJSONEncoder): - def default(self, obj): - if isinstance(obj, bytes) or isinstance(obj, memoryview): - return b64encode(obj) - return super().default(obj) - - -class JSONRenderer(DRFJSONRenderer): - """ - Renderer which serializes to JSON with support for our base64 - """ - - encoder_class = JSONEncoder diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py deleted file mode 100644 index 26ac5a7..0000000 --- a/django_etebase/serializers.py +++ /dev/null @@ -1,598 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 - -from django.core.files.base import ContentFile -from django.core import exceptions as django_exceptions -from django.contrib.auth import get_user_model -from django.db import IntegrityError, transaction -from rest_framework import serializers, status -from . import models -from .utils import get_user_queryset, create_user, CallbackContext - -from .exceptions import EtebaseValidationError - -User = get_user_model() - - -def process_revisions_for_item(item, revision_data): - chunks_objs = [] - chunks = revision_data.pop("chunks_relation") - - revision = models.CollectionItemRevision(**revision_data, item=item) - revision.validate_unique() # Verify there aren't any validation issues - - for chunk in chunks: - uid = chunk[0] - chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() - content = chunk[1] if len(chunk) > 1 else None - # If the chunk already exists we assume it's fine. Otherwise, we upload it. - if chunk_obj is None: - if content is not None: - chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) - chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) - chunk_obj.save() - else: - raise EtebaseValidationError("chunk_no_content", "Tried to create a new chunk without content") - - chunks_objs.append(chunk_obj) - - stoken = models.Stoken.objects.create() - revision.stoken = stoken - revision.save() - - for chunk in chunks_objs: - models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) - return revision - - -def b64encode(value): - return base64.urlsafe_b64encode(value).decode("ascii").strip("=") - - -def b64decode(data): - data += "=" * ((4 - len(data) % 4) % 4) - return base64.urlsafe_b64decode(data) - - -def b64decode_or_bytes(data): - if isinstance(data, bytes): - return data - else: - return b64decode(data) - - -class BinaryBase64Field(serializers.Field): - def to_representation(self, value): - return value - - def to_internal_value(self, data): - return b64decode_or_bytes(data) - - -class CollectionEncryptionKeyField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get("request", None) - if request is not None: - return instance.members.get(user=request.user).encryptionKey - return None - - -class CollectionTypeField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get("request", None) - if request is not None: - collection_type = instance.members.get(user=request.user).collectionType - return collection_type and collection_type.uid - return None - - -class UserSlugRelatedField(serializers.SlugRelatedField): - def get_queryset(self): - view = self.context.get("view", None) - return get_user_queryset(super().get_queryset(), context=CallbackContext(view.kwargs)) - - def __init__(self, **kwargs): - super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) - - def to_internal_value(self, data): - return super().to_internal_value(data.lower()) - - -class ChunksField(serializers.RelatedField): - def to_representation(self, obj): - obj = obj.chunk - if self.context.get("prefetch") == "auto": - with open(obj.chunkFile.path, "rb") as f: - return (obj.uid, f.read()) - else: - return (obj.uid,) - - def to_internal_value(self, data): - content = data[1] if len(data) > 1 else None - if data[0] is None: - raise EtebaseValidationError("no_null", "null is not allowed") - return (data[0], b64decode_or_bytes(content) if content is not None else None) - - -class BetterErrorsMixin: - @property - def errors(self): - nice = [] - errors = super().errors - for error_type in errors: - if error_type == "non_field_errors": - nice.extend(self.flatten_errors(None, errors[error_type])) - else: - nice.extend(self.flatten_errors(error_type, errors[error_type])) - if nice: - return {"code": "field_errors", "detail": "Field validations failed.", "errors": nice} - return {} - - def flatten_errors(self, field_name, errors): - ret = [] - if isinstance(errors, dict): - for error_key in errors: - error = errors[error_key] - ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) - else: - for error in errors: - if getattr(error, "messages", None): - message = error.messages[0] - else: - message = str(error) - ret.append( - { - "field": field_name, - "code": error.code, - "detail": message, - } - ) - return ret - - def transform_validation_error(self, prefix, err): - if hasattr(err, "error_dict"): - errors = self.flatten_errors(prefix, err.error_dict) - elif not hasattr(err, "message"): - errors = self.flatten_errors(prefix, err.error_list) - else: - raise EtebaseValidationError(err.code, err.message) - - raise serializers.ValidationError( - { - "code": "field_errors", - "detail": "Field validations failed.", - "errors": errors, - } - ) - - -class CollectionItemChunkSerializer(BetterErrorsMixin, serializers.ModelSerializer): - class Meta: - model = models.CollectionItemChunk - fields = ("uid", "chunkFile") - - -class CollectionItemRevisionSerializer(BetterErrorsMixin, serializers.ModelSerializer): - chunks = ChunksField( - source="chunks_relation", - queryset=models.RevisionChunkRelation.objects.all(), - style={"base_template": "input.html"}, - many=True, - ) - meta = BinaryBase64Field() - - class Meta: - model = models.CollectionItemRevision - fields = ("chunks", "meta", "uid", "deleted") - extra_kwargs = { - "uid": {"validators": []}, # We deal with it in the serializers - } - - -class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): - encryptionKey = BinaryBase64Field(required=False, default=None, allow_null=True) - etag = serializers.CharField(allow_null=True, write_only=True) - content = CollectionItemRevisionSerializer(many=False) - - class Meta: - model = models.CollectionItem - fields = ("uid", "version", "encryptionKey", "content", "etag") - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - validate_etag = self.context.get("validate_etag", False) - etag = validated_data.pop("etag") - revision_data = validated_data.pop("content") - uid = validated_data.pop("uid") - - Model = self.__class__.Meta.model - - with transaction.atomic(): - instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) - cur_etag = instance.etag if not created else None - - # If we are trying to update an up to date item, abort early and consider it a success - if cur_etag == revision_data.get("uid"): - return instance - - if validate_etag and cur_etag != etag: - raise EtebaseValidationError( - "wrong_etag", - "Wrong etag. Expected {} got {}".format(cur_etag, etag), - status_code=status.HTTP_409_CONFLICT, - ) - - if not created: - # We don't have to use select_for_update here because the unique constraint on current guards against - # the race condition. But it's a good idea because it'll lock and wait rather than fail. - current_revision = instance.revisions.filter(current=True).select_for_update().first() - - # If we are just re-uploading the same revision, consider it a succes and return. - if current_revision.uid == revision_data.get("uid"): - return instance - - current_revision.current = None - current_revision.save() - - try: - process_revisions_for_item(instance, revision_data) - except django_exceptions.ValidationError as e: - self.transform_validation_error("content", e) - - return instance - - def update(self, instance, validated_data): - # We never update, we always update in the create method - raise NotImplementedError() - - -class CollectionItemDepSerializer(BetterErrorsMixin, serializers.ModelSerializer): - etag = serializers.CharField() - - class Meta: - model = models.CollectionItem - fields = ("uid", "etag") - - def validate(self, data): - item = self.__class__.Meta.model.objects.get(uid=data["uid"]) - etag = data["etag"] - if item.etag != etag: - raise EtebaseValidationError( - "wrong_etag", - "Wrong etag. Expected {} got {}".format(item.etag, etag), - status_code=status.HTTP_409_CONFLICT, - ) - - return data - - -class CollectionItemBulkGetSerializer(BetterErrorsMixin, serializers.ModelSerializer): - etag = serializers.CharField(required=False) - - class Meta: - model = models.CollectionItem - fields = ("uid", "etag") - - -class CollectionListMultiSerializer(BetterErrorsMixin, serializers.Serializer): - collectionTypes = serializers.ListField(child=BinaryBase64Field()) - - -class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): - collectionKey = CollectionEncryptionKeyField() - collectionType = CollectionTypeField() - accessLevel = serializers.SerializerMethodField("get_access_level_from_context") - stoken = serializers.CharField(read_only=True) - - item = CollectionItemSerializer(many=False, source="main_item") - - class Meta: - model = models.Collection - fields = ("item", "accessLevel", "collectionKey", "collectionType", "stoken") - - def get_access_level_from_context(self, obj): - request = self.context.get("request", None) - if request is not None: - return obj.members.get(user=request.user).accessLevel - return None - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - collection_key = validated_data.pop("collectionKey") - collection_type = validated_data.pop("collectionType") - - user = validated_data.get("owner") - main_item_data = validated_data.pop("main_item") - uid = main_item_data.get("uid") - etag = main_item_data.pop("etag") - revision_data = main_item_data.pop("content") - - instance = self.__class__.Meta.model(uid=uid, **validated_data) - - with transaction.atomic(): - if etag is not None: - raise EtebaseValidationError("bad_etag", "etag is not null") - - try: - instance.validate_unique() - except django_exceptions.ValidationError: - raise EtebaseValidationError( - "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT - ) - instance.save() - - main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) - - instance.main_item = main_item - instance.save() - - process_revisions_for_item(main_item, revision_data) - - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) - - models.CollectionMember( - collection=instance, - stoken=models.Stoken.objects.create(), - user=user, - accessLevel=models.AccessLevels.ADMIN, - encryptionKey=collection_key, - collectionType=collection_type_obj, - ).save() - - return instance - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField( - source="user", - read_only=True, - style={"base_template": "input.html"}, - ) - - class Meta: - model = models.CollectionMember - fields = ("username", "accessLevel") - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - with transaction.atomic(): - # We only allow updating accessLevel - access_level = validated_data.pop("accessLevel") - if instance.accessLevel != access_level: - instance.stoken = models.Stoken.objects.create() - instance.accessLevel = access_level - instance.save() - - return instance - - -class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField( - source="user", - queryset=User.objects, - style={"base_template": "input.html"}, - ) - collection = serializers.CharField(source="collection.uid") - fromUsername = serializers.CharField(source="fromMember.user.username", read_only=True) - fromPubkey = BinaryBase64Field(source="fromMember.user.userinfo.pubkey", read_only=True) - signedEncryptionKey = BinaryBase64Field() - - class Meta: - model = models.CollectionInvitation - fields = ( - "username", - "uid", - "collection", - "signedEncryptionKey", - "accessLevel", - "fromUsername", - "fromPubkey", - "version", - ) - - def validate_user(self, value): - request = self.context["request"] - - if request.user.username == value.lower(): - raise EtebaseValidationError("no_self_invite", "Inviting yourself is not allowed") - return value - - def create(self, validated_data): - request = self.context["request"] - collection = validated_data.pop("collection") - - member = collection.members.get(user=request.user) - - with transaction.atomic(): - try: - return type(self).Meta.model.objects.create(**validated_data, fromMember=member) - except IntegrityError: - raise EtebaseValidationError("invitation_exists", "Invitation already exists") - - def update(self, instance, validated_data): - with transaction.atomic(): - instance.accessLevel = validated_data.pop("accessLevel") - instance.signedEncryptionKey = validated_data.pop("signedEncryptionKey") - instance.save() - - return instance - - -class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): - collectionType = BinaryBase64Field() - encryptionKey = BinaryBase64Field() - - def create(self, validated_data): - - with transaction.atomic(): - invitation = self.context["invitation"] - encryption_key = validated_data.get("encryptionKey") - collection_type = validated_data.pop("collectionType") - - user = invitation.user - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) - - member = models.CollectionMember.objects.create( - collection=invitation.collection, - stoken=models.Stoken.objects.create(), - user=user, - accessLevel=invitation.accessLevel, - encryptionKey=encryption_key, - collectionType=collection_type_obj, - ) - - models.CollectionMemberRemoved.objects.filter( - user=invitation.user, collection=invitation.collection - ).delete() - - invitation.delete() - - return member - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class UserSerializer(BetterErrorsMixin, serializers.ModelSerializer): - pubkey = BinaryBase64Field(source="userinfo.pubkey") - encryptedContent = BinaryBase64Field(source="userinfo.encryptedContent") - - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, "pubkey", "encryptedContent") - - -class UserInfoPubkeySerializer(BetterErrorsMixin, serializers.ModelSerializer): - pubkey = BinaryBase64Field() - - class Meta: - model = models.UserInfo - fields = ("pubkey",) - - -class UserSignupSerializer(BetterErrorsMixin, serializers.ModelSerializer): - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) - extra_kwargs = { - "username": {"validators": []}, # We specifically validate in SignupSerializer - } - - -class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): - """Used both for creating new accounts and setting up existing ones for the first time. - When setting up existing ones the email is ignored." - """ - - user = UserSignupSerializer(many=False) - salt = BinaryBase64Field() - loginPubkey = BinaryBase64Field() - pubkey = BinaryBase64Field() - encryptedContent = BinaryBase64Field() - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - user_data = validated_data.pop("user") - - with transaction.atomic(): - view = self.context.get("view", None) - try: - user_queryset = get_user_queryset(User.objects.all(), context=CallbackContext(view.kwargs)) - instance = user_queryset.get(**{User.USERNAME_FIELD: user_data["username"].lower()}) - except User.DoesNotExist: - # Create the user and save the casing the user chose as the first name - try: - instance = create_user( - **user_data, - password=None, - first_name=user_data["username"], - context=CallbackContext(view.kwargs) - ) - instance.full_clean() - except EtebaseValidationError as e: - raise e - except django_exceptions.ValidationError as e: - self.transform_validation_error("user", e) - except Exception as e: - raise EtebaseValidationError("generic", str(e)) - - if hasattr(instance, "userinfo"): - raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) - - models.UserInfo.objects.create(**validated_data, owner=instance) - - return instance - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginChallengeSerializer(BetterErrorsMixin, serializers.Serializer): - username = serializers.CharField(required=True) - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginSerializer(BetterErrorsMixin, serializers.Serializer): - response = BinaryBase64Field() - signature = BinaryBase64Field() - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): - challenge = BinaryBase64Field() - host = serializers.CharField() - action = serializers.CharField() - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationChangePasswordInnerSerializer(AuthenticationLoginInnerSerializer): - loginPubkey = BinaryBase64Field() - encryptedContent = BinaryBase64Field() - - class Meta: - model = models.UserInfo - fields = ("loginPubkey", "encryptedContent") - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - with transaction.atomic(): - instance.loginPubkey = validated_data.pop("loginPubkey") - instance.encryptedContent = validated_data.pop("encryptedContent") - instance.save() - - return instance diff --git a/django_etebase/tests.py b/django_etebase/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/django_etebase/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/django_etebase/token_auth/admin.py b/django_etebase/token_auth/admin.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/token_auth/authentication.py b/django_etebase/token_auth/authentication.py deleted file mode 100644 index 7e84956..0000000 --- a/django_etebase/token_auth/authentication.py +++ /dev/null @@ -1,46 +0,0 @@ -from django.utils import timezone -from django.utils.translation import gettext_lazy as _ - -from rest_framework import exceptions -from rest_framework.authentication import TokenAuthentication as DRFTokenAuthentication - -from .models import AuthToken, get_default_expiry - - -AUTO_REFRESH = True -MIN_REFRESH_INTERVAL = 60 - - -class TokenAuthentication(DRFTokenAuthentication): - keyword = "Token" - model = AuthToken - - def authenticate_credentials(self, key): - msg = _("Invalid token.") - model = self.get_model() - try: - token = model.objects.select_related("user").get(key=key) - except model.DoesNotExist: - raise exceptions.AuthenticationFailed(msg) - - if not token.user.is_active: - raise exceptions.AuthenticationFailed(_("User inactive or deleted.")) - - if token.expiry is not None: - if token.expiry < timezone.now(): - token.delete() - raise exceptions.AuthenticationFailed(msg) - - if AUTO_REFRESH: - self.renew_token(token) - - return (token.user, token) - - def renew_token(self, auth_token): - current_expiry = auth_token.expiry - new_expiry = get_default_expiry() - # Throttle refreshing of token to avoid db writes - delta = (new_expiry - current_expiry).total_seconds() - if delta > MIN_REFRESH_INTERVAL: - auth_token.expiry = new_expiry - auth_token.save(update_fields=("expiry",)) diff --git a/django_etebase/urls.py b/django_etebase/urls.py deleted file mode 100644 index 01797c1..0000000 --- a/django_etebase/urls.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.conf import settings -from django.conf.urls import include -from django.urls import path - -from rest_framework_nested import routers - -from django_etebase import views - -router = routers.DefaultRouter() -router.register(r"collection", views.CollectionViewSet) -router.register(r"authentication", views.AuthenticationViewSet, basename="authentication") -router.register(r"invitation/incoming", views.InvitationIncomingViewSet, basename="invitation_incoming") -router.register(r"invitation/outgoing", views.InvitationOutgoingViewSet, basename="invitation_outgoing") - -collections_router = routers.NestedSimpleRouter(router, r"collection", lookup="collection") -collections_router.register(r"item", views.CollectionItemViewSet, basename="collection_item") -collections_router.register(r"member", views.CollectionMemberViewSet, basename="collection_member") - -item_router = routers.NestedSimpleRouter(collections_router, r"item", lookup="collection_item") -item_router.register(r"chunk", views.CollectionItemChunkViewSet, basename="collection_items_chunk") - -if settings.DEBUG: - router.register(r"test/authentication", views.TestAuthenticationViewSet, basename="test_authentication") - -app_name = "django_etebase" -urlpatterns = [ - path("v1/", include(router.urls)), - path("v1/", include(collections_router.urls)), - path("v1/", include(item_router.urls)), -] diff --git a/django_etebase/views.py b/django_etebase/views.py deleted file mode 100644 index 5a03aa4..0000000 --- a/django_etebase/views.py +++ /dev/null @@ -1,861 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import msgpack - -from django.conf import settings -from django.contrib.auth import get_user_model, user_logged_in, user_logged_out -from django.core.exceptions import PermissionDenied -from django.db import transaction, IntegrityError -from django.db.models import Q -from django.http import HttpResponseBadRequest, HttpResponse, Http404 -from django.shortcuts import get_object_or_404 - -from rest_framework import status -from rest_framework import viewsets -from rest_framework.decorators import action as action_decorator -from rest_framework.response import Response -from rest_framework.parsers import JSONParser, FormParser, MultiPartParser -from rest_framework.renderers import BrowsableAPIRenderer -from rest_framework.exceptions import AuthenticationFailed -from rest_framework.permissions import IsAuthenticated - -import nacl.encoding -import nacl.signing -import nacl.secret -import nacl.hash - -from .sendfile import sendfile -from .token_auth.models import AuthToken - -from .drf_msgpack.parsers import MessagePackParser -from .drf_msgpack.renderers import MessagePackRenderer - -from . import app_settings, permissions -from .renderers import JSONRenderer -from .models import ( - Collection, - CollectionItem, - CollectionItemRevision, - CollectionMember, - CollectionMemberRemoved, - CollectionInvitation, - Stoken, - UserInfo, -) -from .serializers import ( - AuthenticationChangePasswordInnerSerializer, - AuthenticationSignupSerializer, - AuthenticationLoginChallengeSerializer, - AuthenticationLoginSerializer, - AuthenticationLoginInnerSerializer, - CollectionSerializer, - CollectionItemSerializer, - CollectionItemBulkGetSerializer, - CollectionItemDepSerializer, - CollectionItemRevisionSerializer, - CollectionItemChunkSerializer, - CollectionListMultiSerializer, - CollectionMemberSerializer, - CollectionInvitationSerializer, - InvitationAcceptSerializer, - UserInfoPubkeySerializer, - UserSerializer, -) -from .utils import get_user_queryset, CallbackContext -from .exceptions import EtebaseValidationError -from .parsers import ChunkUploadParser -from .signals import user_signed_up - -User = get_user_model() - - -def msgpack_encode(content): - return msgpack.packb(content, use_bin_type=True) - - -def msgpack_decode(content): - return msgpack.unpackb(content, raw=False) - - -class BaseViewSet(viewsets.ModelViewSet): - authentication_classes = tuple(app_settings.API_AUTHENTICATORS) - permission_classes = tuple(app_settings.API_PERMISSIONS) - renderer_classes = [JSONRenderer, MessagePackRenderer] + ([BrowsableAPIRenderer] if settings.DEBUG else []) - parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] - stoken_annotation = None - - def get_serializer_class(self): - serializer_class = self.serializer_class - - if self.request.method == "PUT": - serializer_class = getattr(self, "serializer_update_class", serializer_class) - - return serializer_class - - def get_collection_queryset(self, queryset=Collection.objects): - user = self.request.user - return queryset.filter(members__user=user) - - def get_stoken_obj_id(self, request): - return request.GET.get("stoken", None) - - def get_stoken_obj(self, request): - stoken = self.get_stoken_obj_id(request) - - if stoken is not None: - try: - return Stoken.objects.get(uid=stoken) - except Stoken.DoesNotExist: - raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) - - return None - - def filter_by_stoken(self, request, queryset): - stoken_rev = self.get_stoken_obj(request) - - queryset = queryset.annotate(max_stoken=self.stoken_annotation).order_by("max_stoken") - - if stoken_rev is not None: - queryset = queryset.filter(max_stoken__gt=stoken_rev.id) - - return queryset, stoken_rev - - def get_queryset_stoken(self, queryset): - maxid = -1 - for row in queryset: - rowmaxid = getattr(row, "max_stoken") or -1 - maxid = max(maxid, rowmaxid) - new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) - - return new_stoken or None - - def filter_by_stoken_and_limit(self, request, queryset): - limit = int(request.GET.get("limit", 50)) - - queryset, stoken_rev = self.filter_by_stoken(request, queryset) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - new_stoken_obj = self.get_queryset_stoken(result) or stoken_rev - - return result, new_stoken_obj, done - - # Change how our list works by default - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - serializer = self.get_serializer(queryset, many=True) - - ret = { - "data": serializer.data, - "done": True, # we always return all the items, so it's always done - } - - return Response(ret) - - -class CollectionViewSet(BaseViewSet): - allowed_methods = ["GET", "POST"] - permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly,) - queryset = Collection.objects.all() - serializer_class = CollectionSerializer - lookup_field = "uid" - lookup_url_kwarg = "uid" - stoken_annotation = Collection.stoken_annotation - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - return self.get_collection_queryset(queryset) - - def get_serializer_context(self): - context = super().get_serializer_context() - prefetch = self.request.query_params.get("prefetch", "auto") - context.update({"request": self.request, "prefetch": prefetch}) - return context - - def destroy(self, request, uid=None, *args, **kwargs): - # FIXME: implement - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save(owner=self.request.user) - - return Response({}, status=status.HTTP_201_CREATED) - - def list(self, request, *args, **kwargs): - queryset = self.get_queryset() - return self.list_common(request, queryset, *args, **kwargs) - - @action_decorator(detail=False, methods=["POST"]) - def list_multi(self, request, *args, **kwargs): - serializer = CollectionListMultiSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - - collection_types = serializer.validated_data["collectionTypes"] - - queryset = self.get_queryset() - # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") - queryset = queryset.filter( - Q(members__collectionType__uid__in=collection_types) | Q(members__collectionType__isnull=True) - ) - - return self.list_common(request, queryset, *args, **kwargs) - - def list_common(self, request, queryset, *args, **kwargs): - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": done, - } - - stoken_obj = self.get_stoken_obj(request) - if stoken_obj is not None: - # FIXME: honour limit? (the limit should be combined for data and this because of stoken) - remed_qs = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) - if not ret["done"]: - # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken - # can point to the most recent collection change rather than most recent removed membership. - remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) - - remed = remed_qs.values_list("collection__uid", flat=True) - if len(remed) > 0: - ret["removedMemberships"] = [{"uid": x} for x in remed] - - return Response(ret) - - -class CollectionItemViewSet(BaseViewSet): - allowed_methods = ["GET", "POST", "PUT"] - permission_classes = BaseViewSet.permission_classes + (permissions.HasWriteAccessOrReadOnly,) - queryset = CollectionItem.objects.all() - serializer_class = CollectionItemSerializer - lookup_field = "uid" - stoken_annotation = CollectionItem.stoken_annotation - - def get_queryset(self): - collection_uid = self.kwargs["collection_uid"] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = type(self).queryset.filter(collection__pk=collection.pk, revisions__current=True) - - return queryset - - def get_serializer_context(self): - context = super().get_serializer_context() - prefetch = self.request.query_params.get("prefetch", "auto") - context.update({"request": self.request, "prefetch": prefetch}) - return context - - def create(self, request, collection_uid=None, *args, **kwargs): - # We create using batch and transaction - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def destroy(self, request, collection_uid=None, uid=None, *args, **kwargs): - # We can't have destroy because we need to get data from the user (in the body) such as hmac. - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, collection_uid=None, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, collection_uid=None, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - - if not self.request.query_params.get("withCollection", False): - queryset = queryset.filter(parent__isnull=True) - - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": done, - } - return Response(ret) - - @action_decorator(detail=True, methods=["GET"]) - def revision(self, request, collection_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - item = get_object_or_404(col.items, uid=uid) - - limit = int(request.GET.get("limit", 50)) - iterator = request.GET.get("iterator", None) - - queryset = item.revisions.order_by("-id") - - if iterator is not None: - iterator = get_object_or_404(queryset, uid=iterator) - queryset = queryset.filter(id__lt=iterator.id) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - serializer = CollectionItemRevisionSerializer(result, context=self.get_serializer_context(), many=True) - - iterator = serializer.data[-1]["uid"] if len(result) > 0 else None - - ret = { - "data": serializer.data, - "iterator": iterator, - "done": done, - } - return Response(ret) - - # FIXME: rename to something consistent with what the clients have - maybe list_updates? - @action_decorator(detail=False, methods=["POST"]) - def fetch_updates(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - - serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) - serializer.is_valid(raise_exception=True) - # FIXME: make configurable? - item_limit = 200 - - if len(serializer.validated_data) > item_limit: - content = {"code": "too_many_items", "detail": "Request has too many items. Limit: {}".format(item_limit)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - queryset, stoken_rev = self.filter_by_stoken(request, queryset) - - uids, etags = zip(*[(item["uid"], item.get("etag")) for item in serializer.validated_data]) - revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) - queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - - new_stoken_obj = self.get_queryset_stoken(queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - stoken = stoken_rev and getattr(stoken_rev, "uid", None) - new_stoken = new_stoken or stoken - - serializer = self.get_serializer(queryset, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": True, # we always return all the items, so it's always done - } - return Response(ret) - - @action_decorator(detail=False, methods=["POST"]) - def batch(self, request, collection_uid=None, *args, **kwargs): - return self.transaction(request, collection_uid, validate_etag=False) - - @action_decorator(detail=False, methods=["POST"]) - def transaction(self, request, collection_uid=None, validate_etag=True, *args, **kwargs): - stoken = request.GET.get("stoken", None) - with transaction.atomic(): # We need this for locking on the collection object - collection_object = get_object_or_404( - self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection - uid=collection_uid, - ) - - if stoken is not None and stoken != collection_object.stoken: - content = {"code": "stale_stoken", "detail": "Stoken is too old"} - return Response(content, status=status.HTTP_409_CONFLICT) - - items = request.data.get("items") - deps = request.data.get("deps", None) - # FIXME: It should just be one serializer - context = self.get_serializer_context() - context.update({"validate_etag": validate_etag}) - serializer = self.get_serializer_class()(data=items, context=context, many=True) - deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) - - ser_valid = serializer.is_valid() - deps_ser_valid = deps is None or deps_serializer.is_valid() - if ser_valid and deps_ser_valid: - items = serializer.save(collection=collection_object) - - ret = {} - return Response(ret, status=status.HTTP_200_OK) - - return Response( - { - "items": serializer.errors, - "deps": deps_serializer.errors if deps is not None else [], - }, - status=status.HTTP_409_CONFLICT, - ) - - -class CollectionItemChunkViewSet(viewsets.ViewSet): - allowed_methods = ["GET", "PUT"] - authentication_classes = BaseViewSet.authentication_classes - permission_classes = BaseViewSet.permission_classes - renderer_classes = BaseViewSet.renderer_classes - parser_classes = (ChunkUploadParser,) - serializer_class = CollectionItemChunkSerializer - lookup_field = "uid" - - def get_serializer_class(self): - return self.serializer_class - - def get_collection_queryset(self, queryset=Collection.objects): - user = self.request.user - return queryset.filter(members__user=user) - - def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - - data = { - "uid": uid, - "chunkFile": request.data["file"], - } - - serializer = self.get_serializer_class()(data=data) - serializer.is_valid(raise_exception=True) - try: - serializer.save(collection=col) - except IntegrityError: - return Response( - {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT - ) - - return Response({}, status=status.HTTP_201_CREATED) - - @action_decorator(detail=True, methods=["GET"]) - def download(self, request, collection_uid=None, collection_item_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - chunk = get_object_or_404(col.chunks, uid=uid) - - filename = chunk.chunkFile.path - return sendfile(request, filename) - - -class CollectionMemberViewSet(BaseViewSet): - allowed_methods = ["GET", "PUT", "DELETE"] - our_base_permission_classes = BaseViewSet.permission_classes - permission_classes = our_base_permission_classes + (permissions.IsCollectionAdmin,) - queryset = CollectionMember.objects.all() - serializer_class = CollectionMemberSerializer - lookup_field = f"user__{User.USERNAME_FIELD}__iexact" - lookup_url_kwarg = "username" - stoken_annotation = CollectionMember.stoken_annotation - - # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access - # (if we want to transfer, we need to do that specifically) - - def get_queryset(self, queryset=None): - collection_uid = self.kwargs["collection_uid"] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(collection=collection) - - # We override this method because we expect the stoken to be called iterator - def get_stoken_obj_id(self, request): - return request.GET.get("iterator", None) - - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset().order_by("id") - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "iterator": new_stoken, # Here we call it an iterator, it's only stoken for collection/items - "done": done, - } - - return Response(ret) - - def create(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - # FIXME: block leaving if we are the last admins - should be deleted / assigned in this case depending if there - # are other memebers. - def perform_destroy(self, instance): - instance.revoke() - - @action_decorator(detail=False, methods=["POST"], permission_classes=our_base_permission_classes) - def leave(self, request, collection_uid=None, *args, **kwargs): - collection_uid = self.kwargs["collection_uid"] - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - - member = col.members.get(user=request.user) - self.perform_destroy(member) - - return Response({}) - - -class InvitationBaseViewSet(BaseViewSet): - queryset = CollectionInvitation.objects.all() - serializer_class = CollectionInvitationSerializer - lookup_field = "uid" - lookup_url_kwarg = "invitation_uid" - - def list(self, request, collection_uid=None, *args, **kwargs): - limit = int(request.GET.get("limit", 50)) - iterator = request.GET.get("iterator", None) - - queryset = self.get_queryset().order_by("id") - - if iterator is not None: - iterator = get_object_or_404(queryset, uid=iterator) - queryset = queryset.filter(id__gt=iterator.id) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - serializer = self.get_serializer(result, many=True) - - iterator = serializer.data[-1]["uid"] if len(result) > 0 else None - - ret = { - "data": serializer.data, - "iterator": iterator, - "done": done, - } - - return Response(ret) - - -class InvitationOutgoingViewSet(InvitationBaseViewSet): - allowed_methods = ["GET", "POST", "PUT", "DELETE"] - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(fromMember__user=self.request.user) - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - collection_uid = serializer.validated_data.get("collection", {}).get("uid") - - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - - if request.user == serializer.validated_data.get("user"): - content = {"code": "self_invite", "detail": "Inviting yourself is invalid"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - if not permissions.is_collection_admin(collection, request.user): - raise PermissionDenied( - {"code": "admin_access_required", "detail": "User is not an admin of this collection"} - ) - - serializer.save(collection=collection) - - return Response({}, status=status.HTTP_201_CREATED) - - @action_decorator(detail=False, allowed_methods=["GET"], methods=["GET"]) - def fetch_user_profile(self, request, *args, **kwargs): - username = request.GET.get("username") - kwargs = {User.USERNAME_FIELD: username.lower()} - user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)), **kwargs) - user_info = get_object_or_404(UserInfo.objects.all(), owner=user) - serializer = UserInfoPubkeySerializer(user_info) - return Response(serializer.data) - - -class InvitationIncomingViewSet(InvitationBaseViewSet): - allowed_methods = ["GET", "DELETE"] - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(user=self.request.user) - - @action_decorator(detail=True, allowed_methods=["POST"], methods=["POST"]) - def accept(self, request, invitation_uid=None, *args, **kwargs): - invitation = get_object_or_404(self.get_queryset(), uid=invitation_uid) - context = self.get_serializer_context() - context.update({"invitation": invitation}) - - serializer = InvitationAcceptSerializer(data=request.data, context=context) - serializer.is_valid(raise_exception=True) - serializer.save() - return Response(status=status.HTTP_201_CREATED) - - -class AuthenticationViewSet(viewsets.ViewSet): - allowed_methods = ["POST"] - authentication_classes = BaseViewSet.authentication_classes - renderer_classes = BaseViewSet.renderer_classes - parser_classes = BaseViewSet.parser_classes - - def get_encryption_key(self, salt): - key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) - return nacl.hash.blake2b( - b"", - key=key, - salt=salt[: nacl.hash.BLAKE2B_SALTBYTES], - person=b"etebase-auth", - encoder=nacl.encoding.RawEncoder, - ) - - def get_queryset(self): - return get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) - - def get_serializer_context(self): - return {"request": self.request, "format": self.format_kwarg, "view": self} - - def login_response_data(self, user): - return { - "token": AuthToken.objects.create(user=user).key, - "user": UserSerializer(user).data, - } - - def list(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - @action_decorator(detail=False, methods=["POST"]) - def signup(self, request, *args, **kwargs): - serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) - serializer.is_valid(raise_exception=True) - user = serializer.save() - - user_signed_up.send(sender=user.__class__, request=request, user=user) - - data = self.login_response_data(user) - return Response(data, status=status.HTTP_201_CREATED) - - def get_login_user(self, username): - kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} - try: - user = self.get_queryset().get(**kwargs) - if not hasattr(user, "userinfo"): - raise AuthenticationFailed({"code": "user_not_init", "detail": "User not properly init"}) - return user - except User.DoesNotExist: - raise AuthenticationFailed({"code": "user_not_found", "detail": "User not found"}) - - def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): - from datetime import datetime - - username = validated_data.get("username") - user = self.get_login_user(username) - host = validated_data["host"] - challenge = validated_data["challenge"] - action = validated_data["action"] - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = msgpack_decode(box.decrypt(challenge)) - now = int(datetime.now().timestamp()) - if action != expected_action: - content = {"code": "wrong_action", "detail": 'Expected "{}" but got something else'.format(expected_action)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - content = {"code": "challenge_expired", "detail": "Login challange has expired"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif challenge_data["userId"] != user.id: - content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif not settings.DEBUG and host.split(":", 1)[0] != request.get_host().split(":", 1)[0]: - detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) - content = {"code": "wrong_host", "detail": detail} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) - - try: - verify_key.verify(response_raw, signature) - except nacl.exceptions.BadSignatureError: - return Response( - {"code": "login_bad_signature", "detail": "Wrong password for user."}, - status=status.HTTP_401_UNAUTHORIZED, - ) - - return None - - @action_decorator(detail=False, methods=["GET"]) - def is_etebase(self, request, *args, **kwargs): - return Response({}, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"]) - def login_challenge(self, request, *args, **kwargs): - from datetime import datetime - - serializer = AuthenticationLoginChallengeSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - username = serializer.validated_data.get("username") - user = self.get_login_user(username) - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = { - "timestamp": int(datetime.now().timestamp()), - "userId": user.id, - } - challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) - - ret = { - "salt": salt, - "challenge": challenge, - "version": user.userinfo.version, - } - return Response(ret, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"]) - def login(self, request, *args, **kwargs): - outer_serializer = AuthenticationLoginSerializer(data=request.data) - outer_serializer.is_valid(raise_exception=True) - - response_raw = outer_serializer.validated_data["response"] - response = msgpack_decode(response_raw) - signature = outer_serializer.validated_data["signature"] - - context = {"host": request.get_host()} - serializer = AuthenticationLoginInnerSerializer(data=response, context=context) - serializer.is_valid(raise_exception=True) - - bad_login_response = self.validate_login_request( - request, serializer.validated_data, response_raw, signature, "login" - ) - if bad_login_response is not None: - return bad_login_response - - username = serializer.validated_data.get("username") - user = self.get_login_user(username) - - data = self.login_response_data(user) - - user_logged_in.send(sender=user.__class__, request=request, user=user) - - return Response(data, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"], permission_classes=[IsAuthenticated]) - def logout(self, request, *args, **kwargs): - request.auth.delete() - user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) - return Response(status=status.HTTP_204_NO_CONTENT) - - @action_decorator(detail=False, methods=["POST"], permission_classes=BaseViewSet.permission_classes) - def change_password(self, request, *args, **kwargs): - outer_serializer = AuthenticationLoginSerializer(data=request.data) - outer_serializer.is_valid(raise_exception=True) - - response_raw = outer_serializer.validated_data["response"] - response = msgpack_decode(response_raw) - signature = outer_serializer.validated_data["signature"] - - context = {"host": request.get_host()} - serializer = AuthenticationChangePasswordInnerSerializer(request.user.userinfo, data=response, context=context) - serializer.is_valid(raise_exception=True) - - bad_login_response = self.validate_login_request( - request, serializer.validated_data, response_raw, signature, "changePassword" - ) - if bad_login_response is not None: - return bad_login_response - - serializer.save() - - return Response({}, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"], permission_classes=[IsAuthenticated]) - def dashboard_url(self, request, *args, **kwargs): - get_dashboard_url = app_settings.DASHBOARD_URL_FUNC - if get_dashboard_url is None: - raise EtebaseValidationError( - "not_supported", "This server doesn't have a user dashboard.", status_code=status.HTTP_400_BAD_REQUEST - ) - - ret = { - "url": get_dashboard_url(request, *args, **kwargs), - } - return Response(ret) - - -class TestAuthenticationViewSet(viewsets.ViewSet): - allowed_methods = ["POST"] - renderer_classes = BaseViewSet.renderer_classes - parser_classes = BaseViewSet.parser_classes - - def get_serializer_context(self): - return {"request": self.request, "format": self.format_kwarg, "view": self} - - def list(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - @action_decorator(detail=False, methods=["POST"]) - def reset(self, request, *args, **kwargs): - # Only run when in DEBUG mode! It's only used for tests - if not settings.DEBUG: - return HttpResponseBadRequest("Only allowed in debug mode.") - - with transaction.atomic(): - user_queryset = get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) - user = get_object_or_404(user_queryset, username=request.data.get("user").get("username")) - - # Only allow test users for extra safety - if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): - return HttpResponseBadRequest("Endpoint not allowed for user.") - - if hasattr(user, "userinfo"): - user.userinfo.delete() - - serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) - serializer.is_valid(raise_exception=True) - serializer.save() - - # Delete all of the journal data for this user for a clear test env - user.collection_set.all().delete() - user.collectionmember_set.all().delete() - user.incoming_invitations.all().delete() - - # FIXME: also delete chunk files!!! - - return HttpResponse() diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index a13cc51..1c262c8 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -18,16 +18,14 @@ from fastapi import APIRouter, Depends, status, Request from fastapi.security import APIKeyHeader from django_etebase import app_settings, models -from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext -from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute -from .utils import BaseModel, permission_responses +from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -293,7 +291,7 @@ def signup_save(data: SignupIn, request: Request) -> User: context=CallbackContext(request.path_params), ) instance.full_clean() - except EtebaseValidationError as e: + except HttpError as e: raise e except django_exceptions.ValidationError as e: transform_validation_error("user", e) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 165163a..3473fa0 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,5 +1,6 @@ import dataclasses import typing as t +import msgpack from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel @@ -44,6 +45,14 @@ def is_collection_admin(collection, user): return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) +def msgpack_encode(content): + return msgpack.packb(content, use_bin_type=True) + + +def msgpack_decode(content): + return msgpack.unpackb(content, raw=False) + + PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 46ad3c9..5d57ec0 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -53,8 +53,6 @@ INSTALLED_APPS = [ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "corsheaders", - "rest_framework", "myauth.apps.MyauthConfig", "django_etebase.apps.DjangoEtebaseConfig", "django_etebase.token_auth.apps.TokenAuthConfig", @@ -63,7 +61,6 @@ INSTALLED_APPS = [ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", - "corsheaders.middleware.CorsMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", @@ -124,9 +121,6 @@ USE_L10N = True USE_TZ = True -# Cors -CORS_ORIGIN_ALLOW_ALL = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ diff --git a/etebase_server/urls.py b/etebase_server/urls.py index 443763d..7cf5a60 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,7 +1,7 @@ import os from django.conf import settings -from django.conf.urls import include, url +from django.conf.urls import url from django.contrib import admin from django.urls import path, re_path from django.views.generic import TemplateView @@ -9,15 +9,11 @@ from django.views.static import serve from django.contrib.staticfiles import finders urlpatterns = [ - url(r"^api/", include("django_etebase.urls")), url(r"^admin/", admin.site.urls), path("", TemplateView.as_view(template_name="success.html")), ] if settings.DEBUG: - urlpatterns += [ - url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), - ] def serve_static(request, path): filename = finders.find(path) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index ca8dd94..fee4a56 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,9 +1,5 @@ django -django-cors-headers -djangorestframework -drf-nested-routers msgpack -psycopg2-binary pynacl fastapi -uvicorn \ No newline at end of file +uvicorn diff --git a/requirements.txt b/requirements.txt index 3d19eaf..cfce456 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,14 +7,10 @@ asgiref==3.3.1 # via django cffi==1.14.4 # via pynacl click==7.1.2 # via uvicorn -django-cors-headers==3.6.0 # via -r requirements.in/base.txt -django==3.1.4 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers -djangorestframework==3.12.2 # via -r requirements.in/base.txt, drf-nested-routers -drf-nested-routers==0.92.5 # via -r requirements.in/base.txt +django==3.1.4 # via -r requirements.in/base.txt fastapi==0.63.0 # via -r requirements.in/base.txt h11==0.11.0 # via uvicorn msgpack==1.0.2 # via -r requirements.in/base.txt -psycopg2-binary==2.8.6 # via -r requirements.in/base.txt pycparser==2.20 # via cffi pydantic==1.7.3 # via fastapi pynacl==1.4.0 # via -r requirements.in/base.txt From 2e9caf66f960db6d1177183643e6a4e88e3eba1c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:04:07 +0200 Subject: [PATCH 106/170] Remove deprecated settings. --- django_etebase/app_settings.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index c1e8dc9..90225a6 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -31,17 +31,6 @@ class AppSettings: return getattr(settings, self.prefix + name, dflt) - @cached_property - def API_PERMISSIONS(self): # pylint: disable=invalid-name - """ - Deprecated. Do not use. - """ - perms = self._setting("API_PERMISSIONS", ("rest_framework.permissions.IsAuthenticated",)) - ret = [] - for perm in perms: - ret.append(self.import_from_str(perm)) - return ret - @cached_property def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name perms = self._setting("API_PERMISSIONS_READ", tuple()) @@ -58,23 +47,6 @@ class AppSettings: ret.append(self.import_from_str(perm)) return ret - @cached_property - def API_AUTHENTICATORS(self): # pylint: disable=invalid-name - """ - Deprecated. Do not use. - """ - perms = self._setting( - "API_AUTHENTICATORS", - ( - "rest_framework.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", - ), - ) - ret = [] - for perm in perms: - ret.append(self.import_from_str(perm)) - return ret - @cached_property def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None) From c918d3ed076a799e8e5bced6f48dce8f1de5d5e2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:26:44 +0200 Subject: [PATCH 107/170] Add base64 utils. --- etebase_fastapi/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 3473fa0..5e45db7 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,6 +1,7 @@ import dataclasses import typing as t import msgpack +import base64 from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel @@ -53,6 +54,15 @@ def msgpack_decode(content): return msgpack.unpackb(content, raw=False) +def b64encode(value): + return base64.urlsafe_b64encode(value).decode("ascii").strip("=") + + +def b64decode(data): + data += "=" * ((4 - len(data) % 4) % 4) + return base64.urlsafe_b64decode(data) + + PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] From 313dcf072119a52edd01bc514222185445a7edd7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:27:23 +0200 Subject: [PATCH 108/170] django_etebase utils: add optionl user to context. --- django_etebase/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 1c8654b..09028c4 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -15,6 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] + user: t.Optional[User] def get_user_queryset(queryset, context: CallbackContext): From b3c170e10d6581af2c64f9451ca7ff35e5409cd0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:28:42 +0200 Subject: [PATCH 109/170] fix getting dashboard URL. --- etebase_fastapi/authentication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 1c262c8..2f8395e 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -264,13 +264,12 @@ async def change_password(data: ChangePassword, request: Request, user: User = D @authentication_router.post("/dashboard_url/", responses=permission_responses) def dashboard_url(user: User = Depends(get_authenticated_user)): - # XXX-TOM get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: raise HttpError("not_supported", "This server doesn't have a user dashboard.") ret = { - "url": get_dashboard_url(request, *args, **kwargs), + "url": get_dashboard_url(CallbackContext(request.path_params, user=user)), } return ret From 65cd722616a5b7898ef559387463cdadce656482 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:27:23 +0200 Subject: [PATCH 110/170] django_etebase utils: add optionl user to context. --- django_etebase/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 09028c4..e46cbd0 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -15,7 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] - user: t.Optional[User] + user: t.Optional[User] = None def get_user_queryset(queryset, context: CallbackContext): From c1f171bde0ca3b1a908ae30f2fd24efd83914f54 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:47:41 +0200 Subject: [PATCH 111/170] Change how we create applications. --- etebase_fastapi/main.py | 42 +++++++++++++++++++++-------------------- etebase_server/asgi.py | 4 +++- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 2c10854..706081a 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -12,28 +12,30 @@ from .member import member_router from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse -app = FastAPI() -VERSION = "v1" -BASE_PATH = f"/api/{VERSION}" -COLLECTION_UID_MARKER = "{collection_uid}" -app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) -app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) -app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) -app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) -app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) -app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) +def create_application(prefix=""): + app = FastAPI() + VERSION = "v1" + BASE_PATH = f"{prefix}/api/{VERSION}" + COLLECTION_UID_MARKER = "{collection_uid}" + app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) + app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) + app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) + app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) + app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) + app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) -if settings.DEBUG: - from etebase_fastapi.test_reset_view import test_reset_view_router + if settings.DEBUG: + from etebase_fastapi.test_reset_view import test_reset_view_router - app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") + app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") -app.add_middleware( - CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] -) -app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) + app.add_middleware( + CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] + ) + app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) -@app.exception_handler(CustomHttpException) -async def custom_exception_handler(request: Request, exc: CustomHttpException): - return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + @app.exception_handler(CustomHttpException) + async def custom_exception_handler(request: Request, exc: CustomHttpException): + return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + return app diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py index 92fad1c..25dbf77 100644 --- a/etebase_server/asgi.py +++ b/etebase_server/asgi.py @@ -7,7 +7,9 @@ django_application = get_asgi_application() def create_application(): - from etebase_fastapi.main import app + from etebase_fastapi.main import create_application + + app = create_application() app.mount("/", django_application) From 50f89c48e27d0d6d5aecac22d491e53e130a0bba Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:07:18 +0200 Subject: [PATCH 112/170] Dashboard url: fix getting dashboard url. --- etebase_fastapi/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 2f8395e..04aec31 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -263,7 +263,7 @@ async def change_password(data: ChangePassword, request: Request, user: User = D @authentication_router.post("/dashboard_url/", responses=permission_responses) -def dashboard_url(user: User = Depends(get_authenticated_user)): +def dashboard_url(request: Request, user: User = Depends(get_authenticated_user)): get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: raise HttpError("not_supported", "This server doesn't have a user dashboard.") From ca7f2ec73cb559f0875e580b81a1e0f18b06c21a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:08:03 +0200 Subject: [PATCH 113/170] When converting from ORM convert binary fields to bytes. The problem is that some ORMs return memoryview which are more efficient but are not supported by pydantic at the moment. --- etebase_fastapi/authentication.py | 9 +++++---- etebase_fastapi/collection.py | 6 +++--- etebase_fastapi/invitation.py | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 04aec31..2f5a2f1 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -25,7 +25,7 @@ from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute -from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -63,7 +63,7 @@ class UserOut(BaseModel): @classmethod def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": - return cls(pubkey=obj.userinfo.pubkey, encryptedContent=obj.userinfo.encryptedContent) + return cls(pubkey=bytes(obj.userinfo.pubkey), encryptedContent=bytes(obj.userinfo.encryptedContent)) class LoginOut(BaseModel): @@ -228,14 +228,15 @@ async def is_etebase(): @authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) async def login_challenge(user: User = Depends(get_login_user)): - enc_key = get_encryption_key(user.userinfo.salt) + salt = bytes(user.userinfo.salt) + enc_key = get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) challenge_data = { "timestamp": int(datetime.now().timestamp()), "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) + return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version) @authentication_router.post("/login/", response_model=LoginOut) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index e6c10c3..74730ff 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -58,7 +58,7 @@ class CollectionItemRevisionInOut(BaseModel): chunks.append((chunk_obj.uid, f.read())) else: chunks.append((chunk_obj.uid,)) - return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=chunks) + return cls(uid=obj.uid, meta=bytes(obj.meta), deleted=obj.deleted, chunks=chunks) class CollectionItemCommon(BaseModel): @@ -103,8 +103,8 @@ class CollectionOut(CollectionCommon): member: models.CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( - collectionType=collection_type and collection_type.uid, - collectionKey=member.encryptionKey, + collectionType=collection_type and bytes(collection_type.uid), + collectionKey=bytes(member.encryptionKey), accessLevel=member.accessLevel, stoken=obj.stoken, item=CollectionItemOut.from_orm_context(obj.main_item, context), diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 39460a9..9e731bc 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -32,6 +32,10 @@ class UserInfoOut(BaseModel): class Config: orm_mode = True + @classmethod + def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut": + return cls(pubkey=bytes(obj.pubkey)) + class CollectionInvitationAcceptIn(BaseModel): collectionType: bytes @@ -69,8 +73,8 @@ class CollectionInvitationOut(CollectionInvitationCommon): username=obj.user.username, collection=obj.collection.uid, fromUsername=obj.fromMember.user.username, - fromPubkey=obj.fromMember.user.userinfo.pubkey, - signedEncryptionKey=obj.signedEncryptionKey, + fromPubkey=bytes(obj.fromMember.user.userinfo.pubkey), + signedEncryptionKey=bytes(obj.signedEncryptionKey), ) From 59e30ed9884990f33408f85cd1c18666e61d5507 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:17:13 +0200 Subject: [PATCH 114/170] Signup and logout: make sync. --- etebase_fastapi/authentication.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 2f5a2f1..df5dc62 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -250,10 +250,9 @@ async def login(data: Login, request: Request): @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): - await sync_to_async(auth_data.token.delete)() - # XXX-TOM - await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) +def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): + auth_data.token.delete() + user_logged_out.send(sender=auth_data.user.__class__, request=None, user=auth_data.user) @authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) @@ -306,9 +305,8 @@ def signup_save(data: SignupIn, request: Request) -> User: @authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) -async def signup(data: SignupIn, request: Request): - user = await sync_to_async(signup_save)(data, request) - # XXX-TOM - data = await sync_to_async(LoginOut.from_orm)(user) - await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) +def signup(data: SignupIn, request: Request): + user = signup_save(data, request) + data = LoginOut.from_orm(user) + user_signed_up.send(sender=user.__class__, request=None, user=user) return data From 1bca435d740f1c279efa2113e1ec9d59edc995b8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:26:34 +0200 Subject: [PATCH 115/170] Workaround typing issue. --- etebase_fastapi/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 5e45db7..7280018 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -68,4 +68,7 @@ PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API response_model_dict = {"model": HttpErrorOut} -permission_responses = {401: response_model_dict, 403: response_model_dict} +permission_responses: t.Dict[t.Union[int, str], t.Dict[str, t.Any]] = { + 401: response_model_dict, + 403: response_model_dict, +} From 3e39aa88a15579090286cc8a859c6d0256652caa Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:27:29 +0200 Subject: [PATCH 116/170] Remove unused var. --- etebase_fastapi/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index df5dc62..9c66a7a 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -250,7 +250,7 @@ async def login(data: Login, request: Request): @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): +def logout(auth_data: AuthData = Depends(get_auth_data)): auth_data.token.delete() user_logged_out.send(sender=auth_data.user.__class__, request=None, user=auth_data.user) From c2a2e710c9c9cca7b1bcc9b4523fbe095904c26b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:38:00 +0200 Subject: [PATCH 117/170] Move common dependencies to their own file. --- etebase_fastapi/authentication.py | 57 +-------------------- etebase_fastapi/collection.py | 21 +------- etebase_fastapi/dependencies.py | 82 +++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 etebase_fastapi/dependencies.py diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 9c66a7a..eb54f68 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -1,4 +1,3 @@ -import dataclasses import typing as t from datetime import datetime from functools import cached_property @@ -13,33 +12,22 @@ from django.conf import settings from django.contrib.auth import get_user_model, user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction -from django.utils import timezone from fastapi import APIRouter, Depends, status, Request -from fastapi.security import APIKeyHeader from django_etebase import app_settings, models +from django_etebase.token_auth.models import AuthToken from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up -from django_etebase.token_auth.models import AuthToken -from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from .dependencies import AuthData, get_auth_data, get_authenticated_user User = get_user_model() -token_scheme = APIKeyHeader(name="Authorization") -AUTO_REFRESH = True -MIN_REFRESH_INTERVAL = 60 authentication_router = APIRouter(route_class=MsgpackRoute) -@dataclasses.dataclass(frozen=True) -class AuthData: - user: User - token: AuthToken - - class LoginChallengeIn(BaseModel): username: str @@ -115,47 +103,6 @@ class SignupIn(BaseModel): encryptedContent: bytes -def __renew_token(auth_token: AuthToken): - current_expiry = auth_token.expiry - new_expiry = get_default_expiry() - # Throttle refreshing of token to avoid db writes - delta = (new_expiry - current_expiry).total_seconds() - if delta > MIN_REFRESH_INTERVAL: - auth_token.expiry = new_expiry - auth_token.save(update_fields=("expiry",)) - - -@sync_to_async -def __get_authenticated_user(api_token: str): - api_token = api_token.split()[1] - try: - token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) - except AuthToken.DoesNotExist: - raise AuthenticationFailed(detail="Invalid token.") - if not token.user.is_active: - raise AuthenticationFailed(detail="User inactive or deleted.") - - if token.expiry is not None: - if token.expiry < timezone.now(): - token.delete() - raise AuthenticationFailed(detail="Invalid token.") - - if AUTO_REFRESH: - __renew_token(token) - - return token.user, token - - -async def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: - user, token = await __get_authenticated_user(api_token) - return AuthData(user, token) - - -async def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: - user, token = await __get_authenticated_user(api_token) - return user - - @sync_to_async def __get_login_user(username: str) -> User: kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 74730ff..a60c7f0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -5,8 +5,7 @@ from django.contrib.auth import get_user_model from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile from django.db import transaction -from django.db.models import Q -from django.db.models import QuerySet +from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status from django_etebase import models @@ -25,12 +24,11 @@ from .utils import ( PERMISSIONS_READ, PERMISSIONS_READWRITE, ) +from .dependencies import get_collection_queryset, get_item_queryset, get_collection User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) -default_queryset: QuerySet = models.Collection.objects.all() -default_item_queryset: QuerySet = models.CollectionItem.objects.all() class ListMulti(BaseModel): @@ -203,21 +201,6 @@ def collection_list_common( return ret -def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: - return default_queryset.filter(members__user=user) - - -def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: - return get_object_or_404(queryset, uid=collection_uid) - - -def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: - # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) - - return queryset - - # permissions diff --git a/etebase_fastapi/dependencies.py b/etebase_fastapi/dependencies.py new file mode 100644 index 0000000..ddb9b3b --- /dev/null +++ b/etebase_fastapi/dependencies.py @@ -0,0 +1,82 @@ +import dataclasses + +from fastapi import Depends +from fastapi.security import APIKeyHeader + +from django.contrib.auth import get_user_model +from django.utils import timezone +from django.db.models import QuerySet + +from django_etebase import models +from django_etebase.token_auth.models import AuthToken, get_default_expiry +from .exceptions import AuthenticationFailed +from .utils import get_object_or_404 + + +User = get_user_model() +token_scheme = APIKeyHeader(name="Authorization") +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 + + +@dataclasses.dataclass(frozen=True) +class AuthData: + user: User + token: AuthToken + + +def __renew_token(auth_token: AuthToken): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=("expiry",)) + + +def __get_authenticated_user(api_token: str): + api_token = api_token.split()[1] + try: + token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) + except AuthToken.DoesNotExist: + raise AuthenticationFailed(detail="Invalid token.") + if not token.user.is_active: + raise AuthenticationFailed(detail="User inactive or deleted.") + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise AuthenticationFailed(detail="Invalid token.") + + if AUTO_REFRESH: + __renew_token(token) + + return token.user, token + + +def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: + user, token = __get_authenticated_user(api_token) + return AuthData(user, token) + + +def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: + user, _ = __get_authenticated_user(api_token) + return user + + +def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: + default_queryset: QuerySet = models.Collection.objects.all() + return default_queryset.filter(members__user=user) + + +def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: + return get_object_or_404(queryset, uid=collection_uid) + + +def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + default_item_queryset: QuerySet = models.CollectionItem.objects.all() + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) + + return queryset From 586b015eb78a01d40a13c9acad81c274b5be6380 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:23:01 +0200 Subject: [PATCH 118/170] Login: also return username and email upon login. --- etebase_fastapi/authentication.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index eb54f68..a211e9b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -46,12 +46,19 @@ class LoginResponse(BaseModel): class UserOut(BaseModel): + username: str + email: str pubkey: bytes encryptedContent: bytes @classmethod def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": - return cls(pubkey=bytes(obj.userinfo.pubkey), encryptedContent=bytes(obj.userinfo.encryptedContent)) + return cls( + username=obj.username, + email=obj.email, + pubkey=bytes(obj.userinfo.pubkey), + encryptedContent=bytes(obj.userinfo.encryptedContent), + ) class LoginOut(BaseModel): From 151bec0d9e683a0c8da93796a7468c999c6aecd0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:44:13 +0200 Subject: [PATCH 119/170] Fix type error. --- etebase_fastapi/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index a211e9b..559a60a 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -261,6 +261,6 @@ def signup_save(data: SignupIn, request: Request) -> User: @authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) def signup(data: SignupIn, request: Request): user = signup_save(data, request) - data = LoginOut.from_orm(user) + ret = LoginOut.from_orm(user) user_signed_up.send(sender=user.__class__, request=None, user=user) - return data + return ret From fa0979dce17c779321325c6ef173addb47af52f0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:57:09 +0200 Subject: [PATCH 120/170] Test reset: clean reset function. --- etebase_fastapi/test_reset_view.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index f21fd84..3075290 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -2,31 +2,33 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 -from fastapi import APIRouter, Request, Response, status +from fastapi import APIRouter, Request, status from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute +from etebase_fastapi.exceptions import HttpError test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) User = get_user_model() -@test_reset_view_router.post("/reset/") +@test_reset_view_router.post("/reset/", status_code=status.HTTP_204_NO_CONTENT) def reset(data: SignupIn, request: Request): # Only run when in DEBUG mode! It's only used for tests if not settings.DEBUG: - return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError(code="generic", detail="Only allowed in debug mode.") with transaction.atomic(): user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) user = get_object_or_404(user_queryset, username=data.user.username) # Only allow test users for extra safety if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): - return Response("Endpoint not allowed for user.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError(code="generic", detail="Endpoint not allowed for user.") if hasattr(user, "userinfo"): user.userinfo.delete() + signup_save(data, request) # Delete all of the journal data for this user for a clear test env user.collection_set.all().delete() @@ -34,5 +36,3 @@ def reset(data: SignupIn, request: Request): user.incoming_invitations.all().delete() # FIXME: also delete chunk files!!! - - return Response(status_code=status.HTTP_204_NO_CONTENT) From 10ff303b754f8147acdc4f1740d27bf3dbefa915 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 17:09:20 +0200 Subject: [PATCH 121/170] Fix formatting. --- etebase_fastapi/main.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 706081a..a55e2fa 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -12,6 +12,7 @@ from .member import member_router from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse + def create_application(prefix=""): app = FastAPI() VERSION = "v1" @@ -21,8 +22,12 @@ def create_application(prefix=""): app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) - app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) - app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) + app.include_router( + invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"] + ) + app.include_router( + invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"] + ) if settings.DEBUG: from etebase_fastapi.test_reset_view import test_reset_view_router @@ -30,12 +35,16 @@ def create_application(prefix=""): app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") app.add_middleware( - CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] + CORSMiddleware, + allow_origin_regex="https?://.*", + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) - @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + return app From 3d438b9591955e948c71c3737a534392b5b9a0ef Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 17:39:51 +0200 Subject: [PATCH 122/170] Cleanup validation errors. --- etebase_fastapi/collection.py | 41 +++++++++++++++++++++++++++-------- etebase_fastapi/exceptions.py | 19 ++++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index a60c7f0..6c7d9a4 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user -from .exceptions import HttpError, transform_validation_error, PermissionDenied +from .exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken from .utils import ( @@ -151,10 +151,11 @@ class ItemDepIn(BaseModel): item = models.CollectionItem.objects.get(uid=self.uid) etag = self.etag if item.etag != etag: - raise HttpError( + raise ValidationError( "wrong_etag", "Wrong etag. Expected {} got {}".format(item.etag, etag), status_code=status.HTTP_409_CONFLICT, + field=self.uid, ) @@ -164,8 +165,19 @@ class ItemBatchIn(BaseModel): def validate_db(self): if self.deps is not None: + errors: t.List[HttpError] = [] for dep in self.deps: - dep.validate_db() + try: + dep.validate_db() + except ValidationError as e: + errors.append(e) + if len(errors) > 0: + raise ValidationError( + code="dep_failed", + detail="Dependencies failed to validate", + errors=errors, + status_code=status.HTTP_409_CONFLICT, + ) @sync_to_async @@ -293,7 +305,7 @@ def _create(data: CollectionIn, user: User): try: instance.validate_unique() except django_exceptions.ValidationError: - raise HttpError( + raise ValidationError( "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT ) instance.save() @@ -353,10 +365,11 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance if validate_etag and cur_etag != etag: - raise HttpError( + raise ValidationError( "wrong_etag", "Wrong etag. Expected {} got {}".format(cur_etag, etag), status_code=status.HTTP_409_CONFLICT, + field=uid, ) if not created: @@ -426,12 +439,22 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid if stoken is not None and stoken != collection_object.stoken: raise HttpError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) - # XXX-TOM: make sure we return compatible errors data.validate_db() - for item in data.items: - item_create(item, collection_object, validate_etag) - return None + errors: t.List[HttpError] = [] + for item in data.items: + try: + item_create(item, collection_object, validate_etag) + except ValidationError as e: + errors.append(e) + + if len(errors) > 0: + raise ValidationError( + code="item_failed", + detail="Items failed to validate", + errors=errors, + status_code=status.HTTP_409_CONFLICT, + ) @item_router.get( diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index 2c1757c..72a3faf 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -9,12 +9,18 @@ class HttpErrorField(BaseModel): code: str detail: str + class Config: + orm_mode = True + class HttpErrorOut(BaseModel): code: str detail: str errors: t.Optional[t.List[HttpErrorField]] + class Config: + orm_mode = True + class CustomHttpException(Exception): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): @@ -73,6 +79,19 @@ class HttpError(CustomHttpException): return HttpErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() +class ValidationError(HttpError): + def __init__( + self, + code: str, + detail: str, + status_code: int = status.HTTP_400_BAD_REQUEST, + errors: t.Optional[t.List["HttpError"]] = None, + field: t.Optional[str] = None, + ): + self.field = field + super().__init__(code=code, detail=detail, errors=errors, status_code=status_code) + + def flatten_errors(field_name, errors) -> t.List[HttpError]: ret = [] if isinstance(errors, dict): From f7858a20b7c7165908240aaa2ada4278b17260db Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 17:46:20 +0200 Subject: [PATCH 123/170] Fix user creation. --- django_etebase/utils.py | 2 +- etebase_fastapi/authentication.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index e46cbd0..4d36a94 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -28,7 +28,7 @@ def get_user_queryset(queryset, context: CallbackContext): def create_user(context: CallbackContext, *args, **kwargs): custom_func = app_settings.CREATE_USER_FUNC if custom_func is not None: - return custom_func(*args, **kwargs) + return custom_func(context, *args, **kwargs) return User.objects.create_user(*args, **kwargs) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 559a60a..fe522f7 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -181,7 +181,7 @@ async def is_etebase(): @authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) -async def login_challenge(user: User = Depends(get_login_user)): +def login_challenge(user: User = Depends(get_login_user)): salt = bytes(user.userinfo.salt) enc_key = get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) @@ -238,10 +238,10 @@ def signup_save(data: SignupIn, request: Request) -> User: # Create the user and save the casing the user chose as the first name try: instance = create_user( + CallbackContext(request.path_params), **user_data.dict(), password=None, first_name=user_data.username, - context=CallbackContext(request.path_params), ) instance.full_clean() except HttpError as e: From dcf81aa9ceeca7b56f84f47028ef05eda2980406 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:17:41 +0200 Subject: [PATCH 124/170] Fix prefetch medium. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6c7d9a4..07780ae 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -55,7 +55,7 @@ class CollectionItemRevisionInOut(BaseModel): with open(chunk_obj.chunkFile.path, "rb") as f: chunks.append((chunk_obj.uid, f.read())) else: - chunks.append((chunk_obj.uid,)) + chunks.append((chunk_obj.uid, None)) return cls(uid=obj.uid, meta=bytes(obj.meta), deleted=obj.deleted, chunks=chunks) From 53b22602b28bf2582284495d736b2b1672d35a9b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:17:57 +0200 Subject: [PATCH 125/170] Implement chunk_update. --- etebase_fastapi/collection.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 07780ae..b3f8c5f 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -4,9 +4,9 @@ from asgiref.sync import sync_to_async from django.contrib.auth import get_user_model from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile -from django.db import transaction +from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet -from fastapi import APIRouter, Depends, status +from fastapi import APIRouter, Depends, status, Request from django_etebase import models from .authentication import get_authenticated_user @@ -544,28 +544,27 @@ def item_batch( # Chunks +@sync_to_async +def chunk_save(chunk_uid: str, collection: models.Collection, content_file: ContentFile): + chunk_obj = models.CollectionItemChunk(uid=chunk_uid, collection=collection) + chunk_obj.chunkFile.save("IGNORED", content_file) + chunk_obj.save() + return chunk_obj + + @item_router.put( "/item/{item_uid}/chunk/{chunk_uid}/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE], status_code=status.HTTP_201_CREATED, ) -def chunk_update( - limit: int = 50, - iterator: t.Optional[str] = None, - prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), +async def chunk_update( + request: Request, + chunk_uid: str, collection: models.Collection = Depends(get_collection), ): # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - - data = { - "uid": chunk_uid, - "chunkFile": request.data["file"], - } - - serializer = self.get_serializer_class()(data=data) - serializer.is_valid(raise_exception=True) + content_file = ContentFile(await request.body()) try: - serializer.save(collection=col) + await chunk_save(chunk_uid, collection, content_file) except IntegrityError: - return Response({"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT) + raise HttpError("chunk_exists", "Chunk already exists.", status_code=status.HTTP_409_CONFLICT) From c7f09d3fef2935b81ac3a9232a6b88215b73300d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:25:06 +0200 Subject: [PATCH 126/170] implement chunk_download. --- etebase_fastapi/collection.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index b3f8c5f..c5b0801 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,6 +7,7 @@ from django.core.files.base import ContentFile from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request +from fastapi.responses import FileResponse from django_etebase import models from .authentication import get_authenticated_user @@ -568,3 +569,17 @@ async def chunk_update( await chunk_save(chunk_uid, collection, content_file) except IntegrityError: raise HttpError("chunk_exists", "Chunk already exists.", status_code=status.HTTP_409_CONFLICT) + + +@item_router.get( + "/item/{item_uid}/chunk/{chunk_uid}/download/", + dependencies=PERMISSIONS_READ, +) +def chunk_download( + chunk_uid: str, + collection: models.Collection = Depends(get_collection), +): + chunk = get_object_or_404(collection.chunks, uid=chunk_uid) + + filename = chunk.chunkFile.path + return FileResponse(filename, media_type="application/octet-stream") From f0a8689712601ca2ac5b98b5c3c4aad4f6d58808 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:44:55 +0200 Subject: [PATCH 127/170] Implement sendfile. --- .../sendfile/backends/development.py | 17 ------ django_etebase/sendfile/backends/mod_wsgi.py | 17 ------ django_etebase/sendfile/backends/nginx.py | 12 ---- django_etebase/sendfile/backends/simple.py | 60 ------------------- django_etebase/sendfile/backends/xsendfile.py | 9 --- etebase_fastapi/collection.py | 4 +- .../sendfile/LICENSE | 0 .../sendfile/README.md | 0 .../sendfile/__init__.py | 0 .../sendfile/backends/__init__.py | 0 etebase_fastapi/sendfile/backends/mod_wsgi.py | 9 +++ etebase_fastapi/sendfile/backends/nginx.py | 9 +++ etebase_fastapi/sendfile/backends/simple.py | 12 ++++ .../sendfile/backends/xsendfile.py | 6 ++ .../sendfile/utils.py | 15 ++--- 15 files changed, 46 insertions(+), 124 deletions(-) delete mode 100644 django_etebase/sendfile/backends/development.py delete mode 100644 django_etebase/sendfile/backends/mod_wsgi.py delete mode 100644 django_etebase/sendfile/backends/nginx.py delete mode 100644 django_etebase/sendfile/backends/simple.py delete mode 100644 django_etebase/sendfile/backends/xsendfile.py rename {django_etebase => etebase_fastapi}/sendfile/LICENSE (100%) rename {django_etebase => etebase_fastapi}/sendfile/README.md (100%) rename {django_etebase => etebase_fastapi}/sendfile/__init__.py (100%) rename {django_etebase => etebase_fastapi}/sendfile/backends/__init__.py (100%) create mode 100644 etebase_fastapi/sendfile/backends/mod_wsgi.py create mode 100644 etebase_fastapi/sendfile/backends/nginx.py create mode 100644 etebase_fastapi/sendfile/backends/simple.py create mode 100644 etebase_fastapi/sendfile/backends/xsendfile.py rename {django_etebase => etebase_fastapi}/sendfile/utils.py (81%) diff --git a/django_etebase/sendfile/backends/development.py b/django_etebase/sendfile/backends/development.py deleted file mode 100644 index d321932..0000000 --- a/django_etebase/sendfile/backends/development.py +++ /dev/null @@ -1,17 +0,0 @@ -import os.path - -from django.views.static import serve - - -def sendfile(request, filename, **kwargs): - """ - Send file using Django dev static file server. - - .. warning:: - - Do not use in production. This is only to be used when developing and - is provided for convenience only - """ - dirname = os.path.dirname(filename) - basename = os.path.basename(filename) - return serve(request, basename, dirname) diff --git a/django_etebase/sendfile/backends/mod_wsgi.py b/django_etebase/sendfile/backends/mod_wsgi.py deleted file mode 100644 index 07ba3f1..0000000 --- a/django_etebase/sendfile/backends/mod_wsgi.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import absolute_import - -from django.http import HttpResponse - -from ..utils import _convert_file_to_url - - -def sendfile(request, filename, **kwargs): - response = HttpResponse() - response['Location'] = _convert_file_to_url(filename) - # need to destroy get_host() to stop django - # rewriting our location to include http, so that - # mod_wsgi is able to do the internal redirect - request.get_host = lambda: '' - request.build_absolute_uri = lambda location: location - - return response diff --git a/django_etebase/sendfile/backends/nginx.py b/django_etebase/sendfile/backends/nginx.py deleted file mode 100644 index 8764309..0000000 --- a/django_etebase/sendfile/backends/nginx.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import absolute_import - -from django.http import HttpResponse - -from ..utils import _convert_file_to_url - - -def sendfile(request, filename, **kwargs): - response = HttpResponse() - response['X-Accel-Redirect'] = _convert_file_to_url(filename) - - return response diff --git a/django_etebase/sendfile/backends/simple.py b/django_etebase/sendfile/backends/simple.py deleted file mode 100644 index 0549b20..0000000 --- a/django_etebase/sendfile/backends/simple.py +++ /dev/null @@ -1,60 +0,0 @@ -from email.utils import mktime_tz, parsedate_tz -import re - -from django.core.files.base import File -from django.http import HttpResponse, HttpResponseNotModified -from django.utils.http import http_date - - -def sendfile(request, filepath, **kwargs): - '''Use the SENDFILE_ROOT value composed with the path arrived as argument - to build an absolute path with which resolve and return the file contents. - - If the path points to a file out of the root directory (should cover both - situations with '..' and symlinks) then a 404 is raised. - ''' - statobj = filepath.stat() - - # Respect the If-Modified-Since header. - if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), - statobj.st_mtime, statobj.st_size): - return HttpResponseNotModified() - - with File(filepath.open('rb')) as f: - response = HttpResponse(f.chunks()) - - response["Last-Modified"] = http_date(statobj.st_mtime) - return response - - -def was_modified_since(header=None, mtime=0, size=0): - """ - Was something modified since the user last downloaded it? - - header - This is the value of the If-Modified-Since header. If this is None, - I'll just return True. - - mtime - This is the modification time of the item we're talking about. - - size - This is the size of the item we're talking about. - """ - try: - if header is None: - raise ValueError - matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, - re.IGNORECASE) - header_date = parsedate_tz(matches.group(1)) - if header_date is None: - raise ValueError - header_mtime = mktime_tz(header_date) - header_len = matches.group(3) - if header_len and int(header_len) != size: - raise ValueError - if mtime > header_mtime: - raise ValueError - except (AttributeError, ValueError, OverflowError): - return True - return False diff --git a/django_etebase/sendfile/backends/xsendfile.py b/django_etebase/sendfile/backends/xsendfile.py deleted file mode 100644 index 74993ee..0000000 --- a/django_etebase/sendfile/backends/xsendfile.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.http import HttpResponse - - -def sendfile(request, filename, **kwargs): - filename = str(filename) - response = HttpResponse() - response['X-Sendfile'] = filename - - return response diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index c5b0801..3b672cc 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,7 +7,6 @@ from django.core.files.base import ContentFile from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request -from fastapi.responses import FileResponse from django_etebase import models from .authentication import get_authenticated_user @@ -26,6 +25,7 @@ from .utils import ( PERMISSIONS_READWRITE, ) from .dependencies import get_collection_queryset, get_item_queryset, get_collection +from .sendfile import sendfile User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -582,4 +582,4 @@ def chunk_download( chunk = get_object_or_404(collection.chunks, uid=chunk_uid) filename = chunk.chunkFile.path - return FileResponse(filename, media_type="application/octet-stream") + return sendfile(filename) diff --git a/django_etebase/sendfile/LICENSE b/etebase_fastapi/sendfile/LICENSE similarity index 100% rename from django_etebase/sendfile/LICENSE rename to etebase_fastapi/sendfile/LICENSE diff --git a/django_etebase/sendfile/README.md b/etebase_fastapi/sendfile/README.md similarity index 100% rename from django_etebase/sendfile/README.md rename to etebase_fastapi/sendfile/README.md diff --git a/django_etebase/sendfile/__init__.py b/etebase_fastapi/sendfile/__init__.py similarity index 100% rename from django_etebase/sendfile/__init__.py rename to etebase_fastapi/sendfile/__init__.py diff --git a/django_etebase/sendfile/backends/__init__.py b/etebase_fastapi/sendfile/backends/__init__.py similarity index 100% rename from django_etebase/sendfile/backends/__init__.py rename to etebase_fastapi/sendfile/backends/__init__.py diff --git a/etebase_fastapi/sendfile/backends/mod_wsgi.py b/etebase_fastapi/sendfile/backends/mod_wsgi.py new file mode 100644 index 0000000..b8fc6c0 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/mod_wsgi.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from fastapi import Response + +from ..utils import _convert_file_to_url + + +def sendfile(filename, **kwargs): + return Response(headers={"Location": _convert_file_to_url(filename)}) diff --git a/etebase_fastapi/sendfile/backends/nginx.py b/etebase_fastapi/sendfile/backends/nginx.py new file mode 100644 index 0000000..b22e0d0 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/nginx.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from fastapi import Response + +from ..utils import _convert_file_to_url + + +def sendfile(filename, **kwargs): + return Response(headers={"X-Accel-Redirect": _convert_file_to_url(filename)}) diff --git a/etebase_fastapi/sendfile/backends/simple.py b/etebase_fastapi/sendfile/backends/simple.py new file mode 100644 index 0000000..f3a3548 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/simple.py @@ -0,0 +1,12 @@ +from fastapi.responses import FileResponse + + +def sendfile(filename, mimetype, **kwargs): + """Use the SENDFILE_ROOT value composed with the path arrived as argument + to build an absolute path with which resolve and return the file contents. + + If the path points to a file out of the root directory (should cover both + situations with '..' and symlinks) then a 404 is raised. + """ + + return FileResponse(filename, media_type=mimetype) diff --git a/etebase_fastapi/sendfile/backends/xsendfile.py b/etebase_fastapi/sendfile/backends/xsendfile.py new file mode 100644 index 0000000..530f6a1 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/xsendfile.py @@ -0,0 +1,6 @@ +from fastapi import Response + + +def sendfile(filename, **kwargs): + filename = str(filename) + return Response(headers={"X-Sendfile": filename}) diff --git a/django_etebase/sendfile/utils.py b/etebase_fastapi/sendfile/utils.py similarity index 81% rename from django_etebase/sendfile/utils.py rename to etebase_fastapi/sendfile/utils.py index 97c06d7..7c8b1f2 100644 --- a/django_etebase/sendfile/utils.py +++ b/etebase_fastapi/sendfile/utils.py @@ -4,9 +4,11 @@ from pathlib import Path, PurePath from urllib.parse import quote import logging +from fastapi import status +from ..exceptions import HttpError + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.http import Http404 logger = logging.getLogger(__name__) @@ -54,12 +56,12 @@ def _sanitize_path(filepath): try: filepath_abs.relative_to(path_root) except ValueError: - raise Http404("{} wrt {} is impossible".format(filepath_abs, path_root)) + raise HttpError("generic", "{} wrt {} is impossible".format(filepath_abs, path_root), status_code=status.HTTP_404_NOT_FOUND) return filepath_abs -def sendfile(request, filename, mimetype="application/octet-stream", encoding=None): +def sendfile(filename, mimetype="application/octet-stream", encoding=None): """ Create a response to send file using backend configured in ``SENDFILE_BACKEND`` @@ -75,11 +77,10 @@ def sendfile(request, filename, mimetype="application/octet-stream", encoding=No _sendfile = _get_sendfile() if not filepath_obj.exists(): - raise Http404('"%s" does not exist' % filepath_obj) + raise HttpError("does_not_exist", '"%s" does not exist' % filepath_obj, status_code=status.HTTP_404_NOT_FOUND) - response = _sendfile(request, filepath_obj, mimetype=mimetype) + response = _sendfile(filepath_obj, mimetype=mimetype) - response["Content-length"] = filepath_obj.stat().st_size - response["Content-Type"] = mimetype + response.headers["Content-Type"] = mimetype return response From 7714148807a183ef4eb820232d4014662441372a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:49:05 +0200 Subject: [PATCH 128/170] Use ValidationError when appropriate. --- etebase_fastapi/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3b672cc..3e4d3e8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -285,7 +285,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) chunk_obj.save() else: - raise HttpError("chunk_no_content", "Tried to create a new chunk without content") + raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") chunks_objs.append(chunk_obj) @@ -301,7 +301,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle def _create(data: CollectionIn, user: User): with transaction.atomic(): if data.item.etag is not None: - raise HttpError("bad_etag", "etag is not null") + raise ValidationError("bad_etag", "etag is not null") instance = models.Collection(uid=data.item.uid, owner=user) try: instance.validate_unique() From a8b97e60d407659f363d3486cbde1f8ad5d7bdd7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 09:46:20 +0200 Subject: [PATCH 129/170] Docs: improve metadata. --- etebase_fastapi/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index a55e2fa..69303bf 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -14,7 +14,15 @@ from .msgpack import MsgpackResponse def create_application(prefix=""): - app = FastAPI() + app = FastAPI( + title="Etebase", + description="The Etebase server API documentation", + externalDocs={ + "url": "https://docs.etebase.com", + "description": "Docs about the API specifications and clients.", + } + # FIXME: version="2.5.0", + ) VERSION = "v1" BASE_PATH = f"{prefix}/api/{VERSION}" COLLECTION_UID_MARKER = "{collection_uid}" From f67730f42d2a78acadf28db253bf76c0f3630ba2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 10:12:36 +0200 Subject: [PATCH 130/170] Support passing custom middlewares. --- etebase_fastapi/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 69303bf..534798e 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -13,7 +13,7 @@ from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse -def create_application(prefix=""): +def create_application(prefix="", middlewares=[]): app = FastAPI( title="Etebase", description="The Etebase server API documentation", @@ -51,6 +51,9 @@ def create_application(prefix=""): ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) + for middleware in middlewares: + app.add_middleware(middleware) + @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) From e13f26ec56f03321ad5f8471f4a5a33f4a6f0be0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 10:27:35 +0200 Subject: [PATCH 131/170] Fix handling of legacy accounts that don't have collection type. --- etebase_fastapi/collection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3e4d3e8..5c6e6b6 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -88,7 +88,8 @@ class CollectionItemIn(CollectionItemCommon): class CollectionCommon(BaseModel): - collectionType: bytes + # FIXME: remove optional once we finish collection-type-migration + collectionType: t.Optional[bytes] collectionKey: bytes From 794b5f398347985d7a354acc16e8fcd536600358 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 13:22:36 +0200 Subject: [PATCH 132/170] Fix many type errors. --- django_etebase/token_auth/models.py | 4 +-- django_etebase/utils.py | 6 ++-- etebase_fastapi/authentication.py | 27 ++++++++--------- etebase_fastapi/collection.py | 45 +++++++++++++++-------------- etebase_fastapi/dependencies.py | 10 +++---- etebase_fastapi/invitation.py | 15 +++++----- etebase_fastapi/member.py | 8 +++-- etebase_fastapi/msgpack.py | 6 ++-- etebase_fastapi/test_reset_view.py | 4 +-- etebase_fastapi/utils.py | 6 ++-- myauth/forms.py | 4 +-- myauth/models.py | 16 +++++++++- 12 files changed, 87 insertions(+), 64 deletions(-) diff --git a/django_etebase/token_auth/models.py b/django_etebase/token_auth/models.py index ac1efff..dd5ae87 100644 --- a/django_etebase/token_auth/models.py +++ b/django_etebase/token_auth/models.py @@ -1,9 +1,9 @@ -from django.contrib.auth import get_user_model from django.db import models from django.utils import timezone from django.utils.crypto import get_random_string +from myauth.models import get_typed_user_model -User = get_user_model() +User = get_typed_user_model() def generate_key(): diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 4d36a94..d812ae3 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -1,13 +1,13 @@ import typing as t from dataclasses import dataclass -from django.contrib.auth import get_user_model from django.core.exceptions import PermissionDenied +from myauth.models import UserType, get_typed_user_model from . import app_settings -User = get_user_model() +User = get_typed_user_model() @dataclass @@ -15,7 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] - user: t.Optional[User] = None + user: t.Optional[UserType] = None def get_user_queryset(queryset, context: CallbackContext): diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index fe522f7..064d2da 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -9,7 +9,7 @@ import nacl.secret import nacl.signing from asgiref.sync import sync_to_async from django.conf import settings -from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.contrib.auth import user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction from fastapi import APIRouter, Depends, status, Request @@ -19,12 +19,13 @@ from django_etebase.token_auth.models import AuthToken from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.utils import create_user, get_user_queryset, CallbackContext +from myauth.models import UserType, get_typed_user_model from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode from .dependencies import AuthData, get_auth_data, get_authenticated_user -User = get_user_model() +User = get_typed_user_model() authentication_router = APIRouter(route_class=MsgpackRoute) @@ -52,7 +53,7 @@ class UserOut(BaseModel): encryptedContent: bytes @classmethod - def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": + def from_orm(cls: t.Type["UserOut"], obj: UserType) -> "UserOut": return cls( username=obj.username, email=obj.email, @@ -66,7 +67,7 @@ class LoginOut(BaseModel): user: UserOut @classmethod - def from_orm(cls: t.Type["LoginOut"], obj: User) -> "LoginOut": + def from_orm(cls: t.Type["LoginOut"], obj: UserType) -> "LoginOut": token = AuthToken.objects.create(user=obj).key user = UserOut.from_orm(obj) return cls(token=token, user=user) @@ -111,7 +112,7 @@ class SignupIn(BaseModel): @sync_to_async -def __get_login_user(username: str) -> User: +def __get_login_user(username: str) -> UserType: kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} try: user = User.objects.get(**kwargs) @@ -122,7 +123,7 @@ def __get_login_user(username: str) -> User: raise AuthenticationFailed(code="user_not_found", detail="User not found") -async def get_login_user(challenge: LoginChallengeIn) -> User: +async def get_login_user(challenge: LoginChallengeIn) -> UserType: user = await __get_login_user(challenge.username) return user @@ -138,7 +139,7 @@ def get_encryption_key(salt): ) -def save_changed_password(data: ChangePassword, user: User): +def save_changed_password(data: ChangePassword, user: UserType): response_data = data.response_data user_info: UserInfo = user.userinfo user_info.loginPubkey = response_data.loginPubkey @@ -150,7 +151,7 @@ def save_changed_password(data: ChangePassword, user: User): def validate_login_request( validated_data: LoginResponse, challenge_sent_to_user: Authentication, - user: User, + user: UserType, expected_action: str, host_from_request: str, ): @@ -159,7 +160,7 @@ def validate_login_request( challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) now = int(datetime.now().timestamp()) if validated_data.action != expected_action: - raise HttpError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') + raise HttpError("wrong_action", f'Expected "{expected_action}" but got something else') elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: raise HttpError("challenge_expired", "Login challenge has expired") elif challenge_data["userId"] != user.id: @@ -181,7 +182,7 @@ async def is_etebase(): @authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) -def login_challenge(user: User = Depends(get_login_user)): +def login_challenge(user: UserType = Depends(get_login_user)): salt = bytes(user.userinfo.salt) enc_key = get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) @@ -210,14 +211,14 @@ def logout(auth_data: AuthData = Depends(get_auth_data)): @authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): +async def change_password(data: ChangePassword, request: Request, user: UserType = Depends(get_authenticated_user)): host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "changePassword", host) await sync_to_async(save_changed_password)(data, user) @authentication_router.post("/dashboard_url/", responses=permission_responses) -def dashboard_url(request: Request, user: User = Depends(get_authenticated_user)): +def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_user)): get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: raise HttpError("not_supported", "This server doesn't have a user dashboard.") @@ -228,7 +229,7 @@ def dashboard_url(request: Request, user: User = Depends(get_authenticated_user) return ret -def signup_save(data: SignupIn, request: Request) -> User: +def signup_save(data: SignupIn, request: Request) -> UserType: user_data = data.user with transaction.atomic(): try: diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 5c6e6b6..9e25b38 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,7 +1,6 @@ import typing as t from asgiref.sync import sync_to_async -from django.contrib.auth import get_user_model from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile from django.db import transaction, IntegrityError @@ -9,6 +8,7 @@ from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request from django_etebase import models +from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user from .exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError from .msgpack import MsgpackRoute @@ -27,7 +27,7 @@ from .utils import ( from .dependencies import get_collection_queryset, get_item_queryset, get_collection from .sendfile import sendfile -User = get_user_model() +User = get_typed_user_model collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -36,11 +36,14 @@ class ListMulti(BaseModel): collectionTypes: t.List[bytes] +ChunkType = t.Tuple[str, t.Optional[bytes]] + + class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.Tuple[str, t.Optional[bytes]]] + chunks: t.List[ChunkType] class Config: orm_mode = True @@ -49,7 +52,7 @@ class CollectionItemRevisionInOut(BaseModel): def from_orm_context( cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context ) -> "CollectionItemRevisionInOut": - chunks = [] + chunks: t.List[ChunkType] = [] for chunk_relation in obj.chunks_relation.all(): chunk_obj = chunk_relation.chunk if context.prefetch == "auto": @@ -185,7 +188,7 @@ class ItemBatchIn(BaseModel): @sync_to_async def collection_list_common( queryset: QuerySet, - user: User, + user: UserType, stoken: t.Optional[str], limit: int, prefetch: Prefetch, @@ -210,7 +213,7 @@ def collection_list_common( remed = remed_qs.values_list("collection__uid", flat=True) if len(remed) > 0: - ret.removedMemberships = [{"uid": x} for x in remed] + ret.removedMemberships = [RemovedMembershipOut(uid=x) for x in remed] return ret @@ -219,14 +222,14 @@ def collection_list_common( def verify_collection_admin( - collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) + collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) ): if not is_collection_admin(collection, user): raise PermissionDenied("admin_access_required", "Only collection admins can perform this operation.") def has_write_access( - collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) + collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) ): member = collection.members.get(user=user) if member.accessLevel == models.AccessLevels.READ_ONLY: @@ -247,7 +250,7 @@ async def list_multi( stoken: t.Optional[str] = None, limit: int = 50, queryset: QuerySet = Depends(get_collection_queryset), - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") @@ -263,7 +266,7 @@ async def collection_list( stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), queryset: QuerySet = Depends(get_collection_queryset), ): return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -299,7 +302,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle return revision -def _create(data: CollectionIn, user: User): +def _create(data: CollectionIn, user: UserType): with transaction.atomic(): if data.item.etag is not None: raise ValidationError("bad_etag", "etag is not null") @@ -335,14 +338,14 @@ def _create(data: CollectionIn, user: User): @collection_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) -async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): +async def create(data: CollectionIn, user: UserType = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) @collection_router.get("/{collection_uid}/", response_model=CollectionOut, dependencies=PERMISSIONS_READ) def collection_get( obj: models.Collection = Depends(get_collection), - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): return CollectionOut.from_orm_context(obj, Context(user, prefetch)) @@ -393,7 +396,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) @@ -403,7 +406,7 @@ def item_get( @sync_to_async def item_list_common( queryset: QuerySet, - user: User, + user: UserType, stoken: t.Optional[str], limit: int, prefetch: Prefetch, @@ -424,7 +427,7 @@ async def item_list( limit: int = 50, prefetch: Prefetch = PrefetchQuery, withCollection: bool = False, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), ): if not withCollection: queryset = queryset.filter(parent__isnull=True) @@ -433,7 +436,7 @@ async def item_list( return response -def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): +def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) @@ -467,7 +470,7 @@ def item_revisions( limit: int = 50, iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), items: QuerySet = Depends(get_item_queryset), ): item = get_object_or_404(items, uid=item_uid) @@ -501,7 +504,7 @@ def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), queryset: QuerySet = Depends(get_item_queryset), ): # FIXME: make configurable? @@ -531,14 +534,14 @@ def fetch_updates( @item_router.post("/item/transaction/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_transaction( - collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_batch( - collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) diff --git a/etebase_fastapi/dependencies.py b/etebase_fastapi/dependencies.py index ddb9b3b..fb9cec5 100644 --- a/etebase_fastapi/dependencies.py +++ b/etebase_fastapi/dependencies.py @@ -3,17 +3,17 @@ import dataclasses from fastapi import Depends from fastapi.security import APIKeyHeader -from django.contrib.auth import get_user_model from django.utils import timezone from django.db.models import QuerySet from django_etebase import models from django_etebase.token_auth.models import AuthToken, get_default_expiry +from myauth.models import UserType, get_typed_user_model from .exceptions import AuthenticationFailed from .utils import get_object_or_404 -User = get_user_model() +User = get_typed_user_model() token_scheme = APIKeyHeader(name="Authorization") AUTO_REFRESH = True MIN_REFRESH_INTERVAL = 60 @@ -21,7 +21,7 @@ MIN_REFRESH_INTERVAL = 60 @dataclasses.dataclass(frozen=True) class AuthData: - user: User + user: UserType token: AuthToken @@ -60,12 +60,12 @@ def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: return AuthData(user, token) -def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: +def get_authenticated_user(api_token: str = Depends(token_scheme)) -> UserType: user, _ = __get_authenticated_user(api_token) return user -def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: +def get_collection_queryset(user: UserType = Depends(get_authenticated_user)) -> QuerySet: default_queryset: QuerySet = models.Collection.objects.all() return default_queryset.filter(members__user=user) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 9e731bc..eb9f549 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -1,12 +1,12 @@ import typing as t -from django.contrib.auth import get_user_model from django.db import transaction, IntegrityError from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Request from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext +from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute @@ -20,7 +20,7 @@ from .utils import ( PERMISSIONS_READWRITE, ) -User = get_user_model() +User = get_typed_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionInvitation.objects.all() @@ -53,7 +53,8 @@ class CollectionInvitationCommon(BaseModel): class CollectionInvitationIn(CollectionInvitationCommon): def validate_db(self, context: Context): - if context.user.username == self.username.lower(): + user = context.user + if user is not None and (user.username == self.username.lower()): raise HttpError("no_self_invite", "Inviting yourself is not allowed") @@ -84,11 +85,11 @@ class InvitationListResponse(BaseModel): done: bool -def get_incoming_queryset(user: User = Depends(get_authenticated_user)): +def get_incoming_queryset(user: UserType = Depends(get_authenticated_user)): return default_queryset.filter(user=user) -def get_outgoing_queryset(user: User = Depends(get_authenticated_user)): +def get_outgoing_queryset(user: UserType = Depends(get_authenticated_user)): return default_queryset.filter(fromMember__user=user) @@ -183,7 +184,7 @@ def incoming_accept( def outgoing_create( data: CollectionInvitationIn, request: Request, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), ): collection = get_object_or_404(models.Collection.objects, uid=data.collection) to_user = get_object_or_404( @@ -231,7 +232,7 @@ def outgoing_delete( def outgoing_fetch_user_profile( username: str, request: Request, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), ): kwargs = {User.USERNAME_FIELD: username.lower()} user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 725d44b..22977ac 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -1,11 +1,11 @@ import typing as t -from django.contrib.auth import get_user_model from django.db import transaction from django.db.models import QuerySet from fastapi import APIRouter, Depends, status from django_etebase import models +from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user from .msgpack import MsgpackRoute from .utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE @@ -13,7 +13,7 @@ from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin -User = get_user_model() +User = get_typed_user_model() member_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionMember.objects.all() @@ -98,6 +98,8 @@ def member_patch( @member_router.post("/member/leave/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READ) -def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): +def member_leave( + user: UserType = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection) +): obj = get_object_or_404(collection.members, user=user) obj.revoke() diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index edffd7e..915e783 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -19,13 +19,15 @@ class MsgpackRequest(Request): class MsgpackResponse(Response): media_type = "application/msgpack" - def render(self, content: t.Optional[t.Any]) -> t.Optional[bytes]: + def render(self, content: t.Optional[t.Any]) -> bytes: if content is None: return b"" if isinstance(content, BaseModel): content = content.dict() - return msgpack.packb(content, use_bin_type=True) + ret = msgpack.packb(content, use_bin_type=True) + assert ret is not None + return ret class MsgpackRoute(APIRoute): diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index 3075290..e328875 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -1,5 +1,4 @@ from django.conf import settings -from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, status @@ -8,9 +7,10 @@ from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute from etebase_fastapi.exceptions import HttpError +from myauth.models import get_typed_user_model test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) -User = get_user_model() +User = get_typed_user_model() @test_reset_view_router.post("/reset/", status_code=status.HTTP_204_NO_CONTENT) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 7280018..c91c3ec 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -8,14 +8,14 @@ from pydantic import BaseModel as PyBaseModel from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist -from django.contrib.auth import get_user_model from django_etebase import app_settings from django_etebase.models import AccessLevels +from myauth.models import UserType, get_typed_user_model from .exceptions import HttpError, HttpErrorOut -User = get_user_model() +User = get_typed_user_model() Prefetch = t.Literal["auto", "medium"] PrefetchQuery = Query(default="auto") @@ -30,7 +30,7 @@ class BaseModel(PyBaseModel): @dataclasses.dataclass class Context: - user: t.Optional[User] + user: t.Optional[UserType] prefetch: t.Optional[Prefetch] diff --git a/myauth/forms.py b/myauth/forms.py index 7aacb9b..fc2be74 100644 --- a/myauth/forms.py +++ b/myauth/forms.py @@ -1,8 +1,8 @@ from django import forms -from django.contrib.auth import get_user_model from django.contrib.auth.forms import UsernameField +from myauth.models import get_typed_user_model -User = get_user_model() +User = get_typed_user_model() class AdminUserCreationForm(forms.ModelForm): diff --git a/myauth/models.py b/myauth/models.py index d6585a8..5bc4af7 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -1,3 +1,5 @@ +import typing as t + from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager from django.core import validators from django.db import models @@ -28,9 +30,21 @@ class User(AbstractUser): unique=True, help_text=_("Required. 150 characters or fewer. Letters, digits and ./-/_ only."), validators=[username_validator], - error_messages={"unique": _("A user with that username already exists."),}, + error_messages={ + "unique": _("A user with that username already exists."), + }, ) @classmethod def normalize_username(cls, username): return super().normalize_username(username).lower() + + +UserType = t.Type[User] + + +def get_typed_user_model() -> UserType: + from django.contrib.auth import get_user_model + + ret: t.Any = get_user_model() + return ret From 84b6114e99c206e56292386035dfc7ef0ac84ed4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 13:43:11 +0200 Subject: [PATCH 133/170] Requirements: add dev requirements and django-stubs. --- requirements-dev.txt | 28 ++++++++++++++++++++++++++++ requirements.in/development.txt | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..15a8b60 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements-dev.txt requirements.in/development.txt +# +appdirs==1.4.4 # via black +asgiref==3.3.1 # via django +black==20.8b1 # via -r requirements.in/development.txt +click==7.1.2 # via black, pip-tools +coverage==5.3.1 # via -r requirements.in/development.txt +django-stubs==1.7.0 # via -r requirements.in/development.txt +django==3.1.4 # via django-stubs +mypy-extensions==0.4.3 # via black, mypy +mypy==0.790 # via django-stubs +pathspec==0.8.1 # via black +pip-tools==5.4.0 # via -r requirements.in/development.txt +pytz==2020.5 # via django +pywatchman==1.4.1 # via -r requirements.in/development.txt +regex==2020.11.13 # via black +six==1.15.0 # via pip-tools +sqlparse==0.4.1 # via django +toml==0.10.2 # via black +typed-ast==1.4.1 # via black, mypy +typing-extensions==3.7.4.3 # via black, django-stubs, mypy + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements.in/development.txt b/requirements.in/development.txt index a956471..fb281d3 100644 --- a/requirements.in/development.txt +++ b/requirements.in/development.txt @@ -1,4 +1,5 @@ coverage pip-tools pywatchman -black \ No newline at end of file +black +django-stubs From ff55904f49f32bd21c17f5d383149891e24458eb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 14:04:17 +0200 Subject: [PATCH 134/170] Fix user type --- myauth/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myauth/models.py b/myauth/models.py index 5bc4af7..7786bf0 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -40,7 +40,7 @@ class User(AbstractUser): return super().normalize_username(username).lower() -UserType = t.Type[User] +UserType = User def get_typed_user_model() -> UserType: From e6b47ae1a97fb025c6790a201ec201bc14113f4d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 15:17:46 +0200 Subject: [PATCH 135/170] Fix login_challenge to work with get_user_queryset. --- etebase_fastapi/authentication.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 064d2da..e77cdb8 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -111,11 +111,13 @@ class SignupIn(BaseModel): encryptedContent: bytes -@sync_to_async -def __get_login_user(username: str) -> UserType: +def get_login_user(request: Request, challenge: LoginChallengeIn) -> UserType: + username = challenge.username + kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} try: - user = User.objects.get(**kwargs) + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) + user = user_queryset.get(**kwargs) if not hasattr(user, "userinfo"): raise AuthenticationFailed(code="user_not_init", detail="User not properly init") return user @@ -123,11 +125,6 @@ def __get_login_user(username: str) -> UserType: raise AuthenticationFailed(code="user_not_found", detail="User not found") -async def get_login_user(challenge: LoginChallengeIn) -> UserType: - user = await __get_login_user(challenge.username) - return user - - def get_encryption_key(salt): key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) return nacl.hash.blake2b( @@ -196,7 +193,7 @@ def login_challenge(user: UserType = Depends(get_login_user)): @authentication_router.post("/login/", response_model=LoginOut) async def login(data: Login, request: Request): - user = await get_login_user(LoginChallengeIn(username=data.response_data.username)) + user = await sync_to_async(get_login_user)(request, LoginChallengeIn(username=data.response_data.username)) host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "login", host) data = await sync_to_async(LoginOut.from_orm)(user) From 8bf04fc28638609f3b11559409738ffba6c851e2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 15:37:11 +0200 Subject: [PATCH 136/170] Reformat files using black. --- etebase_fastapi/collection.py | 10 ++++++++-- etebase_fastapi/sendfile/utils.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 9e25b38..14abee9 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -534,14 +534,20 @@ def fetch_updates( @item_router.post("/item/transaction/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_transaction( - collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) + collection_uid: str, + data: ItemBatchIn, + stoken: t.Optional[str] = None, + user: UserType = Depends(get_authenticated_user), ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_batch( - collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) + collection_uid: str, + data: ItemBatchIn, + stoken: t.Optional[str] = None, + user: UserType = Depends(get_authenticated_user), ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) diff --git a/etebase_fastapi/sendfile/utils.py b/etebase_fastapi/sendfile/utils.py index 7c8b1f2..c35d6df 100644 --- a/etebase_fastapi/sendfile/utils.py +++ b/etebase_fastapi/sendfile/utils.py @@ -56,7 +56,9 @@ def _sanitize_path(filepath): try: filepath_abs.relative_to(path_root) except ValueError: - raise HttpError("generic", "{} wrt {} is impossible".format(filepath_abs, path_root), status_code=status.HTTP_404_NOT_FOUND) + raise HttpError( + "generic", "{} wrt {} is impossible".format(filepath_abs, path_root), status_code=status.HTTP_404_NOT_FOUND + ) return filepath_abs From dbdff06e686ccba1649f77539e4cf0a48bf9b8f5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 15:42:41 +0200 Subject: [PATCH 137/170] Move all the routers under their own dir. --- etebase_fastapi/main.py | 10 +++++----- etebase_fastapi/routers/__init__.py | 0 etebase_fastapi/{ => routers}/authentication.py | 8 ++++---- etebase_fastapi/{ => routers}/collection.py | 12 ++++++------ etebase_fastapi/{ => routers}/invitation.py | 6 +++--- etebase_fastapi/{ => routers}/member.py | 6 +++--- etebase_fastapi/{ => routers}/test_reset_view.py | 6 +++--- 7 files changed, 24 insertions(+), 24 deletions(-) create mode 100644 etebase_fastapi/routers/__init__.py rename etebase_fastapi/{ => routers}/authentication.py (97%) rename etebase_fastapi/{ => routers}/collection.py (98%) rename etebase_fastapi/{ => routers}/invitation.py (98%) rename etebase_fastapi/{ => routers}/member.py (94%) rename etebase_fastapi/{ => routers}/test_reset_view.py (90%) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 534798e..8e8469c 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -6,11 +6,11 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware from .exceptions import CustomHttpException -from .authentication import authentication_router -from .collection import collection_router, item_router -from .member import member_router -from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse +from .routers.authentication import authentication_router +from .routers.collection import collection_router, item_router +from .routers.member import member_router +from .routers.invitation import invitation_incoming_router, invitation_outgoing_router def create_application(prefix="", middlewares=[]): @@ -38,7 +38,7 @@ def create_application(prefix="", middlewares=[]): ) if settings.DEBUG: - from etebase_fastapi.test_reset_view import test_reset_view_router + from etebase_fastapi.routers.test_reset_view import test_reset_view_router app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") diff --git a/etebase_fastapi/routers/__init__.py b/etebase_fastapi/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/routers/authentication.py similarity index 97% rename from etebase_fastapi/authentication.py rename to etebase_fastapi/routers/authentication.py index e77cdb8..4b1b57a 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/routers/authentication.py @@ -20,10 +20,10 @@ from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.utils import create_user, get_user_queryset, CallbackContext from myauth.models import UserType, get_typed_user_model -from .exceptions import AuthenticationFailed, transform_validation_error, HttpError -from .msgpack import MsgpackRoute -from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode -from .dependencies import AuthData, get_auth_data, get_authenticated_user +from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError +from ..msgpack import MsgpackRoute +from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from ..dependencies import AuthData, get_auth_data, get_authenticated_user User = get_typed_user_model() authentication_router = APIRouter(route_class=MsgpackRoute) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/routers/collection.py similarity index 98% rename from etebase_fastapi/collection.py rename to etebase_fastapi/routers/collection.py index 14abee9..56afd7b 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -10,10 +10,10 @@ from fastapi import APIRouter, Depends, status, Request from django_etebase import models from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user -from .exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError -from .msgpack import MsgpackRoute -from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import ( +from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError +from ..msgpack import MsgpackRoute +from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken +from ..utils import ( get_object_or_404, Context, Prefetch, @@ -24,8 +24,8 @@ from .utils import ( PERMISSIONS_READ, PERMISSIONS_READWRITE, ) -from .dependencies import get_collection_queryset, get_item_queryset, get_collection -from .sendfile import sendfile +from ..dependencies import get_collection_queryset, get_item_queryset, get_collection +from ..sendfile import sendfile User = get_typed_user_model collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/routers/invitation.py similarity index 98% rename from etebase_fastapi/invitation.py rename to etebase_fastapi/routers/invitation.py index eb9f549..6a06c60 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/routers/invitation.py @@ -8,9 +8,9 @@ from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user -from .exceptions import HttpError, PermissionDenied -from .msgpack import MsgpackRoute -from .utils import ( +from ..exceptions import HttpError, PermissionDenied +from ..msgpack import MsgpackRoute +from ..utils import ( get_object_or_404, Context, is_collection_admin, diff --git a/etebase_fastapi/member.py b/etebase_fastapi/routers/member.py similarity index 94% rename from etebase_fastapi/member.py rename to etebase_fastapi/routers/member.py index 22977ac..210374c 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/routers/member.py @@ -7,9 +7,9 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user -from .msgpack import MsgpackRoute -from .utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE -from .stoken_handler import filter_by_stoken_and_limit +from ..msgpack import MsgpackRoute +from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE +from ..stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/routers/test_reset_view.py similarity index 90% rename from etebase_fastapi/test_reset_view.py rename to etebase_fastapi/routers/test_reset_view.py index e328875..09638e4 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/routers/test_reset_view.py @@ -4,9 +4,9 @@ from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, status from django_etebase.utils import get_user_queryset, CallbackContext -from etebase_fastapi.authentication import SignupIn, signup_save -from etebase_fastapi.msgpack import MsgpackRoute -from etebase_fastapi.exceptions import HttpError +from .authentication import SignupIn, signup_save +from ..msgpack import MsgpackRoute +from ..exceptions import HttpError from myauth.models import get_typed_user_model test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) From 62eb46ec4ef036673b1864c871aa477cf6fde002 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 15:44:52 +0200 Subject: [PATCH 138/170] msgpack route: use the encode/decode functions from the utils module. --- etebase_fastapi/msgpack.py | 9 ++++----- etebase_fastapi/utils.py | 4 +++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 915e783..8de8806 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -1,10 +1,11 @@ import typing as t -import msgpack from fastapi.routing import APIRoute, get_request_handler from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response +from .utils import msgpack_encode, msgpack_decode + class MsgpackRequest(Request): media_type = "application/msgpack" @@ -12,7 +13,7 @@ class MsgpackRequest(Request): async def json(self) -> bytes: if not hasattr(self, "_json"): body = await super().body() - self._json = msgpack.unpackb(body, raw=False) + self._json = msgpack_decode(body) return self._json @@ -25,9 +26,7 @@ class MsgpackResponse(Response): if isinstance(content, BaseModel): content = content.dict() - ret = msgpack.packb(content, use_bin_type=True) - assert ret is not None - return ret + return msgpack_encode(content) class MsgpackRoute(APIRoute): diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index c91c3ec..3a091c5 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -47,7 +47,9 @@ def is_collection_admin(collection, user): def msgpack_encode(content): - return msgpack.packb(content, use_bin_type=True) + ret = msgpack.packb(content, use_bin_type=True) + assert ret is not None + return ret def msgpack_decode(content): From 174e54681d8688a2c048ce3d199dea07ae99354d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 16:06:59 +0200 Subject: [PATCH 139/170] Improve type annotations. --- django_etebase/models.py | 1 + myauth/models.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index 096371d..3060fa4 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -124,6 +124,7 @@ def generate_stoken_uid(): class Stoken(models.Model): + id: int uid = models.CharField( db_index=True, unique=True, diff --git a/myauth/models.py b/myauth/models.py index 7786bf0..c9298a4 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -20,9 +20,10 @@ class UserManager(DjangoUserManager): class User(AbstractUser): + id: int username_validator = UnicodeUsernameValidator() - objects = UserManager() + objects: UserManager = UserManager() username = models.CharField( _("username"), From 8245577dfb1117eab0cac81c4c4224b39967d757 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 16:55:21 +0200 Subject: [PATCH 140/170] Rename module to prevent confusion on import. --- django_etebase/__init__.py | 2 +- django_etebase/{app_settings.py => app_settings_inner.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename django_etebase/{app_settings.py => app_settings_inner.py} (100%) diff --git a/django_etebase/__init__.py b/django_etebase/__init__.py index 426fefd..99ee8b6 100644 --- a/django_etebase/__init__.py +++ b/django_etebase/__init__.py @@ -1 +1 @@ -from .app_settings import app_settings +from .app_settings_inner import app_settings diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings_inner.py similarity index 100% rename from django_etebase/app_settings.py rename to django_etebase/app_settings_inner.py From 332f7e2332757e7aa3aa8fec50583679be1a8608 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 17:52:08 +0200 Subject: [PATCH 141/170] Fix Python 3.7 compatibility Both cached_property and Literal were introduced in Python 3.8 so they can't be used. --- etebase_fastapi/routers/authentication.py | 5 +++-- etebase_fastapi/utils.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/routers/authentication.py b/etebase_fastapi/routers/authentication.py index 4b1b57a..efab7ef 100644 --- a/etebase_fastapi/routers/authentication.py +++ b/etebase_fastapi/routers/authentication.py @@ -1,6 +1,6 @@ import typing as t +from typing_extensions import Literal from datetime import datetime -from functools import cached_property import nacl import nacl.encoding @@ -12,6 +12,7 @@ from django.conf import settings from django.contrib.auth import user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction +from django.utils.functional import cached_property from fastapi import APIRouter, Depends, status, Request from django_etebase import app_settings, models @@ -43,7 +44,7 @@ class LoginResponse(BaseModel): username: str challenge: bytes host: str - action: t.Literal["login", "changePassword"] + action: Literal["login", "changePassword"] class UserOut(BaseModel): diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 3a091c5..03f1a7d 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,5 +1,6 @@ import dataclasses import typing as t +from typing_extensions import Literal import msgpack import base64 @@ -17,7 +18,7 @@ from .exceptions import HttpError, HttpErrorOut User = get_typed_user_model() -Prefetch = t.Literal["auto", "medium"] +Prefetch = Literal["auto", "medium"] PrefetchQuery = Query(default="auto") From 709bc6c1fc2be28b6d2ad9e4bc7bd12de2828dfd Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 17:18:09 +0200 Subject: [PATCH 142/170] Improve typing information. --- django_etebase/models.py | 47 ++++++++++++++++++++------- django_etebase/utils.py | 5 +-- etebase_fastapi/routers/collection.py | 18 +++++----- etebase_fastapi/routers/invitation.py | 19 ++++++----- etebase_fastapi/routers/member.py | 9 ++--- etebase_fastapi/utils.py | 7 ++-- myauth/models.py | 4 +-- 7 files changed, 70 insertions(+), 39 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index 3060fa4..7725a19 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -12,6 +12,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import typing as t from pathlib import Path from django.db import models, transaction @@ -28,7 +29,7 @@ from . import app_settings UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID") -def stoken_annotation_builder(stoken_id_fields): +def stoken_annotation_builder(stoken_id_fields: t.List[str]): aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields] return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] @@ -37,6 +38,8 @@ class CollectionType(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) uid = models.BinaryField(editable=True, blank=False, null=False, db_index=True, unique=True) + objects: models.manager.BaseManager["CollectionType"] + class Collection(models.Model): main_item = models.OneToOneField("CollectionItem", related_name="parent", null=True, on_delete=models.SET_NULL) @@ -46,19 +49,21 @@ class Collection(models.Model): stoken_annotation = stoken_annotation_builder(["items__revisions__stoken", "members__stoken"]) + objects: models.manager.BaseManager["Collection"] + def __str__(self): return self.uid @property - def content(self): + def content(self) -> "CollectionItemRevision": return self.main_item.content @property - def etag(self): + def etag(self) -> str: return self.content.uid @cached_property - def stoken(self): + def stoken(self) -> str: stoken_id = ( self.__class__.objects.filter(main_item=self.main_item) .annotate(max_stoken=self.stoken_annotation) @@ -80,6 +85,8 @@ class CollectionItem(models.Model): stoken_annotation = stoken_annotation_builder(["revisions__stoken"]) + objects: models.manager.BaseManager["CollectionItem"] + class Meta: unique_together = ("uid", "collection") @@ -87,23 +94,23 @@ class CollectionItem(models.Model): return "{} {}".format(self.uid, self.collection.uid) @cached_property - def content(self): + def content(self) -> "CollectionItemRevision": return self.revisions.get(current=True) @property - def etag(self): + def etag(self) -> str: return self.content.uid -def chunk_directory_path(instance, filename): +def chunk_directory_path(instance: "CollectionItemChunk", filename: str) -> Path: custom_func = app_settings.CHUNK_PATH_FUNC if custom_func is not None: return custom_func(instance, filename) - col = instance.collection - user_id = col.owner.id - uid_prefix = instance.uid[:2] - uid_rest = instance.uid[2:] + col: Collection = instance.collection + user_id: int = col.owner.id + uid_prefix: str = instance.uid[:2] + uid_rest: str = instance.uid[2:] return Path("user_{}".format(user_id), col.uid, uid_prefix, uid_rest) @@ -112,6 +119,8 @@ class CollectionItemChunk(models.Model): collection = models.ForeignKey(Collection, related_name="chunks", on_delete=models.CASCADE) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) + objects: models.manager.BaseManager["CollectionItemChunk"] + def __str__(self): return self.uid @@ -135,6 +144,8 @@ class Stoken(models.Model): validators=[UidValidator], ) + objects: models.manager.BaseManager["Stoken"] + class CollectionItemRevision(models.Model): stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT) @@ -146,6 +157,8 @@ class CollectionItemRevision(models.Model): current = models.BooleanField(db_index=True, default=True, null=True) deleted = models.BooleanField(default=False) + objects: models.manager.BaseManager["CollectionItemRevision"] + class Meta: unique_together = ("item", "current") @@ -157,6 +170,8 @@ class RevisionChunkRelation(models.Model): chunk = models.ForeignKey(CollectionItemChunk, related_name="revisions_relation", on_delete=models.CASCADE) revision = models.ForeignKey(CollectionItemRevision, related_name="chunks_relation", on_delete=models.CASCADE) + objects: models.manager.BaseManager["RevisionChunkRelation"] + class Meta: ordering = ("id",) @@ -180,6 +195,8 @@ class CollectionMember(models.Model): stoken_annotation = stoken_annotation_builder(["stoken"]) + objects: models.manager.BaseManager["CollectionMember"] + class Meta: unique_together = ("user", "collection") @@ -204,6 +221,8 @@ class CollectionMemberRemoved(models.Model): collection = models.ForeignKey(Collection, related_name="removed_members", on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + objects: models.manager.BaseManager["CollectionMemberRemoved"] + class Meta: unique_together = ("user", "collection") @@ -225,6 +244,8 @@ class CollectionInvitation(models.Model): default=AccessLevels.READ_ONLY, ) + objects: models.manager.BaseManager["CollectionInvitation"] + class Meta: unique_together = ("user", "fromMember") @@ -232,7 +253,7 @@ class CollectionInvitation(models.Model): return "{} {}".format(self.fromMember.collection.uid, self.user) @cached_property - def collection(self): + def collection(self) -> Collection: return self.fromMember.collection @@ -244,5 +265,7 @@ class UserInfo(models.Model): encryptedContent = models.BinaryField(editable=True, blank=False, null=False) salt = models.BinaryField(editable=True, blank=False, null=False) + objects: models.manager.BaseManager["UserInfo"] + def __str__(self): return "UserInfo<{}>".format(self.owner) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index d812ae3..3a05fd4 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -1,6 +1,7 @@ import typing as t from dataclasses import dataclass +from django.db.models import QuerySet from django.core.exceptions import PermissionDenied from myauth.models import UserType, get_typed_user_model @@ -18,14 +19,14 @@ class CallbackContext: user: t.Optional[UserType] = None -def get_user_queryset(queryset, context: CallbackContext): +def get_user_queryset(queryset: QuerySet[UserType], context: CallbackContext) -> QuerySet[UserType]: custom_func = app_settings.GET_USER_QUERYSET_FUNC if custom_func is not None: return custom_func(queryset, context) return queryset -def create_user(context: CallbackContext, *args, **kwargs): +def create_user(context: CallbackContext, *args, **kwargs) -> UserType: custom_func = app_settings.CREATE_USER_FUNC if custom_func is not None: return custom_func(context, *args, **kwargs) diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index 56afd7b..4825626 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -30,6 +30,8 @@ from ..sendfile import sendfile User = get_typed_user_model collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +CollectionQuerySet = QuerySet[models.Collection] +CollectionItemQuerySet = QuerySet[models.CollectionItem] class ListMulti(BaseModel): @@ -187,7 +189,7 @@ class ItemBatchIn(BaseModel): @sync_to_async def collection_list_common( - queryset: QuerySet, + queryset: CollectionQuerySet, user: UserType, stoken: t.Optional[str], limit: int, @@ -249,7 +251,7 @@ async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, - queryset: QuerySet = Depends(get_collection_queryset), + queryset: CollectionQuerySet = Depends(get_collection_queryset), user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): @@ -267,7 +269,7 @@ async def collection_list( limit: int = 50, prefetch: Prefetch = PrefetchQuery, user: UserType = Depends(get_authenticated_user), - queryset: QuerySet = Depends(get_collection_queryset), + queryset: CollectionQuerySet = Depends(get_collection_queryset), ): return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -395,7 +397,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val @item_router.get("/item/{item_uid}/", response_model=CollectionItemOut, dependencies=PERMISSIONS_READ) def item_get( item_uid: str, - queryset: QuerySet = Depends(get_item_queryset), + queryset: CollectionItemQuerySet = Depends(get_item_queryset), user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): @@ -405,7 +407,7 @@ def item_get( @sync_to_async def item_list_common( - queryset: QuerySet, + queryset: CollectionItemQuerySet, user: UserType, stoken: t.Optional[str], limit: int, @@ -422,7 +424,7 @@ def item_list_common( @item_router.get("/item/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) async def item_list( - queryset: QuerySet = Depends(get_item_queryset), + queryset: CollectionItemQuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, @@ -471,7 +473,7 @@ def item_revisions( iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: UserType = Depends(get_authenticated_user), - items: QuerySet = Depends(get_item_queryset), + items: CollectionItemQuerySet = Depends(get_item_queryset), ): item = get_object_or_404(items, uid=item_uid) @@ -505,7 +507,7 @@ def fetch_updates( stoken: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: UserType = Depends(get_authenticated_user), - queryset: QuerySet = Depends(get_item_queryset), + queryset: CollectionItemQuerySet = Depends(get_item_queryset), ): # FIXME: make configurable? item_limit = 200 diff --git a/etebase_fastapi/routers/invitation.py b/etebase_fastapi/routers/invitation.py index 6a06c60..aceb05d 100644 --- a/etebase_fastapi/routers/invitation.py +++ b/etebase_fastapi/routers/invitation.py @@ -23,7 +23,8 @@ from ..utils import ( User = get_typed_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) -default_queryset: QuerySet = models.CollectionInvitation.objects.all() +InvitationQuerySet = QuerySet[models.CollectionInvitation] +default_queryset: InvitationQuerySet = models.CollectionInvitation.objects.all() class UserInfoOut(BaseModel): @@ -94,7 +95,7 @@ def get_outgoing_queryset(user: UserType = Depends(get_authenticated_user)): def list_common( - queryset: QuerySet, + queryset: InvitationQuerySet, iterator: t.Optional[str], limit: int, ) -> InvitationListResponse: @@ -125,7 +126,7 @@ def list_common( def incoming_list( iterator: t.Optional[str] = None, limit: int = 50, - queryset: QuerySet = Depends(get_incoming_queryset), + queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): return list_common(queryset, iterator, limit) @@ -135,7 +136,7 @@ def incoming_list( ) def incoming_get( invitation_uid: str, - queryset: QuerySet = Depends(get_incoming_queryset), + queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) return CollectionInvitationOut.from_orm(obj) @@ -146,7 +147,7 @@ def incoming_get( ) def incoming_delete( invitation_uid: str, - queryset: QuerySet = Depends(get_incoming_queryset), + queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) obj.delete() @@ -158,7 +159,7 @@ def incoming_delete( def incoming_accept( invitation_uid: str, data: CollectionInvitationAcceptIn, - queryset: QuerySet = Depends(get_incoming_queryset), + queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): invitation = get_object_or_404(queryset, uid=invitation_uid) @@ -201,7 +202,7 @@ def outgoing_create( with transaction.atomic(): try: - ret = models.CollectionInvitation.objects.create( + models.CollectionInvitation.objects.create( **data.dict(exclude={"collection", "username"}), user=to_user, fromMember=member ) except IntegrityError: @@ -212,7 +213,7 @@ def outgoing_create( def outgoing_list( iterator: t.Optional[str] = None, limit: int = 50, - queryset: QuerySet = Depends(get_outgoing_queryset), + queryset: InvitationQuerySet = Depends(get_outgoing_queryset), ): return list_common(queryset, iterator, limit) @@ -222,7 +223,7 @@ def outgoing_list( ) def outgoing_delete( invitation_uid: str, - queryset: QuerySet = Depends(get_outgoing_queryset), + queryset: InvitationQuerySet = Depends(get_outgoing_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) obj.delete() diff --git a/etebase_fastapi/routers/member.py b/etebase_fastapi/routers/member.py index 210374c..41393bf 100644 --- a/etebase_fastapi/routers/member.py +++ b/etebase_fastapi/routers/member.py @@ -15,14 +15,15 @@ from .collection import get_collection, verify_collection_admin User = get_typed_user_model() member_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) -default_queryset: QuerySet = models.CollectionMember.objects.all() +MemberQuerySet = QuerySet[models.CollectionMember] +default_queryset: MemberQuerySet = models.CollectionMember.objects.all() -def get_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: +def get_queryset(collection: models.Collection = Depends(get_collection)) -> MemberQuerySet: return default_queryset.filter(collection=collection) -def get_member(username: str, queryset: QuerySet = Depends(get_queryset)) -> QuerySet: +def get_member(username: str, queryset: MemberQuerySet = Depends(get_queryset)) -> models.CollectionMember: return get_object_or_404(queryset, user__username__iexact=username) @@ -54,7 +55,7 @@ class MemberListResponse(BaseModel): def member_list( iterator: t.Optional[str] = None, limit: int = 50, - queryset: QuerySet = Depends(get_queryset), + queryset: MemberQuerySet = Depends(get_queryset), ): queryset = queryset.order_by("id") result, new_stoken_obj, done = filter_by_stoken_and_limit( diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 03f1a7d..c9db61c 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -7,7 +7,7 @@ import base64 from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel -from django.db.models import QuerySet +from django.db.models import Model, QuerySet from django.core.exceptions import ObjectDoesNotExist from django_etebase import app_settings @@ -22,6 +22,9 @@ Prefetch = Literal["auto", "medium"] PrefetchQuery = Query(default="auto") +T = t.TypeVar("T", bound=Model, covariant=True) + + class BaseModel(PyBaseModel): class Config: json_encoders = { @@ -35,7 +38,7 @@ class Context: prefetch: t.Optional[Prefetch] -def get_object_or_404(queryset: QuerySet, **kwargs): +def get_object_or_404(queryset: QuerySet[T], **kwargs) -> T: try: return queryset.get(**kwargs) except ObjectDoesNotExist as e: diff --git a/myauth/models.py b/myauth/models.py index c9298a4..89b94b4 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -15,7 +15,7 @@ class UnicodeUsernameValidator(validators.RegexValidator): class UserManager(DjangoUserManager): - def get_by_natural_key(self, username): + def get_by_natural_key(self, username: str): return self.get(**{self.model.USERNAME_FIELD + "__iexact": username}) @@ -37,7 +37,7 @@ class User(AbstractUser): ) @classmethod - def normalize_username(cls, username): + def normalize_username(cls, username: str): return super().normalize_username(username).lower() From c4235662d8a9b1f7b126f2f5162b9d6cefa815a3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 18:50:18 +0200 Subject: [PATCH 143/170] Use uvicorn[standard] in requirements to get some added extras. --- requirements.in/base.txt | 2 +- requirements.txt | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index fee4a56..81af2a0 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -2,4 +2,4 @@ django msgpack pynacl fastapi -uvicorn +uvicorn[standard] diff --git a/requirements.txt b/requirements.txt index cfce456..22ed527 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,12 +10,18 @@ click==7.1.2 # via uvicorn django==3.1.4 # via -r requirements.in/base.txt fastapi==0.63.0 # via -r requirements.in/base.txt h11==0.11.0 # via uvicorn +httptools==0.1.1 # via uvicorn msgpack==1.0.2 # via -r requirements.in/base.txt pycparser==2.20 # via cffi pydantic==1.7.3 # via fastapi pynacl==1.4.0 # via -r requirements.in/base.txt +python-dotenv==0.15.0 # via uvicorn pytz==2020.4 # via django +pyyaml==5.3.1 # via uvicorn six==1.15.0 # via pynacl sqlparse==0.4.1 # via django starlette==0.13.6 # via fastapi -uvicorn==0.13.2 # via -r requirements.in/base.txt +uvicorn[standard]==0.13.2 # via -r requirements.in/base.txt +uvloop==0.14.0 # via uvicorn +watchgod==0.6 # via uvicorn +websockets==8.1 # via uvicorn From 8bfdbc55a3d8b9dfa5cdb2f38b87d1012c6b894c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 18:53:31 +0200 Subject: [PATCH 144/170] Add typing extensions to requirements. --- requirements.in/base.txt | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 81af2a0..44e2875 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -2,4 +2,5 @@ django msgpack pynacl fastapi +typing_extensions uvicorn[standard] diff --git a/requirements.txt b/requirements.txt index 22ed527..f59431b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ pyyaml==5.3.1 # via uvicorn six==1.15.0 # via pynacl sqlparse==0.4.1 # via django starlette==0.13.6 # via fastapi +typing-extensions==3.7.4.3 # via -r requirements.in/base.txt uvicorn[standard]==0.13.2 # via -r requirements.in/base.txt uvloop==0.14.0 # via uvicorn watchgod==0.6 # via uvicorn From 981e1a9a6accf7913eb925126d4d9c827644360a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 21:00:56 +0200 Subject: [PATCH 145/170] Add mypy config. --- mypy.ini | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..0a96c4c --- /dev/null +++ b/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +plugins = mypy_django_plugin.main + +[mypy.plugins.django-stubs] +django_settings_module = "etebase_server.settings" From 6615b149c5e334302ee21b999183d86b55b4c0c3 Mon Sep 17 00:00:00 2001 From: Simon Vandevelde Date: Tue, 29 Dec 2020 20:04:48 +0100 Subject: [PATCH 146/170] Update README with wiki changes (#82) * Changed link to wiki * Removed unneeded links (as uWSGI won't be supported) * Added --host flag to uvicorn command, as the server is only accessible from the host machine otherwise (and not from another in the local network) --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1787a2f..c110672 100644 --- a/README.md +++ b/README.md @@ -62,18 +62,14 @@ Now you can initialise our django app. And you are done! You can now run the debug server just to see everything works as expected by running: ``` -uvicorn etebase_server.asgi:application --port 8000 +uvicorn etebase_server.asgi:application --host 0.0.0.0 --port 8000 ``` Using the debug server in production is not recommended, so please read the following section for a proper deployment. # Production deployment -There are more details about a proper production setup using Daphne and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-Daphne-and-Nginx). - -Etebase is based on Django so you should refer to one of the following - * The instructions of the Django project [here](https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/). - * Instructions from uwsgi [here](http://uwsgi-docs.readthedocs.io/en/latest/tutorials/Django_and_nginx.html). +There are more details about a proper production setup using uvicorn and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-Nginx). The webserver should also be configured to serve Etebase using TLS. A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-Etebase) as well. From a7fdb4a1082b3d84a6ee7e7eae1086412124b183 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 20:56:36 +0200 Subject: [PATCH 147/170] More typing fixes. --- django_etebase/models.py | 1 + etebase_fastapi/routers/collection.py | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index 7725a19..fa56a95 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -56,6 +56,7 @@ class Collection(models.Model): @property def content(self) -> "CollectionItemRevision": + assert self.main_item is not None return self.main_item.content @property diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index 4825626..e20e955 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -107,6 +107,7 @@ class CollectionOut(CollectionCommon): def from_orm_context(cls: t.Type["CollectionOut"], obj: models.Collection, context: Context) -> "CollectionOut": member: models.CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType + assert obj.main_item is not None ret = cls( collectionType=collection_type and bytes(collection_type.uid), collectionKey=bytes(member.encryptionKey), @@ -299,8 +300,8 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle revision.stoken = stoken revision.save() - for chunk in chunks_objs: - models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) + for chunk2 in chunks_objs: + models.RevisionChunkRelation.objects.create(chunk=chunk2, revision=revision) return revision @@ -383,6 +384,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val # We don't have to use select_for_update here because the unique constraint on current guards against # the race condition. But it's a good idea because it'll lock and wait rather than fail. current_revision = instance.revisions.filter(current=True).select_for_update().first() + assert current_revision is not None current_revision.current = None current_revision.save() @@ -523,8 +525,8 @@ def fetch_updates( new_stoken_obj = get_queryset_stoken(queryset) new_stoken = new_stoken_obj and new_stoken_obj.uid - stoken = stoken_rev and getattr(stoken_rev, "uid", None) - new_stoken = new_stoken or stoken + stoken_rev_uid = stoken_rev and getattr(stoken_rev, "uid", None) + new_stoken = new_stoken or stoken_rev_uid context = Context(user, prefetch) return CollectionItemListResponse( From 473448246f61a540110142f261e5aa2650b3cd12 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 30 Dec 2020 09:20:38 +0200 Subject: [PATCH 148/170] Add a django middleware to cleanup db connections. This severely impacts performance, though without it we are getting django.db.utils.InterfaceError once connections in the pool go stale. --- etebase_fastapi/main.py | 2 ++ etebase_fastapi/middleware.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 etebase_fastapi/middleware.py diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 8e8469c..c07d975 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -5,6 +5,7 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware +from .middleware import DjangoDbConnectionCleanupMiddleware from .exceptions import CustomHttpException from .msgpack import MsgpackResponse from .routers.authentication import authentication_router @@ -42,6 +43,7 @@ def create_application(prefix="", middlewares=[]): app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") + app.add_middleware(DjangoDbConnectionCleanupMiddleware) app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", diff --git a/etebase_fastapi/middleware.py b/etebase_fastapi/middleware.py new file mode 100644 index 0000000..06e347b --- /dev/null +++ b/etebase_fastapi/middleware.py @@ -0,0 +1,15 @@ +from starlette.types import ASGIApp, Receive, Scope, Send +from django.db import close_old_connections, reset_queries + + +class DjangoDbConnectionCleanupMiddleware: + def __init__(self, app: ASGIApp): + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send): + reset_queries() + close_old_connections() + try: + await self.app(scope, receive, send) + finally: + close_old_connections() From 64be7f10bdf3dcddc00b46dbc4e27ea80ebbdd08 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 30 Dec 2020 10:17:01 +0200 Subject: [PATCH 149/170] Remove all of the needless async decorators. The code uses the django ORM which is sync, and fastapi handles sync paths just fine. So having all of this extra code for handling async was unnecessary. --- etebase_fastapi/routers/authentication.py | 20 +++++++++----------- etebase_fastapi/routers/collection.py | 18 ++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/etebase_fastapi/routers/authentication.py b/etebase_fastapi/routers/authentication.py index efab7ef..be6a7e8 100644 --- a/etebase_fastapi/routers/authentication.py +++ b/etebase_fastapi/routers/authentication.py @@ -7,7 +7,6 @@ import nacl.encoding import nacl.hash import nacl.secret import nacl.signing -from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth import user_logged_out, user_logged_in from django.core import exceptions as django_exceptions @@ -145,7 +144,6 @@ def save_changed_password(data: ChangePassword, user: UserType): user_info.save() -@sync_to_async def validate_login_request( validated_data: LoginResponse, challenge_sent_to_user: Authentication, @@ -193,13 +191,13 @@ def login_challenge(user: UserType = Depends(get_login_user)): @authentication_router.post("/login/", response_model=LoginOut) -async def login(data: Login, request: Request): - user = await sync_to_async(get_login_user)(request, LoginChallengeIn(username=data.response_data.username)) +def login(data: Login, request: Request): + user = get_login_user(request, LoginChallengeIn(username=data.response_data.username)) host = request.headers.get("Host") - await validate_login_request(data.response_data, data, user, "login", host) - data = await sync_to_async(LoginOut.from_orm)(user) - await sync_to_async(user_logged_in.send)(sender=user.__class__, request=None, user=user) - return data + validate_login_request(data.response_data, data, user, "login", host) + ret = LoginOut.from_orm(user) + user_logged_in.send(sender=user.__class__, request=None, user=user) + return ret @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) @@ -209,10 +207,10 @@ def logout(auth_data: AuthData = Depends(get_auth_data)): @authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -async def change_password(data: ChangePassword, request: Request, user: UserType = Depends(get_authenticated_user)): +def change_password(data: ChangePassword, request: Request, user: UserType = Depends(get_authenticated_user)): host = request.headers.get("Host") - await validate_login_request(data.response_data, data, user, "changePassword", host) - await sync_to_async(save_changed_password)(data, user) + validate_login_request(data.response_data, data, user, "changePassword", host) + save_changed_password(data, user) @authentication_router.post("/dashboard_url/", responses=permission_responses) diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index e20e955..b716105 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -188,7 +188,6 @@ class ItemBatchIn(BaseModel): ) -@sync_to_async def collection_list_common( queryset: CollectionQuerySet, user: UserType, @@ -248,7 +247,7 @@ def has_write_access( response_model_exclude_unset=True, dependencies=PERMISSIONS_READ, ) -async def list_multi( +def list_multi( data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, @@ -261,18 +260,18 @@ async def list_multi( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - return await collection_list_common(queryset, user, stoken, limit, prefetch) + return collection_list_common(queryset, user, stoken, limit, prefetch) @collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ) -async def collection_list( +def collection_list( stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, user: UserType = Depends(get_authenticated_user), queryset: CollectionQuerySet = Depends(get_collection_queryset), ): - return await collection_list_common(queryset, user, stoken, limit, prefetch) + return collection_list_common(queryset, user, stoken, limit, prefetch) def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): @@ -341,8 +340,8 @@ def _create(data: CollectionIn, user: UserType): @collection_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) -async def create(data: CollectionIn, user: UserType = Depends(get_authenticated_user)): - await sync_to_async(_create)(data, user) +def create(data: CollectionIn, user: UserType = Depends(get_authenticated_user)): + _create(data, user) @collection_router.get("/{collection_uid}/", response_model=CollectionOut, dependencies=PERMISSIONS_READ) @@ -407,7 +406,6 @@ def item_get( return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) -@sync_to_async def item_list_common( queryset: CollectionItemQuerySet, user: UserType, @@ -425,7 +423,7 @@ def item_list_common( @item_router.get("/item/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) -async def item_list( +def item_list( queryset: CollectionItemQuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, limit: int = 50, @@ -436,7 +434,7 @@ async def item_list( if not withCollection: queryset = queryset.filter(parent__isnull=True) - response = await item_list_common(queryset, user, stoken, limit, prefetch) + response = item_list_common(queryset, user, stoken, limit, prefetch) return response From 6738c2cf20bb1d5a0e24c9910bb0345fb59716cf Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 30 Dec 2020 13:55:05 +0200 Subject: [PATCH 150/170] Remove unused variable. --- etebase_fastapi/routers/collection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index b716105..2041ce7 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -8,7 +8,7 @@ from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request from django_etebase import models -from myauth.models import UserType, get_typed_user_model +from myauth.models import UserType from .authentication import get_authenticated_user from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError from ..msgpack import MsgpackRoute @@ -27,7 +27,6 @@ from ..utils import ( from ..dependencies import get_collection_queryset, get_item_queryset, get_collection from ..sendfile import sendfile -User = get_typed_user_model collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) CollectionQuerySet = QuerySet[models.Collection] From 6ec03c3d3494f91a59accfced180b03eb02478a1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 30 Dec 2020 15:37:59 +0200 Subject: [PATCH 151/170] Revert "Add a django middleware to cleanup db connections." This ended up being useless because of the way startlette and fastapi do thread pools. The middleware is called in one thread, the path in another, and the dependency in yet another. This reverts commit 473448246f61a540110142f261e5aa2650b3cd12. --- etebase_fastapi/main.py | 2 -- etebase_fastapi/middleware.py | 15 --------------- 2 files changed, 17 deletions(-) delete mode 100644 etebase_fastapi/middleware.py diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index c07d975..8e8469c 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -5,7 +5,6 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware -from .middleware import DjangoDbConnectionCleanupMiddleware from .exceptions import CustomHttpException from .msgpack import MsgpackResponse from .routers.authentication import authentication_router @@ -43,7 +42,6 @@ def create_application(prefix="", middlewares=[]): app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") - app.add_middleware(DjangoDbConnectionCleanupMiddleware) app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", diff --git a/etebase_fastapi/middleware.py b/etebase_fastapi/middleware.py deleted file mode 100644 index 06e347b..0000000 --- a/etebase_fastapi/middleware.py +++ /dev/null @@ -1,15 +0,0 @@ -from starlette.types import ASGIApp, Receive, Scope, Send -from django.db import close_old_connections, reset_queries - - -class DjangoDbConnectionCleanupMiddleware: - def __init__(self, app: ASGIApp): - self.app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send): - reset_queries() - close_old_connections() - try: - await self.app(scope, receive, send) - finally: - close_old_connections() From 5b8f667e55cef4ea5ed1e2ad78f6df7d7101258c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 30 Dec 2020 17:09:16 +0200 Subject: [PATCH 152/170] Cleanup django db connections before every request and every dependency. This is instead of the commit we reverted in the previous commit. The problem is that django keeps the connection per thread and it relies on django itself to clean them up before/after connections. We can't do this, because django is unaware of fastapi, so we have to manage this ourselves. The easiest way is to call it at the beginning of evenry route and every dep. We need to do it for each because unfortunately fastapi may send them to different worker threads. --- etebase_fastapi/db_hack.py | 27 +++++++++++++++++++++++++++ etebase_fastapi/msgpack.py | 19 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 etebase_fastapi/db_hack.py diff --git a/etebase_fastapi/db_hack.py b/etebase_fastapi/db_hack.py new file mode 100644 index 0000000..24d5824 --- /dev/null +++ b/etebase_fastapi/db_hack.py @@ -0,0 +1,27 @@ +""" +FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned. +Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM. +""" +import typing as t +from functools import wraps + +from django.db import close_old_connections, reset_queries + + +def django_db_cleanup(): + reset_queries() + close_old_connections() + + +def django_db_cleanup_decorator(func: t.Callable[..., t.Any]): + from inspect import iscoroutinefunction + + if iscoroutinefunction(func): + return func + + @wraps(func) + def wrapper(*args, **kwargs): + django_db_cleanup() + return func(*args, **kwargs) + + return wrapper diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 8de8806..67627e1 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -1,10 +1,13 @@ import typing as t + +from fastapi import params from fastapi.routing import APIRoute, get_request_handler from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response from .utils import msgpack_encode, msgpack_decode +from .db_hack import django_db_cleanup_decorator class MsgpackRequest(Request): @@ -35,6 +38,22 @@ class MsgpackRoute(APIRoute): # keep track of content-type -> response classes ROUTES_HANDLERS_CLASSES = {MsgpackResponse.media_type: MsgpackResponse} + def __init__( + self, + path: str, + endpoint: t.Callable[..., t.Any], + *args, + dependencies: t.Optional[t.Sequence[params.Depends]] = None, + **kwargs + ): + if dependencies is not None: + dependencies = [ + params.Depends(django_db_cleanup_decorator(dep.dependency), use_cache=dep.use_cache) + for dep in dependencies + ] + endpoint = django_db_cleanup_decorator(endpoint) + super().__init__(path, endpoint, *args, dependencies=dependencies, **kwargs) + def _get_media_type_route_handler(self, media_type): return get_request_handler( dependant=self.dependant, From 2e21fe4994c2bf9799b36e535db8bdf6943ff957 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 31 Dec 2020 10:03:16 +0200 Subject: [PATCH 153/170] Django db cleanup: explicitly add it to dependencies. We can't really add it manually, because some of the deps are auto included as parameters. These were not being decorated which in turn meeant issues. --- etebase_fastapi/dependencies.py | 6 ++++++ etebase_fastapi/msgpack.py | 17 ++--------------- etebase_fastapi/routers/collection.py | 3 +++ etebase_fastapi/routers/invitation.py | 3 +++ etebase_fastapi/routers/member.py | 3 +++ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/etebase_fastapi/dependencies.py b/etebase_fastapi/dependencies.py index fb9cec5..520d499 100644 --- a/etebase_fastapi/dependencies.py +++ b/etebase_fastapi/dependencies.py @@ -11,6 +11,7 @@ from django_etebase.token_auth.models import AuthToken, get_default_expiry from myauth.models import UserType, get_typed_user_model from .exceptions import AuthenticationFailed from .utils import get_object_or_404 +from .db_hack import django_db_cleanup_decorator User = get_typed_user_model() @@ -55,25 +56,30 @@ def __get_authenticated_user(api_token: str): return token.user, token +@django_db_cleanup_decorator def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: user, token = __get_authenticated_user(api_token) return AuthData(user, token) +@django_db_cleanup_decorator def get_authenticated_user(api_token: str = Depends(token_scheme)) -> UserType: user, _ = __get_authenticated_user(api_token) return user +@django_db_cleanup_decorator def get_collection_queryset(user: UserType = Depends(get_authenticated_user)) -> QuerySet: default_queryset: QuerySet = models.Collection.objects.all() return default_queryset.filter(members__user=user) +@django_db_cleanup_decorator def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: return get_object_or_404(queryset, uid=collection_uid) +@django_db_cleanup_decorator def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: default_item_queryset: QuerySet = models.CollectionItem.objects.all() # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 67627e1..a671e79 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -1,6 +1,5 @@ import typing as t -from fastapi import params from fastapi.routing import APIRoute, get_request_handler from pydantic import BaseModel from starlette.requests import Request @@ -38,21 +37,9 @@ class MsgpackRoute(APIRoute): # keep track of content-type -> response classes ROUTES_HANDLERS_CLASSES = {MsgpackResponse.media_type: MsgpackResponse} - def __init__( - self, - path: str, - endpoint: t.Callable[..., t.Any], - *args, - dependencies: t.Optional[t.Sequence[params.Depends]] = None, - **kwargs - ): - if dependencies is not None: - dependencies = [ - params.Depends(django_db_cleanup_decorator(dep.dependency), use_cache=dep.use_cache) - for dep in dependencies - ] + def __init__(self, path: str, endpoint: t.Callable[..., t.Any], *args, **kwargs): endpoint = django_db_cleanup_decorator(endpoint) - super().__init__(path, endpoint, *args, dependencies=dependencies, **kwargs) + super().__init__(path, endpoint, *args, **kwargs) def _get_media_type_route_handler(self, media_type): return get_request_handler( diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index 2041ce7..4dcb3c6 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -26,6 +26,7 @@ from ..utils import ( ) from ..dependencies import get_collection_queryset, get_item_queryset, get_collection from ..sendfile import sendfile +from ..db_hack import django_db_cleanup_decorator collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -222,6 +223,7 @@ def collection_list_common( # permissions +@django_db_cleanup_decorator def verify_collection_admin( collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) ): @@ -229,6 +231,7 @@ def verify_collection_admin( raise PermissionDenied("admin_access_required", "Only collection admins can perform this operation.") +@django_db_cleanup_decorator def has_write_access( collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/routers/invitation.py b/etebase_fastapi/routers/invitation.py index aceb05d..cbe570b 100644 --- a/etebase_fastapi/routers/invitation.py +++ b/etebase_fastapi/routers/invitation.py @@ -19,6 +19,7 @@ from ..utils import ( PERMISSIONS_READ, PERMISSIONS_READWRITE, ) +from ..db_hack import django_db_cleanup_decorator User = get_typed_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -86,10 +87,12 @@ class InvitationListResponse(BaseModel): done: bool +@django_db_cleanup_decorator def get_incoming_queryset(user: UserType = Depends(get_authenticated_user)): return default_queryset.filter(user=user) +@django_db_cleanup_decorator def get_outgoing_queryset(user: UserType = Depends(get_authenticated_user)): return default_queryset.filter(fromMember__user=user) diff --git a/etebase_fastapi/routers/member.py b/etebase_fastapi/routers/member.py index 41393bf..38beb79 100644 --- a/etebase_fastapi/routers/member.py +++ b/etebase_fastapi/routers/member.py @@ -10,6 +10,7 @@ from .authentication import get_authenticated_user from ..msgpack import MsgpackRoute from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE from ..stoken_handler import filter_by_stoken_and_limit +from ..db_hack import django_db_cleanup_decorator from .collection import get_collection, verify_collection_admin @@ -19,10 +20,12 @@ MemberQuerySet = QuerySet[models.CollectionMember] default_queryset: MemberQuerySet = models.CollectionMember.objects.all() +@django_db_cleanup_decorator def get_queryset(collection: models.Collection = Depends(get_collection)) -> MemberQuerySet: return default_queryset.filter(collection=collection) +@django_db_cleanup_decorator def get_member(username: str, queryset: MemberQuerySet = Depends(get_queryset)) -> models.CollectionMember: return get_object_or_404(queryset, user__username__iexact=username) From 84870d25bfc1a3529e3349ebfccb62d806dfb154 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 3 Jan 2021 09:14:10 +0200 Subject: [PATCH 154/170] README: mention that Windows requires WSL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c110672..f8800ff 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Before installing the Etebase server make sure you install `virtualenv` (for **P * Arch Linux: `pacman -S python-virtualenv` * Debian/Ubuntu: `apt-get install python3-virtualenv` -* Mac/Windows/Other Linux: install virtualenv or just skip the instructions mentioning virtualenv. +* Mac/Windows (WSL)/Other Linux: install virtualenv or just skip the instructions mentioning virtualenv. Then just clone the git repo and set up this app: From 3894fd205b54302d113898ae73a60366807b8b31 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 3 Jan 2021 12:55:24 +0200 Subject: [PATCH 155/170] README: mention minimum requirements --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index f8800ff..1d39872 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ An [Etebase](https://www.etebase.com) (EteSync 2.0) server so you can run your o # Installation +## Requirements + +Etebase requires Python 3.7 or newer and has a few Python dependencies (listed in `requirements.in/base.txt`). + ## From source Before installing the Etebase server make sure you install `virtualenv` (for **Python 3**): From 43dede57d298bc3d1e0943ba0228a097ac0b019b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 4 Jan 2021 10:13:24 +0200 Subject: [PATCH 156/170] Exceptions: fix types. --- etebase_fastapi/exceptions.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index 72a3faf..d85c07e 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -2,6 +2,7 @@ from fastapi import status import typing as t from pydantic import BaseModel +from django.core.exceptions import ValidationError as DjangoValidationError class HttpErrorField(BaseModel): @@ -92,8 +93,8 @@ class ValidationError(HttpError): super().__init__(code=code, detail=detail, errors=errors, status_code=status_code) -def flatten_errors(field_name, errors) -> t.List[HttpError]: - ret = [] +def flatten_errors(field_name: str, errors) -> t.List[HttpError]: + ret: t.List[HttpError] = [] if isinstance(errors, dict): for error_key in errors: error = errors[error_key] @@ -104,11 +105,11 @@ def flatten_errors(field_name, errors) -> t.List[HttpError]: message = error.messages[0] else: message = str(error) - ret.append(dict(code=error.code, detail=message, field=field_name)) + ret.append(ValidationError(code=error.code, detail=message, field=field_name)) return ret -def transform_validation_error(prefix, err): +def transform_validation_error(prefix: str, err: DjangoValidationError): if hasattr(err, "error_dict"): errors = flatten_errors(prefix, err.error_dict) elif not hasattr(err, "message"): From 9bf118225d51c25371c3e9f572f1de7ae4c5ddf7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 4 Jan 2021 10:14:56 +0200 Subject: [PATCH 157/170] Exceptions: fix error when transforming django validation errors. --- etebase_fastapi/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index d85c07e..d38ef42 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -115,5 +115,5 @@ def transform_validation_error(prefix: str, err: DjangoValidationError): elif not hasattr(err, "message"): errors = flatten_errors(prefix, err.error_list) else: - raise HttpError(err.code, err.message) + raise HttpError(err.code or "validation_error", err.message) raise HttpError(code="field_errors", detail="Field validations failed.", errors=errors) From 9f0430a6da8a1de6916edba0b41411e9b0d68cc5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 4 Jan 2021 11:56:17 +0200 Subject: [PATCH 158/170] Improve types. --- etebase_fastapi/routers/authentication.py | 2 +- etebase_fastapi/utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/etebase_fastapi/routers/authentication.py b/etebase_fastapi/routers/authentication.py index be6a7e8..f8628b2 100644 --- a/etebase_fastapi/routers/authentication.py +++ b/etebase_fastapi/routers/authentication.py @@ -125,7 +125,7 @@ def get_login_user(request: Request, challenge: LoginChallengeIn) -> UserType: raise AuthenticationFailed(code="user_not_found", detail="User not found") -def get_encryption_key(salt): +def get_encryption_key(salt: bytes): key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) return nacl.hash.blake2b( b"", diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index c9db61c..9f915e2 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -50,21 +50,21 @@ def is_collection_admin(collection, user): return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) -def msgpack_encode(content): +def msgpack_encode(content) -> bytes: ret = msgpack.packb(content, use_bin_type=True) assert ret is not None return ret -def msgpack_decode(content): +def msgpack_decode(content: bytes): return msgpack.unpackb(content, raw=False) -def b64encode(value): +def b64encode(value: bytes): return base64.urlsafe_b64encode(value).decode("ascii").strip("=") -def b64decode(data): +def b64decode(data: str): data += "=" * ((4 - len(data) % 4) % 4) return base64.urlsafe_b64decode(data) From 1349f99cd39df34df87b0a53aecf8117876fd0a2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 4 Jan 2021 16:31:28 +0200 Subject: [PATCH 159/170] Exceptions: inherit from the fastapi HTTPException. --- etebase_fastapi/exceptions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index d38ef42..61e4439 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -1,4 +1,4 @@ -from fastapi import status +from fastapi import status, HTTPException import typing as t from pydantic import BaseModel @@ -23,11 +23,10 @@ class HttpErrorOut(BaseModel): orm_mode = True -class CustomHttpException(Exception): +class CustomHttpException(HTTPException): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): - self.status_code = status_code self.code = code - self.detail = detail + super().__init__(status_code, detail) @property def as_dict(self) -> dict: From cd4131e890bc9abe8bbedab726b8bed5acb9ef4e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 11 Jan 2021 22:12:31 +0200 Subject: [PATCH 160/170] Exceptions: make sure error codes aren't None. --- etebase_fastapi/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index 61e4439..d38d50a 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -72,7 +72,7 @@ class HttpError(CustomHttpException): errors: t.Optional[t.List["HttpError"]] = None, ): self.errors = errors - super().__init__(code=code, detail=detail, status_code=status_code) + super().__init__(code=code or "generic_error", detail=detail, status_code=status_code) @property def as_dict(self) -> dict: @@ -104,7 +104,7 @@ def flatten_errors(field_name: str, errors) -> t.List[HttpError]: message = error.messages[0] else: message = str(error) - ret.append(ValidationError(code=error.code, detail=message, field=field_name)) + ret.append(ValidationError(code=error.code or "validation_error", detail=message, field=field_name)) return ret From f52facad1c0adaa84b9418428b35c8e4fb067bf8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 4 Jan 2021 10:02:47 +0200 Subject: [PATCH 161/170] Subscriptions: implement live subscriptions for collection items --- django_etebase/app_settings_inner.py | 6 ++ etebase_fastapi/exceptions.py | 10 +++ etebase_fastapi/main.py | 16 ++++ etebase_fastapi/redis.py | 27 ++++++ etebase_fastapi/routers/collection.py | 26 +++++- etebase_fastapi/routers/websocket.py | 114 ++++++++++++++++++++++++++ 6 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 etebase_fastapi/redis.py create mode 100644 etebase_fastapi/routers/websocket.py diff --git a/django_etebase/app_settings_inner.py b/django_etebase/app_settings_inner.py index 90225a6..41fd910 100644 --- a/django_etebase/app_settings_inner.py +++ b/django_etebase/app_settings_inner.py @@ -11,6 +11,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import typing as t + from django.utils.functional import cached_property @@ -31,6 +33,10 @@ class AppSettings: return getattr(settings, self.prefix + name, dflt) + @cached_property + def REDIS_URI(self) -> t.Optional[str]: # pylint: disable=invalid-name + return self._setting("REDIS_URI", None) + @cached_property def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name perms = self._setting("API_PERMISSIONS_READ", tuple()) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index d38d50a..1a98fcb 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -63,6 +63,16 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) +class NotSupported(CustomHttpException): + def __init__( + self, + code="not_implemented", + detail: str = "This server's configuration does not support this request.", + status_code: int = status.HTTP_501_NOT_IMPLEMENTED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + class HttpError(CustomHttpException): def __init__( self, diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 8e8469c..d63c01d 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -5,12 +5,15 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware +from django_etebase import app_settings + from .exceptions import CustomHttpException from .msgpack import MsgpackResponse from .routers.authentication import authentication_router from .routers.collection import collection_router, item_router from .routers.member import member_router from .routers.invitation import invitation_incoming_router, invitation_outgoing_router +from .routers.websocket import websocket_router def create_application(prefix="", middlewares=[]): @@ -36,6 +39,7 @@ def create_application(prefix="", middlewares=[]): app.include_router( invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"] ) + app.include_router(websocket_router, prefix=f"{BASE_PATH}/ws", tags=["websocket"]) if settings.DEBUG: from etebase_fastapi.routers.test_reset_view import test_reset_view_router @@ -54,6 +58,18 @@ def create_application(prefix="", middlewares=[]): for middleware in middlewares: app.add_middleware(middleware) + @app.on_event("startup") + async def on_startup() -> None: + from .redis import redisw + + await redisw.setup() + + @app.on_event("shutdown") + async def on_shutdown(): + from .redis import redisw + + await redisw.close() + @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) diff --git a/etebase_fastapi/redis.py b/etebase_fastapi/redis.py new file mode 100644 index 0000000..3735e36 --- /dev/null +++ b/etebase_fastapi/redis.py @@ -0,0 +1,27 @@ +import typing as t +import aioredis + +from django_etebase import app_settings + + +class RedisWrapper: + redis: aioredis.Redis + + def __init__(self, redis_uri: t.Optional[str]): + self.redis_uri = redis_uri + + async def setup(self): + if self.redis_uri is not None: + self.redis = await aioredis.create_redis_pool(self.redis_uri) + + async def close(self): + if self.redis is not None: + self.redis.close() + await self.redis.wait_closed() + + @property + def is_active(self): + return self.redis_uri is not None + + +redisw = RedisWrapper(app_settings.REDIS_URI) diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index 4dcb3c6..df25541 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -1,6 +1,6 @@ import typing as t -from asgiref.sync import sync_to_async +from asgiref.sync import sync_to_async, async_to_sync from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile from django.db import transaction, IntegrityError @@ -10,6 +10,7 @@ from fastapi import APIRouter, Depends, status, Request from django_etebase import models from myauth.models import UserType from .authentication import get_authenticated_user +from .websocket import get_ticket, TicketRequest, TicketOut from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError from ..msgpack import MsgpackRoute from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken @@ -19,6 +20,7 @@ from ..utils import ( Prefetch, PrefetchQuery, is_collection_admin, + msgpack_encode, BaseModel, permission_responses, PERMISSIONS_READ, @@ -26,6 +28,7 @@ from ..utils import ( ) from ..dependencies import get_collection_queryset, get_item_queryset, get_collection from ..sendfile import sendfile +from ..redis import redisw from ..db_hack import django_db_cleanup_decorator collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -188,6 +191,16 @@ class ItemBatchIn(BaseModel): ) +# FIXME: make it a background task +def report_items_changed(col_uid: str, stoken: str, items: t.List[CollectionItemIn]): + if not redisw.is_active: + return + + redis = redisw.redis + content = msgpack_encode(CollectionItemListResponse(data=items, stoken=stoken, done=True).dict()) + async_to_sync(redis.publish)(f"col.{col_uid}", content) + + def collection_list_common( queryset: CollectionQuerySet, user: UserType, @@ -440,6 +453,15 @@ def item_list( return response +@item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ) +async def item_list_subscription_ticket( + collection: models.Collection = Depends(get_collection), + user: UserType = Depends(get_authenticated_user), +): + """Get an authentication ticket that can be used with the websocket endpoint""" + return await get_ticket(TicketRequest(collection=collection.uid), user) + + def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user) with transaction.atomic(): # We need this for locking the collection object @@ -465,6 +487,8 @@ def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], status_code=status.HTTP_409_CONFLICT, ) + report_items_changed(collection_object.uid, collection_object.stoken, data.items) + @item_router.get( "/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse, dependencies=PERMISSIONS_READ diff --git a/etebase_fastapi/routers/websocket.py b/etebase_fastapi/routers/websocket.py new file mode 100644 index 0000000..2d599db --- /dev/null +++ b/etebase_fastapi/routers/websocket.py @@ -0,0 +1,114 @@ +import asyncio +import typing as t + +import aioredis +from django.db.models import QuerySet +from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status +import nacl.encoding +import nacl.utils + +from django_etebase import models +from django_etebase.utils import CallbackContext, get_user_queryset +from myauth.models import UserType, get_typed_user_model + +from ..exceptions import NotSupported +from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode +from ..redis import redisw +from ..utils import BaseModel, permission_responses + + +User = get_typed_user_model() +websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +CollectionQuerySet = QuerySet[models.Collection] + + +TICKET_VALIDITY_SECONDS = 10 + + +class TicketRequest(BaseModel): + collection: str + + +class TicketOut(BaseModel): + ticket: str + + +class TicketInner(BaseModel): + user: int + req: TicketRequest + + +async def get_ticket( + ticket_request: TicketRequest, + user: UserType, +): + """Get an authentication ticket that can be used with the websocket endpoint for authentication""" + if not redisw.is_active: + raise NotSupported(detail="This end-point requires Redis to be configured") + + uid = nacl.encoding.URLSafeBase64Encoder.encode(nacl.utils.random(32)) + ticket_model = TicketInner(user=user.id, req=ticket_request) + ticket_raw = msgpack_encode(ticket_model.dict()) + await redisw.redis.set(uid, ticket_raw, expire=TICKET_VALIDITY_SECONDS * 1000) + return TicketOut(ticket=uid) + + +async def load_websocket_ticket(websocket: WebSocket, ticket: str) -> t.Optional[TicketInner]: + content = await redisw.redis.get(ticket) + if content is None: + await websocket.close(code=status.WS_1008_POLICY_VIOLATION) + return None + await redisw.redis.delete(ticket) + return TicketInner(**msgpack_decode(content)) + + +def get_websocket_user(websocket: WebSocket, ticket_model: t.Optional[TicketInner] = Depends(load_websocket_ticket)): + if ticket_model is None: + return None + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(websocket.path_params)) + return user_queryset.get(id=ticket_model.user) + + +@websocket_router.websocket("/{ticket}/") +async def websocket_endpoint( + websocket: WebSocket, + user: t.Optional[UserType] = Depends(get_websocket_user), + ticket_model: TicketInner = Depends(load_websocket_ticket), +): + if user is None: + return + await websocket.accept() + await redis_connector(websocket, ticket_model) + + +async def redis_connector(websocket: WebSocket, ticket_model: TicketInner): + async def producer_handler(r: aioredis.Redis, ws: WebSocket): + channel_name = f"col.{ticket_model.req.collection}" + (channel,) = await r.psubscribe(channel_name) + assert isinstance(channel, aioredis.Channel) + try: + while True: + # We wait on the websocket so we fail if web sockets fail or get data + receive = asyncio.create_task(websocket.receive()) + done, pending = await asyncio.wait( + {receive, channel.wait_message()}, return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + if receive in done: + # Web socket should never receieve any data + await websocket.close(code=status.WS_1008_POLICY_VIOLATION) + return + + message_raw = t.cast(t.Optional[t.Tuple[str, bytes]], await channel.get()) + if message_raw: + _, message = message_raw + await ws.send_bytes(message) + + except aioredis.errors.ConnectionClosedError: + await websocket.close(code=status.WS_1012_SERVICE_RESTART) + except WebSocketDisconnect: + pass + + redis = redisw.redis + await producer_handler(redis, websocket) From 61bd82f1e3dc6a993d823b0941fbb6f7a864f3fa Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 11 Jan 2021 18:39:01 +0200 Subject: [PATCH 162/170] Subscriptions: stream missing items if user passed an old stoken. --- etebase_fastapi/routers/websocket.py | 35 ++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/routers/websocket.py b/etebase_fastapi/routers/websocket.py index 2d599db..ad3331b 100644 --- a/etebase_fastapi/routers/websocket.py +++ b/etebase_fastapi/routers/websocket.py @@ -2,6 +2,7 @@ import asyncio import typing as t import aioredis +from asgiref.sync import sync_to_async from django.db.models import QuerySet from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status import nacl.encoding @@ -11,6 +12,7 @@ from django_etebase import models from django_etebase.utils import CallbackContext, get_user_queryset from myauth.models import UserType, get_typed_user_model +from ..dependencies import get_collection_queryset, get_item_queryset from ..exceptions import NotSupported from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode from ..redis import redisw @@ -72,20 +74,49 @@ def get_websocket_user(websocket: WebSocket, ticket_model: t.Optional[TicketInne @websocket_router.websocket("/{ticket}/") async def websocket_endpoint( websocket: WebSocket, + stoken: t.Optional[str], user: t.Optional[UserType] = Depends(get_websocket_user), ticket_model: TicketInner = Depends(load_websocket_ticket), ): if user is None: return await websocket.accept() - await redis_connector(websocket, ticket_model) + await redis_connector(websocket, ticket_model, user, stoken) -async def redis_connector(websocket: WebSocket, ticket_model: TicketInner): +async def send_item_updates( + websocket: WebSocket, + collection: models.Collection, + user: UserType, + stoken: t.Optional[str], +): + from .collection import item_list_common + + done = False + while not done: + queryset = await sync_to_async(get_item_queryset)(collection) + response = await sync_to_async(item_list_common)(queryset, user, stoken, limit=50, prefetch="auto") + done = response.done + if len(response.data) > 0: + await websocket.send_bytes(msgpack_encode(response.dict())) + + +async def redis_connector(websocket: WebSocket, ticket_model: TicketInner, user: UserType, stoken: t.Optional[str]): async def producer_handler(r: aioredis.Redis, ws: WebSocket): channel_name = f"col.{ticket_model.req.collection}" (channel,) = await r.psubscribe(channel_name) assert isinstance(channel, aioredis.Channel) + + # Send missing items if we are not up to date + queryset: QuerySet[models.Collection] = get_collection_queryset(user) + collection: t.Optional[models.Collection] = await sync_to_async( + queryset.filter(uid=ticket_model.req.collection).first + )() + if collection is None: + await websocket.close(code=status.WS_1008_POLICY_VIOLATION) + return + await send_item_updates(websocket, collection, user, stoken) + try: while True: # We wait on the websocket so we fail if web sockets fail or get data From 240469342bd57a32fdd54d0f106ce01dc75ae6c6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 12 Jan 2021 11:57:43 +0200 Subject: [PATCH 163/170] Move reporting item changes to a background task. --- etebase_fastapi/routers/collection.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index df25541..167179d 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -1,11 +1,11 @@ import typing as t -from asgiref.sync import sync_to_async, async_to_sync +from asgiref.sync import sync_to_async from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, status, Request, BackgroundTasks from django_etebase import models from myauth.models import UserType @@ -191,14 +191,13 @@ class ItemBatchIn(BaseModel): ) -# FIXME: make it a background task -def report_items_changed(col_uid: str, stoken: str, items: t.List[CollectionItemIn]): +async def report_items_changed(col_uid: str, stoken: str, items: t.List[CollectionItemIn]): if not redisw.is_active: return redis = redisw.redis content = msgpack_encode(CollectionItemListResponse(data=items, stoken=stoken, done=True).dict()) - async_to_sync(redis.publish)(f"col.{col_uid}", content) + await redis.publish(f"col.{col_uid}", content) def collection_list_common( @@ -462,7 +461,14 @@ async def item_list_subscription_ticket( return await get_ticket(TicketRequest(collection=collection.uid), user) -def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], uid: str, validate_etag: bool): +def item_bulk_common( + data: ItemBatchIn, + user: UserType, + stoken: t.Optional[str], + uid: str, + validate_etag: bool, + background_tasks: BackgroundTasks, +): queryset = get_collection_queryset(user) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) @@ -487,7 +493,7 @@ def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], status_code=status.HTTP_409_CONFLICT, ) - report_items_changed(collection_object.uid, collection_object.stoken, data.items) + background_tasks.add_task(report_items_changed, collection_object.uid, collection_object.stoken, data.items) @item_router.get( @@ -564,20 +570,22 @@ def fetch_updates( def item_transaction( collection_uid: str, data: ItemBatchIn, + background_tasks: BackgroundTasks, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user), ): - return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks) @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_batch( collection_uid: str, data: ItemBatchIn, + background_tasks: BackgroundTasks, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user), ): - return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks) # Chunks From 233aeab98b65118cb606954c242cee55557706ad Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 Jan 2021 09:35:36 +0200 Subject: [PATCH 164/170] Support login and invitations using email rather than just username. --- etebase_fastapi/routers/authentication.py | 4 ++-- etebase_fastapi/routers/invitation.py | 8 ++++---- etebase_fastapi/utils.py | 5 +++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/routers/authentication.py b/etebase_fastapi/routers/authentication.py index f8628b2..7ec6bc6 100644 --- a/etebase_fastapi/routers/authentication.py +++ b/etebase_fastapi/routers/authentication.py @@ -22,7 +22,7 @@ from django_etebase.utils import create_user, get_user_queryset, CallbackContext from myauth.models import UserType, get_typed_user_model from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError from ..msgpack import MsgpackRoute -from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs from ..dependencies import AuthData, get_auth_data, get_authenticated_user User = get_typed_user_model() @@ -114,7 +114,7 @@ class SignupIn(BaseModel): def get_login_user(request: Request, challenge: LoginChallengeIn) -> UserType: username = challenge.username - kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} + kwargs = get_user_username_email_kwargs(username) try: user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) user = user_queryset.get(**kwargs) diff --git a/etebase_fastapi/routers/invitation.py b/etebase_fastapi/routers/invitation.py index cbe570b..7e52978 100644 --- a/etebase_fastapi/routers/invitation.py +++ b/etebase_fastapi/routers/invitation.py @@ -12,6 +12,7 @@ from ..exceptions import HttpError, PermissionDenied from ..msgpack import MsgpackRoute from ..utils import ( get_object_or_404, + get_user_username_email_kwargs, Context, is_collection_admin, BaseModel, @@ -191,9 +192,8 @@ def outgoing_create( user: UserType = Depends(get_authenticated_user), ): collection = get_object_or_404(models.Collection.objects, uid=data.collection) - to_user = get_object_or_404( - get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), username=data.username - ) + kwargs = get_user_username_email_kwargs(data.username) + to_user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) context = Context(user, None) data.validate_db(context) @@ -238,7 +238,7 @@ def outgoing_fetch_user_profile( request: Request, user: UserType = Depends(get_authenticated_user), ): - kwargs = {User.USERNAME_FIELD: username.lower()} + kwargs = get_user_username_email_kwargs(username) user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) return UserInfoOut.from_orm(user_info) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 9f915e2..09c223e 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -69,6 +69,11 @@ def b64decode(data: str): return base64.urlsafe_b64decode(data) +def get_user_username_email_kwargs(username: str): + field_name = User.EMAIL_FIELD if "@" in username else User.USERNAME_FIELD + return {field_name + "__iexact": username.lower()} + + PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] From 0ac41e77e830fd5a64df6b46651cd1828d395fd7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 Jan 2021 14:38:10 +0200 Subject: [PATCH 165/170] Fix DB errors when using MySQL/MariaDB. Fixes #69. --- .../migrations/0037_auto_20210127_1237.py | 18 ++++++++++++++++++ django_etebase/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 django_etebase/migrations/0037_auto_20210127_1237.py diff --git a/django_etebase/migrations/0037_auto_20210127_1237.py b/django_etebase/migrations/0037_auto_20210127_1237.py new file mode 100644 index 0000000..06c31b2 --- /dev/null +++ b/django_etebase/migrations/0037_auto_20210127_1237.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2021-01-27 12:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0036_auto_20201214_1128'), + ] + + operations = [ + migrations.AlterField( + model_name='collectiontype', + name='uid', + field=models.BinaryField(db_index=True, editable=True, max_length=1024, unique=True), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index fa56a95..4640e6d 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -36,7 +36,7 @@ def stoken_annotation_builder(stoken_id_fields: t.List[str]): class CollectionType(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - uid = models.BinaryField(editable=True, blank=False, null=False, db_index=True, unique=True) + uid = models.BinaryField(editable=True, blank=False, null=False, db_index=True, unique=True, max_length=1024) objects: models.manager.BaseManager["CollectionType"] From e4361d2364c3ab0c0b808cb4c546bf6d36e53694 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 Jan 2021 18:48:56 +0200 Subject: [PATCH 166/170] Patch old DB migration in order to fix mysql issues. Really fix #69. --- django_etebase/migrations/0032_auto_20201013_1409.py | 2 +- django_etebase/migrations/0037_auto_20210127_1237.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/django_etebase/migrations/0032_auto_20201013_1409.py b/django_etebase/migrations/0032_auto_20201013_1409.py index 2bb3cb0..c6e92a2 100644 --- a/django_etebase/migrations/0032_auto_20201013_1409.py +++ b/django_etebase/migrations/0032_auto_20201013_1409.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="collectiontype", name="uid", - field=models.BinaryField(db_index=True, editable=True, unique=True), + field=models.BinaryField(db_index=True, editable=True, max_length=1024, unique=True), ), ] diff --git a/django_etebase/migrations/0037_auto_20210127_1237.py b/django_etebase/migrations/0037_auto_20210127_1237.py index 06c31b2..deaba1e 100644 --- a/django_etebase/migrations/0037_auto_20210127_1237.py +++ b/django_etebase/migrations/0037_auto_20210127_1237.py @@ -6,13 +6,13 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('django_etebase', '0036_auto_20201214_1128'), + ("django_etebase", "0036_auto_20201214_1128"), ] operations = [ migrations.AlterField( - model_name='collectiontype', - name='uid', + model_name="collectiontype", + name="uid", field=models.BinaryField(db_index=True, editable=True, max_length=1024, unique=True), ), ] From ef69954b6dd7b1e3e02a0fc148a1d1b9de2a4029 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 28 Jan 2021 17:44:17 +0200 Subject: [PATCH 167/170] requirements.txt: Add missing deps. --- requirements.in/base.txt | 2 ++ requirements.txt | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 44e2875..4b9b85b 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -4,3 +4,5 @@ pynacl fastapi typing_extensions uvicorn[standard] +aiofiles +aioredis diff --git a/requirements.txt b/requirements.txt index f59431b..213462f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,12 +4,16 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # +aiofiles==0.6.0 # via -r requirements.in/base.txt +aioredis==1.3.1 # via -r requirements.in/base.txt asgiref==3.3.1 # via django +async-timeout==3.0.1 # via aioredis cffi==1.14.4 # via pynacl click==7.1.2 # via uvicorn django==3.1.4 # via -r requirements.in/base.txt fastapi==0.63.0 # via -r requirements.in/base.txt h11==0.11.0 # via uvicorn +hiredis==1.1.0 # via aioredis httptools==0.1.1 # via uvicorn msgpack==1.0.2 # via -r requirements.in/base.txt pycparser==2.20 # via cffi From 848580604673f93fc54c9ea76b1aedd305b113af Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 28 Jan 2021 17:55:37 +0200 Subject: [PATCH 168/170] Easy config: add support for setting redis URI. --- etebase-server.ini.example | 1 + etebase_server/settings.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/etebase-server.ini.example b/etebase-server.ini.example index 2b4682a..13903b2 100644 --- a/etebase-server.ini.example +++ b/etebase-server.ini.example @@ -8,6 +8,7 @@ debug = false ;media_url = /user-media/ ;language_code = en-us ;time_zone = UTC +;redis_uri = redis://localhost:6379 [allowed_hosts] allowed_host1 = example.com diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 5d57ec0..80a6ee9 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -154,6 +154,9 @@ if any(os.path.isfile(x) for x in config_locations): TIME_ZONE = section.get("time_zone", TIME_ZONE) DEBUG = section.getboolean("debug", DEBUG) + if "redis_uri" in section: + ETEBASE_REDIS_URI = section.get("redis_uri") + if "allowed_hosts" in config: ALLOWED_HOSTS = [y for x, y in config.items("allowed_hosts")] From b33a3c882ee12e665f9ddfebbfa515315543f532 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 28 Jan 2021 18:10:12 +0200 Subject: [PATCH 169/170] Subscriptions: allow subscribing without setting an stoken. --- etebase_fastapi/routers/websocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/routers/websocket.py b/etebase_fastapi/routers/websocket.py index ad3331b..3fc535f 100644 --- a/etebase_fastapi/routers/websocket.py +++ b/etebase_fastapi/routers/websocket.py @@ -74,7 +74,7 @@ def get_websocket_user(websocket: WebSocket, ticket_model: t.Optional[TicketInne @websocket_router.websocket("/{ticket}/") async def websocket_endpoint( websocket: WebSocket, - stoken: t.Optional[str], + stoken: t.Optional[str] = None, user: t.Optional[UserType] = Depends(get_websocket_user), ticket_model: TicketInner = Depends(load_websocket_ticket), ): From 14b3cfca556ae117e0f28017a0b2516a19c1ec7d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 1 Feb 2021 18:40:11 +0200 Subject: [PATCH 170/170] Handle stoken being the empty string. For whatever reason some users were getting this which was causing this code to fail. --- etebase_fastapi/routers/collection.py | 2 +- etebase_fastapi/stoken_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/routers/collection.py b/etebase_fastapi/routers/collection.py index 167179d..c8146f2 100644 --- a/etebase_fastapi/routers/collection.py +++ b/etebase_fastapi/routers/collection.py @@ -473,7 +473,7 @@ def item_bulk_common( with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) - if stoken is not None and stoken != collection_object.stoken: + if stoken and stoken != collection_object.stoken: raise HttpError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) data.validate_db() diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index 76d348a..b6f2999 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -12,7 +12,7 @@ StokenAnnotation = t.Any def get_stoken_obj(stoken: t.Optional[str]) -> t.Optional[Stoken]: - if stoken is not None: + if stoken: try: return Stoken.objects.get(uid=stoken) except Stoken.DoesNotExist: