From cc05db83cb588c8d71e3270884ec835141b6b34c Mon Sep 17 00:00:00 2001 From: Andrew Nichols Date: Wed, 26 Oct 2022 16:42:11 -0400 Subject: [PATCH 1/2] Extract users, memberships when copying a project I wasn't able to directly use `copy_table` because of the approach that function takes with primary key popping/auto-incrementation/re-insertion, since users have string pkey. Instead, there's a series of execute calls to conditionally insert users into the dest *if* they do not already exist. This allows a project with users A,B,C to safely be exported and imported to a given server and all the links between commands/users/memberships will be preserved. Closes: #248 --- taguette/database/copy.py | 45 +++++++++++++++++++++-------- tests.py | 60 ++++++++++++++++++++------------------- 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/taguette/database/copy.py b/taguette/database/copy.py index 3e9921a1..1063874c 100644 --- a/taguette/database/copy.py +++ b/taguette/database/copy.py @@ -2,7 +2,7 @@ import logging import opentelemetry.trace from .models import Project, Privileges, ProjectMember, TextDirection, \ - Document, Command, Highlight, Tag, highlight_tags + Document, Command, Highlight, Tag, highlight_tags, User from .. import convert from ..utils import DefaultMap from .. import validate @@ -48,15 +48,36 @@ def copy_project( new_project_id, = insert(Project.__table__, project).inserted_primary_key mapping_project = {project_id: new_project_id} - # Add member - insert( - ProjectMember.__table__, - dict( - project_id=new_project_id, - user_login=user_login, - privileges=Privileges.ADMIN, - ), - ) + # Copy all users and memberships that are part of the project + members = src_db.execute( + ProjectMember.__table__ + .select() + .where(ProjectMember.project_id == project_id) + ).all() + for member in members: + existing_user = dest_db.execute( + User.__table__ + .select(). + where(User.login == member.user_login) + ).one_or_none() + if existing_user is None: + new_user = src_db.execute( + User.__table__ + .select() + .where(User.login == member.user_login) + ).one() + insert(User.__table__, dict(new_user)) + + insert( + ProjectMember.__table__, + dict( + project_id=new_project_id, + user_login=member.user_login, + privileges=member.privileges + ), + ) + + mapping_user = {member['user_login']: member['user_login'] for member in members} # Copy documents mapping_document = copy( @@ -178,11 +199,11 @@ def copy_project( return dict(cmd.items(), payload=payload) + copy( Command.__table__, 'id', dict( - # Map all users to the importing user - user_login=DefaultMap(lambda key: user_login, {}), + user_login=mapping_user, project_id=mapping_project, # Map None to None and unknown keys (deleted documents) to negative document_id=DefaultMap( diff --git a/tests.py b/tests.py index fe7da4bb..f98f9a1b 100644 --- a/tests.py +++ b/tests.py @@ -1578,7 +1578,7 @@ class TestMultiuser(MyHTTPTestCase): {row[0] for row in db1.execute( sqlalchemy.select([database.User.__table__.c.login]) )}, - {'admin', 'db1user'}, + {'admin', 'db1user', 'db2user'}, ) self.assertRowsEqualsExceptDates( db1.execute( @@ -1602,7 +1602,8 @@ class TestMultiuser(MyHTTPTestCase): (1, 'db1user', database.Privileges.ADMIN), (2, 'admin', database.Privileges.ADMIN), (2, 'db1user', database.Privileges.ADMIN), - (3, 'db1user', database.Privileges.ADMIN), + (3, 'admin', database.Privileges.ADMIN), + (3, 'db2user', database.Privileges.ADMIN), ], ) self.assertEqual( @@ -1635,42 +1636,42 @@ class TestMultiuser(MyHTTPTestCase): # tags 1, 2, 3 imported as 7, 8, -3 # highlights 1, 2, 3 imported as 7, 8, -3 # commands 1-13 exported as 27-39 - (27, 'db1user', 3, 7, + (27, 'db2user', 3, 7, {'type': 'document_add', 'description': '', 'text_direction': 'LEFT_TO_RIGHT', 'document_name': 'db2doc11.txt'}), - (28, 'db1user', 3, -3, + (28, 'db2user', 3, -3, {'type': 'document_add', 'description': '', 'text_direction': 'LEFT_TO_RIGHT', 'document_name': 'db2doc1100.txt'}), - (29, 'db1user', 3, 8, + (29, 'db2user', 3, 8, {'type': 'document_add', 'description': '', 'text_direction': 'RIGHT_TO_LEFT', 'document_name': 'db2doc12.txt'}), - (30, 'db1user', 3, -3, {'type': 'document_delete'}), - (31, 'db1user', 3, None, + (30, 'db2user', 3, -3, {'type': 'document_delete'}), + (31, 'db2user', 3, None, {'type': 'tag_add', 'description': '', 'tag_id': 7, 'tag_path': 'db2tag11'}), - (32, 'db1user', 3, None, + (32, 'db2user', 3, None, {'type': 'tag_add', 'description': '', 'tag_id': -3, 'tag_path': 'db2tagF'}), - (33, 'db1user', 3, None, + (33, 'db2user', 3, None, {'type': 'tag_add', 'description': '', 'tag_id': 8, 'tag_path': 'db2tag12'}), - (34, 'db1user', 3, None, {'type': 'tag_delete', 'tag_id': -3}), - (35, 'db1user', 3, 7, + (34, 'db2user', 3, None, {'type': 'tag_delete', 'tag_id': -3}), + (35, 'db2user', 3, 7, {'type': 'highlight_add', 'highlight_id': 7, 'start_offset': 3, 'end_offset': 6, 'tags': []}), - (36, 'db1user', 3, 7, + (36, 'db2user', 3, 7, {'type': 'highlight_add', 'highlight_id': 7, 'start_offset': 3, 'end_offset': 6, 'tags': [8]}), - (37, 'db1user', 3, 7, + (37, 'db2user', 3, 7, {'type': 'highlight_add', 'highlight_id': -3, 'start_offset': 3, 'end_offset': 6, 'tags': [7]}), - (38, 'db1user', 3, 7, + (38, 'db2user', 3, 7, {'type': 'highlight_add', 'highlight_id': 8, 'start_offset': 3, 'end_offset': 6, 'tags': [7]}), - (39, 'db1user', 3, 7, + (39, 'db2user', 3, 7, {'type': 'highlight_delete', 'highlight_id': -3}), (40, 'db1user', 3, None, {'type': 'project_import'}), ], @@ -1745,7 +1746,7 @@ class TestMultiuser(MyHTTPTestCase): {row[0] for row in db2.execute( sqlalchemy.select([database.User.__table__.c.login]) )}, - {'admin'}, + {'db1user', 'admin'}, ) self.assertRowsEqualsExceptDates( db2.execute( @@ -1763,6 +1764,7 @@ class TestMultiuser(MyHTTPTestCase): ), [ (1, 'admin', database.Privileges.ADMIN), + (1, 'db1user', database.Privileges.ADMIN), ], ) self.assertEqual( @@ -1790,43 +1792,43 @@ class TestMultiuser(MyHTTPTestCase): # tags 4, 5, 6 exported as 1, 2, -6 # highlights 4, 5, 6 exported as 1, 2, -6 # commands 14-26 exported as 1-13 - (1, 'admin', 1, 1, + (1, 'db1user', 1, 1, {'type': 'document_add', 'description': '', 'text_direction': 'LEFT_TO_RIGHT', 'document_name': 'db1doc21.txt'}), - (2, 'admin', 1, -6, + (2, 'db1user', 1, -6, {'type': 'document_add', 'description': '', 'text_direction': 'LEFT_TO_RIGHT', 'document_name': 'db1doc2100.txt'}), - (3, 'admin', 1, 2, + (3, 'db1user', 1, 2, {'type': 'document_add', 'description': '', 'text_direction': 'RIGHT_TO_LEFT', 'document_name': 'db1doc22.txt'}), - (4, 'admin', 1, -6, {'type': 'document_delete'}), - (5, 'admin', 1, None, + (4, 'db1user', 1, -6, {'type': 'document_delete'}), + (5, 'db1user', 1, None, {'type': 'tag_add', 'description': '', 'tag_id': 1, 'tag_path': 'db1tag21'}), - (6, 'admin', 1, None, + (6, 'db1user', 1, None, {'type': 'tag_add', 'description': '', 'tag_id': -6, 'tag_path': 'db1tagF'}), - (7, 'admin', 1, None, + (7, 'db1user', 1, None, {'type': 'tag_add', 'description': '', 'tag_id': 2, 'tag_path': 'db1tag22'}), - (8, 'admin', 1, None, + (8, 'db1user', 1, None, {'type': 'tag_delete', 'tag_id': -6}), - (9, 'admin', 1, 1, + (9, 'db1user', 1, 1, {'type': 'highlight_add', 'highlight_id': 1, 'start_offset': 3, 'end_offset': 6, 'tags': []}), - (10, 'admin', 1, 1, + (10, 'db1user', 1, 1, {'type': 'highlight_add', 'highlight_id': 1, 'start_offset': 3, 'end_offset': 6, 'tags': [2]}), - (11, 'admin', 1, 1, + (11, 'db1user', 1, 1, {'type': 'highlight_add', 'highlight_id': -6, 'start_offset': 3, 'end_offset': 6, 'tags': [1]}), - (12, 'admin', 1, 1, + (12, 'db1user', 1, 1, {'type': 'highlight_add', 'highlight_id': 2, 'start_offset': 3, 'end_offset': 6, 'tags': [1]}), - (13, 'admin', 1, 1, + (13, 'db1user', 1, 1, {'type': 'highlight_delete', 'highlight_id': -6}), ], ) -- GitLab From fa1998a0fe36c4bc53e2f965962616f8f76161a3 Mon Sep 17 00:00:00 2001 From: Andrew Nichols Date: Wed, 26 Oct 2022 17:21:43 -0400 Subject: [PATCH 2/2] linting --- taguette/database/copy.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/taguette/database/copy.py b/taguette/database/copy.py index 1063874c..c97c2ded 100644 --- a/taguette/database/copy.py +++ b/taguette/database/copy.py @@ -50,34 +50,29 @@ def copy_project( # Copy all users and memberships that are part of the project members = src_db.execute( - ProjectMember.__table__ - .select() - .where(ProjectMember.project_id == project_id) - ).all() + ProjectMember.__table__ + .select() + .where(ProjectMember.project_id == project_id)).all() for member in members: existing_user = dest_db.execute( - User.__table__ - .select(). - where(User.login == member.user_login) - ).one_or_none() + User.__table__ + .select(). + where(User.login == member.user_login)).one_or_none() if existing_user is None: new_user = src_db.execute( - User.__table__ - .select() - .where(User.login == member.user_login) - ).one() + User.__table__ + .select() + .where(User.login == member.user_login)).one() insert(User.__table__, dict(new_user)) - insert( - ProjectMember.__table__, - dict( - project_id=new_project_id, - user_login=member.user_login, - privileges=member.privileges - ), - ) + insert(ProjectMember.__table__, + dict( + project_id=new_project_id, + user_login=member.user_login, + privileges=member.privileges + )) - mapping_user = {member['user_login']: member['user_login'] for member in members} + mapping_user = {m['user_login']: m['user_login'] for m in members} # Copy documents mapping_document = copy( @@ -199,7 +194,6 @@ def copy_project( return dict(cmd.items(), payload=payload) - copy( Command.__table__, 'id', dict( -- GitLab