Skip to content

[DOP-17107] dump connection auth data #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dmitry-pedchenko merged 14 commits into from
Jun 20, 2024
Merged

[DOP-17107] dump connection auth data #63

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
b454bd6
[DOP-17107] bugfix
dmitry-pedchenko Jun 19, 2024
f843128
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jun 19, 2024
2467696
[DOP-17107] bugfix
dmitry-pedchenko Jun 19, 2024
5a0dbeb
[DOP-17107] bugfix
dmitry-pedchenko Jun 19, 2024
824129d
[DOP-17107] bugfix
dmitry-pedchenko Jun 19, 2024
865278a
[DOP-17107] bugfix
dmitry-pedchenko Jun 19, 2024
4f7f92c
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
656bcef
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
5d51a9d
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
28f1dc7
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jun 20, 2024
dac209f
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
84d2ace
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
fb8c2cc
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
e476026
[DOP-17107] bugfix
dmitry-pedchenko Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions syncmaster/backend/api/v1/transfers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,8 @@ async def start_run(

transfer = await unit_of_work.transfer.read_by_id(transfer_id=create_run_data.transfer_id)

# The credentials.read method is used rather than credentials.read_bulk deliberately
# it’s more convenient to transfer credits in this place
credentials_source = await unit_of_work.credentials.read(
transfer.source_connection_id,
)
Expand Down
12 changes: 8 additions & 4 deletions tests/test_integration/test_run_transfer/test_hdfs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,10 @@ async def test_run_transfer_hdfs_to_postgres(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=postgres,
Expand Down Expand Up @@ -213,8 +215,10 @@ async def test_run_transfer_postgres_to_hdfs(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = FileDFReader(
connection=hdfs_file_df_connection,
Expand Down
24 changes: 16 additions & 8 deletions tests/test_integration/test_run_transfer/test_hve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,10 @@ async def test_run_transfer_postgres_to_hive(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=hive,
Expand Down Expand Up @@ -143,8 +145,10 @@ async def test_run_transfer_postgres_to_hive_mixed_naming(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]
reader = DBReader(
connection=hive,
table="default.target_table",
Expand Down Expand Up @@ -188,8 +192,10 @@ async def test_run_transfer_hive_to_postgres(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]
reader = DBReader(
connection=postgres,
table="public.target_table",
Expand Down Expand Up @@ -229,8 +235,10 @@ async def test_run_transfer_hive_to_postgres_mixes_naming(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=postgres,
Expand Down
24 changes: 16 additions & 8 deletions tests/test_integration/test_run_transfer/test_oracle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,10 @@ async def test_run_transfer_postgres_to_oracle(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]
reader = DBReader(
connection=oracle,
table=f"{oracle.user}.target_table",
Expand Down Expand Up @@ -145,8 +147,10 @@ async def test_run_transfer_postgres_to_oracle_mixed_naming(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=oracle,
Expand Down Expand Up @@ -190,8 +194,10 @@ async def test_run_transfer_oracle_to_postgres(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=postgres,
Expand Down Expand Up @@ -233,8 +239,10 @@ async def test_run_transfer_oracle_to_postgres_mixed_naming(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=postgres,
Expand Down
12 changes: 8 additions & 4 deletions tests/test_integration/test_run_transfer/test_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,10 @@ async def test_run_transfer_s3_to_postgres(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
Comment thread
dolfinus marked this conversation as resolved.
Outdated
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = DBReader(
connection=postgres,
Expand Down Expand Up @@ -212,8 +214,10 @@ async def test_run_transfer_postgres_to_s3(
token=group_owner.token,
)
assert run_data["status"] == Status.FINISHED.value
assert run_data["transfer_dump"]["source_connection"]["auth_data"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]
assert run_data["transfer_dump"]["source_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["source_connection"]["auth_data"]["password"]
assert run_data["transfer_dump"]["target_connection"]["auth_data"]["user"]
assert not run_data["transfer_dump"]["target_connection"]["auth_data"]["password"]

reader = FileDFReader(
connection=s3_file_df_connection,
Expand Down