Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions diracx-core/src/diracx/core/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,22 +80,14 @@ def read_raw(self, hexsha: str, modified: datetime) -> T:
"""

def read(self) -> T:
"""Load the source from the backend with appropriate caching.

:raises: diracx.core.exceptions.NotReadyError if the source is being loaded still
:raises: git.exc.BadName if version does not exist
Comment on lines -85 to -86

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Raises have been deleted, is that expected?

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think so because there is no explicit 'raise' in the core of the function, so we don't normally have to put it in, do we?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think @devlink42 is right here.

"""
"""Load the source from the backend with appropriate caching."""
hexsha = self._revision_cache.get(
"latest_revision", self._read_work, blocking=True
)
return self._content_cache[hexsha]

async def read_non_blocking(self) -> T:
"""Load the source from the backend with appropriate caching.

:raises: diracx.core.exceptions.NotReadyError if the source is being loaded still
:raises: git.exc.BadName if version does not exist
Comment on lines -96 to -97

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here, they have been deleted

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No explicit rase.

"""
"""Load the source from the backend with appropriate caching."""
hexsha = self._revision_cache.get(
"latest_revision", self._read_work, blocking=False
)
Expand Down
9 changes: 5 additions & 4 deletions diracx-db/src/diracx/db/sql/auth/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,8 @@ async def device_flow_validate_user_code(

Returns the scope field for the given user_code

:raises:
NoResultFound if no such user code currently Pending
Raises:
NoResultFound: if no such user code currently Pending
"""
stmt = select(DeviceFlows.scope).where(
DeviceFlows.user_code == user_code,
Expand Down Expand Up @@ -271,9 +271,10 @@ async def insert_authorization_flow(
async def authorization_flow_insert_id_token(
self, uuid: str, id_token: dict[str, str], max_validity: int
) -> tuple[str, str]:
"""Return code, redirect_uri.
"""Return the authorization code and redirect URI.

:raises: AuthorizationError if no such uuid or status not pending.
Raises:
AuthorizationError: If no such UUID exists or the flow is not pending.
"""
# Hash the code to avoid leaking information
code = secrets.token_urlsafe()
Expand Down
22 changes: 14 additions & 8 deletions diracx-db/src/diracx/db/sql/job/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,10 +200,12 @@ async def set_properties(
All the jobs must update the same properties.
:param properties: {job_id : {prop1: val1, prop2:val2}
:param update_timestamp: if True, update the LastUpdate to now
Args:
properties: {job_id : {prop1: val1, prop2:val2}
update_timestamp: if True, update the LastUpdate to now
:return rowcount
Returns:
rowcount
Comment thread
devlink42 marked this conversation as resolved.
"""
# Check that all we always update the same set of properties
Expand Down Expand Up @@ -238,9 +240,10 @@ async def add_heartbeat_data(
This is instead handled by the `diracx.logic.jobs.status.set_job_statuses`
as it involves updating multiple databases.
:param job_id: the job id
:param dynamic_data: mapping of the dynamic data to store,
e.g. {"AvailableDiskSpace": 123}
Args:
job_id: the job id
dynamic_data: mapping of the dynamic data to store, e.g.
{"AvailableDiskSpace": 123}
"""
if extra_fields := set(dynamic_data) - self.heartbeat_fields:
raise InvalidQueryError(
Expand All @@ -261,8 +264,11 @@ async def add_heartbeat_data(
async def get_job_commands(self, job_ids: Iterable[int]) -> list[JobCommand]:
"""Get a command to be passed to the job together with the next heartbeat.
:param job_ids: the job ids
:return: mapping of job id to list of commands
Args:
job_ids: the job ids
Returns:
mapping of job id to list of commands
"""
# Get the commands
stmt = (
Expand Down
4 changes: 2 additions & 2 deletions diracx-logic/src/diracx/logic/auth/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,8 @@ def parse_and_validate_scope(
* properties are known
return dict with group and properties.

:raises:
* ValueError in case the scope isn't valid
Raises:
ValueError: in case the scope isn't valid
"""
scopes = set(scope.split(" "))

Expand Down
3 changes: 2 additions & 1 deletion diracx-logic/src/diracx/logic/jobs/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ async def set_job_statuses(
logging information in the JobLoggingDB. The status dict has datetime
as a key and status information dictionary as values.

:raises: JobNotFound if the job is not found in one of the DBs
Raises:
JobNotFound: if the job is not found in one of the DBs
"""
# check that the datetime contains timezone info
for job_id, status in status_changes.items():
Expand Down
14 changes: 9 additions & 5 deletions diracx-logic/src/diracx/logic/task_queues/priority.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,15 @@ async def calculate_priority(
) -> dict[float, list[int]]:
"""Calculate the priority for each TQ given a share.

:param tq_dict: dict of {tq_id: prio}
:param all_tqs_data: dict of {tq_id: {tq_data}}, where tq_data is a dict of {field: value}
:param share: share to be distributed among TQs
:param allow_bg_tqs: allow background TQs to be used
:return: dict of {priority: [tq_ids]}
Args:
tq_dict: dict of {tq_id: prio}
all_tqs_data: dict of {tq_id: {tq_data}}, where tq_data is a
dict of {field: value}
share: share to be distributed among TQs
allow_bg_tqs: allow background TQs to be used

Returns:
dict of {priority: [tq_ids]}
"""

def is_background(tq_priority: float, allow_bg_tqs: bool) -> bool:
Expand Down
10 changes: 6 additions & 4 deletions diracx-routers/src/diracx/routers/access_policies.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
* have the open_access decorator to make explicit that it does not implement policy
* have a callable and call it that will perform the access policy


Adding a new policy:
1. Create a class that inherits from BaseAccessPolicy and implement the ``policy`` and ``enrich_tokens`` methods
2. create an entry in diracx.access_policy entrypoints
Expand Down Expand Up @@ -97,9 +96,12 @@ def enrich_tokens(

Content can be whatever is desired inside the access or refresh payload.

:param access_payload: access token payload
:param refresh_payload: refresh token payload
:returns: extra content for both payload
Args:
access_payload: access token payload
refresh_payload: refresh token payload

Returns:
extra content for both payload
"""
return {}, {}

Expand Down
26 changes: 11 additions & 15 deletions diracx-routers/src/diracx/routers/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,21 +115,17 @@ def create_app_inner(
the actual behavior we are interested in for settings, DBs or policy.
This allows an extension to override any of these components


:param enabled_system:
this contains the name of all the routers we have to load
:param all_service_settings:
list of instance of each Settings type required
:param database_urls:
dict <db_name: url>. When testing, sqlite urls are used
:param os_database_conn_kwargs:
<db_name:dict> containing all the parameters the OpenSearch client takes
:param config_source:
Source of the configuration to use
:param all_access_policies:
<policy_name: [implementations]>


Args:
enabled_systems: this contains the name of all the routers we
have to load
all_service_settings: list of instance of each Settings type
required
database_urls: dict <db_name: url>. When testing, sqlite urls
are used
os_database_conn_kwargs: <db_name:dict> containing all the
parameters the OpenSearch client takes
config_source: Source of the configuration to use
all_access_policies: <policy_name: [implementations]>
"""
app = DiracFastAPI()

Expand Down
8 changes: 2 additions & 6 deletions extensions/gubbins/gubbins-cli/src/gubbins/cli/config.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
"""
This just shows how to extend and modify an existing CLI
"""
"""This just shows how to extend and modify an existing CLI"""

# In order to extend it, just import the app from DiracX
from __future__ import annotations
Expand All @@ -12,9 +10,7 @@

@app.async_command()
async def gubbins_extra():
"""
Add an extra command
"""
"""Add an extra command"""
print("Adding something extra")


Expand Down
15 changes: 4 additions & 11 deletions extensions/gubbins/gubbins-cli/src/gubbins/cli/lollygag.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
"""
This shows how to create a new subcommand
"""
"""This shows how to create a new subcommand"""

from __future__ import annotations

Expand All @@ -15,16 +13,13 @@

@app.command()
def hello():
"""
This is just to make sure that the CLI extension mechanism works
"""
"""This is just to make sure that the CLI extension mechanism works"""
print("Shagadelic, Baby!")


@app.async_command()
async def get_owners():
"""
This makes a proper use of the AsyncGubbinsClient to call
"""This makes a proper use of the AsyncGubbinsClient to call
a method specific to Gubbins
"""
async with AsyncGubbinsClient() as api:
Expand All @@ -34,9 +29,7 @@ async def get_owners():

@app.async_command()
async def sensei():
"""
This function is only here to test the GUBBINS_SENSEI property
"""
"""This function is only here to test the GUBBINS_SENSEI property"""
async with AsyncGubbinsClient() as api:
secrets = await api.lollygag.get_gubbins_secrets()
print(secrets)
13 changes: 4 additions & 9 deletions extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,22 @@


class GubbinsJobDB(JobDB):
"""
This DB extends the diracx JobDB.
"""This DB extends the diracx JobDB.
All methods from the parent DB are accessible

"""

metadata = JobDBBase.metadata

async def insert_gubbins_info(self, job_id: int, info: str):
"""
This is a new method that makes use of a new table.
"""
"""This is a new method that makes use of a new table."""
stmt = insert(GubbinsInfo).values(job_id=job_id, info=info)
await self.conn.execute(stmt)

async def get_job_jdls( # type: ignore[override]
self, job_ids, original: bool = False, with_info=False
) -> dict:
"""
This method modifies the one in the parent class:
"""This method modifies the one in the parent class:
* adds an extra argument
* changes the return type

Expand All @@ -50,8 +46,7 @@ async def get_job_jdls( # type: ignore[override]
return result

async def set_job_attributes(self, job_data):
"""
This method modified the one in the parent class,
"""This method modified the one in the parent class,
without changing the argument nor the return type

Also, this method is called by the router via the status_utility
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@


class LollygagDB(BaseSQLDB):
"""
This LollygagDB is just to illustrate some important aspect of writing
"""This LollygagDB is just to illustrate some important aspect of writing
DB classes in DiracX.

It is mostly pure SQLAlchemy, with a few convention
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""
Lollygag dummy AccessPolicy
"""Lollygag dummy AccessPolicy
Makes sure we can use Gubbins specific property

"""
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""
This router makes use of the new LollygagDB.
"""This router makes use of the new LollygagDB.
It uses the the Lollygag AccessPolicy (which itself requires the Gubbins property)
"""

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""
Illustrate how to extend/overwrite a diracx router.
"""Illustrate how to extend/overwrite a diracx router.
It :
* changes slightly the return type
* uses the Gubbins specific configuration
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ max-line-length = 120
[tool.ruff.format]
docstring-code-format = true

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.isort]
profile = "black"
Expand Down
Loading