From 5fd5c41ebd1780dc895c24d9ec978f63dc848c6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Tue, 10 Sep 2024 22:00:50 +0200
Subject: [PATCH 01/10] add: push file
---
dinamis_sdk/__init__.py | 4 +++-
dinamis_sdk/s3.py | 46 +++++++++++++++++++++++++++++++++--------
dinamis_sdk/upload.py | 34 ++++++++++++++++++++++++++++++
tests/test_push.py | 13 ++++++++++++
4 files changed, 87 insertions(+), 10 deletions(-)
create mode 100644 dinamis_sdk/upload.py
create mode 100644 tests/test_push.py
diff --git a/dinamis_sdk/__init__.py b/dinamis_sdk/__init__.py
index dfa1b29..a20433c 100644
--- a/dinamis_sdk/__init__.py
+++ b/dinamis_sdk/__init__.py
@@ -7,6 +7,8 @@ from dinamis_sdk.s3 import (
sign_urls,
sign_item,
sign_asset,
- sign_item_collection
+ sign_item_collection,
+ sign_url_put
) # noqa
from dinamis_sdk import auth # noqa
+from dinamis_sdk.upload import push
diff --git a/dinamis_sdk/s3.py b/dinamis_sdk/s3.py
index f544a13..b445a4b 100644
--- a/dinamis_sdk/s3.py
+++ b/dinamis_sdk/s3.py
@@ -28,11 +28,11 @@ import pydantic
from .utils import (
log,
- SIGNED_URL_TTL_MARGIN,
- CREDENTIALS,
+ SIGNED_URL_TTL_MARGIN,
+ CREDENTIALS,
MAX_URLS,
- S3_SIGNING_ENDPOINT,
- S3_STORAGE_DOMAIN,
+ S3_SIGNING_ENDPOINT,
+ S3_STORAGE_DOMAIN,
SIGNED_URL_DURATION_SECONDS,
BYPASS_API
)
@@ -154,7 +154,10 @@ def sign_string(url: str, copy: bool = True) -> str:
return sign_urls(urls=[url])[url]
-def sign_urls(urls: List[str]) -> Dict[str, str]:
+def _generic_sign_urls(
+ urls: List[str],
+ route: str
+) -> Dict[str, str]:
"""Sign URLs with a S3 Token.
Signing URL allows read access to files in storage.
@@ -165,6 +168,7 @@ def sign_urls(urls: List[str]) -> Dict[str, str]:
Single URLs can be found on a STAC Item's Asset ``href`` value.
Only URLs to assets in S3 Storage are signed, other URLs are
returned unmodified.
+ route: API route
Returns:
dict of signed HREF: key = original URL, value = signed URL
@@ -193,18 +197,38 @@ def sign_urls(urls: List[str]) -> Dict[str, str]:
not_signed_urls = [url for url in urls if url not in signed_urls]
signed_urls.update({
url: signed_url.href
- for url, signed_url in get_signed_urls(not_signed_urls).items()
+ for url, signed_url in _generic_get_signed_urls(
+ urls=not_signed_urls,
+ route=route
+ ).items()
})
return signed_urls
+def sign_urls(urls: List[str]) -> Dict[str, str]:
+ return _generic_sign_urls(urls=urls, route="sign_urls")
+
+
+def sign_urls_put(urls: List[str]) -> Dict[str, str]:
+ return _generic_sign_urls(urls=urls, route="sign_urls_put")
+
+
+def sign_url_put(url: str) -> str:
+ """
+ Sign a single URL for put
+ """
+ urls = sign_urls_put([url])
+ return urls[url]
+
+
def _repl_vrt(match: re.Match) -> str:
# replace all blob-storages URLs with a signed version.
url = match.string[slice(*match.span())]
return sign_urls(url)[url]
-def sign_vrt_string(vrt: str, copy: bool = True) -> str: # pylint: disable = W0613 # noqa: E501
+def sign_vrt_string(vrt: str,
+ copy: bool = True) -> str: # pylint: disable = W0613 # noqa: E501
"""Sign a VRT-like string containing URLs from the storage.
Signing URLs allows read access to files in storage.
@@ -413,8 +437,9 @@ def sign_mapping(mapping: Mapping, copy: bool = True) -> Mapping:
sign_reference_file = sign_mapping
-def get_signed_urls(
+def _generic_get_signed_urls(
urls: List[str],
+ route: str,
retry_total: int = 10,
retry_backoff_factor: float = .8
) -> Dict[str, SignedURL]:
@@ -426,6 +451,7 @@ def get_signed_urls(
Args:
urls: urls
+ route: route (API)
retry_total (int): The number of allowable retry attempts for REST API
calls. Use retry_total=0 to disable retries. A backoff factor to
apply between attempts.
@@ -501,8 +527,10 @@ def get_signed_urls(
params = {"urls": not_signed_urls_chunk}
if SIGNED_URL_DURATION_SECONDS:
params["duration_seconds"] = SIGNED_URL_DURATION_SECONDS
+ post_url = f"{S3_SIGNING_ENDPOINT}{route}"
+ log.debug("POST %s", post_url)
response = session.post(
- f"{S3_SIGNING_ENDPOINT}sign_urls",
+ post_url,
params=params,
headers=headers,
timeout=10
diff --git a/dinamis_sdk/upload.py b/dinamis_sdk/upload.py
new file mode 100644
index 0000000..df2492d
--- /dev/null
+++ b/dinamis_sdk/upload.py
@@ -0,0 +1,34 @@
+import requests
+import urllib3.util.retry
+from .s3 import sign_url_put
+
+def push(
+ local_filename: str,
+ target_url: str,
+ retry_total: int = 5,
+ retry_backoff_factor: float = .8
+):
+ """
+ Publish a local file to the cloud
+
+ """
+ remote_presigned_url = sign_url_put(target_url)
+
+ session = requests.Session()
+ retry = urllib3.util.retry.Retry(
+ total=retry_total,
+ backoff_factor=retry_backoff_factor,
+ status_forcelist=[404, 429, 500, 502, 503, 504],
+ allowed_methods=False,
+ )
+ adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+ session.mount("http://", adapter)
+ session.mount("https://", adapter)
+
+ with open(local_filename, 'rb') as f:
+ ret = session.put(remote_presigned_url, data=f)
+
+ if ret.status_code == 200:
+ return remote_presigned_url
+
+ ret.raise_for_status()
diff --git a/tests/test_push.py b/tests/test_push.py
new file mode 100644
index 0000000..d7623ea
--- /dev/null
+++ b/tests/test_push.py
@@ -0,0 +1,13 @@
+import dinamis_sdk
+import time
+
+local_filename = "/tmp/toto.txt"
+
+with open(local_filename, "w") as f:
+ f.write("hello world")
+
+pushed = dinamis_sdk.push(
+ local_filename=local_filename,
+ target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt"
+)
+print("Done")
--
GitLab
From 97ac406688311c4be7f4c9c030a8f049c26f3aae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:00:58 +0200
Subject: [PATCH 02/10] refactor with settings
---
dinamis_sdk/auth.py | 38 +++++++++++++-------------
dinamis_sdk/s3.py | 35 +++++++++++-------------
dinamis_sdk/settings.py | 14 ++++++++++
dinamis_sdk/upload.py | 16 ++++-------
dinamis_sdk/utils.py | 59 ++++++++++++++++++++++-------------------
5 files changed, 84 insertions(+), 78 deletions(-)
create mode 100644 dinamis_sdk/settings.py
diff --git a/dinamis_sdk/auth.py b/dinamis_sdk/auth.py
index 25be15e..d17a158 100644
--- a/dinamis_sdk/auth.py
+++ b/dinamis_sdk/auth.py
@@ -10,26 +10,28 @@ import requests
from pydantic import BaseModel, Field # pylint: disable = no-name-in-module
import qrcode
import urllib3
-from .utils import log, JWT_FILE, TOKEN_ENDPOINT, AUTH_BASE_URL, TOKEN_SERVER
+from .utils import (
+ log, JWT_FILE, TOKEN_ENDPOINT, AUTH_BASE_URL, settings, create_session
+)
class JWT(BaseModel): # pylint: disable = R0903
"""JWT model."""
- access_token: str = Field(alias="access_token")
- expires_in: int = Field(alias="expires_in")
- refresh_token: str = Field(alias="refresh_token")
- refresh_expires_in: int = Field(alias="refresh_expires_in")
- token_type: str = Field(alias="token_type")
+ access_token: str
+ expires_in: int
+ refresh_token: str
+ refresh_expires_in: int
+ token_type: str
class DeviceGrantResponse(BaseModel): # pylint: disable = R0903
"""Device grant login response model."""
- verification_uri_complete: str = Field(alias="verification_uri_complete")
- device_code: str = Field(alias="device_code")
- expires_in: int = Field(alias="expires_in")
- interval: int = Field(alias="interval")
+ verification_uri_complete: str
+ device_code: str
+ expires_in: int
+ interval: int
class GrantMethodBase:
@@ -251,24 +253,20 @@ class OAuth2Session:
class TokenServer:
"""Token server."""
- def __init__(self, endpoint: str, total_retry=5, backoff_factor=0.8):
+ def __init__(self, endpoint: str, retry_total=5, retry_backoff_factor=0.8):
self.endpoint = endpoint
- self.session = requests.Session()
- retry = urllib3.util.retry.Retry(
- total=total_retry,
- backoff_factor=backoff_factor,
- status_forcelist=[404, 429, 500, 502, 503, 504],
+ self.session = create_session(
+ retry_total=retry_total,
+ retry_backoff_factor=retry_backoff_factor
)
- adapter = requests.adapters.HTTPAdapter(max_retries=retry)
- self.session.mount("https://", adapter)
- self.session.mount("http://", adapter)
log.info("Using Token Server: %s", self.endpoint)
def get_access_token(self) -> str:
return self.session.get(self.endpoint, timeout=10).json()
-session = TokenServer(TOKEN_SERVER) if TOKEN_SERVER else OAuth2Session()
+session = TokenServer(settings.dinamis_sdk_token_server) \
+ if settings.dinamis_sdk_token_server else OAuth2Session()
def _get_access_token():
diff --git a/dinamis_sdk/s3.py b/dinamis_sdk/s3.py
index b445a4b..324d652 100644
--- a/dinamis_sdk/s3.py
+++ b/dinamis_sdk/s3.py
@@ -28,13 +28,12 @@ import pydantic
from .utils import (
log,
- SIGNED_URL_TTL_MARGIN,
+ settings,
CREDENTIALS,
MAX_URLS,
S3_SIGNING_ENDPOINT,
S3_STORAGE_DOMAIN,
- SIGNED_URL_DURATION_SECONDS,
- BYPASS_API
+ create_session
)
_PYDANTIC_2_0 = packaging.version.parse(
@@ -494,27 +493,25 @@ def _generic_get_signed_urls(
log.debug("URL %s already in cache", url)
ttl = signed_url_in_cache.ttl()
log.debug("URL %s TTL is %s", url, ttl)
- if ttl > SIGNED_URL_TTL_MARGIN:
- log.debug("Using cache (%s > %s)", ttl, SIGNED_URL_TTL_MARGIN)
+ if ttl > settings.dinamis_sdk_ttl_margin:
+ log.debug(
+ "Using cache (%s > %s)",
+ ttl,
+ settings.dinamis_sdk_ttl_margin
+ )
signed_urls[url] = signed_url_in_cache
not_signed_urls = [url for url in urls if url not in signed_urls]
log.debug("Already signed URLs:\n %s", signed_urls)
log.debug("Not signed URLs:\n %s", not_signed_urls)
if not_signed_urls:
- # Refresh the token if there's less than SIGNED_URL_TTL_MARGIN seconds
- # remaining, in order to give a small amount of time to do stuff with
- # the url
- session = requests.Session()
- retry = urllib3.util.retry.Retry(
- total=retry_total,
- backoff_factor=retry_backoff_factor,
- status_forcelist=[404, 429, 500, 502, 503, 504],
- allowed_methods=False,
+ # Refresh the token if there's less than
+ # `settings.dinamis_sdk_ttl_margin seconds` remaining, in order to
+ # give a small amount of time to do stuff with the url
+ session = create_session(
+ retry_backoff_factor=retry_backoff_factor,
+ retry_total=retry_total
)
- adapter = requests.adapters.HTTPAdapter(max_retries=retry)
- session.mount("http://", adapter)
- session.mount("https://", adapter)
n_urls = len(not_signed_urls)
log.debug("Number of URLs to sign: %s", n_urls)
n_chunks = math.ceil(n_urls / MAX_URLS)
@@ -525,8 +522,8 @@ def _generic_get_signed_urls(
chunk_end = min(chunk_start + MAX_URLS, n_urls)
not_signed_urls_chunk = not_signed_urls[chunk_start:chunk_end]
params = {"urls": not_signed_urls_chunk}
- if SIGNED_URL_DURATION_SECONDS:
- params["duration_seconds"] = SIGNED_URL_DURATION_SECONDS
+ if settings.dinamis_sdk_url_duration:
+ params["duration_seconds"] = settings.dinamis_sdk_url_duration
post_url = f"{S3_SIGNING_ENDPOINT}{route}"
log.debug("POST %s", post_url)
response = session.post(
diff --git a/dinamis_sdk/settings.py b/dinamis_sdk/settings.py
new file mode 100644
index 0000000..eb4c137
--- /dev/null
+++ b/dinamis_sdk/settings.py
@@ -0,0 +1,14 @@
+"""
+Environment settings
+"""
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+ """
+ Holds settings
+ """
+ dinamis_sdk_ttl_margin: int = 1800
+ dinamis_sdk_url_duration: int = None
+ dinamis_sdk_bypass_api: str = None
+ dinamis_sdk_token_server: str = None
diff --git a/dinamis_sdk/upload.py b/dinamis_sdk/upload.py
index df2492d..6be1d43 100644
--- a/dinamis_sdk/upload.py
+++ b/dinamis_sdk/upload.py
@@ -1,6 +1,6 @@
-import requests
-import urllib3.util.retry
from .s3 import sign_url_put
+from .utils import create_session
+
def push(
local_filename: str,
@@ -14,16 +14,10 @@ def push(
"""
remote_presigned_url = sign_url_put(target_url)
- session = requests.Session()
- retry = urllib3.util.retry.Retry(
- total=retry_total,
- backoff_factor=retry_backoff_factor,
- status_forcelist=[404, 429, 500, 502, 503, 504],
- allowed_methods=False,
+ session = create_session(
+ retry_total=retry_total,
+ retry_backoff_factor=retry_backoff_factor
)
- adapter = requests.adapters.HTTPAdapter(max_retries=retry)
- session.mount("http://", adapter)
- session.mount("https://", adapter)
with open(local_filename, 'rb') as f:
ret = session.put(remote_presigned_url, data=f)
diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py
index ac636ec..a121525 100644
--- a/dinamis_sdk/utils.py
+++ b/dinamis_sdk/utils.py
@@ -5,33 +5,16 @@ import os
import appdirs
from pydantic import BaseModel # pylint: disable = no-name-in-module
import requests
+from .settings import Settings
+import urllib3.util.retry
-# Env vars
-ENV_TTL_MARGIN = "DINAMIS_SDK_TTL_MARGIN"
-ENV_DURATION_SECS = "DINAMIS_SDK_DURATION_SECONDS"
-ENV_BYPASS_API = "DINAMIS_SDK_BYPASS_API"
+
+settings = Settings()
logging.basicConfig(level=os.environ.get("LOGLEVEL") or "INFO")
log = logging.getLogger("dinamis_sdk")
-def _get_seconds(env_var_name: str, default: int = None) -> int:
- val = os.environ.get(env_var_name)
- if val:
- if val.isdigit():
- log.info(
- "Using %s = %s seconds",
- env_var_name,
- val
- )
- return int(val)
- return default
-
-
-# Signed TTL margin default to 1800 seconds (30 minutes), or env. var.
-SIGNED_URL_TTL_MARGIN = _get_seconds(ENV_TTL_MARGIN, 1800)
-SIGNED_URL_DURATION_SECONDS = _get_seconds(ENV_DURATION_SECS)
-
MAX_URLS = 64
S3_STORAGE_DOMAIN = "meso.umontpellier.fr"
S3_SIGNING_ENDPOINT = \
@@ -85,16 +68,36 @@ def retrieve_token_endpoint(s3_signing_endpoint: str = S3_SIGNING_ENDPOINT):
return oauth2_defs["flows"]["password"]["tokenUrl"]
-BYPASS_API = os.environ.get(ENV_BYPASS_API)
-if BYPASS_API:
- S3_SIGNING_ENDPOINT = BYPASS_API
+if settings.dinamis_sdk_bypass_api:
+ S3_SIGNING_ENDPOINT = settings.dinamis_sdk_bypass_api
# Token endpoint is typically something like: https://keycloak-dinamis.apps.okd
# .crocc.meso.umontpellier.fr/auth/realms/dinamis/protocol/openid-connect/token
-TOKEN_ENDPOINT = None if BYPASS_API else retrieve_token_endpoint()
+TOKEN_ENDPOINT = None if settings.dinamis_sdk_bypass_api \
+ else retrieve_token_endpoint()
# Auth base URL is typically something like: https://keycloak-dinamis.apps.okd.
# crocc.meso.umontpellier.fr/auth/realms/dinamis/protocol/openid-connect
-AUTH_BASE_URL = None if BYPASS_API else TOKEN_ENDPOINT.rsplit('/', 1)[0]
+AUTH_BASE_URL = None if settings.dinamis_sdk_bypass_api \
+ else TOKEN_ENDPOINT.rsplit('/', 1)[0]
+
+
+def create_session(
+ retry_total: int = 5,
+ retry_backoff_factor: float = .8
+):
+ """
+ Create a session for requests
+
+ """
+ session = requests.Session()
+ retry = urllib3.util.retry.Retry(
+ total=retry_total,
+ backoff_factor=retry_backoff_factor,
+ status_forcelist=[404, 429, 500, 502, 503, 504],
+ allowed_methods=False,
+ )
+ adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+ session.mount("http://", adapter)
+ session.mount("https://", adapter)
-# Token server (optional)
-TOKEN_SERVER = os.environ.get("DINAMIS_SDK_TOKEN_SERVER")
+ return session
--
GitLab
From 8e1e45e4c38039bc29eddeb1b849c2ee35b699e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:00:58 +0200
Subject: [PATCH 03/10] bump version
---
setup.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index a549ba0..4d4db85 100644
--- a/setup.py
+++ b/setup.py
@@ -7,12 +7,13 @@ install_requires = [
"pystac",
"pystac_client",
"pydantic>=1.7.3",
+ "pydantic_settings",
"packaging"
]
setup(
name="dinamis-sdk",
- version="0.1.13",
+ version="0.2.0",
description="DINAMIS SDK",
python_requires=">=3.8",
author="Remi Cresson",
--
GitLab
From 691bfc0f0c99454181ac399f114538cfc7d487ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:08:13 +0200
Subject: [PATCH 04/10] refactor with settings
---
dinamis_sdk/settings.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/dinamis_sdk/settings.py b/dinamis_sdk/settings.py
index eb4c137..a0a8f78 100644
--- a/dinamis_sdk/settings.py
+++ b/dinamis_sdk/settings.py
@@ -9,6 +9,6 @@ class Settings(BaseSettings):
Holds settings
"""
dinamis_sdk_ttl_margin: int = 1800
- dinamis_sdk_url_duration: int = None
- dinamis_sdk_bypass_api: str = None
- dinamis_sdk_token_server: str = None
+ dinamis_sdk_url_duration: int = 0
+ dinamis_sdk_bypass_api: str = ""
+ dinamis_sdk_token_server: str = ""
--
GitLab
From 9b82538766544b143cfb0932c3bfc7bd7e28ecb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:37:12 +0200
Subject: [PATCH 05/10] refactor with settings
---
dinamis_sdk/__init__.py | 1 +
dinamis_sdk/auth.py | 8 +++++---
dinamis_sdk/s3.py | 21 +++++++++------------
dinamis_sdk/upload.py | 4 ++++
dinamis_sdk/utils.py | 2 +-
5 files changed, 20 insertions(+), 16 deletions(-)
diff --git a/dinamis_sdk/__init__.py b/dinamis_sdk/__init__.py
index a20433c..d062092 100644
--- a/dinamis_sdk/__init__.py
+++ b/dinamis_sdk/__init__.py
@@ -1,4 +1,5 @@
"""Dinamis SDK module."""
+# flake8: noqa
import pkg_resources
__version__ = pkg_resources.require("dinamis-sdk")[0].version
from dinamis_sdk.s3 import (
diff --git a/dinamis_sdk/auth.py b/dinamis_sdk/auth.py
index d17a158..9e1a03b 100644
--- a/dinamis_sdk/auth.py
+++ b/dinamis_sdk/auth.py
@@ -7,9 +7,8 @@ import time
from abc import abstractmethod
from typing import Dict
import requests
-from pydantic import BaseModel, Field # pylint: disable = no-name-in-module
+from pydantic import BaseModel # pylint: disable = no-name-in-module
import qrcode
-import urllib3
from .utils import (
log, JWT_FILE, TOKEN_ENDPOINT, AUTH_BASE_URL, settings, create_session
)
@@ -235,7 +234,7 @@ class OAuth2Session:
log.debug(
"Credentials from %s still valid", JWT_FILE
)
- except Exception as error:
+ except FileNotFoundError as error:
log.warning(
"Warning: can't use token from file %s (%s)",
JWT_FILE,
@@ -262,6 +261,9 @@ class TokenServer:
log.info("Using Token Server: %s", self.endpoint)
def get_access_token(self) -> str:
+ """
+ Return the access token
+ """
return self.session.get(self.endpoint, timeout=10).json()
diff --git a/dinamis_sdk/s3.py b/dinamis_sdk/s3.py
index 324d652..cf5a63a 100644
--- a/dinamis_sdk/s3.py
+++ b/dinamis_sdk/s3.py
@@ -14,10 +14,7 @@ from functools import singledispatch
from typing import Any, Dict, Mapping, TypeVar, cast, List
from urllib.parse import urlparse, parse_qs
import math
-import urllib3.util.retry
-import requests
-import requests.adapters
-from pydantic import BaseModel, Field # pylint: disable = no-name-in-module
+from pydantic import BaseModel # pylint: disable = no-name-in-module
from pystac import Asset, Item, ItemCollection, STACObjectType, Collection
from pystac.serialization.identify import identify_stac_object_type
from pystac.utils import datetime_to_str
@@ -54,7 +51,7 @@ asset_xpr = re.compile(
class URLBase(BaseModel): # pylint: disable = R0903
"""Base model for responses."""
- expiry: datetime = Field(alias="expiry")
+ expiry: datetime
class Config: # pylint: disable = R0903
"""Config for URLBase model."""
@@ -69,7 +66,7 @@ class URLBase(BaseModel): # pylint: disable = R0903
class SignedURL(URLBase): # pylint: disable = R0903
"""Signed URL response."""
- href: str = Field(alias="href")
+ href: str
def ttl(self) -> float:
"""Return the number of seconds the token is still valid for."""
@@ -79,7 +76,7 @@ class SignedURL(URLBase): # pylint: disable = R0903
class SignedURLBatch(URLBase): # pylint: disable = R0903
"""Signed URLs (batch of URLs) response."""
- hrefs: dict = Field(alias="hrefs")
+ hrefs: dict
# Cache of signing requests so we can reuse them
@@ -205,17 +202,17 @@ def _generic_sign_urls(
def sign_urls(urls: List[str]) -> Dict[str, str]:
+ """Sign multiple URLs for get"""
return _generic_sign_urls(urls=urls, route="sign_urls")
def sign_urls_put(urls: List[str]) -> Dict[str, str]:
+ """Sign multiple URLs for put"""
return _generic_sign_urls(urls=urls, route="sign_urls_put")
def sign_url_put(url: str) -> str:
- """
- Sign a single URL for put
- """
+ """Sign a single URL for put"""
urls = sign_urls_put([url])
return urls[url]
@@ -477,8 +474,8 @@ def _generic_get_signed_urls(
"dinamis-secret-key": CREDENTIALS.secret_key
})
log.debug("Using credentials (access/secret keys)")
- elif BYPASS_API:
- log.debug("Using bypass API %s", BYPASS_API)
+ elif settings.dinamis_sdk_bypass_api:
+ log.debug("Using bypass API %s", settings.dinamis_sdk_bypass_api)
else:
from .auth import get_access_token
access_token = get_access_token()
diff --git a/dinamis_sdk/upload.py b/dinamis_sdk/upload.py
index 6be1d43..a027b5d 100644
--- a/dinamis_sdk/upload.py
+++ b/dinamis_sdk/upload.py
@@ -1,3 +1,6 @@
+"""
+This module is used to upload files using HTTP requests.
+"""
from .s3 import sign_url_put
from .utils import create_session
@@ -26,3 +29,4 @@ def push(
return remote_presigned_url
ret.raise_for_status()
+ return ""
diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py
index a121525..189f0e5 100644
--- a/dinamis_sdk/utils.py
+++ b/dinamis_sdk/utils.py
@@ -5,8 +5,8 @@ import os
import appdirs
from pydantic import BaseModel # pylint: disable = no-name-in-module
import requests
-from .settings import Settings
import urllib3.util.retry
+from .settings import Settings
settings = Settings()
--
GitLab
From c46a054ec19de13b67ab2c3d0e1f1e30a7551d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:40:26 +0200
Subject: [PATCH 06/10] refactor with settings
---
dinamis_sdk/auth.py | 10 ++++------
dinamis_sdk/utils.py | 4 +---
2 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/dinamis_sdk/auth.py b/dinamis_sdk/auth.py
index 9e1a03b..6c0c104 100644
--- a/dinamis_sdk/auth.py
+++ b/dinamis_sdk/auth.py
@@ -58,6 +58,7 @@ class GrantMethodBase:
@property
def data_base(self) -> Dict[str, str]:
"""Base payload."""
+
return {
"client_id": self.client_id,
"scope": "openid offline_access"
@@ -189,6 +190,7 @@ class OAuth2Session:
def refresh_if_needed(self):
"""Refresh the token if ttl is too short."""
+
ttl_margin_seconds = 30
now = datetime.datetime.now()
jwt_expires_in = datetime.timedelta(seconds=self.jwt.expires_in)
@@ -214,13 +216,8 @@ class OAuth2Session:
self.save_token(now)
def get_access_token(self) -> str:
- """
- Get the access token.
-
- Returns:
- access token
+ """Return the access token."""
- """
if not self.jwt:
# First JWT initialisation
if JWT_FILE and os.path.isfile(JWT_FILE):
@@ -252,6 +249,7 @@ class OAuth2Session:
class TokenServer:
"""Token server."""
+
def __init__(self, endpoint: str, retry_total=5, retry_backoff_factor=0.8):
self.endpoint = endpoint
self.session = create_session(
diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py
index 189f0e5..04e0d43 100644
--- a/dinamis_sdk/utils.py
+++ b/dinamis_sdk/utils.py
@@ -85,10 +85,8 @@ def create_session(
retry_total: int = 5,
retry_backoff_factor: float = .8
):
- """
- Create a session for requests
+ """Create a session for requests."""
- """
session = requests.Session()
retry = urllib3.util.retry.Retry(
total=retry_total,
--
GitLab
From 2cf5a96f841880238f34f1f2551262ff61183f56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:46:52 +0200
Subject: [PATCH 07/10] refactor with settings
---
dinamis_sdk/auth.py | 7 +------
dinamis_sdk/s3.py | 6 +++---
dinamis_sdk/settings.py | 9 +++------
3 files changed, 7 insertions(+), 15 deletions(-)
diff --git a/dinamis_sdk/auth.py b/dinamis_sdk/auth.py
index 6c0c104..bdaf718 100644
--- a/dinamis_sdk/auth.py
+++ b/dinamis_sdk/auth.py
@@ -58,7 +58,6 @@ class GrantMethodBase:
@property
def data_base(self) -> Dict[str, str]:
"""Base payload."""
-
return {
"client_id": self.client_id,
"scope": "openid offline_access"
@@ -190,7 +189,6 @@ class OAuth2Session:
def refresh_if_needed(self):
"""Refresh the token if ttl is too short."""
-
ttl_margin_seconds = 30
now = datetime.datetime.now()
jwt_expires_in = datetime.timedelta(seconds=self.jwt.expires_in)
@@ -217,7 +215,6 @@ class OAuth2Session:
def get_access_token(self) -> str:
"""Return the access token."""
-
if not self.jwt:
# First JWT initialisation
if JWT_FILE and os.path.isfile(JWT_FILE):
@@ -259,9 +256,7 @@ class TokenServer:
log.info("Using Token Server: %s", self.endpoint)
def get_access_token(self) -> str:
- """
- Return the access token
- """
+ """Return the access token."""
return self.session.get(self.endpoint, timeout=10).json()
diff --git a/dinamis_sdk/s3.py b/dinamis_sdk/s3.py
index cf5a63a..13098ea 100644
--- a/dinamis_sdk/s3.py
+++ b/dinamis_sdk/s3.py
@@ -202,17 +202,17 @@ def _generic_sign_urls(
def sign_urls(urls: List[str]) -> Dict[str, str]:
- """Sign multiple URLs for get"""
+ """Sign multiple URLs for get."""
return _generic_sign_urls(urls=urls, route="sign_urls")
def sign_urls_put(urls: List[str]) -> Dict[str, str]:
- """Sign multiple URLs for put"""
+ """Sign multiple URLs for put."""
return _generic_sign_urls(urls=urls, route="sign_urls_put")
def sign_url_put(url: str) -> str:
- """Sign a single URL for put"""
+ """Sign a single URL for put."""
urls = sign_urls_put([url])
return urls[url]
diff --git a/dinamis_sdk/settings.py b/dinamis_sdk/settings.py
index a0a8f78..3aba92c 100644
--- a/dinamis_sdk/settings.py
+++ b/dinamis_sdk/settings.py
@@ -1,13 +1,10 @@
-"""
-Environment settings
-"""
+"""Settings from environment variables."""
from pydantic_settings import BaseSettings
class Settings(BaseSettings):
- """
- Holds settings
- """
+ """Holds settings."""
+
dinamis_sdk_ttl_margin: int = 1800
dinamis_sdk_url_duration: int = 0
dinamis_sdk_bypass_api: str = ""
--
GitLab
From 646b340b0e92ec685ef1a41b488f07732cce6c26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 14:49:34 +0200
Subject: [PATCH 08/10] refactor with settings
---
dinamis_sdk/auth.py | 1 +
dinamis_sdk/upload.py | 9 ++-------
dinamis_sdk/utils.py | 1 -
3 files changed, 3 insertions(+), 8 deletions(-)
diff --git a/dinamis_sdk/auth.py b/dinamis_sdk/auth.py
index bdaf718..b809ac2 100644
--- a/dinamis_sdk/auth.py
+++ b/dinamis_sdk/auth.py
@@ -248,6 +248,7 @@ class TokenServer:
"""Token server."""
def __init__(self, endpoint: str, retry_total=5, retry_backoff_factor=0.8):
+ """Initialize the token server."""
self.endpoint = endpoint
self.session = create_session(
retry_total=retry_total,
diff --git a/dinamis_sdk/upload.py b/dinamis_sdk/upload.py
index a027b5d..ec5bc4f 100644
--- a/dinamis_sdk/upload.py
+++ b/dinamis_sdk/upload.py
@@ -1,6 +1,4 @@
-"""
-This module is used to upload files using HTTP requests.
-"""
+"""This module is used to upload files using HTTP requests."""
from .s3 import sign_url_put
from .utils import create_session
@@ -11,10 +9,7 @@ def push(
retry_total: int = 5,
retry_backoff_factor: float = .8
):
- """
- Publish a local file to the cloud
-
- """
+ """Publish a local file to the cloud."""
remote_presigned_url = sign_url_put(target_url)
session = create_session(
diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py
index 04e0d43..dcfa7fa 100644
--- a/dinamis_sdk/utils.py
+++ b/dinamis_sdk/utils.py
@@ -86,7 +86,6 @@ def create_session(
retry_backoff_factor: float = .8
):
"""Create a session for requests."""
-
session = requests.Session()
retry = urllib3.util.retry.Retry(
total=retry_total,
--
GitLab
From da0964669ed65c788811bf53f320fec3ac0a3e1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 15:03:24 +0200
Subject: [PATCH 09/10] refactor with settings
---
dinamis_sdk/utils.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py
index dcfa7fa..65e9a29 100644
--- a/dinamis_sdk/utils.py
+++ b/dinamis_sdk/utils.py
@@ -8,18 +8,21 @@ import requests
import urllib3.util.retry
from .settings import Settings
-
+# Settings
settings = Settings()
-logging.basicConfig(level=os.environ.get("LOGLEVEL") or "INFO")
+# Logger
+logging.basicConfig()
log = logging.getLogger("dinamis_sdk")
+log.setLevel(level=os.environ.get('LOGLEVEL', 'INFO').upper())
-
+# Constants
MAX_URLS = 64
S3_STORAGE_DOMAIN = "meso.umontpellier.fr"
S3_SIGNING_ENDPOINT = \
"https://s3-signing-cdos.apps.okd.crocc.meso.umontpellier.fr/"
+# Config path
CFG_PTH = appdirs.user_config_dir(appname='dinamis_sdk_auth')
if not os.path.exists(CFG_PTH):
try:
@@ -31,9 +34,11 @@ if not os.path.exists(CFG_PTH):
else:
log.debug("Config path already exist in %s", CFG_PTH)
+# JWT File
JWT_FILE = os.path.join(CFG_PTH, ".token") if CFG_PTH else None
log.debug("JWT file is %s", JWT_FILE)
+# Settings file
settings_file = os.path.join(CFG_PTH, ".settings") if CFG_PTH else None
log.debug("Settings file is %s", settings_file)
--
GitLab
From bcf4d6c252a037ee520216ac92c011eaf5bb27cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr>
Date: Wed, 11 Sep 2024 15:16:41 +0200
Subject: [PATCH 10/10] add test for push
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 86c9856..718cc82 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -87,6 +87,7 @@ Tests:
script:
- python tests/test_spot-6-7-drs.py
- python tests/test_super-s2.py
+ - python tests/test_push.py
# --------------------------------- Ship --------------------------------------
--
GitLab