diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 718cc829e8fb8df41d270d29c3bea9a5199c0791..d659a3582bb8a31528952f2ded70c1dba2bb5e6b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,8 @@ workflow: stages: - Static Analysis - Install - - Documentation - Test + - Documentation - Ship # ------------------------------ Static analysis ------------------------------ @@ -77,14 +77,26 @@ pages: # --------------------------------- Test -------------------------------------- -Tests: +.tests_base: stage: Test except: - main before_script: - pip install . - pip install pystac-client + + +OAuth2 Tests: + extends: .tests_base + script: + - python tests/test_spot-6-7-drs.py + - python tests/test_super-s2.py + - python tests/test_push.py + +API key Tests: + extends: .tests_base script: + - dinamis_cli register - python tests/test_spot-6-7-drs.py - python tests/test_super-s2.py - python tests/test_push.py diff --git a/dinamis_sdk/cli.py b/dinamis_sdk/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..f900afa067af6085f4133da37101796b54247f94 --- /dev/null +++ b/dinamis_sdk/cli.py @@ -0,0 +1,92 @@ +"""Dinamis Command Line Interface.""" +import click +from .utils import APIKEY_FILE, create_session, S3_SIGNING_ENDPOINT, log +from .auth import get_access_token +import os +import json +from typing import List, Dict + + +@click.group(help="Dinamis CLI") +def app() -> None: + """Click group for dinamis sdk subcommands.""" + pass + + +def http(route: str): + """Perform an HTTP request.""" + session = create_session() + ret = session.get( + f"{S3_SIGNING_ENDPOINT}{route}", + timeout=5, + headers={"authorization": f"bearer {get_access_token()}"} + ) + ret.raise_for_status() + return ret + + +def create_key() -> Dict[str, str]: + """Create an API key.""" + return http("create_api_key").json() + + +def list_keys() -> List[str]: + """List all generated API keys.""" + return http("list_api_keys").json() + + +def revoke_key(key: str): + """Revoke an API key.""" + http(f"revoke_api_key?access_key={key}") + log.info(f"API key {key} revoked") + + +@app.command(help="Create and show a new API key") +def create(): + """Create and show a new API key.""" + log.info(f"Got a new API key: {create_key()}") + + +@app.command(help="List all API keys") +def list(): + """List all API keys.""" + log.info(f"All generated API keys: {list_keys()}") + + +@app.command(help="Revoke all API keys") +def revoke_all(): + """Revoke all API keys.""" + keys = list_keys() + for key in keys: + revoke_key(key) + if not keys: + log.info("No API key found.") + + +@app.command(help="Revoke an API key") +@click.option( + "--key", + prompt="Please enter the access key to revoke", + help="Access key to revoke", +) +def revoke(key: str): + """Revoke an API key.""" + revoke_key(key) + + +@app.command(help="Get and store an API key") +def register(): + """Get and store an API key.""" + with open(APIKEY_FILE, 'w') as f: + json.dump(create_key(), f) + log.info(f"API key successfully created and stored in {APIKEY_FILE}") + + +@app.command(help="Delete the stored API key") +def delete(): + """Delete the stored API key.""" + if os.path.isfile(APIKEY_FILE): + os.remove(APIKEY_FILE) + log.info(f"File {APIKEY_FILE} deleted!") + else: + log.info("No API key stored!") diff --git a/dinamis_sdk/s3.py b/dinamis_sdk/s3.py index 79481db6efb25d65797d687b7e2f5e27ce1c568c..d08cfdc52bd0e93ba18b1cc8d441fcb965f155a7 100644 --- a/dinamis_sdk/s3.py +++ b/dinamis_sdk/s3.py @@ -26,11 +26,11 @@ import pydantic from .utils import ( log, settings, - CREDENTIALS, MAX_URLS, S3_SIGNING_ENDPOINT, S3_STORAGE_DOMAIN, - create_session + create_session, + APIKEY ) _PYDANTIC_2_0 = packaging.version.parse( @@ -474,12 +474,9 @@ def _generic_get_signed_urls( "Content-Type": "application/json", "Accept": "application/json" } - if CREDENTIALS: - headers.update({ - "dinamis-access-key": CREDENTIALS.access_key, - "dinamis-secret-key": CREDENTIALS.secret_key - }) - log.debug("Using credentials (access/secret keys)") + if APIKEY: + headers.update(APIKEY) + log.debug("Using API key") elif settings.dinamis_sdk_bypass_api: log.debug("Using bypass API %s", settings.dinamis_sdk_bypass_api) else: diff --git a/dinamis_sdk/settings.py b/dinamis_sdk/settings.py index 3aba92c8206c62c533c47fff1e0e7c74365705bd..37ba8b3d2fc377d2d21fa39155156aca7d1517a0 100644 --- a/dinamis_sdk/settings.py +++ b/dinamis_sdk/settings.py @@ -9,3 +9,4 @@ class Settings(BaseSettings): dinamis_sdk_url_duration: int = 0 dinamis_sdk_bypass_api: str = "" dinamis_sdk_token_server: str = "" + dinamis_sdk_settings_dir: str = "" diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py index 65e9a296c50230eff057663032a3fe42c2f6fb6d..bcc93e724c3316e842be11e22c8b609265e52338 100644 --- a/dinamis_sdk/utils.py +++ b/dinamis_sdk/utils.py @@ -3,7 +3,6 @@ import json import logging import os import appdirs -from pydantic import BaseModel # pylint: disable = no-name-in-module import requests import urllib3.util.retry from .settings import Settings @@ -23,47 +22,63 @@ S3_SIGNING_ENDPOINT = \ "https://s3-signing-cdos.apps.okd.crocc.meso.umontpellier.fr/" # Config path -CFG_PTH = appdirs.user_config_dir(appname='dinamis_sdk_auth') +CFG_PTH = settings.dinamis_sdk_settings_dir or \ + appdirs.user_config_dir(appname='dinamis_sdk_auth') if not os.path.exists(CFG_PTH): try: os.makedirs(CFG_PTH) - log.debug("Config path created in %s", CFG_PTH) + log.debug("Settings dir created in %s", CFG_PTH) except PermissionError: - log.warning("Unable to create config path") + log.warning("Unable to create settings dir") CFG_PTH = None else: - log.debug("Config path already exist in %s", CFG_PTH) + log.debug("Using existing settings dir %s", CFG_PTH) # JWT File JWT_FILE = os.path.join(CFG_PTH, ".token") if CFG_PTH else None log.debug("JWT file is %s", JWT_FILE) -# Settings file -settings_file = os.path.join(CFG_PTH, ".settings") if CFG_PTH else None -log.debug("Settings file is %s", settings_file) +# API key File +APIKEY_FILE = os.path.join(CFG_PTH, ".api_key") if CFG_PTH else None +log.debug("API key file is %s", APIKEY_FILE) +APIKEY = None +if APIKEY_FILE and os.path.isfile(APIKEY_FILE): + try: + log.debug("Found a stored API key") + with open(APIKEY_FILE, encoding='UTF-8') as json_file: + APIKEY = json.load(json_file) + log.debug("API key successfully loaded") + except json.decoder.JSONDecodeError: + log.warning("Stored API key file is invalid. Deleting it.") + os.remove(APIKEY_FILE) -class StorageCredentials(BaseModel): # pylint: disable = R0903 - """Credentials model.""" - - access_key: str - secret_key: str +def create_session( + retry_total: int = 5, + retry_backoff_factor: float = .8 +): + """Create a session for requests.""" + session = requests.Session() + retry = urllib3.util.retry.Retry( + total=retry_total, + backoff_factor=retry_backoff_factor, + status_forcelist=[404, 429, 500, 502, 503, 504], + allowed_methods=False, + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) -CREDENTIALS = None -if settings_file and os.path.isfile(settings_file): - try: - with open(settings_file, encoding='UTF-8') as json_file: - CREDENTIALS = StorageCredentials(**json.load(json_file)) - except FileNotFoundError: - log.debug("Setting file %s does not exist", settings_file) + return session def retrieve_token_endpoint(s3_signing_endpoint: str = S3_SIGNING_ENDPOINT): """Retrieve the token endpoint from the s3 signing endpoint.""" openapi_url = s3_signing_endpoint + "openapi.json" log.debug("Fetching OAuth2 endpoint from openapi url %s", openapi_url) - res = requests.get( + session = create_session() + res = session.get( openapi_url, timeout=10, ) @@ -84,22 +99,3 @@ TOKEN_ENDPOINT = None if settings.dinamis_sdk_bypass_api \ # crocc.meso.umontpellier.fr/auth/realms/dinamis/protocol/openid-connect AUTH_BASE_URL = None if settings.dinamis_sdk_bypass_api \ else TOKEN_ENDPOINT.rsplit('/', 1)[0] - - -def create_session( - retry_total: int = 5, - retry_backoff_factor: float = .8 -): - """Create a session for requests.""" - session = requests.Session() - retry = urllib3.util.retry.Retry( - total=retry_total, - backoff_factor=retry_backoff_factor, - status_forcelist=[404, 429, 500, 502, 503, 504], - allowed_methods=False, - ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - session.mount("http://", adapter) - session.mount("https://", adapter) - - return session diff --git a/doc/credentials.md b/doc/credentials.md index 9ff33d6e57c9566ca67266b4adbad7b176d29f26..936d76d96193ba916f93e206d43c3ae1893355e5 100644 --- a/doc/credentials.md +++ b/doc/credentials.md @@ -1,15 +1,41 @@ # Credentials +There is two ways of authenticating to the THEIA-MTP Geospatial data +infrastructure: + +- OAuth2 +- API key + +## OAuth2 + The credentials are retrieved using the device code flow on the first call of -`dinamis_sdk.sign_inplace()`. Just follow the instructions! +`dinamis_sdk.sign_inplace()`. Just follow the instructions, i.e. click on the +HTTP link, or scan the QR-code. + +The credentials are valid for 5 days. Every time `dinamis_sdk.sign_inplace()` +is called, the credentials are renewed for another 5 days. After 5 days idle, +you will have to log in again. + +## API key + +Use `dinamis_cli` to register an API key, that will be created and stored into +your local home directory. + +```commandline +dinamis_cli register +``` + +Just follow the instructions to login a single time, then the API key can be +used forever on your local computer. You can duplicate the API key file on +other computers. -## Renewal +You can delete the API key any time with: -The credentials are valid for 5 days. Every time -`dinamis_sdk.sign_inplace` is called, the credentials are renewed for another -5 days. After 5 days idle, you will have to log in again. +```commandline +dinamis_cli delete +``` -## Signed URLs +## Signed URLs expiry The signed URLs for STAC objects assets are valid during 7 days starting after `dinamis_sdk.sign_inplace` is called. diff --git a/doc/index.md b/doc/index.md index 93e6754029d17c6108cd6ef3b933b0b4efdb06d5..bfb4267e422f7be558ada020e833fcd5614a8476 100644 --- a/doc/index.md +++ b/doc/index.md @@ -30,13 +30,16 @@ pip install dinamis-sdk ## Quickstart -This library assists with signing STAC items assets URLs from the DINAMIS SDI -prototype. The `sign` function operates directly on an HREF string, as well as -several [PySTAC](https://github.com/stac-utils/pystac) objects: `Asset`, -`Item`, and `ItemCollection`. In addition, the `sign` function accepts a -[STAC API Client](https://pystac-client.readthedocs.io/en/stable/) +This library assists with signing STAC items assets URLs from the THEIA-MTP +Geospatial Data Infrastructure. +The `sign_inplace` function operates directly on an HREF string, as well as +several [PySTAC](https://github.com/stac-utils/pystac) objects: `Asset`, `Item`, and `ItemCollection`. +In addition, the `sign_inplace` function accepts a [STAC API Client](https://pystac-client.readthedocs.io/en/stable/) `ItemSearch`, which performs a search and returns the resulting `ItemCollection` with all assets signed. +`sign_inplace()` can be used as a `modifier` in `pystac_client.Client` +instances, as shown in the example below. Alternatively, `sign()` can be used +to sign a single url. ```python import dinamis_sdk @@ -49,8 +52,7 @@ api = pystac_client.Client.open( ``` Follow the instructions to authenticate. -Read the [credentials section](credentials) to know more about credential -expiry. +Read the [credentials section](credentials) to know more about credentials. ## Contribute diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..824ea20b84eaa7343a9e47c57949048e4dfccba8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "dinamis-sdk" +authors = [{name = "inrae", email = "remi.cresson@inrae.fr"}] +version = "0.3.0" +description = "DINAMIS SDK for Python" +requires-python = ">=3.7" +dependencies = [ + "click>=7.1", + "pydantic>=1.7.3", + "pystac>=1.0.0", + "pystac-client>=0.2.0", + "requests>=2.25.1", + "packaging", + "qrcode", + "appdirs", + "pydantic_settings", +] +readme = "README.md" +license = {file = "LICENSE"} + +[project.scripts] +dinamis_cli = "dinamis_sdk.cli:app" + +[tool.setuptools] +include-package-data = false diff --git a/setup.py b/setup.py deleted file mode 100644 index eb3fa2fccf21f137c8d15bd0076c5e21ae7d6ace..0000000000000000000000000000000000000000 --- a/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -from setuptools import setup, find_packages - -install_requires = [ - "requests", - "qrcode", - "appdirs", - "pystac", - "pystac_client", - "pydantic>=1.7.3", - "pydantic_settings", - "packaging" -] - -setup( - name="dinamis-sdk", - version="0.2.2", - description="DINAMIS SDK", - python_requires=">=3.8", - author="Remi Cresson", - author_email="remi.cresson@inrae.fr", - license="MIT", - zip_safe=False, - install_requires=install_requires, - packages=find_packages(), -) -