From 28a1a69435c201918d069f1c029e158bdc23371f Mon Sep 17 00:00:00 2001 From: Maxime Alves LIRMM Date: Sun, 20 Aug 2023 23:32:50 +0200 Subject: [PATCH] [rc] 0.6.28rc3 - fix bugs and general configuration management cleanup (see changelog) --- CHANGELOG.md | 35 +++++++-- halfapi/__init__.py | 2 +- halfapi/app.py | 6 +- halfapi/cli/domain.py | 85 ++++++++++++++++---- halfapi/conf.py | 138 +++++++++++++++++++++------------ halfapi/halfapi.py | 32 ++++++-- halfapi/logging.py | 11 +-- halfapi/testing/test_domain.py | 2 + tests/cli/test_cli_proj.py | 49 +++++++++--- 9 files changed, 262 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a6b974..6bcd36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,13 +48,36 @@ it's use in the "tests/dummy_domain/__init__.py" file. The use of an "HEAD" request to check an ACL is now the norm. Please change all the occurrences of your calls on theses routes with the GET method. -### Commits -- [doc-schema] the "/" route on a domain now returns the OpenAPI-validated Schema (not a list of schemas), the "dummy_domain" test now validates OpenAPI specs -- [doc-schema] In module-based routers, if there is a path parameter, you can specify an OpenAPI documentation for it, or a default will be used -- [dev-deps] openapi-schema-validator, openapi-spec-validator -- [doc] add docstrings for halfapi routes -- [acl] The public acls check routes use the "HEAD" method, deprecated "GET" +### CLI + +Domain command update : + +The `--conftest` flag is now allowed when running the `domain` command, it dumps the current configuration as a TOML string. + +`halfapi domain --conftest my_domain` + + +The `--dry-run` flag was buggy and is now fixed when using the `domai ` command with the `--run` flag. + + +### Configuration + +The `port` option in a `domain.my_domain` section in the TOML config file is now prefered to the one in the `project` section. + +The `project` section is used as a default section for the whole configuration file. - Tests still have to be written - + +The standard configuration precedence is fixed, in this order from the hight to the lower : + +- Argument value (i.e. : --log-level) +- Environment value (i.e. : HALFAPI_LOGLEVEL) +- Configuration value under "domain" key +- Configuration value under "project" key +- Default configuration value given in the "DEFAULT_CONF" dictionary of halfapi/conf.py + +### Logs + +Small cleanup of the logs levels. If you don't want the config to be dumped, just set the HALFAPI_LOGLEVEL to something different than "DEBUG". ## 0.6.27 diff --git a/halfapi/__init__.py b/halfapi/__init__.py index 26215f8..e9d5335 100644 --- a/halfapi/__init__.py +++ b/halfapi/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -__version__ = '0.6.28rc2' +__version__ = '0.6.28rc3' def version(): return f'HalfAPI version:{__version__}' diff --git a/halfapi/app.py b/halfapi/app.py index b215332..6b4d137 100644 --- a/halfapi/app.py +++ b/halfapi/app.py @@ -1,11 +1,7 @@ import os from .halfapi import HalfAPI from .logging import logger -from .conf import read_config def application(): - config_file = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config') - - CONFIG = read_config([config_file]) - + from .conf import CONFIG return HalfAPI(CONFIG).application diff --git a/halfapi/cli/domain.py b/halfapi/cli/domain.py index 6b4e1e3..9eea0ec 100644 --- a/halfapi/cli/domain.py +++ b/halfapi/cli/domain.py @@ -17,11 +17,13 @@ import uvicorn from .cli import cli +from ..conf import CONFIG from ..half_domain import HalfDomain from ..lib.routes import api_routes from ..lib.responses import ORJSONResponse +from ..conf import CONFIG, PROJECT_LEVEL_KEYS from ..logging import logger @@ -119,16 +121,22 @@ def list_api_routes(): # list_routes(domain, m_dom) +@click.option('--devel',default=None, is_flag=True) +@click.option('--watch',default=False, is_flag=True) +@click.option('--production',default=None, is_flag=True) +@click.option('--port',default=None, type=int) +@click.option('--log-level',default=None, type=str) @click.option('--dry-run',default=False, is_flag=True) @click.option('--run',default=False, is_flag=True) @click.option('--read',default=False, is_flag=True) +@click.option('--conftest',default=False, is_flag=True) @click.option('--create',default=False, is_flag=True) @click.option('--update',default=False, is_flag=True) @click.option('--delete',default=False, is_flag=True) @click.argument('config_file', type=click.File(mode='rb'), required=False) @click.argument('domain',default=None, required=False) @cli.command() -def domain(domain, config_file, delete, update, create, read, run, dry_run): #, domains, read, create, update, delete): +def domain(domain, config_file, delete, update, create, conftest, read, run, dry_run, log_level, port, production, watch, devel): """ The "halfapi domain" command @@ -147,11 +155,19 @@ def domain(domain, config_file, delete, update, create, read, run, dry_run): #, raise Exception('Missing domain name') if config_file: - CONFIG = toml.load(config_file.name) + ARG_CONFIG = toml.load(config_file.name) + + if 'project' in ARG_CONFIG: + for key, value in ARG_CONFIG['project'].items(): + if key in PROJECT_LEVEL_KEYS: + CONFIG[key] = value - os.environ['HALFAPI_CONF_FILE'] = config_file.name - else: - from halfapi.conf import CONFIG + if 'domain' in ARG_CONFIG and domain in ARG_CONFIG['domain']: + for key, value in ARG_CONFIG['domain'][domain].items(): + if key in PROJECT_LEVEL_KEYS: + CONFIG[key] = value + + CONFIG['domain'].update(ARG_CONFIG['domain']) if create: raise NotImplementedError @@ -170,14 +186,57 @@ def domain(domain, config_file, delete, update, create, read, run, dry_run): #, ) else: - port = CONFIG.get('port', - CONFIG.get('domain', {}).get('port') - ) - uvicorn.run( - 'halfapi.app:application', - port=port, - factory=True - ) + if dry_run: + CONFIG['dryrun'] = True + domains = CONFIG.get('domain') + for key in domains.keys(): + if key != domain: + domains[key]['enabled'] = False + else: + domains[key]['enabled'] = True + + if not log_level: + log_level = CONFIG.get('domain', {}).get('loglevel', CONFIG.get('loglevel', False)) + else: + CONFIG['loglevel'] = log_level + + if not port: + port = CONFIG.get('domain', {}).get('port', CONFIG.get('port', False)) + else: + CONFIG['port'] = port + + if devel is None and production is not None and (production is False or production is True): + CONFIG['production'] = production + + if devel is not None: + CONFIG['production'] = False + CONFIG['loglevel'] = 'debug' + + + if conftest: + click.echo( + toml.dumps(CONFIG) + ) + + else: + # domain section port is preferred, if it doesn't exist we use the global one + + uvicorn_kwargs = {} + + if CONFIG.get('port'): + uvicorn_kwargs['port'] = CONFIG['port'] + + if CONFIG.get('loglevel'): + uvicorn_kwargs['log_level'] = CONFIG['loglevel'].lower() + + if watch: + uvicorn_kwargs['reload'] = True + + uvicorn.run( + 'halfapi.app:application', + factory=True, + **uvicorn_kwargs + ) sys.exit(0) diff --git a/halfapi/conf.py b/halfapi/conf.py index 3d83706..f6af7fa 100644 --- a/halfapi/conf.py +++ b/halfapi/conf.py @@ -46,19 +46,51 @@ import uuid import toml -PRODUCTION = True -LOGLEVEL = 'info' -CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config') -DRYRUN = bool(os.environ.get('HALFAPI_DRYRUN', False)) - SCHEMA = {} -CONF_DIR = environ.get('HALFAPI_CONF_DIR', '/etc/half_api') +DEFAULT_CONF = { + # Default configuration values + 'SECRET': tempfile.mkstemp()[1], + 'PROJECT_NAME': os.getcwd().split('/')[-1], + 'PRODUCTION': True, + 'HOST': '127.0.0.1', + 'PORT': 3000, + 'LOGLEVEL': 'info', + 'BASE_DIR': os.getcwd(), + 'CONF_FILE': '.halfapi/config', + 'CONF_DIR': '/etc/half_api', + 'DRYRUN': None +} + +PROJECT_LEVEL_KEYS = { + # Allowed keys in "project" section of configuration file + 'project_name', + 'production', + 'secret', + 'host', + 'port', + 'loglevel', + 'dryrun' +} + +DOMAIN_LEVEL_KEYS = PROJECT_LEVEL_KEYS | { + # Allowed keys in "domain" section of configuration file + 'name', + 'module', + 'prefix', + 'enabled' +} + +CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', DEFAULT_CONF['CONF_FILE']) +CONF_DIR = os.environ.get('HALFAPI_CONF_DIR', DEFAULT_CONF['CONF_DIR']) + HALFAPI_ETC_FILE=os.path.join( CONF_DIR, 'config' ) + +BASE_DIR = os.environ.get('HALFAPI_BASE_DIR', DEFAULT_CONF['BASE_DIR']) HALFAPI_DOT_FILE=os.path.join( - os.getcwd(), '.halfapi', 'config') + BASE_DIR, '.halfapi', 'config') HALFAPI_CONFIG_FILES = [] @@ -66,15 +98,36 @@ try: with open(HALFAPI_ETC_FILE, 'r'): HALFAPI_CONFIG_FILES.append(HALFAPI_ETC_FILE) except FileNotFoundError: - logger.error('Cannot find a configuration file under %s', HALFAPI_DOT_FILE) + logger.info('Cannot find a configuration file under %s', HALFAPI_ETC_FILE) try: with open(HALFAPI_DOT_FILE, 'r'): HALFAPI_CONFIG_FILES.append(HALFAPI_DOT_FILE) except FileNotFoundError: - logger.error('Cannot find a configuration file under %s', HALFAPI_DOT_FILE) + logger.info('Cannot find a configuration file under %s', HALFAPI_DOT_FILE) +ENVIRONMENT = {} +# Load environment variables allowed in configuration + +if 'HALFAPI_DRYRUN' in os.environ: + ENVIRONMENT['dryrun'] = True + +if 'HALFAPI_PROD' in os.environ: + ENVIRONMENT['production'] = bool(os.environ.get('HALFAPI_PROD')) + +if 'HALFAPI_LOGLEVEL' in os.environ: + ENVIRONMENT['loglevel'] = os.environ.get('HALFAPI_LOGLEVEL').lower() + +if 'HALFAPI_SECRET' in os.environ: + ENVIRONMENT['secret'] = os.environ.get('HALFAPI_SECRET') + +if 'HALFAPI_HOST' in os.environ: + ENVIRONMENT['host'] = os.environ.get('HALFAPI_HOST') + +if 'HALFAPI_PORT' in os.environ: + ENVIRONMENT['port'] = int(os.environ.get('HALFAPI_PORT')) + def read_config(filenames=HALFAPI_CONFIG_FILES): """ The highest index in "filenames" are the highest priorty @@ -84,18 +137,22 @@ def read_config(filenames=HALFAPI_CONFIG_FILES): logger.info('Reading config files %s', filenames) for CONF_FILE in filenames: if os.path.isfile(CONF_FILE): - d_res.update( toml.load(CONF_FILE) ) + conf_dict = toml.load(CONF_FILE) + d_res.update(conf_dict) logger.info('Read config files (result) %s', d_res) return { **d_res.get('project', {}), 'domain': d_res.get('domain', {}) } CONFIG = read_config() +CONFIG.update(**ENVIRONMENT) PROJECT_NAME = CONFIG.get('project_name', - environ.get('HALFAPI_PROJECT_NAME', os.getcwd().split('/')[-1])) + os.environ.get('HALFAPI_PROJECT_NAME', DEFAULT_CONF['PROJECT_NAME'])) -if environ.get('HALFAPI_DOMAIN_NAME'): - DOMAIN_NAME = environ.get('HALFAPI_DOMAIN_NAME') +if os.environ.get('HALFAPI_DOMAIN_NAME'): + # Force enabled domain by environment variable + + DOMAIN_NAME = os.environ.get('HALFAPI_DOMAIN_NAME') if 'domain' in CONFIG and DOMAIN_NAME in CONFIG['domain'] \ and 'config' in CONFIG['domain'][DOMAIN_NAME]: @@ -113,53 +170,36 @@ if environ.get('HALFAPI_DOMAIN_NAME'): CONFIG['domain'][DOMAIN_NAME]['config'] = domain_config - if environ.get('HALFAPI_DOMAIN_MODULE'): - dom_module = environ.get('HALFAPI_DOMAIN_MODULE') + if os.environ.get('HALFAPI_DOMAIN_MODULE'): + # Specify the pythonpath to import the specified domain (defaults to global) + dom_module = os.environ.get('HALFAPI_DOMAIN_MODULE') CONFIG['domain'][DOMAIN_NAME]['module'] = dom_module if len(CONFIG.get('domain', {}).keys()) == 0: logger.info('No domains') -# Bind -HOST = CONFIG.get('host', - environ.get('HALFAPI_HOST', '127.0.0.1')) -PORT = int(CONFIG.get( - 'port', - environ.get('HALFAPI_PORT', '3000'))) - # Secret -SECRET = CONFIG.get( - 'secret', - environ.get('HALFAPI_SECRET')) - -if not SECRET: +if 'secret' not in CONFIG: # TODO: Create a temporary secret - _, SECRET = tempfile.mkstemp() - with open(SECRET, 'w') as secret_file: + CONFIG['secret'] = DEFAULT_CONF['SECRET'] + with open(CONFIG['secret'], 'w') as secret_file: secret_file.write(str(uuid.uuid4())) try: - with open(SECRET, 'r') as secret_file: - CONFIG['secret'] = SECRET.strip() + with open(CONFIG['secret'], 'r') as secret_file: + CONFIG['secret'] = CONFIG['secret'].strip() except FileNotFoundError as exc: - logger.info('Running without secret file: %s', SECRET or 'no file specified') + logger.warning('Running without secret file: %s', CONFIG['secret'] or 'no file specified') -PRODUCTION = bool(CONFIG.get( - 'production', - environ.get('HALFAPI_PROD', True))) +CONFIG.setdefault('project_name', DEFAULT_CONF['PROJECT_NAME']) +CONFIG.setdefault('production', DEFAULT_CONF['PRODUCTION']) +CONFIG.setdefault('host', DEFAULT_CONF['HOST']) +CONFIG.setdefault('port', DEFAULT_CONF['PORT']) +CONFIG.setdefault('loglevel', DEFAULT_CONF['LOGLEVEL']) +CONFIG.setdefault('dryrun', DEFAULT_CONF['DRYRUN']) -LOGLEVEL = CONFIG.get( - 'loglevel', - environ.get('HALFAPI_LOGLEVEL', 'info')).lower() - -BASE_DIR = CONFIG.get( - 'base_dir', - environ.get('HALFAPI_BASE_DIR', '.')) - -CONFIG['project_name'] = PROJECT_NAME -CONFIG['production'] = PRODUCTION -CONFIG['secret'] = SECRET -CONFIG['host'] = HOST -CONFIG['port'] = PORT -CONFIG['dryrun'] = DRYRUN +# !!!TO REMOVE!!! +SECRET = CONFIG.get('SECRET') +PRODUCTION = CONFIG.get('production') +# !!! diff --git a/halfapi/halfapi.py b/halfapi/halfapi.py index 0118abc..b71c7b9 100644 --- a/halfapi/halfapi.py +++ b/halfapi/halfapi.py @@ -45,15 +45,24 @@ from .half_domain import HalfDomain from halfapi import __version__ class HalfAPI(Starlette): - def __init__(self, config, + def __init__(self, + config, d_routes=None): - config_logging(logging.DEBUG) + # Set log level (defaults to debug) + config_logging( + getattr(logging, config.get('loglevel', 'DEBUG').upper(), 'DEBUG') + ) self.config = config - logger.debug('HalfAPI.config: %s', self.config) - SECRET = self.config.get('secret') PRODUCTION = self.config.get('production', True) DRYRUN = self.config.get('dryrun', False) + TIMINGMIDDLEWARE = self.config.get('timingmiddleware', False) + + if DRYRUN: + logger.info('HalfAPI starting in dry-run mode') + else: + logger.info('HalfAPI starting') + self.PRODUCTION = PRODUCTION self.SECRET = SECRET @@ -67,7 +76,7 @@ class HalfAPI(Starlette): Mount('/halfapi', routes=list(self.halfapi_routes())) ) - logger.info('Config: %s', self.config) + logger.debug('Config: %s', self.config) domains = { key: elt @@ -75,7 +84,7 @@ class HalfAPI(Starlette): if elt.get('enabled', False) } - logger.info('Active domains: %s', domains) + logger.debug('Active domains: %s', domains) if d_routes: # Mount the routes from the d_routes argument - domain-less mode @@ -147,7 +156,7 @@ class HalfAPI(Starlette): on_error=on_auth_error ) - if not PRODUCTION: + if not PRODUCTION and TIMINGMIDDLEWARE: self.add_middleware( TimingMiddleware, client=HTimingClient(), @@ -297,3 +306,12 @@ class HalfAPI(Starlette): self.mount(kwargs.get('path', name), self.__domains[name]) return self.__domains[name] + + +def __main__(): + return HalfAPI(CONFIG).application + +if __name__ == '__main__': + __main__() + + diff --git a/halfapi/logging.py b/halfapi/logging.py index 0e34e02..f064463 100644 --- a/halfapi/logging.py +++ b/halfapi/logging.py @@ -1,8 +1,10 @@ import logging +default_level = logging.DEBUG +default_format = '%(asctime)s [%(process)d] [%(levelname)s] %(message)s' +default_datefmt = '[%Y-%m-%d %H:%M:%S %z]' -def config_logging(level=logging.INFO): - +def config_logging(level=default_level, format=default_format, datefmt=default_datefmt): # When run by 'uvicorn ...', a root handler is already # configured and the basicConfig below does nothing. # To get the desired formatting: @@ -12,8 +14,8 @@ def config_logging(level=logging.INFO): # https://github.com/encode/uvicorn/issues/511 logging.basicConfig( # match gunicorn format - format='%(asctime)s [%(process)d] [%(levelname)s] %(message)s', - datefmt='[%Y-%m-%d %H:%M:%S %z]', + format=format, + datefmt=datefmt, level=level) # When run by 'gunicorn -k uvicorn.workers.UvicornWorker ...', @@ -27,5 +29,4 @@ def config_logging(level=logging.INFO): logging.getLogger('uvicorn.access').propagate = True logging.getLogger('uvicorn.error').propagate = True -config_logging() logger = logging.getLogger() diff --git a/halfapi/testing/test_domain.py b/halfapi/testing/test_domain.py index 295d11c..a61f508 100644 --- a/halfapi/testing/test_domain.py +++ b/halfapi/testing/test_domain.py @@ -11,6 +11,7 @@ from click.testing import CliRunner from ..cli.cli import cli from ..halfapi import HalfAPI from ..half_domain import HalfDomain +from ..conf import DEFAULT_CONF from pprint import pprint import tempfile @@ -62,6 +63,7 @@ class TestDomain(TestCase): self.runner = class_(mix_stderr=False) # HTTP + # Fake default values of default configuration self.halfapi_conf = { 'secret': 'testsecret', 'production': False, diff --git a/tests/cli/test_cli_proj.py b/tests/cli/test_cli_proj.py index ff14ccb..03eb137 100644 --- a/tests/cli/test_cli_proj.py +++ b/tests/cli/test_cli_proj.py @@ -10,6 +10,7 @@ import toml import pytest from click.testing import CliRunner from configparser import ConfigParser +from halfapi.conf import DEFAULT_CONF, PROJECT_LEVEL_KEYS, DOMAIN_LEVEL_KEYS PROJNAME = os.environ.get('PROJ','tmp_api') @@ -24,26 +25,50 @@ class TestCliProj(): def test_domain_commands(self, project_runner): """ TODO: Test create command """ + test_conf = { + 'project': { + 'port': '3010', + 'loglevel': 'warning' + }, + 'domain': { + 'dummy_domain': { + 'port': 4242, + 'name': 'dummy_domain', + 'enabled': True + } + } + } + r = project_runner('domain') print(r.stdout) assert r.exit_code == 1 _, tmp_conf = tempfile.mkstemp() with open(tmp_conf, 'w') as fh: fh.write( - toml.dumps({ - 'domain': { - 'dummy_domain': { - 'port': 4242, - 'name': 'dummy_domain', - 'enabled': True - } - }, - 'project': { - 'dryrun': True - } - }) + toml.dumps(test_conf) ) + r = project_runner(f'domain dummy_domain --conftest {tmp_conf}') + assert r.exit_code == 0 + r_conf = toml.loads(r.stdout) + for key, value in r_conf.items(): + if key == 'domain': + continue + assert key in PROJECT_LEVEL_KEYS + if key == 'port': + assert value == test_conf['domain']['dummy_domain']['port'] + elif key == 'loglevel': + assert value == test_conf['project']['loglevel'] + else: + assert value == DEFAULT_CONF[key.upper()] + + + assert json.dumps(test_conf['domain']) == json.dumps(r_conf['domain']) + + for key in test_conf['domain']['dummy_domain'].keys(): + assert key in DOMAIN_LEVEL_KEYS + + # Default command "run" r = project_runner(f'domain dummy_domain --dry-run {tmp_conf}') print(r.stdout) assert r.exit_code == 0