[rc] 0.6.28rc3 - fix bugs and general configuration management cleanup (see changelog)

CHANGELOG.md

@@ -48,13 +48,36 @@ it's use in the "tests/dummy_domain/__init__.py" file.
 The use of an "HEAD" request to check an ACL is now the norm. Please change all
 the occurrences of your calls on theses routes with the GET method.
 
-### Commits
 
-- [doc-schema] the "/" route on a domain now returns the OpenAPI-validated Schema (not a list of schemas), the "dummy_domain" test now validates OpenAPI specs
+### CLI
-- [doc-schema] In module-based routers, if there is a path parameter, you can specify an OpenAPI documentation for it, or a default will be used
+
-- [dev-deps] openapi-schema-validator, openapi-spec-validator
+Domain command update :
-- [doc] add docstrings for halfapi routes
+
-- [acl] The public acls check routes use the "HEAD" method, deprecated "GET"
+The `--conftest` flag is now allowed when running the `domain` command, it dumps the current configuration as a TOML string.
+
+`halfapi domain --conftest my_domain`
+
+
+The `--dry-run` flag was buggy and is now fixed when using the `domai ` command with the `--run` flag.
+
+
+### Configuration
+
+The `port` option in a `domain.my_domain` section in the TOML config file is now prefered to the one in the `project` section.
+
+The `project` section is used as a default section for the whole configuration file. - Tests still have to be written -
+
+The standard configuration precedence is fixed, in this order from the hight to the lower :
+
+- Argument value (i.e. : --log-level)
+- Environment value (i.e. : HALFAPI_LOGLEVEL)
+- Configuration value under "domain" key
+- Configuration value under "project" key
+- Default configuration value given in the "DEFAULT_CONF" dictionary of halfapi/conf.py
+
+### Logs
+
+Small cleanup of the logs levels. If you don't want the config to be dumped, just set the HALFAPI_LOGLEVEL to something different than "DEBUG".
 
 ## 0.6.27
 

halfapi/__init__.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-__version__ = '0.6.28rc2'
+__version__ = '0.6.28rc3'
 
 def version():
     return f'HalfAPI version:{__version__}'

halfapi/app.py

@@ -1,11 +1,7 @@
 import os
 from .halfapi import HalfAPI
 from .logging import logger
-from .conf import read_config
 
 def application():
-    config_file = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config')
+    from .conf import CONFIG
-
-    CONFIG = read_config([config_file])
-
     return HalfAPI(CONFIG).application

halfapi/cli/domain.py

@@ -17,11 +17,13 @@ import uvicorn
 
 
 from .cli import cli
+from ..conf import CONFIG
 
 from ..half_domain import HalfDomain
 
 from ..lib.routes import api_routes
 from ..lib.responses import ORJSONResponse
+from ..conf import CONFIG, PROJECT_LEVEL_KEYS
 
 
 from ..logging import logger
@@ -119,16 +121,22 @@ def list_api_routes():
     #     list_routes(domain, m_dom)
 
 
+@click.option('--devel',default=None, is_flag=True)
+@click.option('--watch',default=False, is_flag=True)
+@click.option('--production',default=None, is_flag=True)
+@click.option('--port',default=None, type=int)
+@click.option('--log-level',default=None, type=str)
 @click.option('--dry-run',default=False, is_flag=True)
 @click.option('--run',default=False, is_flag=True)
 @click.option('--read',default=False, is_flag=True)
+@click.option('--conftest',default=False, is_flag=True)
 @click.option('--create',default=False, is_flag=True)
 @click.option('--update',default=False, is_flag=True)
 @click.option('--delete',default=False, is_flag=True)
 @click.argument('config_file', type=click.File(mode='rb'), required=False)
 @click.argument('domain',default=None, required=False)
 @cli.command()
-def domain(domain, config_file, delete, update, create, read, run, dry_run):  #, domains, read, create, update, delete):
+def domain(domain, config_file, delete, update, create, conftest, read, run, dry_run, log_level, port, production, watch, devel):
     """
     The "halfapi domain" command
 
@@ -147,11 +155,19 @@ def domain(domain, config_file, delete, update, create, read, run, dry_run):  #,
         raise Exception('Missing domain name')
 
     if config_file:
-        CONFIG = toml.load(config_file.name)
+        ARG_CONFIG = toml.load(config_file.name)
             
-        os.environ['HALFAPI_CONF_FILE'] = config_file.name
+        if 'project' in ARG_CONFIG:
-    else:
+            for key, value in ARG_CONFIG['project'].items():
-        from halfapi.conf import CONFIG
+                if key in PROJECT_LEVEL_KEYS:
+                    CONFIG[key] = value
+
+        if 'domain' in ARG_CONFIG and domain in ARG_CONFIG['domain']:
+            for key, value in ARG_CONFIG['domain'][domain].items():
+                if key in PROJECT_LEVEL_KEYS:
+                    CONFIG[key] = value
+
+        CONFIG['domain'].update(ARG_CONFIG['domain'])
 
     if create:
         raise NotImplementedError
@@ -170,14 +186,57 @@ def domain(domain, config_file, delete, update, create, read, run, dry_run):  #,
         )
 
     else:
-        port = CONFIG.get('port',
+        if dry_run:
-            CONFIG.get('domain', {}).get('port')
+            CONFIG['dryrun'] = True
+
+        domains = CONFIG.get('domain')
+        for key in domains.keys():
+            if key != domain:
+                domains[key]['enabled'] = False
+            else:
+                domains[key]['enabled'] = True
+
+        if not log_level:
+            log_level = CONFIG.get('domain', {}).get('loglevel', CONFIG.get('loglevel', False))
+        else:
+            CONFIG['loglevel'] = log_level
+
+        if not port:
+            port = CONFIG.get('domain', {}).get('port', CONFIG.get('port', False))
+        else:
+            CONFIG['port'] = port
+
+        if devel is None and production is not None and (production is False or production is True):
+            CONFIG['production'] = production
+
+        if devel is not None:
+            CONFIG['production'] = False
+            CONFIG['loglevel'] = 'debug'
+
+
+        if conftest:
+            click.echo(
+                toml.dumps(CONFIG)
             )
+
+        else:
+            # domain section port is preferred, if it doesn't exist we use the global one
+
+            uvicorn_kwargs = {}
+
+            if CONFIG.get('port'):
+                uvicorn_kwargs['port'] = CONFIG['port']
+
+            if CONFIG.get('loglevel'):
+                uvicorn_kwargs['log_level'] = CONFIG['loglevel'].lower()
+
+            if watch:
+                uvicorn_kwargs['reload'] = True
+
             uvicorn.run(
                 'halfapi.app:application',
-            port=port,
+                factory=True,
-            factory=True
+                **uvicorn_kwargs
             )
 
-
     sys.exit(0)

halfapi/conf.py

@@ -46,19 +46,51 @@ import uuid
 
 import toml
 
-PRODUCTION = True
-LOGLEVEL = 'info'
-CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config')
-DRYRUN = bool(os.environ.get('HALFAPI_DRYRUN', False))
-
 SCHEMA = {}
 
-CONF_DIR = environ.get('HALFAPI_CONF_DIR', '/etc/half_api')
+DEFAULT_CONF = {
+    # Default configuration values
+    'SECRET': tempfile.mkstemp()[1],
+    'PROJECT_NAME': os.getcwd().split('/')[-1],
+    'PRODUCTION': True,
+    'HOST': '127.0.0.1',
+    'PORT': 3000,
+    'LOGLEVEL': 'info',
+    'BASE_DIR': os.getcwd(),
+    'CONF_FILE': '.halfapi/config',
+    'CONF_DIR': '/etc/half_api',
+    'DRYRUN': None
+}
+
+PROJECT_LEVEL_KEYS = {
+    # Allowed keys in "project" section of configuration file
+    'project_name',
+    'production',
+    'secret',
+    'host',
+    'port',
+    'loglevel',
+    'dryrun'
+}
+
+DOMAIN_LEVEL_KEYS = PROJECT_LEVEL_KEYS | {
+    # Allowed keys in "domain" section of configuration file
+    'name',
+    'module',
+    'prefix',
+    'enabled'
+}
+
+CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', DEFAULT_CONF['CONF_FILE'])
+CONF_DIR = os.environ.get('HALFAPI_CONF_DIR', DEFAULT_CONF['CONF_DIR'])
+
 HALFAPI_ETC_FILE=os.path.join(
     CONF_DIR, 'config'
 )
+
+BASE_DIR = os.environ.get('HALFAPI_BASE_DIR', DEFAULT_CONF['BASE_DIR'])
 HALFAPI_DOT_FILE=os.path.join(
-    os.getcwd(), '.halfapi', 'config')
+    BASE_DIR, '.halfapi', 'config')
 
 HALFAPI_CONFIG_FILES = []
 
@@ -66,15 +98,36 @@ try:
     with open(HALFAPI_ETC_FILE, 'r'):
         HALFAPI_CONFIG_FILES.append(HALFAPI_ETC_FILE)
 except FileNotFoundError:
-    logger.error('Cannot find a configuration file under %s', HALFAPI_DOT_FILE)
+    logger.info('Cannot find a configuration file under %s', HALFAPI_ETC_FILE)
 
 try:
     with open(HALFAPI_DOT_FILE, 'r'):
         HALFAPI_CONFIG_FILES.append(HALFAPI_DOT_FILE)
 except FileNotFoundError:
-    logger.error('Cannot find a configuration file under %s', HALFAPI_DOT_FILE)
+    logger.info('Cannot find a configuration file under %s', HALFAPI_DOT_FILE)
 
 
+ENVIRONMENT = {}
+# Load environment variables allowed in configuration
+
+if 'HALFAPI_DRYRUN' in os.environ:
+    ENVIRONMENT['dryrun'] = True
+
+if 'HALFAPI_PROD' in os.environ:
+    ENVIRONMENT['production'] = bool(os.environ.get('HALFAPI_PROD'))
+
+if 'HALFAPI_LOGLEVEL' in os.environ:
+    ENVIRONMENT['loglevel'] = os.environ.get('HALFAPI_LOGLEVEL').lower()
+
+if 'HALFAPI_SECRET' in os.environ:
+    ENVIRONMENT['secret'] = os.environ.get('HALFAPI_SECRET')
+
+if 'HALFAPI_HOST' in os.environ:
+    ENVIRONMENT['host'] = os.environ.get('HALFAPI_HOST')
+
+if 'HALFAPI_PORT' in os.environ:
+    ENVIRONMENT['port'] = int(os.environ.get('HALFAPI_PORT'))
+
 def read_config(filenames=HALFAPI_CONFIG_FILES):
     """
     The highest index in "filenames" are the highest priorty
@@ -84,18 +137,22 @@ def read_config(filenames=HALFAPI_CONFIG_FILES):
     logger.info('Reading config files %s', filenames)
     for CONF_FILE in filenames:
         if os.path.isfile(CONF_FILE):
-            d_res.update( toml.load(CONF_FILE) )
+            conf_dict = toml.load(CONF_FILE)
+            d_res.update(conf_dict)
 
     logger.info('Read config files (result) %s', d_res)
     return { **d_res.get('project', {}), 'domain': d_res.get('domain', {}) }
 
 CONFIG = read_config()
+CONFIG.update(**ENVIRONMENT)
 
 PROJECT_NAME = CONFIG.get('project_name',
-    environ.get('HALFAPI_PROJECT_NAME', os.getcwd().split('/')[-1]))
+    os.environ.get('HALFAPI_PROJECT_NAME', DEFAULT_CONF['PROJECT_NAME']))
 
-if environ.get('HALFAPI_DOMAIN_NAME'):
+if os.environ.get('HALFAPI_DOMAIN_NAME'):
-    DOMAIN_NAME = environ.get('HALFAPI_DOMAIN_NAME')
+    # Force enabled domain by environment variable
+
+    DOMAIN_NAME = os.environ.get('HALFAPI_DOMAIN_NAME')
     if 'domain' in CONFIG and DOMAIN_NAME in CONFIG['domain'] \
         and 'config' in CONFIG['domain'][DOMAIN_NAME]:
 
@@ -113,53 +170,36 @@ if environ.get('HALFAPI_DOMAIN_NAME'):
 
     CONFIG['domain'][DOMAIN_NAME]['config'] = domain_config
 
-    if environ.get('HALFAPI_DOMAIN_MODULE'):
+    if os.environ.get('HALFAPI_DOMAIN_MODULE'):
-        dom_module = environ.get('HALFAPI_DOMAIN_MODULE')
+        # Specify the pythonpath to import the specified domain (defaults to global)
+        dom_module = os.environ.get('HALFAPI_DOMAIN_MODULE')
         CONFIG['domain'][DOMAIN_NAME]['module'] = dom_module
 
 if len(CONFIG.get('domain', {}).keys()) == 0:
     logger.info('No domains')
 
-# Bind
-HOST = CONFIG.get('host',
-    environ.get('HALFAPI_HOST', '127.0.0.1'))
-PORT = int(CONFIG.get(
-    'port',
-    environ.get('HALFAPI_PORT', '3000')))
-
 
 # Secret
-SECRET = CONFIG.get(
+if 'secret' not in CONFIG:
-    'secret',
-    environ.get('HALFAPI_SECRET'))
-
-if not SECRET:
     # TODO: Create a temporary secret
-    _, SECRET = tempfile.mkstemp()
+    CONFIG['secret'] = DEFAULT_CONF['SECRET']
-    with open(SECRET, 'w') as secret_file:
+    with open(CONFIG['secret'], 'w') as secret_file:
         secret_file.write(str(uuid.uuid4()))
 
 try:
-    with open(SECRET, 'r') as secret_file:
+    with open(CONFIG['secret'], 'r') as secret_file:
-        CONFIG['secret'] = SECRET.strip()
+        CONFIG['secret'] = CONFIG['secret'].strip()
 except FileNotFoundError as exc:
-    logger.info('Running without secret file: %s', SECRET or 'no file specified')
+    logger.warning('Running without secret file: %s', CONFIG['secret'] or 'no file specified')
 
-PRODUCTION = bool(CONFIG.get(
+CONFIG.setdefault('project_name', DEFAULT_CONF['PROJECT_NAME'])
-    'production',
+CONFIG.setdefault('production', DEFAULT_CONF['PRODUCTION'])
-    environ.get('HALFAPI_PROD', True)))
+CONFIG.setdefault('host', DEFAULT_CONF['HOST'])
+CONFIG.setdefault('port', DEFAULT_CONF['PORT'])
+CONFIG.setdefault('loglevel', DEFAULT_CONF['LOGLEVEL'])
+CONFIG.setdefault('dryrun', DEFAULT_CONF['DRYRUN'])
 
-LOGLEVEL = CONFIG.get(
+# !!!TO REMOVE!!!
-    'loglevel',
+SECRET = CONFIG.get('SECRET')
-    environ.get('HALFAPI_LOGLEVEL', 'info')).lower()
+PRODUCTION = CONFIG.get('production')
-
+# !!!
-BASE_DIR = CONFIG.get(
-    'base_dir',
-    environ.get('HALFAPI_BASE_DIR', '.'))
-
-CONFIG['project_name'] = PROJECT_NAME
-CONFIG['production'] = PRODUCTION
-CONFIG['secret'] = SECRET
-CONFIG['host'] = HOST
-CONFIG['port'] = PORT
-CONFIG['dryrun'] = DRYRUN

halfapi/halfapi.py

@@ -45,15 +45,24 @@ from .half_domain import HalfDomain
 from halfapi import __version__
 
 class HalfAPI(Starlette):
-    def __init__(self, config,
+    def __init__(self,
+        config,
         d_routes=None):
-        config_logging(logging.DEBUG)
+        # Set log level (defaults to debug)
+        config_logging(
+            getattr(logging, config.get('loglevel', 'DEBUG').upper(), 'DEBUG')
+        )
         self.config = config
-        logger.debug('HalfAPI.config: %s', self.config)
-
         SECRET = self.config.get('secret')
         PRODUCTION = self.config.get('production', True)
         DRYRUN = self.config.get('dryrun', False)
+        TIMINGMIDDLEWARE = self.config.get('timingmiddleware', False)
+
+        if DRYRUN:
+            logger.info('HalfAPI starting in dry-run mode')
+        else:
+            logger.info('HalfAPI starting')
+
 
         self.PRODUCTION = PRODUCTION
         self.SECRET = SECRET
@@ -67,7 +76,7 @@ class HalfAPI(Starlette):
             Mount('/halfapi', routes=list(self.halfapi_routes()))
         )
 
-        logger.info('Config: %s', self.config)
+        logger.debug('Config: %s', self.config)
 
         domains = {
             key: elt
@@ -75,7 +84,7 @@ class HalfAPI(Starlette):
             if elt.get('enabled', False)
         }
 
-        logger.info('Active domains: %s', domains)
+        logger.debug('Active domains: %s', domains)
 
         if d_routes:
             # Mount the routes from the d_routes argument - domain-less mode
@@ -147,7 +156,7 @@ class HalfAPI(Starlette):
                 on_error=on_auth_error
             )
 
-        if not PRODUCTION:
+        if not PRODUCTION and TIMINGMIDDLEWARE:
             self.add_middleware(
                 TimingMiddleware,
                 client=HTimingClient(),
@@ -297,3 +306,12 @@ class HalfAPI(Starlette):
         self.mount(kwargs.get('path', name), self.__domains[name])
 
         return self.__domains[name]
+
+
+def __main__():
+    return HalfAPI(CONFIG).application
+
+if __name__ == '__main__':
+    __main__()
+
+

halfapi/logging.py

@@ -1,8 +1,10 @@
 import logging
 
+default_level = logging.DEBUG
+default_format = '%(asctime)s [%(process)d] [%(levelname)s] %(message)s'
+default_datefmt = '[%Y-%m-%d %H:%M:%S %z]'
 
-def config_logging(level=logging.INFO):
+def config_logging(level=default_level, format=default_format, datefmt=default_datefmt):
-
     # When run by 'uvicorn ...', a root handler is already
     # configured and the basicConfig below does nothing.
     # To get the desired formatting:
@@ -12,8 +14,8 @@ def config_logging(level=logging.INFO):
     #   https://github.com/encode/uvicorn/issues/511
     logging.basicConfig(
         # match gunicorn format
-        format='%(asctime)s [%(process)d] [%(levelname)s] %(message)s',
+        format=format,
-        datefmt='[%Y-%m-%d %H:%M:%S %z]',
+        datefmt=datefmt,
         level=level)
 
         # When run by 'gunicorn -k uvicorn.workers.UvicornWorker ...',
@@ -27,5 +29,4 @@ def config_logging(level=logging.INFO):
     logging.getLogger('uvicorn.access').propagate = True
     logging.getLogger('uvicorn.error').propagate = True
 
-config_logging()
 logger = logging.getLogger()

halfapi/testing/test_domain.py

@@ -11,6 +11,7 @@ from click.testing import CliRunner
 from ..cli.cli import cli
 from ..halfapi import HalfAPI
 from ..half_domain import HalfDomain
+from ..conf import DEFAULT_CONF
 from pprint import pprint
 import tempfile
 
@@ -62,6 +63,7 @@ class TestDomain(TestCase):
         self.runner = class_(mix_stderr=False)
 
         # HTTP
+        # Fake default values of default configuration
         self.halfapi_conf = {
             'secret': 'testsecret',
             'production': False,

tests/cli/test_cli_proj.py

@@ -10,6 +10,7 @@ import toml
 import pytest
 from click.testing import CliRunner
 from configparser import ConfigParser
+from halfapi.conf import DEFAULT_CONF, PROJECT_LEVEL_KEYS, DOMAIN_LEVEL_KEYS
 
 PROJNAME = os.environ.get('PROJ','tmp_api')
 
@@ -24,26 +25,50 @@ class TestCliProj():
     def test_domain_commands(self, project_runner):
         """ TODO: Test create command
         """
-        r = project_runner('domain')
+        test_conf = {
-        print(r.stdout)
+            'project': {
-        assert r.exit_code == 1
+                'port': '3010',
-        _, tmp_conf = tempfile.mkstemp()
+                'loglevel': 'warning'
-        with open(tmp_conf, 'w') as fh:
+            },
-            fh.write(
-                toml.dumps({
             'domain': {
                 'dummy_domain': {
                     'port': 4242,
                     'name': 'dummy_domain',
                     'enabled': True
                 }
-                    },
-                    'project': {
-                        'dryrun': True
             }
-                })
+        }
+
+        r = project_runner('domain')
+        print(r.stdout)
+        assert r.exit_code == 1
+        _, tmp_conf = tempfile.mkstemp()
+        with open(tmp_conf, 'w') as fh:
+            fh.write(
+                toml.dumps(test_conf)
             )
 
+        r = project_runner(f'domain dummy_domain --conftest {tmp_conf}')
+        assert r.exit_code == 0
+        r_conf = toml.loads(r.stdout)
+        for key, value in r_conf.items():
+            if key == 'domain':
+                continue
+            assert key in PROJECT_LEVEL_KEYS
+            if key == 'port':
+                assert value == test_conf['domain']['dummy_domain']['port']
+            elif key == 'loglevel':
+                assert value == test_conf['project']['loglevel']
+            else:
+                assert value == DEFAULT_CONF[key.upper()]
+
+
+        assert json.dumps(test_conf['domain']) == json.dumps(r_conf['domain'])
+
+        for key in test_conf['domain']['dummy_domain'].keys():
+            assert key in DOMAIN_LEVEL_KEYS
+
+        # Default command "run"
         r = project_runner(f'domain dummy_domain --dry-run {tmp_conf}')
         print(r.stdout)
         assert r.exit_code == 0