masq
/
halfapi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[rc] 0.6.28rc3 - fix bugs and general configuration management cleanup (see changelog)

This commit is contained in:
Maxime Alves LIRMM 2023-08-20 23:32:50 +02:00
parent 65ecf9817c
commit 28a1a69435
9 changed files with 262 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
CHANGELOG.md
View File

@ -48,13 +48,36 @@ it's use in the "tests/dummy_domain/__init__.py" file.
The use of an "HEAD" request to check an ACL is now the norm. Please change all
the occurrences of your calls on theses routes with the GET method.
### Commits
- [doc-schema] the "/" route on a domain now returns the OpenAPI-validated Schema (not a list of schemas), the "dummy_domain" test now validates OpenAPI specs
- [doc-schema] In module-based routers, if there is a path parameter, you can specify an OpenAPI documentation for it, or a default will be used
- [dev-deps] openapi-schema-validator, openapi-spec-validator
- [doc] add docstrings for halfapi routes
- [acl] The public acls check routes use the "HEAD" method, deprecated "GET"
### CLI
Domain command update :
The `--conftest` flag is now allowed when running the `domain` command, it dumps the current configuration as a TOML string.
`halfapi domain --conftest my_domain`
The `--dry-run` flag was buggy and is now fixed when using the `domai ` command with the `--run` flag.
### Configuration
The `port` option in a `domain.my_domain` section in the TOML config file is now prefered to the one in the `project` section.
The `project` section is used as a default section for the whole configuration file. - Tests still have to be written -
The standard configuration precedence is fixed, in this order from the hight to the lower :
- Argument value (i.e. : --log-level)
- Environment value (i.e. : HALFAPI_LOGLEVEL)
- Configuration value under "domain" key
- Configuration value under "project" key
- Default configuration value given in the "DEFAULT_CONF" dictionary of halfapi/conf.py
### Logs
Small cleanup of the logs levels. If you don't want the config to be dumped, just set the HALFAPI_LOGLEVEL to something different than "DEBUG".
## 0.6.27

2
halfapi/__init__.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3
__version__ = '0.6.28rc2'
__version__ = '0.6.28rc3'
def version():
return f'HalfAPI version:{__version__}'

6
halfapi/app.py
View File

@ -1,11 +1,7 @@
import os
from .halfapi import HalfAPI
from .logging import logger
from .conf import read_config
def application():
config_file = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config')
CONFIG = read_config([config_file])
from .conf import CONFIG
return HalfAPI(CONFIG).application

85
halfapi/cli/domain.py
View File

@ -17,11 +17,13 @@ import uvicorn
from .cli import cli
from ..conf import CONFIG
from ..half_domain import HalfDomain
from ..lib.routes import api_routes
from ..lib.responses import ORJSONResponse
from ..conf import CONFIG, PROJECT_LEVEL_KEYS
from ..logging import logger
@ -119,16 +121,22 @@ def list_api_routes():
# list_routes(domain, m_dom)
@click.option('--devel',default=None, is_flag=True)
@click.option('--watch',default=False, is_flag=True)
@click.option('--production',default=None, is_flag=True)
@click.option('--port',default=None, type=int)
@click.option('--log-level',default=None, type=str)
@click.option('--dry-run',default=False, is_flag=True)
@click.option('--run',default=False, is_flag=True)
@click.option('--read',default=False, is_flag=True)
@click.option('--conftest',default=False, is_flag=True)
@click.option('--create',default=False, is_flag=True)
@click.option('--update',default=False, is_flag=True)
@click.option('--delete',default=False, is_flag=True)
@click.argument('config_file', type=click.File(mode='rb'), required=False)
@click.argument('domain',default=None, required=False)
@cli.command()
def domain(domain, config_file, delete, update, create, read, run, dry_run): #, domains, read, create, update, delete):
def domain(domain, config_file, delete, update, create, conftest, read, run, dry_run, log_level, port, production, watch, devel):
"""
The "halfapi domain" command
@ -147,11 +155,19 @@ def domain(domain, config_file, delete, update, create, read, run, dry_run): #,
raise Exception('Missing domain name')
if config_file:
CONFIG = toml.load(config_file.name)
ARG_CONFIG = toml.load(config_file.name)
if 'project' in ARG_CONFIG:
for key, value in ARG_CONFIG['project'].items():
if key in PROJECT_LEVEL_KEYS:
CONFIG[key] = value
os.environ['HALFAPI_CONF_FILE'] = config_file.name
else:
from halfapi.conf import CONFIG
if 'domain' in ARG_CONFIG and domain in ARG_CONFIG['domain']:
for key, value in ARG_CONFIG['domain'][domain].items():
if key in PROJECT_LEVEL_KEYS:
CONFIG[key] = value
CONFIG['domain'].update(ARG_CONFIG['domain'])
if create:
raise NotImplementedError
@ -170,14 +186,57 @@ def domain(domain, config_file, delete, update, create, read, run, dry_run): #,
)
else:
port = CONFIG.get('port',
CONFIG.get('domain', {}).get('port')
)
uvicorn.run(
'halfapi.app:application',
port=port,
factory=True
)
if dry_run:
CONFIG['dryrun'] = True
domains = CONFIG.get('domain')
for key in domains.keys():
if key != domain:
domains[key]['enabled'] = False
else:
domains[key]['enabled'] = True
if not log_level:
log_level = CONFIG.get('domain', {}).get('loglevel', CONFIG.get('loglevel', False))
else:
CONFIG['loglevel'] = log_level
if not port:
port = CONFIG.get('domain', {}).get('port', CONFIG.get('port', False))
else:
CONFIG['port'] = port
if devel is None and production is not None and (production is False or production is True):
CONFIG['production'] = production
if devel is not None:
CONFIG['production'] = False
CONFIG['loglevel'] = 'debug'
if conftest:
click.echo(
toml.dumps(CONFIG)
)
else:
# domain section port is preferred, if it doesn't exist we use the global one
uvicorn_kwargs = {}
if CONFIG.get('port'):
uvicorn_kwargs['port'] = CONFIG['port']
if CONFIG.get('loglevel'):
uvicorn_kwargs['log_level'] = CONFIG['loglevel'].lower()
if watch:
uvicorn_kwargs['reload'] = True
uvicorn.run(
'halfapi.app:application',
factory=True,
**uvicorn_kwargs
)
sys.exit(0)

138
halfapi/conf.py
View File

@ -46,19 +46,51 @@ import uuid
import toml
PRODUCTION = True
LOGLEVEL = 'info'
CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', '.halfapi/config')
DRYRUN = bool(os.environ.get('HALFAPI_DRYRUN', False))
SCHEMA = {}
CONF_DIR = environ.get('HALFAPI_CONF_DIR', '/etc/half_api')
DEFAULT_CONF = {
# Default configuration values
'SECRET': tempfile.mkstemp()[1],
'PROJECT_NAME': os.getcwd().split('/')[-1],
'PRODUCTION': True,
'HOST': '127.0.0.1',
'PORT': 3000,
'LOGLEVEL': 'info',
'BASE_DIR': os.getcwd(),
'CONF_FILE': '.halfapi/config',
'CONF_DIR': '/etc/half_api',
'DRYRUN': None
}
PROJECT_LEVEL_KEYS = {
# Allowed keys in "project" section of configuration file
'project_name',
'production',
'secret',
'host',
'port',
'loglevel',
'dryrun'
}
DOMAIN_LEVEL_KEYS = PROJECT_LEVEL_KEYS | {
# Allowed keys in "domain" section of configuration file
'name',
'module',
'prefix',
'enabled'
}
CONF_FILE = os.environ.get('HALFAPI_CONF_FILE', DEFAULT_CONF['CONF_FILE'])
CONF_DIR = os.environ.get('HALFAPI_CONF_DIR', DEFAULT_CONF['CONF_DIR'])
HALFAPI_ETC_FILE=os.path.join(
CONF_DIR, 'config'
)
BASE_DIR = os.environ.get('HALFAPI_BASE_DIR', DEFAULT_CONF['BASE_DIR'])
HALFAPI_DOT_FILE=os.path.join(
os.getcwd(), '.halfapi', 'config')
BASE_DIR, '.halfapi', 'config')
HALFAPI_CONFIG_FILES = []
@ -66,15 +98,36 @@ try:
with open(HALFAPI_ETC_FILE, 'r'):
HALFAPI_CONFIG_FILES.append(HALFAPI_ETC_FILE)
except FileNotFoundError:
logger.error('Cannot find a configuration file under %s', HALFAPI_DOT_FILE)
logger.info('Cannot find a configuration file under %s', HALFAPI_ETC_FILE)
try:
with open(HALFAPI_DOT_FILE, 'r'):
HALFAPI_CONFIG_FILES.append(HALFAPI_DOT_FILE)
except FileNotFoundError:
logger.error('Cannot find a configuration file under %s', HALFAPI_DOT_FILE)
logger.info('Cannot find a configuration file under %s', HALFAPI_DOT_FILE)
ENVIRONMENT = {}
# Load environment variables allowed in configuration
if 'HALFAPI_DRYRUN' in os.environ:
ENVIRONMENT['dryrun'] = True
if 'HALFAPI_PROD' in os.environ:
ENVIRONMENT['production'] = bool(os.environ.get('HALFAPI_PROD'))
if 'HALFAPI_LOGLEVEL' in os.environ:
ENVIRONMENT['loglevel'] = os.environ.get('HALFAPI_LOGLEVEL').lower()
if 'HALFAPI_SECRET' in os.environ:
ENVIRONMENT['secret'] = os.environ.get('HALFAPI_SECRET')
if 'HALFAPI_HOST' in os.environ:
ENVIRONMENT['host'] = os.environ.get('HALFAPI_HOST')
if 'HALFAPI_PORT' in os.environ:
ENVIRONMENT['port'] = int(os.environ.get('HALFAPI_PORT'))
def read_config(filenames=HALFAPI_CONFIG_FILES):
"""
The highest index in "filenames" are the highest priorty
@ -84,18 +137,22 @@ def read_config(filenames=HALFAPI_CONFIG_FILES):
logger.info('Reading config files %s', filenames)
for CONF_FILE in filenames:
if os.path.isfile(CONF_FILE):
d_res.update( toml.load(CONF_FILE) )
conf_dict = toml.load(CONF_FILE)
d_res.update(conf_dict)
logger.info('Read config files (result) %s', d_res)
return { **d_res.get('project', {}), 'domain': d_res.get('domain', {}) }
CONFIG = read_config()
CONFIG.update(**ENVIRONMENT)
PROJECT_NAME = CONFIG.get('project_name',
environ.get('HALFAPI_PROJECT_NAME', os.getcwd().split('/')[-1]))
os.environ.get('HALFAPI_PROJECT_NAME', DEFAULT_CONF['PROJECT_NAME']))
if environ.get('HALFAPI_DOMAIN_NAME'):
DOMAIN_NAME = environ.get('HALFAPI_DOMAIN_NAME')
if os.environ.get('HALFAPI_DOMAIN_NAME'):
# Force enabled domain by environment variable
DOMAIN_NAME = os.environ.get('HALFAPI_DOMAIN_NAME')
if 'domain' in CONFIG and DOMAIN_NAME in CONFIG['domain'] \
and 'config' in CONFIG['domain'][DOMAIN_NAME]:
@ -113,53 +170,36 @@ if environ.get('HALFAPI_DOMAIN_NAME'):
CONFIG['domain'][DOMAIN_NAME]['config'] = domain_config
if environ.get('HALFAPI_DOMAIN_MODULE'):
dom_module = environ.get('HALFAPI_DOMAIN_MODULE')
if os.environ.get('HALFAPI_DOMAIN_MODULE'):
# Specify the pythonpath to import the specified domain (defaults to global)
dom_module = os.environ.get('HALFAPI_DOMAIN_MODULE')
CONFIG['domain'][DOMAIN_NAME]['module'] = dom_module
if len(CONFIG.get('domain', {}).keys()) == 0:
logger.info('No domains')
# Bind
HOST = CONFIG.get('host',
environ.get('HALFAPI_HOST', '127.0.0.1'))
PORT = int(CONFIG.get(
'port',
environ.get('HALFAPI_PORT', '3000')))
# Secret
SECRET = CONFIG.get(
'secret',
environ.get('HALFAPI_SECRET'))
if not SECRET:
if 'secret' not in CONFIG:
# TODO: Create a temporary secret
_, SECRET = tempfile.mkstemp()
with open(SECRET, 'w') as secret_file:
CONFIG['secret'] = DEFAULT_CONF['SECRET']
with open(CONFIG['secret'], 'w') as secret_file:
secret_file.write(str(uuid.uuid4()))
try:
with open(SECRET, 'r') as secret_file:
CONFIG['secret'] = SECRET.strip()
with open(CONFIG['secret'], 'r') as secret_file:
CONFIG['secret'] = CONFIG['secret'].strip()
except FileNotFoundError as exc:
logger.info('Running without secret file: %s', SECRET or 'no file specified')
logger.warning('Running without secret file: %s', CONFIG['secret'] or 'no file specified')
PRODUCTION = bool(CONFIG.get(
'production',
environ.get('HALFAPI_PROD', True)))
CONFIG.setdefault('project_name', DEFAULT_CONF['PROJECT_NAME'])
CONFIG.setdefault('production', DEFAULT_CONF['PRODUCTION'])
CONFIG.setdefault('host', DEFAULT_CONF['HOST'])
CONFIG.setdefault('port', DEFAULT_CONF['PORT'])
CONFIG.setdefault('loglevel', DEFAULT_CONF['LOGLEVEL'])
CONFIG.setdefault('dryrun', DEFAULT_CONF['DRYRUN'])
LOGLEVEL = CONFIG.get(
'loglevel',
environ.get('HALFAPI_LOGLEVEL', 'info')).lower()
BASE_DIR = CONFIG.get(
'base_dir',
environ.get('HALFAPI_BASE_DIR', '.'))
CONFIG['project_name'] = PROJECT_NAME
CONFIG['production'] = PRODUCTION
CONFIG['secret'] = SECRET
CONFIG['host'] = HOST
CONFIG['port'] = PORT
CONFIG['dryrun'] = DRYRUN
# !!!TO REMOVE!!!
SECRET = CONFIG.get('SECRET')
PRODUCTION = CONFIG.get('production')
# !!!

32
halfapi/halfapi.py
View File

@ -45,15 +45,24 @@ from .half_domain import HalfDomain
from halfapi import __version__
class HalfAPI(Starlette):
def __init__(self, config,
def __init__(self,
config,
d_routes=None):
config_logging(logging.DEBUG)
# Set log level (defaults to debug)
config_logging(
getattr(logging, config.get('loglevel', 'DEBUG').upper(), 'DEBUG')
)
self.config = config
logger.debug('HalfAPI.config: %s', self.config)
SECRET = self.config.get('secret')
PRODUCTION = self.config.get('production', True)
DRYRUN = self.config.get('dryrun', False)
TIMINGMIDDLEWARE = self.config.get('timingmiddleware', False)
if DRYRUN:
logger.info('HalfAPI starting in dry-run mode')
else:
logger.info('HalfAPI starting')
self.PRODUCTION = PRODUCTION
self.SECRET = SECRET
@ -67,7 +76,7 @@ class HalfAPI(Starlette):
Mount('/halfapi', routes=list(self.halfapi_routes()))
)
logger.info('Config: %s', self.config)
logger.debug('Config: %s', self.config)
domains = {
key: elt
@ -75,7 +84,7 @@ class HalfAPI(Starlette):
if elt.get('enabled', False)
}
logger.info('Active domains: %s', domains)
logger.debug('Active domains: %s', domains)
if d_routes:
# Mount the routes from the d_routes argument - domain-less mode
@ -147,7 +156,7 @@ class HalfAPI(Starlette):
on_error=on_auth_error
)
if not PRODUCTION:
if not PRODUCTION and TIMINGMIDDLEWARE:
self.add_middleware(
TimingMiddleware,
client=HTimingClient(),
@ -297,3 +306,12 @@ class HalfAPI(Starlette):
self.mount(kwargs.get('path', name), self.__domains[name])
return self.__domains[name]
def __main__():
return HalfAPI(CONFIG).application
if __name__ == '__main__':
__main__()

11
halfapi/logging.py
View File

@ -1,8 +1,10 @@
import logging
default_level = logging.DEBUG
default_format = '%(asctime)s [%(process)d] [%(levelname)s] %(message)s'
default_datefmt = '[%Y-%m-%d %H:%M:%S %z]'
def config_logging(level=logging.INFO):
def config_logging(level=default_level, format=default_format, datefmt=default_datefmt):
# When run by 'uvicorn ...', a root handler is already
# configured and the basicConfig below does nothing.
# To get the desired formatting:
@ -12,8 +14,8 @@ def config_logging(level=logging.INFO):
# https://github.com/encode/uvicorn/issues/511
logging.basicConfig(
# match gunicorn format
format='%(asctime)s [%(process)d] [%(levelname)s] %(message)s',
datefmt='[%Y-%m-%d %H:%M:%S %z]',
format=format,
datefmt=datefmt,
level=level)
# When run by 'gunicorn -k uvicorn.workers.UvicornWorker ...',
@ -27,5 +29,4 @@ def config_logging(level=logging.INFO):
logging.getLogger('uvicorn.access').propagate = True
logging.getLogger('uvicorn.error').propagate = True
config_logging()
logger = logging.getLogger()

2
halfapi/testing/test_domain.py
View File

@ -11,6 +11,7 @@ from click.testing import CliRunner
from ..cli.cli import cli
from ..halfapi import HalfAPI
from ..half_domain import HalfDomain
from ..conf import DEFAULT_CONF
from pprint import pprint
import tempfile
@ -62,6 +63,7 @@ class TestDomain(TestCase):
self.runner = class_(mix_stderr=False)
# HTTP
# Fake default values of default configuration
self.halfapi_conf = {
'secret': 'testsecret',
'production': False,

49
tests/cli/test_cli_proj.py
View File

@ -10,6 +10,7 @@ import toml
import pytest
from click.testing import CliRunner
from configparser import ConfigParser
from halfapi.conf import DEFAULT_CONF, PROJECT_LEVEL_KEYS, DOMAIN_LEVEL_KEYS
PROJNAME = os.environ.get('PROJ','tmp_api')
@ -24,26 +25,50 @@ class TestCliProj():
def test_domain_commands(self, project_runner):
""" TODO: Test create command
"""
test_conf = {
'project': {
'port': '3010',
'loglevel': 'warning'
},
'domain': {
'dummy_domain': {
'port': 4242,
'name': 'dummy_domain',
'enabled': True
}
}
}
r = project_runner('domain')
print(r.stdout)
assert r.exit_code == 1
_, tmp_conf = tempfile.mkstemp()
with open(tmp_conf, 'w') as fh:
fh.write(
toml.dumps({
'domain': {
'dummy_domain': {
'port': 4242,
'name': 'dummy_domain',
'enabled': True
}
},
'project': {
'dryrun': True
}
})
toml.dumps(test_conf)
)
r = project_runner(f'domain dummy_domain --conftest {tmp_conf}')
assert r.exit_code == 0
r_conf = toml.loads(r.stdout)
for key, value in r_conf.items():
if key == 'domain':
continue
assert key in PROJECT_LEVEL_KEYS
if key == 'port':
assert value == test_conf['domain']['dummy_domain']['port']
elif key == 'loglevel':
assert value == test_conf['project']['loglevel']
else:
assert value == DEFAULT_CONF[key.upper()]
assert json.dumps(test_conf['domain']) == json.dumps(r_conf['domain'])
for key in test_conf['domain']['dummy_domain'].keys():
assert key in DOMAIN_LEVEL_KEYS
# Default command "run"
r = project_runner(f'domain dummy_domain --dry-run {tmp_conf}')
print(r.stdout)
assert r.exit_code == 0