492 lines
21 KiB
Python
492 lines
21 KiB
Python
import logging
|
|
import re
|
|
from collections import defaultdict
|
|
from concurrent.futures import as_completed, ThreadPoolExecutor
|
|
from markupsafe import Markup
|
|
from socket import IPPROTO_TCP, gaierror, getaddrinfo
|
|
from urllib.parse import urlparse
|
|
|
|
from odoo import api, Command, fields, models
|
|
from odoo.fields import Domain
|
|
from odoo.addons.databases.models.project_project import get_database_api_keys
|
|
|
|
from ..api import ApiError, OdooComApi, OdooDatabaseApi, _humanize_version
|
|
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DatabasesSynchronizationWizard(models.TransientModel):
|
|
_name = 'databases.synchronization.wizard'
|
|
_description = 'Database Synchronization Wizard'
|
|
|
|
error_message = fields.Char(default='')
|
|
summary_message = fields.Char(compute='_compute_summary_message')
|
|
database_ids = fields.Many2many(
|
|
comodel_name='project.project',
|
|
domain=[('database_hosting', 'not in', (False, 'other'))],
|
|
)
|
|
created_database_ids = fields.Many2many(
|
|
comodel_name='project.project',
|
|
relation='databases_synchronization_wizard_created_databases_rel',
|
|
domain=[('database_hosting', 'not in', (False, 'other'))],
|
|
)
|
|
property_definition = fields.Json()
|
|
fetched_values = fields.Json()
|
|
new_properties = fields.Json(string="New KPIs to add:")
|
|
notify_user = fields.Boolean(default=False, required=True,
|
|
help="Whether the user should be notified once the cron has finished synchronizing the databases")
|
|
|
|
@api.depends('database_ids', 'created_database_ids')
|
|
def _compute_summary_message(self):
|
|
for record in self:
|
|
record.summary_message = self.env._(
|
|
"%(nb_new_dbs)s new databases, %(nb_updated_dbs)s updated.",
|
|
nb_new_dbs=len(record.created_database_ids),
|
|
nb_updated_dbs=len(record.database_ids)
|
|
)
|
|
|
|
def _open(self):
|
|
# The wizard was used to select which property fields were to be added.
|
|
# As they are now added inconditionally, is is bypassed and we just reload the client view.
|
|
# We only show the error messages if any.
|
|
# The wizard will be cleaned in saas~19.2, but we keep it in 19.0 and saas~19.1 to avoid changing the model in a stable version.
|
|
action = {
|
|
"type": "ir.actions.client",
|
|
"tag": "soft_reload",
|
|
}
|
|
if self.error_message:
|
|
action = {
|
|
'type': 'ir.actions.act_window',
|
|
'name': self.env._("Synchronization of %d databases", len(self.database_ids)),
|
|
'res_model': self._name,
|
|
'res_id': self.id,
|
|
'view_mode': 'form',
|
|
'target': 'new',
|
|
'views': [[False, 'form']],
|
|
}
|
|
return action
|
|
|
|
def _can_update_from_odoo_com(self):
|
|
ICP = self.env['ir.config_parameter'].sudo()
|
|
apikey = ICP.get_param('databases.odoocom_apikey')
|
|
return self.env.user.has_group('databases.group_databases_manager') and apikey
|
|
|
|
def _do_update_from_odoocom(self):
|
|
"""
|
|
- Get db info from odoo.com
|
|
- Create missing dbs
|
|
- Switch unconfigured on premise/paas to saas
|
|
- Update saas dbs
|
|
- Report configured on premise/paas dbs detected in saas as not updated
|
|
"""
|
|
self.check_access("write")
|
|
|
|
# Get db info from Odoo
|
|
ICP = self.env['ir.config_parameter'].sudo()
|
|
apihost = ICP.get_param('databases.odoocom_apihost', 'https://www.odoo.com')
|
|
apidb = ICP.get_param('databases.odoocom_apidb', 'openerp')
|
|
apikey = ICP.get_param('databases.odoocom_apikey')
|
|
|
|
if not apikey:
|
|
return
|
|
|
|
odoocom_api = OdooComApi(apihost, apidb, apikey)
|
|
try:
|
|
databases = [db for db in odoocom_api.list_databases() if db['name'] != self.env.cr.dbname]
|
|
except ApiError as e:
|
|
self.error_message += self.env._(
|
|
"Error while listing databases from %(dbname)s: %(message)s\n",
|
|
dbname=apihost,
|
|
message=e.args[0],
|
|
)
|
|
return
|
|
|
|
# Create missing db
|
|
Project = self.env['project.project']
|
|
databases_by_url = {db['url']: db for db in databases} # TODO: handle duplicate urls correctly
|
|
existing_databases = Project.with_context(active_test=False).search([
|
|
('database_hosting', '!=', False),
|
|
('database_url', 'in', list(databases_by_url)),
|
|
])
|
|
existing_urls = set(existing_databases.mapped('database_url'))
|
|
template = Project._database_get_project_template()
|
|
for db in databases:
|
|
if db['url'] not in existing_urls:
|
|
values = {
|
|
'name': db['name'],
|
|
'database_name': db['name'],
|
|
'database_hosting': 'saas',
|
|
'database_url': db['url'],
|
|
'database_api_login': db['login'],
|
|
}
|
|
if template:
|
|
self.created_database_ids |= template.action_create_from_template(values)
|
|
else:
|
|
self.created_database_ids |= Project.create(values)
|
|
|
|
# Update saas dbs
|
|
saas_domain = [
|
|
('database_hosting', '=', 'saas'),
|
|
('database_url', 'in', list(databases_by_url)),
|
|
]
|
|
# Switch unconfigured hosted db to saas
|
|
hosting_should_be_saas = [
|
|
('database_hosting', 'in', ('paas', 'premise', 'other')),
|
|
('database_url', 'in', list(databases_by_url)),
|
|
('database_api_login', '=', False),
|
|
('database_api_key', '=', False),
|
|
]
|
|
dbs_to_update = Project.with_context(active_test=False).search(Domain.OR([
|
|
saas_domain,
|
|
hosting_should_be_saas,
|
|
]))
|
|
for db in dbs_to_update.try_lock_for_update(): # don't wait for db already updating
|
|
db_info = databases_by_url[db.database_url]
|
|
vals = {
|
|
'database_hosting': 'saas',
|
|
'database_name': db_info['name'],
|
|
'database_api_login': db_info['login'],
|
|
}
|
|
if version := _humanize_version(db_info['version']):
|
|
vals['database_version'] = version
|
|
db.write(vals)
|
|
|
|
self.database_ids |= self.created_database_ids
|
|
|
|
saas_databases = existing_databases | self.created_database_ids | dbs_to_update
|
|
if not saas_databases:
|
|
return
|
|
|
|
erroneous_dbs = []
|
|
unreachable_tag_id = self._get_unreachable_tag_id()
|
|
odoocom_apikey = self.env['ir.config_parameter'].sudo().get_param('databases.odoocom_apikey', '')
|
|
database_api_keys = get_database_api_keys(saas_databases, fallback=False)
|
|
dbs_awaiting_api_key = saas_databases.filtered(lambda db: (
|
|
db.active
|
|
and not database_api_keys.get(db.id)
|
|
# Avoid wasting 15s: unreachable dbs will most likely stay unreachable.
|
|
# If it becomes reachable, we will try to create a key during the next update.
|
|
and unreachable_tag_id not in db.tag_ids.ids
|
|
))
|
|
for db in dbs_awaiting_api_key.try_lock_for_update():
|
|
database_api_key = database_api_keys.get(db.id, odoocom_apikey)
|
|
args = [db.database_url, db.database_name, db.database_api_login, database_api_key]
|
|
if not all(args):
|
|
continue
|
|
db_api = OdooDatabaseApi(*args)
|
|
key_name = ICP.get_param('databases.remote_api_key_description', f'Access for {self.env.company.name}')
|
|
try:
|
|
db.database_api_key = db_api.create_api_key(key_name)
|
|
except ApiError:
|
|
_logger.exception("Error while creating an API key on %r", db.database_name)
|
|
erroneous_dbs.append(db.database_name)
|
|
if erroneous_dbs:
|
|
dbnames = ''.join(f'- {dbname}\n' for dbname in erroneous_dbs[:10])
|
|
if len(erroneous_dbs) > 10:
|
|
dbnames += self.env._('- … and %(nb_dbs)d more!\n', nb_dbs=len(erroneous_dbs) - 10)
|
|
self.error_message += self.env._(
|
|
"Error while creating an API key on %(nb_dbs)d databases:\n",
|
|
nb_dbs=len(erroneous_dbs),
|
|
) + dbnames
|
|
|
|
def _do_synchronize(self):
|
|
self.check_access("write")
|
|
|
|
if not self.database_ids:
|
|
return self._open()
|
|
|
|
try:
|
|
immediate_sync_limit = int(self.env['ir.config_parameter'].sudo().get_param('databases.immediate_sync_limit', 300))
|
|
except ValueError:
|
|
immediate_sync_limit = 300
|
|
|
|
dbs_to_process = self.database_ids[:immediate_sync_limit]
|
|
dbs_to_postpone = self.database_ids[immediate_sync_limit:]
|
|
|
|
# property.base.definition objects are readable only by role Settings
|
|
database_kpi_base_definition_id = self.database_ids.sudo().database_kpi_base_definition_id
|
|
database_kpi_base_definition_id.ensure_one()
|
|
self.property_definition = database_kpi_base_definition_id.properties_definition
|
|
|
|
unreachable_tag_id = self._get_unreachable_tag_id()
|
|
|
|
def _mark_db_unreachable(db, errors):
|
|
db = db.sudo()
|
|
db.write({'tag_ids': [Command.link(unreachable_tag_id)]})
|
|
db.message_post(body=Markup('<br>').join(errors))
|
|
|
|
database_api_keys = get_database_api_keys(dbs_to_process)
|
|
db_by_url = {db.database_url: db for db in dbs_to_process}
|
|
db_apis = []
|
|
nb_failed_dbs = 0
|
|
for db in dbs_to_process:
|
|
database_api_key = database_api_keys.get(db.id, '')
|
|
args = [db.database_url, db.database_name, db.database_api_login, database_api_key]
|
|
if not all(args):
|
|
nb_failed_dbs += 1
|
|
_mark_db_unreachable(db, [self.env._(
|
|
"Error while connecting to %(url)s: We are missing the database name, the api login or the api key\n",
|
|
url=db.database_url,
|
|
)])
|
|
continue
|
|
db_apis.append(OdooDatabaseApi(*args))
|
|
|
|
# avoid flooding a server with tons of parallel requests in case several dbs are hosted on the same server.
|
|
db_apis_per_ip, errors_by_url = _group_by_ips(db_apis, self.env._)
|
|
nb_failed_dbs += len(errors_by_url)
|
|
for db_url, errors in errors_by_url.items():
|
|
_mark_db_unreachable(db_by_url[db_url], errors)
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
db_apis_by_future = {executor.submit(_fetch_database_info, db_apis, self.env._): (db_apis, ip)
|
|
for ip, db_apis in db_apis_per_ip.items()}
|
|
for future in as_completed(db_apis_by_future):
|
|
try:
|
|
results, errors_by_url = future.result()
|
|
except Exception as e: # noqa: BLE001
|
|
# Meaningful per-database errors are handled in `errors_by_url`.
|
|
# Any exception reaching here is unexpected/system-level,
|
|
# should not stop the synchronization and should be reported in the wizard.
|
|
db_apis, ip = db_apis_by_future[future]
|
|
host_names = ', '.join(db_api.host for db_api in db_apis)
|
|
_logger.warning('Error while fetching information from %s on %s: %s', host_names, ip, e)
|
|
self.error_message += self.env._("Error while fetching information from %(host_names)s on %(ip)s: %(message)s\n",
|
|
host_names=host_names, ip=ip, message=str(e))
|
|
continue
|
|
|
|
for db_url, values in results.items():
|
|
db = db_by_url[db_url]
|
|
values.setdefault('tag_ids', []).append(Command.unlink(unreachable_tag_id))
|
|
kpi_summary = values.pop('kpi_summary', None)
|
|
if kpi_summary is not None:
|
|
self._write_kpis(db, kpi_summary)
|
|
users = values.pop('users', None)
|
|
if users is not None:
|
|
self._write_users(db, users)
|
|
db.sudo().write(values) # databases_user should be able to synchronize, values keys are hardcoded
|
|
|
|
nb_failed_dbs += len(errors_by_url)
|
|
for db_url, errors in errors_by_url.items():
|
|
_mark_db_unreachable(db_by_url[db_url], errors)
|
|
|
|
self.action_add_metrics_to_dashboard()
|
|
self.write({'new_properties': {}})
|
|
|
|
if nb_failed_dbs > 0:
|
|
self.error_message += self.env._("%(nb_failed_dbs)s databases could not be synchronized.\n",
|
|
nb_failed_dbs=nb_failed_dbs)
|
|
|
|
if dbs_to_postpone:
|
|
dbs_to_postpone.sudo().database_last_synchro = False
|
|
self.notify_user = True
|
|
self.env.ref('databases.ir_cron_synchronize_databases')._trigger()
|
|
return {
|
|
'type': 'ir.actions.client',
|
|
'tag': 'display_notification',
|
|
'params': {
|
|
'title': self.env._("Synchronization of %d databases", len(self.database_ids)),
|
|
'message': self.env._("Database synchronization is running in the background. "
|
|
"You will be notified upon completion."),
|
|
'type': 'info',
|
|
'sticky': False,
|
|
'next': self._open(),
|
|
},
|
|
}
|
|
return self._open()
|
|
|
|
def _write_users(self, db, users):
|
|
# sudo in this method are there to allow databases_user to synchronize
|
|
users = {u['login']: u for u in users}
|
|
existing_users = db.database_user_ids
|
|
common_users = existing_users.filtered(lambda u: u.login in users)
|
|
missing_logins = set(users) - set(existing_users.mapped('login'))
|
|
users_to_delete = existing_users - common_users
|
|
|
|
users_to_delete.sudo().unlink()
|
|
existing_users.sudo().create([
|
|
{
|
|
'project_id': db.id,
|
|
'login': user['login'],
|
|
'name': user['name'],
|
|
'latest_authentication': user['login_date']
|
|
}
|
|
for user in users.values() if user['login'] in missing_logins
|
|
])
|
|
for user in common_users:
|
|
user_data = users[user.login]
|
|
user.sudo().write({
|
|
'name': user_data['name'],
|
|
'latest_authentication': user_data['login_date'],
|
|
})
|
|
|
|
def _write_kpis(self, db, kpi_summary):
|
|
property_definition = {x['name']: x for x in self.property_definition or []}
|
|
kpi_properties = {}
|
|
for kpi in kpi_summary:
|
|
if kpi['id'] == 'documents.inbox':
|
|
if db.database_fetch_documents:
|
|
db.sudo().database_nb_documents = kpi['value'] # databases_user should be able to synchronize, values keys are hardcoded
|
|
continue
|
|
|
|
if kpi['id'].startswith('account_move_type.') and not db.database_fetch_draft_entries:
|
|
continue
|
|
|
|
if kpi['id'].startswith('account_return.') and not db.database_fetch_tax_returns:
|
|
continue
|
|
|
|
# property names must match odoo.orm.utils.regex_alphanumeric
|
|
kpi_id = re.sub(r'[^a-z0-9]', '_', kpi['id'].lower())
|
|
if kpi_id not in property_definition:
|
|
property_definition[kpi_id] = {
|
|
'name': kpi_id,
|
|
'string': kpi['name'],
|
|
}
|
|
if kpi['type'] == 'integer':
|
|
property_definition[kpi_id].update({
|
|
'type': 'integer',
|
|
'default': 0,
|
|
})
|
|
elif kpi['type'] == 'kyc_status':
|
|
property_definition[kpi_id].update({
|
|
'type': 'selection',
|
|
'default': False,
|
|
'selection': [
|
|
['not_done', '🔴'],
|
|
['incomplete', '🟡'],
|
|
['done', '🟢'],
|
|
],
|
|
})
|
|
elif kpi['type'] == 'return_status':
|
|
property_definition[kpi_id].update({
|
|
'type': 'selection',
|
|
'default': False,
|
|
'selection': [
|
|
['late', '🔴'],
|
|
['longterm', '⚪'],
|
|
['to_do', '🟡'],
|
|
['to_submit', '🟢⚪'],
|
|
['done', '🟢'],
|
|
],
|
|
})
|
|
|
|
kpi_properties[kpi_id] = kpi['value']
|
|
|
|
# property.base.definition objects are readable only by role Settings
|
|
database_kpi_base_definition_id = self.database_ids.sudo().database_kpi_base_definition_id
|
|
database_kpi_base_definition_id.ensure_one()
|
|
previous_kpi_ids = {x['name'] for x in database_kpi_base_definition_id.properties_definition}
|
|
self.write({
|
|
'new_properties': {kpi_id: {'label': kpi['string'], 'checked': True}
|
|
for kpi_id, kpi in property_definition.items() if kpi_id not in previous_kpi_ids},
|
|
'property_definition': list(property_definition.values()),
|
|
})
|
|
|
|
self.fetched_values = {
|
|
**(self.fetched_values or {}),
|
|
db.id: kpi_properties,
|
|
}
|
|
|
|
def action_add_metrics_to_dashboard(self):
|
|
if not self.database_ids:
|
|
return
|
|
|
|
# property.base.definition objects are readable only by role Settings
|
|
database_kpi_base_definition_id = self.database_ids.sudo().database_kpi_base_definition_id
|
|
database_kpi_base_definition_id.ensure_one()
|
|
existing_keys = {x['name'] for x in database_kpi_base_definition_id.properties_definition}
|
|
properties_definition = [x for x in self.property_definition or []
|
|
if x['name'] in existing_keys or self.new_properties[x['name']]['checked']]
|
|
|
|
# sort the properties definition on the prefix of the name first, then on the string
|
|
prefix_order = ('account_journal_type', 'account_move_type', 'account_return')
|
|
properties_definition = sorted(properties_definition, key=lambda x: (
|
|
# index of the first matching prefix, defaulting to the end
|
|
next((i for i, prefix in enumerate(prefix_order) if x['name'].startswith(prefix)), len(prefix_order)),
|
|
# then the displayed name
|
|
x['string'],
|
|
))
|
|
|
|
database_kpi_base_definition_id.write({
|
|
'properties_definition': properties_definition,
|
|
})
|
|
|
|
for db in self.database_ids:
|
|
fetched_values = (self.fetched_values or {}).get(str(db.id)) # JSON keys are strings
|
|
db.sudo().write({'database_kpi_properties': fetched_values})
|
|
|
|
action = {
|
|
"type": "ir.actions.client",
|
|
"tag": "soft_reload",
|
|
}
|
|
return action
|
|
|
|
def _get_unreachable_tag_id(self):
|
|
# TODO in saas~19.3: add XML data definition and replace calls by `self.env.ref('databases.project_tag_db_unreachable')`
|
|
IMD = self.env['ir.model.data']
|
|
xml_id = 'databases.project_tag_db_unreachable'
|
|
unreachable_tag_id = IMD._xmlid_to_res_id(xml_id, raise_if_not_found=False)
|
|
if unreachable_tag_id:
|
|
return unreachable_tag_id
|
|
|
|
unreachable_tag = self.env['project.tags'].create({
|
|
'name': self.env._('Unreachable'),
|
|
'color': 1,
|
|
})
|
|
IMD._update_xmlids([{
|
|
'xml_id': xml_id,
|
|
'record': unreachable_tag,
|
|
}])
|
|
return unreachable_tag.id
|
|
|
|
|
|
def _group_by_ips(db_apis, translate):
|
|
groups = defaultdict(list)
|
|
errors_by_url = defaultdict(list)
|
|
for db_api in db_apis:
|
|
hostname = urlparse(db_api.host).hostname
|
|
try:
|
|
addrinfo = getaddrinfo(hostname, None, proto=IPPROTO_TCP)
|
|
except gaierror as e:
|
|
errors_by_url[db_api.host].append(translate("Error while resolving %(url)s: %(exception)s\n", url=db_api.host, exception=str(e)))
|
|
continue
|
|
if not addrinfo:
|
|
errors_by_url[db_api.host].append(translate("Error while resolving %(url)s: found no IP\n", url=db_api.host))
|
|
continue
|
|
|
|
_family, _type, _proto, _canonname, (ip, _port, *_) = sorted(addrinfo)[0] # sort IPv4 before IPv6
|
|
groups[ip].append(db_api)
|
|
return groups, errors_by_url
|
|
|
|
|
|
def _fetch_database_info(db_apis, translate):
|
|
results = {}
|
|
errors_by_url = defaultdict(list)
|
|
for db_api in db_apis:
|
|
results[db_api.host] = {
|
|
'database_last_synchro': fields.Datetime.now(),
|
|
'database_nb_synchro_errors': 0,
|
|
}
|
|
if version := OdooDatabaseApi.fetch_version(db_api.host):
|
|
results[db_api.host]['database_version'] = version
|
|
|
|
try:
|
|
results[db_api.host]['users'] = db_api.list_internal_users()
|
|
except ApiError as e:
|
|
errors_by_url[db_api.host].append(translate(
|
|
"Error while getting users from %(dbname)s: %(message)s\n",
|
|
dbname=db_api.database,
|
|
message=e.args[0],
|
|
))
|
|
|
|
try:
|
|
results[db_api.host]['kpi_summary'] = db_api.get_kpi_summary()
|
|
except ApiError as e:
|
|
errors_by_url[db_api.host].append(translate(
|
|
"Error while getting KPIs from %(dbname)s: %(message)s\n",
|
|
dbname=db_api.database,
|
|
message=e.args[0],
|
|
))
|
|
|
|
return results, errors_by_url
|