middlewared can crash on service startup on 13

__plugins_setup() calls ad_extend() which eventually through the deep call stack calls pool.query() which calls pool_extend() which ends up calling label_to_dev_disk_cache() which causes this traceback

[2022/01/27 07:40:50] (ERROR) asyncio.default_exception_handler():1738 - Task exception was never retrieved
future: <Task finished name='Task-1' coro=<Middleware.__initialize() done, defined at /usr/local/lib/python3.9/site-packages/middlewared/main.py:1520> exception=AttributeError("'NoneType' object has no attribute 'iterfind'")>
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1566, in __initialize
await self.__plugins_setup(setup_funcs)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 913, in __plugins_setup
await call
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/cache.py", line 225, in setup
await middleware.call('dscache.initialize')
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1224, in _call
return await self.run_in_executor(prepared_call.executor, methodobj, *prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1126, in run_in_executor
return await loop.run_in_executor(exc, functools.partial(method, *args, **kwargs))
File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/cache.py", line 139, in initialize
if (self.middleware.call_sync(f'{ds[0]}.config'))['enable']:
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1272, in call_sync
return self.run_coroutine(methodobj(*prepared_call.args))
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1312, in run_coroutine
return fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 438, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 373, in config
return await self._get_or_insert(self._config.datastore, options)
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 385, in _get_or_insert
return await self.middleware.call('datastore.config', datastore, options)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 186, in config
return await self.query(name, [], options)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 164, in query
result = await self._queryset_serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 214, in _queryset_serialize
result.append(await self._serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 232, in _serialize
data = await self.middleware.call(extend, data)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/activedirectory.py", line 281, in ad_extend
smb = await self.middleware.call('smb.config')
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 404, in config
return await self._get_or_insert(
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 385, in _get_or_insert
return await self.middleware.call('datastore.config', datastore, options)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 186, in config
return await self.query(name, [], options)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 164, in query
result = await self._queryset_serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 214, in _queryset_serialize
result.append(await self._serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 232, in _serialize
data = await self.middleware.call(extend, data)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/smb.py", line 226, in smb_extend
ha_mode = SMBHAMODE[(await self.get_smb_ha_mode())]
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/smb.py", line 495, in get_smb_ha_mode
system_dataset = await self.middleware.call('systemdataset.config')
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 373, in config
return await self._get_or_insert(self._config.datastore, options)
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 385, in _get_or_insert
return await self.middleware.call('datastore.config', datastore, options)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 186, in config
return await self.query(name, [], options)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 164, in query
result = await self._queryset_serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 214, in _queryset_serialize
result.append(await self._serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 232, in _serialize
data = await self.middleware.call(extend, data)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/sysdataset.py", line 44, in config_extend
pool = await self.middleware.call('pool.query', [('name', '=', config['pool'])])
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/service.py", line 471, in query
result = await self.middleware.call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1213, in _call
return await methodobj(*prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/schema.py", line 975, in nf
return await f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 164, in query
result = await self._queryset_serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 214, in _queryset_serialize
result.append(await self._serialize(
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/datastore/read.py", line 232, in _serialize
data = await self.middleware.call(extend, data)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1256, in call
return await self._call(
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1224, in _call
return await self.run_in_executor(prepared_call.executor, methodobj, *prepared_call.args)
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1126, in run_in_executor
return await loop.run_in_executor(exc, functools.partial(method, *args, **kwargs))
File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/pool.py", line 474, in pool_extend
'topology': self.transform_topology(zpool['groups']),
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/pool.py", line 414, in transform_topology
options['label_to_dev_disk_cache'] = self.middleware.call_sync('disk.label_to_dev_disk_cache')
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1280, in call_sync
return self.run_coroutine(self.run_in_executor(prepared_call.executor, methodobj, *prepared_call.args))
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1312, in run_coroutine
return fut.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 438, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/middlewared/main.py", line 1126, in run_in_executor
return await loop.run_in_executor(exc, functools.partial(method, *args, **kwargs))
File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.9/site-packages/middlewared/plugins/disk_/disk_info_freebsd.py", line 87, in label_to_dev_disk_cache
for label in xml.iterfind('.//class[name="LABEL"]/geom'):
AttributeError: 'NoneType' object has no attribute 'iterfind'

The solution is simple. Change the plugin setup order to load the "geom" plugin after "pwenc".