Skip to content

Rework refresh_vgrid_map in order to address a potential race and missing field init #380

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: next
Choose a base branch
from
Open

Rework refresh_vgrid_map in order to address a potential race and missing field init #380

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e0ac249
Rework `refresh_vgrid_map` in order to init all vgrid entries in `VGR…
jonasbardino Nov 2, 2025
e0da484
Apply same idea to the update of `RESOURCES` and `USERS` in `refresh_…
jonasbardino Nov 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions mig/shared/vgridaccess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
Expand All @@ -20,7 +20,7 @@
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

Check warning on line 23 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

line too long (81 > 80 characters)
#
# -- END_HEADER ---
#
Expand Down Expand Up @@ -48,7 +48,7 @@
from mig.shared.resource import list_resources, real_to_anon_res_map
from mig.shared.serial import load, dump
from mig.shared.user import list_users, real_to_anon_user_map, get_user_conf
from mig.shared.vgrid import vgrid_list_vgrids, vgrid_allowed, vgrid_resources, \

Check warning on line 51 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

line too long (81 > 80 characters)
user_allowed_vgrids, vgrid_owners, vgrid_members, vgrid_settings, \
vgrid_list_subvgrids, vgrid_list_parents, res_allowed_vgrids, \
merge_vgrid_settings
Expand All @@ -72,7 +72,7 @@

last_refresh = {USERS: 0, RESOURCES: 0, VGRIDS: 0}
last_load = {USERS: 0, RESOURCES: 0, VGRIDS: 0}
last_map = {USERS: {}, RESOURCES: {}, VGRIDS: {}}

Check failure on line 75 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

Need type annotation for "last_map"  [var-annotated]


def load_entity_map(configuration, kind, do_lock, caching):
Expand Down Expand Up @@ -159,7 +159,7 @@
lock_path = os.path.join(real_base, "%s.lock" % kind)
map_helpers = {'user': load_user_map, 'resource': load_resource_map,
'vgrid': load_vgrid_map}
if not kind in map_helpers:

Check warning on line 162 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
raise ValueError("invalid kind for load map: %s" % kind)
load_map = map_helpers[kind]
# NOTE: we need exclusive lock for the entire load and update process
Expand Down Expand Up @@ -277,7 +277,7 @@
dirty += [user]
# Remove any missing users from map
missing_user = [user for user in user_map
if not user in all_users]

Check warning on line 280 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
for user in missing_user:
del user_map[user]
dirty += [user]
Expand Down Expand Up @@ -345,7 +345,7 @@
dirty += [res]
# Remove any missing resources from map
missing_res = [res for res in resource_map
if not res in all_resources]

Check warning on line 348 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
for res in missing_res:
del resource_map[res]
dirty += [res]
Expand Down Expand Up @@ -401,18 +401,18 @@
optional_conf = [SETTINGS, ]

for vgrid in all_vgrids:
# Make sure vgrid dict exists before filling it
vgrid_map[VGRIDS][vgrid] = vgrid_map[VGRIDS].get(vgrid, {})
for (field, name, list_call) in conf_read:
conf_path = os.path.join(configuration.vgrid_home, vgrid, name)
if not os.path.isfile(conf_path):
# Make sure vgrid dict exists before filling it
vgrid_map[VGRIDS][vgrid] = vgrid_map[VGRIDS].get(vgrid, {})
vgrid_map[VGRIDS][vgrid][field] = []
if vgrid != default_vgrid and field not in optional_conf:
_logger.warning('missing file: %s' %
conf_path)
dirty[VGRIDS] = dirty.get(VGRIDS, []) + [vgrid]

elif vgrid not in vgrid_map[VGRIDS] or \
elif field not in vgrid_map[VGRIDS][vgrid] or \
os.path.getmtime(conf_path) >= map_stamp:
(status, entries) = list_call(vgrid, configuration,
recursive=False)
Expand All @@ -425,9 +425,11 @@
vgrid_map[VGRIDS][vgrid] = map_entry
vgrid_map[VGRIDS][vgrid][field] = entries
dirty[VGRIDS] = dirty.get(VGRIDS, []) + [vgrid]
else:
_logger.debug("caching %s for %s in vgrid map" % (name, vgrid))
# Remove any missing vgrids from map
missing_vgrids = [vgrid for vgrid in vgrid_map[VGRIDS]
if not vgrid in all_vgrids]

Check warning on line 432 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
for vgrid in missing_vgrids:
vgrid_changes[vgrid] = vgrid_changes.get(vgrid, {})
map_entry = vgrid_map[VGRIDS].get(vgrid, {})
Expand All @@ -449,7 +451,8 @@
conf_path = os.path.join(configuration.resource_home, res, "config")
if not os.path.isfile(conf_path):
continue
if os.path.getmtime(conf_path) >= map_stamp:
if res not in vgrid_map[RESOURCES] or \
os.path.getmtime(conf_path) >= map_stamp:
# Read maps of exe name to vgrid list and of store name to vgrid
# list. Save them separately to be able to distinguish them in
# exe / store access and visibility
Expand Down Expand Up @@ -488,7 +491,7 @@
dirty[RESOURCES] = dirty.get(RESOURCES, []) + [res]
# Remove any missing resources from map
missing_res = [res for res in vgrid_map[RESOURCES]
if not res in all_resources]

Check warning on line 494 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
for res in missing_res:
del vgrid_map[RESOURCES][res]
dirty[RESOURCES] = dirty.get(RESOURCES, []) + [res]
Expand Down Expand Up @@ -560,7 +563,7 @@
else:
conf_mtime = -1
user_conf = {}
if conf_mtime >= map_stamp:
if user not in vgrid_map[USERS] or conf_mtime >= map_stamp:
vgrid_map[USERS][user] = user_conf
vgrid_map[USERS][user][ASSIGN] = vgrid_map[USERS][user].get(ASSIGN,
[])
Expand All @@ -573,7 +576,7 @@
dirty[USERS] = dirty.get(USERS, []) + [user]
# Remove any missing users from map
missing_user = [user for user in vgrid_map[USERS]
if not user in all_users]

Check warning on line 579 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
for user in missing_user:
del vgrid_map[USERS][user]
dirty[USERS] = dirty.get(USERS, []) + [user]
Expand Down Expand Up @@ -615,7 +618,7 @@
return vgrid_map


def force_update_user_map(configuration, clean=False):

Check failure on line 621 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused function 'force_update_user_map' (60% confidence)
"""Refresh user map and update map cache"""
map_stamp = load_stamp = time.time()
user_map = refresh_user_map(configuration, clean=clean)
Expand All @@ -626,7 +629,7 @@
return user_map


def force_update_resource_map(configuration, clean=False):

Check failure on line 632 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused function 'force_update_resource_map' (60% confidence)
"""Refresh resources map and update map cache"""
map_stamp = load_stamp = time.time()
resource_map = refresh_resource_map(configuration, clean=clean)
Expand All @@ -637,7 +640,7 @@
return resource_map


def force_update_vgrid_map(configuration, clean=False):

Check failure on line 643 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused function 'force_update_vgrid_map' (60% confidence)
"""Refresh vgrid map and update map cache"""
map_stamp = load_stamp = time.time()
vgrid_map = refresh_vgrid_map(configuration, clean=clean)
Expand Down Expand Up @@ -668,7 +671,7 @@
for parent_name in parent_vgrid_list:
parent_vgrid = inherit_map[VGRIDS][parent_name]
for field in (OWNERS, MEMBERS, RESOURCES):
vgrid[field] += [i for i in parent_vgrid[field] if not i in

Check warning on line 674 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
vgrid[field]]
settings_list.append(dict(parent_vgrid.get(SETTINGS, [])))
settings_list.append(dict(vgrid.get(SETTINGS, [])))
Expand Down Expand Up @@ -708,7 +711,7 @@
}
}

if not key in map_helpers:

Check warning on line 714 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

test for membership should be 'not in'
raise ValueError("invalid entity last_X key: %s" % key)
helpers = map_helpers[key]
name = helpers['name']
Expand Down Expand Up @@ -843,7 +846,7 @@
return vgrid_access


def check_vgrid_access(configuration, client_id, vgrid_name, recursive=True,

Check failure on line 849 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused function 'check_vgrid_access' (60% confidence)
caching=False):
"""Inspect the vgrid map and check if client_id is either a member or
owner of vgrid_name.
Expand All @@ -853,7 +856,6 @@
the vgrid module and should replace that one everywhere that only vgrid map
(cached) lookups are needed.
"""
vgrid_access = [default_vgrid]
vgrid_map = get_vgrid_map(configuration, recursive, caching)
vgrid_entry = vgrid_map.get(VGRIDS, {}).get(
vgrid_name, {OWNERS: [], MEMBERS: []})
Expand Down Expand Up @@ -1170,7 +1172,7 @@
return resources


def get_re_provider_map(configuration, caching=False):

Check failure on line 1175 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused function 'get_re_provider_map' (60% confidence)
"""Find providers for all runtime environments in one go.

Resources are anonymized unless explicitly configured otherwise.
Expand Down Expand Up @@ -1258,10 +1260,10 @@
client_dir = real_path.replace(configuration.user_home, '')
client_dir = client_dir.lstrip(os.sep).split(os.sep, 1)[0]
client_id = client_dir_id(client_dir)
user_base = os.path.join(configuration.user_home, client_dir)

Check failure on line 1263 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused variable 'user_base' (60% confidence)
else:
client_dir = client_id_dir(client_id)
user_base = os.path.join(configuration.user_home, client_dir)

Check failure on line 1266 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

unused variable 'user_base' (60% confidence)

# Caching - build cache on first use and reuse from there
if isinstance(cache, dict):
Expand Down Expand Up @@ -1446,7 +1448,7 @@
print("inherited vgrid map vgrids: %s" % inherited_map[VGRIDS])

print("= testing in vgrid specials =")
cache = {}

Check failure on line 1451 in mig/shared/vgridaccess.py

View workflow job for this annotation

GitHub Actions / Style check python and annotate 

Need type annotation for "cache" (hint: "cache: Dict[<type>, <type>] = ...")  [var-annotated]
client_dir = client_id_dir(user_id)
print("Testing with user home %r" % client_dir)
for path in ['.', './', 'welcome.txt', 'test', 'test/test'
Expand Down