From f15cd02e2d10d800425d11417b7016b28ffca56d Mon Sep 17 00:00:00 2001 From: Daniel Brosche Date: Tue, 15 Jun 2021 11:15:48 +0200 Subject: [PATCH 1/5] initial version of the flat & flat-recursive resolver --- gitman/cli.py | 2 +- gitman/models/config.py | 122 ++++++++++++++++++++++++++++++++++++---- gitman/models/source.py | 3 + 3 files changed, 115 insertions(+), 12 deletions(-) diff --git a/gitman/cli.py b/gitman/cli.py index 25497a7..2547dc1 100644 --- a/gitman/cli.py +++ b/gitman/cli.py @@ -336,7 +336,7 @@ def _run_command(function, args, kwargs): exit_message = "Run again with '--force' to ignore script errors" except exceptions.InvalidConfig as exception: _show_error(exception) - exit_message = "Adapt config and run again" + exit_message = "Check the gitman config of the already imported repositories and the repository that raises the conflict error" finally: if exit_message: common.show(exit_message, color='message') diff --git a/gitman/models/config.py b/gitman/models/config.py index 49c5321..1fcb3f3 100644 --- a/gitman/models/config.py +++ b/gitman/models/config.py @@ -1,6 +1,7 @@ import os import sys from typing import List, Optional +import inspect import log from datafiles import datafile, field @@ -13,20 +14,44 @@ @datafile("{self.root}/{self.filename}", defaults=True, manual=True) class Config: + RESOLVER_RECURSIVE_NESTED = "recursive-nested" + RESOLVER_RECURSIVE_FLAT = "recursive-flat" + RESOLVER_FLAT = "flat" + """Specifies all dependencies for a project.""" root: Optional[str] = None filename: str = "gitman.yml" - location: str = "gitman_sources" + resolver: str = RESOLVER_RECURSIVE_NESTED sources: List[Source] = field(default_factory=list) sources_locked: List[Source] = field(default_factory=list) default_group: str = field(default_factory=str) groups: List[Group] = field(default_factory=list) + processed_sources: List[Source] = field(default_factory=list) + location_path : str = None def __post_init__(self): if self.root is None: self.root = os.getcwd() + if self.location_path is None: + self.location_path = os.path.normpath(os.path.join(self.root, self.location)) + + def _on_post_load(self): + # update location path because default location may different then loaded value + self.location_path = os.path.normpath(os.path.join(self.root, self.location)) + + # check if any of the valid resolver values is set + # if not then set RESOLVER_RECURSIVE_NESTED as default + if (self.resolver != Config.RESOLVER_RECURSIVE_NESTED and + self.resolver != Config.RESOLVER_RECURSIVE_FLAT and + self.resolver != Config.RESOLVER_FLAT): + self.resolver = Config.RESOLVER_RECURSIVE_NESTED + + for source in self.sources: + source._on_post_load() # pylint: disable=protected-access + for source in self.sources_locked: + source._on_post_load() # pylint: disable=protected-access @property def config_path(self) -> str: @@ -41,12 +66,6 @@ def log_path(self) -> str: """Get the full path to the log file.""" return os.path.normpath(os.path.join(self.location_path, "gitman.log")) - @property - def location_path(self) -> str: - """Get the full path to the dependency storage location.""" - assert self.root - return os.path.normpath(os.path.join(self.root, self.location)) - def validate(self): """Check for conflicts between source names and group names.""" for source in self.sources: @@ -92,6 +111,39 @@ def install_dependencies( *names, sources=sources, skip_default_group=skip_default_group ) + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: + # gather flat sources and check for rev conflicts + for source in sources: + is_srcname_found = False + for processed_source in self.processed_sources: + if source.name == processed_source.name: # check if current source was already processed + is_srcname_found = True + if (source.rev != processed_source.rev or + source.repo != processed_source.repo): + error_msg = ("Repo/rev conflict encountered in" + "flat hierarchy while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, + str(processed_source), + str(source)) + raise exceptions.InvalidConfig(error_msg) + # new source name detected -> store new source name to list (cache) used to check for rev conflicts + if not is_srcname_found: # source.name != processed_source.name and source.repo != processed_source.repo: + self.processed_sources.append(source) + else: + for source in sources: + for processed_source in self.processed_sources: + if source.name == processed_source.name: # check if current source was already processed + error_msg = ("Repo/rev conflict encountered in" + "flat hierarchy while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, + str(processed_source), + str(source)) + raise exceptions.InvalidConfig(error_msg) + + self.processed_sources.append(source) + if not os.path.isdir(self.location_path): shell.mkdir(self.location_path) shell.cd(self.location_path) @@ -117,9 +169,24 @@ def install_dependencies( common.newline() count += 1 + if self.resolver == Config.RESOLVER_FLAT: + # don't process nested configs if flat resolver is active + continue + config = load_config(search=False) if config: common.indent() + + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: + # Top level preference for flat hierarchy should + # forward / propagate resolver settings + config.resolver = self.resolver + # forward / override default location -> always use root location + # to install dependencies all into the same folder + config.location_path = self.location_path + # forward processed sources list to check for global conflicts + config.processed_sources = self.processed_sources + count += config.install_dependencies( depth=None if depth is None else max(0, depth - 1), update=update and recurse, @@ -162,12 +229,27 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): if source.name in sources_filter: shell.cd(source.name) + if self.resolver == Config.RESOLVER_FLAT: + # don't process nested configs if flat resolver is active + continue + config = load_config(search=False) if config: common.indent() remaining_depth = None if depth is None else max(0, depth - 1) if remaining_depth: common.newline() + + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: + # Top level preference for flat hierarchy should + # always propagate resolver settings + config.resolver = self.resolver + # override default location -> always use root location + # to install dependencies all into the same folder + config.location_path = self.location_path + # forward processed sources list to check for global conflicts + config.processed_sources = self.processed_sources + count += config.run_scripts(depth=remaining_depth, force=force) common.dedent() @@ -187,6 +269,9 @@ def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): *names, sources=sources, skip_default_group=False ) + if not os.path.isdir(self.location_path): + raise exceptions.InvalidRepository("No dependecies resolved") + shell.cd(self.location_path) common.newline() common.indent() @@ -310,10 +395,24 @@ def _get_sources(self, *, use_locked=None): sources = self.sources extras = [] - for source in self.sources + self.sources_locked: - if source not in sources: - log.info("Source %r missing from selected section", source.name) - extras.append(source) + + all_sources = self.sources + self.sources_locked + + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: + # here is some extra work to do because all dependencies that are resolved in + # flat hierarchy are needed to resolved in a safe manner to lock them all + # in the sources_locked section + # self.processed_sources contains the complete list of all resolved sources + # only if an update process has been completed (basically this is the desired list to return) + # but this is not the case when directly a lock operation has been executed + # therefore we need to do some generic stuff here which is idependently from update process + pass + else: + for source in all_sources: + if source not in sources: + log.info("Source %r missing from selected section", + source.name) + extras.append(source) return sources + extras @@ -353,6 +452,7 @@ def load_config(start=None, *, search=True): for filename in os.listdir(path): if _valid_filename(filename): config = Config(path, filename) + config._on_post_load() # pylint: disable=protected-access config.validate() log.debug("Found config: %s", config.path) return config diff --git a/gitman/models/source.py b/gitman/models/source.py index 5f07805..72b1d83 100644 --- a/gitman/models/source.py +++ b/gitman/models/source.py @@ -37,6 +37,9 @@ def __post_init__(self): self.name = str(self.name) self.type = self.type or 'git' + def _on_post_load(self): + pass + def __repr__(self): return "".format(self) From 5eecfd239a60cfbf4b188d970f914c9e51ffa1ad Mon Sep 17 00:00:00 2001 From: Daniel Brosche Date: Thu, 24 Jun 2021 16:51:30 +0200 Subject: [PATCH 2/5] implement flat recursive lock --- gitman/models/config.py | 117 +++++++++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 36 deletions(-) diff --git a/gitman/models/config.py b/gitman/models/config.py index 1fcb3f3..fcfda84 100644 --- a/gitman/models/config.py +++ b/gitman/models/config.py @@ -9,13 +9,25 @@ from .. import common, exceptions, shell from ..decorators import preserve_cwd from .group import Group -from .source import Source - +from .source import Source, create_sym_link + +# TODO: use __post_init__ instead of _on_post_load +# TODO: create links for nested dependencies if flat recursive is used +# TODO: create relative nested symlinks +# Use Case 1 all flat recursive -> recursive all repos in a flat way +# Use Case 2 flat recursive top level -> recursive-nested bottom level: gather all repos globally and create nested links +# -> gitman -> imports +# - waf-prototyping +# - lib +# - libex +# Use Case 3 flat recursive top level -> flat bottom level: clone in a flat way but abort recursive resolution in the next stage +global_resolver = None @datafile("{self.root}/{self.filename}", defaults=True, manual=True) class Config: RESOLVER_RECURSIVE_NESTED = "recursive-nested" RESOLVER_RECURSIVE_FLAT = "recursive-flat" + RESOLVER_RECURSIVE_FLAT_NESTED_LINKS = "recursive-flat-nested-links" RESOLVER_FLAT = "flat" """Specifies all dependencies for a project.""" @@ -28,31 +40,44 @@ class Config: sources_locked: List[Source] = field(default_factory=list) default_group: str = field(default_factory=str) groups: List[Group] = field(default_factory=list) - processed_sources: List[Source] = field(default_factory=list) - location_path : str = None + def __post_init__(self): if self.root is None: self.root = os.getcwd() + + # used only internally and should not be serialized + setattr(self, 'location_path', None) + processed_sources: List[Source] = [] + setattr(self, 'processed_sources', processed_sources) + if self.location_path is None: self.location_path = os.path.normpath(os.path.join(self.root, self.location)) def _on_post_load(self): + global global_resolver + # update location path because default location may different then loaded value self.location_path = os.path.normpath(os.path.join(self.root, self.location)) # check if any of the valid resolver values is set # if not then set RESOLVER_RECURSIVE_NESTED as default if (self.resolver != Config.RESOLVER_RECURSIVE_NESTED and - self.resolver != Config.RESOLVER_RECURSIVE_FLAT and - self.resolver != Config.RESOLVER_FLAT): - self.resolver = Config.RESOLVER_RECURSIVE_NESTED + self.resolver != Config.RESOLVER_RECURSIVE_FLAT and + self.resolver != Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS and + self.resolver != Config.RESOLVER_FLAT): + msg = "Unknown resolver name \"{}\"".format(self.resolver) + raise exceptions.InvalidConfig(msg) for source in self.sources: source._on_post_load() # pylint: disable=protected-access for source in self.sources_locked: source._on_post_load() # pylint: disable=protected-access + # set global resolve if not already set + if global_resolver is None: + global_resolver = self.resolver + @property def config_path(self) -> str: """Get the full path to the config file.""" @@ -106,36 +131,43 @@ def install_dependencies( log.info("Skipped directory: %s", self.location_path) return 0 - sources = self._get_sources(use_locked=False if update else None) - sources_filter = self._get_sources_filter( - *names, sources=sources, skip_default_group=skip_default_group - ) + sources = None + sources_filter = None + + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: + sources = self._get_sources(use_locked=False if update else None, use_extra=False) + sources_filter = self._get_sources_filter( + *names, sources=sources, skip_default_group=skip_default_group + ) - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: # gather flat sources and check for rev conflicts for source in sources: - is_srcname_found = False for processed_source in self.processed_sources: if source.name == processed_source.name: # check if current source was already processed - is_srcname_found = True if (source.rev != processed_source.rev or source.repo != processed_source.repo): - error_msg = ("Repo/rev conflict encountered in" + error_msg = ("Repo/rev conflict encountered in " "flat hierarchy while updating {}\n" "Details: {} conflict with {}" ).format(self.root, str(processed_source), str(source)) raise exceptions.InvalidConfig(error_msg) - # new source name detected -> store new source name to list (cache) used to check for rev conflicts - if not is_srcname_found: # source.name != processed_source.name and source.repo != processed_source.repo: - self.processed_sources.append(source) + + # new source name detected -> store new source name to list (cache) used to check for rev conflicts + # source.name != processed_source.name and source.repo != processed_source.repo: + self.processed_sources.append(source) else: + sources = self._get_sources(use_locked=False if update else None) + sources_filter = self._get_sources_filter( + *names, sources=sources, skip_default_group=skip_default_group + ) + for source in sources: for processed_source in self.processed_sources: if source.name == processed_source.name: # check if current source was already processed - error_msg = ("Repo/rev conflict encountered in" - "flat hierarchy while updating {}\n" + error_msg = ("Repo conflict encountered " + "while updating {}\n" "Details: {} conflict with {}" ).format(self.root, str(processed_source), @@ -177,12 +209,13 @@ def install_dependencies( if config: common.indent() - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: # Top level preference for flat hierarchy should # forward / propagate resolver settings config.resolver = self.resolver # forward / override default location -> always use root location # to install dependencies all into the same folder + org_location_path = config.location_path config.location_path = self.location_path # forward processed sources list to check for global conflicts config.processed_sources = self.processed_sources @@ -197,6 +230,14 @@ def install_dependencies( skip_changes=skip_changes, skip_default_group=skip_default_group, ) + + # create nested symlinks + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: + for source in config.sources: + link_src = os.path.join(self.location_path, source.name) + link_target = os.path.join(org_location_path, source.name) + create_sym_link(link_src, link_target, True) + common.dedent() shell.cd(self.location_path, _show=False) @@ -240,7 +281,7 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): if remaining_depth: common.newline() - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: # Top level preference for flat hierarchy should # always propagate resolver settings config.resolver = self.resolver @@ -264,7 +305,8 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): """Lock down the immediate dependency versions.""" - sources = self._get_sources(use_locked=obey_existing).copy() + flat_recursive=self.resolver == Config.RESOLVER_RECURSIVE_FLAT or Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + sources = self._get_sources(use_locked=obey_existing, flat_recursive=flat_recursive).copy() sources_filter = self._get_sources_filter( *names, sources=sources, skip_default_group=False ) @@ -277,6 +319,7 @@ def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): common.indent() count = 0 + for source in sources: if source.name not in sources_filter: log.info("Skipped dependency: %s", source.name) @@ -375,7 +418,7 @@ def log(self, message="", *args): with open(self.log_path, 'a') as outfile: outfile.write(message.format(*args) + '\n') - def _get_sources(self, *, use_locked=None): + def _get_sources(self, *, use_locked=None, use_extra=True, recursive = False): """Merge source lists using the requested section as the base.""" if use_locked is True: if self.sources_locked: @@ -396,18 +439,19 @@ def _get_sources(self, *, use_locked=None): extras = [] - all_sources = self.sources + self.sources_locked - - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT: - # here is some extra work to do because all dependencies that are resolved in - # flat hierarchy are needed to resolved in a safe manner to lock them all - # in the sources_locked section - # self.processed_sources contains the complete list of all resolved sources - # only if an update process has been completed (basically this is the desired list to return) - # but this is not the case when directly a lock operation has been executed - # therefore we need to do some generic stuff here which is idependently from update process - pass - else: + if recursive: + recursive_sources = sources + for source in sources: + config = load_config(start=os.path.join(self.location_path, source.name), search=False) + if config: + recursive_sources = recursive_sources + config._get_sources(use_locked=use_locked, flat_recursive=True) + + for source in recursive_sources: + if source not in sources: + extras.append(source) + + if use_extra: + all_sources = self.sources + self.sources_locked for source in all_sources: if source not in sources: log.info("Source %r missing from selected section", @@ -416,6 +460,7 @@ def _get_sources(self, *, use_locked=None): return sources + extras + def _get_sources_filter(self, *names, sources, skip_default_group): """Get filtered sublist of sources.""" names_list = list(names) From 8fcd979416d0f6a9843927e96e1bd9211be60952 Mon Sep 17 00:00:00 2001 From: Daniel Brosche Date: Thu, 24 Jun 2021 19:21:47 +0200 Subject: [PATCH 3/5] initial support of flat-recursive* update, lock and install --- gitman/models/config.py | 106 +++++++++++++++++++++++----------------- gitman/models/source.py | 3 -- 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/gitman/models/config.py b/gitman/models/config.py index fcfda84..49fe44a 100644 --- a/gitman/models/config.py +++ b/gitman/models/config.py @@ -11,17 +11,7 @@ from .group import Group from .source import Source, create_sym_link -# TODO: use __post_init__ instead of _on_post_load -# TODO: create links for nested dependencies if flat recursive is used -# TODO: create relative nested symlinks -# Use Case 1 all flat recursive -> recursive all repos in a flat way -# Use Case 2 flat recursive top level -> recursive-nested bottom level: gather all repos globally and create nested links -# -> gitman -> imports -# - waf-prototyping -# - lib -# - libex -# Use Case 3 flat recursive top level -> flat bottom level: clone in a flat way but abort recursive resolution in the next stage -global_resolver = None + @datafile("{self.root}/{self.filename}", defaults=True, manual=True) class Config: @@ -50,16 +40,12 @@ def __post_init__(self): setattr(self, 'location_path', None) processed_sources: List[Source] = [] setattr(self, 'processed_sources', processed_sources) - + registered_sources: List[Source] = [] + setattr(self, 'registered_sources', registered_sources) + if self.location_path is None: self.location_path = os.path.normpath(os.path.join(self.root, self.location)) - def _on_post_load(self): - global global_resolver - - # update location path because default location may different then loaded value - self.location_path = os.path.normpath(os.path.join(self.root, self.location)) - # check if any of the valid resolver values is set # if not then set RESOLVER_RECURSIVE_NESTED as default if (self.resolver != Config.RESOLVER_RECURSIVE_NESTED and @@ -68,16 +54,15 @@ def _on_post_load(self): self.resolver != Config.RESOLVER_FLAT): msg = "Unknown resolver name \"{}\"".format(self.resolver) raise exceptions.InvalidConfig(msg) - - for source in self.sources: - source._on_post_load() # pylint: disable=protected-access - for source in self.sources_locked: - source._on_post_load() # pylint: disable=protected-access - + # set global resolve if not already set if global_resolver is None: global_resolver = self.resolver + def __post_load__(self): + # update location path because default location may different then loaded value + self.location_path = os.path.normpath(os.path.join(self.root, self.location)) + @property def config_path(self) -> str: """Get the full path to the config file.""" @@ -141,22 +126,43 @@ def install_dependencies( ) # gather flat sources and check for rev conflicts + new_sources : List[Source] = [] for source in sources: - for processed_source in self.processed_sources: - if source.name == processed_source.name: # check if current source was already processed - if (source.rev != processed_source.rev or - source.repo != processed_source.repo): + add_source : bool = True + for registered_source in self.registered_sources: + if source.name == registered_source.name: # check if current source was already processed + if (source.rev != registered_source.rev or + source.repo != registered_source.repo): + # we skip the detected recursed branch rev if we install locked sources + # always the toplevel rev is leading and to ensure creation order + # we process the locked version next instead of the noted rev + if not update: + # check if we have already processed the matched source + if not registered_source in self.processed_sources: + new_sources.append(registered_source) + else: + # already processed therefore we don't care anymore + sources_filter.remove(source.name) + + add_source = False + continue + error_msg = ("Repo/rev conflict encountered in " - "flat hierarchy while updating {}\n" - "Details: {} conflict with {}" - ).format(self.root, - str(processed_source), - str(source)) + "flat hierarchy while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, + str(registered_source), + str(source)) raise exceptions.InvalidConfig(error_msg) # new source name detected -> store new source name to list (cache) used to check for rev conflicts - # source.name != processed_source.name and source.repo != processed_source.repo: - self.processed_sources.append(source) + if add_source: + self.registered_sources.append(source) + new_sources.append(source) + + # assign filtered and collected sources + sources = new_sources + else: sources = self._get_sources(use_locked=False if update else None) sources_filter = self._get_sources_filter( @@ -164,17 +170,17 @@ def install_dependencies( ) for source in sources: - for processed_source in self.processed_sources: - if source.name == processed_source.name: # check if current source was already processed + for registered_source in self.registered_sources: + if source.name == registered_source.name: # check if current source was already processed error_msg = ("Repo conflict encountered " "while updating {}\n" "Details: {} conflict with {}" ).format(self.root, - str(processed_source), + str(registered_source), str(source)) raise exceptions.InvalidConfig(error_msg) - self.processed_sources.append(source) + self.registered_sources.append(source) if not os.path.isdir(self.location_path): shell.mkdir(self.location_path) @@ -190,6 +196,10 @@ def install_dependencies( log.info("Skipped dependency: %s", source.name) continue + # check if source has not already been processed + if source in self.processed_sources: + continue + source.update_files( force=force, force_interactive=force_interactive, @@ -198,6 +208,10 @@ def install_dependencies( skip_changes=skip_changes, ) source.create_links(self.root, force=force) + + # store processed source + self.processed_sources.append(source) + common.newline() count += 1 @@ -217,8 +231,10 @@ def install_dependencies( # to install dependencies all into the same folder org_location_path = config.location_path config.location_path = self.location_path - # forward processed sources list to check for global conflicts + # forward registered and processed sources list to check for global conflicts + config.registered_sources = self.registered_sources config.processed_sources = self.processed_sources + count += config.install_dependencies( depth=None if depth is None else max(0, depth - 1), @@ -305,14 +321,14 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): """Lock down the immediate dependency versions.""" - flat_recursive=self.resolver == Config.RESOLVER_RECURSIVE_FLAT or Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS - sources = self._get_sources(use_locked=obey_existing, flat_recursive=flat_recursive).copy() + recursive = self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + sources = self._get_sources(use_locked=obey_existing, recursive=recursive).copy() sources_filter = self._get_sources_filter( *names, sources=sources, skip_default_group=False ) if not os.path.isdir(self.location_path): - raise exceptions.InvalidRepository("No dependecies resolved") + raise exceptions.InvalidRepository("No dependencies resolved") shell.cd(self.location_path) common.newline() @@ -444,7 +460,7 @@ def _get_sources(self, *, use_locked=None, use_extra=True, recursive = False): for source in sources: config = load_config(start=os.path.join(self.location_path, source.name), search=False) if config: - recursive_sources = recursive_sources + config._get_sources(use_locked=use_locked, flat_recursive=True) + recursive_sources = recursive_sources + config._get_sources(use_locked=use_locked, use_extra=False, recursive=True) for source in recursive_sources: if source not in sources: @@ -497,7 +513,7 @@ def load_config(start=None, *, search=True): for filename in os.listdir(path): if _valid_filename(filename): config = Config(path, filename) - config._on_post_load() # pylint: disable=protected-access + config.__post_load__() config.validate() log.debug("Found config: %s", config.path) return config diff --git a/gitman/models/source.py b/gitman/models/source.py index 72b1d83..5f07805 100644 --- a/gitman/models/source.py +++ b/gitman/models/source.py @@ -37,9 +37,6 @@ def __post_init__(self): self.name = str(self.name) self.type = self.type or 'git' - def _on_post_load(self): - pass - def __repr__(self): return "".format(self) From c3872ac37249c0e5ac358dffe1d691abc7d6eeea Mon Sep 17 00:00:00 2001 From: Daniel Brosche Date: Fri, 25 Jun 2021 08:15:32 +0200 Subject: [PATCH 4/5] fix linter issues --- gitman/cli.py | 5 +- gitman/models/config.py | 209 +++++++++++++++++++++++----------------- 2 files changed, 126 insertions(+), 88 deletions(-) diff --git a/gitman/cli.py b/gitman/cli.py index 2547dc1..136a30c 100644 --- a/gitman/cli.py +++ b/gitman/cli.py @@ -336,7 +336,10 @@ def _run_command(function, args, kwargs): exit_message = "Run again with '--force' to ignore script errors" except exceptions.InvalidConfig as exception: _show_error(exception) - exit_message = "Check the gitman config of the already imported repositories and the repository that raises the conflict error" + exit_message = ( + "Check the gitman config of the already imported " + "repositories and the repository that raises the conflict error" + ) finally: if exit_message: common.show(exit_message, color='message') diff --git a/gitman/models/config.py b/gitman/models/config.py index 49fe44a..e7858f2 100644 --- a/gitman/models/config.py +++ b/gitman/models/config.py @@ -1,7 +1,6 @@ import os import sys from typing import List, Optional -import inspect import log from datafiles import datafile, field @@ -12,6 +11,16 @@ from .source import Source, create_sym_link +# TODO remove the next pylint statement and adapt code + +# disable because of the dynamics setattr in __post_init__ to avoid +# that datafile persist this values -> is there another way to tell +# datafiles to ignore specified fiels +# pylint: disable=attribute-defined-outside-init, access-member-before-definition + +# due to the protected access of _get_sources -> make them public in the next step +# pylint: disable=protected-access + @datafile("{self.root}/{self.filename}", defaults=True, manual=True) class Config: @@ -31,43 +40,42 @@ class Config: default_group: str = field(default_factory=str) groups: List[Group] = field(default_factory=list) - def __post_init__(self): if self.root is None: self.root = os.getcwd() - + # used only internally and should not be serialized - setattr(self, 'location_path', None) + location_path = os.path.normpath(os.path.join(self.root, self.location)) + setattr(self, 'location_path', location_path) processed_sources: List[Source] = [] setattr(self, 'processed_sources', processed_sources) registered_sources: List[Source] = [] setattr(self, 'registered_sources', registered_sources) - if self.location_path is None: - self.location_path = os.path.normpath(os.path.join(self.root, self.location)) - - # check if any of the valid resolver values is set + # check if any of the valid resolver values is set # if not then set RESOLVER_RECURSIVE_NESTED as default - if (self.resolver != Config.RESOLVER_RECURSIVE_NESTED and - self.resolver != Config.RESOLVER_RECURSIVE_FLAT and - self.resolver != Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS and - self.resolver != Config.RESOLVER_FLAT): + if ( + self.resolver != Config.RESOLVER_RECURSIVE_NESTED + and self.resolver != Config.RESOLVER_RECURSIVE_FLAT + and self.resolver != Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + and self.resolver != Config.RESOLVER_FLAT + ): msg = "Unknown resolver name \"{}\"".format(self.resolver) raise exceptions.InvalidConfig(msg) - - # set global resolve if not already set - if global_resolver is None: - global_resolver = self.resolver def __post_load__(self): # update location path because default location may different then loaded value - self.location_path = os.path.normpath(os.path.join(self.root, self.location)) + self.location_path = os.path.normpath( + os.path.join(self.root, self.location) # type: ignore[arg-type] + ) @property def config_path(self) -> str: """Get the full path to the config file.""" assert self.root - return os.path.normpath(os.path.join(self.root, self.filename)) + return os.path.normpath( + os.path.join(self.root, self.filename) # type: ignore[arg-type] + ) path = config_path @@ -110,35 +118,46 @@ def install_dependencies( clean=True, skip_changes=False, skip_default_group=False, - ): # pylint: disable=too-many-locals + ): # pylint: disable=too-many-locals, too-many-statements """Download or update the specified dependencies.""" if depth == 0: log.info("Skipped directory: %s", self.location_path) return 0 - sources = None + sources = None sources_filter = None - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: - sources = self._get_sources(use_locked=False if update else None, use_extra=False) + if ( # pylint: disable=too-many-nested-blocks + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ): + sources = self._get_sources( + use_locked=False if update else None, use_extra=False + ) sources_filter = self._get_sources_filter( *names, sources=sources, skip_default_group=skip_default_group ) # gather flat sources and check for rev conflicts - new_sources : List[Source] = [] + new_sources: List[Source] = [] for source in sources: - add_source : bool = True - for registered_source in self.registered_sources: - if source.name == registered_source.name: # check if current source was already processed - if (source.rev != registered_source.rev or - source.repo != registered_source.repo): - # we skip the detected recursed branch rev if we install locked sources - # always the toplevel rev is leading and to ensure creation order - # we process the locked version next instead of the noted rev + add_source: bool = True + for registered_source in self.registered_sources: # type: ignore + if ( + source.name == registered_source.name + ): # check if current source was already processed + if ( + source.rev != registered_source.rev + or source.repo != registered_source.repo + ): + # we skip the detected recursed branch rev if we install + # locked sources always the toplevel rev is leading and + # to ensure creation order we process the locked version + # next instead of the noted rev if not update: # check if we have already processed the matched source - if not registered_source in self.processed_sources: + # pylint: disable=line-too-long + if not registered_source in self.processed_sources: # type: ignore new_sources.append(registered_source) else: # already processed therefore we don't care anymore @@ -147,20 +166,20 @@ def install_dependencies( add_source = False continue - error_msg = ("Repo/rev conflict encountered in " - "flat hierarchy while updating {}\n" - "Details: {} conflict with {}" - ).format(self.root, - str(registered_source), - str(source)) - raise exceptions.InvalidConfig(error_msg) - - # new source name detected -> store new source name to list (cache) used to check for rev conflicts + error_msg = ( + "Repo/rev conflict encountered in " + "flat hierarchy while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, str(registered_source), str(source)) + raise exceptions.InvalidConfig(error_msg) + + # new source name detected -> store new source name + # to list (cache) used to check for rev conflicts if add_source: - self.registered_sources.append(source) + self.registered_sources.append(source) # type: ignore new_sources.append(source) - # assign filtered and collected sources + # assign filtered and collected sources sources = new_sources else: @@ -170,17 +189,18 @@ def install_dependencies( ) for source in sources: - for registered_source in self.registered_sources: - if source.name == registered_source.name: # check if current source was already processed - error_msg = ("Repo conflict encountered " - "while updating {}\n" - "Details: {} conflict with {}" - ).format(self.root, - str(registered_source), - str(source)) - raise exceptions.InvalidConfig(error_msg) - - self.registered_sources.append(source) + for registered_source in self.registered_sources: # type: ignore + if ( + source.name == registered_source.name + ): # check if current source was already processed + error_msg = ( + "Repo conflict encountered " + "while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, str(registered_source), str(source)) + raise exceptions.InvalidConfig(error_msg) + + self.registered_sources.append(source) # type: ignore if not os.path.isdir(self.location_path): shell.mkdir(self.location_path) @@ -197,7 +217,7 @@ def install_dependencies( continue # check if source has not already been processed - if source in self.processed_sources: + if source in self.processed_sources: # type: ignore continue source.update_files( @@ -208,9 +228,9 @@ def install_dependencies( skip_changes=skip_changes, ) source.create_links(self.root, force=force) - + # store processed source - self.processed_sources.append(source) + self.processed_sources.append(source) # type: ignore common.newline() count += 1 @@ -222,20 +242,23 @@ def install_dependencies( config = load_config(search=False) if config: common.indent() - - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: - # Top level preference for flat hierarchy should + + if ( + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ): + # Top level preference for flat hierarchy should # forward / propagate resolver settings config.resolver = self.resolver # forward / override default location -> always use root location # to install dependencies all into the same folder org_location_path = config.location_path - config.location_path = self.location_path - # forward registered and processed sources list to check for global conflicts - config.registered_sources = self.registered_sources - config.processed_sources = self.processed_sources - - + config.location_path = self.location_path + # forward registered and processed sources list to + # check for global conflicts + config.registered_sources = self.registered_sources # type: ignore + config.processed_sources = self.processed_sources # type: ignore + count += config.install_dependencies( depth=None if depth is None else max(0, depth - 1), update=update and recurse, @@ -247,11 +270,11 @@ def install_dependencies( skip_default_group=skip_default_group, ) - # create nested symlinks + # create nested symlinks if self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: - for source in config.sources: - link_src = os.path.join(self.location_path, source.name) - link_target = os.path.join(org_location_path, source.name) + for src in config.sources: + link_src = os.path.join(self.location_path, src.name) + link_target = os.path.join(org_location_path, src.name) create_sym_link(link_src, link_target, True) common.dedent() @@ -296,17 +319,21 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): remaining_depth = None if depth is None else max(0, depth - 1) if remaining_depth: common.newline() - - if self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: - # Top level preference for flat hierarchy should + + if ( + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ): + # Top level preference for flat hierarchy should # always propagate resolver settings config.resolver = self.resolver # override default location -> always use root location # to install dependencies all into the same folder - config.location_path = self.location_path + config.location_path = self.location_path # forward processed sources list to check for global conflicts - config.processed_sources = self.processed_sources - + # pylint: disable=line-too-long + config.processed_sources = self.processed_sources # type: ignore + count += config.run_scripts(depth=remaining_depth, force=force) common.dedent() @@ -321,8 +348,13 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): """Lock down the immediate dependency versions.""" - recursive = self.resolver == Config.RESOLVER_RECURSIVE_FLAT or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS - sources = self._get_sources(use_locked=obey_existing, recursive=recursive).copy() + recursive = ( + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ) + sources = self._get_sources( + use_locked=obey_existing, recursive=recursive + ).copy() sources_filter = self._get_sources_filter( *names, sources=sources, skip_default_group=False ) @@ -335,7 +367,7 @@ def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): common.indent() count = 0 - + for source in sources: if source.name not in sources_filter: log.info("Skipped dependency: %s", source.name) @@ -434,7 +466,7 @@ def log(self, message="", *args): with open(self.log_path, 'a') as outfile: outfile.write(message.format(*args) + '\n') - def _get_sources(self, *, use_locked=None, use_extra=True, recursive = False): + def _get_sources(self, *, use_locked=None, use_extra=True, recursive=False): """Merge source lists using the requested section as the base.""" if use_locked is True: if self.sources_locked: @@ -458,25 +490,28 @@ def _get_sources(self, *, use_locked=None, use_extra=True, recursive = False): if recursive: recursive_sources = sources for source in sources: - config = load_config(start=os.path.join(self.location_path, source.name), search=False) + config_path = os.path.join( + self.location_path, source.name # type: ignore[arg-type] + ) + config = load_config(start=config_path, search=False) if config: - recursive_sources = recursive_sources + config._get_sources(use_locked=use_locked, use_extra=False, recursive=True) - + recursive_sources = recursive_sources + config._get_sources( + use_locked=use_locked, use_extra=False, recursive=True + ) + for source in recursive_sources: if source not in sources: extras.append(source) - + if use_extra: all_sources = self.sources + self.sources_locked for source in all_sources: if source not in sources: - log.info("Source %r missing from selected section", - source.name) + log.info("Source %r missing from selected section", source.name) extras.append(source) return sources + extras - def _get_sources_filter(self, *names, sources, skip_default_group): """Get filtered sublist of sources.""" names_list = list(names) From 80df9fef369903459bb161ccfebace92948516a2 Mon Sep 17 00:00:00 2001 From: Daniel Brosche Date: Fri, 25 Jun 2021 08:15:59 +0200 Subject: [PATCH 5/5] fix a couple of tests due to the introduced resolver-field --- tests/test_api.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index 3f0768e..dd247e9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -23,6 +23,7 @@ CONFIG = """ location: deps +resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -93,6 +94,7 @@ def it_creates_a_new_config_file(tmpdir): expect(Config().datafile.text) == strip( """ location: gitman_sources + resolver: recursive-nested sources: - repo: https://github.com/githubtraining/hellogitworld name: sample_dependency @@ -144,6 +146,7 @@ def it_merges_sources(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -182,6 +185,7 @@ def it_can_handle_missing_locked_sources(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -225,6 +229,7 @@ def config_with_link(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 repo: https://github.com/jacebrowning/gitman-demo @@ -268,6 +273,7 @@ def config_with_links(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 repo: https://github.com/jacebrowning/gitman-demo @@ -314,6 +320,7 @@ def config_with_scripts(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -342,6 +349,7 @@ def config_with_scripts(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -376,6 +384,7 @@ def config_with_group(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -438,6 +447,7 @@ def config_with_default_group(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -478,6 +488,7 @@ def config_without_default_group(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -617,6 +628,7 @@ def it_locks_previously_unlocked_dependencies(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -661,6 +673,7 @@ def it_locks_previously_unlocked_dependencies(config): expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -703,6 +716,7 @@ def it_should_not_lock_dependencies_when_disabled(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -746,6 +760,7 @@ def it_should_not_lock_dependencies_when_disabled(config): expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -788,6 +803,7 @@ def it_should_not_allow_source_and_group_name_conflicts(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -813,6 +829,7 @@ def it_locks_previously_locked_dependencies_by_group_name(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -880,6 +897,7 @@ def it_locks_previously_locked_dependencies_by_group_name(config): expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -961,6 +979,7 @@ def git_changes( config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -994,6 +1013,7 @@ def git_changes( expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -1049,6 +1069,7 @@ def git_changes( config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -1083,6 +1104,7 @@ def git_changes( expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -1115,6 +1137,7 @@ def it_merges_sources(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1