depot: move versions to subdirectory

This patch changes the depot layout such that each archive is
represented as a directory that contains the versions of the archive as
subdirectories.

Issue #2610
This commit is contained in:
Norman Feske
2017-12-11 17:19:03 +01:00
committed by Christian Helmuth
parent 2d041f0e9c
commit a52541de18
14 changed files with 218 additions and 188 deletions

View File

@@ -23,12 +23,12 @@ define HELP_MESSAGE
E.g., the user 'alan' may build the following archives:
alan/bin/x86_64/zlib-<version> - a binary archive of the zlib
alan/bin/x86_64/zlib/<version> - a binary archive of the zlib
library with the specified
version, built for the 64-bit
x86 architecture
alan/pkg/x86_32/wm-<version> - all binary archives needed by
alan/pkg/x86_32/wm/<version> - all binary archives needed by
the 'wm' package archive, built
for the 32-bit x86 architecture
@@ -95,7 +95,7 @@ endif
# determine binary-archive path within the depot
_dst_bin_spec_path = $(call archive_user,$1)/bin/$(call bin_archive_spec,$1)/
dst_archive_path = $(call _dst_bin_spec_path,$1)$(call bin_archive_recipe,$1)
dst_archive_path = $(call _dst_bin_spec_path,$1)$(call bin_archive_recipe,$1)/$(call bin_archive_version,$1)
BUILD_MK_FILE := $(DEPOT_DIR)/var/build.mk
@@ -113,11 +113,12 @@ $(BUILD_MK_FILE): checked_source_archives_exist checked_no_uncategorized
target=$(call dst_archive_path,$A); \
user=$(call archive_user,$A); \
recipe=$(call bin_archive_recipe,$A); \
version=$(call bin_archive_version,$A); \
spec=$(call bin_archive_spec,$A); \
echo ""; \
echo "TARGETS += $$target"; \
echo "TOOL($$target) := build_bin_archive"; \
echo "ARGS($$target) := $$recipe USER=$$user SPEC=$$spec"; \
echo "ARGS($$target) := $$recipe/$$version USER=$$user SPEC=$$spec"; \
) \
echo -e "\nall: \$$(TARGETS)"; \
echo -e "\n\$$(TARGETS):"; \

View File

@@ -44,8 +44,8 @@ extract:
$(MAKECMDGOALS): extract
_versioned_src_of_bin = $1-$(call recipe_version,src/$(call bin_archive_recipe,$1))
_versioned_pkg = $1-$(call recipe_version,pkg/$(call bin_archive_recipe,$1))
_versioned_src_of_bin = $1/$(call recipe_version,src/$(call bin_archive_recipe,$1))
_versioned_pkg = $1/$(call recipe_version,pkg/$(call bin_archive_recipe,$1))
versioned_archive = $(if $(call archive_has_type,$1,bin),$(call _versioned_src_of_bin,$1),\
$(if $(call archive_has_type,$1,pkg),$(call _versioned_pkg,$1)))

View File

@@ -47,6 +47,9 @@ endef
export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..)
# the extract tool expects archive paths given without the version element
BIN_PKG_PATH_ELEMS := 4
include $(GENODE_DIR)/tool/depot/mk/front_end.inc
include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc
@@ -86,9 +89,9 @@ include $(GENODE_DIR)/tool/depot/mk/dependencies.inc
#
# Obtain version information from recipes
#
# The 'archive_version' function takes the archive type and name as arguments
# and returns the version identifier as present in the corresponding recipe.
# The nested foreach loop populates 'ARCHIVE_VERSION' with the version
# The 'archive_curr_version' function takes the archive type and name as
# arguments and returns the version identifier as present in the corresponding
# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version
# identifier for each archive.
#
# If an archive is given with a complete (versioned) name, we don't need to
@@ -99,7 +102,7 @@ include $(GENODE_DIR)/tool/depot/mk/dependencies.inc
$(foreach TYPE,api src raw pkg,\
$(foreach PATH,${ARCHIVES(${TYPE})},\
$(eval ARCHIVE_VERSION(${PATH}) := $(call archive_version,$(PATH)))))
$(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH)))))
archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1)
@@ -121,7 +124,7 @@ endif
# return versioned archive path, if 'ARCHIVE_VERSION' is undefined, assume
# that the argument is already a versiond path
versioned_archive = $(if $(ARCHIVE_VERSION($1)),$(addsuffix -${ARCHIVE_VERSION($1)},$1),$1)
versioned_archive = $(if $(ARCHIVE_VERSION($1)),$(addsuffix /${ARCHIVE_VERSION($1)},$1),$1)
EXTRACT_MK_FILE := $(DEPOT_DIR)/var/extract.mk

View File

@@ -76,7 +76,7 @@ checked_src_archive:
@true
ifeq ($(SRC_DIR),)
VERSIONED_ARCHIVE := $(ARCHIVE)-$(SRC_VERSION)
VERSIONED_ARCHIVE := $(ARCHIVE)/$(SRC_VERSION)
SRC_DIR := $(DEPOT_SRC_DIR)/$(VERSIONED_ARCHIVE)
checked_src_archive: checked_src_hash_file
endif

View File

@@ -36,16 +36,23 @@ endif
#
# Sub-categorize source-pkg archives (<user>/pkg/<name>) from binary-pkg
# archives (<user>/pkg/<spec>/<name>) so that 'ARCHIVES(pkg)' contains source
# pkgs only, and 'ARCHIVES(binpkg)' contains binary pkgs.
# Sub-categorize source-pkg archives (<user>/pkg/<name>[/<version>]) from
# binary-pkg archives (<user>/pkg/<spec>/<name>,[<version>]) so that
# 'ARCHIVES(pkg)' contains source pkgs only, and 'ARCHIVES(binpkg)' contains
# binary pkgs.
#
# If the path contains 4 elements, it refers to a binary pkg where the third
# element is the build spec. Otherwise, the path refers to a source pkg.
# If the path contains 'BIN_PKG_PATH_ELEMS' elements, it refers to a binary pkg
# where the third element is the build spec. Otherwise, the path refers to a
# source pkg. By default, the distinction assumes versioned archive paths.
# For the 'extract' tool where the version elements are omitted because they
# refer to the current version as present in the repository, the value is
# customized to '4'.
#
_src_pkg = $(if $(word 4,$(subst /, ,$1)),,$1)
_bin_pkg = $(if $(word 4,$(subst /, ,$1)),$1,)
BIN_PKG_PATH_ELEMS ?= 5
_src_pkg = $(if $(word $(BIN_PKG_PATH_ELEMS),$(subst /, ,$1)),,$1)
_bin_pkg = $(if $(word $(BIN_PKG_PATH_ELEMS),$(subst /, ,$1)),$1,)
ARCHIVES(binpkg) := $(strip $(foreach PKG,${ARCHIVES(pkg)},$(call _bin_pkg,$(PKG))))
ARCHIVES(pkg) := $(strip $(foreach PKG,${ARCHIVES(pkg)},$(call _src_pkg,$(PKG))))

View File

@@ -66,7 +66,7 @@ _src_exists = $(wildcard $(dir $(call used_apis_file,$1)))
# return binary-archive path for architecture $1 and source archive $2
_bin_for_src = $(strip \
$(if $(call _src_exists,$2),\
$(call archive_user,$2)/bin/$1/$(call archive_recipe,$2)))
$(call archive_user,$2)/bin/$1/$(call archive_recipe,$2)/$(call archive_version,$2)))
# return list of binary archives contained in a binary package
_binpkg_bin_archives = $(foreach S,$(call _binpkg_src_archives,$1),\

View File

@@ -85,7 +85,7 @@ ORIG_RECIPE_HASH_VALUE := $(RECIPE_HASH_VALUE)
# archive hash.
#
DEPOT_ARCHIVE_DIR := $(DEPOT_SUB_DIR)/$(ARCHIVE).incomplete
DEPOT_ARCHIVE_DIR := $(DEPOT_SUB_DIR)/$(ARCHIVE)/incomplete
reset_stale_temporary_archive_dir:
ifneq ($(wildcard $(DEPOT_ARCHIVE_DIR)),)
@@ -110,16 +110,16 @@ $(ARCHIVE): _rename_to_final_archive
# discard the just-built archive.
#
_rename_to_final_archive: _check_hash
@$(VERBOSE)final_name=$(ARCHIVE)-$(RECIPE_VERSION); \
rm -rf $(DEPOT_SUB_DIR)/$$final_name; \
mv $(DEPOT_ARCHIVE_DIR) $(DEPOT_SUB_DIR)/$$final_name; \
hash=$$(< $(DEPOT_ARCHIVE_DIR).hash); hint=""; \
test $$hash = $(ORIG_RECIPE_HASH_VALUE) ||\
hint=" $(BRIGHT_COL)(new version)$(DEFAULT_COL)"; \
rm -f $(DEPOT_ARCHIVE_DIR).hash; \
$(ECHO) "$(DARK_COL)created$(DEFAULT_COL)" \
"$(USER)/$(notdir $(DEPOT_SUB_DIR))/$$final_name$$hint"; \
true;
$(VERBOSE)final_name=$(ARCHIVE)/$(RECIPE_VERSION); \
rm -rf $(DEPOT_SUB_DIR)/$$final_name; \
mv $(DEPOT_ARCHIVE_DIR) $(DEPOT_SUB_DIR)/$$final_name; \
hash=$$(< $(DEPOT_ARCHIVE_DIR).hash); hint=""; \
test $$hash = $(ORIG_RECIPE_HASH_VALUE) ||\
hint=" $(BRIGHT_COL)(new version)$(DEFAULT_COL)"; \
rm -f $(DEPOT_ARCHIVE_DIR).hash; \
$(ECHO) "$(DARK_COL)created$(DEFAULT_COL)" \
"$(USER)/$(notdir $(DEPOT_SUB_DIR))/$$final_name$$hint"; \
true;
#
# Generate suggested version name for 'HASH_OUT_OF_DATE_MESSAGE'

View File

@@ -45,7 +45,7 @@ include $(GENODE_DIR)/tool/depot/mk/extract.inc
#
_version = $(call recipe_version,$(call archive_type,$1)/$(call archive_recipe,$1))
_versioned_entry = _/$(call archive_type,$1)/$(call archive_recipe,$1)-$(call _version,$1)
_versioned_entry = _/$(call archive_type,$1)/$(call archive_recipe,$1)/$(call _version,$1)
VERSIONED_ARCHIVES := $(foreach A,$(call file_content,$(RECIPE_DIR)/archives),\
$(if $(call archive_has_user,$A,_),$(call _versioned_entry,$A),$A))

View File

@@ -67,7 +67,7 @@ $(DEPOT_ARCHIVE_DIR)/used_apis: $(RECIPE_DIR)/used_apis
fi; \
hash_file_content=$$(< $$hash_file); \
version=$${hash_file_content%% *}; \
echo "$$api-$$version" >> $@; \
echo "$$api/$$version" >> $@; \
done; $$result
#
@@ -89,5 +89,5 @@ $(DEPOT_ARCHIVE_DIR)/api: $(RECIPE_DIR)/api
fi; \
hash_file_content=$$(< $$hash_file); \
version=$${hash_file_content%% *}; \
echo "$$api-$$version" >> $@;
echo "$$api/$$version" >> $@;

View File

@@ -42,24 +42,34 @@ last_path_element = $(call sanitized,$(lastword $(subst /, ,$1)))
archive_user = $(call path_element,1,$1)
archive_type = $(call path_element,2,$1)
archive_recipe = $(call path_element,3,$1)
archive_version = $(call path_element,4,$1)
archive_has_type = $(filter $(call archive_type,$1),$2)
archive_has_user = $(filter $(call archive_user,$1),$2)
archive_version = $(call recipe_version,$(addprefix $(call archive_type,$1)/,$(call archive_recipe,$1)))
# binary archives have the form <user>/bin/<spec>/<name>{-<version>}
archive_curr_version = $(call recipe_version,$(addprefix $(call archive_type,$1)/,$(call archive_recipe,$1)))
# binary archives have the form <user>/bin/<spec>/<name>/<version>
bin_archive_spec = $(call path_element,3,$1)
bin_archive_recipe = $(call last_path_element,$1)
bin_archive_version = $(call recipe_version,src/$(call bin_archive_recipe,$1))
bin_archive_recipe = $(call path_element,4,$1)
bin_archive_version = $(call path_element,5,$1)
grep_archive_type = $(foreach A,$2,$(if $(call archive_has_type,$A,$1),$A,))
grep_archive_user = $(foreach A,$2,$(if $(call archive_has_user,$A,$1),$A,))
#
# The following functions can be called for archive paths with or without
# the version part. In the latter case, 'bin_archive_version' is empty.
# The 'addprefix' is needed to omit the trailing '/' in the result if a
# path without version is specified.
#
# return pkg-archive path of given binary-pkg archive path
pkg_of_binpkg = $(call archive_user,$1)/pkg/$(call bin_archive_recipe,$1)
pkg_of_binpkg = $(call archive_user,$1)/pkg/$(call bin_archive_recipe,$1)$(addprefix /,$(call bin_archive_version,$1))
# return source-archive path for given binary-archive path
src_of_bin = $(call archive_user,$1)/src/$(call bin_archive_recipe,$1)
src_of_bin = $(call archive_user,$1)/src/$(call bin_archive_recipe,$1)$(addprefix /,$(call bin_archive_version,$1))
# return binary-package archive path for architecture $1 and package archive $2
binpkg_for_pkg = $(call archive_user,$2)/bin/$1/$(call archive_recipe,$2)
binpkg_for_pkg = $(call archive_user,$2)/bin/$1/$(call archive_recipe,$2)$(addprefix /,$(call archive_version,$2))

View File

@@ -22,28 +22,40 @@ proc depot_spec { } {
# depot. The list is populated by calls of 'import_from_depot' and evaluated
# at the boot-image-creation stage via 'check_for_missing_depot_archives'.
#
# Each list element is a list of <user>, <type>, <spec>, <name>, and <version>.
#
set _missing_depot_archives {}
#
# Pattern to parse an archive path into <user>, <type>, <name>
# Pattern to parse an version-less archive path into <user>, <type>, <name>
#
proc _depot_archive_path_pattern { } { return {^([\w\d]+)/([\w]+)/([\w\d\-_]+)$} }
proc _depot_archive_path_pattern { } {
return {^([\w\d]+)/([\w]+)/([\w\d\-_]+)$} }
#
# Pattern to parse an versioned archive path into <user>, <type>, <name>, <version>
#
proc _depot_archive_versioned_path_pattern { } {
return {^([\w\d]+)/([\w]+)/([\w\d\-_]+)/([\w\d\-_]+)$} }
#
# Pattern to parse an binary archive path into <user>, <spec>, <name>.
#
proc _depot_bin_archive_path_pattern { } { return {^([\w\d]+)/bin/([\w\d]+)/([\w\d\-_]+)$} }
proc _depot_bin_archive_path_pattern { } {
return {^([\w\d]+)/bin/([\w\d]+)/([\w\d\-_]+)$} }
##
# Determine content of a pkg archive and its dependencies
#
proc _collect_pkg_archive_from_depot { user name } {
proc _collect_pkg_archive_from_depot { user name version } {
global _missing_depot_archives
set archive_dir "$user/pkg/$name"
set archive_dir "$user/pkg/$name/$version"
set archives_file "[depot_dir]/$archive_dir/archives"
if {![file exists $archives_file]} {
@@ -57,7 +69,7 @@ proc _collect_pkg_archive_from_depot { user name } {
set content "$archive_dir"
foreach archive $archives {
if {[regexp [_depot_archive_path_pattern] $archive dummy user type name]} {
if {[regexp [_depot_archive_versioned_path_pattern] $archive dummy user type name version]} {
if {($type == "pkg") || ($type == "src") || ($type == "raw")} {
set content [concat $content [_collect_from_depot $archive]]
}
@@ -79,23 +91,24 @@ proc _copy_directory_content_to_run_dir { dir } {
}
proc _collect_raw_archive_from_depot { user name } { return "$user/raw/$name" }
proc _collect_raw_archive_from_depot { user name version } {
return "$user/raw/$name/$version" }
##
# Determine binary content for a given source archive
#
proc _collect_src_archive_from_depot { user name } {
proc _collect_src_archive_from_depot { user name version } {
global _missing_depot_archives;
set src_archive_dir "$user/src/$name"
set bin_archive_dir "$user/bin/[depot_spec]/$name"
set src_archive_dir "$user/src/$name/$version"
set bin_archive_dir "$user/bin/[depot_spec]/$name/$version"
if {[file exists [depot_dir]/$bin_archive_dir]} {
return [list $src_archive_dir $bin_archive_dir]
} else {
lappend _missing_depot_archives $bin_archive_dir
lappend _missing_depot_archives [list $user bin [depot_spec] $name $version]
}
return {}
@@ -103,35 +116,30 @@ proc _collect_src_archive_from_depot { user name } {
##
# Determine the version-suffixed name of an archive
# Determine the current version for the given archive
#
# This function return an empty string if the archive is missing from the
# This function tries to determine the version information from the Genode
# source tree. It returns an empty string if the archive is missing from the
# depot.
#
proc _versioned_depot_archive_name { user type name } {
proc _current_depot_archive_version { type name } {
# if correctly versioned archive is specified, use it
if {[file exists [depot_dir]/$user/$type/$name]} { return $name }
#
# The given archive name may lack the version identifier if it refers
# to an archive generated locally from the Genode repository. In this case,
# we try to determine the version information from the Genode source tree.
#
set hash_rel_path "recipes/$type/$name/hash"
set repo [repository_contains $hash_rel_path]
set version ""
if {$repo != ""} {
set fh [open "$repo/$hash_rel_path" "RDONLY"]
set version [lindex [gets $fh] 0]
close $fh
append name "-" $version
if {[file exists [depot_dir]/$user/$type/$name]} {
return $name }
}
return $version
}
return ""
proc _depot_contains_archive { user type name version } {
return [file exists [depot_dir]/$user/$type/$name/$version]
}
@@ -143,37 +151,49 @@ proc _collect_from_depot { archives } {
foreach archive $archives {
if {[regexp [_depot_archive_path_pattern] $archive dummy user type name]} {
set version ""
set versioned_name [_versioned_depot_archive_name $user $type $name]
if {$versioned_name == ""} {
lappend _missing_depot_archives $archive
#
# Try to parse versioned archive path. If no version is specified, use
# the current version as present in the source tree.
#
if {![regexp [_depot_archive_versioned_path_pattern] $archive dummy user type name version]} {
if {[regexp [_depot_archive_path_pattern] $archive dummy user type name]} {
set version [_current_depot_archive_version $type $name]
} else {
set content {}
switch $type {
"pkg" { set content [_collect_pkg_archive_from_depot $user $versioned_name] }
"src" { set content [_collect_src_archive_from_depot $user $versioned_name] }
"raw" { set content [_collect_raw_archive_from_depot $user $versioned_name] }
default {
puts stderr "Error: unknown depot-archive type '$type'"
exit 1
}
}
set all_content [concat $all_content $content]
puts stderr "Error: malformed depot-archive path '$archive',"
puts stderr " expected '<user>/<type>/<name>'"
exit 1
}
} else {
puts stderr "Error: malformed depot-archive path '$archive',"
puts stderr " expected '<user>/<type>/<name>'"
}
if {$version == ""} {
puts stderr "Error: unable to guess version of '$type/$name' archive"
exit 1
}
}
if {![_depot_contains_archive $user $type $name $version]} {
lappend _missing_depot_archives [list $user $type "" $name $version]
continue
}
set content {}
switch $type {
"pkg" { set content [_collect_pkg_archive_from_depot $user $name $version] }
"src" { set content [_collect_src_archive_from_depot $user $name $version] }
"raw" { set content [_collect_raw_archive_from_depot $user $name $version] }
default {
puts stderr "Error: unknown depot-archive type '$type'"
exit 1
}
}
set all_content [concat $all_content $content]
}
return $all_content
}
@@ -187,7 +207,7 @@ proc import_from_depot { args } {
foreach subdir [_collect_from_depot $args] {
# prevent src, api, and pkg archives from inflating the boot image
if {[regexp [_depot_archive_path_pattern] $subdir dummy user type]} {
if {[regexp [_depot_archive_versioned_path_pattern] $subdir dummy user type]} {
if {$type == "src"} continue;
if {$type == "api"} continue;
if {$type == "pkg"} continue;
@@ -206,7 +226,7 @@ proc create_tar_from_depot_binaries { archive_path args } {
# filter out api and src archives from requested depot content
set content {}
foreach subdir [_collect_from_depot $args] {
if {[regexp [_depot_archive_path_pattern] $subdir dummy user type]} {
if {[regexp [_depot_archive_versioned_path_pattern] $subdir dummy user type]} {
if {$type == "src"} continue;
if {$type == "api"} continue;
}
@@ -219,31 +239,17 @@ proc create_tar_from_depot_binaries { archive_path args } {
}
proc _locally_available_recipe { user type name } {
proc _locally_available_recipe { user type name version } {
if {[repository_contains "recipes/$type/$name/hash"] != ""} {
return $name }
if {$type == "bin"} { set type "src" }
#
# If the supplied 'name' is a versioned name (as obtained from a pkg
# 'archives' filed already stored in the depot, we try to find a recipe
# in the source tree that matches the specified name and version. If
# we find the matching recipe, we can build the archive locally using
# the version-less recipe name.
#
set version_suffix_pattern {\-[^\-/]*$}
set versioned_name $name
if {[repository_contains "recipes/$type/$name/hash"] == ""} {
return 0 }
while {[regexp -- $version_suffix_pattern $name dummy]} {
if {$version != [_current_depot_archive_version $type $name]} {
return 0 }
# strip last part of version suffix
regsub -- $version_suffix_pattern $name "" name
if {[repository_contains "recipes/$type/$name/hash"] != ""} {
if {[_versioned_depot_archive_name $user $type $name] == $versioned_name} {
return $name } }
}
return ""
return 1
}
@@ -280,45 +286,45 @@ proc check_for_missing_depot_archives { } {
set foreign_archives {}
foreach archive $_missing_depot_archives {
puts stderr " $archive"
if {[regexp [_depot_archive_path_pattern] $archive dummy user type name]} {
#
# If a pkg archive is missing, suggest to obtain the binary-pkg
# archive (matching the build directory) immediately, which implies
# the pkg archive. Otherwise, the user would first obtain the pkg
# archive and its source dependencies, and then get an error for
# the missing binary archives on the next attempt to execute the
# run script.
#
if {$type == "pkg"} { set archive "$user/pkg/[depot_spec]/$name" }
if {$type == "src"} { set archive "$user/bin/[depot_spec]/$name" }
set user [lindex $archive 0]
set type [lindex $archive 1]
set spec [lindex $archive 2]
set name [lindex $archive 3]
set version [lindex $archive 4]
if {[_locally_available_recipe $user $type $name] != ""} {
lappend local_user_archives $archive
} else {
lappend foreign_archives $archive
}
} elseif {[regexp [_depot_bin_archive_path_pattern] $archive dummy user spec name]} {
#
# If a pkg archive is missing, suggest to obtain the binary-pkg
# archive (matching the build directory) immediately, which implies
# the pkg archive. Otherwise, the user would first obtain the pkg
# archive and its source dependencies, and then get an error for
# the missing binary archives on the next attempt to execute the
# run script.
#
if {$type == "pkg"} { set spec "[depot_spec]" }
if {$type == "src"} {
set type "bin"
set spec "[depot_spec]"
}
# source code is present in the current source tree
set recipe [_locally_available_recipe $user src $name]
if {$recipe != ""} {
lappend local_user_archives $user/bin/$spec/$recipe
set path "$user/$type/$name"
if {$type == "bin"} {
set path "$user/bin/$spec/$name" }
# source code is present in the depot
} elseif {[file isdirectory [depot_dir]/$user/src/$name]} {
lappend local_user_archives $user/bin/$spec/$name
puts stderr " $path/$version"
} else {
lappend foreign_archives $archive
}
if {[_locally_available_recipe $user $type $name $version]} {
lappend local_user_archives $path
} else {
lappend foreign_archives $path/$version
}
}
append create_args " CROSS_DEV_PREFIX=[cross_dev_prefix]"
if {[llength $local_user_archives] || [llength $foreign_archives]} {
puts stderr "" }
if {[llength $local_user_archives]} {
append create_args " CROSS_DEV_PREFIX=[cross_dev_prefix]"
puts stderr "You may create the following archives locally:\n"
puts stderr " [genode_dir]/tool/depot/create $local_user_archives$create_args\n"
}