Implement #624: Use shorter hashes with CPM_SOURCE_CACHE (#631)

* Add ASSERT_CONTENTS_EQUAL test macro in testing.cmake

Checks if the contents of a file matches the given input

* Use shorter hashes with CPM_SOURCE_CACHE (#624)

Uses shorter hashes with CPM_SOURCE_CACHE. Falls back to a longer hash
if necessary (ie, if there's a collision with an existing hash).

See: https://github.com/cpm-cmake/CPM.cmake/issues/624

* Update integration tests to support shorter hashes

* trigger ci

* run cmake-format

* if already available, use the legacy cache hash

* create temporary file in current binary dir

* add test case for legacy hash

---------

Co-authored-by: Lars Melchior <lars.melchior@gmail.com>
Co-authored-by: Lars Melchior <TheLartians@users.noreply.github.com>

README.md

@@ -88,7 +88,7 @@ If an additional optional parameter `SYSTEM` is set to a truthy value, the SYSTE
 See the [add_subdirectory ](https://cmake.org/cmake/help/latest/command/add_subdirectory.html?highlight=add_subdirectory)
 and [SYSTEM](https://cmake.org/cmake/help/latest/prop_tgt/SYSTEM.html#prop_tgt:SYSTEM) target property for details.
 
-A single-argument compact syntax is also supported:
+A shorthand syntax is also supported:
 
 ```cmake
 # A git package from a given uri with a version
@@ -112,6 +112,19 @@ CPMAddPackage("https://example.com/my-package-1.2.3.zip#MD5=68e20f674a48be38d60e
 CPMAddPackage("https://example.com/my-package.zip@1.2.3")
 ```
 
+Additionally, if needed, extra arguments can be provided while using single argument syntax by using the shorthand syntax with the `URI` specifier.
+
+```cmake
+CPMAddPackage(
+  URI "gh:nlohmann/json@3.9.1"
+  OPTIONS "JSON_BuildTests OFF"
+)
+```
+
+The `URI` argument must be the first argument to `CPMAddPackage`.
+`URI` automatically sets `EXCLUDE_FROM_ALL YES` and `SYSTEM YES`.
+If this is not desired, `EXCLUDE_FROM_ALL NO` and `SYSTEM NO` can be set afterwards.
+
 After calling `CPMAddPackage`, the following variables are defined in the local scope, where `<dependency>` is the name of the dependency.
 
 - `<dependency>_SOURCE_DIR` is the path to the source of the dependency.
@@ -412,11 +425,8 @@ CPMAddPackage("gh:jbeder/yaml-cpp#yaml-cpp-0.6.3@0.6.3")
 
 ```cmake
 CPMAddPackage(
-  NAME nlohmann_json
-  VERSION 3.9.1
-  GITHUB_REPOSITORY nlohmann/json
-  OPTIONS
-    "JSON_BuildTests OFF"
+  URI "gh:nlohmann/json@3.9.1"
+  OPTIONS "JSON_BuildTests OFF"
 )
 ```
 
@@ -446,8 +456,7 @@ For a working example of using CPM to download and configure the Boost C++ Libra
 ```cmake
 # the install option has to be explicitly set to allow installation
 CPMAddPackage(
-  GITHUB_REPOSITORY jarro2783/cxxopts
-  VERSION 2.2.1
+  URI "gh:jarro2783/cxxopts@2.2.1"
   OPTIONS "CXXOPTS_BUILD_EXAMPLES NO" "CXXOPTS_BUILD_TESTS NO" "CXXOPTS_ENABLE_INSTALL YES"
 )
 ```
@@ -456,9 +465,7 @@ CPMAddPackage(
 
 ```cmake
 CPMAddPackage(
-  NAME benchmark
-  GITHUB_REPOSITORY google/benchmark
-  VERSION 1.5.2
+  URI "gh:google/benchmark@1.5.2"
   OPTIONS "BENCHMARK_ENABLE_TESTING Off"
 )
 

cmake/CPM.cmake

@@ -202,6 +202,60 @@ function(cpm_package_name_from_git_uri URI RESULT)
   endif()
 endfunction()
 
+# Find the shortest hash that can be used eg, if origin_hash is
+# cccb77ae9609d2768ed80dd42cec54f77b1f1455 the following files will be checked, until one is found
+# that is either empty (allowing us to assign origin_hash), or whose contents matches ${origin_hash}
+#
+# * .../cccb.hash
+# * .../cccb77ae.hash
+# * .../cccb77ae9609.hash
+# * .../cccb77ae9609d276.hash
+# * etc
+#
+# We will be able to use a shorter path with very high probability, but in the (rare) event that the
+# first couple characters collide, we will check longer and longer substrings.
+function(cpm_get_shortest_hash source_cache_dir origin_hash short_hash_output_var)
+  # for compatibility with caches populated by a previous version of CPM, check if a directory using
+  # the full hash already exists
+  if(EXISTS "${source_cache_dir}/${origin_hash}")
+    set(${short_hash_output_var}
+        "${origin_hash}"
+        PARENT_SCOPE
+    )
+    return()
+  endif()
+
+  foreach(len RANGE 4 40 4)
+    string(SUBSTRING "${origin_hash}" 0 ${len} short_hash)
+    set(hash_lock ${source_cache_dir}/${short_hash}.lock)
+    set(hash_fp ${source_cache_dir}/${short_hash}.hash)
+    # Take a lock, so we don't have a race condition with another instance of cmake. We will release
+    # this lock when we can, however, if there is an error, we want to ensure it gets released on
+    # it's own on exit from the function.
+    file(LOCK ${hash_lock} GUARD FUNCTION)
+
+    # Load the contents of .../${short_hash}.hash
+    file(TOUCH ${hash_fp})
+    file(READ ${hash_fp} hash_fp_contents)
+
+    if(hash_fp_contents STREQUAL "")
+      # Write the origin hash
+      file(WRITE ${hash_fp} ${origin_hash})
+      file(LOCK ${hash_lock} RELEASE)
+      break()
+    elseif(hash_fp_contents STREQUAL origin_hash)
+      file(LOCK ${hash_lock} RELEASE)
+      break()
+    else()
+      file(LOCK ${hash_lock} RELEASE)
+    endif()
+  endforeach()
+  set(${short_hash_output_var}
+      "${short_hash}"
+      PARENT_SCOPE
+  )
+endfunction()
+
 # Try to infer package name and version from a url
 function(cpm_package_name_and_ver_from_url url outName outVer)
   if(url MATCHES "[/\\?]([a-zA-Z0-9_\\.-]+)\\.(tar|tar\\.gz|tar\\.bz2|zip|ZIP)(\\?|/|$)")
@@ -594,14 +648,6 @@ endfunction()
 function(CPMAddPackage)
   cpm_set_policies()
 
-  list(LENGTH ARGN argnLength)
-  if(argnLength EQUAL 1)
-    cpm_parse_add_package_single_arg("${ARGN}" ARGN)
-
-    # The shorthand syntax implies EXCLUDE_FROM_ALL and SYSTEM
-    set(ARGN "${ARGN};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;")
-  endif()
-
   set(oneValueArgs
       NAME
       FORCE
@@ -624,10 +670,26 @@ function(CPMAddPackage)
 
   set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND PATCHES)
 
+  list(LENGTH ARGN argnLength)
+
+  # Parse single shorthand argument
+  if(argnLength EQUAL 1)
+    cpm_parse_add_package_single_arg("${ARGN}" ARGN)
+
+    # The shorthand syntax implies EXCLUDE_FROM_ALL and SYSTEM
+    set(ARGN "${ARGN};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;")
+
+    # Parse URI shorthand argument
+  elseif(argnLength GREATER 1 AND "${ARGV0}" STREQUAL "URI")
+    list(REMOVE_AT ARGN 0 1) # remove "URI gh:<...>@version#tag"
+    cpm_parse_add_package_single_arg("${ARGV1}" ARGV0)
+
+    set(ARGN "${ARGV0};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;${ARGN}")
+  endif()
+
   cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
 
   # Set default values for arguments
-
   if(NOT DEFINED CPM_ARGS_VERSION)
     if(DEFINED CPM_ARGS_GIT_TAG)
       cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION)
@@ -798,9 +860,19 @@ function(CPMAddPackage)
       set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${CPM_ARGS_CUSTOM_CACHE_KEY})
     elseif(CPM_USE_NAMED_CACHE_DIRECTORIES)
       string(SHA1 origin_hash "${origin_parameters};NEW_CACHE_STRUCTURE_TAG")
+      cpm_get_shortest_hash(
+        "${CPM_SOURCE_CACHE}/${lower_case_name}" # source cache directory
+        "${origin_hash}" # Input hash
+        origin_hash # Computed hash
+      )
       set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}/${CPM_ARGS_NAME})
     else()
       string(SHA1 origin_hash "${origin_parameters}")
+      cpm_get_shortest_hash(
+        "${CPM_SOURCE_CACHE}/${lower_case_name}" # source cache directory
+        "${origin_hash}" # Input hash
+        origin_hash # Computed hash
+      )
       set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash})
     endif()
     # Expand `download_directory` relative path. This is important because EXISTS doesn't work for
@@ -867,7 +939,9 @@ function(CPMAddPackage)
     endif()
   endif()
 
-  cpm_create_module_file(${CPM_ARGS_NAME} "CPMAddPackage(\"${ARGN}\")")
+  if(NOT "${DOWNLOAD_ONLY}")
+    cpm_create_module_file(${CPM_ARGS_NAME} "CPMAddPackage(\"${ARGN}\")")
+  endif()
 
   if(CPM_PACKAGE_LOCK_ENABLED)
     if((CPM_ARGS_VERSION AND NOT CPM_ARGS_SOURCE_DIR) OR CPM_INCLUDE_ALL_IN_PACKAGE_LOCK)

cmake/testing.cmake

@@ -79,3 +79,16 @@ function(ASSERT_NOT_EXISTS file)
     message(FATAL_ERROR "assertion failed: file ${file} exists")
   endif()
 endfunction()
+
+function(ASSERT_CONTENTS_EQUAL file content)
+  if(EXISTS ${file})
+    file(READ ${file} file_content)
+    if(content STREQUAL file_content)
+      message(STATUS "test passed: '${file}' exists and contains '${content}'")
+    else()
+      message(FATAL_ERROR "assertion failed: file '${file}' does not contain expected content.")
+    endif()
+  else()
+    message(FATAL_ERROR "assertion failed: file '${file} does not exist")
+  endif()
+endfunction()

test/integration/test_shorthand_syntax.rb

@@ -0,0 +1,86 @@
+require_relative './lib'
+
+class TestShorthandSyntax < IntegrationTest
+
+  def get_project_binaries prj
+    exe_dir = File.join(prj.bin_dir, 'bin')
+    assert File.directory? exe_dir
+    return Dir[exe_dir + '/**/*'].filter {
+        # on multi-configuration generators (like Visual Studio) the executables will be in bin/<Config>
+        # also filter-out other artifacts like .pdb or .dsym
+        !File.directory?(_1) && File.stat(_1).executable?
+      }.map {
+        # remove .exe extension if any (there will be one on Windows)
+        File.basename(_1, '.exe')
+      }.sort
+  end
+
+  def test_create_with_commit_sha
+    prj = make_project from_template: 'using-adder'
+    prj.create_lists_from_default_template package:
+    'CPMAddPackage("gh:cpm-cmake/testpack-adder#cad1cd4b4cdf957c5b59e30bc9a1dd200dbfc716")'
+    assert_success prj.configure
+
+    cache = prj.read_cache
+    assert_equal 1, cache.packages.size
+    assert_equal '0', cache.packages['testpack-adder'].ver
+
+    assert_success prj.build
+    exes = get_project_binaries prj
+    # No adder projects were built as EXCLUDE_FROM_ALL is implicitly set
+    assert_equal ['using-adder'], exes
+  end
+
+  def test_create_with_version
+    prj = make_project from_template: 'using-adder'
+    prj.create_lists_from_default_template package:
+    'CPMAddPackage("gh:cpm-cmake/testpack-adder@1.0.0")'
+    assert_success prj.configure
+
+    cache = prj.read_cache
+    assert_equal 1, cache.packages.size
+    assert_equal '1.0.0', cache.packages['testpack-adder'].ver
+
+    assert_success prj.build
+    exes = get_project_binaries prj
+    assert_equal ['using-adder'], exes
+  end
+
+  def test_create_with_all
+    prj = make_project from_template: 'using-adder'
+    prj.create_lists_from_default_template package:
+    'CPMAddPackage(
+      URI "gh:cpm-cmake/testpack-adder@1.0.0"
+      EXCLUDE_FROM_ALL false
+    )'
+    assert_success prj.configure
+
+    cache = prj.read_cache
+    assert_equal cache.packages.size, 1
+    assert_equal cache.packages['testpack-adder'].ver, '1.0.0'
+    
+    assert_success prj.build
+    exes = get_project_binaries prj
+    assert_equal exes, ['simple', 'test-adding', 'using-adder']
+  end
+
+  def test_create_with_tests_but_without_examples
+    prj = make_project from_template: 'using-adder'
+    prj.create_lists_from_default_template package:
+    'CPMAddPackage(
+      URI "gh:cpm-cmake/testpack-adder@1.0.0"
+      OPTIONS "ADDER_BUILD_EXAMPLES OFF" "ADDER_BUILD_TESTS TRUE"
+      EXCLUDE_FROM_ALL false
+    )'
+    assert_success prj.configure
+
+    cache = prj.read_cache
+    assert_equal cache.packages.size, 1
+    assert_equal cache.packages['testpack-adder'].ver, '1.0.0'
+    
+    assert_success prj.build
+    exes = get_project_binaries prj
+    assert_equal exes, ['test-adding', 'using-adder']
+  end
+
+end

test/integration/test_source_cache.rb

@@ -75,6 +75,17 @@ class SourceCache < IntegrationTest
     assert_equal ver, package.ver
     expected_parent_dir = File.join(@cache_dir, name.downcase)
     assert package.src_dir.start_with?(expected_parent_dir), "#{package.src_dir} must be in #{expected_parent_dir}"
-    assert_equal dir_sha1, File.basename(package.src_dir)
+
+    # The hash has been shortened by cpm_get_shortest_hash. The following
+    # should hold:
+    # - The short hash should be a prefix of the input hash
+    # - There should be a file ".../${short_hash}.hash" which matches the full hash
+    short_hash = File.basename(package.src_dir)
+    assert dir_sha1.start_with?(short_hash), "short_hash should be a prefix of dir_sha1"
+
+    # Check that the full hash is stored in the .hash file
+    hash_file = "#{package.src_dir}.hash"
+    assert File.exist?(hash_file), "Hash file #{hash_file} should exist"
+    assert_equal dir_sha1, File.read(hash_file), "Hash file should contain the full original hash"
   end
 end

test/unit/get_shortest_hash.cmake

@@ -0,0 +1,93 @@
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
+
+include(${CPM_PATH}/CPM.cmake)
+include(${CPM_PATH}/testing.cmake)
+
+# Random suffix
+string(
+  RANDOM
+  LENGTH 6
+  ALPHABET "0123456789abcdef" tmpdir_suffix
+)
+
+# Seconds since epoch
+string(TIMESTAMP tmpdir_base "%s" UTC)
+
+set(tmp "${CMAKE_CURRENT_BINARY_DIR}/get_shortest_hash-${tmpdir_base}-${tmpdir_suffix}")
+
+if(IS_DIRECTORY ${tmp})
+  message(FATAL_ERROR "Test directory ${tmp} already exists")
+endif()
+
+file(MAKE_DIRECTORY "${tmp}")
+
+# 1. Sanity check: none of these directories should exist yet
+
+assert_not_exists(${tmp}/cccb.hash)
+assert_not_exists(${tmp}/cccb77ae.hash)
+assert_not_exists(${tmp}/cccb77ae9609.hash)
+assert_not_exists(${tmp}/cccb77ae9608.hash)
+assert_not_exists(${tmp}/cccb77be.hash)
+
+# 1. The directory is empty, so it should get a 4-character hash
+cpm_get_shortest_hash(${tmp} "cccb77ae9609d2768ed80dd42cec54f77b1f1455" hash)
+assert_equal(${hash} "cccb")
+assert_contents_equal(${tmp}/cccb.hash cccb77ae9609d2768ed80dd42cec54f77b1f1455)
+
+# 1. Calling the function with a new hash that differs subtly should result in more characters being
+#   used, enough to uniquely identify the hash
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9609d2768ed80dd42cec54f77b1f1456" hash)
+assert_equal(${hash} "cccb77ae")
+assert_contents_equal(${tmp}/cccb77ae.hash cccb77ae9609d2768ed80dd42cec54f77b1f1456)
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9609d2768ed80dd42cec54f77b1f1457" hash)
+assert_equal(${hash} "cccb77ae9609")
+assert_contents_equal(${tmp}/cccb77ae9609.hash cccb77ae9609d2768ed80dd42cec54f77b1f1457)
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9608d2768ed80dd42cec54f77b1f1455" hash)
+assert_equal(${hash} "cccb77ae9608")
+assert_contents_equal(${tmp}/cccb77ae9608.hash cccb77ae9608d2768ed80dd42cec54f77b1f1455)
+
+cpm_get_shortest_hash(${tmp} "cccb77be9609d2768ed80dd42cec54f77b1f1456" hash)
+assert_equal(${hash} "cccb77be")
+assert_contents_equal(${tmp}/cccb77be.hash cccb77be9609d2768ed80dd42cec54f77b1f1456)
+
+# check that legacy hashs are recognized
+file(MAKE_DIRECTORY "${tmp}/cccb77be9609d2768ed80dd42cec54f77b1f1457")
+cpm_get_shortest_hash(${tmp} "cccb77be9609d2768ed80dd42cec54f77b1f1457" hash)
+assert_equal(${hash} "cccb77be9609d2768ed80dd42cec54f77b1f1457")
+
+# 1. The old file should still exist, and have the same content
+assert_contents_equal(${tmp}/cccb.hash cccb77ae9609d2768ed80dd42cec54f77b1f1455)
+assert_contents_equal(${tmp}/cccb77ae.hash cccb77ae9609d2768ed80dd42cec54f77b1f1456)
+assert_contents_equal(${tmp}/cccb77ae9609.hash cccb77ae9609d2768ed80dd42cec54f77b1f1457)
+assert_contents_equal(${tmp}/cccb77ae9608.hash cccb77ae9608d2768ed80dd42cec54f77b1f1455)
+assert_contents_equal(${tmp}/cccb77be.hash cccb77be9609d2768ed80dd42cec54f77b1f1456)
+
+# 1. Confirm idempotence: calling any of these function should produce the same hash as before (hash
+#   lookups work correctly once the .hash files are created)
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9609d2768ed80dd42cec54f77b1f1455" hash)
+assert_equal(${hash} "cccb")
+assert_contents_equal(${tmp}/cccb.hash cccb77ae9609d2768ed80dd42cec54f77b1f1455)
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9609d2768ed80dd42cec54f77b1f1456" hash)
+assert_equal(${hash} "cccb77ae")
+assert_contents_equal(${tmp}/cccb77ae.hash cccb77ae9609d2768ed80dd42cec54f77b1f1456)
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9609d2768ed80dd42cec54f77b1f1457" hash)
+assert_equal(${hash} "cccb77ae9609")
+assert_contents_equal(${tmp}/cccb77ae9609.hash cccb77ae9609d2768ed80dd42cec54f77b1f1457)
+
+cpm_get_shortest_hash(${tmp} "cccb77ae9608d2768ed80dd42cec54f77b1f1455" hash)
+assert_equal(${hash} "cccb77ae9608")
+assert_contents_equal(${tmp}/cccb77ae9608.hash cccb77ae9608d2768ed80dd42cec54f77b1f1455)
+
+cpm_get_shortest_hash(${tmp} "cccb77be9609d2768ed80dd42cec54f77b1f1456" hash)
+assert_equal(${hash} "cccb77be")
+assert_contents_equal(${tmp}/cccb77be.hash cccb77be9609d2768ed80dd42cec54f77b1f1456)
+
+# 1. Cleanup - remove the temporary directory that we created
+
+file(REMOVE_RECURSE ${tmp})